From 33479ea9afb8c75c3fe2401089284308279cdad8 Mon Sep 17 00:00:00 2001 From: vladp Date: Mon, 22 Dec 2025 17:17:48 +0700 Subject: [PATCH 01/37] add check extensions --- .../nocopy/controller/ApiController.java | 10 ++-- .../ru/soune/nocopy/entity/file/FileType.java | 33 +++++++---- .../nocopy/handler/FileUploadHandler.java | 33 ++++++----- .../validator/FileUploadRequestValidator.java | 58 +++++++++++++++++-- 4 files changed, 99 insertions(+), 35 deletions(-) diff --git a/src/main/java/ru/soune/nocopy/controller/ApiController.java b/src/main/java/ru/soune/nocopy/controller/ApiController.java index 9ecad18..ea129c0 100644 --- a/src/main/java/ru/soune/nocopy/controller/ApiController.java +++ b/src/main/java/ru/soune/nocopy/controller/ApiController.java @@ -387,11 +387,11 @@ public class ApiController { return false; } - return mimeType.startsWith("image/") || - mimeType.startsWith("text/") || - mimeType.equals("application/pdf") || - mimeType.startsWith("video/") || - mimeType.startsWith("audio/"); + return mimeType.startsWith("image") || + mimeType.startsWith("text") || + mimeType.equals("pdf") || + mimeType.startsWith("video") || + mimeType.startsWith("audio"); } private Long getUserIdFromToken(String tokenHeader) { diff --git a/src/main/java/ru/soune/nocopy/entity/file/FileType.java b/src/main/java/ru/soune/nocopy/entity/file/FileType.java index 233a7c2..cfe080d 100644 --- a/src/main/java/ru/soune/nocopy/entity/file/FileType.java +++ b/src/main/java/ru/soune/nocopy/entity/file/FileType.java @@ -2,21 +2,34 @@ package ru.soune.nocopy.entity.file; import lombok.Getter; +import java.util.Arrays; +import java.util.List; + @Getter public enum FileType { - PHOTO("photo"), - IMAGE("image"), - VIDEO("video"), - AUDIO("audio"), - DOCUMENT("document"); + PHOTO("photo", Arrays.asList("jpg", "jpeg", "png", "gif", "bmp", "webp")), + IMAGE("image", Arrays.asList("jpg", "jpeg", "png", "gif", "bmp", "webp", "svg", "tiff")), + VIDEO("video", Arrays.asList("mp4", "avi", "mov", "wmv", "flv", "mkv", "webm")), + AUDIO("audio", Arrays.asList("mp3", "wav", "ogg", "aac", "flac", "m4a")), + DOCUMENT("document", Arrays.asList("pdf", "doc", "docx", "txt", "rtf", "odt", "xls", "xlsx", "ppt", "pptx")); - private final String code; + private final String displayName; + private final List allowedExtensions; - FileType(String code) { - this.code = code; + FileType(String displayName, List allowedExtensions) { + this.displayName = displayName; + this.allowedExtensions = allowedExtensions; } - public String getCode() { - return code; + public String getDisplayName() { + return displayName; + } + + public List getAllowedExtensions() { + return allowedExtensions; + } + + public boolean supportsExtension(String extension) { + return allowedExtensions.contains(extension.toLowerCase()); } } diff --git a/src/main/java/ru/soune/nocopy/handler/FileUploadHandler.java b/src/main/java/ru/soune/nocopy/handler/FileUploadHandler.java index 26f6dba..7674f4c 100644 --- a/src/main/java/ru/soune/nocopy/handler/FileUploadHandler.java +++ b/src/main/java/ru/soune/nocopy/handler/FileUploadHandler.java @@ -6,6 +6,7 @@ import lombok.extern.slf4j.Slf4j; import org.springframework.stereotype.Component; import org.springframework.validation.BeanPropertyBindingResult; import org.springframework.validation.BindingResult; +import org.springframework.validation.FieldError; import ru.soune.nocopy.dto.*; import ru.soune.nocopy.dto.file.*; import ru.soune.nocopy.entity.AuthToken; @@ -17,10 +18,8 @@ import ru.soune.nocopy.repository.AuthTokenRepository; import ru.soune.nocopy.repository.FileUploadSessionRepository; import ru.soune.nocopy.service.file.FileUploadService; -import java.util.Arrays; -import java.util.HashMap; -import java.util.Map; -import java.util.Optional; +import java.util.*; +import java.util.stream.Collectors; @Slf4j @Component @@ -70,18 +69,31 @@ public class FileUploadHandler implements RequestHandler { } private BaseResponse handleInitUpload(BaseRequest request, FileUploadRequest fileRequest) { - try { String token = fileRequest.getToken(); Optional tokenOptional = authTokenRepository.findByToken(token); + if (tokenOptional.isEmpty()) { + return new BaseResponse(request.getMsgId(), MessageCode.INVALID_TOKEN.getCode(), + MessageCode.INVALID_TOKEN.getDescription(), Map.of("token", token)); + } + AuthToken authToken = tokenOptional.orElseThrow(() -> new NotFoundAuthToken("Token not found")); BindingResult bindingResult = new BeanPropertyBindingResult(fileRequest, "fileRequest"); fileUploadRequestValidator.validate(fileRequest, bindingResult); if (bindingResult.hasErrors()) { - throw new ValidationException(bindingResult, request.getMsgId()); + Map fieldErrors = bindingResult.getFieldErrors() + .stream() + .collect(Collectors.toMap( + FieldError::getField, + fieldError -> fieldError.getDefaultMessage() != null + ? fieldError.getDefaultMessage() + : "Validation error")); + + return new BaseResponse(request.getMsgId(), MessageCode.INVALID_FIELD.getCode(), + MessageCode.INVALID_FIELD.getDescription(), fieldErrors); } FileUploadSession session = fileUploadService.initUpload( @@ -101,15 +113,6 @@ public class FileUploadHandler implements RequestHandler { return new BaseResponse(request.getMsgId(), MessageCode.SUCCESS.getCode(), MessageCode.SUCCESS.getDescription(), fileResponse); - - } catch (NotFoundAuthToken e) { - InitFileResponse initFileResponse = InitFileResponse.builder() - .build(); - - throw new NotValidFieldException("Invalid or expired token: " + fileRequest.getToken(), - new BaseResponse(request.getMsgId(), MessageCode.INVALID_TOKEN.getCode(), - MessageCode.INVALID_TOKEN.getDescription(), initFileResponse)); - } } private BaseResponse handleGetProgress(BaseRequest request, FileUploadRequest fileRequest) { diff --git a/src/main/java/ru/soune/nocopy/handler/validator/FileUploadRequestValidator.java b/src/main/java/ru/soune/nocopy/handler/validator/FileUploadRequestValidator.java index f40487a..fa12270 100644 --- a/src/main/java/ru/soune/nocopy/handler/validator/FileUploadRequestValidator.java +++ b/src/main/java/ru/soune/nocopy/handler/validator/FileUploadRequestValidator.java @@ -7,9 +7,12 @@ import ru.soune.nocopy.dto.file.FileUploadRequest; import ru.soune.nocopy.entity.file.FileType; import java.util.Arrays; +import java.util.List; @Component public class FileUploadRequestValidator implements Validator { + private final List supportedFileTypes = Arrays.asList(FileType.values()); + @Override public boolean supports(Class clazz) { return FileUploadRequest.class.isAssignableFrom(clazz); @@ -21,7 +24,7 @@ public class FileUploadRequestValidator implements Validator { validateType(request.getFileType(), errors); validateFileName(request.getFileName(), errors); - validateExtension(request.getExtension(), errors); + validateExtension(request.getExtension(), request.getFileName(), errors); } private void validateType(String fileType, Errors errors) { @@ -31,7 +34,12 @@ public class FileUploadRequestValidator implements Validator { } try { - FileType.valueOf(fileType.toUpperCase()); + FileType parsedType = FileType.valueOf(fileType.toUpperCase()); + + if (!supportedFileTypes.contains(parsedType)) { + errors.rejectValue("fileType", "fileType.unsupported", + "Unsupported file type. Valid types: " + Arrays.toString(FileType.values())); + } } catch (IllegalArgumentException e) { errors.rejectValue("fileType", "fileType.invalid", "Invalid file type. Valid types: " + Arrays.toString(FileType.values())); @@ -50,10 +58,50 @@ public class FileUploadRequestValidator implements Validator { } } - private void validateExtension(String extension, Errors errors) { - if (extension == null || extension.isBlank()) { - errors.rejectValue("extension", "extension.required", "Extension is required"); + private void validateExtension(String fileType, String fileName, Errors errors) { + if (fileType == null || fileType.isBlank()) { + errors.rejectValue("fileType", "fileType.required", "File type is required"); return; } + + String fileExtension = getFileExtension(fileName); + + if (fileExtension == null || fileExtension.isBlank()) { + errors.rejectValue("fileName", "fileName.invalid.extension", "File must have an extension"); + return; + } + + if (fileExtension.contains(".")) { + errors.rejectValue("extension", "extension.required", "Extension contains comma"); + return; + } + + try { + FileType parsedType = FileType.valueOf(fileType.toUpperCase()); + + if (!supportedFileTypes.contains(parsedType)) { + errors.rejectValue("fileType", "fileType.unsupported", + "Unsupported file type. Supported types: " + supportedFileTypes); + return; + } + + if (!parsedType.supportsExtension(fileExtension)) { + errors.rejectValue("fileName", "fileType.extension.mismatch", + String.format("File extension '%s' does not match file type '%s'. Allowed extensions for %s: %s", + fileExtension, parsedType.getDisplayName(), parsedType.getDisplayName(), + parsedType.getAllowedExtensions())); + } + + } catch (IllegalArgumentException e) { + errors.rejectValue("fileType", "fileType.invalid", + "Invalid file type. Valid types: " + Arrays.toString(FileType.values())); + } + } + + private String getFileExtension(String fileName) { + if (fileName == null || fileName.lastIndexOf('.') == -1) { + return null; + } + return fileName.substring(fileName.lastIndexOf('.') + 1).toLowerCase(); } } \ No newline at end of file From ce879fea186772433371de13ede35a6fc270998f Mon Sep 17 00:00:00 2001 From: vladp Date: Tue, 23 Dec 2025 12:03:49 +0700 Subject: [PATCH 02/37] add format for all office,os --- .../ru/soune/nocopy/entity/file/FileType.java | 23 +++++++++++++++---- 1 file changed, 18 insertions(+), 5 deletions(-) diff --git a/src/main/java/ru/soune/nocopy/entity/file/FileType.java b/src/main/java/ru/soune/nocopy/entity/file/FileType.java index cfe080d..569d31d 100644 --- a/src/main/java/ru/soune/nocopy/entity/file/FileType.java +++ b/src/main/java/ru/soune/nocopy/entity/file/FileType.java @@ -7,11 +7,24 @@ import java.util.List; @Getter public enum FileType { - PHOTO("photo", Arrays.asList("jpg", "jpeg", "png", "gif", "bmp", "webp")), - IMAGE("image", Arrays.asList("jpg", "jpeg", "png", "gif", "bmp", "webp", "svg", "tiff")), - VIDEO("video", Arrays.asList("mp4", "avi", "mov", "wmv", "flv", "mkv", "webm")), - AUDIO("audio", Arrays.asList("mp3", "wav", "ogg", "aac", "flac", "m4a")), - DOCUMENT("document", Arrays.asList("pdf", "doc", "docx", "txt", "rtf", "odt", "xls", "xlsx", "ppt", "pptx")); + PHOTO("photo", Arrays.asList("jpg", "jpeg", "png", "gif", "bmp")), + IMAGE("image", Arrays.asList("jpg", "jpeg", "png", "gif", "bmp", "svg", "tiff", "tif", "ico", + "psd", "ai", "eps", "raw", "heic", "heif")), + VIDEO("video", Arrays.asList("mp4", "avi", "mov", "wmv", "flv", "mkv", "webm", "m4v", "mpg", "mpeg", + "3gp", "3g2", "f4v", "m2ts", "mts", "vob", "ogv", "divx")), + AUDIO("audio", Arrays.asList("mp3", "wav", "ogg", "aac", "flac", "m4a", "wma", "aiff", "aif", "amr", + "opus", "mka", "ac3", "alac")), + DOCUMENT("document", Arrays.asList( + "pdf", "txt", "rtf", + "doc", "docx", "xls", "xlsx", "ppt", "pptx", "pps", "ppsx", "dot", "dotx", "xlt", "xltx", "pot", "potx", + "odt", "ods", "odp", "odg", "odf", "odb", "odc", "odi", "odm", "ott", "ots", "otp", "otg", "oth", + "sxw", "sxc", "sxi", "sxd", "sxg", "stc", "sti", "stw", "sxm", + "pages", "numbers", "key", + "csv", "tsv", "xml", "html", "htm", "tex", "md", "markdown", + "epub", "mobi", "azw", "azw3", "fb2", + "wps", "wpt", "et", "dps", "vsd", "vsdx", + "java", "py", "cpp", "c", "h", "js", "css", "php", "sql", "json", "yaml", "yml", "sh", "bat", + "one", "note")); private final String displayName; private final List allowedExtensions; From 4c607f5bf9847e718523ff4efd3d430ee2759ffc Mon Sep 17 00:00:00 2001 From: vladp Date: Tue, 23 Dec 2025 12:46:59 +0700 Subject: [PATCH 03/37] fix check format --- .../validator/FileUploadRequestValidator.java | 28 ++++--------------- 1 file changed, 6 insertions(+), 22 deletions(-) diff --git a/src/main/java/ru/soune/nocopy/handler/validator/FileUploadRequestValidator.java b/src/main/java/ru/soune/nocopy/handler/validator/FileUploadRequestValidator.java index fa12270..201dadf 100644 --- a/src/main/java/ru/soune/nocopy/handler/validator/FileUploadRequestValidator.java +++ b/src/main/java/ru/soune/nocopy/handler/validator/FileUploadRequestValidator.java @@ -24,7 +24,7 @@ public class FileUploadRequestValidator implements Validator { validateType(request.getFileType(), errors); validateFileName(request.getFileName(), errors); - validateExtension(request.getExtension(), request.getFileName(), errors); + validateExtension(request.getExtension(), request.getFileType(), errors); } private void validateType(String fileType, Errors errors) { @@ -58,20 +58,13 @@ public class FileUploadRequestValidator implements Validator { } } - private void validateExtension(String fileType, String fileName, Errors errors) { - if (fileType == null || fileType.isBlank()) { + private void validateExtension(String extension, String fileType, Errors errors) { + if (extension == null || extension.isBlank()) { errors.rejectValue("fileType", "fileType.required", "File type is required"); return; } - String fileExtension = getFileExtension(fileName); - - if (fileExtension == null || fileExtension.isBlank()) { - errors.rejectValue("fileName", "fileName.invalid.extension", "File must have an extension"); - return; - } - - if (fileExtension.contains(".")) { + if (extension.contains(".")) { errors.rejectValue("extension", "extension.required", "Extension contains comma"); return; } @@ -79,16 +72,10 @@ public class FileUploadRequestValidator implements Validator { try { FileType parsedType = FileType.valueOf(fileType.toUpperCase()); - if (!supportedFileTypes.contains(parsedType)) { - errors.rejectValue("fileType", "fileType.unsupported", - "Unsupported file type. Supported types: " + supportedFileTypes); - return; - } - - if (!parsedType.supportsExtension(fileExtension)) { + if (!parsedType.supportsExtension(extension)) { errors.rejectValue("fileName", "fileType.extension.mismatch", String.format("File extension '%s' does not match file type '%s'. Allowed extensions for %s: %s", - fileExtension, parsedType.getDisplayName(), parsedType.getDisplayName(), + extension, parsedType.getDisplayName(), parsedType.getDisplayName(), parsedType.getAllowedExtensions())); } @@ -99,9 +86,6 @@ public class FileUploadRequestValidator implements Validator { } private String getFileExtension(String fileName) { - if (fileName == null || fileName.lastIndexOf('.') == -1) { - return null; - } return fileName.substring(fileName.lastIndexOf('.') + 1).toLowerCase(); } } \ No newline at end of file From 8fe4d854a6ab641c31cb23509df6bdcf084a0807 Mon Sep 17 00:00:00 2001 From: vladp Date: Tue, 23 Dec 2025 13:10:36 +0700 Subject: [PATCH 04/37] add create uploads filedir --- infrastructure/jenkins/deploy.groovy | 1 + 1 file changed, 1 insertion(+) diff --git a/infrastructure/jenkins/deploy.groovy b/infrastructure/jenkins/deploy.groovy index 43b39ff..248b0cf 100644 --- a/infrastructure/jenkins/deploy.groovy +++ b/infrastructure/jenkins/deploy.groovy @@ -114,6 +114,7 @@ EOF --network app-network \\ --network-alias app \\ -p 80:8080 \\ + -v /opt/uploads:/data/uploads:rw \\ -e POSTGRES_DB=no_copy_ \\ -e POSTGRES_USER=postgres \\ -e POSTGRES_PASSWORD=postgres \\ From 19d8f61e1d5c5f9df2b1261dab587bc6d858c257 Mon Sep 17 00:00:00 2001 From: vladp Date: Tue, 23 Dec 2025 16:05:51 +0700 Subject: [PATCH 05/37] NCBACK-26 add docker container with storage --- Dockerfile | 2 ++ docker-compose.yaml | 15 +++++++++++++-- 2 files changed, 15 insertions(+), 2 deletions(-) diff --git a/Dockerfile b/Dockerfile index 8242e14..6cd62ca 100644 --- a/Dockerfile +++ b/Dockerfile @@ -12,6 +12,8 @@ WORKDIR /app COPY --from=build /app/build/libs/*.jar app.jar +RUN mkdir -p /data/uploads && chmod 755 /data/uploads + EXPOSE 8080 CMD ["java", "-jar", "app.jar"] diff --git a/docker-compose.yaml b/docker-compose.yaml index d3f9924..ce90ce2 100644 --- a/docker-compose.yaml +++ b/docker-compose.yaml @@ -1,6 +1,16 @@ version: '3.9' services: + storage: + image: alpine:latest + container_name: file-storage + networks: + - app-network + volumes: + - uploads_data:/storage:rw + command: tail -f /dev/null + restart: unless-stopped + db: image: postgres:17 restart: always @@ -30,6 +40,7 @@ services: POSTGRES_PASSWORD: postgres POSTGRES_PORT: 5432 POSTGRES_HOST: db + STORAGE_SERVICE_URL: http://storage:8081 depends_on: - db ports: @@ -41,7 +52,7 @@ services: - backend - api volumes: - - ./uploads:/data/uploads:rw + - uploads_data:/data/uploads:rw grafana: image: grafana/grafana:10.3.1 @@ -146,7 +157,7 @@ volumes: loki_chunks: loki_index: loki_rules: -# uploads_volume: + uploads_data: networks: app-network: From 1aef1dcd96a0dfaca022d310a70aad57e58e4e58 Mon Sep 17 00:00:00 2001 From: vladp Date: Tue, 23 Dec 2025 16:31:09 +0700 Subject: [PATCH 06/37] add chunk size --- .../java/ru/soune/nocopy/service/file/FileUploadServiceImpl.java | 1 + 1 file changed, 1 insertion(+) diff --git a/src/main/java/ru/soune/nocopy/service/file/FileUploadServiceImpl.java b/src/main/java/ru/soune/nocopy/service/file/FileUploadServiceImpl.java index e8fb956..eb48651 100644 --- a/src/main/java/ru/soune/nocopy/service/file/FileUploadServiceImpl.java +++ b/src/main/java/ru/soune/nocopy/service/file/FileUploadServiceImpl.java @@ -108,6 +108,7 @@ public class FileUploadServiceImpl implements FileUploadService { .extension(extension) .fileName(fileName) .fileType(fileType) + .chunkSize(5242880L) .fileSize(fileSize) .totalChunks(totalChunks) .chunksUploaded(0) From a77a51f519fbdd902b3186072084d53d4cfc3982 Mon Sep 17 00:00:00 2001 From: vladp Date: Tue, 23 Dec 2025 18:33:53 +0700 Subject: [PATCH 07/37] delete photo --- src/main/java/ru/soune/nocopy/entity/file/FileType.java | 1 - 1 file changed, 1 deletion(-) diff --git a/src/main/java/ru/soune/nocopy/entity/file/FileType.java b/src/main/java/ru/soune/nocopy/entity/file/FileType.java index 569d31d..32f9cd6 100644 --- a/src/main/java/ru/soune/nocopy/entity/file/FileType.java +++ b/src/main/java/ru/soune/nocopy/entity/file/FileType.java @@ -7,7 +7,6 @@ import java.util.List; @Getter public enum FileType { - PHOTO("photo", Arrays.asList("jpg", "jpeg", "png", "gif", "bmp")), IMAGE("image", Arrays.asList("jpg", "jpeg", "png", "gif", "bmp", "svg", "tiff", "tif", "ico", "psd", "ai", "eps", "raw", "heic", "heif")), VIDEO("video", Arrays.asList("mp4", "avi", "mov", "wmv", "flv", "mkv", "webm", "m4v", "mpg", "mpeg", From f6d4ebfe0e5fd64ba25ee9bde78b2d70d426e241 Mon Sep 17 00:00:00 2001 From: vladp Date: Wed, 24 Dec 2025 13:58:46 +0700 Subject: [PATCH 08/37] fix after test --- .../exception/ChunkSizeExceededException.java | 2 +- .../service/file/FileEntityService.java | 3 +- .../service/file/FileUploadServiceImpl.java | 40 ++++--------------- 3 files changed, 10 insertions(+), 35 deletions(-) diff --git a/src/main/java/ru/soune/nocopy/exception/ChunkSizeExceededException.java b/src/main/java/ru/soune/nocopy/exception/ChunkSizeExceededException.java index a19cd95..f8a4933 100644 --- a/src/main/java/ru/soune/nocopy/exception/ChunkSizeExceededException.java +++ b/src/main/java/ru/soune/nocopy/exception/ChunkSizeExceededException.java @@ -2,6 +2,6 @@ package ru.soune.nocopy.exception; public class ChunkSizeExceededException extends RuntimeException { public ChunkSizeExceededException(long actualSize, long maxSize) { - super(String.format("Chunk size exceeded: %d > %d", actualSize, maxSize)); + super(String.format("Chunk size not valid: " + actualSize)); } } diff --git a/src/main/java/ru/soune/nocopy/service/file/FileEntityService.java b/src/main/java/ru/soune/nocopy/service/file/FileEntityService.java index 8b239a5..97a6cab 100644 --- a/src/main/java/ru/soune/nocopy/service/file/FileEntityService.java +++ b/src/main/java/ru/soune/nocopy/service/file/FileEntityService.java @@ -40,7 +40,6 @@ public class FileEntityService { long fileSize = Files.size(filePath); String originalName = session.getFileName(); - String extension = extractFileExtension(originalName); String storedName = filePath.getFileName().toString(); FileEntity fileEntity = FileEntity.builder() @@ -50,7 +49,7 @@ public class FileEntityService { .filePath(session.getFilePath()) .fileSize(fileSize) .mimeType(session.getFileType()) - .fileExtension(extension) + .fileExtension(session.getExtension()) .checksum(checksum) .uploadSessionId(session.getUploadId()) .status(FileStatus.ACTIVE) diff --git a/src/main/java/ru/soune/nocopy/service/file/FileUploadServiceImpl.java b/src/main/java/ru/soune/nocopy/service/file/FileUploadServiceImpl.java index eb48651..93c7d67 100644 --- a/src/main/java/ru/soune/nocopy/service/file/FileUploadServiceImpl.java +++ b/src/main/java/ru/soune/nocopy/service/file/FileUploadServiceImpl.java @@ -102,13 +102,14 @@ public class FileUploadServiceImpl implements FileUploadService { int totalChunks = (int) Math.ceil((double) fileSize / chunkSize); log.debug("File will be split into {} chunks (chunk size: {} bytes)", totalChunks, chunkSize); + Long chunkSize = totalChunks == 1 ? fileSize : 5242880L; FileUploadSession session = FileUploadSession.builder() .userId(userId) .extension(extension) .fileName(fileName) .fileType(fileType) - .chunkSize(5242880L) + .chunkSize(chunkSize) .fileSize(fileSize) .totalChunks(totalChunks) .chunksUploaded(0) @@ -148,7 +149,12 @@ public class FileUploadServiceImpl implements FileUploadService { chunkNumber, session.getTotalChunks() - 1)); } - if (chunkFile.getSize() > chunkSize) { + if (chunkNumber == 0 && chunkFile.getSize() > chunkSize || chunkNumber + 1 == (session.getTotalChunks()) + && chunkFile.getSize() > chunkSize) { + throw new ChunkSizeExceededException(chunkFile.getSize(), chunkSize); + } + + if (chunkNumber > 0 && chunkNumber + 1 < session.getTotalChunks() && chunkFile.getSize() != chunkSize) { throw new ChunkSizeExceededException(chunkFile.getSize(), chunkSize); } @@ -333,36 +339,6 @@ public class FileUploadServiceImpl implements FileUploadService { } } -// private void assembleFile(FileUploadSession session) throws IOException { -// log.info("Starting file assembly for session: {} ({})", -// session.getUploadId(), session.getFileName()); -// -// Path finalFilePath = null; -// -// try { -// finalFilePath = prepareFinalFile(session); -// -// validateAllChunksExist(session); -// -// mergeChunksToFile(session, finalFilePath); -// -// validateFinalFile(session, finalFilePath); -// -// updateSessionOnSuccess(session, finalFilePath); -// -// cleanupSessionFiles(session); -// -// log.info("File assembly completed: {} -> {} ({} bytes)", -// session.getFileName(), finalFilePath, session.getFileSize()); -// -// } catch (Exception e) { -// if (finalFilePath != null) { -// Files.deleteIfExists(finalFilePath); -// } -// throw e; -// } -// } - private void assembleFile(FileUploadSession session) throws IOException { log.info("Starting file assembly for session: {} ({})", session.getUploadId(), session.getFileName()); From c5ef58ddd4e6798086a71d06342d24a797b70e6d Mon Sep 17 00:00:00 2001 From: vladp Date: Wed, 24 Dec 2025 14:20:49 +0700 Subject: [PATCH 09/37] fix after test --- .../dto/file/FileExtensionResponse.java | 17 +++++++++++++++ .../nocopy/handler/FileUploadHandler.java | 21 +++++++++++++++++++ 2 files changed, 38 insertions(+) create mode 100644 src/main/java/ru/soune/nocopy/dto/file/FileExtensionResponse.java diff --git a/src/main/java/ru/soune/nocopy/dto/file/FileExtensionResponse.java b/src/main/java/ru/soune/nocopy/dto/file/FileExtensionResponse.java new file mode 100644 index 0000000..026def0 --- /dev/null +++ b/src/main/java/ru/soune/nocopy/dto/file/FileExtensionResponse.java @@ -0,0 +1,17 @@ +package ru.soune.nocopy.dto.file; + +import com.fasterxml.jackson.annotation.JsonProperty; +import lombok.Builder; +import lombok.Data; + +import java.util.List; + +@Data +@Builder +public class FileExtensionResponse { + @JsonProperty("file_extension") + private List extension; + + @JsonProperty("count") + private Integer count; +} diff --git a/src/main/java/ru/soune/nocopy/handler/FileUploadHandler.java b/src/main/java/ru/soune/nocopy/handler/FileUploadHandler.java index 7674f4c..a0a4366 100644 --- a/src/main/java/ru/soune/nocopy/handler/FileUploadHandler.java +++ b/src/main/java/ru/soune/nocopy/handler/FileUploadHandler.java @@ -44,6 +44,8 @@ public class FileUploadHandler implements RequestHandler { return handleInitUpload(request, fileRequest); case "file_types": return handleGetFileTypes(request); + case "file_extension": + return handleGetFileExtensions(fileRequest, request); case "chunks": return handleGetChunkStatus(request, fileRequest); case "cancel": @@ -243,4 +245,23 @@ public class FileUploadHandler implements RequestHandler { "Failed to get file types", null); } } + + private BaseResponse handleGetFileExtensions(FileUploadRequest fileRequest, BaseRequest request) { + try { + FileType fileType = FileType.valueOf(fileRequest.getFileType().toUpperCase()); + List allowedExtensions = fileType.getAllowedExtensions(); + + FileExtensionResponse response = FileExtensionResponse.builder() + .extension(allowedExtensions) + .count(allowedExtensions.size()) + .build(); + + return new BaseResponse(request.getMsgId(), MessageCode.SUCCESS.getCode(), + MessageCode.SUCCESS.getDescription(), response); + } catch (Exception e) { + log.error("Error getting file extensions", e); + return new BaseResponse(request.getMsgId(), MessageCode.FILE_UPLOAD_ERROR.getCode(), + "Failed to get file extensions", null); + } + } } From d842af2c62b3460eaa119b2ac1d10afa146648d0 Mon Sep 17 00:00:00 2001 From: vladp Date: Wed, 24 Dec 2025 16:55:07 +0700 Subject: [PATCH 10/37] add file size --- .../ru/soune/nocopy/dto/file/FileExtensionResponse.java | 3 +++ .../java/ru/soune/nocopy/handler/FileUploadHandler.java | 9 +++++++++ 2 files changed, 12 insertions(+) diff --git a/src/main/java/ru/soune/nocopy/dto/file/FileExtensionResponse.java b/src/main/java/ru/soune/nocopy/dto/file/FileExtensionResponse.java index 026def0..6377de9 100644 --- a/src/main/java/ru/soune/nocopy/dto/file/FileExtensionResponse.java +++ b/src/main/java/ru/soune/nocopy/dto/file/FileExtensionResponse.java @@ -14,4 +14,7 @@ public class FileExtensionResponse { @JsonProperty("count") private Integer count; + + @JsonProperty("max_file_size") + private Long maxFileSize; } diff --git a/src/main/java/ru/soune/nocopy/handler/FileUploadHandler.java b/src/main/java/ru/soune/nocopy/handler/FileUploadHandler.java index a0a4366..27ab858 100644 --- a/src/main/java/ru/soune/nocopy/handler/FileUploadHandler.java +++ b/src/main/java/ru/soune/nocopy/handler/FileUploadHandler.java @@ -3,6 +3,7 @@ package ru.soune.nocopy.handler; import com.fasterxml.jackson.databind.ObjectMapper; import lombok.RequiredArgsConstructor; import lombok.extern.slf4j.Slf4j; +import org.springframework.beans.factory.annotation.Value; import org.springframework.stereotype.Component; import org.springframework.validation.BeanPropertyBindingResult; import org.springframework.validation.BindingResult; @@ -26,11 +27,18 @@ import java.util.stream.Collectors; @RequiredArgsConstructor public class FileUploadHandler implements RequestHandler { private final FileUploadService fileUploadService; + private final AuthTokenRepository authTokenRepository; + private final FileUploadSessionRepository fileUploadSessionRepository; + private final ObjectMapper objectMapper; + private final FileUploadRequestValidator fileUploadRequestValidator; + @Value("${file.storage.max-file-size}") + private long maxFileSize; + @Override public BaseResponse handle(BaseRequest request) { try { @@ -254,6 +262,7 @@ public class FileUploadHandler implements RequestHandler { FileExtensionResponse response = FileExtensionResponse.builder() .extension(allowedExtensions) .count(allowedExtensions.size()) + .maxFileSize(maxFileSize) .build(); return new BaseResponse(request.getMsgId(), MessageCode.SUCCESS.getCode(), From b2cf06a512b179035c3211dd060caedc3d0e6b84 Mon Sep 17 00:00:00 2001 From: vladp Date: Wed, 24 Dec 2025 17:26:13 +0700 Subject: [PATCH 11/37] update name controllers ends --- .../nocopy/controller/ApiController.java | 67 ------------------- .../nocopy/handler/FileEntityHandler.java | 12 ++-- .../nocopy/handler/FileUploadHandler.java | 4 +- 3 files changed, 7 insertions(+), 76 deletions(-) diff --git a/src/main/java/ru/soune/nocopy/controller/ApiController.java b/src/main/java/ru/soune/nocopy/controller/ApiController.java index ea129c0..285857b 100644 --- a/src/main/java/ru/soune/nocopy/controller/ApiController.java +++ b/src/main/java/ru/soune/nocopy/controller/ApiController.java @@ -279,73 +279,6 @@ public class ApiController { } } - @GetMapping("/v{version}/files/info/{fileId}") - public ResponseEntity getFileInfo( - @PathVariable String fileId, - @PathVariable Integer version, - @RequestHeader("Authorization") String tokenHeader) { - - try { - Long userId = getUserIdFromToken(tokenHeader); - FileEntityResponse fileInfo = fileEntityService.getById(fileId, version); - - if (!fileInfo.getUserId().equals(userId)) { - return ResponseEntity.status(403).build(); - } - - return ResponseEntity.ok(fileInfo); - - } catch (NotFoundAuthToken e) { - return ResponseEntity.status(401).build(); - } catch (Exception e) { - log.error("Error getting file info", e); - return ResponseEntity.status(500).build(); - } - } - - @GetMapping("/v{version}/files/preview/{fileId}") - public ResponseEntity previewFile( - @PathVariable String fileId, - @PathVariable Integer version, - @RequestHeader("Authorization") String tokenHeader) { - - try { - Long userId = getUserIdFromToken(tokenHeader); - FileEntityResponse fileInfo = fileEntityService.getById(fileId, version); - - if (!fileInfo.getUserId().equals(userId)) { - return ResponseEntity.status(403).build(); - } - - if (!isPreviewSupported(fileInfo.getMimeType())) { - return ResponseEntity.status(415) - .body(null); - } - - Path filePath = Paths.get(fileInfo.getFilePath()); - Resource resource = new UrlResource(filePath.toUri()); - - if (!resource.exists()) { - return ResponseEntity.status(404).build(); - } - - String contentType = determineContentType(filePath); - - return ResponseEntity.ok() - .contentType(MediaType.parseMediaType(contentType)) - .header(HttpHeaders.CONTENT_DISPOSITION, - "inline; filename=\"" + fileInfo.getOriginalFileName() + "\"") - .body(resource); - - } catch (NotFoundAuthToken e) { - return ResponseEntity.status(401).build(); - } catch (Exception e) { - log.error("Error previewing file", e); - return ResponseEntity.status(500).build(); - } - } - - private ResponseEntity createValidationErrorResponse(BindingResult bindingResult, Integer msgId) { List> fieldErrors = bindingResult.getFieldErrors() .stream() diff --git a/src/main/java/ru/soune/nocopy/handler/FileEntityHandler.java b/src/main/java/ru/soune/nocopy/handler/FileEntityHandler.java index bbc8da9..bf0ace9 100644 --- a/src/main/java/ru/soune/nocopy/handler/FileEntityHandler.java +++ b/src/main/java/ru/soune/nocopy/handler/FileEntityHandler.java @@ -31,15 +31,15 @@ public class FileEntityHandler implements RequestHandler { String action = fileRequest.getAction(); switch (action) { - case "get_file_info": + case "file_info": return handleGetFileInfo(request, fileRequest); - case "get_file_by_session": + case "file_by_session": return handleGetFileBySession(request, fileRequest); - case "get_user_files": + case "user_files": return handleGetUserFiles(request, fileRequest); case "search_files": return handleSearchFiles(request, fileRequest); - case "get_storage_usage": + case "storage_usage": return handleGetStorageUsage(request, fileRequest); case "delete_file": return handleDeleteFile(request, fileRequest); @@ -47,8 +47,8 @@ public class FileEntityHandler implements RequestHandler { ActionResponse response = ActionResponse.builder() .action(action) .availableActions(Arrays.asList( - "get_file_info", "get_file_by_session", "get_user_files", - "search_files", "get_storage_usage", "delete_file")) + "file_info", "file_by_session", "user_files", + "search_files", "storage_usage", "delete_file")) .build(); return new BaseResponse(request.getMsgId(), diff --git a/src/main/java/ru/soune/nocopy/handler/FileUploadHandler.java b/src/main/java/ru/soune/nocopy/handler/FileUploadHandler.java index 27ab858..4cb33b5 100644 --- a/src/main/java/ru/soune/nocopy/handler/FileUploadHandler.java +++ b/src/main/java/ru/soune/nocopy/handler/FileUploadHandler.java @@ -60,12 +60,10 @@ public class FileUploadHandler implements RequestHandler { return handleCancelUpload(request, fileRequest); case "progress": return handleGetProgress(request, fileRequest); - case "retry": - return handleRetryUpload(request, fileRequest); default: ActionResponse response = ActionResponse.builder() .action(action) - .availableActions(Arrays.asList("init", "file_types", "chunks", "cancel", "progress", "retry")) + .availableActions(Arrays.asList("init", "file_types", "chunks", "cancel", "progress")) .build(); return new BaseResponse(request.getMsgId(), MessageCode.INVALID_ACTION.getCode(), From 45b7acc774d763135a4c5790b12d3f7bb6fc1219 Mon Sep 17 00:00:00 2001 From: vladp Date: Thu, 25 Dec 2025 13:04:49 +0700 Subject: [PATCH 12/37] fix after test --- docker-compose.yaml | 2 +- .../nocopy/controller/ApiController.java | 49 ++++++++++--------- .../service/file/FileUploadServiceImpl.java | 2 +- 3 files changed, 29 insertions(+), 24 deletions(-) diff --git a/docker-compose.yaml b/docker-compose.yaml index ce90ce2..91e8d40 100644 --- a/docker-compose.yaml +++ b/docker-compose.yaml @@ -33,7 +33,7 @@ services: container_name: app-backend environment: FILE_STORAGE_PATH: /data/uploads - MAX_FILE_SIZE: 1073741824 + MAX_FILE_SIZE: 10737418240 FILE_CHUNK_SIZE: 5242880 POSTGRES_DB: no_copy_ POSTGRES_USER: postgres diff --git a/src/main/java/ru/soune/nocopy/controller/ApiController.java b/src/main/java/ru/soune/nocopy/controller/ApiController.java index 285857b..153e38a 100644 --- a/src/main/java/ru/soune/nocopy/controller/ApiController.java +++ b/src/main/java/ru/soune/nocopy/controller/ApiController.java @@ -89,29 +89,34 @@ public class ApiController { @PostMapping("/v{version}/files/chunk") public ResponseEntity uploadChunk( @PathVariable("version") int version, - @RequestParam("upload_id") String uploadId, - @RequestParam("chunk_number") Integer chunkNumber, - @RequestParam("chunk") MultipartFile chunk) { - - log.info("Uploading chunk {} for session {}, file size: {} bytes, version: {}", - chunkNumber, uploadId, chunk.getSize(), version); - - if (chunk.isEmpty()) { - return ResponseEntity.ok().body(new BaseResponse(20004, MessageCode.FILE_UPLOAD_ERROR.getCode(), - "Chunk file is empty", null)); - } - - if (uploadId == null || uploadId.isBlank()) { - return ResponseEntity.ok().body(new BaseResponse(20004, MessageCode.FILE_UPLOAD_ERROR.getCode(), - "Upload ID is required", null)); - } - - if (chunkNumber == null || chunkNumber < 0) { - return ResponseEntity.ok().body(new BaseResponse(20004, MessageCode.FILE_UPLOAD_ERROR.getCode(), - "Valid chunk number is required", null)); - } - + @RequestParam(value = "upload_id", required = false) String uploadId, + @RequestParam(value = "chunk_number", required = false) Integer chunkNumber, + @RequestParam(value = "chunk", required = false) MultipartFile chunk) { try { + if (chunk == null || chunk.isEmpty()) { + return ResponseEntity.ok().body(new BaseResponse(20004, MessageCode.FILE_UPLOAD_ERROR.getCode(), + "Chunk file null or empty", ChunkUploadResponse.builder() + .uploadId(uploadId) + .chunkNumber(chunkNumber) + .build())); + } + + if (uploadId == null || uploadId.isBlank()) { + return ResponseEntity.ok().body(new BaseResponse(20004, MessageCode.FILE_UPLOAD_ERROR.getCode(), + "Upload ID is required", ChunkUploadResponse.builder() + .uploadId(uploadId) + .chunkNumber(chunkNumber) + .build())); + } + + if (chunkNumber == null || chunkNumber < 0) { + return ResponseEntity.ok().body(new BaseResponse(20004, MessageCode.FILE_UPLOAD_ERROR.getCode(), + "Valid chunk number is required", ChunkUploadResponse.builder() + .uploadId(uploadId) + .chunkNumber(chunkNumber) + .build())); + } + fileUploadService.uploadChunk(uploadId, chunkNumber, chunk); ChunkUploadResponse responseBody = ChunkUploadResponse.builder() diff --git a/src/main/java/ru/soune/nocopy/service/file/FileUploadServiceImpl.java b/src/main/java/ru/soune/nocopy/service/file/FileUploadServiceImpl.java index 93c7d67..c84cf05 100644 --- a/src/main/java/ru/soune/nocopy/service/file/FileUploadServiceImpl.java +++ b/src/main/java/ru/soune/nocopy/service/file/FileUploadServiceImpl.java @@ -145,7 +145,7 @@ public class FileUploadServiceImpl implements FileUploadService { if (chunkNumber < 0 || chunkNumber >= session.getTotalChunks()) { throw new FileUploadException( - String.format("Invalid chunk number %d. Expected 0-%d", + String.format("Invalid chunk number %d. Expected number: %d", chunkNumber, session.getTotalChunks() - 1)); } From 18ec9b51f3158a7ac82d2fdc47b07480fa8b7cd8 Mon Sep 17 00:00:00 2001 From: vladp Date: Thu, 25 Dec 2025 15:27:45 +0700 Subject: [PATCH 13/37] add clean logic for uploadsessions --- .../ru/soune/nocopy/NoCopyApplication.java | 2 + .../nocopy/controller/ApiController.java | 1 - .../nocopy/entity/file/FileUploadSession.java | 2 +- .../service/file/FileUploadService.java | 2 +- .../service/file/FileUploadServiceImpl.java | 225 +++++++++--------- .../file/UploadSessionCleanupService.java | 35 +++ 6 files changed, 150 insertions(+), 117 deletions(-) create mode 100644 src/main/java/ru/soune/nocopy/service/file/UploadSessionCleanupService.java diff --git a/src/main/java/ru/soune/nocopy/NoCopyApplication.java b/src/main/java/ru/soune/nocopy/NoCopyApplication.java index fd6b0f2..0c9c534 100644 --- a/src/main/java/ru/soune/nocopy/NoCopyApplication.java +++ b/src/main/java/ru/soune/nocopy/NoCopyApplication.java @@ -2,8 +2,10 @@ package ru.soune.nocopy; import org.springframework.boot.SpringApplication; import org.springframework.boot.autoconfigure.SpringBootApplication; +import org.springframework.scheduling.annotation.EnableScheduling; @SpringBootApplication +@EnableScheduling public class NoCopyApplication { public static void main(String[] args) { diff --git a/src/main/java/ru/soune/nocopy/controller/ApiController.java b/src/main/java/ru/soune/nocopy/controller/ApiController.java index 153e38a..20ca112 100644 --- a/src/main/java/ru/soune/nocopy/controller/ApiController.java +++ b/src/main/java/ru/soune/nocopy/controller/ApiController.java @@ -128,7 +128,6 @@ public class ApiController { return ResponseEntity.ok().body(new BaseResponse(20004, MessageCode.SUCCESS.getCode(), "Chunk uploaded successfully", responseBody)); - } catch (Exception e) { log.error("Error uploading chunk", e); diff --git a/src/main/java/ru/soune/nocopy/entity/file/FileUploadSession.java b/src/main/java/ru/soune/nocopy/entity/file/FileUploadSession.java index 996f0de..509d0f7 100644 --- a/src/main/java/ru/soune/nocopy/entity/file/FileUploadSession.java +++ b/src/main/java/ru/soune/nocopy/entity/file/FileUploadSession.java @@ -86,7 +86,7 @@ public class FileUploadSession { @PrePersist public void prePersist() { this.createdAt = LocalDateTime.now(); - this.expiresAt = LocalDateTime.now().plusHours(24); + this.expiresAt = LocalDateTime.now().plusMinutes(1); if (this.chunksUploaded == null) { this.chunksUploaded = 0; } diff --git a/src/main/java/ru/soune/nocopy/service/file/FileUploadService.java b/src/main/java/ru/soune/nocopy/service/file/FileUploadService.java index cfb9aec..b27dede 100644 --- a/src/main/java/ru/soune/nocopy/service/file/FileUploadService.java +++ b/src/main/java/ru/soune/nocopy/service/file/FileUploadService.java @@ -15,7 +15,7 @@ public interface FileUploadService { UploadProgressResponse getUploadProgress(String uploadId); - void cleanupExpiredSessions(); + void handleExpiredSession(FileUploadSession session); public void retryFailedUpload(String uploadId); diff --git a/src/main/java/ru/soune/nocopy/service/file/FileUploadServiceImpl.java b/src/main/java/ru/soune/nocopy/service/file/FileUploadServiceImpl.java index c84cf05..908602f 100644 --- a/src/main/java/ru/soune/nocopy/service/file/FileUploadServiceImpl.java +++ b/src/main/java/ru/soune/nocopy/service/file/FileUploadServiceImpl.java @@ -161,6 +161,87 @@ public class FileUploadServiceImpl implements FileUploadService { return processChunk(session, chunkNumber, chunkFile); } + @Override + @Transactional + public void handleExpiredSession(FileUploadSession session) { + session.setStatus(UploadStatus.FAILED); + session.setLastError("Upload session expired"); + sessionRepository.save(session); + + CompletableFuture.runAsync(() -> cleanupSessionFiles(session)); + } + + @Async("fileUploadTaskExecutor") + @Transactional + public void assembleFileAsync(FileUploadSession session) { + try { + assembleFile(session); + log.info("File assembly completed successfully for session: {}", + session.getUploadId()); + + } catch (Exception e) { + log.error("Failed to assemble file for session {}: {}", + session.getUploadId(), e.getMessage(), e); + + handleAssemblyFailure(session, e); + } + } + + @Override + @Transactional + public void cancelUpload(String uploadId) { + FileUploadSession session = sessionRepository.findById(uploadId) + .orElseThrow(() -> new UploadSessionNotFoundException(uploadId)); + + if (session.getStatus() == UploadStatus.COMPLETED || session.getStatus() == UploadStatus.CANCELLED) { + throw new FileUploadException("Cannot cancel completed or cancelled upload"); + } + + session.setStatus(UploadStatus.CANCELLED); + sessionRepository.save(session); + + CompletableFuture.runAsync(() -> cleanupSessionFiles(session)); + + log.info("Upload cancelled: {}", uploadId); + } + + @Override + @Transactional + public void retryFailedUpload(String uploadId) { + FileUploadSession session = sessionRepository.findById(uploadId) + .orElseThrow(() -> new UploadSessionNotFoundException(uploadId)); + + if (session.getStatus() != UploadStatus.FAILED) { + throw new FileUploadException("Only failed uploads can be retried"); + } + + if (session.getRetryCount() >= maxRetryAttempts) { + throw new FileUploadException("Max retry attempts exceeded"); + } + + session.setStatus(UploadStatus.UPLOADING); + session.setRetryCount(session.getRetryCount() + 1); + session.setLastError(null); + + sessionRepository.save(session); + + if (session.getChunksUploaded().equals(session.getTotalChunks())) { + log.info("Retrying file assembly for session: {}", uploadId); + assembleFileAsync(session); + } + + log.info("Upload retry initiated for session: {} (attempt {})", + uploadId, session.getRetryCount()); + } + + @Override + public UploadProgressResponse getUploadProgress(String uploadId) { + FileUploadSession session = sessionRepository.findById(uploadId) + .orElseThrow(() -> new UploadSessionNotFoundException(uploadId)); + + return UploadProgressResponse.fromSession(session); + } + private void validateSession(FileUploadSession session) { UploadStatus status = session.getStatus(); @@ -190,14 +271,6 @@ public class FileUploadServiceImpl implements FileUploadService { } } - private void handleExpiredSession(FileUploadSession session) { - session.setStatus(UploadStatus.FAILED); - session.setLastError("Upload session expired"); - sessionRepository.save(session); - - CompletableFuture.runAsync(() -> cleanupSessionFiles(session)); - } - private UploadProgressResponse processChunk(FileUploadSession session, Integer chunkNumber, MultipartFile chunkFile) { @@ -213,6 +286,7 @@ public class FileUploadServiceImpl implements FileUploadService { session.getChunkPaths().put(chunkNumber, chunkPath); session.setChunksUploaded(session.getChunksUploaded() + 1); session.setStatus(UploadStatus.UPLOADING); + session.setExpiresAt(LocalDateTime.now().plusMinutes(1)); log.debug("Chunk {} saved successfully. Uploaded: {}/{}", chunkNumber, session.getChunksUploaded(), session.getTotalChunks()); @@ -323,22 +397,6 @@ public class FileUploadServiceImpl implements FileUploadService { } } - @Async("fileUploadTaskExecutor") - @Transactional - public void assembleFileAsync(FileUploadSession session) { - try { - assembleFile(session); - log.info("File assembly completed successfully for session: {}", - session.getUploadId()); - - } catch (Exception e) { - log.error("Failed to assemble file for session {}: {}", - session.getUploadId(), e.getMessage(), e); - - handleAssemblyFailure(session, e); - } - } - private void assembleFile(FileUploadSession session) throws IOException { log.info("Starting file assembly for session: {} ({})", session.getUploadId(), session.getFileName()); @@ -503,14 +561,6 @@ public class FileUploadServiceImpl implements FileUploadService { log.debug("Final file checksum: {}", checksum); } - private void updateSessionOnSuccess(FileUploadSession session, Path finalPath) { - session.setFilePath(finalPath.toString()); - session.setChecksum(calculateChecksum(finalPath)); - session.setStatus(UploadStatus.COMPLETED); - session.setCompletedAt(LocalDateTime.now()); - sessionRepository.save(session); - } - private void cleanupSessionFiles(FileUploadSession session) { try { Path chunkDir = getChunkDirectory(session.getUserId(), session.getUploadId()); @@ -547,90 +597,37 @@ public class FileUploadServiceImpl implements FileUploadService { session.getUploadId(), e.getMessage()); } - @Override - @Transactional - public void cancelUpload(String uploadId) { - FileUploadSession session = sessionRepository.findById(uploadId) - .orElseThrow(() -> new UploadSessionNotFoundException(uploadId)); - if (session.getStatus() == UploadStatus.COMPLETED || session.getStatus() == UploadStatus.CANCELLED) { - throw new FileUploadException("Cannot cancel completed or cancelled upload"); - } - session.setStatus(UploadStatus.CANCELLED); - sessionRepository.save(session); - - CompletableFuture.runAsync(() -> cleanupSessionFiles(session)); - - log.info("Upload cancelled: {}", uploadId); - } - - @Override - @Transactional - public void retryFailedUpload(String uploadId) { - FileUploadSession session = sessionRepository.findById(uploadId) - .orElseThrow(() -> new UploadSessionNotFoundException(uploadId)); - - if (session.getStatus() != UploadStatus.FAILED) { - throw new FileUploadException("Only failed uploads can be retried"); - } - - if (session.getRetryCount() >= maxRetryAttempts) { - throw new FileUploadException("Max retry attempts exceeded"); - } - - session.setStatus(UploadStatus.UPLOADING); - session.setRetryCount(session.getRetryCount() + 1); - session.setLastError(null); - - sessionRepository.save(session); - - if (session.getChunksUploaded().equals(session.getTotalChunks())) { - log.info("Retrying file assembly for session: {}", uploadId); - assembleFileAsync(session); - } - - log.info("Upload retry initiated for session: {} (attempt {})", - uploadId, session.getRetryCount()); - } - - @Override - public UploadProgressResponse getUploadProgress(String uploadId) { - FileUploadSession session = sessionRepository.findById(uploadId) - .orElseThrow(() -> new UploadSessionNotFoundException(uploadId)); - - return UploadProgressResponse.fromSession(session); - } - - @Override - @Transactional - public void cleanupExpiredSessions() { - log.info("Starting cleanup of expired upload sessions"); - - LocalDateTime expiryThreshold = LocalDateTime.now().minusHours(sessionExpiryHours); - - try { - var expiredSessions = sessionRepository.findByStatusInAndCreatedAtBefore( - Set.of(UploadStatus.INITIATED, UploadStatus.UPLOADING, UploadStatus.FAILED), - expiryThreshold - ); - - int cleanedCount = 0; - for (FileUploadSession session : expiredSessions) { - session.setStatus(UploadStatus.FAILED); - session.setLastError("Session expired during cleanup"); - sessionRepository.save(session); - - CompletableFuture.runAsync(() -> cleanupSessionFiles(session)); - cleanedCount++; - } - - log.info("Cleaned up {} expired upload sessions", cleanedCount); - - } catch (Exception e) { - log.error("Error during session cleanup", e); - } - } +// @Override +// @Transactional +// public void cleanupExpiredSessions() { +// log.info("Starting cleanup of expired upload sessions"); +// +// LocalDateTime expiryThreshold = LocalDateTime.now().minusHours(sessionExpiryHours); +// +// try { +// var expiredSessions = sessionRepository.findByStatusInAndCreatedAtBefore( +// Set.of(UploadStatus.INITIATED, UploadStatus.UPLOADING, UploadStatus.FAILED), +// expiryThreshold +// ); +// +// int cleanedCount = 0; +// for (FileUploadSession session : expiredSessions) { +// session.setStatus(UploadStatus.FAILED); +// session.setLastError("Session expired during cleanup"); +// sessionRepository.save(session); +// +// CompletableFuture.runAsync(() -> cleanupSessionFiles(session)); +// cleanedCount++; +// } +// +// log.info("Cleaned up {} expired upload sessions", cleanedCount); +// +// } catch (Exception e) { +// log.error("Error during session cleanup", e); +// } +// } private Path getChunkDirectory(Long userId, String uploadId) { return storageRoot.resolve("temp") diff --git a/src/main/java/ru/soune/nocopy/service/file/UploadSessionCleanupService.java b/src/main/java/ru/soune/nocopy/service/file/UploadSessionCleanupService.java new file mode 100644 index 0000000..2c68604 --- /dev/null +++ b/src/main/java/ru/soune/nocopy/service/file/UploadSessionCleanupService.java @@ -0,0 +1,35 @@ +package ru.soune.nocopy.service.file; + +import lombok.RequiredArgsConstructor; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.scheduling.annotation.Scheduled; +import org.springframework.stereotype.Service; +import org.springframework.transaction.annotation.Transactional; +import ru.soune.nocopy.entity.file.FileUploadSession; +import ru.soune.nocopy.entity.file.UploadStatus; +import ru.soune.nocopy.repository.FileUploadSessionRepository; + +import java.time.LocalDateTime; +import java.util.List; +import java.util.Set; + +@Service +@RequiredArgsConstructor +public class UploadSessionCleanupService { + @Autowired + private FileUploadSessionRepository fileUploadSessionRepository; + + @Autowired + private FileUploadService fileUploadService; + + @Transactional + @Scheduled(fixedDelay = 30000) + public void cleanupSessions() { + List expiredSessions = fileUploadSessionRepository.findExpiredSessions( + LocalDateTime.now(), Set.of(UploadStatus.INITIATED, UploadStatus.UPLOADING)); + + for (FileUploadSession session : expiredSessions) { + fileUploadService.handleExpiredSession(session); + } + } +} From a2c20ae645140cdab9811bd253e5fd789949950a Mon Sep 17 00:00:00 2001 From: vladp Date: Fri, 26 Dec 2025 12:22:40 +0700 Subject: [PATCH 14/37] add exceptions --- .../nocopy/controller/ApiController.java | 50 +++++++++++-------- .../java/ru/soune/nocopy/dto/MessageCode.java | 2 + 2 files changed, 32 insertions(+), 20 deletions(-) diff --git a/src/main/java/ru/soune/nocopy/controller/ApiController.java b/src/main/java/ru/soune/nocopy/controller/ApiController.java index 20ca112..eb1d8a4 100644 --- a/src/main/java/ru/soune/nocopy/controller/ApiController.java +++ b/src/main/java/ru/soune/nocopy/controller/ApiController.java @@ -21,6 +21,7 @@ import ru.soune.nocopy.dto.file.FileEntityResponse; import ru.soune.nocopy.dto.file.UploadProgress; import ru.soune.nocopy.entity.AuthToken; import ru.soune.nocopy.entity.file.UploadStatus; +import ru.soune.nocopy.exception.FileEntityNotFoundException; import ru.soune.nocopy.exception.NotFoundAuthToken; import ru.soune.nocopy.exception.NotValidFieldException; import ru.soune.nocopy.exception.ValidationException; @@ -241,29 +242,32 @@ public class ApiController { @GetMapping("/v{version}/files/download/{fileId}") - public ResponseEntity downloadFile( - @PathVariable String fileId, - @PathVariable Integer version, - @RequestHeader("Authorization") String tokenHeader) { - + public ResponseEntity downloadFile( + @PathVariable(required = false) String fileId, + @PathVariable(required = false) Integer version, + @RequestHeader(value = "Authorization", required = false) String tokenHeader) { try { Long userId = getUserIdFromToken(tokenHeader); - FileEntityResponse fileInfo = fileEntityService.getById(fileId, version); + FileEntityResponse entityResponse = fileEntityService.getById(fileId, version); - if (!fileInfo.getUserId().equals(userId)) { - return ResponseEntity.status(403).build(); + if (!entityResponse.getUserId().equals(userId)) { + return ResponseEntity.ok().body(new BaseResponse(20004, MessageCode.FILE_DOWNLOAD_ERROR.getCode(), + MessageCode.FILE_DOWNLOAD_ERROR.getDescription(), Map.of("token", tokenHeader))); } - if (!fileInfo.isExistsOnDisk()) { - return ResponseEntity.status(404) - .body(null); + if (!entityResponse.isExistsOnDisk()) { + return ResponseEntity.ok().body(new BaseResponse(20004, MessageCode.FILE_DOWNLOAD_ERROR.getCode(), + MessageCode.FILE_DOWNLOAD_ERROR.getDescription(), + Map.of("onDisk", entityResponse.isExistsOnDisk()))); } - Path filePath = Paths.get(fileInfo.getFilePath()); + Path filePath = Paths.get(entityResponse.getFilePath()); Resource resource = new UrlResource(filePath.toUri()); if (!resource.exists()) { - return ResponseEntity.status(404).build(); + return ResponseEntity.ok().body(new BaseResponse(20004, MessageCode.FILE_DOWNLOAD_ERROR.getCode(), + MessageCode.FILE_DOWNLOAD_ERROR.getDescription(), + Map.of("resource", resource.exists()))); } String contentType = determineContentType(filePath); @@ -271,15 +275,21 @@ public class ApiController { return ResponseEntity.ok() .contentType(MediaType.parseMediaType(contentType)) .header(HttpHeaders.CONTENT_DISPOSITION, - "attachment; filename=\"" + fileInfo.getOriginalFileName() + "\"") - .header(HttpHeaders.CONTENT_LENGTH, String.valueOf(fileInfo.getFileSize())) + "attachment; filename=\"" + entityResponse.getOriginalFileName() + "\"") + .header(HttpHeaders.CONTENT_LENGTH, String.valueOf(entityResponse.getFileSize())) .body(resource); - - } catch (NotFoundAuthToken e) { - return ResponseEntity.status(401).build(); + } catch (FileEntityNotFoundException e) { + return ResponseEntity.ok().body(new BaseResponse(20004, + MessageCode.FILE_NOT_FOUND.getCode(), + MessageCode.FILE_NOT_FOUND.getDescription(), + Map.of("fileId", fileId))); } catch (Exception e) { - log.error("Error downloading file", e); - return ResponseEntity.status(500).build(); + return ResponseEntity.ok().body(new BaseResponse(20004, + MessageCode.FILE_DOWNLOAD_ERROR_NOT_CORRECT_FIELD.getCode(), + MessageCode.FILE_DOWNLOAD_ERROR_NOT_CORRECT_FIELD.getDescription(), + Map.of("token", tokenHeader, + "fileId", fileId, + "version", version))); } } diff --git a/src/main/java/ru/soune/nocopy/dto/MessageCode.java b/src/main/java/ru/soune/nocopy/dto/MessageCode.java index a3ac162..eef129c 100644 --- a/src/main/java/ru/soune/nocopy/dto/MessageCode.java +++ b/src/main/java/ru/soune/nocopy/dto/MessageCode.java @@ -8,6 +8,8 @@ public enum MessageCode { INVALID_TOKEN(2, "Invalid token"), INVALID_ACTION(2, "Invalid action"), FILE_UPLOAD_ERROR(2, "File upload error"), + FILE_DOWNLOAD_ERROR(2, "File download error"), + FILE_DOWNLOAD_ERROR_NOT_CORRECT_FIELD(2, "File download error with correct field"), INVALID_JSON_BODY(2, "Invalid fields in JSON object"), INCOMPLETE_UPLOAD(2, "Not load all chunks"), MSG_ID_NOT_FOUND(4, "Message id not found"), From 84993dec00090f8efeedc0fdd1276ea4dc991f24 Mon Sep 17 00:00:00 2001 From: vladp Date: Fri, 26 Dec 2025 13:52:53 +0700 Subject: [PATCH 15/37] fix page_size and filtered values --- .../nocopy/handler/FileEntityHandler.java | 27 ++++++++++--------- .../service/file/FileEntityService.java | 2 +- 2 files changed, 15 insertions(+), 14 deletions(-) diff --git a/src/main/java/ru/soune/nocopy/handler/FileEntityHandler.java b/src/main/java/ru/soune/nocopy/handler/FileEntityHandler.java index bf0ace9..e62dd4a 100644 --- a/src/main/java/ru/soune/nocopy/handler/FileEntityHandler.java +++ b/src/main/java/ru/soune/nocopy/handler/FileEntityHandler.java @@ -12,6 +12,7 @@ import ru.soune.nocopy.repository.AuthTokenRepository; import ru.soune.nocopy.service.file.FileEntityService; import java.util.Arrays; +import java.util.List; @Slf4j @Component @@ -168,17 +169,22 @@ public class FileEntityHandler implements RequestHandler { FileResponse allFiles = fileEntityService.getUserFiles(userId, 1, 1000, request.getVersion()); var filteredFiles = allFiles.getFiles().stream() - .filter(f -> f.getOriginalFileName().toLowerCase().contains(fileRequest.getQuery().toLowerCase())) + .filter(f -> f.getOriginalFileName().toLowerCase() + .contains(fileRequest.getQuery().toLowerCase())) .toList(); int start = (page - 1) * pageSize; int end = Math.min(start + pageSize, filteredFiles.size()); + List fileEntityResponses = filteredFiles.subList(start, Math.min(end, filteredFiles.size())); + long sumSize = fileEntityResponses.stream() + .mapToLong(FileEntityResponse::getFileSize) + .sum(); FileListResponse response = FileListResponse.builder() - .files(filteredFiles.subList(start, Math.min(end, filteredFiles.size()))) + .files(fileEntityResponses) .totalCount(filteredFiles.size()) - .totalSize(allFiles.getTotalSize()) - .formattedTotalSize(allFiles.getFormattedTotalSize()) + .totalSize(sumSize) + .formattedTotalSize(fileEntityService.formatFileSize(sumSize)) .page(page) .pageSize(pageSize) .build(); @@ -187,18 +193,13 @@ public class FileEntityHandler implements RequestHandler { MessageCode.SUCCESS.getCode(), MessageCode.SUCCESS.getDescription(), response); - } catch (NotFoundAuthToken e) { - return new BaseResponse(request.getMsgId(), - MessageCode.INVALID_TOKEN.getCode(), - "Authentication required", - null); + return new BaseResponse(request.getMsgId(), MessageCode.INVALID_TOKEN.getCode(), + "Authentication required", null); } catch (Exception e) { log.error("Error searching files", e); - return new BaseResponse(request.getMsgId(), - MessageCode.FILE_UPLOAD_ERROR.getCode(), - "Failed to search files: " + e.getMessage(), - null); + return new BaseResponse(request.getMsgId(), MessageCode.FILE_UPLOAD_ERROR.getCode(), + "Failed to search files: " + e.getMessage(), null); } } diff --git a/src/main/java/ru/soune/nocopy/service/file/FileEntityService.java b/src/main/java/ru/soune/nocopy/service/file/FileEntityService.java index 97a6cab..d21c63c 100644 --- a/src/main/java/ru/soune/nocopy/service/file/FileEntityService.java +++ b/src/main/java/ru/soune/nocopy/service/file/FileEntityService.java @@ -216,7 +216,7 @@ public class FileEntityService { .build(); } - private String formatFileSize(long size) { + public String formatFileSize(long size) { if (size < 1024) { return size + " B"; } else if (size < 1024 * 1024) { From 1ebe8b852ff85d272a53d93f41744ecd7b43d719 Mon Sep 17 00:00:00 2001 From: vladp Date: Fri, 26 Dec 2025 14:51:02 +0700 Subject: [PATCH 16/37] add file extension to search --- src/main/java/ru/soune/nocopy/handler/FileEntityHandler.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/java/ru/soune/nocopy/handler/FileEntityHandler.java b/src/main/java/ru/soune/nocopy/handler/FileEntityHandler.java index e62dd4a..6af6673 100644 --- a/src/main/java/ru/soune/nocopy/handler/FileEntityHandler.java +++ b/src/main/java/ru/soune/nocopy/handler/FileEntityHandler.java @@ -169,7 +169,7 @@ public class FileEntityHandler implements RequestHandler { FileResponse allFiles = fileEntityService.getUserFiles(userId, 1, 1000, request.getVersion()); var filteredFiles = allFiles.getFiles().stream() - .filter(f -> f.getOriginalFileName().toLowerCase() + .filter(f -> (f.getOriginalFileName() + f.getFileExtension()).toLowerCase() .contains(fileRequest.getQuery().toLowerCase())) .toList(); From 2a9f33068a87e0ca86418fe1c3675fa6af021449 Mon Sep 17 00:00:00 2001 From: vladp Date: Fri, 26 Dec 2025 16:23:17 +0700 Subject: [PATCH 17/37] add logic with hard delete --- .../nocopy/handler/FileEntityHandler.java | 34 ++++++++++++++----- .../service/file/FileEntityService.java | 22 ++++++++---- 2 files changed, 42 insertions(+), 14 deletions(-) diff --git a/src/main/java/ru/soune/nocopy/handler/FileEntityHandler.java b/src/main/java/ru/soune/nocopy/handler/FileEntityHandler.java index 6af6673..968a3b7 100644 --- a/src/main/java/ru/soune/nocopy/handler/FileEntityHandler.java +++ b/src/main/java/ru/soune/nocopy/handler/FileEntityHandler.java @@ -7,8 +7,12 @@ import org.springframework.stereotype.Component; import ru.soune.nocopy.dto.*; import ru.soune.nocopy.dto.file.*; import ru.soune.nocopy.entity.AuthToken; +import ru.soune.nocopy.entity.file.FileEntity; +import ru.soune.nocopy.entity.file.FileStatus; +import ru.soune.nocopy.exception.FileEntityNotFoundException; import ru.soune.nocopy.exception.NotFoundAuthToken; import ru.soune.nocopy.repository.AuthTokenRepository; +import ru.soune.nocopy.repository.FileEntityRepository; import ru.soune.nocopy.service.file.FileEntityService; import java.util.Arrays; @@ -18,11 +22,14 @@ import java.util.List; @Component @RequiredArgsConstructor public class FileEntityHandler implements RequestHandler { - private final FileEntityService fileEntityService; + private final AuthTokenRepository authTokenRepository; + private final ObjectMapper objectMapper; + private final FileEntityRepository fileEntityRepository; + @Override public BaseResponse handle(BaseRequest request) { try { @@ -236,7 +243,9 @@ public class FileEntityHandler implements RequestHandler { private BaseResponse handleDeleteFile(BaseRequest request, FileEntityRequest fileRequest) { try { Long userId = getUserIdFromToken(fileRequest.getToken()); - FileEntityResponse fileInfo = fileEntityService.getById(fileRequest.getFileId(), request.getVersion()); + String fileId = fileRequest.getFileId(); + FileEntityResponse fileInfo = fileEntityService.getById(fileId, request.getVersion()); + DeleteFileResponse response; if (!fileInfo.getUserId().equals(userId)) { return new BaseResponse(request.getMsgId(), @@ -245,18 +254,27 @@ public class FileEntityHandler implements RequestHandler { null); } - fileEntityService.markAsDeleted(fileRequest.getFileId()); + FileEntity fileEntity = fileEntityRepository.findById(fileId) + .orElseThrow(() -> new FileEntityNotFoundException(fileId)); - DeleteFileResponse response = DeleteFileResponse.builder() - .fileId(fileRequest.getFileId()) - .message("File marked as deleted") - .build(); + if (fileEntity.getStatus().equals(FileStatus.DELETED)) { + fileEntityService.deleteFromDisk(fileEntity); + response = DeleteFileResponse.builder() + .fileId(fileRequest.getFileId()) + .message("File deleted from disk") + .build(); + } else { + fileEntityService.markAsDeleted(fileEntity); + response = DeleteFileResponse.builder() + .fileId(fileRequest.getFileId()) + .message("File marked as deleted") + .build(); + } return new BaseResponse(request.getMsgId(), MessageCode.SUCCESS.getCode(), "File marked as deleted", response); - } catch (NotFoundAuthToken e) { return new BaseResponse(request.getMsgId(), MessageCode.INVALID_TOKEN.getCode(), diff --git a/src/main/java/ru/soune/nocopy/service/file/FileEntityService.java b/src/main/java/ru/soune/nocopy/service/file/FileEntityService.java index d21c63c..617eeda 100644 --- a/src/main/java/ru/soune/nocopy/service/file/FileEntityService.java +++ b/src/main/java/ru/soune/nocopy/service/file/FileEntityService.java @@ -17,6 +17,7 @@ import java.nio.file.Files; import java.nio.file.Path; import java.nio.file.Paths; import java.time.LocalDateTime; +import java.time.ZoneOffset; import java.util.List; import java.util.stream.Collectors; @@ -167,15 +168,24 @@ public class FileEntityService { } @Transactional - public void markAsDeleted(String fileId) { - FileEntity fileEntity = fileEntityRepository.findById(fileId) - .orElseThrow(() -> new FileEntityNotFoundException(fileId)); - + public void markAsDeleted(FileEntity fileEntity) { fileEntity.setStatus(FileStatus.DELETED); fileEntity.setUpdatedAt(LocalDateTime.now()); - fileEntityRepository.save(fileEntity); - log.info("FileEntity marked as deleted: {}", fileId); + fileEntityRepository.save(fileEntity); + } + + @Transactional + public boolean deleteFromDisk(FileEntity fileEntity) throws IOException { + Path path = Paths.get(fileEntity.getFilePath()); + + if (!Files.exists(path)) { + return true; + } + + Files.delete(path); + + return true; } @Transactional(readOnly = true) From 2c1a2aeb1689ef3c87276ac1584196c23b32571f Mon Sep 17 00:00:00 2001 From: vladp Date: Fri, 26 Dec 2025 19:10:43 +0700 Subject: [PATCH 18/37] add post size --- src/main/resources/application.yaml | 20 +++++++++++++++----- 1 file changed, 15 insertions(+), 5 deletions(-) diff --git a/src/main/resources/application.yaml b/src/main/resources/application.yaml index 08326e1..511bd2f 100644 --- a/src/main/resources/application.yaml +++ b/src/main/resources/application.yaml @@ -16,10 +16,11 @@ spring: servlet: multipart: - max-file-size: 10MB - max-request-size: 100MB enabled: true - resolve-lazily: false + resolve-lazily: true + file-size-threshold: 0 + max-file-size: 5242880 + max-request-size: 10485760 file: storage: @@ -31,6 +32,12 @@ file: temp-ttl-hours: ${TEMP_TTL_HOURS:72} # 3 дня session-expiry-hours: ${SESSION_EXPIRY_HOURS:24} +server: + port: ${SERVER_PORT:8080} + tomcat: + max-swallow-size: -1 # Отключаем ограничения Tomcat + max-http-form-post-size: 10 + security: cors: allowed-origins: "*" @@ -38,8 +45,11 @@ security: allowed-headers: "*" allow-credentials: true -server: - port: ${SERVER_PORT:8080} +#server: +# port: ${SERVER_PORT:8080} +# +# + logging: level: From 4fa6b83a10ee04f6cdd24b2d0cedc99645df34fe Mon Sep 17 00:00:00 2001 From: vladp Date: Fri, 26 Dec 2025 19:15:45 +0700 Subject: [PATCH 19/37] add post size --- src/main/resources/application.yaml | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/src/main/resources/application.yaml b/src/main/resources/application.yaml index 511bd2f..0ad1403 100644 --- a/src/main/resources/application.yaml +++ b/src/main/resources/application.yaml @@ -16,11 +16,10 @@ spring: servlet: multipart: + max-file-size: 10MB + max-request-size: 100MB enabled: true - resolve-lazily: true - file-size-threshold: 0 - max-file-size: 5242880 - max-request-size: 10485760 + resolve-lazily: false file: storage: @@ -35,7 +34,7 @@ file: server: port: ${SERVER_PORT:8080} tomcat: - max-swallow-size: -1 # Отключаем ограничения Tomcat + max-swallow-size: -1 max-http-form-post-size: 10 security: From 91e9b0c785c1557691913629d8cd160248111010 Mon Sep 17 00:00:00 2001 From: vladp Date: Fri, 26 Dec 2025 19:26:39 +0700 Subject: [PATCH 20/37] add post size --- src/main/resources/application.yaml | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/src/main/resources/application.yaml b/src/main/resources/application.yaml index 0ad1403..020a696 100644 --- a/src/main/resources/application.yaml +++ b/src/main/resources/application.yaml @@ -16,10 +16,11 @@ spring: servlet: multipart: - max-file-size: 10MB - max-request-size: 100MB enabled: true - resolve-lazily: false + resolve-lazily: true + file-size-threshold: 0 + max-file-size: 5242880 + max-request-size: 10485760 file: storage: @@ -34,8 +35,9 @@ file: server: port: ${SERVER_PORT:8080} tomcat: - max-swallow-size: -1 - max-http-form-post-size: 10 + max-swallow-size: 20MB + max-http-form-post-size: 20MB + max-http-header-size: 16KB security: cors: From 291586dbf4437993648c03180bbbd64e2e3e1540 Mon Sep 17 00:00:00 2001 From: vladp Date: Fri, 26 Dec 2025 19:29:47 +0700 Subject: [PATCH 21/37] add post size --- src/main/resources/application.yaml | 21 +++++---------------- 1 file changed, 5 insertions(+), 16 deletions(-) diff --git a/src/main/resources/application.yaml b/src/main/resources/application.yaml index 020a696..08326e1 100644 --- a/src/main/resources/application.yaml +++ b/src/main/resources/application.yaml @@ -16,11 +16,10 @@ spring: servlet: multipart: + max-file-size: 10MB + max-request-size: 100MB enabled: true - resolve-lazily: true - file-size-threshold: 0 - max-file-size: 5242880 - max-request-size: 10485760 + resolve-lazily: false file: storage: @@ -32,13 +31,6 @@ file: temp-ttl-hours: ${TEMP_TTL_HOURS:72} # 3 дня session-expiry-hours: ${SESSION_EXPIRY_HOURS:24} -server: - port: ${SERVER_PORT:8080} - tomcat: - max-swallow-size: 20MB - max-http-form-post-size: 20MB - max-http-header-size: 16KB - security: cors: allowed-origins: "*" @@ -46,11 +38,8 @@ security: allowed-headers: "*" allow-credentials: true -#server: -# port: ${SERVER_PORT:8080} -# -# - +server: + port: ${SERVER_PORT:8080} logging: level: From 9541e8965149aa6eb11fd789ff45bb65751e6679 Mon Sep 17 00:00:00 2001 From: vladp Date: Fri, 26 Dec 2025 19:40:08 +0700 Subject: [PATCH 22/37] add post size --- src/main/resources/application.yaml | 17 ++++++++++++----- 1 file changed, 12 insertions(+), 5 deletions(-) diff --git a/src/main/resources/application.yaml b/src/main/resources/application.yaml index 08326e1..3c96660 100644 --- a/src/main/resources/application.yaml +++ b/src/main/resources/application.yaml @@ -16,10 +16,11 @@ spring: servlet: multipart: + enabled: true + resolve-lazily: true # + file-size-threshold: 2KB # max-file-size: 10MB max-request-size: 100MB - enabled: true - resolve-lazily: false file: storage: @@ -31,6 +32,13 @@ file: temp-ttl-hours: ${TEMP_TTL_HOURS:72} # 3 дня session-expiry-hours: ${SESSION_EXPIRY_HOURS:24} +server: + port: ${SERVER_PORT:8080} + tomcat: + max-swallow-size: -1 # + connection-timeout: 300000 # + max-http-header-size: 16KB + security: cors: allowed-origins: "*" @@ -38,13 +46,12 @@ security: allowed-headers: "*" allow-credentials: true -server: - port: ${SERVER_PORT:8080} - logging: level: com.example.fileupload: DEBUG org.springframework.web: DEBUG + org.springframework.web.multipart: TRACE # ← ДЛЯ ОТЛАДКИ + org.apache.tomcat: DEBUG # ← ЛОГИ TOMCAT org.hibernate.SQL: DEBUG file: name: logs/application.log From 86e7371ffe65e57d954f29461304797d7a435020 Mon Sep 17 00:00:00 2001 From: vladp Date: Fri, 26 Dec 2025 19:50:32 +0700 Subject: [PATCH 23/37] set chunk-size to 1000000 --- docker-compose.yaml | 2 +- .../service/file/FileUploadServiceImpl.java | 2 +- src/main/resources/application.yaml | 20 +++++++------------ 3 files changed, 9 insertions(+), 15 deletions(-) diff --git a/docker-compose.yaml b/docker-compose.yaml index 91e8d40..6136ac0 100644 --- a/docker-compose.yaml +++ b/docker-compose.yaml @@ -34,7 +34,7 @@ services: environment: FILE_STORAGE_PATH: /data/uploads MAX_FILE_SIZE: 10737418240 - FILE_CHUNK_SIZE: 5242880 + FILE_CHUNK_SIZE: 1000000 POSTGRES_DB: no_copy_ POSTGRES_USER: postgres POSTGRES_PASSWORD: postgres diff --git a/src/main/java/ru/soune/nocopy/service/file/FileUploadServiceImpl.java b/src/main/java/ru/soune/nocopy/service/file/FileUploadServiceImpl.java index 908602f..3e09822 100644 --- a/src/main/java/ru/soune/nocopy/service/file/FileUploadServiceImpl.java +++ b/src/main/java/ru/soune/nocopy/service/file/FileUploadServiceImpl.java @@ -102,7 +102,7 @@ public class FileUploadServiceImpl implements FileUploadService { int totalChunks = (int) Math.ceil((double) fileSize / chunkSize); log.debug("File will be split into {} chunks (chunk size: {} bytes)", totalChunks, chunkSize); - Long chunkSize = totalChunks == 1 ? fileSize : 5242880L; + Long chunkSize = totalChunks == 1 ? fileSize : 1000000L; FileUploadSession session = FileUploadSession.builder() .userId(userId) diff --git a/src/main/resources/application.yaml b/src/main/resources/application.yaml index 3c96660..57b803b 100644 --- a/src/main/resources/application.yaml +++ b/src/main/resources/application.yaml @@ -16,29 +16,22 @@ spring: servlet: multipart: - enabled: true - resolve-lazily: true # - file-size-threshold: 2KB # max-file-size: 10MB max-request-size: 100MB + enabled: true + resolve-lazily: false file: storage: base-path: ${FILE_STORAGE_PATH:/data/uploads} - chunk-size: ${FILE_CHUNK_SIZE:5242880} # 5MB +# chunk-size: ${FILE_CHUNK_SIZE:5242880} # 5MB + chunk-size: ${FILE_CHUNK_SIZE:1000000} # 5MB max-file-size: ${MAX_FILE_SIZE:10737418240} # 10GB max-retry-attempts: ${MAX_RETRY_ATTEMPTS:3} chunk-timeout-ms: ${CHUNK_TIMEOUT_MS:300000} # 5 минут temp-ttl-hours: ${TEMP_TTL_HOURS:72} # 3 дня session-expiry-hours: ${SESSION_EXPIRY_HOURS:24} -server: - port: ${SERVER_PORT:8080} - tomcat: - max-swallow-size: -1 # - connection-timeout: 300000 # - max-http-header-size: 16KB - security: cors: allowed-origins: "*" @@ -46,12 +39,13 @@ security: allowed-headers: "*" allow-credentials: true +server: + port: ${SERVER_PORT:8080} + logging: level: com.example.fileupload: DEBUG org.springframework.web: DEBUG - org.springframework.web.multipart: TRACE # ← ДЛЯ ОТЛАДКИ - org.apache.tomcat: DEBUG # ← ЛОГИ TOMCAT org.hibernate.SQL: DEBUG file: name: logs/application.log From a33c53fa6bad437665e5c2c909a150b8903dca9d Mon Sep 17 00:00:00 2001 From: vladp Date: Fri, 26 Dec 2025 20:18:18 +0700 Subject: [PATCH 24/37] set chunk-size to 5 --- docker-compose.yaml | 2 +- .../file/MultipartConfiguration.java | 24 +++++++++++++++ src/main/resources/application.yaml | 30 +++++++++++-------- 3 files changed, 42 insertions(+), 14 deletions(-) create mode 100644 src/main/java/ru/soune/nocopy/configuration/file/MultipartConfiguration.java diff --git a/docker-compose.yaml b/docker-compose.yaml index 6136ac0..91e8d40 100644 --- a/docker-compose.yaml +++ b/docker-compose.yaml @@ -34,7 +34,7 @@ services: environment: FILE_STORAGE_PATH: /data/uploads MAX_FILE_SIZE: 10737418240 - FILE_CHUNK_SIZE: 1000000 + FILE_CHUNK_SIZE: 5242880 POSTGRES_DB: no_copy_ POSTGRES_USER: postgres POSTGRES_PASSWORD: postgres diff --git a/src/main/java/ru/soune/nocopy/configuration/file/MultipartConfiguration.java b/src/main/java/ru/soune/nocopy/configuration/file/MultipartConfiguration.java new file mode 100644 index 0000000..1164256 --- /dev/null +++ b/src/main/java/ru/soune/nocopy/configuration/file/MultipartConfiguration.java @@ -0,0 +1,24 @@ +package ru.soune.nocopy.configuration.file; + +import jakarta.servlet.MultipartConfigElement; +import org.springframework.boot.web.servlet.MultipartConfigFactory; +import org.springframework.context.annotation.Bean; +import org.springframework.context.annotation.Configuration; +import org.springframework.util.unit.DataSize; + +@Configuration +public class MultipartConfiguration { + + @Bean + public MultipartConfigElement multipartConfigElement() { + MultipartConfigFactory factory = new MultipartConfigFactory(); + + factory.setMaxFileSize(DataSize.ofGigabytes(10)); + factory.setMaxRequestSize(DataSize.ofGigabytes(10)); + + factory.setFileSizeThreshold(DataSize.ofBytes(0)); + + + return factory.createMultipartConfig(); + } +} diff --git a/src/main/resources/application.yaml b/src/main/resources/application.yaml index 57b803b..f087418 100644 --- a/src/main/resources/application.yaml +++ b/src/main/resources/application.yaml @@ -16,22 +16,29 @@ spring: servlet: multipart: - max-file-size: 10MB - max-request-size: 100MB enabled: true - resolve-lazily: false + resolve-lazily: true + file-size-threshold: 0 + max-file-size: 10GB + max-request-size: 10GB file: storage: base-path: ${FILE_STORAGE_PATH:/data/uploads} -# chunk-size: ${FILE_CHUNK_SIZE:5242880} # 5MB - chunk-size: ${FILE_CHUNK_SIZE:1000000} # 5MB + chunk-size: ${FILE_CHUNK_SIZE:5242880} # 5MB max-file-size: ${MAX_FILE_SIZE:10737418240} # 10GB max-retry-attempts: ${MAX_RETRY_ATTEMPTS:3} - chunk-timeout-ms: ${CHUNK_TIMEOUT_MS:300000} # 5 минут - temp-ttl-hours: ${TEMP_TTL_HOURS:72} # 3 дня + chunk-timeout-ms: ${CHUNK_TIMEOUT_MS:300000} + temp-ttl-hours: ${TEMP_TTL_HOURS:72} session-expiry-hours: ${SESSION_EXPIRY_HOURS:24} +server: + port: ${SERVER_PORT:8080} + tomcat: + max-swallow-size: -1 + max-http-form-post-size: -1 + connection-timeout: 300000 + security: cors: allowed-origins: "*" @@ -39,15 +46,12 @@ security: allowed-headers: "*" allow-credentials: true -server: - port: ${SERVER_PORT:8080} - logging: level: com.example.fileupload: DEBUG org.springframework.web: DEBUG + org.springframework.web.multipart: TRACE + org.apache.tomcat: DEBUG org.hibernate.SQL: DEBUG file: - name: logs/application.log - pattern: - console: "%d{yyyy-MM-dd HH:mm:ss} [%thread] %-5level %logger{36} - %msg%n" \ No newline at end of file + name: logs/application.log \ No newline at end of file From 254fdc0fbd2068d901fafb93eef79866bc59555c Mon Sep 17 00:00:00 2001 From: vladp Date: Fri, 26 Dec 2025 20:30:38 +0700 Subject: [PATCH 25/37] set chunk-size to 5 --- docker-compose.yaml | 2 +- src/main/resources/application.yaml | 30 +++++++++++++---------------- 2 files changed, 14 insertions(+), 18 deletions(-) diff --git a/docker-compose.yaml b/docker-compose.yaml index 91e8d40..6136ac0 100644 --- a/docker-compose.yaml +++ b/docker-compose.yaml @@ -34,7 +34,7 @@ services: environment: FILE_STORAGE_PATH: /data/uploads MAX_FILE_SIZE: 10737418240 - FILE_CHUNK_SIZE: 5242880 + FILE_CHUNK_SIZE: 1000000 POSTGRES_DB: no_copy_ POSTGRES_USER: postgres POSTGRES_PASSWORD: postgres diff --git a/src/main/resources/application.yaml b/src/main/resources/application.yaml index f087418..1dd8dbe 100644 --- a/src/main/resources/application.yaml +++ b/src/main/resources/application.yaml @@ -16,29 +16,22 @@ spring: servlet: multipart: + max-file-size: 10MB + max-request-size: 100MB enabled: true - resolve-lazily: true - file-size-threshold: 0 - max-file-size: 10GB - max-request-size: 10GB + resolve-lazily: false file: storage: base-path: ${FILE_STORAGE_PATH:/data/uploads} - chunk-size: ${FILE_CHUNK_SIZE:5242880} # 5MB + # chunk-size: ${FILE_CHUNK_SIZE:5242880} # 5MB + chunk-size: ${FILE_CHUNK_SIZE:1000000} # 15MB max-file-size: ${MAX_FILE_SIZE:10737418240} # 10GB max-retry-attempts: ${MAX_RETRY_ATTEMPTS:3} - chunk-timeout-ms: ${CHUNK_TIMEOUT_MS:300000} - temp-ttl-hours: ${TEMP_TTL_HOURS:72} + chunk-timeout-ms: ${CHUNK_TIMEOUT_MS:300000} # 5 минут + temp-ttl-hours: ${TEMP_TTL_HOURS:72} # 3 дня session-expiry-hours: ${SESSION_EXPIRY_HOURS:24} -server: - port: ${SERVER_PORT:8080} - tomcat: - max-swallow-size: -1 - max-http-form-post-size: -1 - connection-timeout: 300000 - security: cors: allowed-origins: "*" @@ -46,12 +39,15 @@ security: allowed-headers: "*" allow-credentials: true +server: + port: ${SERVER_PORT:8080} + logging: level: com.example.fileupload: DEBUG org.springframework.web: DEBUG - org.springframework.web.multipart: TRACE - org.apache.tomcat: DEBUG org.hibernate.SQL: DEBUG file: - name: logs/application.log \ No newline at end of file + name: logs/application.log + pattern: + console: "%d{yyyy-MM-dd HH:mm:ss} [%thread] %-5level %logger{36} - %msg%n" \ No newline at end of file From 851676b89546a8e601033b665982f8ae10a42824 Mon Sep 17 00:00:00 2001 From: vladp Date: Fri, 26 Dec 2025 20:32:15 +0700 Subject: [PATCH 26/37] set chunk-size to 5 --- .../file/MultipartConfiguration.java | 24 ------------------- 1 file changed, 24 deletions(-) delete mode 100644 src/main/java/ru/soune/nocopy/configuration/file/MultipartConfiguration.java diff --git a/src/main/java/ru/soune/nocopy/configuration/file/MultipartConfiguration.java b/src/main/java/ru/soune/nocopy/configuration/file/MultipartConfiguration.java deleted file mode 100644 index 1164256..0000000 --- a/src/main/java/ru/soune/nocopy/configuration/file/MultipartConfiguration.java +++ /dev/null @@ -1,24 +0,0 @@ -package ru.soune.nocopy.configuration.file; - -import jakarta.servlet.MultipartConfigElement; -import org.springframework.boot.web.servlet.MultipartConfigFactory; -import org.springframework.context.annotation.Bean; -import org.springframework.context.annotation.Configuration; -import org.springframework.util.unit.DataSize; - -@Configuration -public class MultipartConfiguration { - - @Bean - public MultipartConfigElement multipartConfigElement() { - MultipartConfigFactory factory = new MultipartConfigFactory(); - - factory.setMaxFileSize(DataSize.ofGigabytes(10)); - factory.setMaxRequestSize(DataSize.ofGigabytes(10)); - - factory.setFileSizeThreshold(DataSize.ofBytes(0)); - - - return factory.createMultipartConfig(); - } -} From 72a4516b7dab59299e9ba890a2e7994356101afc Mon Sep 17 00:00:00 2001 From: vladp Date: Tue, 30 Dec 2025 01:05:59 +0700 Subject: [PATCH 27/37] set correct file types --- src/main/java/ru/soune/nocopy/entity/file/FileType.java | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/src/main/java/ru/soune/nocopy/entity/file/FileType.java b/src/main/java/ru/soune/nocopy/entity/file/FileType.java index 32f9cd6..a7071ef 100644 --- a/src/main/java/ru/soune/nocopy/entity/file/FileType.java +++ b/src/main/java/ru/soune/nocopy/entity/file/FileType.java @@ -7,12 +7,10 @@ import java.util.List; @Getter public enum FileType { - IMAGE("image", Arrays.asList("jpg", "jpeg", "png", "gif", "bmp", "svg", "tiff", "tif", "ico", - "psd", "ai", "eps", "raw", "heic", "heif")), + IMAGE("image", Arrays.asList("jpg", "jpeg", "png", "gif", "bmp", "webp")), VIDEO("video", Arrays.asList("mp4", "avi", "mov", "wmv", "flv", "mkv", "webm", "m4v", "mpg", "mpeg", "3gp", "3g2", "f4v", "m2ts", "mts", "vob", "ogv", "divx")), - AUDIO("audio", Arrays.asList("mp3", "wav", "ogg", "aac", "flac", "m4a", "wma", "aiff", "aif", "amr", - "opus", "mka", "ac3", "alac")), + AUDIO("audio", Arrays.asList("mp3", "wav", "flac")), DOCUMENT("document", Arrays.asList( "pdf", "txt", "rtf", "doc", "docx", "xls", "xlsx", "ppt", "pptx", "pps", "ppsx", "dot", "dotx", "xlt", "xltx", "pot", "potx", From 28b0356b0331ca9f08cd6226b9a9c078d4ec6e6d Mon Sep 17 00:00:00 2001 From: vladp Date: Tue, 30 Dec 2025 02:19:07 +0700 Subject: [PATCH 28/37] fix check status deleted --- .../nocopy/controller/ApiController.java | 69 +++++++++++++++---- .../java/ru/soune/nocopy/dto/MessageCode.java | 2 +- 2 files changed, 55 insertions(+), 16 deletions(-) diff --git a/src/main/java/ru/soune/nocopy/controller/ApiController.java b/src/main/java/ru/soune/nocopy/controller/ApiController.java index eb1d8a4..dbbe9df 100644 --- a/src/main/java/ru/soune/nocopy/controller/ApiController.java +++ b/src/main/java/ru/soune/nocopy/controller/ApiController.java @@ -20,11 +20,9 @@ import ru.soune.nocopy.dto.file.CompleteUploadResponse; import ru.soune.nocopy.dto.file.FileEntityResponse; import ru.soune.nocopy.dto.file.UploadProgress; import ru.soune.nocopy.entity.AuthToken; +import ru.soune.nocopy.entity.file.FileStatus; import ru.soune.nocopy.entity.file.UploadStatus; -import ru.soune.nocopy.exception.FileEntityNotFoundException; -import ru.soune.nocopy.exception.NotFoundAuthToken; -import ru.soune.nocopy.exception.NotValidFieldException; -import ru.soune.nocopy.exception.ValidationException; +import ru.soune.nocopy.exception.*; import ru.soune.nocopy.handler.*; import ru.soune.nocopy.repository.AuthTokenRepository; import ru.soune.nocopy.service.file.FileEntityService; @@ -247,27 +245,62 @@ public class ApiController { @PathVariable(required = false) Integer version, @RequestHeader(value = "Authorization", required = false) String tokenHeader) { try { + if (tokenHeader == null) { + Map errorData = new HashMap<>(); + errorData.put("token", tokenHeader); + errorData.put("fileId", fileId); + errorData.put("version", version); + + return ResponseEntity.ok().body(new BaseResponse(20004, + MessageCode.FILE_DOWNLOAD_ERROR_NOT_CORRECT_FIELD.getCode(), + MessageCode.FILE_DOWNLOAD_ERROR_NOT_CORRECT_FIELD.getDescription(), + errorData)); + } + Long userId = getUserIdFromToken(tokenHeader); FileEntityResponse entityResponse = fileEntityService.getById(fileId, version); if (!entityResponse.getUserId().equals(userId)) { - return ResponseEntity.ok().body(new BaseResponse(20004, MessageCode.FILE_DOWNLOAD_ERROR.getCode(), - MessageCode.FILE_DOWNLOAD_ERROR.getDescription(), Map.of("token", tokenHeader))); + Map errorData = new HashMap<>(); + errorData.put("token", tokenHeader); + + return ResponseEntity.ok().body(new BaseResponse(20004, + MessageCode.FILE_DOWNLOAD_ERROR.getCode(), + MessageCode.FILE_DOWNLOAD_ERROR.getDescription(), + errorData)); + } + + if (entityResponse.getStatus().equals(FileStatus.DELETED)) { + Map errorData = new HashMap<>(); + errorData.put("file_status", entityResponse.getStatus()); + + return ResponseEntity.ok().body(new BaseResponse(20004, + MessageCode.FILE_DOWNLOAD_ERROR.getCode(), + MessageCode.FILE_DOWNLOAD_ERROR.getDescription(), + errorData)); } if (!entityResponse.isExistsOnDisk()) { - return ResponseEntity.ok().body(new BaseResponse(20004, MessageCode.FILE_DOWNLOAD_ERROR.getCode(), + Map errorData = new HashMap<>(); + errorData.put("onDisk", entityResponse.isExistsOnDisk()); + + return ResponseEntity.ok().body(new BaseResponse(20004, + MessageCode.FILE_DOWNLOAD_ERROR.getCode(), MessageCode.FILE_DOWNLOAD_ERROR.getDescription(), - Map.of("onDisk", entityResponse.isExistsOnDisk()))); + errorData)); } Path filePath = Paths.get(entityResponse.getFilePath()); Resource resource = new UrlResource(filePath.toUri()); if (!resource.exists()) { - return ResponseEntity.ok().body(new BaseResponse(20004, MessageCode.FILE_DOWNLOAD_ERROR.getCode(), + Map errorData = new HashMap<>(); + errorData.put("resource", resource.exists()); + + return ResponseEntity.ok().body(new BaseResponse(20004, + MessageCode.FILE_DOWNLOAD_ERROR.getCode(), MessageCode.FILE_DOWNLOAD_ERROR.getDescription(), - Map.of("resource", resource.exists()))); + errorData)); } String contentType = determineContentType(filePath); @@ -279,17 +312,23 @@ public class ApiController { .header(HttpHeaders.CONTENT_LENGTH, String.valueOf(entityResponse.getFileSize())) .body(resource); } catch (FileEntityNotFoundException e) { + Map errorData = new HashMap<>(); + errorData.put("fileId", fileId); + return ResponseEntity.ok().body(new BaseResponse(20004, MessageCode.FILE_NOT_FOUND.getCode(), MessageCode.FILE_NOT_FOUND.getDescription(), - Map.of("fileId", fileId))); - } catch (Exception e) { + errorData)); + } catch (NotFoundAuthToken | IOException e) { + Map errorData = new HashMap<>(); + errorData.put("token", tokenHeader); + errorData.put("fileId", fileId); + errorData.put("version", version); + return ResponseEntity.ok().body(new BaseResponse(20004, MessageCode.FILE_DOWNLOAD_ERROR_NOT_CORRECT_FIELD.getCode(), MessageCode.FILE_DOWNLOAD_ERROR_NOT_CORRECT_FIELD.getDescription(), - Map.of("token", tokenHeader, - "fileId", fileId, - "version", version))); + errorData)); } } diff --git a/src/main/java/ru/soune/nocopy/dto/MessageCode.java b/src/main/java/ru/soune/nocopy/dto/MessageCode.java index eef129c..b945f64 100644 --- a/src/main/java/ru/soune/nocopy/dto/MessageCode.java +++ b/src/main/java/ru/soune/nocopy/dto/MessageCode.java @@ -9,7 +9,7 @@ public enum MessageCode { INVALID_ACTION(2, "Invalid action"), FILE_UPLOAD_ERROR(2, "File upload error"), FILE_DOWNLOAD_ERROR(2, "File download error"), - FILE_DOWNLOAD_ERROR_NOT_CORRECT_FIELD(2, "File download error with correct field"), + FILE_DOWNLOAD_ERROR_NOT_CORRECT_FIELD(2, "Not correct field"), INVALID_JSON_BODY(2, "Invalid fields in JSON object"), INCOMPLETE_UPLOAD(2, "Not load all chunks"), MSG_ID_NOT_FOUND(4, "Message id not found"), From 0f73c8319e40e45da602a439065dbc1857756c19 Mon Sep 17 00:00:00 2001 From: vladp Date: Tue, 30 Dec 2025 12:08:23 +0700 Subject: [PATCH 29/37] add ram size --- docker-compose.yaml | 71 +++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 71 insertions(+) diff --git a/docker-compose.yaml b/docker-compose.yaml index 6136ac0..f1e3e94 100644 --- a/docker-compose.yaml +++ b/docker-compose.yaml @@ -4,6 +4,12 @@ services: storage: image: alpine:latest container_name: file-storage + deploy: + resources: + limits: + memory: 128M + reservations: + memory: 64M networks: - app-network volumes: @@ -14,10 +20,20 @@ services: db: image: postgres:17 restart: always + deploy: + resources: + limits: + cpus: '2.0' + memory: 2G + reservations: + cpus: '1.0' + memory: 1G environment: POSTGRES_DB: no_copy_ POSTGRES_USER: postgres POSTGRES_PASSWORD: postgres + POSTGRES_SHARED_BUFFERS: 512MB + POSTGRES_EFFECTIVE_CACHE_SIZE: 1536MB ports: - "54320:5432" volumes: @@ -31,6 +47,14 @@ services: app: build: . container_name: app-backend + deploy: + resources: + limits: + cpus: '1.5' + memory: 1G + reservations: + cpus: '0.5' + memory: 512M environment: FILE_STORAGE_PATH: /data/uploads MAX_FILE_SIZE: 10737418240 @@ -57,6 +81,13 @@ services: grafana: image: grafana/grafana:10.3.1 container_name: grafana + deploy: + resources: + limits: + cpus: '0.5' + memory: 512M + reservations: + memory: 256M ports: - "3000:3000" depends_on: @@ -73,6 +104,8 @@ services: GF_SECURITY_ADMIN_USER: admin GF_SECURITY_ADMIN_PASSWORD: admin GF_METRICS_ENABLED: "true" + GF_DATABASE_MAX_IDLE_CONN: "2" + GF_DATABASE_MAX_OPEN_CONN: "10" volumes: - ./infrastructure/grafana/provisioning:/etc/grafana/provisioning - ./infrastructure/grafana/dashboards:/var/lib/grafana/dashboards @@ -87,10 +120,23 @@ services: prometheus: image: prom/prometheus:latest container_name: prometheus + deploy: # ← ДОБАВЛЕНО + resources: + limits: + cpus: '0.5' + memory: 1G + reservations: + memory: 512M ports: - "9090:9090" volumes: - ./infrastructure/prometheus/prometheus.yml:/etc/prometheus/prometheus.yml + - prometheus_data:/prometheus + command: + - '--config.file=/etc/prometheus/prometheus.yml' + - '--storage.tsdb.path=/prometheus' + - '--storage.tsdb.retention.time=15d' # Удерживать 15 дней + - '--web.enable-lifecycle' healthcheck: test: [ "CMD", "wget", "--spider", "http://localhost:9090/-/healthy" ] interval: 10s @@ -102,10 +148,20 @@ services: loki: image: grafana/loki:2.9.2 container_name: loki + deploy: # ← ДОБАВЛЕНО + resources: + limits: + cpus: '1.0' + memory: 2G + reservations: + memory: 1G ports: - "3100:3100" volumes: - ./infrastructure/loki/loki-config.yaml:/etc/loki/loki-config.yaml:ro + - loki_data:/loki + command: + - -config.file=/etc/loki/loki-config.yaml healthcheck: test: [ "CMD", "wget", "--spider", "-q", "http://localhost:3100/ready" ] interval: 10s @@ -117,6 +173,13 @@ services: tempo: image: grafana/tempo:2.4.1 container_name: tempo + deploy: # ← ДОБАВЛЕНО + resources: + limits: + cpus: '0.5' + memory: 512M + reservations: + memory: 256M command: [ "-config.file=/etc/tempo/tempo.yaml" ] volumes: - ./infrastructure/tempo/tempo.yaml:/etc/tempo/tempo.yaml @@ -129,6 +192,13 @@ services: alloy: image: grafana/alloy:latest container_name: alloy + deploy: + resources: + limits: + cpus: '0.5' + memory: 256M + reservations: + memory: 128M user: root ports: - "9080:9080" @@ -158,6 +228,7 @@ volumes: loki_index: loki_rules: uploads_data: + prometheus_data: networks: app-network: From bd9ecd4f84ad076b371cb5e93e9230257371ccac Mon Sep 17 00:00:00 2001 From: vladp Date: Tue, 30 Dec 2025 12:25:22 +0700 Subject: [PATCH 30/37] change chunk size --- docker-compose.yaml | 2 +- infrastructure/jenkins/deploy.groovy | 241 +++++++++++------- .../service/file/FileUploadServiceImpl.java | 2 +- src/main/resources/application.yaml | 2 +- 4 files changed, 146 insertions(+), 101 deletions(-) diff --git a/docker-compose.yaml b/docker-compose.yaml index f1e3e94..f64115c 100644 --- a/docker-compose.yaml +++ b/docker-compose.yaml @@ -58,7 +58,7 @@ services: environment: FILE_STORAGE_PATH: /data/uploads MAX_FILE_SIZE: 10737418240 - FILE_CHUNK_SIZE: 1000000 + FILE_CHUNK_SIZE: 1048576 POSTGRES_DB: no_copy_ POSTGRES_USER: postgres POSTGRES_PASSWORD: postgres diff --git a/infrastructure/jenkins/deploy.groovy b/infrastructure/jenkins/deploy.groovy index 248b0cf..a05e8ca 100644 --- a/infrastructure/jenkins/deploy.groovy +++ b/infrastructure/jenkins/deploy.groovy @@ -45,119 +45,162 @@ pipeline { echo "Deploying branch: ${params.BRANCH}" echo "Copying files to server..." - sshpass -p '$SSH_PASS' ssh $SSH_USER@$SERVER "mkdir -p /opt/deployments/${params.BRANCH}" - sshpass -p '$SSH_PASS' scp -r ./* $SSH_USER@$SERVER:/opt/deployments/${params.BRANCH}/ + sshpass -p '$SSH_PASS' ssh -o StrictHostKeyChecking=no $SSH_USER@$SERVER "mkdir -p /opt/deployments/${params.BRANCH}" + sshpass -p '$SSH_PASS' scp -r -o StrictHostKeyChecking=no ./* $SSH_USER@$SERVER:/opt/deployments/${params.BRANCH}/ - sshpass -p '$SSH_PASS' ssh $SSH_USER@$SERVER " + sshpass -p '$SSH_PASS' ssh -o StrictHostKeyChecking=no $SSH_USER@$SERVER " cd /opt/deployments/${params.BRANCH} + echo '========================================' echo 'Step 1: Checking current state...' + echo '========================================' - # Смотрим что запущено - docker ps --filter 'name=postgres|app-backend' --format 'table {{.Names}}\\t{{.Image}}\\t{{.Status}}\\t{{.Ports}}' + docker ps --filter 'name=postgres|app-backend|storage|grafana|prometheus|loki|tempo|alloy' --format 'table {{.Names}}\\t{{.Image}}\\t{{.Status}}' - echo 'Step 2: Force cleanup old application...' + echo '========================================' + echo 'Step 2: Setup environment for compatibility...' + echo '========================================' - # Принудительно удаляем старый контейнер и образ - docker stop app-backend 2>/dev/null || echo 'App not running' - docker rm app-backend 2>/dev/null || echo 'App not found' - docker rmi app-backend:latest 2>/dev/null || echo 'Image not found' + docker network create app-network 2>/dev/null || echo 'Network app-network already exists' - # Удаляем все образы app-backend - docker images --filter 'reference=app-backend*' -q | xargs -r docker rmi 2>/dev/null || echo 'No images to remove' + echo 'COMPOSE_COMPATIBILITY=true' > .env - echo 'Step 3: Verify copied files...' - echo 'Files in directory:' - ls -la - echo '' - echo 'Checking Java sources:' - find . -name '*.java' | head -2 + echo '========================================' + echo 'Step 3: Force cleanup old application...' + echo '========================================' - echo 'Step 4: Check if PostgreSQL is running...' - if ! docker ps | grep -q postgres; then - echo 'Starting PostgreSQL...' - docker-compose up -d db - sleep 10 - else - echo 'PostgreSQL already running' - fi + docker-compose --compatibility down 2>/dev/null || echo 'No previous compose stack' - echo 'Step 5: Build application with forced rebuild...' - echo 'Checking Dockerfile exists:' - ls -la Dockerfile 2>/dev/null || echo 'Dockerfile not found, creating simple one' + docker images --filter 'reference=*app-backend*' -q | xargs -r docker rmi -f 2>/dev/null || echo 'No images to remove' - # Если нет Dockerfile, создаем простой - if [ ! -f Dockerfile ]; then - echo 'Creating simple Dockerfile...' - cat > Dockerfile << 'EOF' -FROM eclipse-temurin:21-jre -WORKDIR /app -COPY . /app/ -CMD ["java", "-jar", "app.jar"] -EOF - fi + echo '========================================' + echo 'Step 4: Verify files and fix compose...' + echo '========================================' - # Собираем с подробным выводом - echo 'Building Docker image...' - docker build --no-cache --progress=plain -t app-backend:latest . 2>&1 | tail -50 - - echo 'Step 6: Verify new image...' - echo 'Current app-backend images:' - docker images | grep app-backend - - echo 'Image creation time:' - docker inspect app-backend:latest --format='{{.Created}}' 2>/dev/null || echo 'Cannot inspect image' - - echo 'Step 7: Starting application with alias...' - docker run -d \\ - --name app-backend \\ - --network app-network \\ - --network-alias app \\ - -p 80:8080 \\ - -v /opt/uploads:/data/uploads:rw \\ - -e POSTGRES_DB=no_copy_ \\ - -e POSTGRES_USER=postgres \\ - -e POSTGRES_PASSWORD=postgres \\ - -e POSTGRES_PORT=5432 \\ - -e POSTGRES_HOST=db \\ - --restart unless-stopped \\ - app-backend:latest - - echo 'Step 8: Checking deployment...' - sleep 10 - - echo 'Containers status:' - docker ps --filter 'name=postgres|app-backend' --format 'table {{.Names}}\\t{{.Image}}\\t{{.Status}}\\t{{.CreatedAt}}' - - echo 'Checking image inside container:' - docker exec app-backend ls -la /app/ 2>/dev/null || echo 'Cannot check container files' - - echo 'Application logs (last 10 lines):' - docker logs --tail=10 app-backend 2>/dev/null || echo 'Logs not available yet' - - echo 'Step 9: Health check...' - if docker ps | grep -q app-backend; then - echo 'Container is running' - echo 'Testing application health...' - for i in {1..5}; do - if curl -s -f http://localhost:80/health > /dev/null 2>&1; then - echo 'Health check passed' - echo 'Deployment successful' - echo 'Application URL: http://${params.SERVER}:80' - exit 0 - fi - echo 'Waiting for application to start... attempt ' \$i - sleep 5 - done - - echo 'Health check failed, but container is running' - echo 'Application might be starting slowly' - echo 'Application URL: http://${params.SERVER}:80' - else - echo 'Application failed to start' - docker logs app-backend + if [ ! -f docker-compose.yaml ]; then + echo 'ERROR: docker-compose.yaml not found!' + ls -la exit 1 fi + + echo 'Checking docker-compose.yaml syntax...' + docker-compose --compatibility config 2>&1 | head -20 || echo 'Config check output' + + echo '========================================' + echo 'Step 5: Start infrastructure services...' + echo '========================================' + + echo 'Starting PostgreSQL with resource limits...' + docker-compose --compatibility up -d db + + echo 'Starting storage service...' + docker-compose --compatibility up -d storage + + echo 'Waiting for PostgreSQL to be ready...' + for i in {1..30}; do + if docker-compose --compatibility exec db pg_isready -U postgres 2>/dev/null; then + echo 'PostgreSQL is ready' + break + fi + echo 'Waiting... attempt ' \$i + sleep 2 + done + + echo '========================================' + echo 'Step 6: Build application image...' + echo '========================================' + + if [ ! -f Dockerfile ]; then + echo 'ERROR: Dockerfile not found!' + exit 1 + fi + + echo 'Building Docker image...' + docker build --no-cache -t app-backend:latest . 2>&1 | tail -30 + + echo 'Image created successfully' + docker images app-backend:latest --format 'table {{.Repository}}\\t{{.Tag}}\\t{{.Size}}\\t{{.CreatedAt}}' + + echo '========================================' + echo 'Step 7: Start application with resource limits...' + echo '========================================' + + docker volume create uploads_data 2>/dev/null || echo 'Volume uploads_data already exists' + + echo 'Starting main application...' + docker-compose --compatibility up -d app + + echo 'Waiting for application to start...' + sleep 15 + + echo '========================================' + echo 'Step 8: Start monitoring stack...' + echo '========================================' + + docker-compose --compatibility up -d grafana prometheus loki tempo alloy + + echo '========================================' + echo 'Step 9: Verify deployment...' + echo '========================================' + + echo 'All containers status:' + docker-compose --compatibility ps + + echo '' + echo 'Resource limits check:' + echo 'App backend limits:' + docker inspect app-backend --format='{{.HostConfig.Memory}} {{.HostConfig.CpuQuota}} {{.HostConfig.CpuPeriod}}' 2>/dev/null || echo 'Cannot inspect' + + echo '' + echo 'PostgreSQL limits:' + docker inspect postgres --format='{{.HostConfig.Memory}} {{.HostConfig.CpuQuota}} {{.HostConfig.CpuPeriod}}' 2>/dev/null || echo 'Cannot inspect' + + echo '' + echo 'Container resource usage:' + docker stats --no-stream --format 'table {{.Name}}\\t{{.CPUPerc}}\\t{{.MemUsage}}\\t{{.MemPerc}}' postgres app-backend storage 2>/dev/null || echo 'Stats not available' + + echo '========================================' + echo 'Step 10: Health checks...' + echo '========================================' + + echo 'Application health check...' + for i in {1..10}; do + if curl -s -f http://localhost:80/health 2>/dev/null; then + echo '✓ Application health check PASSED' + break + elif [ \$i -eq 10 ]; then + echo '✗ Application health check FAILED' + echo 'Last logs:' + docker-compose --compatibility logs --tail=20 app + else + echo 'Waiting for application... attempt ' \$i + sleep 5 + fi + done + + echo 'Checking monitoring services...' + if curl -s -f http://localhost:3000/api/health 2>/dev/null; then + echo '✓ Grafana is healthy' + else + echo '⚠ Grafana may be starting' + fi + + if curl -s -f http://localhost:9090/-/healthy 2>/dev/null; then + echo '✓ Prometheus is healthy' + fi + + echo '========================================' + echo 'Deployment Summary:' + echo '========================================' + echo 'Application URL: http://${params.SERVER}:80' + echo 'Grafana: http://${params.SERVER}:3000 (admin/admin)' + echo 'Prometheus: http://${params.SERVER}:9090' + echo 'Loki: http://${params.SERVER}:3100' + echo '' + echo 'To view logs: docker-compose logs -f app' + echo 'To view all containers: docker-compose ps' + echo 'To stop: docker-compose down' + echo '========================================' " """ } @@ -170,6 +213,8 @@ EOF success { echo "Deployment successful" echo "Application URL: http://${params.SERVER}:80" + echo "Grafana: http://${params.SERVER}:3000" + echo "Prometheus: http://${params.SERVER}:9090" } failure { echo "Deployment failed for branch ${params.BRANCH}" diff --git a/src/main/java/ru/soune/nocopy/service/file/FileUploadServiceImpl.java b/src/main/java/ru/soune/nocopy/service/file/FileUploadServiceImpl.java index 3e09822..f54cec9 100644 --- a/src/main/java/ru/soune/nocopy/service/file/FileUploadServiceImpl.java +++ b/src/main/java/ru/soune/nocopy/service/file/FileUploadServiceImpl.java @@ -102,7 +102,7 @@ public class FileUploadServiceImpl implements FileUploadService { int totalChunks = (int) Math.ceil((double) fileSize / chunkSize); log.debug("File will be split into {} chunks (chunk size: {} bytes)", totalChunks, chunkSize); - Long chunkSize = totalChunks == 1 ? fileSize : 1000000L; + Long chunkSize = totalChunks == 1 ? fileSize : 1048576L; FileUploadSession session = FileUploadSession.builder() .userId(userId) diff --git a/src/main/resources/application.yaml b/src/main/resources/application.yaml index 1dd8dbe..1fd4fab 100644 --- a/src/main/resources/application.yaml +++ b/src/main/resources/application.yaml @@ -25,7 +25,7 @@ file: storage: base-path: ${FILE_STORAGE_PATH:/data/uploads} # chunk-size: ${FILE_CHUNK_SIZE:5242880} # 5MB - chunk-size: ${FILE_CHUNK_SIZE:1000000} # 15MB + chunk-size: ${FILE_CHUNK_SIZE:1048576} # 15MB max-file-size: ${MAX_FILE_SIZE:10737418240} # 10GB max-retry-attempts: ${MAX_RETRY_ATTEMPTS:3} chunk-timeout-ms: ${CHUNK_TIMEOUT_MS:300000} # 5 минут From bf4d1ca450629fb7f200e03b94e8b78118e7d8ee Mon Sep 17 00:00:00 2001 From: vladp Date: Tue, 6 Jan 2026 12:50:39 +0700 Subject: [PATCH 31/37] add endpoint with files user info --- .../nocopy/dto/file/FileInfoUserResponse.java | 61 ++++++++++++ .../soune/nocopy/entity/file/FileStatus.java | 2 + .../nocopy/handler/FileEntityHandler.java | 17 ++++ .../nocopy/service/file/FileStatsService.java | 93 +++++++++++++++++++ 4 files changed, 173 insertions(+) create mode 100644 src/main/java/ru/soune/nocopy/dto/file/FileInfoUserResponse.java create mode 100644 src/main/java/ru/soune/nocopy/service/file/FileStatsService.java diff --git a/src/main/java/ru/soune/nocopy/dto/file/FileInfoUserResponse.java b/src/main/java/ru/soune/nocopy/dto/file/FileInfoUserResponse.java new file mode 100644 index 0000000..8b2f932 --- /dev/null +++ b/src/main/java/ru/soune/nocopy/dto/file/FileInfoUserResponse.java @@ -0,0 +1,61 @@ +package ru.soune.nocopy.dto.file; + +import com.fasterxml.jackson.annotation.JsonProperty; +import lombok.AllArgsConstructor; +import lombok.Builder; +import lombok.Data; +import lombok.NoArgsConstructor; + +@Data +@Builder +@NoArgsConstructor +@AllArgsConstructor +public class FileInfoUserResponse { + @JsonProperty("all_files_size") + private Long allFileSize; + + @JsonProperty("all_files_quantity") + private Integer fileCount; + + @JsonProperty("all_files_check") + private Integer filesCheck; + + @JsonProperty("all_files_violation") + private Integer filesViolation; + + @JsonProperty("images_size") + private Long imagesSize; + + @JsonProperty("images_quantity") + private Integer imagesCount; + + @JsonProperty("images_check") + private Integer imagesCheck; + + @JsonProperty("images_violations") + private Integer imagesViolations; + + @JsonProperty("videos_size") + private Long videosSize; + + @JsonProperty("videos_quantity") + private Integer videosCount; + + @JsonProperty("videos_check") + private Integer videosCheck; + + @JsonProperty("videos_violations") + private Integer videosViolations; + + @JsonProperty("audios_size") + private Long audiosSize; + + @JsonProperty("audios_quantity") + private Integer audiosCount; + + @JsonProperty("audios_check") + private Integer audiosCheck; + + @JsonProperty("audios_violations") + private Integer audiosViolations; +} diff --git a/src/main/java/ru/soune/nocopy/entity/file/FileStatus.java b/src/main/java/ru/soune/nocopy/entity/file/FileStatus.java index 8e96247..b0b2351 100644 --- a/src/main/java/ru/soune/nocopy/entity/file/FileStatus.java +++ b/src/main/java/ru/soune/nocopy/entity/file/FileStatus.java @@ -4,5 +4,7 @@ public enum FileStatus { ACTIVE, DELETED, PROCESSING, + VIOLATION, + CHECKED, ERROR } diff --git a/src/main/java/ru/soune/nocopy/handler/FileEntityHandler.java b/src/main/java/ru/soune/nocopy/handler/FileEntityHandler.java index 968a3b7..9bddff0 100644 --- a/src/main/java/ru/soune/nocopy/handler/FileEntityHandler.java +++ b/src/main/java/ru/soune/nocopy/handler/FileEntityHandler.java @@ -9,11 +9,13 @@ import ru.soune.nocopy.dto.file.*; import ru.soune.nocopy.entity.AuthToken; import ru.soune.nocopy.entity.file.FileEntity; import ru.soune.nocopy.entity.file.FileStatus; +import ru.soune.nocopy.entity.file.FileType; import ru.soune.nocopy.exception.FileEntityNotFoundException; import ru.soune.nocopy.exception.NotFoundAuthToken; import ru.soune.nocopy.repository.AuthTokenRepository; import ru.soune.nocopy.repository.FileEntityRepository; import ru.soune.nocopy.service.file.FileEntityService; +import ru.soune.nocopy.service.file.FileStatsService; import java.util.Arrays; import java.util.List; @@ -22,8 +24,11 @@ import java.util.List; @Component @RequiredArgsConstructor public class FileEntityHandler implements RequestHandler { + private final FileEntityService fileEntityService; + private final FileStatsService fileStatsService; + private final AuthTokenRepository authTokenRepository; private final ObjectMapper objectMapper; @@ -41,6 +46,8 @@ public class FileEntityHandler implements RequestHandler { switch (action) { case "file_info": return handleGetFileInfo(request, fileRequest); + case "user_files_info": + return handleGetFilesUserInfo(request, fileRequest); case "file_by_session": return handleGetFileBySession(request, fileRequest); case "user_files": @@ -73,6 +80,16 @@ public class FileEntityHandler implements RequestHandler { } } + private BaseResponse handleGetFilesUserInfo(BaseRequest request, FileEntityRequest fileRequest) { + Long userId = getUserIdFromToken(fileRequest.getToken()); + FileInfoUserResponse userFileStats = fileStatsService.getUserFileStats(userId); + + return new BaseResponse(request.getMsgId(), + MessageCode.SUCCESS.getCode(), + MessageCode.SUCCESS.getDescription(), + userFileStats); + } + private BaseResponse handleGetFileInfo(BaseRequest request, FileEntityRequest fileRequest) { try { Long userId = getUserIdFromToken(fileRequest.getToken()); diff --git a/src/main/java/ru/soune/nocopy/service/file/FileStatsService.java b/src/main/java/ru/soune/nocopy/service/file/FileStatsService.java new file mode 100644 index 0000000..a32f04d --- /dev/null +++ b/src/main/java/ru/soune/nocopy/service/file/FileStatsService.java @@ -0,0 +1,93 @@ +package ru.soune.nocopy.service.file; + +import lombok.RequiredArgsConstructor; +import org.springframework.stereotype.Service; +import ru.soune.nocopy.dto.file.FileInfoUserResponse; +import ru.soune.nocopy.entity.file.FileEntity; +import ru.soune.nocopy.entity.file.FileStatus; +import ru.soune.nocopy.entity.file.FileType; +import ru.soune.nocopy.repository.FileEntityRepository; + +import java.util.List; + +@Service +@RequiredArgsConstructor +public class FileStatsService { + private final FileEntityRepository fileEntityRepository; + + public FileInfoUserResponse getUserFileStats(Long userId) { + List userFiles = fileEntityRepository.findByUserId(userId); + return calculateStats(userFiles); + } + + public FileInfoUserResponse calculateStats(List files) { + return FileInfoUserResponse.builder() + .allFileSize(calculateTotalSize(files)) + .fileCount(calculateTotalCount(files)) + .filesCheck(calculateByStatus(files, FileStatus.CHECKED)) + .filesViolation(calculateByStatus(files, FileStatus.VIOLATION)) + + .imagesSize(calculateMediaSize(files, FileType.IMAGE)) + .imagesCount(calculateMediaCount(files, FileType.IMAGE)) + .imagesCheck(calculateMediaByStatus(files, FileType.IMAGE, FileStatus.CHECKED)) + .imagesViolations(calculateMediaByStatus(files, FileType.IMAGE, FileStatus.VIOLATION)) + + .videosSize(calculateMediaSize(files, FileType.VIDEO)) + .videosCount(calculateMediaCount(files, FileType.VIDEO)) + .videosCheck(calculateMediaByStatus(files, FileType.VIDEO, FileStatus.CHECKED)) + .videosViolations(calculateMediaByStatus(files, FileType.VIDEO, FileStatus.VIOLATION)) + + .audiosSize(calculateMediaSize(files, FileType.AUDIO)) + .audiosCount(calculateMediaCount(files, FileType.AUDIO)) + .audiosCheck(calculateMediaByStatus(files, FileType.AUDIO, FileStatus.CHECKED)) + .audiosViolations(calculateMediaByStatus(files, FileType.AUDIO, FileStatus.VIOLATION)) + .build(); + } + private Long calculateTotalSize(List files) { + return files.stream() + .filter(file -> file.getStatus() != FileStatus.DELETED) + .mapToLong(FileEntity::getFileSize) + .sum(); + } + + private Integer calculateTotalCount(List files) { + return (int) files.stream() + .filter(file -> file.getStatus() != FileStatus.DELETED) + .count(); + } + + private Integer calculateByStatus(List files, FileStatus status) { + return (int) files.stream() + .filter(file -> file.getStatus() == status) + .count(); + } + + private Long calculateMediaSize(List files, FileType fileType) { + return files.stream() + .filter(file -> file.getStatus() != FileStatus.DELETED) + .filter(file -> isFileType(file, fileType)) + .mapToLong(FileEntity::getFileSize) + .sum(); + } + + private Integer calculateMediaCount(List files, FileType fileType) { + return (int) files.stream() + .filter(file -> file.getStatus() != FileStatus.DELETED) + .filter(file -> isFileType(file, fileType)) + .count(); + } + + private Integer calculateMediaByStatus(List files, FileType fileType, FileStatus status) { + return (int) files.stream() + .filter(file -> isFileType(file, fileType)) + .filter(file -> file.getStatus() == status) + .count(); + } + + private boolean isFileType(FileEntity file, FileType fileType) { + if (file.getFileExtension() == null) return false; + + String extension = file.getFileExtension().toLowerCase().replace(".", ""); + return fileType.supportsExtension(extension); + } +} From 918b4894fc7ea54126fe09947fb97e35066c10bc Mon Sep 17 00:00:00 2001 From: vladp Date: Tue, 6 Jan 2026 17:27:38 +0700 Subject: [PATCH 32/37] add endpoint with files user info --- docker-compose.yaml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/docker-compose.yaml b/docker-compose.yaml index f64115c..8df1c29 100644 --- a/docker-compose.yaml +++ b/docker-compose.yaml @@ -120,7 +120,7 @@ services: prometheus: image: prom/prometheus:latest container_name: prometheus - deploy: # ← ДОБАВЛЕНО + deploy: resources: limits: cpus: '0.5' @@ -148,7 +148,7 @@ services: loki: image: grafana/loki:2.9.2 container_name: loki - deploy: # ← ДОБАВЛЕНО + deploy: resources: limits: cpus: '1.0' @@ -173,7 +173,7 @@ services: tempo: image: grafana/tempo:2.4.1 container_name: tempo - deploy: # ← ДОБАВЛЕНО + deploy: resources: limits: cpus: '0.5' From b9a401f11378966c1b910c2ed8b988dabecf5cc0 Mon Sep 17 00:00:00 2001 From: vladp Date: Wed, 7 Jan 2026 11:56:03 +0700 Subject: [PATCH 33/37] correct file_extensions --- .../ru/soune/nocopy/controller/ApiController.java | 12 ------------ .../java/ru/soune/nocopy/entity/file/FileType.java | 13 +------------ .../ru/soune/nocopy/handler/FileEntityHandler.java | 1 - 3 files changed, 1 insertion(+), 25 deletions(-) diff --git a/src/main/java/ru/soune/nocopy/controller/ApiController.java b/src/main/java/ru/soune/nocopy/controller/ApiController.java index dbbe9df..4f9239d 100644 --- a/src/main/java/ru/soune/nocopy/controller/ApiController.java +++ b/src/main/java/ru/soune/nocopy/controller/ApiController.java @@ -368,18 +368,6 @@ public class ApiController { return contentType; } - private boolean isPreviewSupported(String mimeType) { - if (mimeType == null) { - return false; - } - - return mimeType.startsWith("image") || - mimeType.startsWith("text") || - mimeType.equals("pdf") || - mimeType.startsWith("video") || - mimeType.startsWith("audio"); - } - private Long getUserIdFromToken(String tokenHeader) { String token = tokenHeader.replace("Bearer ", ""); AuthToken authToken = authTokenRepository.findByToken(token) diff --git a/src/main/java/ru/soune/nocopy/entity/file/FileType.java b/src/main/java/ru/soune/nocopy/entity/file/FileType.java index a7071ef..82f1ebc 100644 --- a/src/main/java/ru/soune/nocopy/entity/file/FileType.java +++ b/src/main/java/ru/soune/nocopy/entity/file/FileType.java @@ -10,18 +10,7 @@ public enum FileType { IMAGE("image", Arrays.asList("jpg", "jpeg", "png", "gif", "bmp", "webp")), VIDEO("video", Arrays.asList("mp4", "avi", "mov", "wmv", "flv", "mkv", "webm", "m4v", "mpg", "mpeg", "3gp", "3g2", "f4v", "m2ts", "mts", "vob", "ogv", "divx")), - AUDIO("audio", Arrays.asList("mp3", "wav", "flac")), - DOCUMENT("document", Arrays.asList( - "pdf", "txt", "rtf", - "doc", "docx", "xls", "xlsx", "ppt", "pptx", "pps", "ppsx", "dot", "dotx", "xlt", "xltx", "pot", "potx", - "odt", "ods", "odp", "odg", "odf", "odb", "odc", "odi", "odm", "ott", "ots", "otp", "otg", "oth", - "sxw", "sxc", "sxi", "sxd", "sxg", "stc", "sti", "stw", "sxm", - "pages", "numbers", "key", - "csv", "tsv", "xml", "html", "htm", "tex", "md", "markdown", - "epub", "mobi", "azw", "azw3", "fb2", - "wps", "wpt", "et", "dps", "vsd", "vsdx", - "java", "py", "cpp", "c", "h", "js", "css", "php", "sql", "json", "yaml", "yml", "sh", "bat", - "one", "note")); + AUDIO("audio", Arrays.asList("mp3", "wav", "flac")); private final String displayName; private final List allowedExtensions; diff --git a/src/main/java/ru/soune/nocopy/handler/FileEntityHandler.java b/src/main/java/ru/soune/nocopy/handler/FileEntityHandler.java index 9bddff0..cacf0eb 100644 --- a/src/main/java/ru/soune/nocopy/handler/FileEntityHandler.java +++ b/src/main/java/ru/soune/nocopy/handler/FileEntityHandler.java @@ -9,7 +9,6 @@ import ru.soune.nocopy.dto.file.*; import ru.soune.nocopy.entity.AuthToken; import ru.soune.nocopy.entity.file.FileEntity; import ru.soune.nocopy.entity.file.FileStatus; -import ru.soune.nocopy.entity.file.FileType; import ru.soune.nocopy.exception.FileEntityNotFoundException; import ru.soune.nocopy.exception.NotFoundAuthToken; import ru.soune.nocopy.repository.AuthTokenRepository; From f8673dd5817cc65253d1cf98e3414cf00256e690 Mon Sep 17 00:00:00 2001 From: vladp Date: Wed, 7 Jan 2026 12:50:40 +0700 Subject: [PATCH 34/37] set chunk --- docker-compose.yaml | 2 +- .../ru/soune/nocopy/service/file/FileUploadServiceImpl.java | 2 +- src/main/resources/application.yaml | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/docker-compose.yaml b/docker-compose.yaml index 8df1c29..1225275 100644 --- a/docker-compose.yaml +++ b/docker-compose.yaml @@ -58,7 +58,7 @@ services: environment: FILE_STORAGE_PATH: /data/uploads MAX_FILE_SIZE: 10737418240 - FILE_CHUNK_SIZE: 1048576 + FILE_CHUNK_SIZE: 1000000 POSTGRES_DB: no_copy_ POSTGRES_USER: postgres POSTGRES_PASSWORD: postgres diff --git a/src/main/java/ru/soune/nocopy/service/file/FileUploadServiceImpl.java b/src/main/java/ru/soune/nocopy/service/file/FileUploadServiceImpl.java index f54cec9..3e09822 100644 --- a/src/main/java/ru/soune/nocopy/service/file/FileUploadServiceImpl.java +++ b/src/main/java/ru/soune/nocopy/service/file/FileUploadServiceImpl.java @@ -102,7 +102,7 @@ public class FileUploadServiceImpl implements FileUploadService { int totalChunks = (int) Math.ceil((double) fileSize / chunkSize); log.debug("File will be split into {} chunks (chunk size: {} bytes)", totalChunks, chunkSize); - Long chunkSize = totalChunks == 1 ? fileSize : 1048576L; + Long chunkSize = totalChunks == 1 ? fileSize : 1000000L; FileUploadSession session = FileUploadSession.builder() .userId(userId) diff --git a/src/main/resources/application.yaml b/src/main/resources/application.yaml index 1fd4fab..a9970b3 100644 --- a/src/main/resources/application.yaml +++ b/src/main/resources/application.yaml @@ -25,7 +25,7 @@ file: storage: base-path: ${FILE_STORAGE_PATH:/data/uploads} # chunk-size: ${FILE_CHUNK_SIZE:5242880} # 5MB - chunk-size: ${FILE_CHUNK_SIZE:1048576} # 15MB + chunk-size: ${FILE_CHUNK_SIZE:1000000} # 1MB max-file-size: ${MAX_FILE_SIZE:10737418240} # 10GB max-retry-attempts: ${MAX_RETRY_ATTEMPTS:3} chunk-timeout-ms: ${CHUNK_TIMEOUT_MS:300000} # 5 минут From 6fff5884d2fc0e33ec9460c363a56feb8796d71b Mon Sep 17 00:00:00 2001 From: vladp Date: Wed, 7 Jan 2026 13:16:53 +0700 Subject: [PATCH 35/37] set chunk --- docker-compose.yaml | 2 +- .../ru/soune/nocopy/service/file/FileUploadServiceImpl.java | 2 +- src/main/resources/application.yaml | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/docker-compose.yaml b/docker-compose.yaml index 1225275..8df1c29 100644 --- a/docker-compose.yaml +++ b/docker-compose.yaml @@ -58,7 +58,7 @@ services: environment: FILE_STORAGE_PATH: /data/uploads MAX_FILE_SIZE: 10737418240 - FILE_CHUNK_SIZE: 1000000 + FILE_CHUNK_SIZE: 1048576 POSTGRES_DB: no_copy_ POSTGRES_USER: postgres POSTGRES_PASSWORD: postgres diff --git a/src/main/java/ru/soune/nocopy/service/file/FileUploadServiceImpl.java b/src/main/java/ru/soune/nocopy/service/file/FileUploadServiceImpl.java index 3e09822..f54cec9 100644 --- a/src/main/java/ru/soune/nocopy/service/file/FileUploadServiceImpl.java +++ b/src/main/java/ru/soune/nocopy/service/file/FileUploadServiceImpl.java @@ -102,7 +102,7 @@ public class FileUploadServiceImpl implements FileUploadService { int totalChunks = (int) Math.ceil((double) fileSize / chunkSize); log.debug("File will be split into {} chunks (chunk size: {} bytes)", totalChunks, chunkSize); - Long chunkSize = totalChunks == 1 ? fileSize : 1000000L; + Long chunkSize = totalChunks == 1 ? fileSize : 1048576L; FileUploadSession session = FileUploadSession.builder() .userId(userId) diff --git a/src/main/resources/application.yaml b/src/main/resources/application.yaml index a9970b3..363d191 100644 --- a/src/main/resources/application.yaml +++ b/src/main/resources/application.yaml @@ -25,7 +25,7 @@ file: storage: base-path: ${FILE_STORAGE_PATH:/data/uploads} # chunk-size: ${FILE_CHUNK_SIZE:5242880} # 5MB - chunk-size: ${FILE_CHUNK_SIZE:1000000} # 1MB + chunk-size: ${FILE_CHUNK_SIZE:1048576} # 1MB max-file-size: ${MAX_FILE_SIZE:10737418240} # 10GB max-retry-attempts: ${MAX_RETRY_ATTEMPTS:3} chunk-timeout-ms: ${CHUNK_TIMEOUT_MS:300000} # 5 минут From 3a9a5b48a2244d26ac65cff297874b2ddb4297a4 Mon Sep 17 00:00:00 2001 From: vladp Date: Wed, 7 Jan 2026 14:10:35 +0700 Subject: [PATCH 36/37] update types --- infrastructure/jenkins/deploy.groovy | 180 ++++-------------- .../ru/soune/nocopy/entity/file/FileType.java | 2 +- 2 files changed, 40 insertions(+), 142 deletions(-) diff --git a/infrastructure/jenkins/deploy.groovy b/infrastructure/jenkins/deploy.groovy index a05e8ca..7611efe 100644 --- a/infrastructure/jenkins/deploy.groovy +++ b/infrastructure/jenkins/deploy.groovy @@ -7,7 +7,6 @@ pipeline { defaultValue: 'dev', description: 'Ветка для деплоя' ) - string( name: 'SERVER', defaultValue: '92.242.61.23', @@ -51,156 +50,57 @@ pipeline { sshpass -p '$SSH_PASS' ssh -o StrictHostKeyChecking=no $SSH_USER@$SERVER " cd /opt/deployments/${params.BRANCH} - echo '========================================' - echo 'Step 1: Checking current state...' - echo '========================================' + echo '1. Остановка старого приложения...' + docker stop app-backend 2>/dev/null || echo 'Контейнер не найден' + docker rm app-backend 2>/dev/null || echo 'Контейнер не найден' - docker ps --filter 'name=postgres|app-backend|storage|grafana|prometheus|loki|tempo|alloy' --format 'table {{.Names}}\\t{{.Image}}\\t{{.Status}}' + echo '2. Удаление старых образов...' + docker images --filter 'reference=*app*' -q | xargs -r docker rmi -f 2>/dev/null || echo 'Нет образов для удаления' - echo '========================================' - echo 'Step 2: Setup environment for compatibility...' - echo '========================================' + echo '3. Создание сети если нужно...' + docker network create app-network 2>/dev/null || echo 'Сеть уже существует' - docker network create app-network 2>/dev/null || echo 'Network app-network already exists' + echo '4. Запуск инфраструктуры...' + docker-compose up -d db storage - echo 'COMPOSE_COMPATIBILITY=true' > .env + echo '5. Ожидание PostgreSQL...' + sleep 10 - echo '========================================' - echo 'Step 3: Force cleanup old application...' - echo '========================================' + echo '6. Сборка нового образа приложения...' + docker build --no-cache -t app-backend:latest . - docker-compose --compatibility down 2>/dev/null || echo 'No previous compose stack' + echo '7. Запуск приложения...' + docker run -d \\ + --name app-backend \\ + --network app-network \\ + --network-alias app \\ + -p 80:8080 \\ + -v uploads_data:/data/uploads:rw \\ + -e POSTGRES_DB=no_copy_ \\ + -e POSTGRES_USER=postgres \\ + -e POSTGRES_PASSWORD=postgres \\ + -e POSTGRES_PORT=5432 \\ + -e POSTGRES_HOST=db \\ + --restart unless-stopped \\ + app-backend:latest - docker images --filter 'reference=*app-backend*' -q | xargs -r docker rmi -f 2>/dev/null || echo 'No images to remove' + echo '8. Запуск мониторинга...' + docker-compose up -d grafana prometheus loki tempo alloy - echo '========================================' - echo 'Step 4: Verify files and fix compose...' - echo '========================================' + echo '9. Проверка...' + sleep 5 - if [ ! -f docker-compose.yaml ]; then - echo 'ERROR: docker-compose.yaml not found!' - ls -la - exit 1 - fi + echo 'Статус контейнеров:' + docker ps --format 'table {{.Names}}\\t{{.Image}}\\t{{.Status}}' - echo 'Checking docker-compose.yaml syntax...' - docker-compose --compatibility config 2>&1 | head -20 || echo 'Config check output' - - echo '========================================' - echo 'Step 5: Start infrastructure services...' - echo '========================================' - - echo 'Starting PostgreSQL with resource limits...' - docker-compose --compatibility up -d db - - echo 'Starting storage service...' - docker-compose --compatibility up -d storage - - echo 'Waiting for PostgreSQL to be ready...' - for i in {1..30}; do - if docker-compose --compatibility exec db pg_isready -U postgres 2>/dev/null; then - echo 'PostgreSQL is ready' - break - fi - echo 'Waiting... attempt ' \$i - sleep 2 - done - - echo '========================================' - echo 'Step 6: Build application image...' - echo '========================================' - - if [ ! -f Dockerfile ]; then - echo 'ERROR: Dockerfile not found!' - exit 1 - fi - - echo 'Building Docker image...' - docker build --no-cache -t app-backend:latest . 2>&1 | tail -30 - - echo 'Image created successfully' - docker images app-backend:latest --format 'table {{.Repository}}\\t{{.Tag}}\\t{{.Size}}\\t{{.CreatedAt}}' - - echo '========================================' - echo 'Step 7: Start application with resource limits...' - echo '========================================' - - docker volume create uploads_data 2>/dev/null || echo 'Volume uploads_data already exists' - - echo 'Starting main application...' - docker-compose --compatibility up -d app - - echo 'Waiting for application to start...' - sleep 15 - - echo '========================================' - echo 'Step 8: Start monitoring stack...' - echo '========================================' - - docker-compose --compatibility up -d grafana prometheus loki tempo alloy - - echo '========================================' - echo 'Step 9: Verify deployment...' - echo '========================================' - - echo 'All containers status:' - docker-compose --compatibility ps - - echo '' - echo 'Resource limits check:' - echo 'App backend limits:' - docker inspect app-backend --format='{{.HostConfig.Memory}} {{.HostConfig.CpuQuota}} {{.HostConfig.CpuPeriod}}' 2>/dev/null || echo 'Cannot inspect' - - echo '' - echo 'PostgreSQL limits:' - docker inspect postgres --format='{{.HostConfig.Memory}} {{.HostConfig.CpuQuota}} {{.HostConfig.CpuPeriod}}' 2>/dev/null || echo 'Cannot inspect' - - echo '' - echo 'Container resource usage:' - docker stats --no-stream --format 'table {{.Name}}\\t{{.CPUPerc}}\\t{{.MemUsage}}\\t{{.MemPerc}}' postgres app-backend storage 2>/dev/null || echo 'Stats not available' - - echo '========================================' - echo 'Step 10: Health checks...' - echo '========================================' - - echo 'Application health check...' - for i in {1..10}; do - if curl -s -f http://localhost:80/health 2>/dev/null; then - echo '✓ Application health check PASSED' - break - elif [ \$i -eq 10 ]; then - echo '✗ Application health check FAILED' - echo 'Last logs:' - docker-compose --compatibility logs --tail=20 app - else - echo 'Waiting for application... attempt ' \$i - sleep 5 - fi - done - - echo 'Checking monitoring services...' - if curl -s -f http://localhost:3000/api/health 2>/dev/null; then - echo '✓ Grafana is healthy' + echo '10. Проверка health...' + if curl -s -f http://localhost:80/health > /dev/null 2>&1; then + echo 'Приложение работает' + echo 'URL: http://${params.SERVER}:80' else - echo '⚠ Grafana may be starting' + echo 'Проверка health не удалась' + docker logs app-backend --tail=20 fi - - if curl -s -f http://localhost:9090/-/healthy 2>/dev/null; then - echo '✓ Prometheus is healthy' - fi - - echo '========================================' - echo 'Deployment Summary:' - echo '========================================' - echo 'Application URL: http://${params.SERVER}:80' - echo 'Grafana: http://${params.SERVER}:3000 (admin/admin)' - echo 'Prometheus: http://${params.SERVER}:9090' - echo 'Loki: http://${params.SERVER}:3100' - echo '' - echo 'To view logs: docker-compose logs -f app' - echo 'To view all containers: docker-compose ps' - echo 'To stop: docker-compose down' - echo '========================================' " """ } @@ -213,8 +113,6 @@ pipeline { success { echo "Deployment successful" echo "Application URL: http://${params.SERVER}:80" - echo "Grafana: http://${params.SERVER}:3000" - echo "Prometheus: http://${params.SERVER}:9090" } failure { echo "Deployment failed for branch ${params.BRANCH}" diff --git a/src/main/java/ru/soune/nocopy/entity/file/FileType.java b/src/main/java/ru/soune/nocopy/entity/file/FileType.java index 82f1ebc..8844013 100644 --- a/src/main/java/ru/soune/nocopy/entity/file/FileType.java +++ b/src/main/java/ru/soune/nocopy/entity/file/FileType.java @@ -7,7 +7,7 @@ import java.util.List; @Getter public enum FileType { - IMAGE("image", Arrays.asList("jpg", "jpeg", "png", "gif", "bmp", "webp")), + IMAGE("image", Arrays.asList("jpg", "jpeg", "png", "gif", "bmp", "webp", "jfif")), VIDEO("video", Arrays.asList("mp4", "avi", "mov", "wmv", "flv", "mkv", "webm", "m4v", "mpg", "mpeg", "3gp", "3g2", "f4v", "m2ts", "mts", "vob", "ogv", "divx")), AUDIO("audio", Arrays.asList("mp3", "wav", "flac")); From 63baaf7c807eb9eebe1763e9349c2da339408080 Mon Sep 17 00:00:00 2001 From: vladp Date: Thu, 8 Jan 2026 12:59:30 +0700 Subject: [PATCH 37/37] update types --- docker-compose.yaml | 3 ++- .../ru/soune/nocopy/service/file/FileUploadServiceImpl.java | 2 +- src/main/resources/application.yaml | 3 ++- 3 files changed, 5 insertions(+), 3 deletions(-) diff --git a/docker-compose.yaml b/docker-compose.yaml index 8df1c29..1ce8455 100644 --- a/docker-compose.yaml +++ b/docker-compose.yaml @@ -58,7 +58,8 @@ services: environment: FILE_STORAGE_PATH: /data/uploads MAX_FILE_SIZE: 10737418240 - FILE_CHUNK_SIZE: 1048576 +# FILE_CHUNK_SIZE: 1048576 + FILE_CHUNK_SIZE: 1000000 POSTGRES_DB: no_copy_ POSTGRES_USER: postgres POSTGRES_PASSWORD: postgres diff --git a/src/main/java/ru/soune/nocopy/service/file/FileUploadServiceImpl.java b/src/main/java/ru/soune/nocopy/service/file/FileUploadServiceImpl.java index f54cec9..3e09822 100644 --- a/src/main/java/ru/soune/nocopy/service/file/FileUploadServiceImpl.java +++ b/src/main/java/ru/soune/nocopy/service/file/FileUploadServiceImpl.java @@ -102,7 +102,7 @@ public class FileUploadServiceImpl implements FileUploadService { int totalChunks = (int) Math.ceil((double) fileSize / chunkSize); log.debug("File will be split into {} chunks (chunk size: {} bytes)", totalChunks, chunkSize); - Long chunkSize = totalChunks == 1 ? fileSize : 1048576L; + Long chunkSize = totalChunks == 1 ? fileSize : 1000000L; FileUploadSession session = FileUploadSession.builder() .userId(userId) diff --git a/src/main/resources/application.yaml b/src/main/resources/application.yaml index 363d191..da03a23 100644 --- a/src/main/resources/application.yaml +++ b/src/main/resources/application.yaml @@ -25,7 +25,8 @@ file: storage: base-path: ${FILE_STORAGE_PATH:/data/uploads} # chunk-size: ${FILE_CHUNK_SIZE:5242880} # 5MB - chunk-size: ${FILE_CHUNK_SIZE:1048576} # 1MB +# chunk-size: ${FILE_CHUNK_SIZE:1048576} # 1MB + chunk-size: ${FILE_CHUNK_SIZE:1000000} # 1MB max-file-size: ${MAX_FILE_SIZE:10737418240} # 10GB max-retry-attempts: ${MAX_RETRY_ATTEMPTS:3} chunk-timeout-ms: ${CHUNK_TIMEOUT_MS:300000} # 5 минут