diff --git a/Dockerfile b/Dockerfile index 8242e14..6cd62ca 100644 --- a/Dockerfile +++ b/Dockerfile @@ -12,6 +12,8 @@ WORKDIR /app COPY --from=build /app/build/libs/*.jar app.jar +RUN mkdir -p /data/uploads && chmod 755 /data/uploads + EXPOSE 8080 CMD ["java", "-jar", "app.jar"] diff --git a/docker-compose.yaml b/docker-compose.yaml index eb4724b..0a65adf 100644 --- a/docker-compose.yaml +++ b/docker-compose.yaml @@ -1,13 +1,39 @@ version: '3.9' services: + storage: + image: alpine:latest + container_name: file-storage + deploy: + resources: + limits: + memory: 128M + reservations: + memory: 64M + networks: + - app-network + volumes: + - uploads_data:/storage:rw + command: tail -f /dev/null + restart: unless-stopped + db: image: postgres:17 restart: always + deploy: + resources: + limits: + cpus: '2.0' + memory: 2G + reservations: + cpus: '1.0' + memory: 1G environment: POSTGRES_DB: no_copy_ POSTGRES_USER: postgres POSTGRES_PASSWORD: postgres + POSTGRES_SHARED_BUFFERS: 512MB + POSTGRES_EFFECTIVE_CACHE_SIZE: 1536MB ports: - "54320:5432" volumes: @@ -21,10 +47,19 @@ services: app: build: . container_name: app-backend + deploy: + resources: + limits: + cpus: '1.5' + memory: 1G + reservations: + cpus: '0.5' + memory: 512M environment: FILE_STORAGE_PATH: /data/uploads - MAX_FILE_SIZE: 1073741824 - FILE_CHUNK_SIZE: 5242880 + MAX_FILE_SIZE: 10737418240 +# FILE_CHUNK_SIZE: 1048576 + FILE_CHUNK_SIZE: 1000000 POSTGRES_DB: no_copy_ POSTGRES_USER: postgres POSTGRES_PASSWORD: postgres @@ -32,6 +67,7 @@ services: POSTGRES_HOST: db YANDEX_SEARCH_API_KEY: AQVNyaVaUmgUb1GMCtf5zSEqFxy0woXrcMOOB43q YANDEX_SEARCH_FOLDER_ID: b1gokpdbm6qfpsou8pcd + STORAGE_SERVICE_URL: http://storage:8081 depends_on: - db ports: @@ -43,11 +79,18 @@ services: - backend - api volumes: - - ./uploads:/data/uploads:rw + - uploads_data:/data/uploads:rw grafana: image: grafana/grafana:10.3.1 container_name: grafana + deploy: + resources: + limits: + cpus: '0.5' + memory: 512M + reservations: + memory: 256M ports: - "3000:3000" depends_on: @@ -64,6 +107,8 @@ services: GF_SECURITY_ADMIN_USER: admin GF_SECURITY_ADMIN_PASSWORD: admin GF_METRICS_ENABLED: "true" + GF_DATABASE_MAX_IDLE_CONN: "2" + GF_DATABASE_MAX_OPEN_CONN: "10" volumes: - ./infrastructure/grafana/provisioning:/etc/grafana/provisioning - ./infrastructure/grafana/dashboards:/var/lib/grafana/dashboards @@ -78,10 +123,23 @@ services: prometheus: image: prom/prometheus:latest container_name: prometheus + deploy: + resources: + limits: + cpus: '0.5' + memory: 1G + reservations: + memory: 512M ports: - "9090:9090" volumes: - ./infrastructure/prometheus/prometheus.yml:/etc/prometheus/prometheus.yml + - prometheus_data:/prometheus + command: + - '--config.file=/etc/prometheus/prometheus.yml' + - '--storage.tsdb.path=/prometheus' + - '--storage.tsdb.retention.time=15d' # Удерживать 15 дней + - '--web.enable-lifecycle' healthcheck: test: [ "CMD", "wget", "--spider", "http://localhost:9090/-/healthy" ] interval: 10s @@ -93,10 +151,20 @@ services: loki: image: grafana/loki:2.9.2 container_name: loki + deploy: + resources: + limits: + cpus: '1.0' + memory: 2G + reservations: + memory: 1G ports: - "3100:3100" volumes: - ./infrastructure/loki/loki-config.yaml:/etc/loki/loki-config.yaml:ro + - loki_data:/loki + command: + - -config.file=/etc/loki/loki-config.yaml healthcheck: test: [ "CMD", "wget", "--spider", "-q", "http://localhost:3100/ready" ] interval: 10s @@ -108,6 +176,13 @@ services: tempo: image: grafana/tempo:2.4.1 container_name: tempo + deploy: + resources: + limits: + cpus: '0.5' + memory: 512M + reservations: + memory: 256M command: [ "-config.file=/etc/tempo/tempo.yaml" ] volumes: - ./infrastructure/tempo/tempo.yaml:/etc/tempo/tempo.yaml @@ -120,6 +195,13 @@ services: alloy: image: grafana/alloy:latest container_name: alloy + deploy: + resources: + limits: + cpus: '0.5' + memory: 256M + reservations: + memory: 128M user: root ports: - "9080:9080" @@ -148,7 +230,8 @@ volumes: loki_chunks: loki_index: loki_rules: -# uploads_volume: + uploads_data: + prometheus_data: networks: app-network: diff --git a/infrastructure/jenkins/deploy.groovy b/infrastructure/jenkins/deploy.groovy index 43b39ff..7611efe 100644 --- a/infrastructure/jenkins/deploy.groovy +++ b/infrastructure/jenkins/deploy.groovy @@ -7,7 +7,6 @@ pipeline { defaultValue: 'dev', description: 'Ветка для деплоя' ) - string( name: 'SERVER', defaultValue: '92.242.61.23', @@ -45,75 +44,38 @@ pipeline { echo "Deploying branch: ${params.BRANCH}" echo "Copying files to server..." - sshpass -p '$SSH_PASS' ssh $SSH_USER@$SERVER "mkdir -p /opt/deployments/${params.BRANCH}" - sshpass -p '$SSH_PASS' scp -r ./* $SSH_USER@$SERVER:/opt/deployments/${params.BRANCH}/ + sshpass -p '$SSH_PASS' ssh -o StrictHostKeyChecking=no $SSH_USER@$SERVER "mkdir -p /opt/deployments/${params.BRANCH}" + sshpass -p '$SSH_PASS' scp -r -o StrictHostKeyChecking=no ./* $SSH_USER@$SERVER:/opt/deployments/${params.BRANCH}/ - sshpass -p '$SSH_PASS' ssh $SSH_USER@$SERVER " + sshpass -p '$SSH_PASS' ssh -o StrictHostKeyChecking=no $SSH_USER@$SERVER " cd /opt/deployments/${params.BRANCH} - echo 'Step 1: Checking current state...' + echo '1. Остановка старого приложения...' + docker stop app-backend 2>/dev/null || echo 'Контейнер не найден' + docker rm app-backend 2>/dev/null || echo 'Контейнер не найден' - # Смотрим что запущено - docker ps --filter 'name=postgres|app-backend' --format 'table {{.Names}}\\t{{.Image}}\\t{{.Status}}\\t{{.Ports}}' + echo '2. Удаление старых образов...' + docker images --filter 'reference=*app*' -q | xargs -r docker rmi -f 2>/dev/null || echo 'Нет образов для удаления' - echo 'Step 2: Force cleanup old application...' + echo '3. Создание сети если нужно...' + docker network create app-network 2>/dev/null || echo 'Сеть уже существует' - # Принудительно удаляем старый контейнер и образ - docker stop app-backend 2>/dev/null || echo 'App not running' - docker rm app-backend 2>/dev/null || echo 'App not found' - docker rmi app-backend:latest 2>/dev/null || echo 'Image not found' + echo '4. Запуск инфраструктуры...' + docker-compose up -d db storage - # Удаляем все образы app-backend - docker images --filter 'reference=app-backend*' -q | xargs -r docker rmi 2>/dev/null || echo 'No images to remove' + echo '5. Ожидание PostgreSQL...' + sleep 10 - echo 'Step 3: Verify copied files...' - echo 'Files in directory:' - ls -la - echo '' - echo 'Checking Java sources:' - find . -name '*.java' | head -2 + echo '6. Сборка нового образа приложения...' + docker build --no-cache -t app-backend:latest . - echo 'Step 4: Check if PostgreSQL is running...' - if ! docker ps | grep -q postgres; then - echo 'Starting PostgreSQL...' - docker-compose up -d db - sleep 10 - else - echo 'PostgreSQL already running' - fi - - echo 'Step 5: Build application with forced rebuild...' - echo 'Checking Dockerfile exists:' - ls -la Dockerfile 2>/dev/null || echo 'Dockerfile not found, creating simple one' - - # Если нет Dockerfile, создаем простой - if [ ! -f Dockerfile ]; then - echo 'Creating simple Dockerfile...' - cat > Dockerfile << 'EOF' -FROM eclipse-temurin:21-jre -WORKDIR /app -COPY . /app/ -CMD ["java", "-jar", "app.jar"] -EOF - fi - - # Собираем с подробным выводом - echo 'Building Docker image...' - docker build --no-cache --progress=plain -t app-backend:latest . 2>&1 | tail -50 - - echo 'Step 6: Verify new image...' - echo 'Current app-backend images:' - docker images | grep app-backend - - echo 'Image creation time:' - docker inspect app-backend:latest --format='{{.Created}}' 2>/dev/null || echo 'Cannot inspect image' - - echo 'Step 7: Starting application with alias...' + echo '7. Запуск приложения...' docker run -d \\ --name app-backend \\ --network app-network \\ --network-alias app \\ -p 80:8080 \\ + -v uploads_data:/data/uploads:rw \\ -e POSTGRES_DB=no_copy_ \\ -e POSTGRES_USER=postgres \\ -e POSTGRES_PASSWORD=postgres \\ @@ -122,40 +84,22 @@ EOF --restart unless-stopped \\ app-backend:latest - echo 'Step 8: Checking deployment...' - sleep 10 + echo '8. Запуск мониторинга...' + docker-compose up -d grafana prometheus loki tempo alloy - echo 'Containers status:' - docker ps --filter 'name=postgres|app-backend' --format 'table {{.Names}}\\t{{.Image}}\\t{{.Status}}\\t{{.CreatedAt}}' + echo '9. Проверка...' + sleep 5 - echo 'Checking image inside container:' - docker exec app-backend ls -la /app/ 2>/dev/null || echo 'Cannot check container files' + echo 'Статус контейнеров:' + docker ps --format 'table {{.Names}}\\t{{.Image}}\\t{{.Status}}' - echo 'Application logs (last 10 lines):' - docker logs --tail=10 app-backend 2>/dev/null || echo 'Logs not available yet' - - echo 'Step 9: Health check...' - if docker ps | grep -q app-backend; then - echo 'Container is running' - echo 'Testing application health...' - for i in {1..5}; do - if curl -s -f http://localhost:80/health > /dev/null 2>&1; then - echo 'Health check passed' - echo 'Deployment successful' - echo 'Application URL: http://${params.SERVER}:80' - exit 0 - fi - echo 'Waiting for application to start... attempt ' \$i - sleep 5 - done - - echo 'Health check failed, but container is running' - echo 'Application might be starting slowly' - echo 'Application URL: http://${params.SERVER}:80' + echo '10. Проверка health...' + if curl -s -f http://localhost:80/health > /dev/null 2>&1; then + echo 'Приложение работает' + echo 'URL: http://${params.SERVER}:80' else - echo 'Application failed to start' - docker logs app-backend - exit 1 + echo 'Проверка health не удалась' + docker logs app-backend --tail=20 fi " """ diff --git a/src/main/java/ru/soune/nocopy/NoCopyApplication.java b/src/main/java/ru/soune/nocopy/NoCopyApplication.java index fd6b0f2..0c9c534 100644 --- a/src/main/java/ru/soune/nocopy/NoCopyApplication.java +++ b/src/main/java/ru/soune/nocopy/NoCopyApplication.java @@ -2,8 +2,10 @@ package ru.soune.nocopy; import org.springframework.boot.SpringApplication; import org.springframework.boot.autoconfigure.SpringBootApplication; +import org.springframework.scheduling.annotation.EnableScheduling; @SpringBootApplication +@EnableScheduling public class NoCopyApplication { public static void main(String[] args) { diff --git a/src/main/java/ru/soune/nocopy/controller/ApiController.java b/src/main/java/ru/soune/nocopy/controller/ApiController.java index 9ecad18..4f9239d 100644 --- a/src/main/java/ru/soune/nocopy/controller/ApiController.java +++ b/src/main/java/ru/soune/nocopy/controller/ApiController.java @@ -20,10 +20,9 @@ import ru.soune.nocopy.dto.file.CompleteUploadResponse; import ru.soune.nocopy.dto.file.FileEntityResponse; import ru.soune.nocopy.dto.file.UploadProgress; import ru.soune.nocopy.entity.AuthToken; +import ru.soune.nocopy.entity.file.FileStatus; import ru.soune.nocopy.entity.file.UploadStatus; -import ru.soune.nocopy.exception.NotFoundAuthToken; -import ru.soune.nocopy.exception.NotValidFieldException; -import ru.soune.nocopy.exception.ValidationException; +import ru.soune.nocopy.exception.*; import ru.soune.nocopy.handler.*; import ru.soune.nocopy.repository.AuthTokenRepository; import ru.soune.nocopy.service.file.FileEntityService; @@ -89,29 +88,34 @@ public class ApiController { @PostMapping("/v{version}/files/chunk") public ResponseEntity uploadChunk( @PathVariable("version") int version, - @RequestParam("upload_id") String uploadId, - @RequestParam("chunk_number") Integer chunkNumber, - @RequestParam("chunk") MultipartFile chunk) { - - log.info("Uploading chunk {} for session {}, file size: {} bytes, version: {}", - chunkNumber, uploadId, chunk.getSize(), version); - - if (chunk.isEmpty()) { - return ResponseEntity.ok().body(new BaseResponse(20004, MessageCode.FILE_UPLOAD_ERROR.getCode(), - "Chunk file is empty", null)); - } - - if (uploadId == null || uploadId.isBlank()) { - return ResponseEntity.ok().body(new BaseResponse(20004, MessageCode.FILE_UPLOAD_ERROR.getCode(), - "Upload ID is required", null)); - } - - if (chunkNumber == null || chunkNumber < 0) { - return ResponseEntity.ok().body(new BaseResponse(20004, MessageCode.FILE_UPLOAD_ERROR.getCode(), - "Valid chunk number is required", null)); - } - + @RequestParam(value = "upload_id", required = false) String uploadId, + @RequestParam(value = "chunk_number", required = false) Integer chunkNumber, + @RequestParam(value = "chunk", required = false) MultipartFile chunk) { try { + if (chunk == null || chunk.isEmpty()) { + return ResponseEntity.ok().body(new BaseResponse(20004, MessageCode.FILE_UPLOAD_ERROR.getCode(), + "Chunk file null or empty", ChunkUploadResponse.builder() + .uploadId(uploadId) + .chunkNumber(chunkNumber) + .build())); + } + + if (uploadId == null || uploadId.isBlank()) { + return ResponseEntity.ok().body(new BaseResponse(20004, MessageCode.FILE_UPLOAD_ERROR.getCode(), + "Upload ID is required", ChunkUploadResponse.builder() + .uploadId(uploadId) + .chunkNumber(chunkNumber) + .build())); + } + + if (chunkNumber == null || chunkNumber < 0) { + return ResponseEntity.ok().body(new BaseResponse(20004, MessageCode.FILE_UPLOAD_ERROR.getCode(), + "Valid chunk number is required", ChunkUploadResponse.builder() + .uploadId(uploadId) + .chunkNumber(chunkNumber) + .build())); + } + fileUploadService.uploadChunk(uploadId, chunkNumber, chunk); ChunkUploadResponse responseBody = ChunkUploadResponse.builder() @@ -123,7 +127,6 @@ public class ApiController { return ResponseEntity.ok().body(new BaseResponse(20004, MessageCode.SUCCESS.getCode(), "Chunk uploaded successfully", responseBody)); - } catch (Exception e) { log.error("Error uploading chunk", e); @@ -237,29 +240,67 @@ public class ApiController { @GetMapping("/v{version}/files/download/{fileId}") - public ResponseEntity downloadFile( - @PathVariable String fileId, - @PathVariable Integer version, - @RequestHeader("Authorization") String tokenHeader) { - + public ResponseEntity downloadFile( + @PathVariable(required = false) String fileId, + @PathVariable(required = false) Integer version, + @RequestHeader(value = "Authorization", required = false) String tokenHeader) { try { + if (tokenHeader == null) { + Map errorData = new HashMap<>(); + errorData.put("token", tokenHeader); + errorData.put("fileId", fileId); + errorData.put("version", version); + + return ResponseEntity.ok().body(new BaseResponse(20004, + MessageCode.FILE_DOWNLOAD_ERROR_NOT_CORRECT_FIELD.getCode(), + MessageCode.FILE_DOWNLOAD_ERROR_NOT_CORRECT_FIELD.getDescription(), + errorData)); + } + Long userId = getUserIdFromToken(tokenHeader); - FileEntityResponse fileInfo = fileEntityService.getById(fileId, version); + FileEntityResponse entityResponse = fileEntityService.getById(fileId, version); - if (!fileInfo.getUserId().equals(userId)) { - return ResponseEntity.status(403).build(); + if (!entityResponse.getUserId().equals(userId)) { + Map errorData = new HashMap<>(); + errorData.put("token", tokenHeader); + + return ResponseEntity.ok().body(new BaseResponse(20004, + MessageCode.FILE_DOWNLOAD_ERROR.getCode(), + MessageCode.FILE_DOWNLOAD_ERROR.getDescription(), + errorData)); } - if (!fileInfo.isExistsOnDisk()) { - return ResponseEntity.status(404) - .body(null); + if (entityResponse.getStatus().equals(FileStatus.DELETED)) { + Map errorData = new HashMap<>(); + errorData.put("file_status", entityResponse.getStatus()); + + return ResponseEntity.ok().body(new BaseResponse(20004, + MessageCode.FILE_DOWNLOAD_ERROR.getCode(), + MessageCode.FILE_DOWNLOAD_ERROR.getDescription(), + errorData)); } - Path filePath = Paths.get(fileInfo.getFilePath()); + if (!entityResponse.isExistsOnDisk()) { + Map errorData = new HashMap<>(); + errorData.put("onDisk", entityResponse.isExistsOnDisk()); + + return ResponseEntity.ok().body(new BaseResponse(20004, + MessageCode.FILE_DOWNLOAD_ERROR.getCode(), + MessageCode.FILE_DOWNLOAD_ERROR.getDescription(), + errorData)); + } + + Path filePath = Paths.get(entityResponse.getFilePath()); Resource resource = new UrlResource(filePath.toUri()); if (!resource.exists()) { - return ResponseEntity.status(404).build(); + Map errorData = new HashMap<>(); + errorData.put("resource", resource.exists()); + + return ResponseEntity.ok().body(new BaseResponse(20004, + MessageCode.FILE_DOWNLOAD_ERROR.getCode(), + MessageCode.FILE_DOWNLOAD_ERROR.getDescription(), + errorData)); } String contentType = determineContentType(filePath); @@ -267,85 +308,30 @@ public class ApiController { return ResponseEntity.ok() .contentType(MediaType.parseMediaType(contentType)) .header(HttpHeaders.CONTENT_DISPOSITION, - "attachment; filename=\"" + fileInfo.getOriginalFileName() + "\"") - .header(HttpHeaders.CONTENT_LENGTH, String.valueOf(fileInfo.getFileSize())) + "attachment; filename=\"" + entityResponse.getOriginalFileName() + "\"") + .header(HttpHeaders.CONTENT_LENGTH, String.valueOf(entityResponse.getFileSize())) .body(resource); + } catch (FileEntityNotFoundException e) { + Map errorData = new HashMap<>(); + errorData.put("fileId", fileId); - } catch (NotFoundAuthToken e) { - return ResponseEntity.status(401).build(); - } catch (Exception e) { - log.error("Error downloading file", e); - return ResponseEntity.status(500).build(); + return ResponseEntity.ok().body(new BaseResponse(20004, + MessageCode.FILE_NOT_FOUND.getCode(), + MessageCode.FILE_NOT_FOUND.getDescription(), + errorData)); + } catch (NotFoundAuthToken | IOException e) { + Map errorData = new HashMap<>(); + errorData.put("token", tokenHeader); + errorData.put("fileId", fileId); + errorData.put("version", version); + + return ResponseEntity.ok().body(new BaseResponse(20004, + MessageCode.FILE_DOWNLOAD_ERROR_NOT_CORRECT_FIELD.getCode(), + MessageCode.FILE_DOWNLOAD_ERROR_NOT_CORRECT_FIELD.getDescription(), + errorData)); } } - @GetMapping("/v{version}/files/info/{fileId}") - public ResponseEntity getFileInfo( - @PathVariable String fileId, - @PathVariable Integer version, - @RequestHeader("Authorization") String tokenHeader) { - - try { - Long userId = getUserIdFromToken(tokenHeader); - FileEntityResponse fileInfo = fileEntityService.getById(fileId, version); - - if (!fileInfo.getUserId().equals(userId)) { - return ResponseEntity.status(403).build(); - } - - return ResponseEntity.ok(fileInfo); - - } catch (NotFoundAuthToken e) { - return ResponseEntity.status(401).build(); - } catch (Exception e) { - log.error("Error getting file info", e); - return ResponseEntity.status(500).build(); - } - } - - @GetMapping("/v{version}/files/preview/{fileId}") - public ResponseEntity previewFile( - @PathVariable String fileId, - @PathVariable Integer version, - @RequestHeader("Authorization") String tokenHeader) { - - try { - Long userId = getUserIdFromToken(tokenHeader); - FileEntityResponse fileInfo = fileEntityService.getById(fileId, version); - - if (!fileInfo.getUserId().equals(userId)) { - return ResponseEntity.status(403).build(); - } - - if (!isPreviewSupported(fileInfo.getMimeType())) { - return ResponseEntity.status(415) - .body(null); - } - - Path filePath = Paths.get(fileInfo.getFilePath()); - Resource resource = new UrlResource(filePath.toUri()); - - if (!resource.exists()) { - return ResponseEntity.status(404).build(); - } - - String contentType = determineContentType(filePath); - - return ResponseEntity.ok() - .contentType(MediaType.parseMediaType(contentType)) - .header(HttpHeaders.CONTENT_DISPOSITION, - "inline; filename=\"" + fileInfo.getOriginalFileName() + "\"") - .body(resource); - - } catch (NotFoundAuthToken e) { - return ResponseEntity.status(401).build(); - } catch (Exception e) { - log.error("Error previewing file", e); - return ResponseEntity.status(500).build(); - } - } - - private ResponseEntity createValidationErrorResponse(BindingResult bindingResult, Integer msgId) { List> fieldErrors = bindingResult.getFieldErrors() .stream() @@ -382,18 +368,6 @@ public class ApiController { return contentType; } - private boolean isPreviewSupported(String mimeType) { - if (mimeType == null) { - return false; - } - - return mimeType.startsWith("image/") || - mimeType.startsWith("text/") || - mimeType.equals("application/pdf") || - mimeType.startsWith("video/") || - mimeType.startsWith("audio/"); - } - private Long getUserIdFromToken(String tokenHeader) { String token = tokenHeader.replace("Bearer ", ""); AuthToken authToken = authTokenRepository.findByToken(token) diff --git a/src/main/java/ru/soune/nocopy/dto/MessageCode.java b/src/main/java/ru/soune/nocopy/dto/MessageCode.java index ee9d40b..5c75246 100644 --- a/src/main/java/ru/soune/nocopy/dto/MessageCode.java +++ b/src/main/java/ru/soune/nocopy/dto/MessageCode.java @@ -8,6 +8,8 @@ public enum MessageCode { INVALID_TOKEN(2, "Invalid token"), INVALID_ACTION(2, "Invalid action"), FILE_UPLOAD_ERROR(2, "File upload error"), + FILE_DOWNLOAD_ERROR(2, "File download error"), + FILE_DOWNLOAD_ERROR_NOT_CORRECT_FIELD(2, "Not correct field"), IMAGE_FOUND_ERROR(2, "Image found error"), INVALID_JSON_BODY(2, "Invalid fields in JSON object"), INCOMPLETE_UPLOAD(2, "Not load all chunks"), diff --git a/src/main/java/ru/soune/nocopy/dto/file/FileExtensionResponse.java b/src/main/java/ru/soune/nocopy/dto/file/FileExtensionResponse.java new file mode 100644 index 0000000..6377de9 --- /dev/null +++ b/src/main/java/ru/soune/nocopy/dto/file/FileExtensionResponse.java @@ -0,0 +1,20 @@ +package ru.soune.nocopy.dto.file; + +import com.fasterxml.jackson.annotation.JsonProperty; +import lombok.Builder; +import lombok.Data; + +import java.util.List; + +@Data +@Builder +public class FileExtensionResponse { + @JsonProperty("file_extension") + private List extension; + + @JsonProperty("count") + private Integer count; + + @JsonProperty("max_file_size") + private Long maxFileSize; +} diff --git a/src/main/java/ru/soune/nocopy/dto/file/FileInfoUserResponse.java b/src/main/java/ru/soune/nocopy/dto/file/FileInfoUserResponse.java new file mode 100644 index 0000000..8b2f932 --- /dev/null +++ b/src/main/java/ru/soune/nocopy/dto/file/FileInfoUserResponse.java @@ -0,0 +1,61 @@ +package ru.soune.nocopy.dto.file; + +import com.fasterxml.jackson.annotation.JsonProperty; +import lombok.AllArgsConstructor; +import lombok.Builder; +import lombok.Data; +import lombok.NoArgsConstructor; + +@Data +@Builder +@NoArgsConstructor +@AllArgsConstructor +public class FileInfoUserResponse { + @JsonProperty("all_files_size") + private Long allFileSize; + + @JsonProperty("all_files_quantity") + private Integer fileCount; + + @JsonProperty("all_files_check") + private Integer filesCheck; + + @JsonProperty("all_files_violation") + private Integer filesViolation; + + @JsonProperty("images_size") + private Long imagesSize; + + @JsonProperty("images_quantity") + private Integer imagesCount; + + @JsonProperty("images_check") + private Integer imagesCheck; + + @JsonProperty("images_violations") + private Integer imagesViolations; + + @JsonProperty("videos_size") + private Long videosSize; + + @JsonProperty("videos_quantity") + private Integer videosCount; + + @JsonProperty("videos_check") + private Integer videosCheck; + + @JsonProperty("videos_violations") + private Integer videosViolations; + + @JsonProperty("audios_size") + private Long audiosSize; + + @JsonProperty("audios_quantity") + private Integer audiosCount; + + @JsonProperty("audios_check") + private Integer audiosCheck; + + @JsonProperty("audios_violations") + private Integer audiosViolations; +} diff --git a/src/main/java/ru/soune/nocopy/entity/file/FileStatus.java b/src/main/java/ru/soune/nocopy/entity/file/FileStatus.java index 8e96247..b0b2351 100644 --- a/src/main/java/ru/soune/nocopy/entity/file/FileStatus.java +++ b/src/main/java/ru/soune/nocopy/entity/file/FileStatus.java @@ -4,5 +4,7 @@ public enum FileStatus { ACTIVE, DELETED, PROCESSING, + VIOLATION, + CHECKED, ERROR } diff --git a/src/main/java/ru/soune/nocopy/entity/file/FileType.java b/src/main/java/ru/soune/nocopy/entity/file/FileType.java index 233a7c2..8844013 100644 --- a/src/main/java/ru/soune/nocopy/entity/file/FileType.java +++ b/src/main/java/ru/soune/nocopy/entity/file/FileType.java @@ -2,21 +2,33 @@ package ru.soune.nocopy.entity.file; import lombok.Getter; +import java.util.Arrays; +import java.util.List; + @Getter public enum FileType { - PHOTO("photo"), - IMAGE("image"), - VIDEO("video"), - AUDIO("audio"), - DOCUMENT("document"); + IMAGE("image", Arrays.asList("jpg", "jpeg", "png", "gif", "bmp", "webp", "jfif")), + VIDEO("video", Arrays.asList("mp4", "avi", "mov", "wmv", "flv", "mkv", "webm", "m4v", "mpg", "mpeg", + "3gp", "3g2", "f4v", "m2ts", "mts", "vob", "ogv", "divx")), + AUDIO("audio", Arrays.asList("mp3", "wav", "flac")); - private final String code; + private final String displayName; + private final List allowedExtensions; - FileType(String code) { - this.code = code; + FileType(String displayName, List allowedExtensions) { + this.displayName = displayName; + this.allowedExtensions = allowedExtensions; } - public String getCode() { - return code; + public String getDisplayName() { + return displayName; + } + + public List getAllowedExtensions() { + return allowedExtensions; + } + + public boolean supportsExtension(String extension) { + return allowedExtensions.contains(extension.toLowerCase()); } } diff --git a/src/main/java/ru/soune/nocopy/entity/file/FileUploadSession.java b/src/main/java/ru/soune/nocopy/entity/file/FileUploadSession.java index 996f0de..509d0f7 100644 --- a/src/main/java/ru/soune/nocopy/entity/file/FileUploadSession.java +++ b/src/main/java/ru/soune/nocopy/entity/file/FileUploadSession.java @@ -86,7 +86,7 @@ public class FileUploadSession { @PrePersist public void prePersist() { this.createdAt = LocalDateTime.now(); - this.expiresAt = LocalDateTime.now().plusHours(24); + this.expiresAt = LocalDateTime.now().plusMinutes(1); if (this.chunksUploaded == null) { this.chunksUploaded = 0; } diff --git a/src/main/java/ru/soune/nocopy/exception/ChunkSizeExceededException.java b/src/main/java/ru/soune/nocopy/exception/ChunkSizeExceededException.java index a19cd95..f8a4933 100644 --- a/src/main/java/ru/soune/nocopy/exception/ChunkSizeExceededException.java +++ b/src/main/java/ru/soune/nocopy/exception/ChunkSizeExceededException.java @@ -2,6 +2,6 @@ package ru.soune.nocopy.exception; public class ChunkSizeExceededException extends RuntimeException { public ChunkSizeExceededException(long actualSize, long maxSize) { - super(String.format("Chunk size exceeded: %d > %d", actualSize, maxSize)); + super(String.format("Chunk size not valid: " + actualSize)); } } diff --git a/src/main/java/ru/soune/nocopy/handler/FileEntityHandler.java b/src/main/java/ru/soune/nocopy/handler/FileEntityHandler.java index bbc8da9..cacf0eb 100644 --- a/src/main/java/ru/soune/nocopy/handler/FileEntityHandler.java +++ b/src/main/java/ru/soune/nocopy/handler/FileEntityHandler.java @@ -7,11 +7,17 @@ import org.springframework.stereotype.Component; import ru.soune.nocopy.dto.*; import ru.soune.nocopy.dto.file.*; import ru.soune.nocopy.entity.AuthToken; +import ru.soune.nocopy.entity.file.FileEntity; +import ru.soune.nocopy.entity.file.FileStatus; +import ru.soune.nocopy.exception.FileEntityNotFoundException; import ru.soune.nocopy.exception.NotFoundAuthToken; import ru.soune.nocopy.repository.AuthTokenRepository; +import ru.soune.nocopy.repository.FileEntityRepository; import ru.soune.nocopy.service.file.FileEntityService; +import ru.soune.nocopy.service.file.FileStatsService; import java.util.Arrays; +import java.util.List; @Slf4j @Component @@ -19,9 +25,15 @@ import java.util.Arrays; public class FileEntityHandler implements RequestHandler { private final FileEntityService fileEntityService; + + private final FileStatsService fileStatsService; + private final AuthTokenRepository authTokenRepository; + private final ObjectMapper objectMapper; + private final FileEntityRepository fileEntityRepository; + @Override public BaseResponse handle(BaseRequest request) { try { @@ -31,15 +43,17 @@ public class FileEntityHandler implements RequestHandler { String action = fileRequest.getAction(); switch (action) { - case "get_file_info": + case "file_info": return handleGetFileInfo(request, fileRequest); - case "get_file_by_session": + case "user_files_info": + return handleGetFilesUserInfo(request, fileRequest); + case "file_by_session": return handleGetFileBySession(request, fileRequest); - case "get_user_files": + case "user_files": return handleGetUserFiles(request, fileRequest); case "search_files": return handleSearchFiles(request, fileRequest); - case "get_storage_usage": + case "storage_usage": return handleGetStorageUsage(request, fileRequest); case "delete_file": return handleDeleteFile(request, fileRequest); @@ -47,8 +61,8 @@ public class FileEntityHandler implements RequestHandler { ActionResponse response = ActionResponse.builder() .action(action) .availableActions(Arrays.asList( - "get_file_info", "get_file_by_session", "get_user_files", - "search_files", "get_storage_usage", "delete_file")) + "file_info", "file_by_session", "user_files", + "search_files", "storage_usage", "delete_file")) .build(); return new BaseResponse(request.getMsgId(), @@ -65,6 +79,16 @@ public class FileEntityHandler implements RequestHandler { } } + private BaseResponse handleGetFilesUserInfo(BaseRequest request, FileEntityRequest fileRequest) { + Long userId = getUserIdFromToken(fileRequest.getToken()); + FileInfoUserResponse userFileStats = fileStatsService.getUserFileStats(userId); + + return new BaseResponse(request.getMsgId(), + MessageCode.SUCCESS.getCode(), + MessageCode.SUCCESS.getDescription(), + userFileStats); + } + private BaseResponse handleGetFileInfo(BaseRequest request, FileEntityRequest fileRequest) { try { Long userId = getUserIdFromToken(fileRequest.getToken()); @@ -168,17 +192,22 @@ public class FileEntityHandler implements RequestHandler { FileResponse allFiles = fileEntityService.getUserFiles(userId, 1, 1000, request.getVersion()); var filteredFiles = allFiles.getFiles().stream() - .filter(f -> f.getOriginalFileName().toLowerCase().contains(fileRequest.getQuery().toLowerCase())) + .filter(f -> (f.getOriginalFileName() + f.getFileExtension()).toLowerCase() + .contains(fileRequest.getQuery().toLowerCase())) .toList(); int start = (page - 1) * pageSize; int end = Math.min(start + pageSize, filteredFiles.size()); + List fileEntityResponses = filteredFiles.subList(start, Math.min(end, filteredFiles.size())); + long sumSize = fileEntityResponses.stream() + .mapToLong(FileEntityResponse::getFileSize) + .sum(); FileListResponse response = FileListResponse.builder() - .files(filteredFiles.subList(start, Math.min(end, filteredFiles.size()))) + .files(fileEntityResponses) .totalCount(filteredFiles.size()) - .totalSize(allFiles.getTotalSize()) - .formattedTotalSize(allFiles.getFormattedTotalSize()) + .totalSize(sumSize) + .formattedTotalSize(fileEntityService.formatFileSize(sumSize)) .page(page) .pageSize(pageSize) .build(); @@ -187,18 +216,13 @@ public class FileEntityHandler implements RequestHandler { MessageCode.SUCCESS.getCode(), MessageCode.SUCCESS.getDescription(), response); - } catch (NotFoundAuthToken e) { - return new BaseResponse(request.getMsgId(), - MessageCode.INVALID_TOKEN.getCode(), - "Authentication required", - null); + return new BaseResponse(request.getMsgId(), MessageCode.INVALID_TOKEN.getCode(), + "Authentication required", null); } catch (Exception e) { log.error("Error searching files", e); - return new BaseResponse(request.getMsgId(), - MessageCode.FILE_UPLOAD_ERROR.getCode(), - "Failed to search files: " + e.getMessage(), - null); + return new BaseResponse(request.getMsgId(), MessageCode.FILE_UPLOAD_ERROR.getCode(), + "Failed to search files: " + e.getMessage(), null); } } @@ -235,7 +259,9 @@ public class FileEntityHandler implements RequestHandler { private BaseResponse handleDeleteFile(BaseRequest request, FileEntityRequest fileRequest) { try { Long userId = getUserIdFromToken(fileRequest.getToken()); - FileEntityResponse fileInfo = fileEntityService.getById(fileRequest.getFileId(), request.getVersion()); + String fileId = fileRequest.getFileId(); + FileEntityResponse fileInfo = fileEntityService.getById(fileId, request.getVersion()); + DeleteFileResponse response; if (!fileInfo.getUserId().equals(userId)) { return new BaseResponse(request.getMsgId(), @@ -244,18 +270,27 @@ public class FileEntityHandler implements RequestHandler { null); } - fileEntityService.markAsDeleted(fileRequest.getFileId()); + FileEntity fileEntity = fileEntityRepository.findById(fileId) + .orElseThrow(() -> new FileEntityNotFoundException(fileId)); - DeleteFileResponse response = DeleteFileResponse.builder() - .fileId(fileRequest.getFileId()) - .message("File marked as deleted") - .build(); + if (fileEntity.getStatus().equals(FileStatus.DELETED)) { + fileEntityService.deleteFromDisk(fileEntity); + response = DeleteFileResponse.builder() + .fileId(fileRequest.getFileId()) + .message("File deleted from disk") + .build(); + } else { + fileEntityService.markAsDeleted(fileEntity); + response = DeleteFileResponse.builder() + .fileId(fileRequest.getFileId()) + .message("File marked as deleted") + .build(); + } return new BaseResponse(request.getMsgId(), MessageCode.SUCCESS.getCode(), "File marked as deleted", response); - } catch (NotFoundAuthToken e) { return new BaseResponse(request.getMsgId(), MessageCode.INVALID_TOKEN.getCode(), diff --git a/src/main/java/ru/soune/nocopy/handler/FileUploadHandler.java b/src/main/java/ru/soune/nocopy/handler/FileUploadHandler.java index 26f6dba..4cb33b5 100644 --- a/src/main/java/ru/soune/nocopy/handler/FileUploadHandler.java +++ b/src/main/java/ru/soune/nocopy/handler/FileUploadHandler.java @@ -3,9 +3,11 @@ package ru.soune.nocopy.handler; import com.fasterxml.jackson.databind.ObjectMapper; import lombok.RequiredArgsConstructor; import lombok.extern.slf4j.Slf4j; +import org.springframework.beans.factory.annotation.Value; import org.springframework.stereotype.Component; import org.springframework.validation.BeanPropertyBindingResult; import org.springframework.validation.BindingResult; +import org.springframework.validation.FieldError; import ru.soune.nocopy.dto.*; import ru.soune.nocopy.dto.file.*; import ru.soune.nocopy.entity.AuthToken; @@ -17,21 +19,26 @@ import ru.soune.nocopy.repository.AuthTokenRepository; import ru.soune.nocopy.repository.FileUploadSessionRepository; import ru.soune.nocopy.service.file.FileUploadService; -import java.util.Arrays; -import java.util.HashMap; -import java.util.Map; -import java.util.Optional; +import java.util.*; +import java.util.stream.Collectors; @Slf4j @Component @RequiredArgsConstructor public class FileUploadHandler implements RequestHandler { private final FileUploadService fileUploadService; + private final AuthTokenRepository authTokenRepository; + private final FileUploadSessionRepository fileUploadSessionRepository; + private final ObjectMapper objectMapper; + private final FileUploadRequestValidator fileUploadRequestValidator; + @Value("${file.storage.max-file-size}") + private long maxFileSize; + @Override public BaseResponse handle(BaseRequest request) { try { @@ -45,18 +52,18 @@ public class FileUploadHandler implements RequestHandler { return handleInitUpload(request, fileRequest); case "file_types": return handleGetFileTypes(request); + case "file_extension": + return handleGetFileExtensions(fileRequest, request); case "chunks": return handleGetChunkStatus(request, fileRequest); case "cancel": return handleCancelUpload(request, fileRequest); case "progress": return handleGetProgress(request, fileRequest); - case "retry": - return handleRetryUpload(request, fileRequest); default: ActionResponse response = ActionResponse.builder() .action(action) - .availableActions(Arrays.asList("init", "file_types", "chunks", "cancel", "progress", "retry")) + .availableActions(Arrays.asList("init", "file_types", "chunks", "cancel", "progress")) .build(); return new BaseResponse(request.getMsgId(), MessageCode.INVALID_ACTION.getCode(), @@ -70,18 +77,31 @@ public class FileUploadHandler implements RequestHandler { } private BaseResponse handleInitUpload(BaseRequest request, FileUploadRequest fileRequest) { - try { String token = fileRequest.getToken(); Optional tokenOptional = authTokenRepository.findByToken(token); + if (tokenOptional.isEmpty()) { + return new BaseResponse(request.getMsgId(), MessageCode.INVALID_TOKEN.getCode(), + MessageCode.INVALID_TOKEN.getDescription(), Map.of("token", token)); + } + AuthToken authToken = tokenOptional.orElseThrow(() -> new NotFoundAuthToken("Token not found")); BindingResult bindingResult = new BeanPropertyBindingResult(fileRequest, "fileRequest"); fileUploadRequestValidator.validate(fileRequest, bindingResult); if (bindingResult.hasErrors()) { - throw new ValidationException(bindingResult, request.getMsgId()); + Map fieldErrors = bindingResult.getFieldErrors() + .stream() + .collect(Collectors.toMap( + FieldError::getField, + fieldError -> fieldError.getDefaultMessage() != null + ? fieldError.getDefaultMessage() + : "Validation error")); + + return new BaseResponse(request.getMsgId(), MessageCode.INVALID_FIELD.getCode(), + MessageCode.INVALID_FIELD.getDescription(), fieldErrors); } FileUploadSession session = fileUploadService.initUpload( @@ -101,15 +121,6 @@ public class FileUploadHandler implements RequestHandler { return new BaseResponse(request.getMsgId(), MessageCode.SUCCESS.getCode(), MessageCode.SUCCESS.getDescription(), fileResponse); - - } catch (NotFoundAuthToken e) { - InitFileResponse initFileResponse = InitFileResponse.builder() - .build(); - - throw new NotValidFieldException("Invalid or expired token: " + fileRequest.getToken(), - new BaseResponse(request.getMsgId(), MessageCode.INVALID_TOKEN.getCode(), - MessageCode.INVALID_TOKEN.getDescription(), initFileResponse)); - } } private BaseResponse handleGetProgress(BaseRequest request, FileUploadRequest fileRequest) { @@ -240,4 +251,24 @@ public class FileUploadHandler implements RequestHandler { "Failed to get file types", null); } } + + private BaseResponse handleGetFileExtensions(FileUploadRequest fileRequest, BaseRequest request) { + try { + FileType fileType = FileType.valueOf(fileRequest.getFileType().toUpperCase()); + List allowedExtensions = fileType.getAllowedExtensions(); + + FileExtensionResponse response = FileExtensionResponse.builder() + .extension(allowedExtensions) + .count(allowedExtensions.size()) + .maxFileSize(maxFileSize) + .build(); + + return new BaseResponse(request.getMsgId(), MessageCode.SUCCESS.getCode(), + MessageCode.SUCCESS.getDescription(), response); + } catch (Exception e) { + log.error("Error getting file extensions", e); + return new BaseResponse(request.getMsgId(), MessageCode.FILE_UPLOAD_ERROR.getCode(), + "Failed to get file extensions", null); + } + } } diff --git a/src/main/java/ru/soune/nocopy/handler/validator/FileUploadRequestValidator.java b/src/main/java/ru/soune/nocopy/handler/validator/FileUploadRequestValidator.java index f40487a..201dadf 100644 --- a/src/main/java/ru/soune/nocopy/handler/validator/FileUploadRequestValidator.java +++ b/src/main/java/ru/soune/nocopy/handler/validator/FileUploadRequestValidator.java @@ -7,9 +7,12 @@ import ru.soune.nocopy.dto.file.FileUploadRequest; import ru.soune.nocopy.entity.file.FileType; import java.util.Arrays; +import java.util.List; @Component public class FileUploadRequestValidator implements Validator { + private final List supportedFileTypes = Arrays.asList(FileType.values()); + @Override public boolean supports(Class clazz) { return FileUploadRequest.class.isAssignableFrom(clazz); @@ -21,7 +24,7 @@ public class FileUploadRequestValidator implements Validator { validateType(request.getFileType(), errors); validateFileName(request.getFileName(), errors); - validateExtension(request.getExtension(), errors); + validateExtension(request.getExtension(), request.getFileType(), errors); } private void validateType(String fileType, Errors errors) { @@ -31,7 +34,12 @@ public class FileUploadRequestValidator implements Validator { } try { - FileType.valueOf(fileType.toUpperCase()); + FileType parsedType = FileType.valueOf(fileType.toUpperCase()); + + if (!supportedFileTypes.contains(parsedType)) { + errors.rejectValue("fileType", "fileType.unsupported", + "Unsupported file type. Valid types: " + Arrays.toString(FileType.values())); + } } catch (IllegalArgumentException e) { errors.rejectValue("fileType", "fileType.invalid", "Invalid file type. Valid types: " + Arrays.toString(FileType.values())); @@ -50,10 +58,34 @@ public class FileUploadRequestValidator implements Validator { } } - private void validateExtension(String extension, Errors errors) { + private void validateExtension(String extension, String fileType, Errors errors) { if (extension == null || extension.isBlank()) { - errors.rejectValue("extension", "extension.required", "Extension is required"); + errors.rejectValue("fileType", "fileType.required", "File type is required"); return; } + + if (extension.contains(".")) { + errors.rejectValue("extension", "extension.required", "Extension contains comma"); + return; + } + + try { + FileType parsedType = FileType.valueOf(fileType.toUpperCase()); + + if (!parsedType.supportsExtension(extension)) { + errors.rejectValue("fileName", "fileType.extension.mismatch", + String.format("File extension '%s' does not match file type '%s'. Allowed extensions for %s: %s", + extension, parsedType.getDisplayName(), parsedType.getDisplayName(), + parsedType.getAllowedExtensions())); + } + + } catch (IllegalArgumentException e) { + errors.rejectValue("fileType", "fileType.invalid", + "Invalid file type. Valid types: " + Arrays.toString(FileType.values())); + } + } + + private String getFileExtension(String fileName) { + return fileName.substring(fileName.lastIndexOf('.') + 1).toLowerCase(); } } \ No newline at end of file diff --git a/src/main/java/ru/soune/nocopy/service/file/FileEntityService.java b/src/main/java/ru/soune/nocopy/service/file/FileEntityService.java index 8b239a5..617eeda 100644 --- a/src/main/java/ru/soune/nocopy/service/file/FileEntityService.java +++ b/src/main/java/ru/soune/nocopy/service/file/FileEntityService.java @@ -17,6 +17,7 @@ import java.nio.file.Files; import java.nio.file.Path; import java.nio.file.Paths; import java.time.LocalDateTime; +import java.time.ZoneOffset; import java.util.List; import java.util.stream.Collectors; @@ -40,7 +41,6 @@ public class FileEntityService { long fileSize = Files.size(filePath); String originalName = session.getFileName(); - String extension = extractFileExtension(originalName); String storedName = filePath.getFileName().toString(); FileEntity fileEntity = FileEntity.builder() @@ -50,7 +50,7 @@ public class FileEntityService { .filePath(session.getFilePath()) .fileSize(fileSize) .mimeType(session.getFileType()) - .fileExtension(extension) + .fileExtension(session.getExtension()) .checksum(checksum) .uploadSessionId(session.getUploadId()) .status(FileStatus.ACTIVE) @@ -168,15 +168,24 @@ public class FileEntityService { } @Transactional - public void markAsDeleted(String fileId) { - FileEntity fileEntity = fileEntityRepository.findById(fileId) - .orElseThrow(() -> new FileEntityNotFoundException(fileId)); - + public void markAsDeleted(FileEntity fileEntity) { fileEntity.setStatus(FileStatus.DELETED); fileEntity.setUpdatedAt(LocalDateTime.now()); - fileEntityRepository.save(fileEntity); - log.info("FileEntity marked as deleted: {}", fileId); + fileEntityRepository.save(fileEntity); + } + + @Transactional + public boolean deleteFromDisk(FileEntity fileEntity) throws IOException { + Path path = Paths.get(fileEntity.getFilePath()); + + if (!Files.exists(path)) { + return true; + } + + Files.delete(path); + + return true; } @Transactional(readOnly = true) @@ -217,7 +226,7 @@ public class FileEntityService { .build(); } - private String formatFileSize(long size) { + public String formatFileSize(long size) { if (size < 1024) { return size + " B"; } else if (size < 1024 * 1024) { diff --git a/src/main/java/ru/soune/nocopy/service/file/FileStatsService.java b/src/main/java/ru/soune/nocopy/service/file/FileStatsService.java new file mode 100644 index 0000000..a32f04d --- /dev/null +++ b/src/main/java/ru/soune/nocopy/service/file/FileStatsService.java @@ -0,0 +1,93 @@ +package ru.soune.nocopy.service.file; + +import lombok.RequiredArgsConstructor; +import org.springframework.stereotype.Service; +import ru.soune.nocopy.dto.file.FileInfoUserResponse; +import ru.soune.nocopy.entity.file.FileEntity; +import ru.soune.nocopy.entity.file.FileStatus; +import ru.soune.nocopy.entity.file.FileType; +import ru.soune.nocopy.repository.FileEntityRepository; + +import java.util.List; + +@Service +@RequiredArgsConstructor +public class FileStatsService { + private final FileEntityRepository fileEntityRepository; + + public FileInfoUserResponse getUserFileStats(Long userId) { + List userFiles = fileEntityRepository.findByUserId(userId); + return calculateStats(userFiles); + } + + public FileInfoUserResponse calculateStats(List files) { + return FileInfoUserResponse.builder() + .allFileSize(calculateTotalSize(files)) + .fileCount(calculateTotalCount(files)) + .filesCheck(calculateByStatus(files, FileStatus.CHECKED)) + .filesViolation(calculateByStatus(files, FileStatus.VIOLATION)) + + .imagesSize(calculateMediaSize(files, FileType.IMAGE)) + .imagesCount(calculateMediaCount(files, FileType.IMAGE)) + .imagesCheck(calculateMediaByStatus(files, FileType.IMAGE, FileStatus.CHECKED)) + .imagesViolations(calculateMediaByStatus(files, FileType.IMAGE, FileStatus.VIOLATION)) + + .videosSize(calculateMediaSize(files, FileType.VIDEO)) + .videosCount(calculateMediaCount(files, FileType.VIDEO)) + .videosCheck(calculateMediaByStatus(files, FileType.VIDEO, FileStatus.CHECKED)) + .videosViolations(calculateMediaByStatus(files, FileType.VIDEO, FileStatus.VIOLATION)) + + .audiosSize(calculateMediaSize(files, FileType.AUDIO)) + .audiosCount(calculateMediaCount(files, FileType.AUDIO)) + .audiosCheck(calculateMediaByStatus(files, FileType.AUDIO, FileStatus.CHECKED)) + .audiosViolations(calculateMediaByStatus(files, FileType.AUDIO, FileStatus.VIOLATION)) + .build(); + } + private Long calculateTotalSize(List files) { + return files.stream() + .filter(file -> file.getStatus() != FileStatus.DELETED) + .mapToLong(FileEntity::getFileSize) + .sum(); + } + + private Integer calculateTotalCount(List files) { + return (int) files.stream() + .filter(file -> file.getStatus() != FileStatus.DELETED) + .count(); + } + + private Integer calculateByStatus(List files, FileStatus status) { + return (int) files.stream() + .filter(file -> file.getStatus() == status) + .count(); + } + + private Long calculateMediaSize(List files, FileType fileType) { + return files.stream() + .filter(file -> file.getStatus() != FileStatus.DELETED) + .filter(file -> isFileType(file, fileType)) + .mapToLong(FileEntity::getFileSize) + .sum(); + } + + private Integer calculateMediaCount(List files, FileType fileType) { + return (int) files.stream() + .filter(file -> file.getStatus() != FileStatus.DELETED) + .filter(file -> isFileType(file, fileType)) + .count(); + } + + private Integer calculateMediaByStatus(List files, FileType fileType, FileStatus status) { + return (int) files.stream() + .filter(file -> isFileType(file, fileType)) + .filter(file -> file.getStatus() == status) + .count(); + } + + private boolean isFileType(FileEntity file, FileType fileType) { + if (file.getFileExtension() == null) return false; + + String extension = file.getFileExtension().toLowerCase().replace(".", ""); + return fileType.supportsExtension(extension); + } +} diff --git a/src/main/java/ru/soune/nocopy/service/file/FileUploadService.java b/src/main/java/ru/soune/nocopy/service/file/FileUploadService.java index cfb9aec..b27dede 100644 --- a/src/main/java/ru/soune/nocopy/service/file/FileUploadService.java +++ b/src/main/java/ru/soune/nocopy/service/file/FileUploadService.java @@ -15,7 +15,7 @@ public interface FileUploadService { UploadProgressResponse getUploadProgress(String uploadId); - void cleanupExpiredSessions(); + void handleExpiredSession(FileUploadSession session); public void retryFailedUpload(String uploadId); diff --git a/src/main/java/ru/soune/nocopy/service/file/FileUploadServiceImpl.java b/src/main/java/ru/soune/nocopy/service/file/FileUploadServiceImpl.java index e8fb956..3e09822 100644 --- a/src/main/java/ru/soune/nocopy/service/file/FileUploadServiceImpl.java +++ b/src/main/java/ru/soune/nocopy/service/file/FileUploadServiceImpl.java @@ -102,12 +102,14 @@ public class FileUploadServiceImpl implements FileUploadService { int totalChunks = (int) Math.ceil((double) fileSize / chunkSize); log.debug("File will be split into {} chunks (chunk size: {} bytes)", totalChunks, chunkSize); + Long chunkSize = totalChunks == 1 ? fileSize : 1000000L; FileUploadSession session = FileUploadSession.builder() .userId(userId) .extension(extension) .fileName(fileName) .fileType(fileType) + .chunkSize(chunkSize) .fileSize(fileSize) .totalChunks(totalChunks) .chunksUploaded(0) @@ -143,17 +145,103 @@ public class FileUploadServiceImpl implements FileUploadService { if (chunkNumber < 0 || chunkNumber >= session.getTotalChunks()) { throw new FileUploadException( - String.format("Invalid chunk number %d. Expected 0-%d", + String.format("Invalid chunk number %d. Expected number: %d", chunkNumber, session.getTotalChunks() - 1)); } - if (chunkFile.getSize() > chunkSize) { + if (chunkNumber == 0 && chunkFile.getSize() > chunkSize || chunkNumber + 1 == (session.getTotalChunks()) + && chunkFile.getSize() > chunkSize) { + throw new ChunkSizeExceededException(chunkFile.getSize(), chunkSize); + } + + if (chunkNumber > 0 && chunkNumber + 1 < session.getTotalChunks() && chunkFile.getSize() != chunkSize) { throw new ChunkSizeExceededException(chunkFile.getSize(), chunkSize); } return processChunk(session, chunkNumber, chunkFile); } + @Override + @Transactional + public void handleExpiredSession(FileUploadSession session) { + session.setStatus(UploadStatus.FAILED); + session.setLastError("Upload session expired"); + sessionRepository.save(session); + + CompletableFuture.runAsync(() -> cleanupSessionFiles(session)); + } + + @Async("fileUploadTaskExecutor") + @Transactional + public void assembleFileAsync(FileUploadSession session) { + try { + assembleFile(session); + log.info("File assembly completed successfully for session: {}", + session.getUploadId()); + + } catch (Exception e) { + log.error("Failed to assemble file for session {}: {}", + session.getUploadId(), e.getMessage(), e); + + handleAssemblyFailure(session, e); + } + } + + @Override + @Transactional + public void cancelUpload(String uploadId) { + FileUploadSession session = sessionRepository.findById(uploadId) + .orElseThrow(() -> new UploadSessionNotFoundException(uploadId)); + + if (session.getStatus() == UploadStatus.COMPLETED || session.getStatus() == UploadStatus.CANCELLED) { + throw new FileUploadException("Cannot cancel completed or cancelled upload"); + } + + session.setStatus(UploadStatus.CANCELLED); + sessionRepository.save(session); + + CompletableFuture.runAsync(() -> cleanupSessionFiles(session)); + + log.info("Upload cancelled: {}", uploadId); + } + + @Override + @Transactional + public void retryFailedUpload(String uploadId) { + FileUploadSession session = sessionRepository.findById(uploadId) + .orElseThrow(() -> new UploadSessionNotFoundException(uploadId)); + + if (session.getStatus() != UploadStatus.FAILED) { + throw new FileUploadException("Only failed uploads can be retried"); + } + + if (session.getRetryCount() >= maxRetryAttempts) { + throw new FileUploadException("Max retry attempts exceeded"); + } + + session.setStatus(UploadStatus.UPLOADING); + session.setRetryCount(session.getRetryCount() + 1); + session.setLastError(null); + + sessionRepository.save(session); + + if (session.getChunksUploaded().equals(session.getTotalChunks())) { + log.info("Retrying file assembly for session: {}", uploadId); + assembleFileAsync(session); + } + + log.info("Upload retry initiated for session: {} (attempt {})", + uploadId, session.getRetryCount()); + } + + @Override + public UploadProgressResponse getUploadProgress(String uploadId) { + FileUploadSession session = sessionRepository.findById(uploadId) + .orElseThrow(() -> new UploadSessionNotFoundException(uploadId)); + + return UploadProgressResponse.fromSession(session); + } + private void validateSession(FileUploadSession session) { UploadStatus status = session.getStatus(); @@ -183,14 +271,6 @@ public class FileUploadServiceImpl implements FileUploadService { } } - private void handleExpiredSession(FileUploadSession session) { - session.setStatus(UploadStatus.FAILED); - session.setLastError("Upload session expired"); - sessionRepository.save(session); - - CompletableFuture.runAsync(() -> cleanupSessionFiles(session)); - } - private UploadProgressResponse processChunk(FileUploadSession session, Integer chunkNumber, MultipartFile chunkFile) { @@ -206,6 +286,7 @@ public class FileUploadServiceImpl implements FileUploadService { session.getChunkPaths().put(chunkNumber, chunkPath); session.setChunksUploaded(session.getChunksUploaded() + 1); session.setStatus(UploadStatus.UPLOADING); + session.setExpiresAt(LocalDateTime.now().plusMinutes(1)); log.debug("Chunk {} saved successfully. Uploaded: {}/{}", chunkNumber, session.getChunksUploaded(), session.getTotalChunks()); @@ -316,52 +397,6 @@ public class FileUploadServiceImpl implements FileUploadService { } } - @Async("fileUploadTaskExecutor") - @Transactional - public void assembleFileAsync(FileUploadSession session) { - try { - assembleFile(session); - log.info("File assembly completed successfully for session: {}", - session.getUploadId()); - - } catch (Exception e) { - log.error("Failed to assemble file for session {}: {}", - session.getUploadId(), e.getMessage(), e); - - handleAssemblyFailure(session, e); - } - } - -// private void assembleFile(FileUploadSession session) throws IOException { -// log.info("Starting file assembly for session: {} ({})", -// session.getUploadId(), session.getFileName()); -// -// Path finalFilePath = null; -// -// try { -// finalFilePath = prepareFinalFile(session); -// -// validateAllChunksExist(session); -// -// mergeChunksToFile(session, finalFilePath); -// -// validateFinalFile(session, finalFilePath); -// -// updateSessionOnSuccess(session, finalFilePath); -// -// cleanupSessionFiles(session); -// -// log.info("File assembly completed: {} -> {} ({} bytes)", -// session.getFileName(), finalFilePath, session.getFileSize()); -// -// } catch (Exception e) { -// if (finalFilePath != null) { -// Files.deleteIfExists(finalFilePath); -// } -// throw e; -// } -// } - private void assembleFile(FileUploadSession session) throws IOException { log.info("Starting file assembly for session: {} ({})", session.getUploadId(), session.getFileName()); @@ -526,14 +561,6 @@ public class FileUploadServiceImpl implements FileUploadService { log.debug("Final file checksum: {}", checksum); } - private void updateSessionOnSuccess(FileUploadSession session, Path finalPath) { - session.setFilePath(finalPath.toString()); - session.setChecksum(calculateChecksum(finalPath)); - session.setStatus(UploadStatus.COMPLETED); - session.setCompletedAt(LocalDateTime.now()); - sessionRepository.save(session); - } - private void cleanupSessionFiles(FileUploadSession session) { try { Path chunkDir = getChunkDirectory(session.getUserId(), session.getUploadId()); @@ -570,90 +597,37 @@ public class FileUploadServiceImpl implements FileUploadService { session.getUploadId(), e.getMessage()); } - @Override - @Transactional - public void cancelUpload(String uploadId) { - FileUploadSession session = sessionRepository.findById(uploadId) - .orElseThrow(() -> new UploadSessionNotFoundException(uploadId)); - if (session.getStatus() == UploadStatus.COMPLETED || session.getStatus() == UploadStatus.CANCELLED) { - throw new FileUploadException("Cannot cancel completed or cancelled upload"); - } - session.setStatus(UploadStatus.CANCELLED); - sessionRepository.save(session); - - CompletableFuture.runAsync(() -> cleanupSessionFiles(session)); - - log.info("Upload cancelled: {}", uploadId); - } - - @Override - @Transactional - public void retryFailedUpload(String uploadId) { - FileUploadSession session = sessionRepository.findById(uploadId) - .orElseThrow(() -> new UploadSessionNotFoundException(uploadId)); - - if (session.getStatus() != UploadStatus.FAILED) { - throw new FileUploadException("Only failed uploads can be retried"); - } - - if (session.getRetryCount() >= maxRetryAttempts) { - throw new FileUploadException("Max retry attempts exceeded"); - } - - session.setStatus(UploadStatus.UPLOADING); - session.setRetryCount(session.getRetryCount() + 1); - session.setLastError(null); - - sessionRepository.save(session); - - if (session.getChunksUploaded().equals(session.getTotalChunks())) { - log.info("Retrying file assembly for session: {}", uploadId); - assembleFileAsync(session); - } - - log.info("Upload retry initiated for session: {} (attempt {})", - uploadId, session.getRetryCount()); - } - - @Override - public UploadProgressResponse getUploadProgress(String uploadId) { - FileUploadSession session = sessionRepository.findById(uploadId) - .orElseThrow(() -> new UploadSessionNotFoundException(uploadId)); - - return UploadProgressResponse.fromSession(session); - } - - @Override - @Transactional - public void cleanupExpiredSessions() { - log.info("Starting cleanup of expired upload sessions"); - - LocalDateTime expiryThreshold = LocalDateTime.now().minusHours(sessionExpiryHours); - - try { - var expiredSessions = sessionRepository.findByStatusInAndCreatedAtBefore( - Set.of(UploadStatus.INITIATED, UploadStatus.UPLOADING, UploadStatus.FAILED), - expiryThreshold - ); - - int cleanedCount = 0; - for (FileUploadSession session : expiredSessions) { - session.setStatus(UploadStatus.FAILED); - session.setLastError("Session expired during cleanup"); - sessionRepository.save(session); - - CompletableFuture.runAsync(() -> cleanupSessionFiles(session)); - cleanedCount++; - } - - log.info("Cleaned up {} expired upload sessions", cleanedCount); - - } catch (Exception e) { - log.error("Error during session cleanup", e); - } - } +// @Override +// @Transactional +// public void cleanupExpiredSessions() { +// log.info("Starting cleanup of expired upload sessions"); +// +// LocalDateTime expiryThreshold = LocalDateTime.now().minusHours(sessionExpiryHours); +// +// try { +// var expiredSessions = sessionRepository.findByStatusInAndCreatedAtBefore( +// Set.of(UploadStatus.INITIATED, UploadStatus.UPLOADING, UploadStatus.FAILED), +// expiryThreshold +// ); +// +// int cleanedCount = 0; +// for (FileUploadSession session : expiredSessions) { +// session.setStatus(UploadStatus.FAILED); +// session.setLastError("Session expired during cleanup"); +// sessionRepository.save(session); +// +// CompletableFuture.runAsync(() -> cleanupSessionFiles(session)); +// cleanedCount++; +// } +// +// log.info("Cleaned up {} expired upload sessions", cleanedCount); +// +// } catch (Exception e) { +// log.error("Error during session cleanup", e); +// } +// } private Path getChunkDirectory(Long userId, String uploadId) { return storageRoot.resolve("temp") diff --git a/src/main/java/ru/soune/nocopy/service/file/UploadSessionCleanupService.java b/src/main/java/ru/soune/nocopy/service/file/UploadSessionCleanupService.java new file mode 100644 index 0000000..2c68604 --- /dev/null +++ b/src/main/java/ru/soune/nocopy/service/file/UploadSessionCleanupService.java @@ -0,0 +1,35 @@ +package ru.soune.nocopy.service.file; + +import lombok.RequiredArgsConstructor; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.scheduling.annotation.Scheduled; +import org.springframework.stereotype.Service; +import org.springframework.transaction.annotation.Transactional; +import ru.soune.nocopy.entity.file.FileUploadSession; +import ru.soune.nocopy.entity.file.UploadStatus; +import ru.soune.nocopy.repository.FileUploadSessionRepository; + +import java.time.LocalDateTime; +import java.util.List; +import java.util.Set; + +@Service +@RequiredArgsConstructor +public class UploadSessionCleanupService { + @Autowired + private FileUploadSessionRepository fileUploadSessionRepository; + + @Autowired + private FileUploadService fileUploadService; + + @Transactional + @Scheduled(fixedDelay = 30000) + public void cleanupSessions() { + List expiredSessions = fileUploadSessionRepository.findExpiredSessions( + LocalDateTime.now(), Set.of(UploadStatus.INITIATED, UploadStatus.UPLOADING)); + + for (FileUploadSession session : expiredSessions) { + fileUploadService.handleExpiredSession(session); + } + } +} diff --git a/src/main/resources/application.yaml b/src/main/resources/application.yaml index 45fcaaa..b58b4bc 100644 --- a/src/main/resources/application.yaml +++ b/src/main/resources/application.yaml @@ -24,7 +24,9 @@ spring: file: storage: base-path: ${FILE_STORAGE_PATH:/data/uploads} - chunk-size: ${FILE_CHUNK_SIZE:5242880} # 5MB + # chunk-size: ${FILE_CHUNK_SIZE:5242880} # 5MB +# chunk-size: ${FILE_CHUNK_SIZE:1048576} # 1MB + chunk-size: ${FILE_CHUNK_SIZE:1000000} # 1MB max-file-size: ${MAX_FILE_SIZE:10737418240} # 10GB max-retry-attempts: ${MAX_RETRY_ATTEMPTS:3} chunk-timeout-ms: ${CHUNK_TIMEOUT_MS:300000} # 5 минут