diff --git a/docker-compose.yaml b/docker-compose.yaml index f64115c..1ce8455 100644 --- a/docker-compose.yaml +++ b/docker-compose.yaml @@ -58,7 +58,8 @@ services: environment: FILE_STORAGE_PATH: /data/uploads MAX_FILE_SIZE: 10737418240 - FILE_CHUNK_SIZE: 1048576 +# FILE_CHUNK_SIZE: 1048576 + FILE_CHUNK_SIZE: 1000000 POSTGRES_DB: no_copy_ POSTGRES_USER: postgres POSTGRES_PASSWORD: postgres @@ -120,7 +121,7 @@ services: prometheus: image: prom/prometheus:latest container_name: prometheus - deploy: # ← ДОБАВЛЕНО + deploy: resources: limits: cpus: '0.5' @@ -148,7 +149,7 @@ services: loki: image: grafana/loki:2.9.2 container_name: loki - deploy: # ← ДОБАВЛЕНО + deploy: resources: limits: cpus: '1.0' @@ -173,7 +174,7 @@ services: tempo: image: grafana/tempo:2.4.1 container_name: tempo - deploy: # ← ДОБАВЛЕНО + deploy: resources: limits: cpus: '0.5' diff --git a/infrastructure/jenkins/deploy.groovy b/infrastructure/jenkins/deploy.groovy index a05e8ca..7611efe 100644 --- a/infrastructure/jenkins/deploy.groovy +++ b/infrastructure/jenkins/deploy.groovy @@ -7,7 +7,6 @@ pipeline { defaultValue: 'dev', description: 'Ветка для деплоя' ) - string( name: 'SERVER', defaultValue: '92.242.61.23', @@ -51,156 +50,57 @@ pipeline { sshpass -p '$SSH_PASS' ssh -o StrictHostKeyChecking=no $SSH_USER@$SERVER " cd /opt/deployments/${params.BRANCH} - echo '========================================' - echo 'Step 1: Checking current state...' - echo '========================================' + echo '1. Остановка старого приложения...' + docker stop app-backend 2>/dev/null || echo 'Контейнер не найден' + docker rm app-backend 2>/dev/null || echo 'Контейнер не найден' - docker ps --filter 'name=postgres|app-backend|storage|grafana|prometheus|loki|tempo|alloy' --format 'table {{.Names}}\\t{{.Image}}\\t{{.Status}}' + echo '2. Удаление старых образов...' + docker images --filter 'reference=*app*' -q | xargs -r docker rmi -f 2>/dev/null || echo 'Нет образов для удаления' - echo '========================================' - echo 'Step 2: Setup environment for compatibility...' - echo '========================================' + echo '3. Создание сети если нужно...' + docker network create app-network 2>/dev/null || echo 'Сеть уже существует' - docker network create app-network 2>/dev/null || echo 'Network app-network already exists' + echo '4. Запуск инфраструктуры...' + docker-compose up -d db storage - echo 'COMPOSE_COMPATIBILITY=true' > .env + echo '5. Ожидание PostgreSQL...' + sleep 10 - echo '========================================' - echo 'Step 3: Force cleanup old application...' - echo '========================================' + echo '6. Сборка нового образа приложения...' + docker build --no-cache -t app-backend:latest . - docker-compose --compatibility down 2>/dev/null || echo 'No previous compose stack' + echo '7. Запуск приложения...' + docker run -d \\ + --name app-backend \\ + --network app-network \\ + --network-alias app \\ + -p 80:8080 \\ + -v uploads_data:/data/uploads:rw \\ + -e POSTGRES_DB=no_copy_ \\ + -e POSTGRES_USER=postgres \\ + -e POSTGRES_PASSWORD=postgres \\ + -e POSTGRES_PORT=5432 \\ + -e POSTGRES_HOST=db \\ + --restart unless-stopped \\ + app-backend:latest - docker images --filter 'reference=*app-backend*' -q | xargs -r docker rmi -f 2>/dev/null || echo 'No images to remove' + echo '8. Запуск мониторинга...' + docker-compose up -d grafana prometheus loki tempo alloy - echo '========================================' - echo 'Step 4: Verify files and fix compose...' - echo '========================================' + echo '9. Проверка...' + sleep 5 - if [ ! -f docker-compose.yaml ]; then - echo 'ERROR: docker-compose.yaml not found!' - ls -la - exit 1 - fi + echo 'Статус контейнеров:' + docker ps --format 'table {{.Names}}\\t{{.Image}}\\t{{.Status}}' - echo 'Checking docker-compose.yaml syntax...' - docker-compose --compatibility config 2>&1 | head -20 || echo 'Config check output' - - echo '========================================' - echo 'Step 5: Start infrastructure services...' - echo '========================================' - - echo 'Starting PostgreSQL with resource limits...' - docker-compose --compatibility up -d db - - echo 'Starting storage service...' - docker-compose --compatibility up -d storage - - echo 'Waiting for PostgreSQL to be ready...' - for i in {1..30}; do - if docker-compose --compatibility exec db pg_isready -U postgres 2>/dev/null; then - echo 'PostgreSQL is ready' - break - fi - echo 'Waiting... attempt ' \$i - sleep 2 - done - - echo '========================================' - echo 'Step 6: Build application image...' - echo '========================================' - - if [ ! -f Dockerfile ]; then - echo 'ERROR: Dockerfile not found!' - exit 1 - fi - - echo 'Building Docker image...' - docker build --no-cache -t app-backend:latest . 2>&1 | tail -30 - - echo 'Image created successfully' - docker images app-backend:latest --format 'table {{.Repository}}\\t{{.Tag}}\\t{{.Size}}\\t{{.CreatedAt}}' - - echo '========================================' - echo 'Step 7: Start application with resource limits...' - echo '========================================' - - docker volume create uploads_data 2>/dev/null || echo 'Volume uploads_data already exists' - - echo 'Starting main application...' - docker-compose --compatibility up -d app - - echo 'Waiting for application to start...' - sleep 15 - - echo '========================================' - echo 'Step 8: Start monitoring stack...' - echo '========================================' - - docker-compose --compatibility up -d grafana prometheus loki tempo alloy - - echo '========================================' - echo 'Step 9: Verify deployment...' - echo '========================================' - - echo 'All containers status:' - docker-compose --compatibility ps - - echo '' - echo 'Resource limits check:' - echo 'App backend limits:' - docker inspect app-backend --format='{{.HostConfig.Memory}} {{.HostConfig.CpuQuota}} {{.HostConfig.CpuPeriod}}' 2>/dev/null || echo 'Cannot inspect' - - echo '' - echo 'PostgreSQL limits:' - docker inspect postgres --format='{{.HostConfig.Memory}} {{.HostConfig.CpuQuota}} {{.HostConfig.CpuPeriod}}' 2>/dev/null || echo 'Cannot inspect' - - echo '' - echo 'Container resource usage:' - docker stats --no-stream --format 'table {{.Name}}\\t{{.CPUPerc}}\\t{{.MemUsage}}\\t{{.MemPerc}}' postgres app-backend storage 2>/dev/null || echo 'Stats not available' - - echo '========================================' - echo 'Step 10: Health checks...' - echo '========================================' - - echo 'Application health check...' - for i in {1..10}; do - if curl -s -f http://localhost:80/health 2>/dev/null; then - echo '✓ Application health check PASSED' - break - elif [ \$i -eq 10 ]; then - echo '✗ Application health check FAILED' - echo 'Last logs:' - docker-compose --compatibility logs --tail=20 app - else - echo 'Waiting for application... attempt ' \$i - sleep 5 - fi - done - - echo 'Checking monitoring services...' - if curl -s -f http://localhost:3000/api/health 2>/dev/null; then - echo '✓ Grafana is healthy' + echo '10. Проверка health...' + if curl -s -f http://localhost:80/health > /dev/null 2>&1; then + echo 'Приложение работает' + echo 'URL: http://${params.SERVER}:80' else - echo '⚠ Grafana may be starting' + echo 'Проверка health не удалась' + docker logs app-backend --tail=20 fi - - if curl -s -f http://localhost:9090/-/healthy 2>/dev/null; then - echo '✓ Prometheus is healthy' - fi - - echo '========================================' - echo 'Deployment Summary:' - echo '========================================' - echo 'Application URL: http://${params.SERVER}:80' - echo 'Grafana: http://${params.SERVER}:3000 (admin/admin)' - echo 'Prometheus: http://${params.SERVER}:9090' - echo 'Loki: http://${params.SERVER}:3100' - echo '' - echo 'To view logs: docker-compose logs -f app' - echo 'To view all containers: docker-compose ps' - echo 'To stop: docker-compose down' - echo '========================================' " """ } @@ -213,8 +113,6 @@ pipeline { success { echo "Deployment successful" echo "Application URL: http://${params.SERVER}:80" - echo "Grafana: http://${params.SERVER}:3000" - echo "Prometheus: http://${params.SERVER}:9090" } failure { echo "Deployment failed for branch ${params.BRANCH}" diff --git a/src/main/java/ru/soune/nocopy/dto/file/FileInfoUserResponse.java b/src/main/java/ru/soune/nocopy/dto/file/FileInfoUserResponse.java new file mode 100644 index 0000000..8b2f932 --- /dev/null +++ b/src/main/java/ru/soune/nocopy/dto/file/FileInfoUserResponse.java @@ -0,0 +1,61 @@ +package ru.soune.nocopy.dto.file; + +import com.fasterxml.jackson.annotation.JsonProperty; +import lombok.AllArgsConstructor; +import lombok.Builder; +import lombok.Data; +import lombok.NoArgsConstructor; + +@Data +@Builder +@NoArgsConstructor +@AllArgsConstructor +public class FileInfoUserResponse { + @JsonProperty("all_files_size") + private Long allFileSize; + + @JsonProperty("all_files_quantity") + private Integer fileCount; + + @JsonProperty("all_files_check") + private Integer filesCheck; + + @JsonProperty("all_files_violation") + private Integer filesViolation; + + @JsonProperty("images_size") + private Long imagesSize; + + @JsonProperty("images_quantity") + private Integer imagesCount; + + @JsonProperty("images_check") + private Integer imagesCheck; + + @JsonProperty("images_violations") + private Integer imagesViolations; + + @JsonProperty("videos_size") + private Long videosSize; + + @JsonProperty("videos_quantity") + private Integer videosCount; + + @JsonProperty("videos_check") + private Integer videosCheck; + + @JsonProperty("videos_violations") + private Integer videosViolations; + + @JsonProperty("audios_size") + private Long audiosSize; + + @JsonProperty("audios_quantity") + private Integer audiosCount; + + @JsonProperty("audios_check") + private Integer audiosCheck; + + @JsonProperty("audios_violations") + private Integer audiosViolations; +} diff --git a/src/main/java/ru/soune/nocopy/entity/file/FileStatus.java b/src/main/java/ru/soune/nocopy/entity/file/FileStatus.java index 8e96247..b0b2351 100644 --- a/src/main/java/ru/soune/nocopy/entity/file/FileStatus.java +++ b/src/main/java/ru/soune/nocopy/entity/file/FileStatus.java @@ -4,5 +4,7 @@ public enum FileStatus { ACTIVE, DELETED, PROCESSING, + VIOLATION, + CHECKED, ERROR } diff --git a/src/main/java/ru/soune/nocopy/entity/file/FileType.java b/src/main/java/ru/soune/nocopy/entity/file/FileType.java index a7071ef..8844013 100644 --- a/src/main/java/ru/soune/nocopy/entity/file/FileType.java +++ b/src/main/java/ru/soune/nocopy/entity/file/FileType.java @@ -7,21 +7,10 @@ import java.util.List; @Getter public enum FileType { - IMAGE("image", Arrays.asList("jpg", "jpeg", "png", "gif", "bmp", "webp")), + IMAGE("image", Arrays.asList("jpg", "jpeg", "png", "gif", "bmp", "webp", "jfif")), VIDEO("video", Arrays.asList("mp4", "avi", "mov", "wmv", "flv", "mkv", "webm", "m4v", "mpg", "mpeg", "3gp", "3g2", "f4v", "m2ts", "mts", "vob", "ogv", "divx")), - AUDIO("audio", Arrays.asList("mp3", "wav", "flac")), - DOCUMENT("document", Arrays.asList( - "pdf", "txt", "rtf", - "doc", "docx", "xls", "xlsx", "ppt", "pptx", "pps", "ppsx", "dot", "dotx", "xlt", "xltx", "pot", "potx", - "odt", "ods", "odp", "odg", "odf", "odb", "odc", "odi", "odm", "ott", "ots", "otp", "otg", "oth", - "sxw", "sxc", "sxi", "sxd", "sxg", "stc", "sti", "stw", "sxm", - "pages", "numbers", "key", - "csv", "tsv", "xml", "html", "htm", "tex", "md", "markdown", - "epub", "mobi", "azw", "azw3", "fb2", - "wps", "wpt", "et", "dps", "vsd", "vsdx", - "java", "py", "cpp", "c", "h", "js", "css", "php", "sql", "json", "yaml", "yml", "sh", "bat", - "one", "note")); + AUDIO("audio", Arrays.asList("mp3", "wav", "flac")); private final String displayName; private final List allowedExtensions; diff --git a/src/main/java/ru/soune/nocopy/handler/FileEntityHandler.java b/src/main/java/ru/soune/nocopy/handler/FileEntityHandler.java index 526ff9f..040b037 100644 --- a/src/main/java/ru/soune/nocopy/handler/FileEntityHandler.java +++ b/src/main/java/ru/soune/nocopy/handler/FileEntityHandler.java @@ -14,6 +14,7 @@ import ru.soune.nocopy.repository.AuthTokenRepository; import ru.soune.nocopy.repository.FileEntityRepository; import ru.soune.nocopy.service.auth.AuthService; import ru.soune.nocopy.service.file.FileEntityService; +import ru.soune.nocopy.service.file.FileStatsService; import java.util.Arrays; import java.util.List; @@ -22,10 +23,13 @@ import java.util.List; @Component @RequiredArgsConstructor public class FileEntityHandler implements RequestHandler { + private final FileEntityService fileEntityService; private final AuthService authService; + private final FileStatsService fileStatsService; + private final AuthTokenRepository authTokenRepository; private final ObjectMapper objectMapper; @@ -43,6 +47,8 @@ public class FileEntityHandler implements RequestHandler { switch (action) { case "file_info": return handleGetFileInfo(request, fileRequest); + case "user_files_info": + return handleGetFilesUserInfo(request, fileRequest); case "file_by_session": return handleGetFileBySession(request, fileRequest); case "user_files": @@ -75,6 +81,16 @@ public class FileEntityHandler implements RequestHandler { } } + private BaseResponse handleGetFilesUserInfo(BaseRequest request, FileEntityRequest fileRequest) { + Long userId = getUserIdFromToken(fileRequest.getToken()); + FileInfoUserResponse userFileStats = fileStatsService.getUserFileStats(userId); + + return new BaseResponse(request.getMsgId(), + MessageCode.SUCCESS.getCode(), + MessageCode.SUCCESS.getDescription(), + userFileStats); + } + private BaseResponse handleGetFileInfo(BaseRequest request, FileEntityRequest fileRequest) { try { Long userId = authService.useUserAuthToken(fileRequest.getToken()); diff --git a/src/main/java/ru/soune/nocopy/service/file/FileStatsService.java b/src/main/java/ru/soune/nocopy/service/file/FileStatsService.java new file mode 100644 index 0000000..a32f04d --- /dev/null +++ b/src/main/java/ru/soune/nocopy/service/file/FileStatsService.java @@ -0,0 +1,93 @@ +package ru.soune.nocopy.service.file; + +import lombok.RequiredArgsConstructor; +import org.springframework.stereotype.Service; +import ru.soune.nocopy.dto.file.FileInfoUserResponse; +import ru.soune.nocopy.entity.file.FileEntity; +import ru.soune.nocopy.entity.file.FileStatus; +import ru.soune.nocopy.entity.file.FileType; +import ru.soune.nocopy.repository.FileEntityRepository; + +import java.util.List; + +@Service +@RequiredArgsConstructor +public class FileStatsService { + private final FileEntityRepository fileEntityRepository; + + public FileInfoUserResponse getUserFileStats(Long userId) { + List userFiles = fileEntityRepository.findByUserId(userId); + return calculateStats(userFiles); + } + + public FileInfoUserResponse calculateStats(List files) { + return FileInfoUserResponse.builder() + .allFileSize(calculateTotalSize(files)) + .fileCount(calculateTotalCount(files)) + .filesCheck(calculateByStatus(files, FileStatus.CHECKED)) + .filesViolation(calculateByStatus(files, FileStatus.VIOLATION)) + + .imagesSize(calculateMediaSize(files, FileType.IMAGE)) + .imagesCount(calculateMediaCount(files, FileType.IMAGE)) + .imagesCheck(calculateMediaByStatus(files, FileType.IMAGE, FileStatus.CHECKED)) + .imagesViolations(calculateMediaByStatus(files, FileType.IMAGE, FileStatus.VIOLATION)) + + .videosSize(calculateMediaSize(files, FileType.VIDEO)) + .videosCount(calculateMediaCount(files, FileType.VIDEO)) + .videosCheck(calculateMediaByStatus(files, FileType.VIDEO, FileStatus.CHECKED)) + .videosViolations(calculateMediaByStatus(files, FileType.VIDEO, FileStatus.VIOLATION)) + + .audiosSize(calculateMediaSize(files, FileType.AUDIO)) + .audiosCount(calculateMediaCount(files, FileType.AUDIO)) + .audiosCheck(calculateMediaByStatus(files, FileType.AUDIO, FileStatus.CHECKED)) + .audiosViolations(calculateMediaByStatus(files, FileType.AUDIO, FileStatus.VIOLATION)) + .build(); + } + private Long calculateTotalSize(List files) { + return files.stream() + .filter(file -> file.getStatus() != FileStatus.DELETED) + .mapToLong(FileEntity::getFileSize) + .sum(); + } + + private Integer calculateTotalCount(List files) { + return (int) files.stream() + .filter(file -> file.getStatus() != FileStatus.DELETED) + .count(); + } + + private Integer calculateByStatus(List files, FileStatus status) { + return (int) files.stream() + .filter(file -> file.getStatus() == status) + .count(); + } + + private Long calculateMediaSize(List files, FileType fileType) { + return files.stream() + .filter(file -> file.getStatus() != FileStatus.DELETED) + .filter(file -> isFileType(file, fileType)) + .mapToLong(FileEntity::getFileSize) + .sum(); + } + + private Integer calculateMediaCount(List files, FileType fileType) { + return (int) files.stream() + .filter(file -> file.getStatus() != FileStatus.DELETED) + .filter(file -> isFileType(file, fileType)) + .count(); + } + + private Integer calculateMediaByStatus(List files, FileType fileType, FileStatus status) { + return (int) files.stream() + .filter(file -> isFileType(file, fileType)) + .filter(file -> file.getStatus() == status) + .count(); + } + + private boolean isFileType(FileEntity file, FileType fileType) { + if (file.getFileExtension() == null) return false; + + String extension = file.getFileExtension().toLowerCase().replace(".", ""); + return fileType.supportsExtension(extension); + } +} diff --git a/src/main/java/ru/soune/nocopy/service/file/FileUploadServiceImpl.java b/src/main/java/ru/soune/nocopy/service/file/FileUploadServiceImpl.java index f54cec9..3e09822 100644 --- a/src/main/java/ru/soune/nocopy/service/file/FileUploadServiceImpl.java +++ b/src/main/java/ru/soune/nocopy/service/file/FileUploadServiceImpl.java @@ -102,7 +102,7 @@ public class FileUploadServiceImpl implements FileUploadService { int totalChunks = (int) Math.ceil((double) fileSize / chunkSize); log.debug("File will be split into {} chunks (chunk size: {} bytes)", totalChunks, chunkSize); - Long chunkSize = totalChunks == 1 ? fileSize : 1048576L; + Long chunkSize = totalChunks == 1 ? fileSize : 1000000L; FileUploadSession session = FileUploadSession.builder() .userId(userId) diff --git a/src/main/resources/application.yaml b/src/main/resources/application.yaml index 68701ca..8122079 100644 --- a/src/main/resources/application.yaml +++ b/src/main/resources/application.yaml @@ -25,7 +25,8 @@ file: storage: base-path: ${FILE_STORAGE_PATH:/data/uploads} # chunk-size: ${FILE_CHUNK_SIZE:5242880} # 5MB - chunk-size: ${FILE_CHUNK_SIZE:1048576} # 15MB +# chunk-size: ${FILE_CHUNK_SIZE:1048576} # 1MB + chunk-size: ${FILE_CHUNK_SIZE:1000000} # 1MB max-file-size: ${MAX_FILE_SIZE:10737418240} # 10GB max-retry-attempts: ${MAX_RETRY_ATTEMPTS:3} chunk-timeout-ms: ${CHUNK_TIMEOUT_MS:300000} # 5 минут