Merge branch 'dev' into NCBACK-27

# Conflicts: # src/main/java/ru/soune/nocopy/controller/ApiController.java # src/main/java/ru/soune/nocopy/handler/FileEntityHandler.java
2026-01-09 12:41:32 +07:00
parent f1222ede57 63baaf7c80
commit 18d35cb0fb
9 changed files with 221 additions and 160 deletions
@@ -58,7 +58,8 @@ services:
    environment:
      FILE_STORAGE_PATH: /data/uploads
      MAX_FILE_SIZE: 10737418240
-      FILE_CHUNK_SIZE: 1048576
+#      FILE_CHUNK_SIZE: 1048576
+      FILE_CHUNK_SIZE: 1000000
      POSTGRES_DB: no_copy_
      POSTGRES_USER: postgres
      POSTGRES_PASSWORD: postgres
@@ -120,7 +121,7 @@ services:
  prometheus:
    image: prom/prometheus:latest
    container_name: prometheus
-    deploy:  # ← ДОБАВЛЕНО
+    deploy:  
      resources:
        limits:
          cpus: '0.5'
@@ -148,7 +149,7 @@ services:
  loki:
    image: grafana/loki:2.9.2
    container_name: loki
-    deploy:  # ← ДОБАВЛЕНО
+    deploy:  
      resources:
        limits:
          cpus: '1.0'
@@ -173,7 +174,7 @@ services:
  tempo:
    image: grafana/tempo:2.4.1
    container_name: tempo
-    deploy:  # ← ДОБАВЛЕНО
+    deploy:  
      resources:
        limits:
          cpus: '0.5'
@@ -7,7 +7,6 @@ pipeline {
                defaultValue: 'dev',
                description: 'Ветка для деплоя'
        )
-
        string(
                name: 'SERVER',
                defaultValue: '92.242.61.23',
@@ -51,156 +50,57 @@ pipeline {
                            sshpass -p '$SSH_PASS' ssh -o StrictHostKeyChecking=no $SSH_USER@$SERVER "
                                cd /opt/deployments/${params.BRANCH}
                                
-                                echo '========================================'
-                                echo 'Step 1: Checking current state...'
-                                echo '========================================'
+                                echo '1. Остановка старого приложения...'
+                                docker stop app-backend 2>/dev/null || echo 'Контейнер не найден'
+                                docker rm app-backend 2>/dev/null || echo 'Контейнер не найден'
                                
-                                docker ps --filter 'name=postgres|app-backend|storage|grafana|prometheus|loki|tempo|alloy' --format 'table {{.Names}}\\t{{.Image}}\\t{{.Status}}'
+                                echo '2. Удаление старых образов...'
+                                docker images --filter 'reference=*app*' -q | xargs -r docker rmi -f 2>/dev/null || echo 'Нет образов для удаления'
                                
-                                echo '========================================'
-                                echo 'Step 2: Setup environment for compatibility...'
-                                echo '========================================'
+                                echo '3. Создание сети если нужно...'
+                                docker network create app-network 2>/dev/null || echo 'Сеть уже существует'
                                
-                                docker network create app-network 2>/dev/null || echo 'Network app-network already exists'
+                                echo '4. Запуск инфраструктуры...'
+                                docker-compose up -d db storage
                                
-                                echo 'COMPOSE_COMPATIBILITY=true' > .env
+                                echo '5. Ожидание PostgreSQL...'
+                                sleep 10
                                
-                                echo '========================================'
-                                echo 'Step 3: Force cleanup old application...'
-                                echo '========================================'
+                                echo '6. Сборка нового образа приложения...'
+                                docker build --no-cache -t app-backend:latest .
                                
-                                docker-compose --compatibility down 2>/dev/null || echo 'No previous compose stack'
+                                echo '7. Запуск приложения...'
+                                docker run -d \\
+                                  --name app-backend \\
+                                  --network app-network \\
+                                  --network-alias app \\
+                                  -p 80:8080 \\
+                                  -v uploads_data:/data/uploads:rw \\
+                                  -e POSTGRES_DB=no_copy_ \\
+                                  -e POSTGRES_USER=postgres \\
+                                  -e POSTGRES_PASSWORD=postgres \\
+                                  -e POSTGRES_PORT=5432 \\
+                                  -e POSTGRES_HOST=db \\
+                                  --restart unless-stopped \\
+                                  app-backend:latest
                                
-                                docker images --filter 'reference=*app-backend*' -q | xargs -r docker rmi -f 2>/dev/null || echo 'No images to remove'
+                                echo '8. Запуск мониторинга...'
+                                docker-compose up -d grafana prometheus loki tempo alloy
                                
-                                echo '========================================'
-                                echo 'Step 4: Verify files and fix compose...'
-                                echo '========================================'
+                                echo '9. Проверка...'
+                                sleep 5
                                
-                                if [ ! -f docker-compose.yaml ]; then
-                                    echo 'ERROR: docker-compose.yaml not found!'
-                                    ls -la
-                                    exit 1
-                                fi
+                                echo 'Статус контейнеров:'
+                                docker ps --format 'table {{.Names}}\\t{{.Image}}\\t{{.Status}}'
                                
-                                echo 'Checking docker-compose.yaml syntax...'
-                                docker-compose --compatibility config 2>&1 | head -20 || echo 'Config check output'
-                                
-                                echo '========================================'
-                                echo 'Step 5: Start infrastructure services...'
-                                echo '========================================'
-                                
-                                echo 'Starting PostgreSQL with resource limits...'
-                                docker-compose --compatibility up -d db
-                                
-                                echo 'Starting storage service...'
-                                docker-compose --compatibility up -d storage
-                                
-                                echo 'Waiting for PostgreSQL to be ready...'
-                                for i in {1..30}; do
-                                    if docker-compose --compatibility exec db pg_isready -U postgres 2>/dev/null; then
-                                        echo 'PostgreSQL is ready'
-                                        break
-                                    fi
-                                    echo 'Waiting... attempt ' \$i
-                                    sleep 2
-                                done
-                                
-                                echo '========================================'
-                                echo 'Step 6: Build application image...'
-                                echo '========================================'
-                                
-                                if [ ! -f Dockerfile ]; then
-                                    echo 'ERROR: Dockerfile not found!'
-                                    exit 1
-                                fi
-                                
-                                echo 'Building Docker image...'
-                                docker build --no-cache -t app-backend:latest . 2>&1 | tail -30
-                                
-                                echo 'Image created successfully'
-                                docker images app-backend:latest --format 'table {{.Repository}}\\t{{.Tag}}\\t{{.Size}}\\t{{.CreatedAt}}'
-                                
-                                echo '========================================'
-                                echo 'Step 7: Start application with resource limits...'
-                                echo '========================================'
-                                
-                                docker volume create uploads_data 2>/dev/null || echo 'Volume uploads_data already exists'
-                                
-                                echo 'Starting main application...'
-                                docker-compose --compatibility up -d app
-                                
-                                echo 'Waiting for application to start...'
-                                sleep 15
-                                
-                                echo '========================================'
-                                echo 'Step 8: Start monitoring stack...'
-                                echo '========================================'
-                                
-                                docker-compose --compatibility up -d grafana prometheus loki tempo alloy
-                                
-                                echo '========================================'
-                                echo 'Step 9: Verify deployment...'
-                                echo '========================================'
-                                
-                                echo 'All containers status:'
-                                docker-compose --compatibility ps
-                                
-                                echo ''
-                                echo 'Resource limits check:'
-                                echo 'App backend limits:'
-                                docker inspect app-backend --format='{{.HostConfig.Memory}} {{.HostConfig.CpuQuota}} {{.HostConfig.CpuPeriod}}' 2>/dev/null || echo 'Cannot inspect'
-                                
-                                echo ''
-                                echo 'PostgreSQL limits:'
-                                docker inspect postgres --format='{{.HostConfig.Memory}} {{.HostConfig.CpuQuota}} {{.HostConfig.CpuPeriod}}' 2>/dev/null || echo 'Cannot inspect'
-                                
-                                echo ''
-                                echo 'Container resource usage:'
-                                docker stats --no-stream --format 'table {{.Name}}\\t{{.CPUPerc}}\\t{{.MemUsage}}\\t{{.MemPerc}}' postgres app-backend storage 2>/dev/null || echo 'Stats not available'
-                                
-                                echo '========================================'
-                                echo 'Step 10: Health checks...'
-                                echo '========================================'
-                                
-                                echo 'Application health check...'
-                                for i in {1..10}; do
-                                    if curl -s -f http://localhost:80/health 2>/dev/null; then
-                                        echo '✓ Application health check PASSED'
-                                        break
-                                    elif [ \$i -eq 10 ]; then
-                                        echo '✗ Application health check FAILED'
-                                        echo 'Last logs:'
-                                        docker-compose --compatibility logs --tail=20 app
-                                    else
-                                        echo 'Waiting for application... attempt ' \$i
-                                        sleep 5
-                                    fi
-                                done
-                                
-                                echo 'Checking monitoring services...'
-                                if curl -s -f http://localhost:3000/api/health 2>/dev/null; then
-                                    echo '✓ Grafana is healthy'
+                                echo '10. Проверка health...'
+                                if curl -s -f http://localhost:80/health > /dev/null 2>&1; then
+                                    echo 'Приложение работает'
+                                    echo 'URL: http://${params.SERVER}:80'
                                else
-                                    echo '⚠ Grafana may be starting'
+                                    echo 'Проверка health не удалась'
+                                    docker logs app-backend --tail=20
                                fi
-                                
-                                if curl -s -f http://localhost:9090/-/healthy 2>/dev/null; then
-                                    echo '✓ Prometheus is healthy'
-                                fi
-                                
-                                echo '========================================'
-                                echo 'Deployment Summary:'
-                                echo '========================================'
-                                echo 'Application URL: http://${params.SERVER}:80'
-                                echo 'Grafana: http://${params.SERVER}:3000 (admin/admin)'
-                                echo 'Prometheus: http://${params.SERVER}:9090'
-                                echo 'Loki: http://${params.SERVER}:3100'
-                                echo ''
-                                echo 'To view logs: docker-compose logs -f app'
-                                echo 'To view all containers: docker-compose ps'
-                                echo 'To stop: docker-compose down'
-                                echo '========================================'
                            "
                        """
                    }
@@ -213,8 +113,6 @@ pipeline {
        success {
            echo "Deployment successful"
            echo "Application URL: http://${params.SERVER}:80"
-            echo "Grafana: http://${params.SERVER}:3000"
-            echo "Prometheus: http://${params.SERVER}:9090"
        }
        failure {
            echo "Deployment failed for branch ${params.BRANCH}"
@@ -0,0 +1,61 @@
+package ru.soune.nocopy.dto.file;
+
+import com.fasterxml.jackson.annotation.JsonProperty;
+import lombok.AllArgsConstructor;
+import lombok.Builder;
+import lombok.Data;
+import lombok.NoArgsConstructor;
+
+@Data
+@Builder
+@NoArgsConstructor
+@AllArgsConstructor
+public class FileInfoUserResponse {
+    @JsonProperty("all_files_size")
+    private Long allFileSize;
+
+    @JsonProperty("all_files_quantity")
+    private Integer fileCount;
+
+    @JsonProperty("all_files_check")
+    private Integer filesCheck;
+
+    @JsonProperty("all_files_violation")
+    private Integer filesViolation;
+
+    @JsonProperty("images_size")
+    private Long imagesSize;
+
+    @JsonProperty("images_quantity")
+    private Integer imagesCount;
+
+    @JsonProperty("images_check")
+    private Integer imagesCheck;
+
+    @JsonProperty("images_violations")
+    private Integer imagesViolations;
+
+    @JsonProperty("videos_size")
+    private Long videosSize;
+
+    @JsonProperty("videos_quantity")
+    private Integer videosCount;
+
+    @JsonProperty("videos_check")
+    private Integer videosCheck;
+
+    @JsonProperty("videos_violations")
+    private Integer videosViolations;
+
+    @JsonProperty("audios_size")
+    private Long audiosSize;
+
+    @JsonProperty("audios_quantity")
+    private Integer audiosCount;
+
+    @JsonProperty("audios_check")
+    private Integer audiosCheck;
+
+    @JsonProperty("audios_violations")
+    private Integer audiosViolations;
+}
@@ -4,5 +4,7 @@ public enum FileStatus {
    ACTIVE,
    DELETED,
    PROCESSING,
+    VIOLATION,
+    CHECKED,
    ERROR
 }
@@ -7,21 +7,10 @@ import java.util.List;

@Getter
 public enum FileType {
-    IMAGE("image", Arrays.asList("jpg", "jpeg", "png", "gif", "bmp", "webp")),
+    IMAGE("image", Arrays.asList("jpg", "jpeg", "png", "gif", "bmp", "webp", "jfif")),
    VIDEO("video", Arrays.asList("mp4", "avi", "mov", "wmv", "flv", "mkv", "webm", "m4v", "mpg", "mpeg",
            "3gp", "3g2", "f4v", "m2ts", "mts", "vob", "ogv", "divx")),
-    AUDIO("audio", Arrays.asList("mp3", "wav", "flac")),
-    DOCUMENT("document", Arrays.asList(
-            "pdf", "txt", "rtf",
-            "doc", "docx", "xls", "xlsx", "ppt", "pptx", "pps", "ppsx", "dot", "dotx", "xlt", "xltx", "pot", "potx",
-            "odt", "ods", "odp", "odg", "odf", "odb", "odc", "odi", "odm", "ott", "ots", "otp", "otg", "oth",
-            "sxw", "sxc", "sxi", "sxd", "sxg", "stc", "sti", "stw", "sxm",
-            "pages", "numbers", "key",
-            "csv", "tsv", "xml", "html", "htm", "tex", "md", "markdown",
-            "epub", "mobi", "azw", "azw3", "fb2",
-            "wps", "wpt", "et", "dps", "vsd", "vsdx",
-            "java", "py", "cpp", "c", "h", "js", "css", "php", "sql", "json", "yaml", "yml", "sh", "bat",
-            "one", "note"));
+    AUDIO("audio", Arrays.asList("mp3", "wav", "flac"));

    private final String displayName;
    private final List<String> allowedExtensions;
@@ -14,6 +14,7 @@ import ru.soune.nocopy.repository.AuthTokenRepository;
 import ru.soune.nocopy.repository.FileEntityRepository;
 import ru.soune.nocopy.service.auth.AuthService;
 import ru.soune.nocopy.service.file.FileEntityService;
+import ru.soune.nocopy.service.file.FileStatsService;

 import java.util.Arrays;
 import java.util.List;
@@ -22,10 +23,13 @@ import java.util.List;
@Component
@RequiredArgsConstructor
 public class FileEntityHandler implements RequestHandler {
+
    private final FileEntityService fileEntityService;

    private final AuthService authService;

+    private final FileStatsService fileStatsService;
+
    private final AuthTokenRepository authTokenRepository;

    private final ObjectMapper objectMapper;
@@ -43,6 +47,8 @@ public class FileEntityHandler implements RequestHandler {
            switch (action) {
                case "file_info":
                    return handleGetFileInfo(request, fileRequest);
+                case "user_files_info":
+                    return handleGetFilesUserInfo(request, fileRequest);
                case "file_by_session":
                    return handleGetFileBySession(request, fileRequest);
                case "user_files":
@@ -75,6 +81,16 @@ public class FileEntityHandler implements RequestHandler {
        }
    }

+    private BaseResponse handleGetFilesUserInfo(BaseRequest request, FileEntityRequest fileRequest) {
+        Long userId = getUserIdFromToken(fileRequest.getToken());
+        FileInfoUserResponse userFileStats = fileStatsService.getUserFileStats(userId);
+
+        return new BaseResponse(request.getMsgId(),
+                MessageCode.SUCCESS.getCode(),
+                MessageCode.SUCCESS.getDescription(),
+                userFileStats);
+    }
+
    private BaseResponse handleGetFileInfo(BaseRequest request, FileEntityRequest fileRequest) {
        try {
            Long userId =  authService.useUserAuthToken(fileRequest.getToken());
@@ -0,0 +1,93 @@
+package ru.soune.nocopy.service.file;
+
+import lombok.RequiredArgsConstructor;
+import org.springframework.stereotype.Service;
+import ru.soune.nocopy.dto.file.FileInfoUserResponse;
+import ru.soune.nocopy.entity.file.FileEntity;
+import ru.soune.nocopy.entity.file.FileStatus;
+import ru.soune.nocopy.entity.file.FileType;
+import ru.soune.nocopy.repository.FileEntityRepository;
+
+import java.util.List;
+
+@Service
+@RequiredArgsConstructor
+public class FileStatsService {
+    private final FileEntityRepository fileEntityRepository;
+
+    public FileInfoUserResponse getUserFileStats(Long userId) {
+        List<FileEntity> userFiles = fileEntityRepository.findByUserId(userId);
+        return calculateStats(userFiles);
+    }
+
+    public FileInfoUserResponse calculateStats(List<FileEntity> files) {
+        return FileInfoUserResponse.builder()
+                .allFileSize(calculateTotalSize(files))
+                .fileCount(calculateTotalCount(files))
+                .filesCheck(calculateByStatus(files, FileStatus.CHECKED))
+                .filesViolation(calculateByStatus(files, FileStatus.VIOLATION))
+
+                .imagesSize(calculateMediaSize(files, FileType.IMAGE))
+                .imagesCount(calculateMediaCount(files, FileType.IMAGE))
+                .imagesCheck(calculateMediaByStatus(files, FileType.IMAGE, FileStatus.CHECKED))
+                .imagesViolations(calculateMediaByStatus(files, FileType.IMAGE, FileStatus.VIOLATION))
+
+                .videosSize(calculateMediaSize(files, FileType.VIDEO))
+                .videosCount(calculateMediaCount(files, FileType.VIDEO))
+                .videosCheck(calculateMediaByStatus(files, FileType.VIDEO, FileStatus.CHECKED))
+                .videosViolations(calculateMediaByStatus(files, FileType.VIDEO, FileStatus.VIOLATION))
+
+                .audiosSize(calculateMediaSize(files, FileType.AUDIO))
+                .audiosCount(calculateMediaCount(files, FileType.AUDIO))
+                .audiosCheck(calculateMediaByStatus(files, FileType.AUDIO, FileStatus.CHECKED))
+                .audiosViolations(calculateMediaByStatus(files, FileType.AUDIO, FileStatus.VIOLATION))
+                .build();
+    }
+    private Long calculateTotalSize(List<FileEntity> files) {
+        return files.stream()
+                .filter(file -> file.getStatus() != FileStatus.DELETED)
+                .mapToLong(FileEntity::getFileSize)
+                .sum();
+    }
+
+    private Integer calculateTotalCount(List<FileEntity> files) {
+        return (int) files.stream()
+                .filter(file -> file.getStatus() != FileStatus.DELETED)
+                .count();
+    }
+
+    private Integer calculateByStatus(List<FileEntity> files, FileStatus status) {
+        return (int) files.stream()
+                .filter(file -> file.getStatus() == status)
+                .count();
+    }
+
+    private Long calculateMediaSize(List<FileEntity> files, FileType fileType) {
+        return files.stream()
+                .filter(file -> file.getStatus() != FileStatus.DELETED)
+                .filter(file -> isFileType(file, fileType))
+                .mapToLong(FileEntity::getFileSize)
+                .sum();
+    }
+
+    private Integer calculateMediaCount(List<FileEntity> files, FileType fileType) {
+        return (int) files.stream()
+                .filter(file -> file.getStatus() != FileStatus.DELETED)
+                .filter(file -> isFileType(file, fileType))
+                .count();
+    }
+
+    private Integer calculateMediaByStatus(List<FileEntity> files, FileType fileType, FileStatus status) {
+        return (int) files.stream()
+                .filter(file -> isFileType(file, fileType))
+                .filter(file -> file.getStatus() == status)
+                .count();
+    }
+
+    private boolean isFileType(FileEntity file, FileType fileType) {
+        if (file.getFileExtension() == null) return false;
+
+        String extension = file.getFileExtension().toLowerCase().replace(".", "");
+        return fileType.supportsExtension(extension);
+    }
+}
@@ -102,7 +102,7 @@ public class FileUploadServiceImpl implements FileUploadService {
        int totalChunks = (int) Math.ceil((double) fileSize / chunkSize);
        log.debug("File will be split into {} chunks (chunk size: {} bytes)",
                totalChunks, chunkSize);
-        Long chunkSize = totalChunks == 1 ? fileSize : 1048576L;
+        Long chunkSize = totalChunks == 1 ? fileSize : 1000000L;

        FileUploadSession session = FileUploadSession.builder()
                .userId(userId)
@@ -25,7 +25,8 @@ file:
  storage:
    base-path: ${FILE_STORAGE_PATH:/data/uploads}
    #    chunk-size: ${FILE_CHUNK_SIZE:5242880}  # 5MB
-    chunk-size: ${FILE_CHUNK_SIZE:1048576}  # 15MB
+#    chunk-size: ${FILE_CHUNK_SIZE:1048576}  # 1MB
+    chunk-size: ${FILE_CHUNK_SIZE:1000000}  # 1MB
    max-file-size: ${MAX_FILE_SIZE:10737418240}  # 10GB
    max-retry-attempts: ${MAX_RETRY_ATTEMPTS:3}
    chunk-timeout-ms: ${CHUNK_TIMEOUT_MS:300000}  # 5 минут