diff --git a/.env b/.env index d3e4347..e54b26f 100644 --- a/.env +++ b/.env @@ -1,10 +1,10 @@ POSTGRES_DB=no_copy_ -POSTGRES_USER=postgres -POSTGRES_PASSWORD=postgres +POSTGRES_USER=ncp_db +POSTGRES_PASSWORD=ncpDbApp POSTGRES_PORT=5432 POSTGRES_HOST=postgres REDIS_HOST=redis REDIS_PORT=6379 -SERVER_PORT=8080 +SERVER_PORT=8080 \ No newline at end of file diff --git a/Dockerfile b/Dockerfile index 6cd62ca..03d3325 100644 --- a/Dockerfile +++ b/Dockerfile @@ -16,4 +16,4 @@ RUN mkdir -p /data/uploads && chmod 755 /data/uploads EXPOSE 8080 -CMD ["java", "-jar", "app.jar"] +CMD ["java", "-jar", "app.jar"] \ No newline at end of file diff --git a/README.md b/README.md index e69de29..aa5e26c 100644 --- a/README.md +++ b/README.md @@ -0,0 +1,78 @@ +Создаем сеть: +docker network create app-network + +Войти в БД: +docker exec -it postgres psql -U ncp_db -d no_copy_ + +Выполнить команду: +\du + +Должен быть один пользователь: + List of roles +Role name | Attributes +-----------+------------------------------------------------------------ +ncp_db | Superuser, Create role, Create DB, Replication, Bypass RLS + +суперпользователь БД: +postgres/postgres +пользователь бд для бэка +ncp_db / ncpDbApp + +Подключение к БД для приложения,если volume не существует : + +Создаться автоматически. + +Если уже существует volume : + +docker exec -it postgres psql -U postgres -d postgres + +CREATE USER ncp_db WITH PASSWORD 'ncpDbApp'; + +GRANT ALL PRIVILEGES ON DATABASE no_copy_ TO ncp_db; + +\du + +\q + +------- + +psql -U postgres -d no_copy_ + +-- база + +GRANT CONNECT ON DATABASE no_copy_ TO ncp_db; + +-- схема + +GRANT USAGE ON SCHEMA public TO ncp_db; + +-- существующие таблицы + +GRANT ALL PRIVILEGES ON ALL TABLES IN SCHEMA public TO ncp_db; + +-- существующие sequence (очень важно для id) + +GRANT ALL PRIVILEGES ON ALL SEQUENCES IN SCHEMA public TO ncp_db; + +-- будущие таблицы + +ALTER DEFAULT PRIVILEGES IN SCHEMA public +GRANT ALL ON TABLES TO ncp_db; + +ALTER DEFAULT PRIVILEGES IN SCHEMA public +GRANT ALL ON SEQUENCES TO ncp_db; + + +Зайти в контейнер: +docker exec -it {name} bash + +------ + +Создать для сущности FileEntity sequence для икремента индетификатора + +CREATE SEQUENCE IF NOT EXISTS file_support_id_seq START 1; + +ALTER TABLE file_entities +ALTER COLUMN support_id +SET DEFAULT nextval('file_support_id_seq'); + diff --git a/build.gradle b/build.gradle index 4277882..0c7f38c 100644 --- a/build.gradle +++ b/build.gradle @@ -41,6 +41,12 @@ dependencies { implementation 'org.mapstruct:mapstruct:1.5.5.Final' implementation 'commons-validator:commons-validator:1.7' + implementation 'org.flywaydb:flyway-core:9.22.0' + + implementation 'com.fasterxml.jackson.dataformat:jackson-dataformat-xml:2.17.0' + implementation 'tools.jackson.core:jackson-core:3.0.3' + implementation 'jakarta.xml.bind:jakarta.xml.bind-api:4.0.0' + annotationProcessor 'org.mapstruct:mapstruct-processor:1.5.5.Final' annotationProcessor 'org.projectlombok:lombok' diff --git a/docker-compose.yaml b/docker-compose.yaml index 8df1c29..bb1e1b7 100644 --- a/docker-compose.yaml +++ b/docker-compose.yaml @@ -18,7 +18,7 @@ services: restart: unless-stopped db: - image: postgres:17 + image: postgres:17.7 restart: always deploy: resources: @@ -30,14 +30,15 @@ services: memory: 1G environment: POSTGRES_DB: no_copy_ - POSTGRES_USER: postgres - POSTGRES_PASSWORD: postgres + POSTGRES_USER: ${POSTGRES_USER} + POSTGRES_PASSWORD: ${POSTGRES_PASSWORD} POSTGRES_SHARED_BUFFERS: 512MB POSTGRES_EFFECTIVE_CACHE_SIZE: 1536MB ports: - "54320:5432" volumes: - pgdata:/var/lib/postgresql/data + - ./init-scripts:/docker-entrypoint-initdb.d:ro container_name: postgres networks: app-network: @@ -58,13 +59,17 @@ services: environment: FILE_STORAGE_PATH: /data/uploads MAX_FILE_SIZE: 10737418240 - FILE_CHUNK_SIZE: 1048576 + FILE_CHUNK_SIZE: 1000000 POSTGRES_DB: no_copy_ - POSTGRES_USER: postgres - POSTGRES_PASSWORD: postgres + POSTGRES_USER: ${POSTGRES_USER} + POSTGRES_PASSWORD: ${POSTGRES_PASSWORD} POSTGRES_PORT: 5432 POSTGRES_HOST: db STORAGE_SERVICE_URL: http://storage:8081 + SPRING_PROFILES_ACTIVE: docker + YANDEX_API_KEY: AQVNyaVaUmgUb1GMCtf5zSEqFxy0woXrcMOOB43q + YANDEX_FOLDER_ID: b1gokpdbm6qfpsou8pcd + YANDEX_SEARCH_URL: "https://searchapi.api.cloud.yandex.net/v2/image/search_by_image" depends_on: - db ports: @@ -77,6 +82,10 @@ services: - api volumes: - uploads_data:/data/uploads:rw + healthcheck: + test: [ "CMD-SHELL", "pg_isready -U ${POSTGRES_USER} -d no_copy_" ] + interval: 10s + retries: 5 grafana: image: grafana/grafana:10.3.1 @@ -120,7 +129,7 @@ services: prometheus: image: prom/prometheus:latest container_name: prometheus - deploy: + deploy: resources: limits: cpus: '0.5' @@ -148,7 +157,7 @@ services: loki: image: grafana/loki:2.9.2 container_name: loki - deploy: + deploy: resources: limits: cpus: '1.0' @@ -173,7 +182,7 @@ services: tempo: image: grafana/tempo:2.4.1 container_name: tempo - deploy: + deploy: resources: limits: cpus: '0.5' diff --git a/infrastructure/jenkins/deploy.groovy b/infrastructure/jenkins/deploy.groovy index 7611efe..dc09c2e 100644 --- a/infrastructure/jenkins/deploy.groovy +++ b/infrastructure/jenkins/deploy.groovy @@ -30,6 +30,7 @@ pipeline { } } + stage('Deploy with docker-compose') { steps { script { @@ -38,71 +39,73 @@ pipeline { credentialsId: 'server-root-password', usernameVariable: 'SSH_USER', passwordVariable: 'SSH_PASS' - ) + ), + string(credentialsId: 'DB_USER', variable: 'DB_USER'), + string(credentialsId: 'DB_PASSWORD', variable: 'DB_PASSWORD') ]) { sh """ - echo "Deploying branch: ${params.BRANCH}" - - echo "Copying files to server..." - sshpass -p '$SSH_PASS' ssh -o StrictHostKeyChecking=no $SSH_USER@$SERVER "mkdir -p /opt/deployments/${params.BRANCH}" - sshpass -p '$SSH_PASS' scp -r -o StrictHostKeyChecking=no ./* $SSH_USER@$SERVER:/opt/deployments/${params.BRANCH}/ - - sshpass -p '$SSH_PASS' ssh -o StrictHostKeyChecking=no $SSH_USER@$SERVER " - cd /opt/deployments/${params.BRANCH} - - echo '1. Остановка старого приложения...' - docker stop app-backend 2>/dev/null || echo 'Контейнер не найден' - docker rm app-backend 2>/dev/null || echo 'Контейнер не найден' - - echo '2. Удаление старых образов...' - docker images --filter 'reference=*app*' -q | xargs -r docker rmi -f 2>/dev/null || echo 'Нет образов для удаления' - - echo '3. Создание сети если нужно...' - docker network create app-network 2>/dev/null || echo 'Сеть уже существует' - - echo '4. Запуск инфраструктуры...' - docker-compose up -d db storage - - echo '5. Ожидание PostgreSQL...' - sleep 10 - - echo '6. Сборка нового образа приложения...' - docker build --no-cache -t app-backend:latest . - - echo '7. Запуск приложения...' - docker run -d \\ - --name app-backend \\ - --network app-network \\ - --network-alias app \\ - -p 80:8080 \\ - -v uploads_data:/data/uploads:rw \\ - -e POSTGRES_DB=no_copy_ \\ - -e POSTGRES_USER=postgres \\ - -e POSTGRES_PASSWORD=postgres \\ - -e POSTGRES_PORT=5432 \\ - -e POSTGRES_HOST=db \\ - --restart unless-stopped \\ - app-backend:latest - - echo '8. Запуск мониторинга...' - docker-compose up -d grafana prometheus loki tempo alloy - - echo '9. Проверка...' - sleep 5 - - echo 'Статус контейнеров:' - docker ps --format 'table {{.Names}}\\t{{.Image}}\\t{{.Status}}' - - echo '10. Проверка health...' - if curl -s -f http://localhost:80/health > /dev/null 2>&1; then - echo 'Приложение работает' - echo 'URL: http://${params.SERVER}:80' - else - echo 'Проверка health не удалась' - docker logs app-backend --tail=20 - fi - " - """ + echo "Deploying branch: ${params.BRANCH}" + + echo "Copying files to server..." + sshpass -p '$SSH_PASS' ssh -o StrictHostKeyChecking=no $SSH_USER@$SERVER "mkdir -p /opt/deployments/${params.BRANCH}" + sshpass -p '$SSH_PASS' scp -r -o StrictHostKeyChecking=no ./* $SSH_USER@$SERVER:/opt/deployments/${params.BRANCH}/ + + sshpass -p '$SSH_PASS' ssh -o StrictHostKeyChecking=no $SSH_USER@$SERVER " + cd /opt/deployments/${params.BRANCH} + + echo '1. Остановка старого приложения...' + docker stop app-backend 2>/dev/null || echo 'Контейнер не найден' + docker rm app-backend 2>/dev/null || echo 'Контейнер не найден' + + echo '2. Удаление старых образов...' + docker images --filter 'reference=*app*' -q | xargs -r docker rmi -f 2>/dev/null || echo 'Нет образов для удаления' + + echo '3. Создание сети если нужно...' + docker network create app-network 2>/dev/null || echo 'Сеть уже существует' + + echo '4. Запуск инфраструктуры...' + docker-compose up -d db storage + + echo '5. Ожидание PostgreSQL...' + sleep 10 + + echo '6. Сборка нового образа приложения...' + docker build --no-cache -t app-backend:latest . + + echo '7. Запуск приложения...' + docker run -d \\ + --name app-backend \\ + --network app-network \\ + --network-alias app \\ + -p 80:8080 \\ + -v uploads_data:/data/uploads:rw \\ + -e POSTGRES_DB=no_copy_ \\ + -e POSTGRES_USER=$DB_USER \\ + -e POSTGRES_PASSWORD=$DB_PASSWORD \\ + -e POSTGRES_PORT=5432 \\ + -e POSTGRES_HOST=db \\ + --restart unless-stopped \\ + app-backend:latest + + echo '8. Запуск мониторинга...' + docker-compose up -d grafana prometheus loki tempo alloy + + echo '9. Проверка...' + sleep 5 + + echo 'Статус контейнеров:' + docker ps --format 'table {{.Names}}\\t{{.Image}}\\t{{.Status}}' + + echo '10. Проверка health...' + if curl -s -f http://localhost:80/health > /dev/null 2>&1; then + echo 'Приложение работает' + echo 'URL: http://${params.SERVER}:80' + else + echo 'Проверка health не удалась' + docker logs app-backend --tail=20 + fi + " + """ } } } diff --git a/init-scripts/01-init.sql b/init-scripts/01-init.sql new file mode 100644 index 0000000..60bedbc --- /dev/null +++ b/init-scripts/01-init.sql @@ -0,0 +1,19 @@ +DO $$ +BEGIN + IF NOT EXISTS (SELECT FROM pg_roles WHERE rolname = 'ncp_db') THEN + CREATE USER ncp_db WITH PASSWORD 'ncpDbApp'; +END IF; +END +$$; + +GRANT CONNECT ON DATABASE no_copy_ TO ncp_db; +GRANT USAGE ON SCHEMA public TO ncp_db; + +GRANT ALL PRIVILEGES ON ALL TABLES IN SCHEMA public TO ncp_db; +GRANT ALL PRIVILEGES ON ALL SEQUENCES IN SCHEMA public TO ncp_db; + +ALTER DEFAULT PRIVILEGES IN SCHEMA public +GRANT ALL ON TABLES TO ncp_db; + +ALTER DEFAULT PRIVILEGES IN SCHEMA public +GRANT ALL ON SEQUENCES TO ncp_db; \ No newline at end of file diff --git a/src/main/java/ru/soune/nocopy/configuration/HandlerConfig.java b/src/main/java/ru/soune/nocopy/configuration/HandlerConfig.java index 5b3c744..d8cba33 100644 --- a/src/main/java/ru/soune/nocopy/configuration/HandlerConfig.java +++ b/src/main/java/ru/soune/nocopy/configuration/HandlerConfig.java @@ -16,7 +16,9 @@ public class HandlerConfig { LoginRequestHandler login, FileUploadHandler upload, FileEntityHandler file, - LogoutRequestHandler logoutHandler + LogoutRequestHandler logoutHandler, + ImageFoundRequestHandler imageFoundRequestHandler, + AuthRequestHandler authRequestHandler ) { Map map = new HashMap<>(); map.put(20001, login); @@ -24,6 +26,8 @@ public class HandlerConfig { map.put(20004, upload); map.put(20005, file); map.put(20006, logoutHandler); + map.put(20007, imageFoundRequestHandler); + map.put(20008, authRequestHandler); return map; } diff --git a/src/main/java/ru/soune/nocopy/configuration/JacksonConfig.java b/src/main/java/ru/soune/nocopy/configuration/JacksonConfig.java index 99c5c00..db00da1 100644 --- a/src/main/java/ru/soune/nocopy/configuration/JacksonConfig.java +++ b/src/main/java/ru/soune/nocopy/configuration/JacksonConfig.java @@ -35,6 +35,8 @@ public class JacksonConfig { mapper.registerModule(javaTimeModule); mapper.disable(SerializationFeature.WRITE_DATES_AS_TIMESTAMPS); + mapper.configure(com.fasterxml.jackson.databind.DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, false); + mapper.enable(SerializationFeature.INDENT_OUTPUT); return mapper; } diff --git a/src/main/java/ru/soune/nocopy/controller/ApiController.java b/src/main/java/ru/soune/nocopy/controller/ApiController.java index 1528058..414dba6 100644 --- a/src/main/java/ru/soune/nocopy/controller/ApiController.java +++ b/src/main/java/ru/soune/nocopy/controller/ApiController.java @@ -6,6 +6,9 @@ import lombok.RequiredArgsConstructor; import lombok.extern.slf4j.Slf4j; import org.springframework.core.io.Resource; import org.springframework.core.io.UrlResource; +import org.springframework.data.domain.Page; +import org.springframework.data.domain.Pageable; +import org.springframework.data.web.PageableDefault; import org.springframework.http.HttpHeaders; import org.springframework.http.MediaType; import org.springframework.http.ResponseEntity; @@ -17,6 +20,8 @@ import ru.soune.nocopy.dto.BaseRequest; import ru.soune.nocopy.dto.BaseResponse; import ru.soune.nocopy.dto.MessageCode; import ru.soune.nocopy.dto.RegAnswer; +import ru.soune.nocopy.dto.file.*; +import ru.soune.nocopy.entity.file.FileEntity; import ru.soune.nocopy.dto.file.ChunkUploadResponse; import ru.soune.nocopy.dto.file.CompleteUploadResponse; import ru.soune.nocopy.dto.file.FileEntityResponse; @@ -30,6 +35,9 @@ import ru.soune.nocopy.exception.*; import ru.soune.nocopy.handler.*; import ru.soune.nocopy.repository.AuthTokenRepository; import ru.soune.nocopy.repository.FileEntityRepository; +import ru.soune.nocopy.repository.FileEntityRepository; +import ru.soune.nocopy.service.FileSimilarityService; +import ru.soune.nocopy.service.auth.AuthService; import ru.soune.nocopy.service.file.FileEntityService; import ru.soune.nocopy.service.file.FileUploadService; @@ -54,7 +62,9 @@ public class ApiController { private final FileEntityService fileEntityService; - private final AuthTokenRepository authTokenRepository; + private final AuthService authService; + + private final FileSimilarityService fileSimilarityService; private final FileEntityRepository fileEntityRepository; @@ -81,6 +91,9 @@ public class ApiController { return ResponseEntity.ok().body(response); } catch (ValidationException e) { return createValidationErrorResponse(e.getBindingResult(), e.getMsgId()); + } catch (NotFoundAuthToken e) { + return ResponseEntity.ok().body(new BaseResponse(msgId, MessageCode.AUTH_TOKEN_NOT_FOUND.getCode(), + MessageCode.AUTH_TOKEN_NOT_FOUND.getDescription(), new HashMap<>())); } catch (NotValidFieldException e) { throw e; } catch (Exception e) { @@ -103,54 +116,63 @@ public class ApiController { @RequestParam(value = "chunk", required = false) MultipartFile chunk) { try { if (chunk == null || chunk.isEmpty()) { - return ResponseEntity.ok().body(new BaseResponse(20004, MessageCode.FILE_UPLOAD_ERROR.getCode(), - "Chunk file null or empty", ChunkUploadResponse.builder() - .uploadId(uploadId) - .chunkNumber(chunkNumber) - .build())); + return buildErrorResponse(uploadId, chunkNumber, "Chunk file null or empty"); } - if (uploadId == null || uploadId.isBlank()) { - return ResponseEntity.ok().body(new BaseResponse(20004, MessageCode.FILE_UPLOAD_ERROR.getCode(), - "Upload ID is required", ChunkUploadResponse.builder() - .uploadId(uploadId) - .chunkNumber(chunkNumber) - .build())); + return buildErrorResponse(uploadId, chunkNumber, "Upload ID is required"); } - if (chunkNumber == null || chunkNumber < 0) { - return ResponseEntity.ok().body(new BaseResponse(20004, MessageCode.FILE_UPLOAD_ERROR.getCode(), - "Valid chunk number is required", ChunkUploadResponse.builder() - .uploadId(uploadId) - .chunkNumber(chunkNumber) - .build())); + return buildErrorResponse(uploadId, chunkNumber, "Valid chunk number is required"); } fileUploadService.uploadChunk(uploadId, chunkNumber, chunk); - ChunkUploadResponse responseBody = ChunkUploadResponse.builder() - .uploadId(uploadId) - .chunkNumber(chunkNumber) - .chunkSize(chunk.getSize()) - .message("Chunk uploaded successfully") - .build(); - - return ResponseEntity.ok().body(new BaseResponse(20004, MessageCode.SUCCESS.getCode(), - "Chunk uploaded successfully", responseBody)); + return buildSuccessResponse(uploadId, chunkNumber, chunk); + } catch (DuplicateImageException e) { + return ResponseEntity.ok().body(new BaseResponse(20004, MessageCode.DUPLICATE_FILE_UPLOAD.getCode(), + MessageCode.DUPLICATE_FILE_UPLOAD.getDescription(), + Map.of("duplicateOwnerId", e.userId(), "duplicateFileId", e.duplicateFileId()))); } catch (Exception e) { log.error("Error uploading chunk", e); - - ChunkUploadResponse responseBody = ChunkUploadResponse.builder() - .uploadId(uploadId) - .chunkNumber(chunkNumber) - .build(); - - return ResponseEntity.ok().body(new BaseResponse( - 20004, MessageCode.FILE_UPLOAD_ERROR.getCode(), - "Failed to upload chunk: " + e.getMessage(), responseBody)); + return buildErrorResponse(uploadId, chunkNumber, "Failed to upload chunk: " + e.getMessage()); } } + @GetMapping("/v{version}/files/{fileId}/similar") + public ResponseEntity findSimilarFiles( + @PathVariable("version") int version, + @PathVariable String fileId, + @RequestParam(required = false) List similarityLevels, + @PageableDefault(size = 20, sort = "hammingDistance") Pageable pageable) { + SimilarityFilter filter = SimilarityFilter.builder() + .similarityLevels(similarityLevels) + .build(); + Page similarFiles = fileSimilarityService.findSimilarFiles(fileId, filter, pageable); + + String messageDesc; + MessageCode success; + + if (similarFiles.isEmpty()) { + messageDesc = MessageCode.FILE_NOT_FOUND.getDescription(); + success = MessageCode.FILE_NOT_FOUND; + } else { + messageDesc = MessageCode.SIMILAR_FILES_FOUND.getDescription(); + success = MessageCode.SIMILAR_FILES_FOUND; + } + + Map responseData = new HashMap<>(); + responseData.put("content", similarFiles.getContent()); + responseData.put("page", similarFiles.getNumber()); + responseData.put("size", similarFiles.getSize()); + responseData.put("totalElements", similarFiles.getTotalElements()); + responseData.put("totalPages", similarFiles.getTotalPages()); + responseData.put("hasNext", similarFiles.hasNext()); + responseData.put("hasPrevious", similarFiles.hasPrevious()); + + return ResponseEntity.ok() + .body(new BaseResponse(20004, success.getCode(), messageDesc, responseData)); + } + @GetMapping("/v{version}/files/progress/{uploadId}") public ResponseEntity getUploadProgress( @PathVariable("version") int version, @@ -174,7 +196,6 @@ public class ApiController { return ResponseEntity.ok().body(new BaseResponse( 20004, MessageCode.SUCCESS.getCode(), MessageCode.SUCCESS.getDescription(), responseBody)); - } catch (Exception e) { log.error("Error getting progress for upload: {}", uploadId, e); @@ -267,7 +288,7 @@ public class ApiController { errorData)); } - Long userId = getUserIdFromToken(tokenHeader); + Long userId = authService.useUserAuthToken(tokenHeader); FileEntityResponse entityResponse = fileEntityService.getById(fileId, version); if (!entityResponse.getUserId().equals(userId)) { @@ -383,6 +404,78 @@ public class ApiController { } } + private ResponseEntity checkForDuplicates(String uploadId) throws IOException { + Optional uploadedFile = fileEntityRepository.findByUploadSessionId(uploadId); + + if (uploadedFile.isEmpty() || !uploadedFile.get().getMimeType().equals("image")) { + return null; + } + + FileEntity fileEntity = uploadedFile.get(); + List similarFiles = fileSimilarityService.findSimilarFiles(fileEntity.getId()); + + if (hasDuplicate(similarFiles)) { + return handleDuplicate(fileEntity, similarFiles); + } + + return null; + } + + private boolean hasDuplicate(List similarFiles) { + return similarFiles.stream().anyMatch(f -> f.getHammingDistance() <= 5); + } + + private ResponseEntity handleDuplicate(FileEntity fileEntity, List similarFiles) + throws IOException { + fileEntityService.deleteFromDisk(fileEntity); + + fileEntityService.markAsDeleted(fileEntity); + + Optional originalFile = fileEntityRepository.findById(similarFiles.get(0).getFileId()); + + if (originalFile.isPresent()) { + Map duplicateInfo = Map.of( + "duplicate_file_id", originalFile.get().getId(), + "owner_user_id", String.valueOf(originalFile.get().getUserId())); + + return ResponseEntity.ok().body(new BaseResponse( + 20004, + MessageCode.DUPLICATE_FILE_UPLOAD.getCode(), + "Failed to upload chunk, duplicate", + duplicateInfo)); + } + + return null; + } + + private ResponseEntity buildSuccessResponse(String uploadId, Integer chunkNumber, MultipartFile chunk) { + ChunkUploadResponse responseBody = ChunkUploadResponse.builder() + .uploadId(uploadId) + .chunkNumber(chunkNumber) + .chunkSize(chunk.getSize()) + .message("Chunk uploaded successfully") + .build(); + + return ResponseEntity.ok().body(new BaseResponse( + 20000, + MessageCode.SUCCESS.getCode(), + "Chunk uploaded successfully", + responseBody)); + } + + private ResponseEntity buildErrorResponse(String uploadId, Integer chunkNumber, String errorMessage) { + ChunkUploadResponse responseBody = ChunkUploadResponse.builder() + .uploadId(uploadId) + .chunkNumber(chunkNumber) + .build(); + + return ResponseEntity.ok().body(new BaseResponse( + 20004, + MessageCode.FILE_UPLOAD_ERROR.getCode(), + errorMessage, + responseBody)); + } + private ResponseEntity createValidationErrorResponse(BindingResult bindingResult, Integer msgId) { List> fieldErrors = bindingResult.getFieldErrors() .stream() @@ -425,4 +518,16 @@ public class ApiController { .orElseThrow(() -> new NotFoundAuthToken("Token not found")); return authToken.getUser().getId(); } + + private boolean isPreviewSupported(String mimeType) { + if (mimeType == null) { + return false; + } + + return mimeType.startsWith("image") || + mimeType.startsWith("text") || + mimeType.equals("pdf") || + mimeType.startsWith("video") || + mimeType.startsWith("audio"); + } } \ No newline at end of file diff --git a/src/main/java/ru/soune/nocopy/dto/BaseRequest.java b/src/main/java/ru/soune/nocopy/dto/BaseRequest.java index b02ad8c..684e79a 100644 --- a/src/main/java/ru/soune/nocopy/dto/BaseRequest.java +++ b/src/main/java/ru/soune/nocopy/dto/BaseRequest.java @@ -1,9 +1,11 @@ package ru.soune.nocopy.dto; +import com.fasterxml.jackson.annotation.JsonFormat; import com.fasterxml.jackson.annotation.JsonProperty; import lombok.Data; @Data +@JsonFormat(with = JsonFormat.Feature.ACCEPT_CASE_INSENSITIVE_PROPERTIES) public class BaseRequest { @JsonProperty("version") Integer version; diff --git a/src/main/java/ru/soune/nocopy/dto/BaseResponse.java b/src/main/java/ru/soune/nocopy/dto/BaseResponse.java index b60419a..75f1543 100644 --- a/src/main/java/ru/soune/nocopy/dto/BaseResponse.java +++ b/src/main/java/ru/soune/nocopy/dto/BaseResponse.java @@ -1,11 +1,13 @@ package ru.soune.nocopy.dto; +import com.fasterxml.jackson.annotation.JsonFormat; import com.fasterxml.jackson.annotation.JsonProperty; import lombok.AllArgsConstructor; import lombok.Data; @Data @AllArgsConstructor +@JsonFormat(with = JsonFormat.Feature.ACCEPT_CASE_INSENSITIVE_PROPERTIES) public class BaseResponse { @JsonProperty("msg_id") private Integer msgId; diff --git a/src/main/java/ru/soune/nocopy/dto/LoginRequest.java b/src/main/java/ru/soune/nocopy/dto/LoginRequest.java index d64fa8b..b0548bc 100644 --- a/src/main/java/ru/soune/nocopy/dto/LoginRequest.java +++ b/src/main/java/ru/soune/nocopy/dto/LoginRequest.java @@ -1,5 +1,6 @@ package ru.soune.nocopy.dto; +import com.fasterxml.jackson.annotation.JsonFormat; import jakarta.validation.constraints.Email; import jakarta.validation.constraints.NotBlank; import jakarta.validation.constraints.Size; @@ -10,6 +11,7 @@ import lombok.NoArgsConstructor; @Data @AllArgsConstructor @NoArgsConstructor +@JsonFormat(with = JsonFormat.Feature.ACCEPT_CASE_INSENSITIVE_PROPERTIES) public class LoginRequest { @NotBlank(message = "error.not.blank") @Email(message = "error.not.email") @Size(max = 128) diff --git a/src/main/java/ru/soune/nocopy/dto/MessageCode.java b/src/main/java/ru/soune/nocopy/dto/MessageCode.java index b945f64..3a4c540 100644 --- a/src/main/java/ru/soune/nocopy/dto/MessageCode.java +++ b/src/main/java/ru/soune/nocopy/dto/MessageCode.java @@ -8,8 +8,10 @@ public enum MessageCode { INVALID_TOKEN(2, "Invalid token"), INVALID_ACTION(2, "Invalid action"), FILE_UPLOAD_ERROR(2, "File upload error"), + DUPLICATE_FILE_UPLOAD(2, "Duplicate file upload"), FILE_DOWNLOAD_ERROR(2, "File download error"), FILE_DOWNLOAD_ERROR_NOT_CORRECT_FIELD(2, "Not correct field"), + IMAGE_FOUND_ERROR(2, "Image found error"), INVALID_JSON_BODY(2, "Invalid fields in JSON object"), INCOMPLETE_UPLOAD(2, "Not load all chunks"), MSG_ID_NOT_FOUND(4, "Message id not found"), @@ -18,8 +20,10 @@ public enum MessageCode { AUTH_EMAIL_NOT_FOUND(4, "Email not found"), AUTH_EMAIL_OR_TOKEN_NOT_FOUND(4, "Email or Token not found "), AUTH_TOKEN_MISMATCH(4, "Token mismatch"), + AUTH_TOKEN_NOT_FOUND(4, "Token not found"), FILE_NOT_FOUND(4, "File not found"), - AUTH_PASSWORD_NOT_MATCHES(2, "Password does not match"); + AUTH_PASSWORD_NOT_MATCHES(2, "Password does not match"), + SIMILAR_FILES_FOUND(0, "Similar files found"); private final Integer code; diff --git a/src/main/java/ru/soune/nocopy/dto/RegRequest.java b/src/main/java/ru/soune/nocopy/dto/RegRequest.java index e08f404..d7adfea 100644 --- a/src/main/java/ru/soune/nocopy/dto/RegRequest.java +++ b/src/main/java/ru/soune/nocopy/dto/RegRequest.java @@ -1,10 +1,12 @@ package ru.soune.nocopy.dto; +import com.fasterxml.jackson.annotation.JsonFormat; import jakarta.validation.constraints.*; import lombok.Data; @Data +@JsonFormat(with = JsonFormat.Feature.ACCEPT_CASE_INSENSITIVE_PROPERTIES) public class RegRequest { @NotEmpty(message = "Full name is required") private String fullName; diff --git a/src/main/java/ru/soune/nocopy/dto/TokenProcessRequest.java b/src/main/java/ru/soune/nocopy/dto/TokenProcessRequest.java new file mode 100644 index 0000000..646f9c6 --- /dev/null +++ b/src/main/java/ru/soune/nocopy/dto/TokenProcessRequest.java @@ -0,0 +1,11 @@ +package ru.soune.nocopy.dto; + +import com.fasterxml.jackson.annotation.JsonProperty; +import lombok.Data; + +@Data +public class TokenProcessRequest { + + @JsonProperty("token") + private String token; +} diff --git a/src/main/java/ru/soune/nocopy/dto/UserRequest.java b/src/main/java/ru/soune/nocopy/dto/UserRequest.java index c5f1662..91f7379 100644 --- a/src/main/java/ru/soune/nocopy/dto/UserRequest.java +++ b/src/main/java/ru/soune/nocopy/dto/UserRequest.java @@ -10,6 +10,7 @@ import ru.soune.nocopy.entity.GenderType; import java.time.LocalDate; @Data +@JsonFormat(with = JsonFormat.Feature.ACCEPT_CASE_INSENSITIVE_PROPERTIES) public class UserRequest { @Size(min = 2) private String fullName; diff --git a/src/main/java/ru/soune/nocopy/dto/file/FileApiResponse.java b/src/main/java/ru/soune/nocopy/dto/file/FileApiResponse.java deleted file mode 100644 index 9feb546..0000000 --- a/src/main/java/ru/soune/nocopy/dto/file/FileApiResponse.java +++ /dev/null @@ -1,41 +0,0 @@ -package ru.soune.nocopy.dto.file; - -import lombok.AllArgsConstructor; -import lombok.Builder; -import lombok.Data; -import lombok.NoArgsConstructor; - -@Data -@Builder -@NoArgsConstructor -@AllArgsConstructor -public class FileApiResponse { - - private boolean success; - private String message; - private T data; - private String error; - - public static FileApiResponse success(T data) { - return FileApiResponse.builder() - .success(true) - .message("Operation completed successfully") - .data(data) - .build(); - } - - public static FileApiResponse success(String message, T data) { - return FileApiResponse.builder() - .success(true) - .message(message) - .data(data) - .build(); - } - - public static FileApiResponse error(String error) { - return FileApiResponse.builder() - .success(false) - .error(error) - .build(); - } -} diff --git a/src/main/java/ru/soune/nocopy/dto/file/FileEntityResponse.java b/src/main/java/ru/soune/nocopy/dto/file/FileEntityResponse.java index 3e89725..cbfae88 100644 --- a/src/main/java/ru/soune/nocopy/dto/file/FileEntityResponse.java +++ b/src/main/java/ru/soune/nocopy/dto/file/FileEntityResponse.java @@ -30,4 +30,5 @@ public class FileEntityResponse { private String formattedSize; private String downloadUrl; private boolean existsOnDisk; + private Integer supportId; } diff --git a/src/main/java/ru/soune/nocopy/dto/file/ImageSearchRequest.java b/src/main/java/ru/soune/nocopy/dto/file/ImageSearchRequest.java new file mode 100644 index 0000000..f943989 --- /dev/null +++ b/src/main/java/ru/soune/nocopy/dto/file/ImageSearchRequest.java @@ -0,0 +1,10 @@ +package ru.soune.nocopy.dto.file; + +import com.fasterxml.jackson.annotation.JsonProperty; +import lombok.Data; + +@Data +public class ImageSearchRequest { + @JsonProperty("file_id") + private String fileId; +} diff --git a/src/main/java/ru/soune/nocopy/dto/file/SimilarFileResponse.java b/src/main/java/ru/soune/nocopy/dto/file/SimilarFileResponse.java new file mode 100644 index 0000000..f7aae6e --- /dev/null +++ b/src/main/java/ru/soune/nocopy/dto/file/SimilarFileResponse.java @@ -0,0 +1,14 @@ +package ru.soune.nocopy.dto.file; + +import lombok.Builder; +import lombok.Value; + +@Value +@Builder +public class SimilarFileResponse { + String fileId; + String originalFileName; + Long fileSize; + Integer hammingDistance; + String similarityLevel; +} diff --git a/src/main/java/ru/soune/nocopy/dto/file/SimilarityFilter.java b/src/main/java/ru/soune/nocopy/dto/file/SimilarityFilter.java new file mode 100644 index 0000000..48a4433 --- /dev/null +++ b/src/main/java/ru/soune/nocopy/dto/file/SimilarityFilter.java @@ -0,0 +1,18 @@ +package ru.soune.nocopy.dto.file; + +import lombok.AllArgsConstructor; +import lombok.Builder; +import lombok.Data; +import lombok.NoArgsConstructor; + +import java.util.List; + +@Data +@Builder +@NoArgsConstructor +@AllArgsConstructor +public class SimilarityFilter { + private List similarityLevels; + private Integer page; + private Integer size; +} diff --git a/src/main/java/ru/soune/nocopy/dto/file/YandexSearchResponse.java b/src/main/java/ru/soune/nocopy/dto/file/YandexSearchResponse.java new file mode 100644 index 0000000..760440f --- /dev/null +++ b/src/main/java/ru/soune/nocopy/dto/file/YandexSearchResponse.java @@ -0,0 +1,36 @@ +package ru.soune.nocopy.dto.file; + +import com.fasterxml.jackson.annotation.JsonIgnoreProperties; +import com.fasterxml.jackson.annotation.JsonProperty; +import lombok.Data; +import java.util.List; + +@Data +@JsonIgnoreProperties(ignoreUnknown = true) +public class YandexSearchResponse { + + @JsonProperty("images") + private List images; + + @Data + @JsonIgnoreProperties(ignoreUnknown = true) + public static class ImageResult { + @JsonProperty("url") + private String url; + + @JsonProperty("pageUrl") + private String pageUrl; + + @JsonProperty("pageTitle") + private String pageTitle; + + @JsonProperty("width") + private Integer width; + + @JsonProperty("height") + private Integer height; + + @JsonProperty("host") + private String host; + } +} \ No newline at end of file diff --git a/src/main/java/ru/soune/nocopy/entity/AuthToken.java b/src/main/java/ru/soune/nocopy/entity/AuthToken.java index 63cf34a..6fa1a9c 100644 --- a/src/main/java/ru/soune/nocopy/entity/AuthToken.java +++ b/src/main/java/ru/soune/nocopy/entity/AuthToken.java @@ -30,7 +30,7 @@ public class AuthToken { private String token; @Column(name = "expires_at", nullable = false) - private LocalDateTime expiresAt = LocalDateTime.now().plusDays(30); + private LocalDateTime expiresAt = LocalDateTime.now().plusHours(1); @CreatedDate @Column(name = "created_at", updatable = false, nullable = false) @@ -41,8 +41,4 @@ public class AuthToken { @Column(name = "is_active") private Boolean isActive = true; - - public boolean isValid() { - return Boolean.TRUE.equals(isActive) && expiresAt.isAfter(LocalDateTime.now()); - } } diff --git a/src/main/java/ru/soune/nocopy/entity/file/FileEntity.java b/src/main/java/ru/soune/nocopy/entity/file/FileEntity.java index eb68451..724af3f 100644 --- a/src/main/java/ru/soune/nocopy/entity/file/FileEntity.java +++ b/src/main/java/ru/soune/nocopy/entity/file/FileEntity.java @@ -1,11 +1,15 @@ package ru.soune.nocopy.entity.file; +import com.fasterxml.jackson.annotation.JsonIgnore; import jakarta.persistence.*; import lombok.*; +import org.hibernate.annotations.GenerationTime; import org.springframework.data.annotation.CreatedDate; import org.springframework.data.annotation.LastModifiedDate; import org.springframework.data.jpa.domain.support.AuditingEntityListener; +import java.sql.DriverManager; +import java.sql.SQLException; import java.time.LocalDateTime; @Data @@ -16,11 +20,13 @@ import java.time.LocalDateTime; @Table(name = "file_entities") @EntityListeners(AuditingEntityListener.class) public class FileEntity { - @Id @GeneratedValue(strategy = GenerationType.UUID) private String id; + @Column(name = "support_id", unique = true, updatable = false) + private Integer supportId; + @Column(name = "user_id", nullable = false) private Long userId; @@ -70,6 +76,10 @@ public class FileEntity { @Column(name = "protected_at") private LocalDateTime protectedAt; + @OneToOne(mappedBy = "file", cascade = CascadeType.ALL, orphanRemoval = true) + @JsonIgnore + private ImageHashEntity imageHash; + @PrePersist public void prePersist() { if (this.status == null) { diff --git a/src/main/java/ru/soune/nocopy/entity/file/FileUploadSession.java b/src/main/java/ru/soune/nocopy/entity/file/FileUploadSession.java index 509d0f7..80dc12f 100644 --- a/src/main/java/ru/soune/nocopy/entity/file/FileUploadSession.java +++ b/src/main/java/ru/soune/nocopy/entity/file/FileUploadSession.java @@ -69,6 +69,7 @@ public class FileUploadSession { private String extension; @Column(name = "retry_count") + @Builder.Default private Integer retryCount = 0; @Column(name = "completed_at") @@ -81,6 +82,7 @@ public class FileUploadSession { ) @MapKeyColumn(name = "chunk_number") @Column(name = "chunk_path") + @Builder.Default private Map chunkPaths = new HashMap<>(); @PrePersist diff --git a/src/main/java/ru/soune/nocopy/entity/file/ImageHashEntity.java b/src/main/java/ru/soune/nocopy/entity/file/ImageHashEntity.java new file mode 100644 index 0000000..37f1b5a --- /dev/null +++ b/src/main/java/ru/soune/nocopy/entity/file/ImageHashEntity.java @@ -0,0 +1,39 @@ +package ru.soune.nocopy.entity.file; + +import jakarta.persistence.*; +import lombok.AllArgsConstructor; +import lombok.Builder; +import lombok.Data; +import lombok.NoArgsConstructor; + +import java.time.LocalDateTime; + +@Entity +@Table(name = "image_hashes") +@Data +@NoArgsConstructor +@AllArgsConstructor +@Builder +public class ImageHashEntity { + + @Id + @Column(name = "file_id") + private String fileId; + + @OneToOne(fetch = FetchType.LAZY) + @MapsId + @JoinColumn(name = "file_id") + private FileEntity file; + + @Column(name = "hash64_hi") + private Integer hash64Hi; + + @Column(name = "hash64_lo") + private Integer hash64Lo; + + @Column(name = "hash_algorithm", nullable = false) + private String hashAlgorithm; + + @Column(name = "created_at") + private LocalDateTime createdAt; +} diff --git a/src/main/java/ru/soune/nocopy/exception/DuplicateImageException.java b/src/main/java/ru/soune/nocopy/exception/DuplicateImageException.java new file mode 100644 index 0000000..3ac1325 --- /dev/null +++ b/src/main/java/ru/soune/nocopy/exception/DuplicateImageException.java @@ -0,0 +1,20 @@ +package ru.soune.nocopy.exception; + +public class DuplicateImageException extends RuntimeException { + private final String duplicateFileId; + private final Long userId; + + public DuplicateImageException(String message, String duplicateFileId, Long userId) { + super(message); + this.duplicateFileId = duplicateFileId; + this.userId = userId; + } + + public String duplicateFileId() { + return duplicateFileId; + } + + public Long userId() { + return userId; + } +} diff --git a/src/main/java/ru/soune/nocopy/handler/AuthRequestHandler.java b/src/main/java/ru/soune/nocopy/handler/AuthRequestHandler.java new file mode 100644 index 0000000..c79ec06 --- /dev/null +++ b/src/main/java/ru/soune/nocopy/handler/AuthRequestHandler.java @@ -0,0 +1,37 @@ +package ru.soune.nocopy.handler; + +import com.fasterxml.jackson.databind.ObjectMapper; +import lombok.RequiredArgsConstructor; +import lombok.extern.slf4j.Slf4j; +import org.springframework.stereotype.Component; +import ru.soune.nocopy.dto.BaseRequest; +import ru.soune.nocopy.dto.BaseResponse; +import ru.soune.nocopy.dto.MessageCode; +import ru.soune.nocopy.dto.TokenProcessRequest; +import ru.soune.nocopy.exception.NotFoundAuthToken; +import ru.soune.nocopy.service.auth.AuthService; + +import java.util.Map; + +@Slf4j +@Component +@RequiredArgsConstructor +public class AuthRequestHandler implements RequestHandler { + + private final ObjectMapper objectMapper; + + private final AuthService authService; + + @Override + public BaseResponse handle(BaseRequest request) throws NotFoundAuthToken { + TokenProcessRequest tokenProcessRequest = objectMapper.convertValue(request.getMessageBody(), + TokenProcessRequest.class); + + authService.useUserAuthToken(tokenProcessRequest.getToken()); + + return new BaseResponse(request.getMsgId(), + MessageCode.SUCCESS.getCode(), + "Token processed successfully", + Map.of("token", tokenProcessRequest.getToken())); + } +} diff --git a/src/main/java/ru/soune/nocopy/handler/FileEntityHandler.java b/src/main/java/ru/soune/nocopy/handler/FileEntityHandler.java index cacf0eb..4676310 100644 --- a/src/main/java/ru/soune/nocopy/handler/FileEntityHandler.java +++ b/src/main/java/ru/soune/nocopy/handler/FileEntityHandler.java @@ -6,13 +6,12 @@ import lombok.extern.slf4j.Slf4j; import org.springframework.stereotype.Component; import ru.soune.nocopy.dto.*; import ru.soune.nocopy.dto.file.*; -import ru.soune.nocopy.entity.AuthToken; import ru.soune.nocopy.entity.file.FileEntity; import ru.soune.nocopy.entity.file.FileStatus; import ru.soune.nocopy.exception.FileEntityNotFoundException; import ru.soune.nocopy.exception.NotFoundAuthToken; -import ru.soune.nocopy.repository.AuthTokenRepository; import ru.soune.nocopy.repository.FileEntityRepository; +import ru.soune.nocopy.service.auth.AuthService; import ru.soune.nocopy.service.file.FileEntityService; import ru.soune.nocopy.service.file.FileStatsService; @@ -26,9 +25,9 @@ public class FileEntityHandler implements RequestHandler { private final FileEntityService fileEntityService; - private final FileStatsService fileStatsService; + private final AuthService authService; - private final AuthTokenRepository authTokenRepository; + private final FileStatsService fileStatsService; private final ObjectMapper objectMapper; @@ -62,7 +61,8 @@ public class FileEntityHandler implements RequestHandler { .action(action) .availableActions(Arrays.asList( "file_info", "file_by_session", "user_files", - "search_files", "storage_usage", "delete_file")) + "search_files", "storage_usage", "delete_file", + "user_files_info")) .build(); return new BaseResponse(request.getMsgId(), @@ -80,7 +80,7 @@ public class FileEntityHandler implements RequestHandler { } private BaseResponse handleGetFilesUserInfo(BaseRequest request, FileEntityRequest fileRequest) { - Long userId = getUserIdFromToken(fileRequest.getToken()); + Long userId = authService.useUserAuthToken(fileRequest.getToken()); FileInfoUserResponse userFileStats = fileStatsService.getUserFileStats(userId); return new BaseResponse(request.getMsgId(), @@ -91,7 +91,7 @@ public class FileEntityHandler implements RequestHandler { private BaseResponse handleGetFileInfo(BaseRequest request, FileEntityRequest fileRequest) { try { - Long userId = getUserIdFromToken(fileRequest.getToken()); + Long userId = authService.useUserAuthToken(fileRequest.getToken()); FileEntityResponse fileInfo = fileEntityService.getById(fileRequest.getFileId(), request.getVersion()); if (!fileInfo.getUserId().equals(userId)) { @@ -122,7 +122,7 @@ public class FileEntityHandler implements RequestHandler { private BaseResponse handleGetFileBySession(BaseRequest request, FileEntityRequest fileRequest) { try { - Long userId = getUserIdFromToken(fileRequest.getToken()); + Long userId = authService.useUserAuthToken(fileRequest.getToken()); FileEntityResponse fileInfo = fileEntityService.getByUploadSessionId(fileRequest.getUploadSessionId(), request.getVersion()); @@ -154,7 +154,7 @@ public class FileEntityHandler implements RequestHandler { private BaseResponse handleGetUserFiles(BaseRequest request, FileEntityRequest fileRequest) { try { - Long userId = getUserIdFromToken(fileRequest.getToken()); + Long userId = authService.useUserAuthToken(fileRequest.getToken()); int page = fileRequest.getPage() != null ? fileRequest.getPage() : 1; int pageSize = fileRequest.getPageSize() != null ? fileRequest.getPageSize() : 20; @@ -181,7 +181,7 @@ public class FileEntityHandler implements RequestHandler { private BaseResponse handleSearchFiles(BaseRequest request, FileEntityRequest fileRequest) { try { - Long userId = getUserIdFromToken(fileRequest.getToken()); + Long userId = authService.useUserAuthToken(fileRequest.getToken()); int page = fileRequest.getPage() != null ? fileRequest.getPage() : 1; int pageSize = fileRequest.getPageSize() != null ? fileRequest.getPageSize() : 20; @@ -228,7 +228,7 @@ public class FileEntityHandler implements RequestHandler { private BaseResponse handleGetStorageUsage(BaseRequest request, FileEntityRequest fileRequest) { try { - Long userId = getUserIdFromToken(fileRequest.getToken()); + Long userId = authService.useUserAuthToken(fileRequest.getToken()); long usage = fileEntityService.getUserStorageUsed(userId); StorageUsageResponse response = StorageUsageResponse.builder() @@ -258,7 +258,7 @@ public class FileEntityHandler implements RequestHandler { private BaseResponse handleDeleteFile(BaseRequest request, FileEntityRequest fileRequest) { try { - Long userId = getUserIdFromToken(fileRequest.getToken()); + Long userId = authService.useUserAuthToken(fileRequest.getToken()); String fileId = fileRequest.getFileId(); FileEntityResponse fileInfo = fileEntityService.getById(fileId, request.getVersion()); DeleteFileResponse response; @@ -305,20 +305,6 @@ public class FileEntityHandler implements RequestHandler { } } - private Long getUserIdFromToken(String token) { - if (token == null || token.isBlank()) { - throw new NotFoundAuthToken("Token is required"); - } - - if (token.startsWith("Bearer ")) { - token = token.replace("Bearer ", ""); - } - - AuthToken authToken = authTokenRepository.findByToken(token) - .orElseThrow(() -> new NotFoundAuthToken("Token not found")); - return authToken.getUser().getId(); - } - private String formatFileSize(long size) { if (size < 1024) return size + " B"; int exp = (int) (Math.log(size) / Math.log(1024)); diff --git a/src/main/java/ru/soune/nocopy/handler/FileUploadHandler.java b/src/main/java/ru/soune/nocopy/handler/FileUploadHandler.java index 4cb33b5..7b98da3 100644 --- a/src/main/java/ru/soune/nocopy/handler/FileUploadHandler.java +++ b/src/main/java/ru/soune/nocopy/handler/FileUploadHandler.java @@ -17,6 +17,7 @@ import ru.soune.nocopy.exception.*; import ru.soune.nocopy.handler.validator.FileUploadRequestValidator; import ru.soune.nocopy.repository.AuthTokenRepository; import ru.soune.nocopy.repository.FileUploadSessionRepository; +import ru.soune.nocopy.service.auth.AuthService; import ru.soune.nocopy.service.file.FileUploadService; import java.util.*; @@ -28,6 +29,8 @@ import java.util.stream.Collectors; public class FileUploadHandler implements RequestHandler { private final FileUploadService fileUploadService; + private final AuthService authService; + private final AuthTokenRepository authTokenRepository; private final FileUploadSessionRepository fileUploadSessionRepository; @@ -86,10 +89,9 @@ public class FileUploadHandler implements RequestHandler { MessageCode.INVALID_TOKEN.getDescription(), Map.of("token", token)); } - AuthToken authToken = tokenOptional.orElseThrow(() -> new NotFoundAuthToken("Token not found")); - + Long userId = authService.useUserAuthToken(token); BindingResult bindingResult = new BeanPropertyBindingResult(fileRequest, "fileRequest"); - fileUploadRequestValidator.validate(fileRequest, bindingResult); + fileUploadRequestValidator.validate(fileRequest, bindingResult); if (bindingResult.hasErrors()) { Map fieldErrors = bindingResult.getFieldErrors() @@ -105,7 +107,7 @@ public class FileUploadHandler implements RequestHandler { } FileUploadSession session = fileUploadService.initUpload( - authToken.getUser().getId(), + userId, fileRequest.getFileName(), fileRequest.getFileType(), fileRequest.getExtension(), @@ -175,40 +177,13 @@ public class FileUploadHandler implements RequestHandler { } } - private BaseResponse handleRetryUpload(BaseRequest request, FileUploadRequest fileRequest) { - try { - fileUploadService.retryFailedUpload(fileRequest.getUploadId()); - ru.soune.nocopy.dto.file.UploadProgressResponse progress = - fileUploadService.getUploadProgress(fileRequest.getUploadId()); - - RetryUploadResponse response = RetryUploadResponse.builder() - .uploadId(progress.getUploadId()) - .message("Upload retry initiated") - .status(progress.getStatus().toString()) - .build(); - - return new BaseResponse(request.getMsgId(), MessageCode.SUCCESS.getCode(), "Upload retry initiated", - response); - - } catch (UploadSessionNotFoundException e) { - return new BaseResponse(request.getMsgId(), MessageCode.FILE_NOT_FOUND.getCode(), - "Upload session not found", null); - } catch (FileUploadException e) { - return new BaseResponse(request.getMsgId(), MessageCode.FILE_UPLOAD_ERROR.getCode(), e.getMessage(), - null); - } catch (Exception e) { - log.error("Error retrying upload", e); - return new BaseResponse(request.getMsgId(), MessageCode.FILE_UPLOAD_ERROR.getCode(), - "Failed to retry upload", null); - } - } - private BaseResponse handleGetChunkStatus(BaseRequest request, FileUploadRequest fileRequest) { try { FileUploadSession session = fileUploadSessionRepository.findById(fileRequest.getUploadId()) .orElseThrow(() -> new UploadSessionNotFoundException(fileRequest.getUploadId())); Map chunkStatus = new HashMap<>(); + for (int i = 0; i < session.getTotalChunks(); i++) { chunkStatus.put("chunk_" + i, session.getChunkPaths().containsKey(i)); } @@ -223,7 +198,6 @@ public class FileUploadHandler implements RequestHandler { return new BaseResponse(request.getMsgId(), MessageCode.SUCCESS.getCode(), MessageCode.SUCCESS.getDescription(), response); - } catch (UploadSessionNotFoundException e) { return new BaseResponse(request.getMsgId(), MessageCode.FILE_NOT_FOUND.getCode(), "Upload session not found", null); diff --git a/src/main/java/ru/soune/nocopy/handler/GlobalExceptionHandler.java b/src/main/java/ru/soune/nocopy/handler/GlobalExceptionHandler.java index 071b9ae..7303fb5 100644 --- a/src/main/java/ru/soune/nocopy/handler/GlobalExceptionHandler.java +++ b/src/main/java/ru/soune/nocopy/handler/GlobalExceptionHandler.java @@ -5,15 +5,16 @@ import org.springframework.context.support.DefaultMessageSourceResolvable; import org.springframework.http.HttpStatus; import org.springframework.http.ResponseEntity; import org.springframework.web.bind.MethodArgumentNotValidException; -import org.springframework.web.bind.annotation.ExceptionHandler; -import org.springframework.web.bind.annotation.ResponseStatus; -import org.springframework.web.bind.annotation.RestControllerAdvice; +import org.springframework.web.bind.annotation.*; +import ru.soune.nocopy.dto.BaseResponse; +import ru.soune.nocopy.dto.MessageCode; import ru.soune.nocopy.exception.*; import java.util.Map; @RestControllerAdvice @AllArgsConstructor +@ControllerAdvice public class GlobalExceptionHandler { @ExceptionHandler(MethodArgumentNotValidException.class) @@ -81,4 +82,12 @@ public class GlobalExceptionHandler { "message" ,ex.getMessage() )); } + + @ExceptionHandler(DuplicateImageException.class) + @ResponseBody + public ResponseEntity handleDuplicateImage(DuplicateImageException e) { + return ResponseEntity.ok().body(new BaseResponse( + 20004, MessageCode.DUPLICATE_FILE_UPLOAD.getCode(), + "Duplicate image detected", e.getMessage())); + } } diff --git a/src/main/java/ru/soune/nocopy/handler/ImageFoundRequestHandler.java b/src/main/java/ru/soune/nocopy/handler/ImageFoundRequestHandler.java new file mode 100644 index 0000000..3d13d4a --- /dev/null +++ b/src/main/java/ru/soune/nocopy/handler/ImageFoundRequestHandler.java @@ -0,0 +1,36 @@ +package ru.soune.nocopy.handler; + +import com.fasterxml.jackson.databind.ObjectMapper; +import lombok.RequiredArgsConstructor; +import lombok.extern.slf4j.Slf4j; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.stereotype.Component; +import ru.soune.nocopy.dto.BaseRequest; +import ru.soune.nocopy.dto.BaseResponse; +import ru.soune.nocopy.dto.MessageCode; +import ru.soune.nocopy.dto.file.ImageSearchRequest; +import ru.soune.nocopy.dto.file.YandexSearchResponse; +import ru.soune.nocopy.service.YandexSearchService; + +@Slf4j +@Component +@RequiredArgsConstructor +public class ImageFoundRequestHandler implements RequestHandler { + + private final ObjectMapper objectMapper; + + private final YandexSearchService yandexSearchService; + + @Override + public BaseResponse handle(BaseRequest request) throws Exception { + ImageSearchRequest imageSearchRequest = objectMapper.convertValue(request.getMessageBody(), + ImageSearchRequest.class); + + String fileId = imageSearchRequest.getFileId(); + + YandexSearchResponse response = yandexSearchService.searchByFileEntity(fileId); + + return new BaseResponse(request.getMsgId(), MessageCode.SUCCESS.getCode(), + MessageCode.SUCCESS.getDescription(), response); + } +} diff --git a/src/main/java/ru/soune/nocopy/handler/LoginRequestHandler.java b/src/main/java/ru/soune/nocopy/handler/LoginRequestHandler.java index 95abf63..6add2a0 100644 --- a/src/main/java/ru/soune/nocopy/handler/LoginRequestHandler.java +++ b/src/main/java/ru/soune/nocopy/handler/LoginRequestHandler.java @@ -7,7 +7,7 @@ import ru.soune.nocopy.dto.*; import ru.soune.nocopy.entity.AuthToken; import ru.soune.nocopy.exception.NotValidFieldException; import ru.soune.nocopy.repository.UserRepository; -import ru.soune.nocopy.service.AuthService; +import ru.soune.nocopy.service.auth.AuthService; import java.util.Arrays; import java.util.Map; @@ -17,7 +17,9 @@ import java.util.Map; public class LoginRequestHandler implements RequestHandler { private final UserRepository userRepository; + private final AuthService authService; + private final ObjectMapper objectMapper; @Override @@ -34,9 +36,12 @@ public class LoginRequestHandler implements RequestHandler { } AuthToken authToken = authService.login(loginRequest); + String token = authToken.getToken(); LoginAnswer loginAnswer = new LoginAnswer(); - loginAnswer.setToken(authToken.getToken()); + loginAnswer.setToken(token); + + authService.useUserAuthToken(token); return new BaseResponse(request.getMsgId(), MessageCode.SUCCESS.getCode(), MessageCode.SUCCESS.getDescription(), loginAnswer); diff --git a/src/main/java/ru/soune/nocopy/handler/LogoutRequestHandler.java b/src/main/java/ru/soune/nocopy/handler/LogoutRequestHandler.java index 44fa253..e12284c 100644 --- a/src/main/java/ru/soune/nocopy/handler/LogoutRequestHandler.java +++ b/src/main/java/ru/soune/nocopy/handler/LogoutRequestHandler.java @@ -9,7 +9,7 @@ import ru.soune.nocopy.entity.User; import ru.soune.nocopy.exception.NotValidFieldException; import ru.soune.nocopy.repository.AuthTokenRepository; import ru.soune.nocopy.repository.UserRepository; -import ru.soune.nocopy.service.AuthService; +import ru.soune.nocopy.service.auth.AuthService; import java.util.Objects; import java.util.Optional; diff --git a/src/main/java/ru/soune/nocopy/handler/RegRequestHandler.java b/src/main/java/ru/soune/nocopy/handler/RegRequestHandler.java index 349f0a7..86fcbc7 100644 --- a/src/main/java/ru/soune/nocopy/handler/RegRequestHandler.java +++ b/src/main/java/ru/soune/nocopy/handler/RegRequestHandler.java @@ -12,7 +12,7 @@ import ru.soune.nocopy.exception.NotValidFieldException; import ru.soune.nocopy.exception.ValidationException; import ru.soune.nocopy.handler.validator.RegRequestValidator; import ru.soune.nocopy.repository.UserRepository; -import ru.soune.nocopy.service.AuthService; +import ru.soune.nocopy.service.auth.AuthService; import java.util.Arrays; import java.util.Map; @@ -52,9 +52,11 @@ public class RegRequestHandler implements RequestHandler { } AuthToken authToken = authService.register(regRequest); + String token = authToken.getToken(); + authService.useUserAuthToken(token); RegAnswer regAnswer = new RegAnswer(); - regAnswer.setToken(authToken.getToken()); + regAnswer.setToken(token); return new BaseResponse(request.getMsgId(), MessageCode.SUCCESS.getCode(), MessageCode.SUCCESS.getDescription(), regAnswer); diff --git a/src/main/java/ru/soune/nocopy/handler/validator/RegRequestValidator.java b/src/main/java/ru/soune/nocopy/handler/validator/RegRequestValidator.java index 9b7ca63..22e4b62 100644 --- a/src/main/java/ru/soune/nocopy/handler/validator/RegRequestValidator.java +++ b/src/main/java/ru/soune/nocopy/handler/validator/RegRequestValidator.java @@ -10,7 +10,7 @@ import java.util.*; @Component public class RegRequestValidator implements Validator { - private static final String COMPANY_REGEX = "^[a-zA-Zа-яА-ЯёЁ0-9\\s\\-&.,'()]{0,200}$"; + private static final String COMPANY_REGEX = "^[a-zA-Zа-яА-ЯёЁ0-9\\s\\-&.,'()\"\"«»„‟”“”‘'’‹›❛❜❝❞〝〞〟]{1,200}$"; private static final String NAME_REGEX = "^[a-zA-Zа-яА-ЯёЁ\\s\\-'.]{2,100}$"; @@ -71,21 +71,6 @@ public class RegRequestValidator implements Validator { errors.rejectValue("companyName", "companyName.invalid.chars", "Company name contains invalid characters"); } - - validateForbiddenWords(trimmedCompany, "companyName", errors, - Arrays.asList("admin", "root", "system", "test")); - } - - private void validateForbiddenWords(String text, String fieldName, - Errors errors, List forbiddenWords) { - String lowerText = text.toLowerCase(); - for (String word : forbiddenWords) { - if (lowerText.contains(word)) { - errors.rejectValue(fieldName, fieldName + ".forbidden.word", - "Contains forbidden word: " + word); - break; - } - } } private void validateEmail(String email, Errors errors) { diff --git a/src/main/java/ru/soune/nocopy/repository/AuthTokenRepository.java b/src/main/java/ru/soune/nocopy/repository/AuthTokenRepository.java index 74826fd..cbd7529 100644 --- a/src/main/java/ru/soune/nocopy/repository/AuthTokenRepository.java +++ b/src/main/java/ru/soune/nocopy/repository/AuthTokenRepository.java @@ -4,11 +4,13 @@ import org.springframework.data.jpa.repository.JpaRepository; import org.springframework.stereotype.Repository; import ru.soune.nocopy.entity.AuthToken; -import java.time.LocalDate; +import java.time.LocalDateTime; +import java.util.List; import java.util.Optional; @Repository public interface AuthTokenRepository extends JpaRepository { - Optional findByTokenAndExpiresAtAfter(String token, LocalDate expiresAtAfter); + List findByExpiresAtBefore(LocalDateTime expiresAtBefore); + Optional findByLastUsedAtBefore(LocalDateTime lastUsedAt); Optional findByToken(String token); } diff --git a/src/main/java/ru/soune/nocopy/repository/FileEntityRepository.java b/src/main/java/ru/soune/nocopy/repository/FileEntityRepository.java index d806273..abfb7d8 100644 --- a/src/main/java/ru/soune/nocopy/repository/FileEntityRepository.java +++ b/src/main/java/ru/soune/nocopy/repository/FileEntityRepository.java @@ -15,6 +15,7 @@ import java.util.Optional; public interface FileEntityRepository extends JpaRepository { List findByUserId(Long userId); + Optional findByUserIdAndChecksum(Long userId, String imageHash); List findByUserIdAndStatus(Long userId, FileStatus status); @@ -36,5 +37,8 @@ public interface FileEntityRepository extends JpaRepository @Query("SELECT f FROM FileEntity f WHERE f.userId = :userId AND f.originalFileName LIKE %:keyword%") List searchByFileName(@Param("userId") Long userId, @Param("keyword") String keyword); + @Query("SELECT f FROM FileEntity f WHERE f.status = :status") + List searchFileEntityByStatus(@Param("status") FileStatus status); + long countByUserId(Long userId); } diff --git a/src/main/java/ru/soune/nocopy/repository/ImageHashRepository.java b/src/main/java/ru/soune/nocopy/repository/ImageHashRepository.java new file mode 100644 index 0000000..ee0aac2 --- /dev/null +++ b/src/main/java/ru/soune/nocopy/repository/ImageHashRepository.java @@ -0,0 +1,10 @@ +package ru.soune.nocopy.repository; + +import org.springframework.data.jpa.repository.JpaRepository; +import org.springframework.stereotype.Repository; +import ru.soune.nocopy.entity.file.ImageHashEntity; + +@Repository +public interface ImageHashRepository extends JpaRepository { + void deleteByFileId(String fileId); +} diff --git a/src/main/java/ru/soune/nocopy/repository/ImageSimilarityRepository.java b/src/main/java/ru/soune/nocopy/repository/ImageSimilarityRepository.java new file mode 100644 index 0000000..a6ef973 --- /dev/null +++ b/src/main/java/ru/soune/nocopy/repository/ImageSimilarityRepository.java @@ -0,0 +1,54 @@ +package ru.soune.nocopy.repository; + +import org.springframework.data.jpa.repository.JpaRepository; +import org.springframework.data.jpa.repository.Query; +import org.springframework.data.repository.query.Param; +import org.springframework.stereotype.Repository; +import ru.soune.nocopy.entity.file.FileEntity; + +import java.util.List; + +@Repository +public interface ImageSimilarityRepository + extends JpaRepository { + + @Query(value = """ + + SELECT + f.id AS similarFileId, + f.original_file_name AS originalFileName, + f.file_size AS fileSize, + h.hash64_hi AS hash64Hi, + h.hash64_lo AS hash64Lo + FROM image_hashes ref + JOIN image_hashes h + ON ref.file_id <> h.file_id + JOIN file_entities f + ON f.id = h.file_id + WHERE ref.file_id = :fileId + """, + nativeQuery = true) + List findCandidates( + @Param("fileId") String fileId + ); + + @Query(value = """ + SELECT + h.hash64_hi AS hash64Hi, + h.hash64_lo AS hash64Lo, + h.file_id AS fileId, + f.user_id AS userId, + f.original_file_name AS originalFileName, + f.file_size AS fileSize, + f.stored_file_name AS similarFileId + FROM image_hashes h + JOIN file_entities f ON f.id = h.file_id + WHERE h.hash64_hi = :hash64Hi + AND h.hash64_lo = :hash64Lo + """, + nativeQuery = true) + List findExactDuplicates( + @Param("hash64Hi") Integer hash64_hi, + @Param("hash64Lo") Integer hash64_lo); +} + diff --git a/src/main/java/ru/soune/nocopy/repository/SimilarImageProjection.java b/src/main/java/ru/soune/nocopy/repository/SimilarImageProjection.java new file mode 100644 index 0000000..ed2fdb4 --- /dev/null +++ b/src/main/java/ru/soune/nocopy/repository/SimilarImageProjection.java @@ -0,0 +1,10 @@ +package ru.soune.nocopy.repository; + +public interface SimilarImageProjection { + Integer getHash64Hi(); + Integer getHash64Lo(); + String getFileId(); + Long getUserId(); + String getOriginalFileName(); + Long getFileSize(); +} \ No newline at end of file diff --git a/src/main/java/ru/soune/nocopy/service/FileSimilarityService.java b/src/main/java/ru/soune/nocopy/service/FileSimilarityService.java new file mode 100644 index 0000000..dea9334 --- /dev/null +++ b/src/main/java/ru/soune/nocopy/service/FileSimilarityService.java @@ -0,0 +1,135 @@ +package ru.soune.nocopy.service; + +import lombok.RequiredArgsConstructor; +import org.springframework.data.domain.Page; +import org.springframework.data.domain.PageImpl; +import org.springframework.data.domain.Pageable; +import org.springframework.stereotype.Service; +import ru.soune.nocopy.dto.file.SimilarFileResponse; +import ru.soune.nocopy.dto.file.SimilarityFilter; +import ru.soune.nocopy.repository.ImageHashRepository; +import ru.soune.nocopy.repository.ImageSimilarityRepository; +import ru.soune.nocopy.repository.SimilarImageProjection; + +import java.util.ArrayList; +import java.util.Comparator; +import java.util.List; +import java.util.stream.Collectors; + +@Service +@RequiredArgsConstructor +public class FileSimilarityService { + private final ImageSimilarityRepository repository; + + private final ImageHashRepository hashRepository; + + public List findSimilarFiles(String fileId) { + var imageHashEntity = hashRepository.findById(fileId) + .orElseThrow(() -> new RuntimeException("Hash not found")); + + Integer hash64Hi = imageHashEntity.getHash64Hi(); + Integer hash64Lo = imageHashEntity.getHash64Lo(); + + List candidates = + repository.findCandidates(fileId); + + return candidates.stream() + .map(c -> { + Integer cHi = c.getHash64Hi(); + Integer cLo = c.getHash64Lo(); + + int hamming = hamming64(hash64Hi, hash64Lo, cHi, cLo); + + String level; + if (hamming <= 5) { + level = "DUPLICATE"; + } else if (hamming <= 12) { + level = "SIMILAR"; + } else { + level = "DIFFERENT"; + } + + return SimilarFileResponse.builder() + .fileId(c.getFileId()) + .originalFileName(c.getOriginalFileName()) + .fileSize(c.getFileSize()) + .hammingDistance(hamming) + .similarityLevel(level) + .build(); + }) + .sorted((a, b) -> + Integer.compare(a.getHammingDistance(), b.getHammingDistance())) + .toList(); + } + + public List findDuplicatedByHash(Integer hash64Hi, Integer hash64Lo) { + List duplicates = repository.findExactDuplicates(hash64Hi,hash64Lo); + + if (duplicates.isEmpty()) { + return new ArrayList<>(); + } + + return duplicates; + } + + public Page findSimilarFiles(String fileId, SimilarityFilter filter, Pageable pageable) { + var imageHashEntity = hashRepository.findById(fileId) + .orElseThrow(() -> new RuntimeException("Hash not found")); + + Integer hash64Hi = imageHashEntity.getHash64Hi(); + Integer hash64Lo = imageHashEntity.getHash64Lo(); + + List candidates = repository.findCandidates(fileId); + + List similarityLevels = (filter != null && filter.getSimilarityLevels() != null) + ? filter.getSimilarityLevels() + : List.of("DUPLICATE", "SIMILAR", "DIFFERENT"); + + List allResults = candidates.stream() + .map(c -> createSimilarFileResponse(c, hash64Hi, hash64Lo)) + .filter(response -> similarityLevels.contains(response.getSimilarityLevel())) + .sorted(Comparator.comparingInt(SimilarFileResponse::getHammingDistance)) + .collect(Collectors.toList()); + + int total = allResults.size(); + int page = (pageable != null) ? pageable.getPageNumber() : 0; + int size = (pageable != null) ? pageable.getPageSize() : 20; + + int fromIndex = Math.min(page * size, total); + int toIndex = Math.min(fromIndex + size, total); + + List pageContent = allResults.subList(fromIndex, toIndex); + + return new PageImpl<>(pageContent, pageable, total); + } + + private SimilarFileResponse createSimilarFileResponse(SimilarImageProjection similarImageProjection, + Integer hash64Hi, Integer hash64Lo) { + Integer imageProjectionHash64Hi = similarImageProjection.getHash64Hi(); + Integer similarImageProjectionHash64Lo = similarImageProjection.getHash64Lo(); + + int hamming = hamming64(hash64Hi, hash64Lo, imageProjectionHash64Hi, similarImageProjectionHash64Lo); + + String level; + if (hamming <= 5) { + level = "DUPLICATE"; + } else if (hamming <= 12) { + level = "SIMILAR"; + } else { + level = "DIFFERENT"; + } + + return SimilarFileResponse.builder() + .fileId(similarImageProjection.getFileId()) + .originalFileName(similarImageProjection.getOriginalFileName()) + .fileSize(similarImageProjection.getFileSize()) + .hammingDistance(hamming) + .similarityLevel(level) + .build(); + } + + private int hamming64(int aHi, int aLo, int bHi, int bLo) { + return Integer.bitCount(aHi ^ bHi) + + Integer.bitCount(aLo ^ bLo); + } +} diff --git a/src/main/java/ru/soune/nocopy/service/ImageHashService.java b/src/main/java/ru/soune/nocopy/service/ImageHashService.java new file mode 100644 index 0000000..b7b6882 --- /dev/null +++ b/src/main/java/ru/soune/nocopy/service/ImageHashService.java @@ -0,0 +1,154 @@ +package ru.soune.nocopy.service; + +import lombok.RequiredArgsConstructor; +import lombok.extern.slf4j.Slf4j; +import org.springframework.stereotype.Service; +import ru.soune.nocopy.entity.file.FileEntity; +import ru.soune.nocopy.entity.file.ImageHashEntity; +import ru.soune.nocopy.repository.ImageHashRepository; + +import javax.imageio.ImageIO; +import java.awt.*; +import java.awt.image.BufferedImage; +import java.io.IOException; +import java.io.InputStream; +import java.nio.file.Files; +import java.nio.file.Path; +import java.time.LocalDateTime; +import java.util.Arrays; +import java.util.Map; + +@Service +@RequiredArgsConstructor +@Slf4j +public class ImageHashService { + + private final ImageHashRepository repository; + + public Map calculateHash(Path imagePath) throws IOException { + long hash64 = computePhash64(imagePath); + + return Map.of( + "hi", high32(hash64), + "low", low32(hash64)); + } + + public void create(FileEntity file, Map stringIntegerMap) { + ImageHashEntity entity = ImageHashEntity.builder() + .file(file) + .hash64Hi(stringIntegerMap.get("hi")) + .hash64Lo(stringIntegerMap.get("low")) + .hashAlgorithm("PHASH64") + .createdAt(LocalDateTime.now()) + .build(); + + repository.save(entity); + } + + private long computePhash64(Path path) throws IOException { + try (InputStream is = Files.newInputStream(path)) { + BufferedImage image = ImageIO.read(is); + if (image == null) throw new IOException("Cannot read image file: " + path); + + BufferedImage gray = toGrayScale(image); + BufferedImage resized = resizeImage(gray, 32, 32); + + double[][] dct = applyDCT(resized); + double[][] topLeft = extractTopLeft8x8(dct); + double median = calculateMedian(topLeft, true); + + return create64BitHashLong(topLeft, median); + } + } + + private BufferedImage toGrayScale(BufferedImage image) { + BufferedImage gray = new BufferedImage(image.getWidth(), image.getHeight(), BufferedImage.TYPE_BYTE_GRAY); + Graphics2D g = gray.createGraphics(); + g.drawImage(image, 0, 0, null); + g.dispose(); + return gray; + } + + private BufferedImage resizeImage(BufferedImage image, int width, int height) { + BufferedImage resized = new BufferedImage(width, height, BufferedImage.TYPE_BYTE_GRAY); + Graphics2D g = resized.createGraphics(); + g.setRenderingHint(RenderingHints.KEY_INTERPOLATION, RenderingHints.VALUE_INTERPOLATION_BILINEAR); + g.drawImage(image, 0, 0, width, height, null); + g.dispose(); + return resized; + } + + private double[][] applyDCT(BufferedImage image) { + int w = image.getWidth(); + int h = image.getHeight(); + double[][] pixels = new double[h][w]; + + for (int y = 0; y < h; y++) { + for (int x = 0; x < w; x++) { + int gray = image.getRGB(x, y) & 0xFF; + pixels[y][x] = gray; + } + } + + double[][] dct = new double[h][w]; + for (int u = 0; u < h; u++) { + double au = u == 0 ? 1.0 / Math.sqrt(h) : Math.sqrt(2.0 / h); + for (int v = 0; v < w; v++) { + double av = v == 0 ? 1.0 / Math.sqrt(w) : Math.sqrt(2.0 / w); + double sum = 0.0; + for (int x = 0; x < h; x++) { + for (int y = 0; y < w; y++) { + sum += pixels[x][y] * + Math.cos((2 * x + 1) * u * Math.PI / (2.0 * h)) * + Math.cos((2 * y + 1) * v * Math.PI / (2.0 * w)); + } + } + dct[u][v] = au * av * sum; + } + } + return dct; + } + + private double[][] extractTopLeft8x8(double[][] dct) { + double[][] block = new double[8][8]; + for (int i = 0; i < 8; i++) System.arraycopy(dct[i], 0, block[i], 0, 8); + return block; + } + + private double calculateMedian(double[][] block, boolean excludeDC) { + double[] vals = new double[63]; + int idx = 0; + for (int i = 0; i < 8; i++) { + for (int j = 0; j < 8; j++) { + if (i == 0 && j == 0 && excludeDC) continue; + vals[idx++] = block[i][j]; + } + } + Arrays.sort(vals); + return (vals.length % 2 == 0) ? + (vals[vals.length / 2 - 1] + vals[vals.length / 2]) / 2.0 : + vals[vals.length / 2]; + } + + private long create64BitHashLong(double[][] block, double median) { + long hash = 0L; + + for (int i = 0; i < 8; i++) { + for (int j = 0; j < 8; j++) { + if (i == 0 && j == 0) continue; + hash <<= 1; + if (block[i][j] > median) hash |= 1; + } + } + + return hash; + } + + private int high32(long hash64) { + return (int) (hash64 >>> 32); + } + + private int low32(long hash64) { + return (int) hash64; + } +} diff --git a/src/main/java/ru/soune/nocopy/service/YandexSearchService.java b/src/main/java/ru/soune/nocopy/service/YandexSearchService.java new file mode 100644 index 0000000..d0f4710 --- /dev/null +++ b/src/main/java/ru/soune/nocopy/service/YandexSearchService.java @@ -0,0 +1,162 @@ +package ru.soune.nocopy.service; + +import com.fasterxml.jackson.core.JsonProcessingException; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.fasterxml.jackson.databind.DeserializationFeature; +import jakarta.annotation.PostConstruct; +import lombok.RequiredArgsConstructor; +import lombok.extern.slf4j.Slf4j; +import org.springframework.beans.factory.annotation.Value; +import org.springframework.stereotype.Service; +import ru.soune.nocopy.dto.BaseResponse; +import ru.soune.nocopy.dto.MessageCode; +import ru.soune.nocopy.dto.file.YandexSearchResponse; +import ru.soune.nocopy.entity.file.FileEntity; +import ru.soune.nocopy.exception.FileEntityNotFoundException; +import ru.soune.nocopy.exception.NotValidFieldException; +import ru.soune.nocopy.repository.FileEntityRepository; + +import java.io.*; +import java.net.HttpURLConnection; +import java.net.URL; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.Base64; +import java.util.Map; + +@Slf4j +@Service +@RequiredArgsConstructor +public class YandexSearchService { + private final FileEntityRepository fileEntityRepository; + + private final ObjectMapper objectMapper; + + @Value("${yandex.api-key}") + private String apiKey; + + @Value("${yandex.folder-id}") + private String folderId; + + @Value("${yandex.search-url}") + private String searchUrl; + + @PostConstruct + public void init() { + objectMapper.configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, false); + objectMapper.configure(DeserializationFeature.ACCEPT_SINGLE_VALUE_AS_ARRAY, true); + } + + public YandexSearchResponse searchByFileEntity(String fileId) throws IOException { + FileEntity fileEntity = fileEntityRepository.findById(fileId) + .orElseThrow(() -> { + throw new NotValidFieldException("File not found", new BaseResponse(20007, + MessageCode.FILE_NOT_FOUND.getCode(), MessageCode.FILE_NOT_FOUND.getDescription(), + Map.of("fileId",fileId))); + }); + byte[] fileBytes; + + if (!isImageFile(fileEntity)) { + log.error("File not image: {}", fileEntity.getMimeType()); + throw new NotValidFieldException("File not image", new BaseResponse(20007, + MessageCode.INVALID_FIELD.getCode(), MessageCode.INVALID_FIELD.getDescription(), + Map.of("file_type", fileEntity.getMimeType()))); + } + + try { + fileBytes = readFileFromDisk(fileEntity); + } catch (IOException e) { + throw new NotValidFieldException("File not found or cannot read file", new BaseResponse(20007, + MessageCode.FILE_NOT_FOUND.getCode(), MessageCode.FILE_NOT_FOUND.getDescription(), + Map.of("fileId", fileId, + "filePath", fileEntity.getFilePath()))); + } + + return callYandexApi(fileBytes); + } + + private boolean isImageFile(FileEntity fileEntity) { + String mimeType = fileEntity.getMimeType(); + return mimeType != null && mimeType.startsWith("image"); + } + + private byte[] readFileFromDisk(FileEntity fileEntity) throws IOException { + Path filePath = Path.of(fileEntity.getFilePath()); + + if (!Files.exists(filePath)) { + throw new IOException("File not found: " + fileEntity.getFilePath()); + } + + if (!Files.isReadable(filePath)) { + throw new IOException("Cannot read file: " + fileEntity.getFilePath()); + } + + return Files.readAllBytes(filePath); + } + + private YandexSearchResponse callYandexApi(byte[] imageBytes) throws IOException { + String imageBase64 = Base64.getEncoder().encodeToString(imageBytes); + String jsonRequest = String.format("{\"folderId\":\"%s\",\"data\":\"%s\",\"page\":0,\"showSimilarImages\":true}", + folderId, imageBase64); + + URL url = new URL(searchUrl); + HttpURLConnection connection = (HttpURLConnection) url.openConnection(); + + try { + connection.setRequestMethod("POST"); + connection.setRequestProperty("Authorization", "Api-Key " + apiKey); + connection.setRequestProperty("Content-Type", "application/json"); + connection.setRequestProperty("Accept", "application/json"); + connection.setConnectTimeout(30000); + connection.setReadTimeout(30000); + connection.setDoOutput(true); + + try (OutputStream os = connection.getOutputStream()) { + byte[] input = jsonRequest.getBytes("utf-8"); + os.write(input, 0, input.length); + os.flush(); + } + + int responseCode = connection.getResponseCode(); + + String responseBody; + if (responseCode == HttpURLConnection.HTTP_OK) { + try (BufferedReader br = new BufferedReader( + new InputStreamReader(connection.getInputStream(), "utf-8"))) { + responseBody = readAll(br); + } + } else { + try (BufferedReader br = new BufferedReader( + new InputStreamReader(connection.getErrorStream(), "utf-8"))) { + responseBody = readAll(br); + } + throw new IOException("Error Yandex API: " + responseCode + " - " + responseBody); + } + + return parseJsonResponse(responseBody); + + } finally { + connection.disconnect(); + } + } + + private String readAll(BufferedReader reader) throws IOException { + StringBuilder sb = new StringBuilder(); + String line; + while ((line = reader.readLine()) != null) { + sb.append(line); + } + return sb.toString(); + } + + private YandexSearchResponse parseJsonResponse(String json) { + YandexSearchResponse response = null; + try { + response = objectMapper.readValue(json, YandexSearchResponse.class); + } catch (JsonProcessingException e) { + throw new RuntimeException(e); + } + + return response; + } +} diff --git a/src/main/java/ru/soune/nocopy/service/AuthService.java b/src/main/java/ru/soune/nocopy/service/auth/AuthService.java similarity index 80% rename from src/main/java/ru/soune/nocopy/service/AuthService.java rename to src/main/java/ru/soune/nocopy/service/auth/AuthService.java index 2377b2b..f14d6d1 100644 --- a/src/main/java/ru/soune/nocopy/service/AuthService.java +++ b/src/main/java/ru/soune/nocopy/service/auth/AuthService.java @@ -1,13 +1,13 @@ -package ru.soune.nocopy.service; +package ru.soune.nocopy.service.auth; import lombok.RequiredArgsConstructor; -import org.springframework.context.MessageSource; import org.springframework.security.crypto.password.PasswordEncoder; import org.springframework.stereotype.Service; import org.springframework.transaction.annotation.Transactional; import ru.soune.nocopy.dto.*; import ru.soune.nocopy.entity.AuthToken; import ru.soune.nocopy.entity.User; +import ru.soune.nocopy.exception.NotFoundAuthToken; import ru.soune.nocopy.exception.NotValidFieldException; import ru.soune.nocopy.repository.AuthTokenRepository; import ru.soune.nocopy.repository.UserRepository; @@ -29,8 +29,6 @@ public class AuthService { private final PasswordEncoder passwordEncoder; - private final MessageSource messageSource; - private final SecureRandom secureRandom = new SecureRandom(); @Transactional @@ -81,6 +79,26 @@ public class AuthService { return authTokenRepository.save(authToken); } + @Transactional + public Long useUserAuthToken(String token) { + if (token == null || token.isBlank()) { + throw new NotFoundAuthToken("Token is required"); + } + + if (token.startsWith("Bearer ")) { + token = token.replace("Bearer ", ""); + } + + AuthToken authToken = authTokenRepository.findByToken(token) + .orElseThrow(() -> new NotFoundAuthToken("Token not found")); + authToken.setLastUsedAt(LocalDateTime.now()); + authToken.setExpiresAt(LocalDateTime.now().plusHours(1)); + + authTokenRepository.save(authToken); + + return authToken.getUser().getId(); + } + @Transactional public void logout(String token) { authTokenRepository.findByToken(token) diff --git a/src/main/java/ru/soune/nocopy/service/auth/CleanupTokenSessionsService.java b/src/main/java/ru/soune/nocopy/service/auth/CleanupTokenSessionsService.java new file mode 100644 index 0000000..5ac5128 --- /dev/null +++ b/src/main/java/ru/soune/nocopy/service/auth/CleanupTokenSessionsService.java @@ -0,0 +1,37 @@ +package ru.soune.nocopy.service.auth; + +import lombok.RequiredArgsConstructor; +import lombok.extern.slf4j.Slf4j; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.beans.factory.annotation.Value; +import org.springframework.scheduling.annotation.Scheduled; +import org.springframework.stereotype.Component; +import org.springframework.transaction.annotation.Transactional; +import ru.soune.nocopy.entity.AuthToken; +import ru.soune.nocopy.repository.AuthTokenRepository; + +import java.time.LocalDateTime; +import java.util.List; + +@Slf4j +@Component +@RequiredArgsConstructor +public class CleanupTokenSessionsService { + @Autowired + private AuthTokenRepository authTokenRepository; + + @Value("${file.storage.auth-token-life-hours}") + private int authTokenLife; + + @Transactional + @Scheduled(fixedDelay = 30000) + public void cleanupExpiredTokens() { + LocalDateTime now = LocalDateTime.now(); + + List expiredTokens = authTokenRepository.findByExpiresAtBefore(now); + + if (!expiredTokens.isEmpty()) { + authTokenRepository.deleteAll(expiredTokens); + } + } +} diff --git a/src/main/java/ru/soune/nocopy/service/file/FileEntityService.java b/src/main/java/ru/soune/nocopy/service/file/FileEntityService.java index 617eeda..1f3464e 100644 --- a/src/main/java/ru/soune/nocopy/service/file/FileEntityService.java +++ b/src/main/java/ru/soune/nocopy/service/file/FileEntityService.java @@ -9,16 +9,20 @@ import ru.soune.nocopy.dto.file.FileResponse; import ru.soune.nocopy.entity.file.FileEntity; import ru.soune.nocopy.entity.file.FileStatus; import ru.soune.nocopy.entity.file.FileUploadSession; +import ru.soune.nocopy.exception.DuplicateImageException; import ru.soune.nocopy.exception.FileEntityNotFoundException; import ru.soune.nocopy.repository.FileEntityRepository; +import ru.soune.nocopy.repository.SimilarImageProjection; +import ru.soune.nocopy.service.FileSimilarityService; +import ru.soune.nocopy.service.ImageHashService; import java.io.IOException; import java.nio.file.Files; import java.nio.file.Path; import java.nio.file.Paths; import java.time.LocalDateTime; -import java.time.ZoneOffset; import java.util.List; +import java.util.Map; import java.util.stream.Collectors; @Slf4j @@ -28,7 +32,11 @@ public class FileEntityService { private final FileEntityRepository fileEntityRepository; - @Transactional + private final ImageHashService imageHashService; + + private final FileSimilarityService fileSimilarityService; + + @Transactional(noRollbackFor = DuplicateImageException.class) public FileEntity createFromUploadSession(FileUploadSession session, String checksum) { log.info("Creating FileEntity for upload session: {}", session.getUploadId()); @@ -39,6 +47,21 @@ public class FileEntityService { throw new IOException("File not found on disk: " + filePath); } + Map imageHash = Map.of(); + + if (session.getFileType().startsWith("image")) { + imageHash = imageHashService.calculateHash(filePath); + List duplicatedByHash = fileSimilarityService.findDuplicatedByHash( + imageHash.get("hi"), imageHash.get("low")); + + if (!duplicatedByHash.isEmpty()) { + SimilarImageProjection similarImageProjection = duplicatedByHash.get(0); + + throw new DuplicateImageException("Duplicate", similarImageProjection.getFileId(), + similarImageProjection.getUserId()); + } + } + long fileSize = Files.size(filePath); String originalName = session.getFileName(); String storedName = filePath.getFileName().toString(); @@ -57,11 +80,12 @@ public class FileEntityService { .build(); FileEntity saved = fileEntityRepository.save(fileEntity); - log.info("FileEntity created successfully: {} (size: {} bytes)", - saved.getId(), fileSize); + + if (!imageHash.isEmpty()) { + imageHashService.create(saved, imageHash); + } return saved; - } catch (IOException e) { log.error("Failed to create FileEntity for session {}: {}", session.getUploadId(), e.getMessage(), e); @@ -69,15 +93,6 @@ public class FileEntityService { } } - - private String extractFileExtension(String fileName) { - int dotIndex = fileName.lastIndexOf('.'); - if (dotIndex > 0 && dotIndex < fileName.length() - 1) { - return fileName.substring(dotIndex + 1).toLowerCase(); - } - return ""; - } - @Transactional(readOnly = true) public FileEntityResponse getById(String fileId, int version) { FileEntity fileEntity = fileEntityRepository.findById(fileId) @@ -95,39 +110,6 @@ public class FileEntityService { return convertToResponse(fileEntity, version); } - @Transactional(readOnly = true) - public FileEntityResponse getByFilePath(String filePath, int version) { - FileEntity fileEntity = fileEntityRepository.findByFilePath(filePath) - .orElseThrow(() -> new FileEntityNotFoundException("Path: " + filePath)); - - return convertToResponse(fileEntity, version); - } - - - @Transactional(readOnly = true) - public FileResponse getAllUserFiles(Long userId, int version) { - List fileEntities = fileEntityRepository.findByUserIdAndStatus( - userId, FileStatus.ACTIVE); - - List files = fileEntities.stream() - .map(file -> convertToResponse(file, version)) - .collect(Collectors.toList()); - - long totalSize = fileEntities.stream() - .mapToLong(FileEntity::getFileSize) - .sum(); - - return FileResponse.builder() - .files(files) - .totalCount(files.size()) - .totalSize(totalSize) - .formattedTotalSize(formatFileSize(totalSize)) - .page(1) - .pageSize(files.size()) - .build(); - } - - @Transactional(readOnly = true) public FileResponse getUserFiles(Long userId, int page, int pageSize, int version) { List allFiles = fileEntityRepository.findByUserIdAndStatus( @@ -185,6 +167,8 @@ public class FileEntityService { Files.delete(path); + markAsDeleted(fileEntity); + return true; } @@ -223,6 +207,7 @@ public class FileEntityService { .formattedSize(formatFileSize(fileEntity.getFileSize())) .downloadUrl("/api/v" + version + "/files/download/" + fileEntity.getId()) .existsOnDisk(existsOnDisk) + .supportId(fileEntity.getSupportId()) .build(); } diff --git a/src/main/java/ru/soune/nocopy/service/file/FileUploadService.java b/src/main/java/ru/soune/nocopy/service/file/FileUploadService.java index b27dede..714ef21 100644 --- a/src/main/java/ru/soune/nocopy/service/file/FileUploadService.java +++ b/src/main/java/ru/soune/nocopy/service/file/FileUploadService.java @@ -17,7 +17,5 @@ public interface FileUploadService { void handleExpiredSession(FileUploadSession session); - public void retryFailedUpload(String uploadId); - public void cancelUpload(String uploadId); } diff --git a/src/main/java/ru/soune/nocopy/service/file/FileUploadServiceImpl.java b/src/main/java/ru/soune/nocopy/service/file/FileUploadServiceImpl.java index f54cec9..587cdc1 100644 --- a/src/main/java/ru/soune/nocopy/service/file/FileUploadServiceImpl.java +++ b/src/main/java/ru/soune/nocopy/service/file/FileUploadServiceImpl.java @@ -10,12 +10,19 @@ import org.springframework.stereotype.Service; import org.springframework.transaction.annotation.Transactional; import org.springframework.web.multipart.MultipartFile; import ru.soune.nocopy.dto.file.UploadProgressResponse; +import ru.soune.nocopy.entity.file.FileEntity; +import ru.soune.nocopy.entity.file.FileStatus; import ru.soune.nocopy.entity.file.FileUploadSession; import ru.soune.nocopy.entity.file.UploadStatus; import ru.soune.nocopy.exception.ChunkSizeExceededException; +import ru.soune.nocopy.exception.DuplicateImageException; import ru.soune.nocopy.exception.FileUploadException; import ru.soune.nocopy.exception.UploadSessionNotFoundException; +import ru.soune.nocopy.repository.FileEntityRepository; import ru.soune.nocopy.repository.FileUploadSessionRepository; +import ru.soune.nocopy.repository.SimilarImageProjection; +import ru.soune.nocopy.service.FileSimilarityService; +import ru.soune.nocopy.service.ImageHashService; import java.io.*; import java.nio.file.*; @@ -23,10 +30,7 @@ import java.nio.file.attribute.PosixFilePermission; import java.security.MessageDigest; import java.security.NoSuchAlgorithmException; import java.time.LocalDateTime; -import java.util.Comparator; -import java.util.HashSet; -import java.util.Set; -import java.util.UUID; +import java.util.*; import java.util.concurrent.CompletableFuture; import java.util.stream.Stream; @@ -57,6 +61,15 @@ public class FileUploadServiceImpl implements FileUploadService { private Path storageRoot; + @Autowired + private ImageHashService imageHashService; + + @Autowired + private FileSimilarityService fileSimilarityService; + + @Autowired + private FileEntityRepository fileEntityRepository; + @Autowired private FileEntityService fileEntityService; @@ -102,7 +115,7 @@ public class FileUploadServiceImpl implements FileUploadService { int totalChunks = (int) Math.ceil((double) fileSize / chunkSize); log.debug("File will be split into {} chunks (chunk size: {} bytes)", totalChunks, chunkSize); - Long chunkSize = totalChunks == 1 ? fileSize : 1048576L; + Long chunkSize = totalChunks == 1 ? fileSize : 1000000L; FileUploadSession session = FileUploadSession.builder() .userId(userId) @@ -129,10 +142,6 @@ public class FileUploadServiceImpl implements FileUploadService { @Transactional public UploadProgressResponse uploadChunk(String uploadId, Integer chunkNumber, MultipartFile chunkFile) { - - log.info("Processing chunk {} for session {}, size: {} bytes", - chunkNumber, uploadId, chunkFile.getSize()); - FileUploadSession session = sessionRepository.findById(uploadId) .orElseThrow(() -> new UploadSessionNotFoundException(uploadId)); @@ -173,12 +182,13 @@ public class FileUploadServiceImpl implements FileUploadService { @Async("fileUploadTaskExecutor") @Transactional - public void assembleFileAsync(FileUploadSession session) { + public void assembleFileAsync(FileUploadSession session) throws DuplicateImageException { try { assembleFile(session); log.info("File assembly completed successfully for session: {}", session.getUploadId()); - + } catch (DuplicateImageException e) { + throw new DuplicateImageException("DUBL", e.duplicateFileId(), e.userId()); } catch (Exception e) { log.error("Failed to assemble file for session {}: {}", session.getUploadId(), e.getMessage(), e); @@ -205,35 +215,6 @@ public class FileUploadServiceImpl implements FileUploadService { log.info("Upload cancelled: {}", uploadId); } - @Override - @Transactional - public void retryFailedUpload(String uploadId) { - FileUploadSession session = sessionRepository.findById(uploadId) - .orElseThrow(() -> new UploadSessionNotFoundException(uploadId)); - - if (session.getStatus() != UploadStatus.FAILED) { - throw new FileUploadException("Only failed uploads can be retried"); - } - - if (session.getRetryCount() >= maxRetryAttempts) { - throw new FileUploadException("Max retry attempts exceeded"); - } - - session.setStatus(UploadStatus.UPLOADING); - session.setRetryCount(session.getRetryCount() + 1); - session.setLastError(null); - - sessionRepository.save(session); - - if (session.getChunksUploaded().equals(session.getTotalChunks())) { - log.info("Retrying file assembly for session: {}", uploadId); - assembleFileAsync(session); - } - - log.info("Upload retry initiated for session: {} (attempt {})", - uploadId, session.getRetryCount()); - } - @Override public UploadProgressResponse getUploadProgress(String uploadId) { FileUploadSession session = sessionRepository.findById(uploadId) @@ -271,9 +252,42 @@ public class FileUploadServiceImpl implements FileUploadService { } } - private UploadProgressResponse processChunk(FileUploadSession session, - Integer chunkNumber, - MultipartFile chunkFile) { + @Async("fileUploadTaskExecutor") + @Transactional + public void completeFileProcessingAsync(FileUploadSession session) { + try { + Path filePath = Paths.get(session.getFilePath()); + String checksum = calculateChecksum(filePath); + + FileEntity fileEntity = FileEntity.builder() + .userId(session.getUserId()) + .originalFileName(session.getFileName()) + .storedFileName(filePath.getFileName().toString()) + .filePath(session.getFilePath()) + .fileSize(session.getFileSize()) + .mimeType(session.getFileType()) + .fileExtension(session.getExtension()) + .checksum(checksum) + .uploadSessionId(session.getUploadId()) + .status(FileStatus.ACTIVE) + .build(); + + FileEntity saved = fileEntityRepository.save(fileEntity); + + Map hash = imageHashService.calculateHash(filePath); + imageHashService.create(saved, hash); + + cleanupSessionFiles(session); + + log.info("File processing completed for session: {}", session.getUploadId()); + + } catch (Exception e) { + log.error("Failed to complete file processing for session {}: {}", + session.getUploadId(), e.getMessage(), e); + } + } + + private UploadProgressResponse processChunk(FileUploadSession session, Integer chunkNumber, MultipartFile chunkFile) { String chunkPath = null; try { @@ -291,17 +305,42 @@ public class FileUploadServiceImpl implements FileUploadService { log.debug("Chunk {} saved successfully. Uploaded: {}/{}", chunkNumber, session.getChunksUploaded(), session.getTotalChunks()); - if (session.getChunksUploaded().equals(session.getTotalChunks())) { + boolean isLastChunk = session.getChunksUploaded().equals(session.getTotalChunks()); + + if (isLastChunk) { log.info("All chunks uploaded for session {}. Starting assembly...", session.getUploadId()); - sessionRepository.save(session); - assembleFileAsync(session); + + String finalFilePath = assembleFileSynchronously(session); + + if (session.getFileType().startsWith("image")) { + checkForDuplicatesSynchronously(finalFilePath); + } + + session.setStatus(UploadStatus.COMPLETED); + session.setFilePath(finalFilePath); + + completeFileProcessingAsync(session); + } else { sessionRepository.save(session); } return UploadProgressResponse.fromSession(session); + } catch (DuplicateImageException e) { + if (chunkPath != null) { + cleanupFailedChunk(chunkPath); + } + + log.warn("Duplicate image found for session {}: {}", + session.getUploadId(), e.getMessage()); + + session.setStatus(UploadStatus.FAILED); + session.setLastError(e.getMessage()); + sessionRepository.save(session); + + throw e; } catch (Exception e) { if (chunkPath != null) { cleanupFailedChunk(chunkPath); @@ -318,6 +357,29 @@ public class FileUploadServiceImpl implements FileUploadService { } } + private String assembleFileSynchronously(FileUploadSession session) throws IOException { + Path finalFilePath = prepareFinalFile(session); + + validateAllChunksExist(session); + mergeChunksToFile(session, finalFilePath); + validateFinalFile(session, finalFilePath); + + return finalFilePath.toString(); + } + + private void checkForDuplicatesSynchronously(String filePath) + throws IOException { + Path path = Paths.get(filePath); + Map hash = imageHashService.calculateHash(path); + List duplicates = fileSimilarityService.findDuplicatedByHash( + hash.get("hi"), hash.get("low")); + + if (!duplicates.isEmpty()) { + throw new DuplicateImageException("Duplicate", duplicates.get(0).getFileId(), + duplicates.get(0).getUserId()); + } + } + private UploadProgressResponse handleExistingChunk(FileUploadSession session, Integer chunkNumber, MultipartFile chunkFile) throws IOException { @@ -325,25 +387,24 @@ public class FileUploadServiceImpl implements FileUploadService { Path chunkPath = Paths.get(existingPath); if (!Files.exists(chunkPath)) { - log.warn("Chunk file missing, re-uploading: {}", chunkPath); session.getChunkPaths().remove(chunkNumber); session.setChunksUploaded(session.getChunksUploaded() - 1); sessionRepository.save(session); + return processChunk(session, chunkNumber, chunkFile); } long existingSize = Files.size(chunkPath); + if (existingSize != chunkFile.getSize()) { - log.warn("Chunk size mismatch, re-uploading: {} != {}", - existingSize, chunkFile.getSize()); Files.deleteIfExists(chunkPath); session.getChunkPaths().remove(chunkNumber); session.setChunksUploaded(session.getChunksUploaded() - 1); sessionRepository.save(session); + return processChunk(session, chunkNumber, chunkFile); } - log.debug("Chunk {} already uploaded and valid", chunkNumber); return UploadProgressResponse.fromSession(session); } @@ -402,7 +463,7 @@ public class FileUploadServiceImpl implements FileUploadService { session.getUploadId(), session.getFileName()); Path finalFilePath = null; - String checksum = null; + String checksum; try { finalFilePath = prepareFinalFile(session); @@ -428,6 +489,8 @@ public class FileUploadServiceImpl implements FileUploadService { fileEntityService.createFromUploadSession(session, checksum); log.info("FileEntity successfully created for session: {}", session.getUploadId()); + } catch (DuplicateImageException e) { + throw new DuplicateImageException("Duplicate", e.duplicateFileId(), e.userId()); } catch (Exception e) { log.error("Failed to create FileEntity for session {} (file uploaded, but metadata not saved): {}", session.getUploadId(), e.getMessage()); @@ -591,44 +654,10 @@ public class FileUploadServiceImpl implements FileUploadService { private void handleAssemblyFailure(FileUploadSession session, Exception e) { session.setStatus(UploadStatus.FAILED); session.setLastError(e.getMessage()); + sessionRepository.save(session); - - log.error("File assembly failed for session {}: {}", - session.getUploadId(), e.getMessage()); } - - -// @Override -// @Transactional -// public void cleanupExpiredSessions() { -// log.info("Starting cleanup of expired upload sessions"); -// -// LocalDateTime expiryThreshold = LocalDateTime.now().minusHours(sessionExpiryHours); -// -// try { -// var expiredSessions = sessionRepository.findByStatusInAndCreatedAtBefore( -// Set.of(UploadStatus.INITIATED, UploadStatus.UPLOADING, UploadStatus.FAILED), -// expiryThreshold -// ); -// -// int cleanedCount = 0; -// for (FileUploadSession session : expiredSessions) { -// session.setStatus(UploadStatus.FAILED); -// session.setLastError("Session expired during cleanup"); -// sessionRepository.save(session); -// -// CompletableFuture.runAsync(() -> cleanupSessionFiles(session)); -// cleanedCount++; -// } -// -// log.info("Cleaned up {} expired upload sessions", cleanedCount); -// -// } catch (Exception e) { -// log.error("Error during session cleanup", e); -// } -// } - private Path getChunkDirectory(Long userId, String uploadId) { return storageRoot.resolve("temp") .resolve("user") diff --git a/src/main/java/ru/soune/nocopy/service/file/NotExistFilesCleanupService.java b/src/main/java/ru/soune/nocopy/service/file/NotExistFilesCleanupService.java new file mode 100644 index 0000000..1be7b9e --- /dev/null +++ b/src/main/java/ru/soune/nocopy/service/file/NotExistFilesCleanupService.java @@ -0,0 +1,35 @@ +package ru.soune.nocopy.service.file; + +import lombok.RequiredArgsConstructor; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.scheduling.annotation.Scheduled; +import org.springframework.stereotype.Service; +import org.springframework.transaction.annotation.Transactional; +import ru.soune.nocopy.entity.file.FileEntity; +import ru.soune.nocopy.entity.file.FileStatus; +import ru.soune.nocopy.repository.FileEntityRepository; + +import java.nio.file.Files; +import java.nio.file.Paths; +import java.util.List; + +@Service +@RequiredArgsConstructor +public class NotExistFilesCleanupService { + + @Autowired + private FileEntityRepository fileEntityRepository; + + @Transactional + @Scheduled(fixedDelay = 30000) + public void cleanUpNotExistFiles() { + List fileEntities = fileEntityRepository.searchFileEntityByStatus(FileStatus.DELETED); + + for (FileEntity fileEntity : fileEntities) { + boolean exists = Files.exists(Paths.get(fileEntity.getFilePath())); + if (!exists) { + fileEntityRepository.delete(fileEntity); + } + } + } +} diff --git a/src/main/resources/application.yaml b/src/main/resources/application.yaml index 363d191..c460c1f 100644 --- a/src/main/resources/application.yaml +++ b/src/main/resources/application.yaml @@ -1,4 +1,8 @@ spring: + flyway: + enabled: true + baseline-on-migrate: true + datasource: url: jdbc:postgresql://${POSTGRES_HOST}:${POSTGRES_PORT}/${POSTGRES_DB} username: ${POSTGRES_USER} @@ -24,12 +28,12 @@ spring: file: storage: base-path: ${FILE_STORAGE_PATH:/data/uploads} - # chunk-size: ${FILE_CHUNK_SIZE:5242880} # 5MB - chunk-size: ${FILE_CHUNK_SIZE:1048576} # 1MB + chunk-size: ${FILE_CHUNK_SIZE:1000000} # 1MB max-file-size: ${MAX_FILE_SIZE:10737418240} # 10GB max-retry-attempts: ${MAX_RETRY_ATTEMPTS:3} chunk-timeout-ms: ${CHUNK_TIMEOUT_MS:300000} # 5 минут temp-ttl-hours: ${TEMP_TTL_HOURS:72} # 3 дня + auth-token-life-hours: ${AUTH_TOKEN_LIFE:1} # 1 час session-expiry-hours: ${SESSION_EXPIRY_HOURS:24} security: @@ -50,4 +54,9 @@ logging: file: name: logs/application.log pattern: - console: "%d{yyyy-MM-dd HH:mm:ss} [%thread] %-5level %logger{36} - %msg%n" \ No newline at end of file + console: "%d{yyyy-MM-dd HH:mm:ss} [%thread] %-5level %logger{36} - %msg%n" + +yandex: + api-key: ${YANDEX_API_KEY:AQVNyaVaUmgUb1GMCtf5zSEqFxy0woXrcMOOB43q} + folder-id: ${YANDEX_FOLDER_ID:b1gokpdbm6qfpsou8pcd} + search-url: ${YANDEX_SEARCH_URL:https://searchapi.api.cloud.yandex.net/v2/image/search_by_image} \ No newline at end of file diff --git a/src/main/resources/db/migration/V1__add_image_hash_prefix_index.sql b/src/main/resources/db/migration/V1__add_image_hash_prefix_index.sql new file mode 100644 index 0000000..2700671 --- /dev/null +++ b/src/main/resources/db/migration/V1__add_image_hash_prefix_index.sql @@ -0,0 +1,2 @@ +CREATE INDEX idx_image_hash_prefix + ON image_hashes ((hash_value >> 48)); \ No newline at end of file diff --git a/src/test/java/ru/soune/nocopy/service/AuthServiceTest.java b/src/test/java/ru/soune/nocopy/service/AuthServiceTest.java deleted file mode 100644 index 499d694..0000000 --- a/src/test/java/ru/soune/nocopy/service/AuthServiceTest.java +++ /dev/null @@ -1,101 +0,0 @@ -package ru.soune.nocopy.service; - -import org.junit.jupiter.api.BeforeEach; -import org.junit.jupiter.api.Test; -import org.mockito.ArgumentCaptor; -import org.mockito.InjectMocks; -import org.mockito.Mock; -import org.mockito.MockitoAnnotations; -import org.springframework.context.MessageSource; -import org.springframework.security.crypto.password.PasswordEncoder; -import ru.soune.nocopy.dto.LoginRequest; -import ru.soune.nocopy.entity.AuthToken; -import ru.soune.nocopy.entity.User; -import ru.soune.nocopy.exception.NotValidationPasswordException; -import ru.soune.nocopy.exception.UserNotFoundException; -import ru.soune.nocopy.repository.AuthTokenRepository; -import ru.soune.nocopy.repository.UserRepository; - -import java.util.Optional; - -import static org.junit.jupiter.api.Assertions.*; -import static org.mockito.ArgumentMatchers.any; -import static org.mockito.Mockito.verify; -import static org.mockito.Mockito.when; - -public class AuthServiceTest { - - @Mock - private UserRepository userRepository; - - @Mock - private AuthTokenRepository authTokenRepository; - - @Mock - private PasswordEncoder passwordEncoder; - - @Mock - private MessageSource messageSource; - - @InjectMocks - private AuthService authService; - - @BeforeEach - void setUp() { - MockitoAnnotations.openMocks(this); - } - - @Test - void login_success() { - LoginRequest request = new LoginRequest("test@mail.com", "password"); - - User user = new User(); - user.setId(1L); - user.setEmail("test@mail.com"); - user.setPassword("encoded_pass"); - - when(userRepository.findByEmail("test@mail.com")) - .thenReturn(Optional.of(user)); - when(passwordEncoder.matches("password", "encoded_pass")) - .thenReturn(true); - when(authTokenRepository.save(any(AuthToken.class))) - .thenAnswer(invocation -> invocation.getArgument(0)); - - AuthToken token = authService.login(request); - - assertNotNull(token); - assertNotNull(token.getToken()); -// assertEquals(user.getTokens().get(0), token.getUser()); - - ArgumentCaptor captor = ArgumentCaptor.forClass(AuthToken.class); - verify(authTokenRepository).save(captor.capture()); - -// assertEquals(user, captor.getValue().getUser()); - assertNotNull(captor.getValue().getToken()); - } - - - @Test - void login_userNotFound() { - LoginRequest request = new LoginRequest("missing@mail.com", "pwd"); - - when(userRepository.findByEmail("missing@mail.com")).thenReturn(Optional.empty()); - - assertThrows(UserNotFoundException.class, () -> authService.login(request)); - } - - - @Test - void login_invalidPassword() { - LoginRequest request = new LoginRequest("test@mail.com", "wrong"); - - User user = new User(); - user.setEmail("test@mail.com"); - user.setPassword("encoded_pass"); - - when(userRepository.findByEmail("test@mail.com")).thenReturn(Optional.of(user)); - when(passwordEncoder.matches("wrong", "encoded_pass")).thenReturn(false); - - assertThrows(NotValidationPasswordException.class, () -> authService.login(request)); - } -}