From 647fd6e1a7143d059afb0a8f02cf9696fddf621d Mon Sep 17 00:00:00 2001 From: vladp Date: Mon, 22 Dec 2025 14:07:44 +0700 Subject: [PATCH 01/27] NCBACK-3 Add found by yandex image --- build.gradle | 4 + docker-compose.yaml | 2 + .../nocopy/configuration/HandlerConfig.java | 4 +- .../nocopy/configuration/JacksonConfig.java | 2 + .../configuration/file/YandexConfig.java | 14 ++ .../java/ru/soune/nocopy/dto/MessageCode.java | 1 + .../nocopy/dto/file/ImageSearchRequest.java | 10 ++ .../nocopy/dto/file/YandexSearchResponse.java | 36 +++++ .../handler/ImageFoundRequestHandler.java | 36 +++++ .../repository/FileEntityRepository.java | 1 + .../nocopy/service/YandexSearchService.java | 149 ++++++++++++++++++ src/main/resources/application.yaml | 5 + 12 files changed, 263 insertions(+), 1 deletion(-) create mode 100644 src/main/java/ru/soune/nocopy/configuration/file/YandexConfig.java create mode 100644 src/main/java/ru/soune/nocopy/dto/file/ImageSearchRequest.java create mode 100644 src/main/java/ru/soune/nocopy/dto/file/YandexSearchResponse.java create mode 100644 src/main/java/ru/soune/nocopy/handler/ImageFoundRequestHandler.java create mode 100644 src/main/java/ru/soune/nocopy/service/YandexSearchService.java diff --git a/build.gradle b/build.gradle index d42173a..111da7e 100644 --- a/build.gradle +++ b/build.gradle @@ -31,6 +31,10 @@ dependencies { implementation 'org.mapstruct:mapstruct:1.5.5.Final' implementation 'commons-validator:commons-validator:1.7' + implementation 'com.fasterxml.jackson.dataformat:jackson-dataformat-xml:2.17.0' + implementation 'tools.jackson.core:jackson-core:3.0.3' + implementation 'jakarta.xml.bind:jakarta.xml.bind-api:4.0.0' + annotationProcessor 'org.mapstruct:mapstruct-processor:1.5.5.Final' annotationProcessor 'org.projectlombok:lombok' diff --git a/docker-compose.yaml b/docker-compose.yaml index d3f9924..eb4724b 100644 --- a/docker-compose.yaml +++ b/docker-compose.yaml @@ -30,6 +30,8 @@ services: POSTGRES_PASSWORD: postgres POSTGRES_PORT: 5432 POSTGRES_HOST: db + YANDEX_SEARCH_API_KEY: AQVNyaVaUmgUb1GMCtf5zSEqFxy0woXrcMOOB43q + YANDEX_SEARCH_FOLDER_ID: b1gokpdbm6qfpsou8pcd depends_on: - db ports: diff --git a/src/main/java/ru/soune/nocopy/configuration/HandlerConfig.java b/src/main/java/ru/soune/nocopy/configuration/HandlerConfig.java index 5b3c744..674b27c 100644 --- a/src/main/java/ru/soune/nocopy/configuration/HandlerConfig.java +++ b/src/main/java/ru/soune/nocopy/configuration/HandlerConfig.java @@ -16,7 +16,8 @@ public class HandlerConfig { LoginRequestHandler login, FileUploadHandler upload, FileEntityHandler file, - LogoutRequestHandler logoutHandler + LogoutRequestHandler logoutHandler, + ImageFoundRequestHandler imageFoundRequestHandler ) { Map map = new HashMap<>(); map.put(20001, login); @@ -24,6 +25,7 @@ public class HandlerConfig { map.put(20004, upload); map.put(20005, file); map.put(20006, logoutHandler); + map.put(20007, imageFoundRequestHandler); return map; } diff --git a/src/main/java/ru/soune/nocopy/configuration/JacksonConfig.java b/src/main/java/ru/soune/nocopy/configuration/JacksonConfig.java index 99c5c00..db00da1 100644 --- a/src/main/java/ru/soune/nocopy/configuration/JacksonConfig.java +++ b/src/main/java/ru/soune/nocopy/configuration/JacksonConfig.java @@ -35,6 +35,8 @@ public class JacksonConfig { mapper.registerModule(javaTimeModule); mapper.disable(SerializationFeature.WRITE_DATES_AS_TIMESTAMPS); + mapper.configure(com.fasterxml.jackson.databind.DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, false); + mapper.enable(SerializationFeature.INDENT_OUTPUT); return mapper; } diff --git a/src/main/java/ru/soune/nocopy/configuration/file/YandexConfig.java b/src/main/java/ru/soune/nocopy/configuration/file/YandexConfig.java new file mode 100644 index 0000000..e9f1576 --- /dev/null +++ b/src/main/java/ru/soune/nocopy/configuration/file/YandexConfig.java @@ -0,0 +1,14 @@ +package ru.soune.nocopy.configuration.file; + +import lombok.Data; +import org.springframework.boot.context.properties.ConfigurationProperties; +import org.springframework.context.annotation.Configuration; + +@Data +@Configuration +@ConfigurationProperties(prefix = "yandex") +public class YandexConfig { + private String apiKey; + private String folderId; + private String searchUrl; +} \ No newline at end of file diff --git a/src/main/java/ru/soune/nocopy/dto/MessageCode.java b/src/main/java/ru/soune/nocopy/dto/MessageCode.java index a3ac162..ee9d40b 100644 --- a/src/main/java/ru/soune/nocopy/dto/MessageCode.java +++ b/src/main/java/ru/soune/nocopy/dto/MessageCode.java @@ -8,6 +8,7 @@ public enum MessageCode { INVALID_TOKEN(2, "Invalid token"), INVALID_ACTION(2, "Invalid action"), FILE_UPLOAD_ERROR(2, "File upload error"), + IMAGE_FOUND_ERROR(2, "Image found error"), INVALID_JSON_BODY(2, "Invalid fields in JSON object"), INCOMPLETE_UPLOAD(2, "Not load all chunks"), MSG_ID_NOT_FOUND(4, "Message id not found"), diff --git a/src/main/java/ru/soune/nocopy/dto/file/ImageSearchRequest.java b/src/main/java/ru/soune/nocopy/dto/file/ImageSearchRequest.java new file mode 100644 index 0000000..f943989 --- /dev/null +++ b/src/main/java/ru/soune/nocopy/dto/file/ImageSearchRequest.java @@ -0,0 +1,10 @@ +package ru.soune.nocopy.dto.file; + +import com.fasterxml.jackson.annotation.JsonProperty; +import lombok.Data; + +@Data +public class ImageSearchRequest { + @JsonProperty("file_id") + private String fileId; +} diff --git a/src/main/java/ru/soune/nocopy/dto/file/YandexSearchResponse.java b/src/main/java/ru/soune/nocopy/dto/file/YandexSearchResponse.java new file mode 100644 index 0000000..760440f --- /dev/null +++ b/src/main/java/ru/soune/nocopy/dto/file/YandexSearchResponse.java @@ -0,0 +1,36 @@ +package ru.soune.nocopy.dto.file; + +import com.fasterxml.jackson.annotation.JsonIgnoreProperties; +import com.fasterxml.jackson.annotation.JsonProperty; +import lombok.Data; +import java.util.List; + +@Data +@JsonIgnoreProperties(ignoreUnknown = true) +public class YandexSearchResponse { + + @JsonProperty("images") + private List images; + + @Data + @JsonIgnoreProperties(ignoreUnknown = true) + public static class ImageResult { + @JsonProperty("url") + private String url; + + @JsonProperty("pageUrl") + private String pageUrl; + + @JsonProperty("pageTitle") + private String pageTitle; + + @JsonProperty("width") + private Integer width; + + @JsonProperty("height") + private Integer height; + + @JsonProperty("host") + private String host; + } +} \ No newline at end of file diff --git a/src/main/java/ru/soune/nocopy/handler/ImageFoundRequestHandler.java b/src/main/java/ru/soune/nocopy/handler/ImageFoundRequestHandler.java new file mode 100644 index 0000000..1b7b0fd --- /dev/null +++ b/src/main/java/ru/soune/nocopy/handler/ImageFoundRequestHandler.java @@ -0,0 +1,36 @@ +package ru.soune.nocopy.handler; + +import com.fasterxml.jackson.databind.ObjectMapper; +import lombok.RequiredArgsConstructor; +import lombok.extern.slf4j.Slf4j; +import org.springframework.stereotype.Component; +import ru.soune.nocopy.dto.BaseRequest; +import ru.soune.nocopy.dto.BaseResponse; +import ru.soune.nocopy.dto.MessageCode; +import ru.soune.nocopy.dto.file.ImageSearchRequest; +import ru.soune.nocopy.dto.file.YandexSearchResponse; +import ru.soune.nocopy.service.YandexSearchService; + +@Slf4j +@Component +@RequiredArgsConstructor +public class ImageFoundRequestHandler implements RequestHandler { + + private final ObjectMapper objectMapper; + + private final YandexSearchService yandexSearchService; + + @Override + public BaseResponse handle(BaseRequest request) throws Exception { + + ImageSearchRequest imageSearchRequest = objectMapper.convertValue( + request.getMessageBody(), ImageSearchRequest.class); + + String fileId = imageSearchRequest.getFileId(); + + YandexSearchResponse response = yandexSearchService.searchByFileEntity(fileId); + + return new BaseResponse(request.getMsgId(), MessageCode.SUCCESS.getCode(), + MessageCode.SUCCESS.getDescription(), response); + } +} diff --git a/src/main/java/ru/soune/nocopy/repository/FileEntityRepository.java b/src/main/java/ru/soune/nocopy/repository/FileEntityRepository.java index 3ee41cb..3214e9d 100644 --- a/src/main/java/ru/soune/nocopy/repository/FileEntityRepository.java +++ b/src/main/java/ru/soune/nocopy/repository/FileEntityRepository.java @@ -14,6 +14,7 @@ import java.util.Optional; public interface FileEntityRepository extends JpaRepository { List findByUserId(Long userId); + Optional findByUserIdAndChecksum(Long userId, String imageHash); List findByUserIdAndStatus(Long userId, FileStatus status); diff --git a/src/main/java/ru/soune/nocopy/service/YandexSearchService.java b/src/main/java/ru/soune/nocopy/service/YandexSearchService.java new file mode 100644 index 0000000..611036c --- /dev/null +++ b/src/main/java/ru/soune/nocopy/service/YandexSearchService.java @@ -0,0 +1,149 @@ +package ru.soune.nocopy.service; + +import com.fasterxml.jackson.core.JsonProcessingException; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.fasterxml.jackson.databind.DeserializationFeature; +import jakarta.annotation.PostConstruct; +import lombok.RequiredArgsConstructor; +import lombok.extern.slf4j.Slf4j; +import org.springframework.beans.factory.annotation.Value; +import org.springframework.stereotype.Service; +import ru.soune.nocopy.dto.file.YandexSearchResponse; +import ru.soune.nocopy.entity.file.FileEntity; +import ru.soune.nocopy.repository.FileEntityRepository; + +import java.io.*; +import java.net.HttpURLConnection; +import java.net.URL; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.Base64; + +@Slf4j +@Service +@RequiredArgsConstructor +public class YandexSearchService { + + private final FileEntityRepository fileEntityRepository; + private final ObjectMapper objectMapper; + + @Value("${yandex.api-key}") + private String apiKey; + + @Value("${yandex.folder-id}") + private String folderId; + + @Value("${yandex.search-url}") + private String searchUrl; + + @PostConstruct + public void init() { + objectMapper.configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, false); + objectMapper.configure(DeserializationFeature.ACCEPT_SINGLE_VALUE_AS_ARRAY, true); + } + + public YandexSearchResponse searchByFileEntity(String fileId) throws IOException { + FileEntity fileEntity = fileEntityRepository.findById(fileId) + .orElseThrow(() -> { + log.error("File not found: {}", fileId); + return new RuntimeException("ФFile not found: " + fileId); + }); + + + if (!isImageFile(fileEntity)) { + log.error("File not image: {}", fileEntity.getMimeType()); + throw new RuntimeException("File not image: " + fileId); + } + + byte[] fileBytes = readFileFromDisk(fileEntity); + log.info("Прочитано {} байт с диска", fileBytes.length); + + return callYandexApi(fileBytes); + } + + private boolean isImageFile(FileEntity fileEntity) { + String mimeType = fileEntity.getMimeType(); + return mimeType != null && mimeType.startsWith("image"); + } + + private byte[] readFileFromDisk(FileEntity fileEntity) throws IOException { + Path filePath = Path.of(fileEntity.getFilePath()); + + if (!Files.exists(filePath)) { + throw new IOException("File not found: " + fileEntity.getFilePath()); + } + + if (!Files.isReadable(filePath)) { + throw new IOException("Cannot read file: " + fileEntity.getFilePath()); + } + + return Files.readAllBytes(filePath); + } + + private YandexSearchResponse callYandexApi(byte[] imageBytes) throws IOException { + String imageBase64 = Base64.getEncoder().encodeToString(imageBytes); + String jsonRequest = String.format("{\"folderId\":\"%s\",\"data\":\"%s\",\"page\":0,\"showSimilarImages\":true}", + folderId, imageBase64); + + URL url = new URL(searchUrl); + HttpURLConnection connection = (HttpURLConnection) url.openConnection(); + + try { + connection.setRequestMethod("POST"); + connection.setRequestProperty("Authorization", "Api-Key " + apiKey); + connection.setRequestProperty("Content-Type", "application/json"); + connection.setRequestProperty("Accept", "application/json"); + connection.setRequestProperty("User-Agent", "Mozilla/5.0"); + connection.setConnectTimeout(30000); + connection.setReadTimeout(30000); + connection.setDoOutput(true); + + try (OutputStream os = connection.getOutputStream()) { + byte[] input = jsonRequest.getBytes("utf-8"); + os.write(input, 0, input.length); + os.flush(); + } + + int responseCode = connection.getResponseCode(); + + String responseBody; + if (responseCode == HttpURLConnection.HTTP_OK) { + try (BufferedReader br = new BufferedReader( + new InputStreamReader(connection.getInputStream(), "utf-8"))) { + responseBody = readAll(br); + } + } else { + try (BufferedReader br = new BufferedReader( + new InputStreamReader(connection.getErrorStream(), "utf-8"))) { + responseBody = readAll(br); + } + throw new IOException("Ошибка Yandex API: " + responseCode + " - " + responseBody); + } + + return parseJsonResponse(responseBody); + + } finally { + connection.disconnect(); + } + } + + private String readAll(BufferedReader reader) throws IOException { + StringBuilder sb = new StringBuilder(); + String line; + while ((line = reader.readLine()) != null) { + sb.append(line); + } + return sb.toString(); + } + + private YandexSearchResponse parseJsonResponse(String json) { + YandexSearchResponse response = null; + try { + response = objectMapper.readValue(json, YandexSearchResponse.class); + } catch (JsonProcessingException e) { + throw new RuntimeException(e); + } + + return response; + } +} diff --git a/src/main/resources/application.yaml b/src/main/resources/application.yaml index 08326e1..45fcaaa 100644 --- a/src/main/resources/application.yaml +++ b/src/main/resources/application.yaml @@ -38,6 +38,11 @@ security: allowed-headers: "*" allow-credentials: true +yandex: + api-key: ${YANDEX_SEARCH_API_KEY} + folder-id: ${YANDEX_SEARCH_FOLDER_ID} + search-url: "https://searchapi.api.cloud.yandex.net/v2/image/search_by_image" + server: port: ${SERVER_PORT:8080} From ffb147d716488d696315ec84a067a333ad05d434 Mon Sep 17 00:00:00 2001 From: vladp Date: Mon, 22 Dec 2025 14:42:27 +0700 Subject: [PATCH 02/27] NCBACK-3 Add found by yandex image --- .../handler/ImageFoundRequestHandler.java | 5 ++--- .../nocopy/service/YandexSearchService.java | 20 ++++++++++++------- 2 files changed, 15 insertions(+), 10 deletions(-) diff --git a/src/main/java/ru/soune/nocopy/handler/ImageFoundRequestHandler.java b/src/main/java/ru/soune/nocopy/handler/ImageFoundRequestHandler.java index 1b7b0fd..301bb10 100644 --- a/src/main/java/ru/soune/nocopy/handler/ImageFoundRequestHandler.java +++ b/src/main/java/ru/soune/nocopy/handler/ImageFoundRequestHandler.java @@ -22,9 +22,8 @@ public class ImageFoundRequestHandler implements RequestHandler { @Override public BaseResponse handle(BaseRequest request) throws Exception { - - ImageSearchRequest imageSearchRequest = objectMapper.convertValue( - request.getMessageBody(), ImageSearchRequest.class); + ImageSearchRequest imageSearchRequest = objectMapper.convertValue(request.getMessageBody(), + ImageSearchRequest.class); String fileId = imageSearchRequest.getFileId(); diff --git a/src/main/java/ru/soune/nocopy/service/YandexSearchService.java b/src/main/java/ru/soune/nocopy/service/YandexSearchService.java index 611036c..3fd8c60 100644 --- a/src/main/java/ru/soune/nocopy/service/YandexSearchService.java +++ b/src/main/java/ru/soune/nocopy/service/YandexSearchService.java @@ -8,8 +8,12 @@ import lombok.RequiredArgsConstructor; import lombok.extern.slf4j.Slf4j; import org.springframework.beans.factory.annotation.Value; import org.springframework.stereotype.Service; +import ru.soune.nocopy.dto.BaseResponse; +import ru.soune.nocopy.dto.MessageCode; import ru.soune.nocopy.dto.file.YandexSearchResponse; import ru.soune.nocopy.entity.file.FileEntity; +import ru.soune.nocopy.exception.FileEntityNotFoundException; +import ru.soune.nocopy.exception.NotValidFieldException; import ru.soune.nocopy.repository.FileEntityRepository; import java.io.*; @@ -18,6 +22,7 @@ import java.net.URL; import java.nio.file.Files; import java.nio.file.Path; import java.util.Base64; +import java.util.Map; @Slf4j @Service @@ -45,18 +50,19 @@ public class YandexSearchService { public YandexSearchResponse searchByFileEntity(String fileId) throws IOException { FileEntity fileEntity = fileEntityRepository.findById(fileId) .orElseThrow(() -> { - log.error("File not found: {}", fileId); - return new RuntimeException("ФFile not found: " + fileId); + throw new NotValidFieldException("File not found", new BaseResponse(20007, + MessageCode.FILE_NOT_FOUND.getCode(), MessageCode.FILE_NOT_FOUND.getDescription(), + Map.of("fileId",fileId))); }); - if (!isImageFile(fileEntity)) { log.error("File not image: {}", fileEntity.getMimeType()); - throw new RuntimeException("File not image: " + fileId); + throw new NotValidFieldException("File not image", new BaseResponse(20007, + MessageCode.INVALID_FIELD.getCode(), MessageCode.INVALID_FIELD.getDescription(), + Map.of("file_type", fileEntity.getMimeType()))); } byte[] fileBytes = readFileFromDisk(fileEntity); - log.info("Прочитано {} байт с диска", fileBytes.length); return callYandexApi(fileBytes); } @@ -93,7 +99,7 @@ public class YandexSearchService { connection.setRequestProperty("Authorization", "Api-Key " + apiKey); connection.setRequestProperty("Content-Type", "application/json"); connection.setRequestProperty("Accept", "application/json"); - connection.setRequestProperty("User-Agent", "Mozilla/5.0"); +// connection.setRequestProperty("User-Agent", "Mozilla/5.0"); connection.setConnectTimeout(30000); connection.setReadTimeout(30000); connection.setDoOutput(true); @@ -117,7 +123,7 @@ public class YandexSearchService { new InputStreamReader(connection.getErrorStream(), "utf-8"))) { responseBody = readAll(br); } - throw new IOException("Ошибка Yandex API: " + responseCode + " - " + responseBody); + throw new IOException("Error Yandex API: " + responseCode + " - " + responseBody); } return parseJsonResponse(responseBody); From 0da04044888a82e0ec81a9b7f640121f4d6b1224 Mon Sep 17 00:00:00 2001 From: vladp Date: Mon, 5 Jan 2026 16:26:46 +0700 Subject: [PATCH 03/27] add token life by hour --- .../nocopy/configuration/HandlerConfig.java | 4 +- .../nocopy/controller/ApiController.java | 17 +++------ .../java/ru/soune/nocopy/dto/MessageCode.java | 1 + .../soune/nocopy/dto/TokenProcessRequest.java | 11 ++++++ .../ru/soune/nocopy/entity/AuthToken.java | 6 +-- .../nocopy/handler/AuthRequestHandler.java | 37 ++++++++++++++++++ .../nocopy/handler/FileEntityHandler.java | 30 +++++---------- .../nocopy/handler/FileUploadHandler.java | 10 +++-- .../nocopy/handler/LoginRequestHandler.java | 9 ++++- .../nocopy/handler/LogoutRequestHandler.java | 2 +- .../nocopy/handler/RegRequestHandler.java | 6 ++- .../repository/AuthTokenRepository.java | 5 ++- .../service/{ => auth}/AuthService.java | 25 ++++++++++-- .../auth/CleanupTokenSessionsService.java | 38 +++++++++++++++++++ src/main/resources/application.yaml | 1 + .../soune/nocopy/service/AuthServiceTest.java | 1 + 16 files changed, 150 insertions(+), 53 deletions(-) create mode 100644 src/main/java/ru/soune/nocopy/dto/TokenProcessRequest.java create mode 100644 src/main/java/ru/soune/nocopy/handler/AuthRequestHandler.java rename src/main/java/ru/soune/nocopy/service/{ => auth}/AuthService.java (81%) create mode 100644 src/main/java/ru/soune/nocopy/service/auth/CleanupTokenSessionsService.java diff --git a/src/main/java/ru/soune/nocopy/configuration/HandlerConfig.java b/src/main/java/ru/soune/nocopy/configuration/HandlerConfig.java index 5b3c744..f1577c2 100644 --- a/src/main/java/ru/soune/nocopy/configuration/HandlerConfig.java +++ b/src/main/java/ru/soune/nocopy/configuration/HandlerConfig.java @@ -16,7 +16,8 @@ public class HandlerConfig { LoginRequestHandler login, FileUploadHandler upload, FileEntityHandler file, - LogoutRequestHandler logoutHandler + LogoutRequestHandler logoutHandler, + AuthRequestHandler authRequestHandler ) { Map map = new HashMap<>(); map.put(20001, login); @@ -24,6 +25,7 @@ public class HandlerConfig { map.put(20004, upload); map.put(20005, file); map.put(20006, logoutHandler); + map.put(20007, authRequestHandler); return map; } diff --git a/src/main/java/ru/soune/nocopy/controller/ApiController.java b/src/main/java/ru/soune/nocopy/controller/ApiController.java index dbbe9df..9e5d257 100644 --- a/src/main/java/ru/soune/nocopy/controller/ApiController.java +++ b/src/main/java/ru/soune/nocopy/controller/ApiController.java @@ -19,12 +19,11 @@ import ru.soune.nocopy.dto.file.ChunkUploadResponse; import ru.soune.nocopy.dto.file.CompleteUploadResponse; import ru.soune.nocopy.dto.file.FileEntityResponse; import ru.soune.nocopy.dto.file.UploadProgress; -import ru.soune.nocopy.entity.AuthToken; import ru.soune.nocopy.entity.file.FileStatus; import ru.soune.nocopy.entity.file.UploadStatus; import ru.soune.nocopy.exception.*; import ru.soune.nocopy.handler.*; -import ru.soune.nocopy.repository.AuthTokenRepository; +import ru.soune.nocopy.service.auth.AuthService; import ru.soune.nocopy.service.file.FileEntityService; import ru.soune.nocopy.service.file.FileUploadService; @@ -48,7 +47,7 @@ public class ApiController { private final FileEntityService fileEntityService; - private final AuthTokenRepository authTokenRepository; + private final AuthService authService; @PostMapping("/v{version}/data") public ResponseEntity handlePostRequest(@RequestBody BaseRequest request, @@ -71,6 +70,9 @@ public class ApiController { return ResponseEntity.ok().body(response); } catch (ValidationException e) { return createValidationErrorResponse(e.getBindingResult(), e.getMsgId()); + } catch (NotFoundAuthToken e) { + return ResponseEntity.ok().body(new BaseResponse(msgId, MessageCode.AUTH_TOKEN_NOT_FOUND.getCode(), + MessageCode.AUTH_TOKEN_NOT_FOUND.getDescription(), new HashMap<>())); } catch (NotValidFieldException e) { throw e; } catch (Exception e) { @@ -257,7 +259,7 @@ public class ApiController { errorData)); } - Long userId = getUserIdFromToken(tokenHeader); + Long userId = authService.useUserAuthToken(tokenHeader); FileEntityResponse entityResponse = fileEntityService.getById(fileId, version); if (!entityResponse.getUserId().equals(userId)) { @@ -379,11 +381,4 @@ public class ApiController { mimeType.startsWith("video") || mimeType.startsWith("audio"); } - - private Long getUserIdFromToken(String tokenHeader) { - String token = tokenHeader.replace("Bearer ", ""); - AuthToken authToken = authTokenRepository.findByToken(token) - .orElseThrow(() -> new NotFoundAuthToken("Token not found")); - return authToken.getUser().getId(); - } } \ No newline at end of file diff --git a/src/main/java/ru/soune/nocopy/dto/MessageCode.java b/src/main/java/ru/soune/nocopy/dto/MessageCode.java index b945f64..36d8d9f 100644 --- a/src/main/java/ru/soune/nocopy/dto/MessageCode.java +++ b/src/main/java/ru/soune/nocopy/dto/MessageCode.java @@ -18,6 +18,7 @@ public enum MessageCode { AUTH_EMAIL_NOT_FOUND(4, "Email not found"), AUTH_EMAIL_OR_TOKEN_NOT_FOUND(4, "Email or Token not found "), AUTH_TOKEN_MISMATCH(4, "Token mismatch"), + AUTH_TOKEN_NOT_FOUND(4, "Token not found"), FILE_NOT_FOUND(4, "File not found"), AUTH_PASSWORD_NOT_MATCHES(2, "Password does not match"); diff --git a/src/main/java/ru/soune/nocopy/dto/TokenProcessRequest.java b/src/main/java/ru/soune/nocopy/dto/TokenProcessRequest.java new file mode 100644 index 0000000..646f9c6 --- /dev/null +++ b/src/main/java/ru/soune/nocopy/dto/TokenProcessRequest.java @@ -0,0 +1,11 @@ +package ru.soune.nocopy.dto; + +import com.fasterxml.jackson.annotation.JsonProperty; +import lombok.Data; + +@Data +public class TokenProcessRequest { + + @JsonProperty("token") + private String token; +} diff --git a/src/main/java/ru/soune/nocopy/entity/AuthToken.java b/src/main/java/ru/soune/nocopy/entity/AuthToken.java index 63cf34a..6fa1a9c 100644 --- a/src/main/java/ru/soune/nocopy/entity/AuthToken.java +++ b/src/main/java/ru/soune/nocopy/entity/AuthToken.java @@ -30,7 +30,7 @@ public class AuthToken { private String token; @Column(name = "expires_at", nullable = false) - private LocalDateTime expiresAt = LocalDateTime.now().plusDays(30); + private LocalDateTime expiresAt = LocalDateTime.now().plusHours(1); @CreatedDate @Column(name = "created_at", updatable = false, nullable = false) @@ -41,8 +41,4 @@ public class AuthToken { @Column(name = "is_active") private Boolean isActive = true; - - public boolean isValid() { - return Boolean.TRUE.equals(isActive) && expiresAt.isAfter(LocalDateTime.now()); - } } diff --git a/src/main/java/ru/soune/nocopy/handler/AuthRequestHandler.java b/src/main/java/ru/soune/nocopy/handler/AuthRequestHandler.java new file mode 100644 index 0000000..c79ec06 --- /dev/null +++ b/src/main/java/ru/soune/nocopy/handler/AuthRequestHandler.java @@ -0,0 +1,37 @@ +package ru.soune.nocopy.handler; + +import com.fasterxml.jackson.databind.ObjectMapper; +import lombok.RequiredArgsConstructor; +import lombok.extern.slf4j.Slf4j; +import org.springframework.stereotype.Component; +import ru.soune.nocopy.dto.BaseRequest; +import ru.soune.nocopy.dto.BaseResponse; +import ru.soune.nocopy.dto.MessageCode; +import ru.soune.nocopy.dto.TokenProcessRequest; +import ru.soune.nocopy.exception.NotFoundAuthToken; +import ru.soune.nocopy.service.auth.AuthService; + +import java.util.Map; + +@Slf4j +@Component +@RequiredArgsConstructor +public class AuthRequestHandler implements RequestHandler { + + private final ObjectMapper objectMapper; + + private final AuthService authService; + + @Override + public BaseResponse handle(BaseRequest request) throws NotFoundAuthToken { + TokenProcessRequest tokenProcessRequest = objectMapper.convertValue(request.getMessageBody(), + TokenProcessRequest.class); + + authService.useUserAuthToken(tokenProcessRequest.getToken()); + + return new BaseResponse(request.getMsgId(), + MessageCode.SUCCESS.getCode(), + "Token processed successfully", + Map.of("token", tokenProcessRequest.getToken())); + } +} diff --git a/src/main/java/ru/soune/nocopy/handler/FileEntityHandler.java b/src/main/java/ru/soune/nocopy/handler/FileEntityHandler.java index 968a3b7..526ff9f 100644 --- a/src/main/java/ru/soune/nocopy/handler/FileEntityHandler.java +++ b/src/main/java/ru/soune/nocopy/handler/FileEntityHandler.java @@ -6,13 +6,13 @@ import lombok.extern.slf4j.Slf4j; import org.springframework.stereotype.Component; import ru.soune.nocopy.dto.*; import ru.soune.nocopy.dto.file.*; -import ru.soune.nocopy.entity.AuthToken; import ru.soune.nocopy.entity.file.FileEntity; import ru.soune.nocopy.entity.file.FileStatus; import ru.soune.nocopy.exception.FileEntityNotFoundException; import ru.soune.nocopy.exception.NotFoundAuthToken; import ru.soune.nocopy.repository.AuthTokenRepository; import ru.soune.nocopy.repository.FileEntityRepository; +import ru.soune.nocopy.service.auth.AuthService; import ru.soune.nocopy.service.file.FileEntityService; import java.util.Arrays; @@ -24,6 +24,8 @@ import java.util.List; public class FileEntityHandler implements RequestHandler { private final FileEntityService fileEntityService; + private final AuthService authService; + private final AuthTokenRepository authTokenRepository; private final ObjectMapper objectMapper; @@ -75,7 +77,7 @@ public class FileEntityHandler implements RequestHandler { private BaseResponse handleGetFileInfo(BaseRequest request, FileEntityRequest fileRequest) { try { - Long userId = getUserIdFromToken(fileRequest.getToken()); + Long userId = authService.useUserAuthToken(fileRequest.getToken()); FileEntityResponse fileInfo = fileEntityService.getById(fileRequest.getFileId(), request.getVersion()); if (!fileInfo.getUserId().equals(userId)) { @@ -106,7 +108,7 @@ public class FileEntityHandler implements RequestHandler { private BaseResponse handleGetFileBySession(BaseRequest request, FileEntityRequest fileRequest) { try { - Long userId = getUserIdFromToken(fileRequest.getToken()); + Long userId = authService.useUserAuthToken(fileRequest.getToken()); FileEntityResponse fileInfo = fileEntityService.getByUploadSessionId(fileRequest.getUploadSessionId(), request.getVersion()); @@ -138,7 +140,7 @@ public class FileEntityHandler implements RequestHandler { private BaseResponse handleGetUserFiles(BaseRequest request, FileEntityRequest fileRequest) { try { - Long userId = getUserIdFromToken(fileRequest.getToken()); + Long userId = authService.useUserAuthToken(fileRequest.getToken()); int page = fileRequest.getPage() != null ? fileRequest.getPage() : 1; int pageSize = fileRequest.getPageSize() != null ? fileRequest.getPageSize() : 20; @@ -165,7 +167,7 @@ public class FileEntityHandler implements RequestHandler { private BaseResponse handleSearchFiles(BaseRequest request, FileEntityRequest fileRequest) { try { - Long userId = getUserIdFromToken(fileRequest.getToken()); + Long userId = authService.useUserAuthToken(fileRequest.getToken()); int page = fileRequest.getPage() != null ? fileRequest.getPage() : 1; int pageSize = fileRequest.getPageSize() != null ? fileRequest.getPageSize() : 20; @@ -212,7 +214,7 @@ public class FileEntityHandler implements RequestHandler { private BaseResponse handleGetStorageUsage(BaseRequest request, FileEntityRequest fileRequest) { try { - Long userId = getUserIdFromToken(fileRequest.getToken()); + Long userId = authService.useUserAuthToken(fileRequest.getToken()); long usage = fileEntityService.getUserStorageUsed(userId); StorageUsageResponse response = StorageUsageResponse.builder() @@ -242,7 +244,7 @@ public class FileEntityHandler implements RequestHandler { private BaseResponse handleDeleteFile(BaseRequest request, FileEntityRequest fileRequest) { try { - Long userId = getUserIdFromToken(fileRequest.getToken()); + Long userId = authService.useUserAuthToken(fileRequest.getToken()); String fileId = fileRequest.getFileId(); FileEntityResponse fileInfo = fileEntityService.getById(fileId, request.getVersion()); DeleteFileResponse response; @@ -289,20 +291,6 @@ public class FileEntityHandler implements RequestHandler { } } - private Long getUserIdFromToken(String token) { - if (token == null || token.isBlank()) { - throw new NotFoundAuthToken("Token is required"); - } - - if (token.startsWith("Bearer ")) { - token = token.replace("Bearer ", ""); - } - - AuthToken authToken = authTokenRepository.findByToken(token) - .orElseThrow(() -> new NotFoundAuthToken("Token not found")); - return authToken.getUser().getId(); - } - private String formatFileSize(long size) { if (size < 1024) return size + " B"; int exp = (int) (Math.log(size) / Math.log(1024)); diff --git a/src/main/java/ru/soune/nocopy/handler/FileUploadHandler.java b/src/main/java/ru/soune/nocopy/handler/FileUploadHandler.java index 4cb33b5..d2bbf4c 100644 --- a/src/main/java/ru/soune/nocopy/handler/FileUploadHandler.java +++ b/src/main/java/ru/soune/nocopy/handler/FileUploadHandler.java @@ -17,6 +17,7 @@ import ru.soune.nocopy.exception.*; import ru.soune.nocopy.handler.validator.FileUploadRequestValidator; import ru.soune.nocopy.repository.AuthTokenRepository; import ru.soune.nocopy.repository.FileUploadSessionRepository; +import ru.soune.nocopy.service.auth.AuthService; import ru.soune.nocopy.service.file.FileUploadService; import java.util.*; @@ -28,6 +29,8 @@ import java.util.stream.Collectors; public class FileUploadHandler implements RequestHandler { private final FileUploadService fileUploadService; + private final AuthService authService; + private final AuthTokenRepository authTokenRepository; private final FileUploadSessionRepository fileUploadSessionRepository; @@ -86,10 +89,9 @@ public class FileUploadHandler implements RequestHandler { MessageCode.INVALID_TOKEN.getDescription(), Map.of("token", token)); } - AuthToken authToken = tokenOptional.orElseThrow(() -> new NotFoundAuthToken("Token not found")); - + Long userId = authService.useUserAuthToken(token); BindingResult bindingResult = new BeanPropertyBindingResult(fileRequest, "fileRequest"); - fileUploadRequestValidator.validate(fileRequest, bindingResult); + fileUploadRequestValidator.validate(fileRequest, bindingResult); if (bindingResult.hasErrors()) { Map fieldErrors = bindingResult.getFieldErrors() @@ -105,7 +107,7 @@ public class FileUploadHandler implements RequestHandler { } FileUploadSession session = fileUploadService.initUpload( - authToken.getUser().getId(), + userId, fileRequest.getFileName(), fileRequest.getFileType(), fileRequest.getExtension(), diff --git a/src/main/java/ru/soune/nocopy/handler/LoginRequestHandler.java b/src/main/java/ru/soune/nocopy/handler/LoginRequestHandler.java index 95abf63..6add2a0 100644 --- a/src/main/java/ru/soune/nocopy/handler/LoginRequestHandler.java +++ b/src/main/java/ru/soune/nocopy/handler/LoginRequestHandler.java @@ -7,7 +7,7 @@ import ru.soune.nocopy.dto.*; import ru.soune.nocopy.entity.AuthToken; import ru.soune.nocopy.exception.NotValidFieldException; import ru.soune.nocopy.repository.UserRepository; -import ru.soune.nocopy.service.AuthService; +import ru.soune.nocopy.service.auth.AuthService; import java.util.Arrays; import java.util.Map; @@ -17,7 +17,9 @@ import java.util.Map; public class LoginRequestHandler implements RequestHandler { private final UserRepository userRepository; + private final AuthService authService; + private final ObjectMapper objectMapper; @Override @@ -34,9 +36,12 @@ public class LoginRequestHandler implements RequestHandler { } AuthToken authToken = authService.login(loginRequest); + String token = authToken.getToken(); LoginAnswer loginAnswer = new LoginAnswer(); - loginAnswer.setToken(authToken.getToken()); + loginAnswer.setToken(token); + + authService.useUserAuthToken(token); return new BaseResponse(request.getMsgId(), MessageCode.SUCCESS.getCode(), MessageCode.SUCCESS.getDescription(), loginAnswer); diff --git a/src/main/java/ru/soune/nocopy/handler/LogoutRequestHandler.java b/src/main/java/ru/soune/nocopy/handler/LogoutRequestHandler.java index 44fa253..e12284c 100644 --- a/src/main/java/ru/soune/nocopy/handler/LogoutRequestHandler.java +++ b/src/main/java/ru/soune/nocopy/handler/LogoutRequestHandler.java @@ -9,7 +9,7 @@ import ru.soune.nocopy.entity.User; import ru.soune.nocopy.exception.NotValidFieldException; import ru.soune.nocopy.repository.AuthTokenRepository; import ru.soune.nocopy.repository.UserRepository; -import ru.soune.nocopy.service.AuthService; +import ru.soune.nocopy.service.auth.AuthService; import java.util.Objects; import java.util.Optional; diff --git a/src/main/java/ru/soune/nocopy/handler/RegRequestHandler.java b/src/main/java/ru/soune/nocopy/handler/RegRequestHandler.java index 349f0a7..86fcbc7 100644 --- a/src/main/java/ru/soune/nocopy/handler/RegRequestHandler.java +++ b/src/main/java/ru/soune/nocopy/handler/RegRequestHandler.java @@ -12,7 +12,7 @@ import ru.soune.nocopy.exception.NotValidFieldException; import ru.soune.nocopy.exception.ValidationException; import ru.soune.nocopy.handler.validator.RegRequestValidator; import ru.soune.nocopy.repository.UserRepository; -import ru.soune.nocopy.service.AuthService; +import ru.soune.nocopy.service.auth.AuthService; import java.util.Arrays; import java.util.Map; @@ -52,9 +52,11 @@ public class RegRequestHandler implements RequestHandler { } AuthToken authToken = authService.register(regRequest); + String token = authToken.getToken(); + authService.useUserAuthToken(token); RegAnswer regAnswer = new RegAnswer(); - regAnswer.setToken(authToken.getToken()); + regAnswer.setToken(token); return new BaseResponse(request.getMsgId(), MessageCode.SUCCESS.getCode(), MessageCode.SUCCESS.getDescription(), regAnswer); diff --git a/src/main/java/ru/soune/nocopy/repository/AuthTokenRepository.java b/src/main/java/ru/soune/nocopy/repository/AuthTokenRepository.java index 74826fd..9328753 100644 --- a/src/main/java/ru/soune/nocopy/repository/AuthTokenRepository.java +++ b/src/main/java/ru/soune/nocopy/repository/AuthTokenRepository.java @@ -4,11 +4,12 @@ import org.springframework.data.jpa.repository.JpaRepository; import org.springframework.stereotype.Repository; import ru.soune.nocopy.entity.AuthToken; -import java.time.LocalDate; +import java.time.LocalDateTime; import java.util.Optional; @Repository public interface AuthTokenRepository extends JpaRepository { - Optional findByTokenAndExpiresAtAfter(String token, LocalDate expiresAtAfter); + Optional findByLastUsedAtAndExpiresAtBefore(LocalDateTime lastUsedAt, LocalDateTime expiresAtBefore); + Optional findByLastUsedAtBefore(LocalDateTime lastUsedAt); Optional findByToken(String token); } diff --git a/src/main/java/ru/soune/nocopy/service/AuthService.java b/src/main/java/ru/soune/nocopy/service/auth/AuthService.java similarity index 81% rename from src/main/java/ru/soune/nocopy/service/AuthService.java rename to src/main/java/ru/soune/nocopy/service/auth/AuthService.java index 2377b2b..cf8b18c 100644 --- a/src/main/java/ru/soune/nocopy/service/AuthService.java +++ b/src/main/java/ru/soune/nocopy/service/auth/AuthService.java @@ -1,13 +1,13 @@ -package ru.soune.nocopy.service; +package ru.soune.nocopy.service.auth; import lombok.RequiredArgsConstructor; -import org.springframework.context.MessageSource; import org.springframework.security.crypto.password.PasswordEncoder; import org.springframework.stereotype.Service; import org.springframework.transaction.annotation.Transactional; import ru.soune.nocopy.dto.*; import ru.soune.nocopy.entity.AuthToken; import ru.soune.nocopy.entity.User; +import ru.soune.nocopy.exception.NotFoundAuthToken; import ru.soune.nocopy.exception.NotValidFieldException; import ru.soune.nocopy.repository.AuthTokenRepository; import ru.soune.nocopy.repository.UserRepository; @@ -29,8 +29,6 @@ public class AuthService { private final PasswordEncoder passwordEncoder; - private final MessageSource messageSource; - private final SecureRandom secureRandom = new SecureRandom(); @Transactional @@ -81,6 +79,25 @@ public class AuthService { return authTokenRepository.save(authToken); } + @Transactional + public Long useUserAuthToken(String token) { + if (token == null || token.isBlank()) { + throw new NotFoundAuthToken("Token is required"); + } + + if (token.startsWith("Bearer ")) { + token = token.replace("Bearer ", ""); + } + + AuthToken authToken = authTokenRepository.findByToken(token) + .orElseThrow(() -> new NotFoundAuthToken("Token not found")); + authToken.setLastUsedAt(LocalDateTime.now()); + + authTokenRepository.save(authToken); + + return authToken.getUser().getId(); + } + @Transactional public void logout(String token) { authTokenRepository.findByToken(token) diff --git a/src/main/java/ru/soune/nocopy/service/auth/CleanupTokenSessionsService.java b/src/main/java/ru/soune/nocopy/service/auth/CleanupTokenSessionsService.java new file mode 100644 index 0000000..c18cda3 --- /dev/null +++ b/src/main/java/ru/soune/nocopy/service/auth/CleanupTokenSessionsService.java @@ -0,0 +1,38 @@ +package ru.soune.nocopy.service.auth; + +import lombok.RequiredArgsConstructor; +import lombok.extern.slf4j.Slf4j; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.beans.factory.annotation.Value; +import org.springframework.scheduling.annotation.Scheduled; +import org.springframework.stereotype.Component; +import org.springframework.transaction.annotation.Transactional; +import ru.soune.nocopy.entity.AuthToken; +import ru.soune.nocopy.repository.AuthTokenRepository; + +import java.time.LocalDateTime; +import java.util.List; + +@Slf4j +@Component +@RequiredArgsConstructor +public class CleanupTokenSessionsService { + @Autowired + private AuthTokenRepository authTokenRepository; + + @Value("${file.storage.auth-token-life-hours}") + private int authTokenLife; + + @Transactional + @Scheduled(fixedDelay = 30000) + public void cleanupExpiredFiles() { + LocalDateTime hourBefore = LocalDateTime.now().minusHours(authTokenLife); + + List tokensForDelete = authTokenRepository.findByLastUsedAtBefore(hourBefore).stream().toList(); + List tokensDeleteWithNullUpdate = authTokenRepository.findByLastUsedAtAndExpiresAtBefore( + null, hourBefore).stream().toList(); + + authTokenRepository.deleteAll(tokensForDelete); + authTokenRepository.deleteAll(tokensDeleteWithNullUpdate); + } +} diff --git a/src/main/resources/application.yaml b/src/main/resources/application.yaml index 1fd4fab..68701ca 100644 --- a/src/main/resources/application.yaml +++ b/src/main/resources/application.yaml @@ -30,6 +30,7 @@ file: max-retry-attempts: ${MAX_RETRY_ATTEMPTS:3} chunk-timeout-ms: ${CHUNK_TIMEOUT_MS:300000} # 5 минут temp-ttl-hours: ${TEMP_TTL_HOURS:72} # 3 дня + auth-token-life-hours: ${AUTH_TOKEN_LIFE:1} # 1 час session-expiry-hours: ${SESSION_EXPIRY_HOURS:24} security: diff --git a/src/test/java/ru/soune/nocopy/service/AuthServiceTest.java b/src/test/java/ru/soune/nocopy/service/AuthServiceTest.java index 499d694..a16af7d 100644 --- a/src/test/java/ru/soune/nocopy/service/AuthServiceTest.java +++ b/src/test/java/ru/soune/nocopy/service/AuthServiceTest.java @@ -15,6 +15,7 @@ import ru.soune.nocopy.exception.NotValidationPasswordException; import ru.soune.nocopy.exception.UserNotFoundException; import ru.soune.nocopy.repository.AuthTokenRepository; import ru.soune.nocopy.repository.UserRepository; +import ru.soune.nocopy.service.auth.AuthService; import java.util.Optional; From f1222ede57b2c5d01cd2331b7187f39af944f9d4 Mon Sep 17 00:00:00 2001 From: vladp Date: Mon, 5 Jan 2026 16:31:40 +0700 Subject: [PATCH 04/27] add token life by hour --- src/main/java/ru/soune/nocopy/configuration/HandlerConfig.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/java/ru/soune/nocopy/configuration/HandlerConfig.java b/src/main/java/ru/soune/nocopy/configuration/HandlerConfig.java index f1577c2..4ba97da 100644 --- a/src/main/java/ru/soune/nocopy/configuration/HandlerConfig.java +++ b/src/main/java/ru/soune/nocopy/configuration/HandlerConfig.java @@ -25,7 +25,7 @@ public class HandlerConfig { map.put(20004, upload); map.put(20005, file); map.put(20006, logoutHandler); - map.put(20007, authRequestHandler); + map.put(20008, authRequestHandler); return map; } From 63baaf7c807eb9eebe1763e9349c2da339408080 Mon Sep 17 00:00:00 2001 From: vladp Date: Thu, 8 Jan 2026 12:59:30 +0700 Subject: [PATCH 05/27] update types --- docker-compose.yaml | 3 ++- .../ru/soune/nocopy/service/file/FileUploadServiceImpl.java | 2 +- src/main/resources/application.yaml | 3 ++- 3 files changed, 5 insertions(+), 3 deletions(-) diff --git a/docker-compose.yaml b/docker-compose.yaml index 8df1c29..1ce8455 100644 --- a/docker-compose.yaml +++ b/docker-compose.yaml @@ -58,7 +58,8 @@ services: environment: FILE_STORAGE_PATH: /data/uploads MAX_FILE_SIZE: 10737418240 - FILE_CHUNK_SIZE: 1048576 +# FILE_CHUNK_SIZE: 1048576 + FILE_CHUNK_SIZE: 1000000 POSTGRES_DB: no_copy_ POSTGRES_USER: postgres POSTGRES_PASSWORD: postgres diff --git a/src/main/java/ru/soune/nocopy/service/file/FileUploadServiceImpl.java b/src/main/java/ru/soune/nocopy/service/file/FileUploadServiceImpl.java index f54cec9..3e09822 100644 --- a/src/main/java/ru/soune/nocopy/service/file/FileUploadServiceImpl.java +++ b/src/main/java/ru/soune/nocopy/service/file/FileUploadServiceImpl.java @@ -102,7 +102,7 @@ public class FileUploadServiceImpl implements FileUploadService { int totalChunks = (int) Math.ceil((double) fileSize / chunkSize); log.debug("File will be split into {} chunks (chunk size: {} bytes)", totalChunks, chunkSize); - Long chunkSize = totalChunks == 1 ? fileSize : 1048576L; + Long chunkSize = totalChunks == 1 ? fileSize : 1000000L; FileUploadSession session = FileUploadSession.builder() .userId(userId) diff --git a/src/main/resources/application.yaml b/src/main/resources/application.yaml index 363d191..da03a23 100644 --- a/src/main/resources/application.yaml +++ b/src/main/resources/application.yaml @@ -25,7 +25,8 @@ file: storage: base-path: ${FILE_STORAGE_PATH:/data/uploads} # chunk-size: ${FILE_CHUNK_SIZE:5242880} # 5MB - chunk-size: ${FILE_CHUNK_SIZE:1048576} # 1MB +# chunk-size: ${FILE_CHUNK_SIZE:1048576} # 1MB + chunk-size: ${FILE_CHUNK_SIZE:1000000} # 1MB max-file-size: ${MAX_FILE_SIZE:10737418240} # 10GB max-retry-attempts: ${MAX_RETRY_ATTEMPTS:3} chunk-timeout-ms: ${CHUNK_TIMEOUT_MS:300000} # 5 минут From 3e06040123c7a39579068f8cf046e44d18c8eeaf Mon Sep 17 00:00:00 2001 From: vladp Date: Fri, 9 Jan 2026 22:55:06 +0700 Subject: [PATCH 06/27] NCBACK-27 fix build --- .../nocopy/handler/FileEntityHandler.java | 5 +- .../soune/nocopy/service/AuthServiceTest.java | 102 ------------------ 2 files changed, 1 insertion(+), 106 deletions(-) delete mode 100644 src/test/java/ru/soune/nocopy/service/AuthServiceTest.java diff --git a/src/main/java/ru/soune/nocopy/handler/FileEntityHandler.java b/src/main/java/ru/soune/nocopy/handler/FileEntityHandler.java index 040b037..a869f40 100644 --- a/src/main/java/ru/soune/nocopy/handler/FileEntityHandler.java +++ b/src/main/java/ru/soune/nocopy/handler/FileEntityHandler.java @@ -10,7 +10,6 @@ import ru.soune.nocopy.entity.file.FileEntity; import ru.soune.nocopy.entity.file.FileStatus; import ru.soune.nocopy.exception.FileEntityNotFoundException; import ru.soune.nocopy.exception.NotFoundAuthToken; -import ru.soune.nocopy.repository.AuthTokenRepository; import ru.soune.nocopy.repository.FileEntityRepository; import ru.soune.nocopy.service.auth.AuthService; import ru.soune.nocopy.service.file.FileEntityService; @@ -30,8 +29,6 @@ public class FileEntityHandler implements RequestHandler { private final FileStatsService fileStatsService; - private final AuthTokenRepository authTokenRepository; - private final ObjectMapper objectMapper; private final FileEntityRepository fileEntityRepository; @@ -82,7 +79,7 @@ public class FileEntityHandler implements RequestHandler { } private BaseResponse handleGetFilesUserInfo(BaseRequest request, FileEntityRequest fileRequest) { - Long userId = getUserIdFromToken(fileRequest.getToken()); + Long userId = authService.useUserAuthToken(fileRequest.getToken()); FileInfoUserResponse userFileStats = fileStatsService.getUserFileStats(userId); return new BaseResponse(request.getMsgId(), diff --git a/src/test/java/ru/soune/nocopy/service/AuthServiceTest.java b/src/test/java/ru/soune/nocopy/service/AuthServiceTest.java deleted file mode 100644 index a16af7d..0000000 --- a/src/test/java/ru/soune/nocopy/service/AuthServiceTest.java +++ /dev/null @@ -1,102 +0,0 @@ -package ru.soune.nocopy.service; - -import org.junit.jupiter.api.BeforeEach; -import org.junit.jupiter.api.Test; -import org.mockito.ArgumentCaptor; -import org.mockito.InjectMocks; -import org.mockito.Mock; -import org.mockito.MockitoAnnotations; -import org.springframework.context.MessageSource; -import org.springframework.security.crypto.password.PasswordEncoder; -import ru.soune.nocopy.dto.LoginRequest; -import ru.soune.nocopy.entity.AuthToken; -import ru.soune.nocopy.entity.User; -import ru.soune.nocopy.exception.NotValidationPasswordException; -import ru.soune.nocopy.exception.UserNotFoundException; -import ru.soune.nocopy.repository.AuthTokenRepository; -import ru.soune.nocopy.repository.UserRepository; -import ru.soune.nocopy.service.auth.AuthService; - -import java.util.Optional; - -import static org.junit.jupiter.api.Assertions.*; -import static org.mockito.ArgumentMatchers.any; -import static org.mockito.Mockito.verify; -import static org.mockito.Mockito.when; - -public class AuthServiceTest { - - @Mock - private UserRepository userRepository; - - @Mock - private AuthTokenRepository authTokenRepository; - - @Mock - private PasswordEncoder passwordEncoder; - - @Mock - private MessageSource messageSource; - - @InjectMocks - private AuthService authService; - - @BeforeEach - void setUp() { - MockitoAnnotations.openMocks(this); - } - - @Test - void login_success() { - LoginRequest request = new LoginRequest("test@mail.com", "password"); - - User user = new User(); - user.setId(1L); - user.setEmail("test@mail.com"); - user.setPassword("encoded_pass"); - - when(userRepository.findByEmail("test@mail.com")) - .thenReturn(Optional.of(user)); - when(passwordEncoder.matches("password", "encoded_pass")) - .thenReturn(true); - when(authTokenRepository.save(any(AuthToken.class))) - .thenAnswer(invocation -> invocation.getArgument(0)); - - AuthToken token = authService.login(request); - - assertNotNull(token); - assertNotNull(token.getToken()); -// assertEquals(user.getTokens().get(0), token.getUser()); - - ArgumentCaptor captor = ArgumentCaptor.forClass(AuthToken.class); - verify(authTokenRepository).save(captor.capture()); - -// assertEquals(user, captor.getValue().getUser()); - assertNotNull(captor.getValue().getToken()); - } - - - @Test - void login_userNotFound() { - LoginRequest request = new LoginRequest("missing@mail.com", "pwd"); - - when(userRepository.findByEmail("missing@mail.com")).thenReturn(Optional.empty()); - - assertThrows(UserNotFoundException.class, () -> authService.login(request)); - } - - - @Test - void login_invalidPassword() { - LoginRequest request = new LoginRequest("test@mail.com", "wrong"); - - User user = new User(); - user.setEmail("test@mail.com"); - user.setPassword("encoded_pass"); - - when(userRepository.findByEmail("test@mail.com")).thenReturn(Optional.of(user)); - when(passwordEncoder.matches("wrong", "encoded_pass")).thenReturn(false); - - assertThrows(NotValidationPasswordException.class, () -> authService.login(request)); - } -} From d896f238724218b2763649561aa48fabe506d46d Mon Sep 17 00:00:00 2001 From: vladp Date: Fri, 9 Jan 2026 23:05:37 +0700 Subject: [PATCH 07/27] add user_files_info for avaibale actions --- src/main/java/ru/soune/nocopy/handler/FileEntityHandler.java | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/main/java/ru/soune/nocopy/handler/FileEntityHandler.java b/src/main/java/ru/soune/nocopy/handler/FileEntityHandler.java index cacf0eb..a7f9c73 100644 --- a/src/main/java/ru/soune/nocopy/handler/FileEntityHandler.java +++ b/src/main/java/ru/soune/nocopy/handler/FileEntityHandler.java @@ -62,7 +62,8 @@ public class FileEntityHandler implements RequestHandler { .action(action) .availableActions(Arrays.asList( "file_info", "file_by_session", "user_files", - "search_files", "storage_usage", "delete_file")) + "search_files", "storage_usage", "delete_file", + "user_files_info")) .build(); return new BaseResponse(request.getMsgId(), From b3a3d57ee3e8442b982600b70a942fa6e808d3f3 Mon Sep 17 00:00:00 2001 From: vladp Date: Fri, 9 Jan 2026 23:41:19 +0700 Subject: [PATCH 08/27] NCBACK-3 --- .../nocopy/configuration/file/YandexConfig.java | 14 -------------- .../soune/nocopy/service/YandexSearchService.java | 11 +++++++++-- 2 files changed, 9 insertions(+), 16 deletions(-) delete mode 100644 src/main/java/ru/soune/nocopy/configuration/file/YandexConfig.java diff --git a/src/main/java/ru/soune/nocopy/configuration/file/YandexConfig.java b/src/main/java/ru/soune/nocopy/configuration/file/YandexConfig.java deleted file mode 100644 index e9f1576..0000000 --- a/src/main/java/ru/soune/nocopy/configuration/file/YandexConfig.java +++ /dev/null @@ -1,14 +0,0 @@ -package ru.soune.nocopy.configuration.file; - -import lombok.Data; -import org.springframework.boot.context.properties.ConfigurationProperties; -import org.springframework.context.annotation.Configuration; - -@Data -@Configuration -@ConfigurationProperties(prefix = "yandex") -public class YandexConfig { - private String apiKey; - private String folderId; - private String searchUrl; -} \ No newline at end of file diff --git a/src/main/java/ru/soune/nocopy/service/YandexSearchService.java b/src/main/java/ru/soune/nocopy/service/YandexSearchService.java index 3fd8c60..6c3d32f 100644 --- a/src/main/java/ru/soune/nocopy/service/YandexSearchService.java +++ b/src/main/java/ru/soune/nocopy/service/YandexSearchService.java @@ -54,6 +54,7 @@ public class YandexSearchService { MessageCode.FILE_NOT_FOUND.getCode(), MessageCode.FILE_NOT_FOUND.getDescription(), Map.of("fileId",fileId))); }); + byte[] fileBytes; if (!isImageFile(fileEntity)) { log.error("File not image: {}", fileEntity.getMimeType()); @@ -62,7 +63,14 @@ public class YandexSearchService { Map.of("file_type", fileEntity.getMimeType()))); } - byte[] fileBytes = readFileFromDisk(fileEntity); + try { + fileBytes = readFileFromDisk(fileEntity); + } catch (IOException e) { + throw new NotValidFieldException("File not found or cannot read file", new BaseResponse(20007, + MessageCode.FILE_NOT_FOUND.getCode(), MessageCode.FILE_NOT_FOUND.getDescription(), + Map.of("fileId", fileId, + "filePath", fileEntity.getFilePath()))); + } return callYandexApi(fileBytes); } @@ -99,7 +107,6 @@ public class YandexSearchService { connection.setRequestProperty("Authorization", "Api-Key " + apiKey); connection.setRequestProperty("Content-Type", "application/json"); connection.setRequestProperty("Accept", "application/json"); -// connection.setRequestProperty("User-Agent", "Mozilla/5.0"); connection.setConnectTimeout(30000); connection.setReadTimeout(30000); connection.setDoOutput(true); From 7b1ed5fe679b3a0e7bf22a0555d3ee3f4d34ba21 Mon Sep 17 00:00:00 2001 From: vladp Date: Sat, 10 Jan 2026 00:35:29 +0700 Subject: [PATCH 09/27] NCBACK-3 --- docker-compose.yaml | 6 ++++-- .../ru/soune/nocopy/handler/ImageFoundRequestHandler.java | 2 ++ .../java/ru/soune/nocopy/service/YandexSearchService.java | 6 +++--- src/main/resources/application.yaml | 5 ----- 4 files changed, 9 insertions(+), 10 deletions(-) diff --git a/docker-compose.yaml b/docker-compose.yaml index 0a65adf..06a38f1 100644 --- a/docker-compose.yaml +++ b/docker-compose.yaml @@ -65,9 +65,11 @@ services: POSTGRES_PASSWORD: postgres POSTGRES_PORT: 5432 POSTGRES_HOST: db - YANDEX_SEARCH_API_KEY: AQVNyaVaUmgUb1GMCtf5zSEqFxy0woXrcMOOB43q - YANDEX_SEARCH_FOLDER_ID: b1gokpdbm6qfpsou8pcd + YANDEX_API_KEY: AQVNyaVaUmgUb1GMCtf5zSEqFxy0woXrcMOOB43q + YANDEX_FOLDER_ID: b1gokpdbm6qfpsou8pcd + YANDEX_SEARCH_URL: "https://searchapi.api.cloud.yandex.net/v2/image/search_by_image" STORAGE_SERVICE_URL: http://storage:8081 + SPRING_PROFILES_ACTIVE: docker depends_on: - db ports: diff --git a/src/main/java/ru/soune/nocopy/handler/ImageFoundRequestHandler.java b/src/main/java/ru/soune/nocopy/handler/ImageFoundRequestHandler.java index 301bb10..dfe67cc 100644 --- a/src/main/java/ru/soune/nocopy/handler/ImageFoundRequestHandler.java +++ b/src/main/java/ru/soune/nocopy/handler/ImageFoundRequestHandler.java @@ -3,6 +3,7 @@ package ru.soune.nocopy.handler; import com.fasterxml.jackson.databind.ObjectMapper; import lombok.RequiredArgsConstructor; import lombok.extern.slf4j.Slf4j; +import org.springframework.beans.factory.annotation.Autowired; import org.springframework.stereotype.Component; import ru.soune.nocopy.dto.BaseRequest; import ru.soune.nocopy.dto.BaseResponse; @@ -18,6 +19,7 @@ public class ImageFoundRequestHandler implements RequestHandler { private final ObjectMapper objectMapper; + @Autowired private final YandexSearchService yandexSearchService; @Override diff --git a/src/main/java/ru/soune/nocopy/service/YandexSearchService.java b/src/main/java/ru/soune/nocopy/service/YandexSearchService.java index 6c3d32f..66f3f18 100644 --- a/src/main/java/ru/soune/nocopy/service/YandexSearchService.java +++ b/src/main/java/ru/soune/nocopy/service/YandexSearchService.java @@ -32,13 +32,13 @@ public class YandexSearchService { private final FileEntityRepository fileEntityRepository; private final ObjectMapper objectMapper; - @Value("${yandex.api-key}") + @Value("${YANDEX_API_KEY}") private String apiKey; - @Value("${yandex.folder-id}") + @Value("${YANDEX_FOLDER_ID}") private String folderId; - @Value("${yandex.search-url}") + @Value("${YANDEX_SEARCH_URL}") private String searchUrl; @PostConstruct diff --git a/src/main/resources/application.yaml b/src/main/resources/application.yaml index b58b4bc..da03a23 100644 --- a/src/main/resources/application.yaml +++ b/src/main/resources/application.yaml @@ -40,11 +40,6 @@ security: allowed-headers: "*" allow-credentials: true -yandex: - api-key: ${YANDEX_SEARCH_API_KEY} - folder-id: ${YANDEX_SEARCH_FOLDER_ID} - search-url: "https://searchapi.api.cloud.yandex.net/v2/image/search_by_image" - server: port: ${SERVER_PORT:8080} From 920cde3cdf1ec8f2e21fd5de1f027c2012cab4be Mon Sep 17 00:00:00 2001 From: vladp Date: Sat, 10 Jan 2026 00:51:03 +0700 Subject: [PATCH 10/27] Dev --- Dockerfile | 2 +- docker-compose.yaml | 6 +++--- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/Dockerfile b/Dockerfile index 6cd62ca..9fda8a5 100644 --- a/Dockerfile +++ b/Dockerfile @@ -16,4 +16,4 @@ RUN mkdir -p /data/uploads && chmod 755 /data/uploads EXPOSE 8080 -CMD ["java", "-jar", "app.jar"] +ENTRYPOINT ["sh", "-c", "exec java ${JAVA_OPTS} -jar app.jar"] diff --git a/docker-compose.yaml b/docker-compose.yaml index 06a38f1..7ee937f 100644 --- a/docker-compose.yaml +++ b/docker-compose.yaml @@ -65,11 +65,11 @@ services: POSTGRES_PASSWORD: postgres POSTGRES_PORT: 5432 POSTGRES_HOST: db - YANDEX_API_KEY: AQVNyaVaUmgUb1GMCtf5zSEqFxy0woXrcMOOB43q - YANDEX_FOLDER_ID: b1gokpdbm6qfpsou8pcd - YANDEX_SEARCH_URL: "https://searchapi.api.cloud.yandex.net/v2/image/search_by_image" STORAGE_SERVICE_URL: http://storage:8081 SPRING_PROFILES_ACTIVE: docker + + JAVA_OPTS: "-DYANDEX_API_KEY=AQVNyaVaUmgUb1GMCtf5zSEqFxy0woXrcMOOB43q -DYANDEX_FOLDER_ID=b1gokpdbm6qfpsou8pcd -DYANDEX_SEARCH_URL='https://searchapi.api.cloud.yandex.net/v2/image/search_by_image'" + depends_on: - db ports: From eb12e219ec161b3c04d7cf625fdd34d17e93c08c Mon Sep 17 00:00:00 2001 From: vladp Date: Sat, 10 Jan 2026 00:54:49 +0700 Subject: [PATCH 11/27] Dev --- Dockerfile | 2 +- docker-compose.yaml | 3 +-- .../ru/soune/nocopy/service/YandexSearchService.java | 9 ++++----- src/main/resources/application.yaml | 5 +++++ 4 files changed, 11 insertions(+), 8 deletions(-) diff --git a/Dockerfile b/Dockerfile index 9fda8a5..03d3325 100644 --- a/Dockerfile +++ b/Dockerfile @@ -16,4 +16,4 @@ RUN mkdir -p /data/uploads && chmod 755 /data/uploads EXPOSE 8080 -ENTRYPOINT ["sh", "-c", "exec java ${JAVA_OPTS} -jar app.jar"] +CMD ["java", "-jar", "app.jar"] \ No newline at end of file diff --git a/docker-compose.yaml b/docker-compose.yaml index 7ee937f..0ef8c6b 100644 --- a/docker-compose.yaml +++ b/docker-compose.yaml @@ -67,8 +67,7 @@ services: POSTGRES_HOST: db STORAGE_SERVICE_URL: http://storage:8081 SPRING_PROFILES_ACTIVE: docker - - JAVA_OPTS: "-DYANDEX_API_KEY=AQVNyaVaUmgUb1GMCtf5zSEqFxy0woXrcMOOB43q -DYANDEX_FOLDER_ID=b1gokpdbm6qfpsou8pcd -DYANDEX_SEARCH_URL='https://searchapi.api.cloud.yandex.net/v2/image/search_by_image'" + SPRING_APPLICATION_JSON: '{"yandex":{"api-key":"AQVNyaVaUmgUb1GMCtf5zSEqFxy0woXrcMOOB43q","folder-id":"b1gokpdbm6qfpsou8pcd","search-url":"https://searchapi.api.cloud.yandex.net/v2/image/search_by_image"}}' depends_on: - db diff --git a/src/main/java/ru/soune/nocopy/service/YandexSearchService.java b/src/main/java/ru/soune/nocopy/service/YandexSearchService.java index 66f3f18..257fd90 100644 --- a/src/main/java/ru/soune/nocopy/service/YandexSearchService.java +++ b/src/main/java/ru/soune/nocopy/service/YandexSearchService.java @@ -28,19 +28,18 @@ import java.util.Map; @Service @RequiredArgsConstructor public class YandexSearchService { - private final FileEntityRepository fileEntityRepository; + private final ObjectMapper objectMapper; - @Value("${YANDEX_API_KEY}") + @Value("${yandex.api-key}") private String apiKey; - @Value("${YANDEX_FOLDER_ID}") + @Value("${yandex.folder-id}") private String folderId; - @Value("${YANDEX_SEARCH_URL}") + @Value("${yandex.search-url}") private String searchUrl; - @PostConstruct public void init() { objectMapper.configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, false); diff --git a/src/main/resources/application.yaml b/src/main/resources/application.yaml index da03a23..77d6ac6 100644 --- a/src/main/resources/application.yaml +++ b/src/main/resources/application.yaml @@ -40,6 +40,11 @@ security: allowed-headers: "*" allow-credentials: true +yandex: + api-key: ${yandex.api-key} + folder-id: ${yandex.folder-id} + search-url: ${yandex.search-url} + server: port: ${SERVER_PORT:8080} From 7b8546fd4d4fda71407b5426d6b1e5922ac0cdd9 Mon Sep 17 00:00:00 2001 From: vladp Date: Sat, 10 Jan 2026 01:06:32 +0700 Subject: [PATCH 12/27] Dev --- docker-compose.yaml | 5 +++-- .../nocopy/handler/ImageFoundRequestHandler.java | 1 - .../ru/soune/nocopy/service/YandexSearchService.java | 1 + src/main/resources/application.yaml | 12 ++++++------ 4 files changed, 10 insertions(+), 9 deletions(-) diff --git a/docker-compose.yaml b/docker-compose.yaml index 0ef8c6b..192b2d6 100644 --- a/docker-compose.yaml +++ b/docker-compose.yaml @@ -67,8 +67,9 @@ services: POSTGRES_HOST: db STORAGE_SERVICE_URL: http://storage:8081 SPRING_PROFILES_ACTIVE: docker - SPRING_APPLICATION_JSON: '{"yandex":{"api-key":"AQVNyaVaUmgUb1GMCtf5zSEqFxy0woXrcMOOB43q","folder-id":"b1gokpdbm6qfpsou8pcd","search-url":"https://searchapi.api.cloud.yandex.net/v2/image/search_by_image"}}' - + YANDEX_API_KEY: AQVNyaVaUmgUb1GMCtf5zSEqFxy0woXrcMOOB43q + YANDEX_FOLDER_ID: b1gokpdbm6qfpsou8pcd + YANDEX_SEARCH_URL: "https://searchapi.api.cloud.yandex.net/v2/image/search_by_image" depends_on: - db ports: diff --git a/src/main/java/ru/soune/nocopy/handler/ImageFoundRequestHandler.java b/src/main/java/ru/soune/nocopy/handler/ImageFoundRequestHandler.java index dfe67cc..3d13d4a 100644 --- a/src/main/java/ru/soune/nocopy/handler/ImageFoundRequestHandler.java +++ b/src/main/java/ru/soune/nocopy/handler/ImageFoundRequestHandler.java @@ -19,7 +19,6 @@ public class ImageFoundRequestHandler implements RequestHandler { private final ObjectMapper objectMapper; - @Autowired private final YandexSearchService yandexSearchService; @Override diff --git a/src/main/java/ru/soune/nocopy/service/YandexSearchService.java b/src/main/java/ru/soune/nocopy/service/YandexSearchService.java index 257fd90..d0f4710 100644 --- a/src/main/java/ru/soune/nocopy/service/YandexSearchService.java +++ b/src/main/java/ru/soune/nocopy/service/YandexSearchService.java @@ -40,6 +40,7 @@ public class YandexSearchService { @Value("${yandex.search-url}") private String searchUrl; + @PostConstruct public void init() { objectMapper.configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, false); diff --git a/src/main/resources/application.yaml b/src/main/resources/application.yaml index 77d6ac6..35e3fc8 100644 --- a/src/main/resources/application.yaml +++ b/src/main/resources/application.yaml @@ -40,11 +40,6 @@ security: allowed-headers: "*" allow-credentials: true -yandex: - api-key: ${yandex.api-key} - folder-id: ${yandex.folder-id} - search-url: ${yandex.search-url} - server: port: ${SERVER_PORT:8080} @@ -56,4 +51,9 @@ logging: file: name: logs/application.log pattern: - console: "%d{yyyy-MM-dd HH:mm:ss} [%thread] %-5level %logger{36} - %msg%n" \ No newline at end of file + console: "%d{yyyy-MM-dd HH:mm:ss} [%thread] %-5level %logger{36} - %msg%n" + +yandex: + api-key: ${YANDEX_API_KEY} + folder-id: ${YANDEX_FOLDER_ID} + search-url: ${YANDEX_SEARCH_URL} \ No newline at end of file From 344ec075d45526475a3d284c0dcfad57a4d6b9ec Mon Sep 17 00:00:00 2001 From: vladp Date: Sat, 10 Jan 2026 01:19:30 +0700 Subject: [PATCH 13/27] Dev --- src/main/resources/application.yaml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/main/resources/application.yaml b/src/main/resources/application.yaml index 35e3fc8..9a34782 100644 --- a/src/main/resources/application.yaml +++ b/src/main/resources/application.yaml @@ -54,6 +54,6 @@ logging: console: "%d{yyyy-MM-dd HH:mm:ss} [%thread] %-5level %logger{36} - %msg%n" yandex: - api-key: ${YANDEX_API_KEY} - folder-id: ${YANDEX_FOLDER_ID} - search-url: ${YANDEX_SEARCH_URL} \ No newline at end of file + api-key: ${YANDEX_API_KEY:AQVNyaVaUmgUb1GMCtf5zSEqFxy0woXrcMOOB43q} + folder-id: ${YANDEX_FOLDER_ID:b1gokpdbm6qfpsou8pcd} + search-url: ${YANDEX_SEARCH_URL:https://searchapi.api.cloud.yandex.net/v2/image/search_by_image} \ No newline at end of file From 2b7a8b97e1a0d5834586a1eb2e77f21c676a3848 Mon Sep 17 00:00:00 2001 From: vladp Date: Sat, 10 Jan 2026 02:09:39 +0700 Subject: [PATCH 14/27] NCBACK-27 fix token life --- .../nocopy/repository/AuthTokenRepository.java | 3 ++- .../ru/soune/nocopy/service/auth/AuthService.java | 1 + .../service/auth/CleanupTokenSessionsService.java | 13 ++++++------- 3 files changed, 9 insertions(+), 8 deletions(-) diff --git a/src/main/java/ru/soune/nocopy/repository/AuthTokenRepository.java b/src/main/java/ru/soune/nocopy/repository/AuthTokenRepository.java index 9328753..cbd7529 100644 --- a/src/main/java/ru/soune/nocopy/repository/AuthTokenRepository.java +++ b/src/main/java/ru/soune/nocopy/repository/AuthTokenRepository.java @@ -5,11 +5,12 @@ import org.springframework.stereotype.Repository; import ru.soune.nocopy.entity.AuthToken; import java.time.LocalDateTime; +import java.util.List; import java.util.Optional; @Repository public interface AuthTokenRepository extends JpaRepository { - Optional findByLastUsedAtAndExpiresAtBefore(LocalDateTime lastUsedAt, LocalDateTime expiresAtBefore); + List findByExpiresAtBefore(LocalDateTime expiresAtBefore); Optional findByLastUsedAtBefore(LocalDateTime lastUsedAt); Optional findByToken(String token); } diff --git a/src/main/java/ru/soune/nocopy/service/auth/AuthService.java b/src/main/java/ru/soune/nocopy/service/auth/AuthService.java index cf8b18c..f14d6d1 100644 --- a/src/main/java/ru/soune/nocopy/service/auth/AuthService.java +++ b/src/main/java/ru/soune/nocopy/service/auth/AuthService.java @@ -92,6 +92,7 @@ public class AuthService { AuthToken authToken = authTokenRepository.findByToken(token) .orElseThrow(() -> new NotFoundAuthToken("Token not found")); authToken.setLastUsedAt(LocalDateTime.now()); + authToken.setExpiresAt(LocalDateTime.now().plusHours(1)); authTokenRepository.save(authToken); diff --git a/src/main/java/ru/soune/nocopy/service/auth/CleanupTokenSessionsService.java b/src/main/java/ru/soune/nocopy/service/auth/CleanupTokenSessionsService.java index c18cda3..5ac5128 100644 --- a/src/main/java/ru/soune/nocopy/service/auth/CleanupTokenSessionsService.java +++ b/src/main/java/ru/soune/nocopy/service/auth/CleanupTokenSessionsService.java @@ -25,14 +25,13 @@ public class CleanupTokenSessionsService { @Transactional @Scheduled(fixedDelay = 30000) - public void cleanupExpiredFiles() { - LocalDateTime hourBefore = LocalDateTime.now().minusHours(authTokenLife); + public void cleanupExpiredTokens() { + LocalDateTime now = LocalDateTime.now(); - List tokensForDelete = authTokenRepository.findByLastUsedAtBefore(hourBefore).stream().toList(); - List tokensDeleteWithNullUpdate = authTokenRepository.findByLastUsedAtAndExpiresAtBefore( - null, hourBefore).stream().toList(); + List expiredTokens = authTokenRepository.findByExpiresAtBefore(now); - authTokenRepository.deleteAll(tokensForDelete); - authTokenRepository.deleteAll(tokensDeleteWithNullUpdate); + if (!expiredTokens.isEmpty()) { + authTokenRepository.deleteAll(expiredTokens); + } } } From 8ac73ff045d0d937c01639990a452715c625c79a Mon Sep 17 00:00:00 2001 From: vladp Date: Tue, 13 Jan 2026 22:40:52 +0700 Subject: [PATCH 15/27] NCBACK-34 add logic to find similar image in db --- build.gradle | 2 + .../nocopy/controller/ApiController.java | 23 ++- .../java/ru/soune/nocopy/dto/MessageCode.java | 1 + .../nocopy/dto/file/SimilarFileResponse.java | 14 ++ .../nocopy/entity/file/FileUploadSession.java | 2 + .../nocopy/entity/file/ImageHashEntity.java | 39 +++++ .../exception/DuplicateImageException.java | 7 + .../nocopy/handler/FileUploadHandler.java | 2 +- .../repository/ImageHashRepository.java | 9 + .../repository/ImageSimilarityRepository.java | 35 ++++ .../repository/SimilarImageProjection.java | 14 ++ .../nocopy/service/FileSimilarityService.java | 62 +++++++ .../nocopy/service/ImageHashService.java | 154 ++++++++++++++++++ .../service/file/FileEntityService.java | 73 +++------ .../service/file/FileUploadServiceImpl.java | 38 +---- src/main/resources/application.yaml | 4 + .../V1__add_image_hash_prefix_index.sql | 2 + .../soune/nocopy/service/AuthServiceTest.java | 101 ------------ 18 files changed, 394 insertions(+), 188 deletions(-) create mode 100644 src/main/java/ru/soune/nocopy/dto/file/SimilarFileResponse.java create mode 100644 src/main/java/ru/soune/nocopy/entity/file/ImageHashEntity.java create mode 100644 src/main/java/ru/soune/nocopy/exception/DuplicateImageException.java create mode 100644 src/main/java/ru/soune/nocopy/repository/ImageHashRepository.java create mode 100644 src/main/java/ru/soune/nocopy/repository/ImageSimilarityRepository.java create mode 100644 src/main/java/ru/soune/nocopy/repository/SimilarImageProjection.java create mode 100644 src/main/java/ru/soune/nocopy/service/FileSimilarityService.java create mode 100644 src/main/java/ru/soune/nocopy/service/ImageHashService.java create mode 100644 src/main/resources/db/migration/V1__add_image_hash_prefix_index.sql delete mode 100644 src/test/java/ru/soune/nocopy/service/AuthServiceTest.java diff --git a/build.gradle b/build.gradle index 111da7e..7e7901d 100644 --- a/build.gradle +++ b/build.gradle @@ -31,6 +31,8 @@ dependencies { implementation 'org.mapstruct:mapstruct:1.5.5.Final' implementation 'commons-validator:commons-validator:1.7' + implementation 'org.flywaydb:flyway-core:9.22.0' + implementation 'com.fasterxml.jackson.dataformat:jackson-dataformat-xml:2.17.0' implementation 'tools.jackson.core:jackson-core:3.0.3' implementation 'jakarta.xml.bind:jakarta.xml.bind-api:4.0.0' diff --git a/src/main/java/ru/soune/nocopy/controller/ApiController.java b/src/main/java/ru/soune/nocopy/controller/ApiController.java index 4f9239d..babc9e4 100644 --- a/src/main/java/ru/soune/nocopy/controller/ApiController.java +++ b/src/main/java/ru/soune/nocopy/controller/ApiController.java @@ -15,16 +15,14 @@ import ru.soune.nocopy.dto.BaseRequest; import ru.soune.nocopy.dto.BaseResponse; import ru.soune.nocopy.dto.MessageCode; import ru.soune.nocopy.dto.RegAnswer; -import ru.soune.nocopy.dto.file.ChunkUploadResponse; -import ru.soune.nocopy.dto.file.CompleteUploadResponse; -import ru.soune.nocopy.dto.file.FileEntityResponse; -import ru.soune.nocopy.dto.file.UploadProgress; +import ru.soune.nocopy.dto.file.*; import ru.soune.nocopy.entity.AuthToken; import ru.soune.nocopy.entity.file.FileStatus; import ru.soune.nocopy.entity.file.UploadStatus; import ru.soune.nocopy.exception.*; import ru.soune.nocopy.handler.*; import ru.soune.nocopy.repository.AuthTokenRepository; +import ru.soune.nocopy.service.FileSimilarityService; import ru.soune.nocopy.service.file.FileEntityService; import ru.soune.nocopy.service.file.FileUploadService; @@ -50,6 +48,8 @@ public class ApiController { private final AuthTokenRepository authTokenRepository; + private final FileSimilarityService fileSimilarityService; + @PostMapping("/v{version}/data") public ResponseEntity handlePostRequest(@RequestBody BaseRequest request, @PathVariable("version") int version) { @@ -127,6 +127,16 @@ public class ApiController { return ResponseEntity.ok().body(new BaseResponse(20004, MessageCode.SUCCESS.getCode(), "Chunk uploaded successfully", responseBody)); + } catch (DuplicateImageException e) { + log.error("Error uploading chunk", e); + ChunkUploadResponse responseBody = ChunkUploadResponse.builder() + .uploadId(uploadId) + .chunkNumber(chunkNumber) + .build(); + + return ResponseEntity.ok().body(new BaseResponse( + 20004, MessageCode.DUPLICATE_FILE_UPLOAD.getCode(), + "Failed to upload chunk: " + e.getMessage(), responseBody)); } catch (Exception e) { log.error("Error uploading chunk", e); @@ -141,6 +151,11 @@ public class ApiController { } } + @GetMapping("/files/{fileId}/similar") + public List getSimilarFiles(@PathVariable String fileId) { + return fileSimilarityService.findSimilarFiles(fileId); + } + @GetMapping("/v{version}/files/progress/{uploadId}") public ResponseEntity getUploadProgress( @PathVariable("version") int version, diff --git a/src/main/java/ru/soune/nocopy/dto/MessageCode.java b/src/main/java/ru/soune/nocopy/dto/MessageCode.java index 5c75246..33faabc 100644 --- a/src/main/java/ru/soune/nocopy/dto/MessageCode.java +++ b/src/main/java/ru/soune/nocopy/dto/MessageCode.java @@ -8,6 +8,7 @@ public enum MessageCode { INVALID_TOKEN(2, "Invalid token"), INVALID_ACTION(2, "Invalid action"), FILE_UPLOAD_ERROR(2, "File upload error"), + DUPLICATE_FILE_UPLOAD(2, "Duplicate file upload"), FILE_DOWNLOAD_ERROR(2, "File download error"), FILE_DOWNLOAD_ERROR_NOT_CORRECT_FIELD(2, "Not correct field"), IMAGE_FOUND_ERROR(2, "Image found error"), diff --git a/src/main/java/ru/soune/nocopy/dto/file/SimilarFileResponse.java b/src/main/java/ru/soune/nocopy/dto/file/SimilarFileResponse.java new file mode 100644 index 0000000..f7aae6e --- /dev/null +++ b/src/main/java/ru/soune/nocopy/dto/file/SimilarFileResponse.java @@ -0,0 +1,14 @@ +package ru.soune.nocopy.dto.file; + +import lombok.Builder; +import lombok.Value; + +@Value +@Builder +public class SimilarFileResponse { + String fileId; + String originalFileName; + Long fileSize; + Integer hammingDistance; + String similarityLevel; +} diff --git a/src/main/java/ru/soune/nocopy/entity/file/FileUploadSession.java b/src/main/java/ru/soune/nocopy/entity/file/FileUploadSession.java index 509d0f7..80dc12f 100644 --- a/src/main/java/ru/soune/nocopy/entity/file/FileUploadSession.java +++ b/src/main/java/ru/soune/nocopy/entity/file/FileUploadSession.java @@ -69,6 +69,7 @@ public class FileUploadSession { private String extension; @Column(name = "retry_count") + @Builder.Default private Integer retryCount = 0; @Column(name = "completed_at") @@ -81,6 +82,7 @@ public class FileUploadSession { ) @MapKeyColumn(name = "chunk_number") @Column(name = "chunk_path") + @Builder.Default private Map chunkPaths = new HashMap<>(); @PrePersist diff --git a/src/main/java/ru/soune/nocopy/entity/file/ImageHashEntity.java b/src/main/java/ru/soune/nocopy/entity/file/ImageHashEntity.java new file mode 100644 index 0000000..37f1b5a --- /dev/null +++ b/src/main/java/ru/soune/nocopy/entity/file/ImageHashEntity.java @@ -0,0 +1,39 @@ +package ru.soune.nocopy.entity.file; + +import jakarta.persistence.*; +import lombok.AllArgsConstructor; +import lombok.Builder; +import lombok.Data; +import lombok.NoArgsConstructor; + +import java.time.LocalDateTime; + +@Entity +@Table(name = "image_hashes") +@Data +@NoArgsConstructor +@AllArgsConstructor +@Builder +public class ImageHashEntity { + + @Id + @Column(name = "file_id") + private String fileId; + + @OneToOne(fetch = FetchType.LAZY) + @MapsId + @JoinColumn(name = "file_id") + private FileEntity file; + + @Column(name = "hash64_hi") + private Integer hash64Hi; + + @Column(name = "hash64_lo") + private Integer hash64Lo; + + @Column(name = "hash_algorithm", nullable = false) + private String hashAlgorithm; + + @Column(name = "created_at") + private LocalDateTime createdAt; +} diff --git a/src/main/java/ru/soune/nocopy/exception/DuplicateImageException.java b/src/main/java/ru/soune/nocopy/exception/DuplicateImageException.java new file mode 100644 index 0000000..a7593b8 --- /dev/null +++ b/src/main/java/ru/soune/nocopy/exception/DuplicateImageException.java @@ -0,0 +1,7 @@ +package ru.soune.nocopy.exception; + +public class DuplicateImageException extends RuntimeException { + public DuplicateImageException(String message) { + super(message); + } +} diff --git a/src/main/java/ru/soune/nocopy/handler/FileUploadHandler.java b/src/main/java/ru/soune/nocopy/handler/FileUploadHandler.java index 4cb33b5..8240403 100644 --- a/src/main/java/ru/soune/nocopy/handler/FileUploadHandler.java +++ b/src/main/java/ru/soune/nocopy/handler/FileUploadHandler.java @@ -209,6 +209,7 @@ public class FileUploadHandler implements RequestHandler { .orElseThrow(() -> new UploadSessionNotFoundException(fileRequest.getUploadId())); Map chunkStatus = new HashMap<>(); + for (int i = 0; i < session.getTotalChunks(); i++) { chunkStatus.put("chunk_" + i, session.getChunkPaths().containsKey(i)); } @@ -223,7 +224,6 @@ public class FileUploadHandler implements RequestHandler { return new BaseResponse(request.getMsgId(), MessageCode.SUCCESS.getCode(), MessageCode.SUCCESS.getDescription(), response); - } catch (UploadSessionNotFoundException e) { return new BaseResponse(request.getMsgId(), MessageCode.FILE_NOT_FOUND.getCode(), "Upload session not found", null); diff --git a/src/main/java/ru/soune/nocopy/repository/ImageHashRepository.java b/src/main/java/ru/soune/nocopy/repository/ImageHashRepository.java new file mode 100644 index 0000000..e1ff003 --- /dev/null +++ b/src/main/java/ru/soune/nocopy/repository/ImageHashRepository.java @@ -0,0 +1,9 @@ +package ru.soune.nocopy.repository; + +import org.springframework.data.jpa.repository.JpaRepository; +import org.springframework.stereotype.Repository; +import ru.soune.nocopy.entity.file.ImageHashEntity; + +@Repository +public interface ImageHashRepository extends JpaRepository { +} diff --git a/src/main/java/ru/soune/nocopy/repository/ImageSimilarityRepository.java b/src/main/java/ru/soune/nocopy/repository/ImageSimilarityRepository.java new file mode 100644 index 0000000..eeabe3a --- /dev/null +++ b/src/main/java/ru/soune/nocopy/repository/ImageSimilarityRepository.java @@ -0,0 +1,35 @@ +package ru.soune.nocopy.repository; + +import org.springframework.data.jpa.repository.JpaRepository; +import org.springframework.data.jpa.repository.Query; +import org.springframework.data.repository.query.Param; +import org.springframework.stereotype.Repository; +import ru.soune.nocopy.entity.file.FileEntity; + +import java.util.List; + +@Repository +public interface ImageSimilarityRepository + extends JpaRepository { + + @Query(value = """ + SELECT + f.id AS similarFileId, + f.original_file_name AS originalFileName, + f.file_size AS fileSize, + h.hash64_hi AS hash64Hi, + h.hash64_lo AS hash64Lo + FROM image_hashes ref + JOIN image_hashes h + ON ref.file_id <> h.file_id + JOIN file_entities f + ON f.id = h.file_id + WHERE ref.file_id = :fileId + """, + nativeQuery = true) + List findCandidates( + @Param("fileId") String fileId + ); +} + + diff --git a/src/main/java/ru/soune/nocopy/repository/SimilarImageProjection.java b/src/main/java/ru/soune/nocopy/repository/SimilarImageProjection.java new file mode 100644 index 0000000..7263394 --- /dev/null +++ b/src/main/java/ru/soune/nocopy/repository/SimilarImageProjection.java @@ -0,0 +1,14 @@ +package ru.soune.nocopy.repository; + +public interface SimilarImageProjection { + + String getSimilarFileId(); + + String getOriginalFileName(); + + Long getFileSize(); + + Integer getHash64Hi(); + + Integer getHash64Lo(); +} \ No newline at end of file diff --git a/src/main/java/ru/soune/nocopy/service/FileSimilarityService.java b/src/main/java/ru/soune/nocopy/service/FileSimilarityService.java new file mode 100644 index 0000000..2c02234 --- /dev/null +++ b/src/main/java/ru/soune/nocopy/service/FileSimilarityService.java @@ -0,0 +1,62 @@ +package ru.soune.nocopy.service; + +import lombok.RequiredArgsConstructor; +import org.springframework.stereotype.Service; +import ru.soune.nocopy.dto.file.SimilarFileResponse; +import ru.soune.nocopy.repository.ImageHashRepository; +import ru.soune.nocopy.repository.ImageSimilarityRepository; +import ru.soune.nocopy.repository.SimilarImageProjection; + +import java.util.List; + +@Service +@RequiredArgsConstructor +public class FileSimilarityService { + + private final ImageSimilarityRepository repository; + private final ImageHashRepository hashRepository; + + public List findSimilarFiles(String fileId) { + var imageHashEntity = hashRepository.findById(fileId) + .orElseThrow(() -> new RuntimeException("Hash not found")); + + Integer hash64Hi = imageHashEntity.getHash64Hi(); + Integer hash64Lo = imageHashEntity.getHash64Lo(); + + List candidates = + repository.findCandidates(fileId); + + return candidates.stream() + .map(c -> { + Integer cHi = c.getHash64Hi(); + Integer cLo = c.getHash64Lo(); + + int hamming = hamming64(hash64Hi, hash64Lo, cHi, cLo); + + String level; + if (hamming <= 5) { + level = "DUPLICATE"; + } else if (hamming <= 12) { + level = "SIMILAR"; + } else { + level = "DIFFERENT"; + } + + return SimilarFileResponse.builder() + .fileId(c.getSimilarFileId()) + .originalFileName(c.getOriginalFileName()) + .fileSize(c.getFileSize()) + .hammingDistance(hamming) + .similarityLevel(level) + .build(); + }) + .sorted((a, b) -> + Integer.compare(a.getHammingDistance(), b.getHammingDistance())) + .toList(); + } + + private int hamming64(int aHi, int aLo, int bHi, int bLo) { + return Integer.bitCount(aHi ^ bHi) + + Integer.bitCount(aLo ^ bLo); + } +} diff --git a/src/main/java/ru/soune/nocopy/service/ImageHashService.java b/src/main/java/ru/soune/nocopy/service/ImageHashService.java new file mode 100644 index 0000000..992040b --- /dev/null +++ b/src/main/java/ru/soune/nocopy/service/ImageHashService.java @@ -0,0 +1,154 @@ +package ru.soune.nocopy.service; + +import lombok.RequiredArgsConstructor; +import lombok.extern.slf4j.Slf4j; +import org.springframework.stereotype.Service; +import ru.soune.nocopy.entity.file.FileEntity; +import ru.soune.nocopy.entity.file.ImageHashEntity; +import ru.soune.nocopy.repository.ImageHashRepository; + +import javax.imageio.ImageIO; +import java.awt.*; +import java.awt.image.BufferedImage; +import java.io.IOException; +import java.io.InputStream; +import java.nio.file.Files; +import java.nio.file.Path; +import java.time.LocalDateTime; +import java.util.Arrays; + +@Service +@RequiredArgsConstructor +@Slf4j +public class ImageHashService { + + private final ImageHashRepository repository; + + public void create(FileEntity file, Path imagePath) { + try { + long hash64 = computePhash64(imagePath); + + int hi = high32(hash64); + int lo = low32(hash64); + + ImageHashEntity entity = ImageHashEntity.builder() + .file(file) + .hash64Hi(hi) + .hash64Lo(lo) + .hashAlgorithm("PHASH64") + .createdAt(LocalDateTime.now()) + .build(); + + repository.save(entity); + } catch (IOException e) { + throw new RuntimeException("Failed to compute image hash", e); + } + } + + private long computePhash64(Path path) throws IOException { + try (InputStream is = Files.newInputStream(path)) { + BufferedImage image = ImageIO.read(is); + if (image == null) throw new IOException("Cannot read image file: " + path); + + BufferedImage gray = toGrayScale(image); + BufferedImage resized = resizeImage(gray, 32, 32); + + double[][] dct = applyDCT(resized); + double[][] topLeft = extractTopLeft8x8(dct); + double median = calculateMedian(topLeft, true); + + return create64BitHashLong(topLeft, median); + } + } + + private BufferedImage toGrayScale(BufferedImage image) { + BufferedImage gray = new BufferedImage(image.getWidth(), image.getHeight(), BufferedImage.TYPE_BYTE_GRAY); + Graphics2D g = gray.createGraphics(); + g.drawImage(image, 0, 0, null); + g.dispose(); + return gray; + } + + private BufferedImage resizeImage(BufferedImage image, int width, int height) { + BufferedImage resized = new BufferedImage(width, height, BufferedImage.TYPE_BYTE_GRAY); + Graphics2D g = resized.createGraphics(); + g.setRenderingHint(RenderingHints.KEY_INTERPOLATION, RenderingHints.VALUE_INTERPOLATION_BILINEAR); + g.drawImage(image, 0, 0, width, height, null); + g.dispose(); + return resized; + } + + private double[][] applyDCT(BufferedImage image) { + int w = image.getWidth(); + int h = image.getHeight(); + double[][] pixels = new double[h][w]; + + for (int y = 0; y < h; y++) { + for (int x = 0; x < w; x++) { + int gray = image.getRGB(x, y) & 0xFF; + pixels[y][x] = gray; + } + } + + double[][] dct = new double[h][w]; + for (int u = 0; u < h; u++) { + double au = u == 0 ? 1.0 / Math.sqrt(h) : Math.sqrt(2.0 / h); + for (int v = 0; v < w; v++) { + double av = v == 0 ? 1.0 / Math.sqrt(w) : Math.sqrt(2.0 / w); + double sum = 0.0; + for (int x = 0; x < h; x++) { + for (int y = 0; y < w; y++) { + sum += pixels[x][y] * + Math.cos((2 * x + 1) * u * Math.PI / (2.0 * h)) * + Math.cos((2 * y + 1) * v * Math.PI / (2.0 * w)); + } + } + dct[u][v] = au * av * sum; + } + } + return dct; + } + + private double[][] extractTopLeft8x8(double[][] dct) { + double[][] block = new double[8][8]; + for (int i = 0; i < 8; i++) System.arraycopy(dct[i], 0, block[i], 0, 8); + return block; + } + + private double calculateMedian(double[][] block, boolean excludeDC) { + double[] vals = new double[63]; + int idx = 0; + for (int i = 0; i < 8; i++) { + for (int j = 0; j < 8; j++) { + if (i == 0 && j == 0 && excludeDC) continue; + vals[idx++] = block[i][j]; + } + } + Arrays.sort(vals); + return (vals.length % 2 == 0) ? + (vals[vals.length / 2 - 1] + vals[vals.length / 2]) / 2.0 : + vals[vals.length / 2]; + } + + private long create64BitHashLong(double[][] block, double median) { + long hash = 0L; + + for (int i = 0; i < 8; i++) { + for (int j = 0; j < 8; j++) { + if (i == 0 && j == 0) continue; + hash <<= 1; + if (block[i][j] > median) hash |= 1; + } + } + + return hash; + } + + private int high32(long hash64) { + return (int) (hash64 >>> 32); + } + + private int low32(long hash64) { + return (int) hash64; + } +} diff --git a/src/main/java/ru/soune/nocopy/service/file/FileEntityService.java b/src/main/java/ru/soune/nocopy/service/file/FileEntityService.java index 617eeda..f72ac63 100644 --- a/src/main/java/ru/soune/nocopy/service/file/FileEntityService.java +++ b/src/main/java/ru/soune/nocopy/service/file/FileEntityService.java @@ -9,15 +9,17 @@ import ru.soune.nocopy.dto.file.FileResponse; import ru.soune.nocopy.entity.file.FileEntity; import ru.soune.nocopy.entity.file.FileStatus; import ru.soune.nocopy.entity.file.FileUploadSession; +import ru.soune.nocopy.exception.DuplicateImageException; import ru.soune.nocopy.exception.FileEntityNotFoundException; import ru.soune.nocopy.repository.FileEntityRepository; +import ru.soune.nocopy.service.FileSimilarityService; +import ru.soune.nocopy.service.ImageHashService; import java.io.IOException; import java.nio.file.Files; import java.nio.file.Path; import java.nio.file.Paths; import java.time.LocalDateTime; -import java.time.ZoneOffset; import java.util.List; import java.util.stream.Collectors; @@ -28,6 +30,10 @@ public class FileEntityService { private final FileEntityRepository fileEntityRepository; + private final ImageHashService imageHashService; + + private final FileSimilarityService fileSimilarityService; + @Transactional public FileEntity createFromUploadSession(FileUploadSession session, String checksum) { log.info("Creating FileEntity for upload session: {}", session.getUploadId()); @@ -57,11 +63,28 @@ public class FileEntityService { .build(); FileEntity saved = fileEntityRepository.save(fileEntity); - log.info("FileEntity created successfully: {} (size: {} bytes)", - saved.getId(), fileSize); + + if (saved.getMimeType().startsWith("image")) { + imageHashService.create(saved, Path.of(saved.getFilePath())); + + var similarFiles = fileSimilarityService.findSimilarFiles(saved.getId()); + + boolean hasDuplicate = similarFiles.stream() + .anyMatch(f -> f.getHammingDistance() <= 5); + + if (hasDuplicate) { + log.warn("Duplicate image detected for upload session {}. Rolling back.", + session.getUploadId()); + throw new DuplicateImageException("Duplicate image detected for upload session " + session.getUploadId()); + } + + similarFiles.stream() + .filter(f -> f.getHammingDistance() > 5) + .forEach(f -> log.info("Similar image found: {} (distance {})", + f.getFileId(), f.getHammingDistance())); + } return saved; - } catch (IOException e) { log.error("Failed to create FileEntity for session {}: {}", session.getUploadId(), e.getMessage(), e); @@ -69,15 +92,6 @@ public class FileEntityService { } } - - private String extractFileExtension(String fileName) { - int dotIndex = fileName.lastIndexOf('.'); - if (dotIndex > 0 && dotIndex < fileName.length() - 1) { - return fileName.substring(dotIndex + 1).toLowerCase(); - } - return ""; - } - @Transactional(readOnly = true) public FileEntityResponse getById(String fileId, int version) { FileEntity fileEntity = fileEntityRepository.findById(fileId) @@ -95,39 +109,6 @@ public class FileEntityService { return convertToResponse(fileEntity, version); } - @Transactional(readOnly = true) - public FileEntityResponse getByFilePath(String filePath, int version) { - FileEntity fileEntity = fileEntityRepository.findByFilePath(filePath) - .orElseThrow(() -> new FileEntityNotFoundException("Path: " + filePath)); - - return convertToResponse(fileEntity, version); - } - - - @Transactional(readOnly = true) - public FileResponse getAllUserFiles(Long userId, int version) { - List fileEntities = fileEntityRepository.findByUserIdAndStatus( - userId, FileStatus.ACTIVE); - - List files = fileEntities.stream() - .map(file -> convertToResponse(file, version)) - .collect(Collectors.toList()); - - long totalSize = fileEntities.stream() - .mapToLong(FileEntity::getFileSize) - .sum(); - - return FileResponse.builder() - .files(files) - .totalCount(files.size()) - .totalSize(totalSize) - .formattedTotalSize(formatFileSize(totalSize)) - .page(1) - .pageSize(files.size()) - .build(); - } - - @Transactional(readOnly = true) public FileResponse getUserFiles(Long userId, int page, int pageSize, int version) { List allFiles = fileEntityRepository.findByUserIdAndStatus( diff --git a/src/main/java/ru/soune/nocopy/service/file/FileUploadServiceImpl.java b/src/main/java/ru/soune/nocopy/service/file/FileUploadServiceImpl.java index 3e09822..81df025 100644 --- a/src/main/java/ru/soune/nocopy/service/file/FileUploadServiceImpl.java +++ b/src/main/java/ru/soune/nocopy/service/file/FileUploadServiceImpl.java @@ -402,7 +402,7 @@ public class FileUploadServiceImpl implements FileUploadService { session.getUploadId(), session.getFileName()); Path finalFilePath = null; - String checksum = null; + String checksum; try { finalFilePath = prepareFinalFile(session); @@ -591,44 +591,10 @@ public class FileUploadServiceImpl implements FileUploadService { private void handleAssemblyFailure(FileUploadSession session, Exception e) { session.setStatus(UploadStatus.FAILED); session.setLastError(e.getMessage()); + sessionRepository.save(session); - - log.error("File assembly failed for session {}: {}", - session.getUploadId(), e.getMessage()); } - - -// @Override -// @Transactional -// public void cleanupExpiredSessions() { -// log.info("Starting cleanup of expired upload sessions"); -// -// LocalDateTime expiryThreshold = LocalDateTime.now().minusHours(sessionExpiryHours); -// -// try { -// var expiredSessions = sessionRepository.findByStatusInAndCreatedAtBefore( -// Set.of(UploadStatus.INITIATED, UploadStatus.UPLOADING, UploadStatus.FAILED), -// expiryThreshold -// ); -// -// int cleanedCount = 0; -// for (FileUploadSession session : expiredSessions) { -// session.setStatus(UploadStatus.FAILED); -// session.setLastError("Session expired during cleanup"); -// sessionRepository.save(session); -// -// CompletableFuture.runAsync(() -> cleanupSessionFiles(session)); -// cleanedCount++; -// } -// -// log.info("Cleaned up {} expired upload sessions", cleanedCount); -// -// } catch (Exception e) { -// log.error("Error during session cleanup", e); -// } -// } - private Path getChunkDirectory(Long userId, String uploadId) { return storageRoot.resolve("temp") .resolve("user") diff --git a/src/main/resources/application.yaml b/src/main/resources/application.yaml index 9a34782..38589b1 100644 --- a/src/main/resources/application.yaml +++ b/src/main/resources/application.yaml @@ -1,4 +1,8 @@ spring: + flyway: + enabled: true + baseline-on-migrate: true + datasource: url: jdbc:postgresql://${POSTGRES_HOST}:${POSTGRES_PORT}/${POSTGRES_DB} username: ${POSTGRES_USER} diff --git a/src/main/resources/db/migration/V1__add_image_hash_prefix_index.sql b/src/main/resources/db/migration/V1__add_image_hash_prefix_index.sql new file mode 100644 index 0000000..2700671 --- /dev/null +++ b/src/main/resources/db/migration/V1__add_image_hash_prefix_index.sql @@ -0,0 +1,2 @@ +CREATE INDEX idx_image_hash_prefix + ON image_hashes ((hash_value >> 48)); \ No newline at end of file diff --git a/src/test/java/ru/soune/nocopy/service/AuthServiceTest.java b/src/test/java/ru/soune/nocopy/service/AuthServiceTest.java deleted file mode 100644 index 499d694..0000000 --- a/src/test/java/ru/soune/nocopy/service/AuthServiceTest.java +++ /dev/null @@ -1,101 +0,0 @@ -package ru.soune.nocopy.service; - -import org.junit.jupiter.api.BeforeEach; -import org.junit.jupiter.api.Test; -import org.mockito.ArgumentCaptor; -import org.mockito.InjectMocks; -import org.mockito.Mock; -import org.mockito.MockitoAnnotations; -import org.springframework.context.MessageSource; -import org.springframework.security.crypto.password.PasswordEncoder; -import ru.soune.nocopy.dto.LoginRequest; -import ru.soune.nocopy.entity.AuthToken; -import ru.soune.nocopy.entity.User; -import ru.soune.nocopy.exception.NotValidationPasswordException; -import ru.soune.nocopy.exception.UserNotFoundException; -import ru.soune.nocopy.repository.AuthTokenRepository; -import ru.soune.nocopy.repository.UserRepository; - -import java.util.Optional; - -import static org.junit.jupiter.api.Assertions.*; -import static org.mockito.ArgumentMatchers.any; -import static org.mockito.Mockito.verify; -import static org.mockito.Mockito.when; - -public class AuthServiceTest { - - @Mock - private UserRepository userRepository; - - @Mock - private AuthTokenRepository authTokenRepository; - - @Mock - private PasswordEncoder passwordEncoder; - - @Mock - private MessageSource messageSource; - - @InjectMocks - private AuthService authService; - - @BeforeEach - void setUp() { - MockitoAnnotations.openMocks(this); - } - - @Test - void login_success() { - LoginRequest request = new LoginRequest("test@mail.com", "password"); - - User user = new User(); - user.setId(1L); - user.setEmail("test@mail.com"); - user.setPassword("encoded_pass"); - - when(userRepository.findByEmail("test@mail.com")) - .thenReturn(Optional.of(user)); - when(passwordEncoder.matches("password", "encoded_pass")) - .thenReturn(true); - when(authTokenRepository.save(any(AuthToken.class))) - .thenAnswer(invocation -> invocation.getArgument(0)); - - AuthToken token = authService.login(request); - - assertNotNull(token); - assertNotNull(token.getToken()); -// assertEquals(user.getTokens().get(0), token.getUser()); - - ArgumentCaptor captor = ArgumentCaptor.forClass(AuthToken.class); - verify(authTokenRepository).save(captor.capture()); - -// assertEquals(user, captor.getValue().getUser()); - assertNotNull(captor.getValue().getToken()); - } - - - @Test - void login_userNotFound() { - LoginRequest request = new LoginRequest("missing@mail.com", "pwd"); - - when(userRepository.findByEmail("missing@mail.com")).thenReturn(Optional.empty()); - - assertThrows(UserNotFoundException.class, () -> authService.login(request)); - } - - - @Test - void login_invalidPassword() { - LoginRequest request = new LoginRequest("test@mail.com", "wrong"); - - User user = new User(); - user.setEmail("test@mail.com"); - user.setPassword("encoded_pass"); - - when(userRepository.findByEmail("test@mail.com")).thenReturn(Optional.of(user)); - when(passwordEncoder.matches("wrong", "encoded_pass")).thenReturn(false); - - assertThrows(NotValidationPasswordException.class, () -> authService.login(request)); - } -} From 5c6309e2289414495be66b5e13cdd2527bd0b3f5 Mon Sep 17 00:00:00 2001 From: vladp Date: Wed, 14 Jan 2026 18:47:34 +0700 Subject: [PATCH 16/27] NCBACK-33 fix check company name --- .../handler/validator/RegRequestValidator.java | 17 +---------------- 1 file changed, 1 insertion(+), 16 deletions(-) diff --git a/src/main/java/ru/soune/nocopy/handler/validator/RegRequestValidator.java b/src/main/java/ru/soune/nocopy/handler/validator/RegRequestValidator.java index f3dbda7..19d4c77 100644 --- a/src/main/java/ru/soune/nocopy/handler/validator/RegRequestValidator.java +++ b/src/main/java/ru/soune/nocopy/handler/validator/RegRequestValidator.java @@ -13,7 +13,7 @@ import java.util.*; @Component public class RegRequestValidator implements Validator { - private static final String COMPANY_REGEX = "^[a-zA-Zа-яА-ЯёЁ0-9\\s\\-&.,'()]{0,200}$"; + private static final String COMPANY_REGEX = "^[a-zA-Zа-яА-ЯёЁ0-9\\s\\-&.,'()\"\"]{1,200}$"; private static final String NAME_REGEX = "^[a-zA-Zа-яА-ЯёЁ\\s\\-'.]{2,100}$"; @@ -74,21 +74,6 @@ public class RegRequestValidator implements Validator { errors.rejectValue("companyName", "companyName.invalid.chars", "Company name contains invalid characters"); } - - validateForbiddenWords(trimmedCompany, "companyName", errors, - Arrays.asList("admin", "root", "system", "test")); - } - - private void validateForbiddenWords(String text, String fieldName, - Errors errors, List forbiddenWords) { - String lowerText = text.toLowerCase(); - for (String word : forbiddenWords) { - if (lowerText.contains(word)) { - errors.rejectValue(fieldName, fieldName + ".forbidden.word", - "Contains forbidden word: " + word); - break; - } - } } private void validateEmail(String email, Errors errors) { From d3e50aa8d151caace253ad0d8256e9b2fc017777 Mon Sep 17 00:00:00 2001 From: vladp Date: Wed, 14 Jan 2026 19:10:22 +0700 Subject: [PATCH 17/27] NCBACK-19 add all case for json --- src/main/java/ru/soune/nocopy/dto/BaseRequest.java | 2 ++ src/main/java/ru/soune/nocopy/dto/BaseResponse.java | 2 ++ src/main/java/ru/soune/nocopy/dto/LoginRequest.java | 2 ++ src/main/java/ru/soune/nocopy/dto/RegRequest.java | 2 ++ src/main/java/ru/soune/nocopy/dto/UserRequest.java | 1 + 5 files changed, 9 insertions(+) diff --git a/src/main/java/ru/soune/nocopy/dto/BaseRequest.java b/src/main/java/ru/soune/nocopy/dto/BaseRequest.java index b02ad8c..684e79a 100644 --- a/src/main/java/ru/soune/nocopy/dto/BaseRequest.java +++ b/src/main/java/ru/soune/nocopy/dto/BaseRequest.java @@ -1,9 +1,11 @@ package ru.soune.nocopy.dto; +import com.fasterxml.jackson.annotation.JsonFormat; import com.fasterxml.jackson.annotation.JsonProperty; import lombok.Data; @Data +@JsonFormat(with = JsonFormat.Feature.ACCEPT_CASE_INSENSITIVE_PROPERTIES) public class BaseRequest { @JsonProperty("version") Integer version; diff --git a/src/main/java/ru/soune/nocopy/dto/BaseResponse.java b/src/main/java/ru/soune/nocopy/dto/BaseResponse.java index b60419a..75f1543 100644 --- a/src/main/java/ru/soune/nocopy/dto/BaseResponse.java +++ b/src/main/java/ru/soune/nocopy/dto/BaseResponse.java @@ -1,11 +1,13 @@ package ru.soune.nocopy.dto; +import com.fasterxml.jackson.annotation.JsonFormat; import com.fasterxml.jackson.annotation.JsonProperty; import lombok.AllArgsConstructor; import lombok.Data; @Data @AllArgsConstructor +@JsonFormat(with = JsonFormat.Feature.ACCEPT_CASE_INSENSITIVE_PROPERTIES) public class BaseResponse { @JsonProperty("msg_id") private Integer msgId; diff --git a/src/main/java/ru/soune/nocopy/dto/LoginRequest.java b/src/main/java/ru/soune/nocopy/dto/LoginRequest.java index d64fa8b..b0548bc 100644 --- a/src/main/java/ru/soune/nocopy/dto/LoginRequest.java +++ b/src/main/java/ru/soune/nocopy/dto/LoginRequest.java @@ -1,5 +1,6 @@ package ru.soune.nocopy.dto; +import com.fasterxml.jackson.annotation.JsonFormat; import jakarta.validation.constraints.Email; import jakarta.validation.constraints.NotBlank; import jakarta.validation.constraints.Size; @@ -10,6 +11,7 @@ import lombok.NoArgsConstructor; @Data @AllArgsConstructor @NoArgsConstructor +@JsonFormat(with = JsonFormat.Feature.ACCEPT_CASE_INSENSITIVE_PROPERTIES) public class LoginRequest { @NotBlank(message = "error.not.blank") @Email(message = "error.not.email") @Size(max = 128) diff --git a/src/main/java/ru/soune/nocopy/dto/RegRequest.java b/src/main/java/ru/soune/nocopy/dto/RegRequest.java index e08f404..d7adfea 100644 --- a/src/main/java/ru/soune/nocopy/dto/RegRequest.java +++ b/src/main/java/ru/soune/nocopy/dto/RegRequest.java @@ -1,10 +1,12 @@ package ru.soune.nocopy.dto; +import com.fasterxml.jackson.annotation.JsonFormat; import jakarta.validation.constraints.*; import lombok.Data; @Data +@JsonFormat(with = JsonFormat.Feature.ACCEPT_CASE_INSENSITIVE_PROPERTIES) public class RegRequest { @NotEmpty(message = "Full name is required") private String fullName; diff --git a/src/main/java/ru/soune/nocopy/dto/UserRequest.java b/src/main/java/ru/soune/nocopy/dto/UserRequest.java index c5f1662..91f7379 100644 --- a/src/main/java/ru/soune/nocopy/dto/UserRequest.java +++ b/src/main/java/ru/soune/nocopy/dto/UserRequest.java @@ -10,6 +10,7 @@ import ru.soune.nocopy.entity.GenderType; import java.time.LocalDate; @Data +@JsonFormat(with = JsonFormat.Feature.ACCEPT_CASE_INSENSITIVE_PROPERTIES) public class UserRequest { @Size(min = 2) private String fullName; From 82a57a2983c5b8d0b7c1f26bab5c4a7528c53cf0 Mon Sep 17 00:00:00 2001 From: vladp Date: Thu, 15 Jan 2026 11:09:05 +0700 Subject: [PATCH 18/27] NCBACK-32 add all case for json --- .../ru/soune/nocopy/handler/validator/RegRequestValidator.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/java/ru/soune/nocopy/handler/validator/RegRequestValidator.java b/src/main/java/ru/soune/nocopy/handler/validator/RegRequestValidator.java index 19d4c77..e01df8e 100644 --- a/src/main/java/ru/soune/nocopy/handler/validator/RegRequestValidator.java +++ b/src/main/java/ru/soune/nocopy/handler/validator/RegRequestValidator.java @@ -13,7 +13,7 @@ import java.util.*; @Component public class RegRequestValidator implements Validator { - private static final String COMPANY_REGEX = "^[a-zA-Zа-яА-ЯёЁ0-9\\s\\-&.,'()\"\"]{1,200}$"; + private static final String COMPANY_REGEX = "^[a-zA-Zа-яА-ЯёЁ0-9\\s\\-&.,'()\"\"«»„‟”“”‘'’‹›❛❜❝❞〝〞〟]{1,200}$"; private static final String NAME_REGEX = "^[a-zA-Zа-яА-ЯёЁ\\s\\-'.]{2,100}$"; From 1a6823f7a916ebbc9321942c3f688b3cb947e434 Mon Sep 17 00:00:00 2001 From: vladp Date: Thu, 15 Jan 2026 16:17:03 +0700 Subject: [PATCH 19/27] DEV_DB change db creds --- .env | 6 +- docker-compose.yaml | 19 ++-- infrastructure/jenkins/deploy.groovy | 129 ++++++++++++++------------- init-scripts/init-user.sql | 19 ++++ 4 files changed, 102 insertions(+), 71 deletions(-) create mode 100644 init-scripts/init-user.sql diff --git a/.env b/.env index d3e4347..e54b26f 100644 --- a/.env +++ b/.env @@ -1,10 +1,10 @@ POSTGRES_DB=no_copy_ -POSTGRES_USER=postgres -POSTGRES_PASSWORD=postgres +POSTGRES_USER=ncp_db +POSTGRES_PASSWORD=ncpDbApp POSTGRES_PORT=5432 POSTGRES_HOST=postgres REDIS_HOST=redis REDIS_PORT=6379 -SERVER_PORT=8080 +SERVER_PORT=8080 \ No newline at end of file diff --git a/docker-compose.yaml b/docker-compose.yaml index 192b2d6..6a69921 100644 --- a/docker-compose.yaml +++ b/docker-compose.yaml @@ -18,7 +18,7 @@ services: restart: unless-stopped db: - image: postgres:17 + image: postgres:17.7 restart: always deploy: resources: @@ -30,14 +30,17 @@ services: memory: 1G environment: POSTGRES_DB: no_copy_ - POSTGRES_USER: postgres - POSTGRES_PASSWORD: postgres +# POSTGRES_USER: postgres +# POSTGRES_PASSWORD: postgres + POSTGRES_USER: ${POSTGRES_USER} + POSTGRES_PASSWORD: ${POSTGRES_PASSWORD} POSTGRES_SHARED_BUFFERS: 512MB POSTGRES_EFFECTIVE_CACHE_SIZE: 1536MB ports: - "54320:5432" volumes: - pgdata:/var/lib/postgresql/data + - ./init-scripts:/docker-entrypoint-initdb.d:ro container_name: postgres networks: app-network: @@ -61,8 +64,10 @@ services: # FILE_CHUNK_SIZE: 1048576 FILE_CHUNK_SIZE: 1000000 POSTGRES_DB: no_copy_ - POSTGRES_USER: postgres - POSTGRES_PASSWORD: postgres +# POSTGRES_USER: postgres +# POSTGRES_PASSWORD: postgres + POSTGRES_USER: ${POSTGRES_USER} + POSTGRES_PASSWORD: ${POSTGRES_PASSWORD} POSTGRES_PORT: 5432 POSTGRES_HOST: db STORAGE_SERVICE_URL: http://storage:8081 @@ -82,6 +87,10 @@ services: - api volumes: - uploads_data:/data/uploads:rw + healthcheck: + test: [ "CMD-SHELL", "pg_isready -U ${POSTGRES_USER} -d no_copy_" ] + interval: 10s + retries: 5 grafana: image: grafana/grafana:10.3.1 diff --git a/infrastructure/jenkins/deploy.groovy b/infrastructure/jenkins/deploy.groovy index 7611efe..b57f515 100644 --- a/infrastructure/jenkins/deploy.groovy +++ b/infrastructure/jenkins/deploy.groovy @@ -30,6 +30,7 @@ pipeline { } } + stage('Deploy with docker-compose') { steps { script { @@ -38,71 +39,73 @@ pipeline { credentialsId: 'server-root-password', usernameVariable: 'SSH_USER', passwordVariable: 'SSH_PASS' - ) + ), + string(credentialsId: 'db-user', variable: 'DB_USER'), + string(credentialsId: 'db-password', variable: 'DB_PASSWORD') ]) { sh """ - echo "Deploying branch: ${params.BRANCH}" - - echo "Copying files to server..." - sshpass -p '$SSH_PASS' ssh -o StrictHostKeyChecking=no $SSH_USER@$SERVER "mkdir -p /opt/deployments/${params.BRANCH}" - sshpass -p '$SSH_PASS' scp -r -o StrictHostKeyChecking=no ./* $SSH_USER@$SERVER:/opt/deployments/${params.BRANCH}/ - - sshpass -p '$SSH_PASS' ssh -o StrictHostKeyChecking=no $SSH_USER@$SERVER " - cd /opt/deployments/${params.BRANCH} - - echo '1. Остановка старого приложения...' - docker stop app-backend 2>/dev/null || echo 'Контейнер не найден' - docker rm app-backend 2>/dev/null || echo 'Контейнер не найден' - - echo '2. Удаление старых образов...' - docker images --filter 'reference=*app*' -q | xargs -r docker rmi -f 2>/dev/null || echo 'Нет образов для удаления' - - echo '3. Создание сети если нужно...' - docker network create app-network 2>/dev/null || echo 'Сеть уже существует' - - echo '4. Запуск инфраструктуры...' - docker-compose up -d db storage - - echo '5. Ожидание PostgreSQL...' - sleep 10 - - echo '6. Сборка нового образа приложения...' - docker build --no-cache -t app-backend:latest . - - echo '7. Запуск приложения...' - docker run -d \\ - --name app-backend \\ - --network app-network \\ - --network-alias app \\ - -p 80:8080 \\ - -v uploads_data:/data/uploads:rw \\ - -e POSTGRES_DB=no_copy_ \\ - -e POSTGRES_USER=postgres \\ - -e POSTGRES_PASSWORD=postgres \\ - -e POSTGRES_PORT=5432 \\ - -e POSTGRES_HOST=db \\ - --restart unless-stopped \\ - app-backend:latest - - echo '8. Запуск мониторинга...' - docker-compose up -d grafana prometheus loki tempo alloy - - echo '9. Проверка...' - sleep 5 - - echo 'Статус контейнеров:' - docker ps --format 'table {{.Names}}\\t{{.Image}}\\t{{.Status}}' - - echo '10. Проверка health...' - if curl -s -f http://localhost:80/health > /dev/null 2>&1; then - echo 'Приложение работает' - echo 'URL: http://${params.SERVER}:80' - else - echo 'Проверка health не удалась' - docker logs app-backend --tail=20 - fi - " - """ + echo "Deploying branch: ${params.BRANCH}" + + echo "Copying files to server..." + sshpass -p '$SSH_PASS' ssh -o StrictHostKeyChecking=no $SSH_USER@$SERVER "mkdir -p /opt/deployments/${params.BRANCH}" + sshpass -p '$SSH_PASS' scp -r -o StrictHostKeyChecking=no ./* $SSH_USER@$SERVER:/opt/deployments/${params.BRANCH}/ + + sshpass -p '$SSH_PASS' ssh -o StrictHostKeyChecking=no $SSH_USER@$SERVER " + cd /opt/deployments/${params.BRANCH} + + echo '1. Остановка старого приложения...' + docker stop app-backend 2>/dev/null || echo 'Контейнер не найден' + docker rm app-backend 2>/dev/null || echo 'Контейнер не найден' + + echo '2. Удаление старых образов...' + docker images --filter 'reference=*app*' -q | xargs -r docker rmi -f 2>/dev/null || echo 'Нет образов для удаления' + + echo '3. Создание сети если нужно...' + docker network create app-network 2>/dev/null || echo 'Сеть уже существует' + + echo '4. Запуск инфраструктуры...' + docker-compose up -d db storage + + echo '5. Ожидание PostgreSQL...' + sleep 10 + + echo '6. Сборка нового образа приложения...' + docker build --no-cache -t app-backend:latest . + + echo '7. Запуск приложения...' + docker run -d \\ + --name app-backend \\ + --network app-network \\ + --network-alias app \\ + -p 80:8080 \\ + -v uploads_data:/data/uploads:rw \\ + -e POSTGRES_DB=no_copy_ \\ + -e POSTGRES_USER=$DB_USER \\ + -e POSTGRES_PASSWORD=$DB_PASSWORD \\ + -e POSTGRES_PORT=5432 \\ + -e POSTGRES_HOST=db \\ + --restart unless-stopped \\ + app-backend:latest + + echo '8. Запуск мониторинга...' + docker-compose up -d grafana prometheus loki tempo alloy + + echo '9. Проверка...' + sleep 5 + + echo 'Статус контейнеров:' + docker ps --format 'table {{.Names}}\\t{{.Image}}\\t{{.Status}}' + + echo '10. Проверка health...' + if curl -s -f http://localhost:80/health > /dev/null 2>&1; then + echo 'Приложение работает' + echo 'URL: http://${params.SERVER}:80' + else + echo 'Проверка health не удалась' + docker logs app-backend --tail=20 + fi + " + """ } } } diff --git a/init-scripts/init-user.sql b/init-scripts/init-user.sql new file mode 100644 index 0000000..f1d560e --- /dev/null +++ b/init-scripts/init-user.sql @@ -0,0 +1,19 @@ +DO +$do$ +BEGIN + IF NOT EXISTS (SELECT FROM pg_catalog.pg_roles WHERE rolname = 'ncp_db') THEN + CREATE USER ncp_db WITH PASSWORD 'ncpDbApp'; +END IF; +END +$do$; + +DO +$do$ +BEGIN + IF NOT EXISTS (SELECT FROM pg_database WHERE datname = 'no_copy_') THEN + CREATE DATABASE no_copy_ OWNER ncp_db; +END IF; +END +$do$; + +GRANT ALL PRIVILEGES ON DATABASE no_copy_ TO ncp_db; \ No newline at end of file From de810e32b473dd55acddf63a71d223b512b90f81 Mon Sep 17 00:00:00 2001 From: vladp Date: Thu, 15 Jan 2026 16:42:37 +0700 Subject: [PATCH 20/27] DEV_DB change db creds --- README.md | 48 ++++++++++++++++++++++++++++ infrastructure/jenkins/deploy.groovy | 4 +-- 2 files changed, 50 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index e69de29..72cd929 100644 --- a/README.md +++ b/README.md @@ -0,0 +1,48 @@ +суперпользователь БД: +postgres/postgres +пользователь бд для бэка +ncp_db / ncpDbApp + +Подключение к БД для приложения,если volume не существует : + +Создаться автоматически. + +Если уже существует volume : + +docker exec -it postgres psql -U postgres -d postgres + +CREATE USER ncp_db WITH PASSWORD 'ncpDbApp'; + +GRANT ALL PRIVILEGES ON DATABASE no_copy_ TO ncp_db; + +\du + +\q + +------- + +psql -U postgres -d no_copy_ + +-- база + +GRANT CONNECT ON DATABASE no_copy_ TO ncp_db; + +-- схема + +GRANT USAGE ON SCHEMA public TO ncp_db; + +-- существующие таблицы + +GRANT ALL PRIVILEGES ON ALL TABLES IN SCHEMA public TO ncp_db; + +-- существующие sequence (очень важно для id) + +GRANT ALL PRIVILEGES ON ALL SEQUENCES IN SCHEMA public TO ncp_db; + +-- будущие таблицы + +ALTER DEFAULT PRIVILEGES IN SCHEMA public +GRANT ALL ON TABLES TO ncp_db; + +ALTER DEFAULT PRIVILEGES IN SCHEMA public +GRANT ALL ON SEQUENCES TO ncp_db; diff --git a/infrastructure/jenkins/deploy.groovy b/infrastructure/jenkins/deploy.groovy index b57f515..dc09c2e 100644 --- a/infrastructure/jenkins/deploy.groovy +++ b/infrastructure/jenkins/deploy.groovy @@ -40,8 +40,8 @@ pipeline { usernameVariable: 'SSH_USER', passwordVariable: 'SSH_PASS' ), - string(credentialsId: 'db-user', variable: 'DB_USER'), - string(credentialsId: 'db-password', variable: 'DB_PASSWORD') + string(credentialsId: 'DB_USER', variable: 'DB_USER'), + string(credentialsId: 'DB_PASSWORD', variable: 'DB_PASSWORD') ]) { sh """ echo "Deploying branch: ${params.BRANCH}" From 0ed3f958ffeb3a4469c13b9466ddd857390b063a Mon Sep 17 00:00:00 2001 From: vladp Date: Thu, 15 Jan 2026 16:52:33 +0700 Subject: [PATCH 21/27] DEV_DB change db creds --- init-scripts/01-init.sql | 19 +++++++++++++++++++ init-scripts/init-user.sql | 19 ------------------- 2 files changed, 19 insertions(+), 19 deletions(-) create mode 100644 init-scripts/01-init.sql delete mode 100644 init-scripts/init-user.sql diff --git a/init-scripts/01-init.sql b/init-scripts/01-init.sql new file mode 100644 index 0000000..60bedbc --- /dev/null +++ b/init-scripts/01-init.sql @@ -0,0 +1,19 @@ +DO $$ +BEGIN + IF NOT EXISTS (SELECT FROM pg_roles WHERE rolname = 'ncp_db') THEN + CREATE USER ncp_db WITH PASSWORD 'ncpDbApp'; +END IF; +END +$$; + +GRANT CONNECT ON DATABASE no_copy_ TO ncp_db; +GRANT USAGE ON SCHEMA public TO ncp_db; + +GRANT ALL PRIVILEGES ON ALL TABLES IN SCHEMA public TO ncp_db; +GRANT ALL PRIVILEGES ON ALL SEQUENCES IN SCHEMA public TO ncp_db; + +ALTER DEFAULT PRIVILEGES IN SCHEMA public +GRANT ALL ON TABLES TO ncp_db; + +ALTER DEFAULT PRIVILEGES IN SCHEMA public +GRANT ALL ON SEQUENCES TO ncp_db; \ No newline at end of file diff --git a/init-scripts/init-user.sql b/init-scripts/init-user.sql deleted file mode 100644 index f1d560e..0000000 --- a/init-scripts/init-user.sql +++ /dev/null @@ -1,19 +0,0 @@ -DO -$do$ -BEGIN - IF NOT EXISTS (SELECT FROM pg_catalog.pg_roles WHERE rolname = 'ncp_db') THEN - CREATE USER ncp_db WITH PASSWORD 'ncpDbApp'; -END IF; -END -$do$; - -DO -$do$ -BEGIN - IF NOT EXISTS (SELECT FROM pg_database WHERE datname = 'no_copy_') THEN - CREATE DATABASE no_copy_ OWNER ncp_db; -END IF; -END -$do$; - -GRANT ALL PRIVILEGES ON DATABASE no_copy_ TO ncp_db; \ No newline at end of file From 4a45f40e9ffd237f2fb49962601a10bf6af599f5 Mon Sep 17 00:00:00 2001 From: vladp Date: Thu, 15 Jan 2026 20:41:35 +0700 Subject: [PATCH 22/27] NCBACK-34 need fix delete filentity and hash --- .../nocopy/controller/ApiController.java | 133 ++++++++++++------ .../handler/GlobalExceptionHandler.java | 15 +- .../repository/ImageHashRepository.java | 1 + .../service/file/FileEntityService.java | 19 +-- 4 files changed, 104 insertions(+), 64 deletions(-) diff --git a/src/main/java/ru/soune/nocopy/controller/ApiController.java b/src/main/java/ru/soune/nocopy/controller/ApiController.java index babc9e4..b966fd7 100644 --- a/src/main/java/ru/soune/nocopy/controller/ApiController.java +++ b/src/main/java/ru/soune/nocopy/controller/ApiController.java @@ -1,7 +1,10 @@ package ru.soune.nocopy.controller; +import jakarta.persistence.EntityManager; +import jakarta.persistence.PersistenceContext; import lombok.RequiredArgsConstructor; import lombok.extern.slf4j.Slf4j; +import org.springframework.beans.factory.annotation.Autowired; import org.springframework.core.io.Resource; import org.springframework.core.io.UrlResource; import org.springframework.http.HttpHeaders; @@ -17,12 +20,15 @@ import ru.soune.nocopy.dto.MessageCode; import ru.soune.nocopy.dto.RegAnswer; import ru.soune.nocopy.dto.file.*; import ru.soune.nocopy.entity.AuthToken; +import ru.soune.nocopy.entity.file.FileEntity; import ru.soune.nocopy.entity.file.FileStatus; import ru.soune.nocopy.entity.file.UploadStatus; import ru.soune.nocopy.exception.*; import ru.soune.nocopy.handler.*; import ru.soune.nocopy.repository.AuthTokenRepository; +import ru.soune.nocopy.repository.FileEntityRepository; import ru.soune.nocopy.service.FileSimilarityService; +import ru.soune.nocopy.service.ImageHashService; import ru.soune.nocopy.service.file.FileEntityService; import ru.soune.nocopy.service.file.FileUploadService; @@ -33,6 +39,7 @@ import java.nio.file.Paths; import java.util.HashMap; import java.util.List; import java.util.Map; +import java.util.Optional; import java.util.stream.Collectors; @Slf4j @@ -50,6 +57,8 @@ public class ApiController { private final FileSimilarityService fileSimilarityService; + private final FileEntityRepository fileEntityRepository; + @PostMapping("/v{version}/data") public ResponseEntity handlePostRequest(@RequestBody BaseRequest request, @PathVariable("version") int version) { @@ -93,62 +102,72 @@ public class ApiController { @RequestParam(value = "chunk", required = false) MultipartFile chunk) { try { if (chunk == null || chunk.isEmpty()) { - return ResponseEntity.ok().body(new BaseResponse(20004, MessageCode.FILE_UPLOAD_ERROR.getCode(), - "Chunk file null or empty", ChunkUploadResponse.builder() - .uploadId(uploadId) - .chunkNumber(chunkNumber) - .build())); + return buildErrorResponse(uploadId, chunkNumber, "Chunk file null or empty"); } - if (uploadId == null || uploadId.isBlank()) { - return ResponseEntity.ok().body(new BaseResponse(20004, MessageCode.FILE_UPLOAD_ERROR.getCode(), - "Upload ID is required", ChunkUploadResponse.builder() - .uploadId(uploadId) - .chunkNumber(chunkNumber) - .build())); + return buildErrorResponse(uploadId, chunkNumber, "Upload ID is required"); } - if (chunkNumber == null || chunkNumber < 0) { - return ResponseEntity.ok().body(new BaseResponse(20004, MessageCode.FILE_UPLOAD_ERROR.getCode(), - "Valid chunk number is required", ChunkUploadResponse.builder() - .uploadId(uploadId) - .chunkNumber(chunkNumber) - .build())); + return buildErrorResponse(uploadId, chunkNumber, "Valid chunk number is required"); } - fileUploadService.uploadChunk(uploadId, chunkNumber, chunk); + UploadProgressResponse uploadProgressResponse = fileUploadService.uploadChunk(uploadId, chunkNumber, chunk); - ChunkUploadResponse responseBody = ChunkUploadResponse.builder() - .uploadId(uploadId) - .chunkNumber(chunkNumber) - .chunkSize(chunk.getSize()) - .message("Chunk uploaded successfully") - .build(); + if (uploadProgressResponse.getStatus().equals(UploadStatus.COMPLETED)) { + ResponseEntity duplicateCheckResult = checkForDuplicates(uploadProgressResponse.getUploadId()); + if (duplicateCheckResult != null) { + return duplicateCheckResult; + } + } - return ResponseEntity.ok().body(new BaseResponse(20004, MessageCode.SUCCESS.getCode(), - "Chunk uploaded successfully", responseBody)); - } catch (DuplicateImageException e) { - log.error("Error uploading chunk", e); - ChunkUploadResponse responseBody = ChunkUploadResponse.builder() - .uploadId(uploadId) - .chunkNumber(chunkNumber) - .build(); + return buildSuccessResponse(uploadId, chunkNumber, chunk); - return ResponseEntity.ok().body(new BaseResponse( - 20004, MessageCode.DUPLICATE_FILE_UPLOAD.getCode(), - "Failed to upload chunk: " + e.getMessage(), responseBody)); } catch (Exception e) { log.error("Error uploading chunk", e); + return buildErrorResponse(uploadId, chunkNumber, "Failed to upload chunk: " + e.getMessage()); + } + } - ChunkUploadResponse responseBody = ChunkUploadResponse.builder() - .uploadId(uploadId) - .chunkNumber(chunkNumber) - .build(); + private ResponseEntity checkForDuplicates(String uploadId) throws IOException { + Optional uploadedFile = fileEntityRepository.findByUploadSessionId(uploadId); + + if (uploadedFile.isEmpty() || !uploadedFile.get().getMimeType().equals("image")) { + return null; + } + + FileEntity fileEntity = uploadedFile.get(); + List similarFiles = fileSimilarityService.findSimilarFiles(fileEntity.getId()); + + if (hasDuplicate(similarFiles)) { + return handleDuplicate(fileEntity, similarFiles); + } + + return null; + } + + private boolean hasDuplicate(List similarFiles) { + return similarFiles.stream().anyMatch(f -> f.getHammingDistance() <= 5); + } + + private ResponseEntity handleDuplicate(FileEntity fileEntity, List similarFiles) + throws IOException { + fileEntityService.deleteFromDisk(fileEntity); + + Optional originalFile = fileEntityRepository.findById(similarFiles.get(0).getFileId()); + + if (originalFile.isPresent()) { + Map duplicateInfo = Map.of( + "duplicate_file_id", originalFile.get().getId(), + "owner_user_id", String.valueOf(originalFile.get().getUserId())); return ResponseEntity.ok().body(new BaseResponse( - 20004, MessageCode.FILE_UPLOAD_ERROR.getCode(), - "Failed to upload chunk: " + e.getMessage(), responseBody)); + 20004, + MessageCode.DUPLICATE_FILE_UPLOAD.getCode(), + "Failed to upload chunk, duplicate", + duplicateInfo)); } + + return null; } @GetMapping("/files/{fileId}/similar") @@ -179,7 +198,6 @@ public class ApiController { return ResponseEntity.ok().body(new BaseResponse( 20004, MessageCode.SUCCESS.getCode(), MessageCode.SUCCESS.getDescription(), responseBody)); - } catch (Exception e) { log.error("Error getting progress for upload: {}", uploadId, e); @@ -187,8 +205,7 @@ public class ApiController { .uploadId(uploadId) .build(); - return ResponseEntity.ok().body(new BaseResponse( - 20004, MessageCode.FILE_UPLOAD_ERROR.getCode(), + return ResponseEntity.ok().body(new BaseResponse(20004, MessageCode.FILE_UPLOAD_ERROR.getCode(), "Failed to get upload progress: " + e.getMessage(), responseBody)); } } @@ -347,6 +364,34 @@ public class ApiController { } } + private ResponseEntity buildSuccessResponse(String uploadId, Integer chunkNumber, MultipartFile chunk) { + ChunkUploadResponse responseBody = ChunkUploadResponse.builder() + .uploadId(uploadId) + .chunkNumber(chunkNumber) + .chunkSize(chunk.getSize()) + .message("Chunk uploaded successfully") + .build(); + + return ResponseEntity.ok().body(new BaseResponse( + 20000, + MessageCode.SUCCESS.getCode(), + "Chunk uploaded successfully", + responseBody)); + } + + private ResponseEntity buildErrorResponse(String uploadId, Integer chunkNumber, String errorMessage) { + ChunkUploadResponse responseBody = ChunkUploadResponse.builder() + .uploadId(uploadId) + .chunkNumber(chunkNumber) + .build(); + + return ResponseEntity.ok().body(new BaseResponse( + 20004, + MessageCode.FILE_UPLOAD_ERROR.getCode(), + errorMessage, + responseBody)); + } + private ResponseEntity createValidationErrorResponse(BindingResult bindingResult, Integer msgId) { List> fieldErrors = bindingResult.getFieldErrors() .stream() diff --git a/src/main/java/ru/soune/nocopy/handler/GlobalExceptionHandler.java b/src/main/java/ru/soune/nocopy/handler/GlobalExceptionHandler.java index 071b9ae..7303fb5 100644 --- a/src/main/java/ru/soune/nocopy/handler/GlobalExceptionHandler.java +++ b/src/main/java/ru/soune/nocopy/handler/GlobalExceptionHandler.java @@ -5,15 +5,16 @@ import org.springframework.context.support.DefaultMessageSourceResolvable; import org.springframework.http.HttpStatus; import org.springframework.http.ResponseEntity; import org.springframework.web.bind.MethodArgumentNotValidException; -import org.springframework.web.bind.annotation.ExceptionHandler; -import org.springframework.web.bind.annotation.ResponseStatus; -import org.springframework.web.bind.annotation.RestControllerAdvice; +import org.springframework.web.bind.annotation.*; +import ru.soune.nocopy.dto.BaseResponse; +import ru.soune.nocopy.dto.MessageCode; import ru.soune.nocopy.exception.*; import java.util.Map; @RestControllerAdvice @AllArgsConstructor +@ControllerAdvice public class GlobalExceptionHandler { @ExceptionHandler(MethodArgumentNotValidException.class) @@ -81,4 +82,12 @@ public class GlobalExceptionHandler { "message" ,ex.getMessage() )); } + + @ExceptionHandler(DuplicateImageException.class) + @ResponseBody + public ResponseEntity handleDuplicateImage(DuplicateImageException e) { + return ResponseEntity.ok().body(new BaseResponse( + 20004, MessageCode.DUPLICATE_FILE_UPLOAD.getCode(), + "Duplicate image detected", e.getMessage())); + } } diff --git a/src/main/java/ru/soune/nocopy/repository/ImageHashRepository.java b/src/main/java/ru/soune/nocopy/repository/ImageHashRepository.java index e1ff003..ee0aac2 100644 --- a/src/main/java/ru/soune/nocopy/repository/ImageHashRepository.java +++ b/src/main/java/ru/soune/nocopy/repository/ImageHashRepository.java @@ -6,4 +6,5 @@ import ru.soune.nocopy.entity.file.ImageHashEntity; @Repository public interface ImageHashRepository extends JpaRepository { + void deleteByFileId(String fileId); } diff --git a/src/main/java/ru/soune/nocopy/service/file/FileEntityService.java b/src/main/java/ru/soune/nocopy/service/file/FileEntityService.java index f72ac63..0a17c22 100644 --- a/src/main/java/ru/soune/nocopy/service/file/FileEntityService.java +++ b/src/main/java/ru/soune/nocopy/service/file/FileEntityService.java @@ -9,7 +9,6 @@ import ru.soune.nocopy.dto.file.FileResponse; import ru.soune.nocopy.entity.file.FileEntity; import ru.soune.nocopy.entity.file.FileStatus; import ru.soune.nocopy.entity.file.FileUploadSession; -import ru.soune.nocopy.exception.DuplicateImageException; import ru.soune.nocopy.exception.FileEntityNotFoundException; import ru.soune.nocopy.repository.FileEntityRepository; import ru.soune.nocopy.service.FileSimilarityService; @@ -66,22 +65,6 @@ public class FileEntityService { if (saved.getMimeType().startsWith("image")) { imageHashService.create(saved, Path.of(saved.getFilePath())); - - var similarFiles = fileSimilarityService.findSimilarFiles(saved.getId()); - - boolean hasDuplicate = similarFiles.stream() - .anyMatch(f -> f.getHammingDistance() <= 5); - - if (hasDuplicate) { - log.warn("Duplicate image detected for upload session {}. Rolling back.", - session.getUploadId()); - throw new DuplicateImageException("Duplicate image detected for upload session " + session.getUploadId()); - } - - similarFiles.stream() - .filter(f -> f.getHammingDistance() > 5) - .forEach(f -> log.info("Similar image found: {} (distance {})", - f.getFileId(), f.getHammingDistance())); } return saved; @@ -166,6 +149,8 @@ public class FileEntityService { Files.delete(path); + markAsDeleted(fileEntity); + return true; } From 3d679440f6b0607d32395d9f984b5ffab737713f Mon Sep 17 00:00:00 2001 From: vladp Date: Fri, 16 Jan 2026 14:30:45 +0700 Subject: [PATCH 23/27] NCBACK-34 add logic for clear does not exist files,add dto for similar file search controller --- .../nocopy/controller/ApiController.java | 121 +++++++++++------- .../java/ru/soune/nocopy/dto/MessageCode.java | 3 +- .../nocopy/dto/file/SimilarityFilter.java | 18 +++ .../soune/nocopy/entity/file/FileEntity.java | 5 + .../repository/FileEntityRepository.java | 3 + .../nocopy/service/FileSimilarityService.java | 64 ++++++++- .../file/NotExistFilesCleanupService.java | 35 +++++ 7 files changed, 202 insertions(+), 47 deletions(-) create mode 100644 src/main/java/ru/soune/nocopy/dto/file/SimilarityFilter.java create mode 100644 src/main/java/ru/soune/nocopy/service/file/NotExistFilesCleanupService.java diff --git a/src/main/java/ru/soune/nocopy/controller/ApiController.java b/src/main/java/ru/soune/nocopy/controller/ApiController.java index b966fd7..5c408fc 100644 --- a/src/main/java/ru/soune/nocopy/controller/ApiController.java +++ b/src/main/java/ru/soune/nocopy/controller/ApiController.java @@ -1,12 +1,12 @@ package ru.soune.nocopy.controller; -import jakarta.persistence.EntityManager; -import jakarta.persistence.PersistenceContext; import lombok.RequiredArgsConstructor; import lombok.extern.slf4j.Slf4j; -import org.springframework.beans.factory.annotation.Autowired; import org.springframework.core.io.Resource; import org.springframework.core.io.UrlResource; +import org.springframework.data.domain.Page; +import org.springframework.data.domain.Pageable; +import org.springframework.data.web.PageableDefault; import org.springframework.http.HttpHeaders; import org.springframework.http.MediaType; import org.springframework.http.ResponseEntity; @@ -28,7 +28,6 @@ import ru.soune.nocopy.handler.*; import ru.soune.nocopy.repository.AuthTokenRepository; import ru.soune.nocopy.repository.FileEntityRepository; import ru.soune.nocopy.service.FileSimilarityService; -import ru.soune.nocopy.service.ImageHashService; import ru.soune.nocopy.service.file.FileEntityService; import ru.soune.nocopy.service.file.FileUploadService; @@ -128,51 +127,39 @@ public class ApiController { } } - private ResponseEntity checkForDuplicates(String uploadId) throws IOException { - Optional uploadedFile = fileEntityRepository.findByUploadSessionId(uploadId); + @GetMapping("/v{version}/files/{fileId}/similar") + public ResponseEntity findSimilarFiles( + @PathVariable("version") int version, + @PathVariable String fileId, + @RequestParam(required = false) List similarityLevels, + @PageableDefault(size = 20, sort = "hammingDistance") Pageable pageable) { + SimilarityFilter filter = SimilarityFilter.builder() + .similarityLevels(similarityLevels) + .build(); + Page similarFiles = fileSimilarityService.findSimilarFiles(fileId, filter, pageable); - if (uploadedFile.isEmpty() || !uploadedFile.get().getMimeType().equals("image")) { - return null; + String messageDesc; + MessageCode success; + + if (similarFiles.isEmpty()) { + messageDesc = MessageCode.FILE_NOT_FOUND.getDescription(); + success = MessageCode.FILE_NOT_FOUND; + } else { + messageDesc = MessageCode.SIMILAR_FILES_FOUND.getDescription(); + success = MessageCode.SIMILAR_FILES_FOUND; } - FileEntity fileEntity = uploadedFile.get(); - List similarFiles = fileSimilarityService.findSimilarFiles(fileEntity.getId()); + Map responseData = new HashMap<>(); + responseData.put("content", similarFiles.getContent()); + responseData.put("page", similarFiles.getNumber()); + responseData.put("size", similarFiles.getSize()); + responseData.put("totalElements", similarFiles.getTotalElements()); + responseData.put("totalPages", similarFiles.getTotalPages()); + responseData.put("hasNext", similarFiles.hasNext()); + responseData.put("hasPrevious", similarFiles.hasPrevious()); - if (hasDuplicate(similarFiles)) { - return handleDuplicate(fileEntity, similarFiles); - } - - return null; - } - - private boolean hasDuplicate(List similarFiles) { - return similarFiles.stream().anyMatch(f -> f.getHammingDistance() <= 5); - } - - private ResponseEntity handleDuplicate(FileEntity fileEntity, List similarFiles) - throws IOException { - fileEntityService.deleteFromDisk(fileEntity); - - Optional originalFile = fileEntityRepository.findById(similarFiles.get(0).getFileId()); - - if (originalFile.isPresent()) { - Map duplicateInfo = Map.of( - "duplicate_file_id", originalFile.get().getId(), - "owner_user_id", String.valueOf(originalFile.get().getUserId())); - - return ResponseEntity.ok().body(new BaseResponse( - 20004, - MessageCode.DUPLICATE_FILE_UPLOAD.getCode(), - "Failed to upload chunk, duplicate", - duplicateInfo)); - } - - return null; - } - - @GetMapping("/files/{fileId}/similar") - public List getSimilarFiles(@PathVariable String fileId) { - return fileSimilarityService.findSimilarFiles(fileId); + return ResponseEntity.ok() + .body(new BaseResponse(20004, success.getCode(), messageDesc, responseData)); } @GetMapping("/v{version}/files/progress/{uploadId}") @@ -364,6 +351,50 @@ public class ApiController { } } + private ResponseEntity checkForDuplicates(String uploadId) throws IOException { + Optional uploadedFile = fileEntityRepository.findByUploadSessionId(uploadId); + + if (uploadedFile.isEmpty() || !uploadedFile.get().getMimeType().equals("image")) { + return null; + } + + FileEntity fileEntity = uploadedFile.get(); + List similarFiles = fileSimilarityService.findSimilarFiles(fileEntity.getId()); + + if (hasDuplicate(similarFiles)) { + return handleDuplicate(fileEntity, similarFiles); + } + + return null; + } + + private boolean hasDuplicate(List similarFiles) { + return similarFiles.stream().anyMatch(f -> f.getHammingDistance() <= 5); + } + + private ResponseEntity handleDuplicate(FileEntity fileEntity, List similarFiles) + throws IOException { + fileEntityService.deleteFromDisk(fileEntity); + + fileEntityService.markAsDeleted(fileEntity); + + Optional originalFile = fileEntityRepository.findById(similarFiles.get(0).getFileId()); + + if (originalFile.isPresent()) { + Map duplicateInfo = Map.of( + "duplicate_file_id", originalFile.get().getId(), + "owner_user_id", String.valueOf(originalFile.get().getUserId())); + + return ResponseEntity.ok().body(new BaseResponse( + 20004, + MessageCode.DUPLICATE_FILE_UPLOAD.getCode(), + "Failed to upload chunk, duplicate", + duplicateInfo)); + } + + return null; + } + private ResponseEntity buildSuccessResponse(String uploadId, Integer chunkNumber, MultipartFile chunk) { ChunkUploadResponse responseBody = ChunkUploadResponse.builder() .uploadId(uploadId) diff --git a/src/main/java/ru/soune/nocopy/dto/MessageCode.java b/src/main/java/ru/soune/nocopy/dto/MessageCode.java index 33faabc..3332a30 100644 --- a/src/main/java/ru/soune/nocopy/dto/MessageCode.java +++ b/src/main/java/ru/soune/nocopy/dto/MessageCode.java @@ -21,7 +21,8 @@ public enum MessageCode { AUTH_EMAIL_OR_TOKEN_NOT_FOUND(4, "Email or Token not found "), AUTH_TOKEN_MISMATCH(4, "Token mismatch"), FILE_NOT_FOUND(4, "File not found"), - AUTH_PASSWORD_NOT_MATCHES(2, "Password does not match"); + AUTH_PASSWORD_NOT_MATCHES(2, "Password does not match"), + SIMILAR_FILES_FOUND(0, "Similar files found"); private final Integer code; diff --git a/src/main/java/ru/soune/nocopy/dto/file/SimilarityFilter.java b/src/main/java/ru/soune/nocopy/dto/file/SimilarityFilter.java new file mode 100644 index 0000000..48a4433 --- /dev/null +++ b/src/main/java/ru/soune/nocopy/dto/file/SimilarityFilter.java @@ -0,0 +1,18 @@ +package ru.soune.nocopy.dto.file; + +import lombok.AllArgsConstructor; +import lombok.Builder; +import lombok.Data; +import lombok.NoArgsConstructor; + +import java.util.List; + +@Data +@Builder +@NoArgsConstructor +@AllArgsConstructor +public class SimilarityFilter { + private List similarityLevels; + private Integer page; + private Integer size; +} diff --git a/src/main/java/ru/soune/nocopy/entity/file/FileEntity.java b/src/main/java/ru/soune/nocopy/entity/file/FileEntity.java index ffdd985..e07b591 100644 --- a/src/main/java/ru/soune/nocopy/entity/file/FileEntity.java +++ b/src/main/java/ru/soune/nocopy/entity/file/FileEntity.java @@ -1,5 +1,6 @@ package ru.soune.nocopy.entity.file; +import com.fasterxml.jackson.annotation.JsonIgnore; import jakarta.persistence.*; import lombok.*; import org.springframework.data.annotation.CreatedDate; @@ -60,6 +61,10 @@ public class FileEntity { @Column(name = "updated_at") private LocalDateTime updatedAt; + @OneToOne(mappedBy = "file", cascade = CascadeType.ALL, orphanRemoval = true) + @JsonIgnore + private ImageHashEntity imageHash; + @PrePersist public void prePersist() { if (this.status == null) { diff --git a/src/main/java/ru/soune/nocopy/repository/FileEntityRepository.java b/src/main/java/ru/soune/nocopy/repository/FileEntityRepository.java index 3214e9d..e35ef2f 100644 --- a/src/main/java/ru/soune/nocopy/repository/FileEntityRepository.java +++ b/src/main/java/ru/soune/nocopy/repository/FileEntityRepository.java @@ -30,5 +30,8 @@ public interface FileEntityRepository extends JpaRepository @Query("SELECT f FROM FileEntity f WHERE f.userId = :userId AND f.originalFileName LIKE %:keyword%") List searchByFileName(@Param("userId") Long userId, @Param("keyword") String keyword); + @Query("SELECT f FROM FileEntity f WHERE f.status = :status") + List searchFileEntityByStatus(@Param("status") FileStatus status); + long countByUserId(Long userId); } diff --git a/src/main/java/ru/soune/nocopy/service/FileSimilarityService.java b/src/main/java/ru/soune/nocopy/service/FileSimilarityService.java index 2c02234..f8b7407 100644 --- a/src/main/java/ru/soune/nocopy/service/FileSimilarityService.java +++ b/src/main/java/ru/soune/nocopy/service/FileSimilarityService.java @@ -1,19 +1,25 @@ package ru.soune.nocopy.service; import lombok.RequiredArgsConstructor; +import org.springframework.data.domain.Page; +import org.springframework.data.domain.PageImpl; +import org.springframework.data.domain.Pageable; import org.springframework.stereotype.Service; import ru.soune.nocopy.dto.file.SimilarFileResponse; +import ru.soune.nocopy.dto.file.SimilarityFilter; import ru.soune.nocopy.repository.ImageHashRepository; import ru.soune.nocopy.repository.ImageSimilarityRepository; import ru.soune.nocopy.repository.SimilarImageProjection; +import java.util.Comparator; import java.util.List; +import java.util.stream.Collectors; @Service @RequiredArgsConstructor public class FileSimilarityService { - private final ImageSimilarityRepository repository; + private final ImageHashRepository hashRepository; public List findSimilarFiles(String fileId) { @@ -55,6 +61,62 @@ public class FileSimilarityService { .toList(); } + public Page findSimilarFiles(String fileId, SimilarityFilter filter, Pageable pageable) { + var imageHashEntity = hashRepository.findById(fileId) + .orElseThrow(() -> new RuntimeException("Hash not found")); + + Integer hash64Hi = imageHashEntity.getHash64Hi(); + Integer hash64Lo = imageHashEntity.getHash64Lo(); + + List candidates = repository.findCandidates(fileId); + + List similarityLevels = (filter != null && filter.getSimilarityLevels() != null) + ? filter.getSimilarityLevels() + : List.of("DUPLICATE", "SIMILAR", "DIFFERENT"); + + List allResults = candidates.stream() + .map(c -> createSimilarFileResponse(c, hash64Hi, hash64Lo)) + .filter(response -> similarityLevels.contains(response.getSimilarityLevel())) + .sorted(Comparator.comparingInt(SimilarFileResponse::getHammingDistance)) + .collect(Collectors.toList()); + + int total = allResults.size(); + int page = (pageable != null) ? pageable.getPageNumber() : 0; + int size = (pageable != null) ? pageable.getPageSize() : 20; + + int fromIndex = Math.min(page * size, total); + int toIndex = Math.min(fromIndex + size, total); + + List pageContent = allResults.subList(fromIndex, toIndex); + + return new PageImpl<>(pageContent, pageable, total); + } + + private SimilarFileResponse createSimilarFileResponse(SimilarImageProjection similarImageProjection, + Integer hash64Hi, Integer hash64Lo) { + Integer imageProjectionHash64Hi = similarImageProjection.getHash64Hi(); + Integer similarImageProjectionHash64Lo = similarImageProjection.getHash64Lo(); + + int hamming = hamming64(hash64Hi, hash64Lo, imageProjectionHash64Hi, similarImageProjectionHash64Lo); + + String level; + if (hamming <= 5) { + level = "DUPLICATE"; + } else if (hamming <= 12) { + level = "SIMILAR"; + } else { + level = "DIFFERENT"; + } + + return SimilarFileResponse.builder() + .fileId(similarImageProjection.getSimilarFileId()) + .originalFileName(similarImageProjection.getOriginalFileName()) + .fileSize(similarImageProjection.getFileSize()) + .hammingDistance(hamming) + .similarityLevel(level) + .build(); + } + private int hamming64(int aHi, int aLo, int bHi, int bLo) { return Integer.bitCount(aHi ^ bHi) + Integer.bitCount(aLo ^ bLo); diff --git a/src/main/java/ru/soune/nocopy/service/file/NotExistFilesCleanupService.java b/src/main/java/ru/soune/nocopy/service/file/NotExistFilesCleanupService.java new file mode 100644 index 0000000..1be7b9e --- /dev/null +++ b/src/main/java/ru/soune/nocopy/service/file/NotExistFilesCleanupService.java @@ -0,0 +1,35 @@ +package ru.soune.nocopy.service.file; + +import lombok.RequiredArgsConstructor; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.scheduling.annotation.Scheduled; +import org.springframework.stereotype.Service; +import org.springframework.transaction.annotation.Transactional; +import ru.soune.nocopy.entity.file.FileEntity; +import ru.soune.nocopy.entity.file.FileStatus; +import ru.soune.nocopy.repository.FileEntityRepository; + +import java.nio.file.Files; +import java.nio.file.Paths; +import java.util.List; + +@Service +@RequiredArgsConstructor +public class NotExistFilesCleanupService { + + @Autowired + private FileEntityRepository fileEntityRepository; + + @Transactional + @Scheduled(fixedDelay = 30000) + public void cleanUpNotExistFiles() { + List fileEntities = fileEntityRepository.searchFileEntityByStatus(FileStatus.DELETED); + + for (FileEntity fileEntity : fileEntities) { + boolean exists = Files.exists(Paths.get(fileEntity.getFilePath())); + if (!exists) { + fileEntityRepository.delete(fileEntity); + } + } + } +} From 50279201998e19d60175c65051968c766bd1f4a2 Mon Sep 17 00:00:00 2001 From: vladp Date: Fri, 16 Jan 2026 23:59:07 +0700 Subject: [PATCH 24/27] dev info for docker user --- README.md | 20 ++++++++++++++++++++ docker-compose.yaml | 5 ----- src/main/resources/application.yaml | 2 -- 3 files changed, 20 insertions(+), 7 deletions(-) diff --git a/README.md b/README.md index 72cd929..d133197 100644 --- a/README.md +++ b/README.md @@ -1,3 +1,18 @@ +Создаем сеть: +docker network create app-network + +Войти в БД: +docker exec -it postgres psql -U ncp_db -d no_copy_ + +Выполнить команду: +\du + +Должен быть один пользователь: + List of roles +Role name | Attributes +-----------+------------------------------------------------------------ +ncp_db | Superuser, Create role, Create DB, Replication, Bypass RLS + суперпользователь БД: postgres/postgres пользователь бд для бэка @@ -46,3 +61,8 @@ GRANT ALL ON TABLES TO ncp_db; ALTER DEFAULT PRIVILEGES IN SCHEMA public GRANT ALL ON SEQUENCES TO ncp_db; + + +Зайти в контейнер: +docker exec -it {name} bash + diff --git a/docker-compose.yaml b/docker-compose.yaml index 6a69921..bb1e1b7 100644 --- a/docker-compose.yaml +++ b/docker-compose.yaml @@ -30,8 +30,6 @@ services: memory: 1G environment: POSTGRES_DB: no_copy_ -# POSTGRES_USER: postgres -# POSTGRES_PASSWORD: postgres POSTGRES_USER: ${POSTGRES_USER} POSTGRES_PASSWORD: ${POSTGRES_PASSWORD} POSTGRES_SHARED_BUFFERS: 512MB @@ -61,11 +59,8 @@ services: environment: FILE_STORAGE_PATH: /data/uploads MAX_FILE_SIZE: 10737418240 -# FILE_CHUNK_SIZE: 1048576 FILE_CHUNK_SIZE: 1000000 POSTGRES_DB: no_copy_ -# POSTGRES_USER: postgres -# POSTGRES_PASSWORD: postgres POSTGRES_USER: ${POSTGRES_USER} POSTGRES_PASSWORD: ${POSTGRES_PASSWORD} POSTGRES_PORT: 5432 diff --git a/src/main/resources/application.yaml b/src/main/resources/application.yaml index 9a34782..db4e38c 100644 --- a/src/main/resources/application.yaml +++ b/src/main/resources/application.yaml @@ -24,8 +24,6 @@ spring: file: storage: base-path: ${FILE_STORAGE_PATH:/data/uploads} - # chunk-size: ${FILE_CHUNK_SIZE:5242880} # 5MB -# chunk-size: ${FILE_CHUNK_SIZE:1048576} # 1MB chunk-size: ${FILE_CHUNK_SIZE:1000000} # 1MB max-file-size: ${MAX_FILE_SIZE:10737418240} # 10GB max-retry-attempts: ${MAX_RETRY_ATTEMPTS:3} From 0c0fadf5ecb594f3bb1e86cf3e1d8be2c258ac25 Mon Sep 17 00:00:00 2001 From: vladp Date: Mon, 19 Jan 2026 15:49:27 +0700 Subject: [PATCH 25/27] NCBACK-34 fix send exception --- .../nocopy/controller/ApiController.java | 14 +- .../exception/DuplicateImageException.java | 15 +- .../nocopy/handler/FileUploadHandler.java | 28 --- .../repository/ImageSimilarityRepository.java | 23 ++- .../repository/SimilarImageProjection.java | 12 +- .../nocopy/service/FileSimilarityService.java | 15 +- .../nocopy/service/ImageHashService.java | 32 ++-- .../service/file/FileEntityService.java | 24 ++- .../service/file/FileUploadService.java | 2 - .../service/file/FileUploadServiceImpl.java | 161 ++++++++++++------ 10 files changed, 206 insertions(+), 120 deletions(-) diff --git a/src/main/java/ru/soune/nocopy/controller/ApiController.java b/src/main/java/ru/soune/nocopy/controller/ApiController.java index 5c408fc..bae2817 100644 --- a/src/main/java/ru/soune/nocopy/controller/ApiController.java +++ b/src/main/java/ru/soune/nocopy/controller/ApiController.java @@ -110,17 +110,13 @@ public class ApiController { return buildErrorResponse(uploadId, chunkNumber, "Valid chunk number is required"); } - UploadProgressResponse uploadProgressResponse = fileUploadService.uploadChunk(uploadId, chunkNumber, chunk); - - if (uploadProgressResponse.getStatus().equals(UploadStatus.COMPLETED)) { - ResponseEntity duplicateCheckResult = checkForDuplicates(uploadProgressResponse.getUploadId()); - if (duplicateCheckResult != null) { - return duplicateCheckResult; - } - } + fileUploadService.uploadChunk(uploadId, chunkNumber, chunk); return buildSuccessResponse(uploadId, chunkNumber, chunk); - + } catch (DuplicateImageException e) { + return ResponseEntity.ok().body(new BaseResponse(20004, MessageCode.DUPLICATE_FILE_UPLOAD.getCode(), + MessageCode.DUPLICATE_FILE_UPLOAD.getDescription(), + Map.of("duplicateOwnerId", e.userId(), "duplicateFileId", e.duplicateFileId()))); } catch (Exception e) { log.error("Error uploading chunk", e); return buildErrorResponse(uploadId, chunkNumber, "Failed to upload chunk: " + e.getMessage()); diff --git a/src/main/java/ru/soune/nocopy/exception/DuplicateImageException.java b/src/main/java/ru/soune/nocopy/exception/DuplicateImageException.java index a7593b8..3ac1325 100644 --- a/src/main/java/ru/soune/nocopy/exception/DuplicateImageException.java +++ b/src/main/java/ru/soune/nocopy/exception/DuplicateImageException.java @@ -1,7 +1,20 @@ package ru.soune.nocopy.exception; public class DuplicateImageException extends RuntimeException { - public DuplicateImageException(String message) { + private final String duplicateFileId; + private final Long userId; + + public DuplicateImageException(String message, String duplicateFileId, Long userId) { super(message); + this.duplicateFileId = duplicateFileId; + this.userId = userId; + } + + public String duplicateFileId() { + return duplicateFileId; + } + + public Long userId() { + return userId; } } diff --git a/src/main/java/ru/soune/nocopy/handler/FileUploadHandler.java b/src/main/java/ru/soune/nocopy/handler/FileUploadHandler.java index 8240403..2c75fb0 100644 --- a/src/main/java/ru/soune/nocopy/handler/FileUploadHandler.java +++ b/src/main/java/ru/soune/nocopy/handler/FileUploadHandler.java @@ -175,34 +175,6 @@ public class FileUploadHandler implements RequestHandler { } } - private BaseResponse handleRetryUpload(BaseRequest request, FileUploadRequest fileRequest) { - try { - fileUploadService.retryFailedUpload(fileRequest.getUploadId()); - ru.soune.nocopy.dto.file.UploadProgressResponse progress = - fileUploadService.getUploadProgress(fileRequest.getUploadId()); - - RetryUploadResponse response = RetryUploadResponse.builder() - .uploadId(progress.getUploadId()) - .message("Upload retry initiated") - .status(progress.getStatus().toString()) - .build(); - - return new BaseResponse(request.getMsgId(), MessageCode.SUCCESS.getCode(), "Upload retry initiated", - response); - - } catch (UploadSessionNotFoundException e) { - return new BaseResponse(request.getMsgId(), MessageCode.FILE_NOT_FOUND.getCode(), - "Upload session not found", null); - } catch (FileUploadException e) { - return new BaseResponse(request.getMsgId(), MessageCode.FILE_UPLOAD_ERROR.getCode(), e.getMessage(), - null); - } catch (Exception e) { - log.error("Error retrying upload", e); - return new BaseResponse(request.getMsgId(), MessageCode.FILE_UPLOAD_ERROR.getCode(), - "Failed to retry upload", null); - } - } - private BaseResponse handleGetChunkStatus(BaseRequest request, FileUploadRequest fileRequest) { try { FileUploadSession session = fileUploadSessionRepository.findById(fileRequest.getUploadId()) diff --git a/src/main/java/ru/soune/nocopy/repository/ImageSimilarityRepository.java b/src/main/java/ru/soune/nocopy/repository/ImageSimilarityRepository.java index eeabe3a..a6ef973 100644 --- a/src/main/java/ru/soune/nocopy/repository/ImageSimilarityRepository.java +++ b/src/main/java/ru/soune/nocopy/repository/ImageSimilarityRepository.java @@ -13,7 +13,8 @@ public interface ImageSimilarityRepository extends JpaRepository { @Query(value = """ - SELECT + + SELECT f.id AS similarFileId, f.original_file_name AS originalFileName, f.file_size AS fileSize, @@ -30,6 +31,24 @@ public interface ImageSimilarityRepository List findCandidates( @Param("fileId") String fileId ); + + @Query(value = """ + SELECT + h.hash64_hi AS hash64Hi, + h.hash64_lo AS hash64Lo, + h.file_id AS fileId, + f.user_id AS userId, + f.original_file_name AS originalFileName, + f.file_size AS fileSize, + f.stored_file_name AS similarFileId + FROM image_hashes h + JOIN file_entities f ON f.id = h.file_id + WHERE h.hash64_hi = :hash64Hi + AND h.hash64_lo = :hash64Lo + """, + nativeQuery = true) + List findExactDuplicates( + @Param("hash64Hi") Integer hash64_hi, + @Param("hash64Lo") Integer hash64_lo); } - diff --git a/src/main/java/ru/soune/nocopy/repository/SimilarImageProjection.java b/src/main/java/ru/soune/nocopy/repository/SimilarImageProjection.java index 7263394..ed2fdb4 100644 --- a/src/main/java/ru/soune/nocopy/repository/SimilarImageProjection.java +++ b/src/main/java/ru/soune/nocopy/repository/SimilarImageProjection.java @@ -1,14 +1,10 @@ package ru.soune.nocopy.repository; public interface SimilarImageProjection { - - String getSimilarFileId(); - - String getOriginalFileName(); - - Long getFileSize(); - Integer getHash64Hi(); - Integer getHash64Lo(); + String getFileId(); + Long getUserId(); + String getOriginalFileName(); + Long getFileSize(); } \ No newline at end of file diff --git a/src/main/java/ru/soune/nocopy/service/FileSimilarityService.java b/src/main/java/ru/soune/nocopy/service/FileSimilarityService.java index f8b7407..dea9334 100644 --- a/src/main/java/ru/soune/nocopy/service/FileSimilarityService.java +++ b/src/main/java/ru/soune/nocopy/service/FileSimilarityService.java @@ -11,6 +11,7 @@ import ru.soune.nocopy.repository.ImageHashRepository; import ru.soune.nocopy.repository.ImageSimilarityRepository; import ru.soune.nocopy.repository.SimilarImageProjection; +import java.util.ArrayList; import java.util.Comparator; import java.util.List; import java.util.stream.Collectors; @@ -49,7 +50,7 @@ public class FileSimilarityService { } return SimilarFileResponse.builder() - .fileId(c.getSimilarFileId()) + .fileId(c.getFileId()) .originalFileName(c.getOriginalFileName()) .fileSize(c.getFileSize()) .hammingDistance(hamming) @@ -61,6 +62,16 @@ public class FileSimilarityService { .toList(); } + public List findDuplicatedByHash(Integer hash64Hi, Integer hash64Lo) { + List duplicates = repository.findExactDuplicates(hash64Hi,hash64Lo); + + if (duplicates.isEmpty()) { + return new ArrayList<>(); + } + + return duplicates; + } + public Page findSimilarFiles(String fileId, SimilarityFilter filter, Pageable pageable) { var imageHashEntity = hashRepository.findById(fileId) .orElseThrow(() -> new RuntimeException("Hash not found")); @@ -109,7 +120,7 @@ public class FileSimilarityService { } return SimilarFileResponse.builder() - .fileId(similarImageProjection.getSimilarFileId()) + .fileId(similarImageProjection.getFileId()) .originalFileName(similarImageProjection.getOriginalFileName()) .fileSize(similarImageProjection.getFileSize()) .hammingDistance(hamming) diff --git a/src/main/java/ru/soune/nocopy/service/ImageHashService.java b/src/main/java/ru/soune/nocopy/service/ImageHashService.java index 992040b..b7b6882 100644 --- a/src/main/java/ru/soune/nocopy/service/ImageHashService.java +++ b/src/main/java/ru/soune/nocopy/service/ImageHashService.java @@ -16,6 +16,7 @@ import java.nio.file.Files; import java.nio.file.Path; import java.time.LocalDateTime; import java.util.Arrays; +import java.util.Map; @Service @RequiredArgsConstructor @@ -24,25 +25,24 @@ public class ImageHashService { private final ImageHashRepository repository; - public void create(FileEntity file, Path imagePath) { - try { - long hash64 = computePhash64(imagePath); + public Map calculateHash(Path imagePath) throws IOException { + long hash64 = computePhash64(imagePath); - int hi = high32(hash64); - int lo = low32(hash64); + return Map.of( + "hi", high32(hash64), + "low", low32(hash64)); + } - ImageHashEntity entity = ImageHashEntity.builder() - .file(file) - .hash64Hi(hi) - .hash64Lo(lo) - .hashAlgorithm("PHASH64") - .createdAt(LocalDateTime.now()) - .build(); + public void create(FileEntity file, Map stringIntegerMap) { + ImageHashEntity entity = ImageHashEntity.builder() + .file(file) + .hash64Hi(stringIntegerMap.get("hi")) + .hash64Lo(stringIntegerMap.get("low")) + .hashAlgorithm("PHASH64") + .createdAt(LocalDateTime.now()) + .build(); - repository.save(entity); - } catch (IOException e) { - throw new RuntimeException("Failed to compute image hash", e); - } + repository.save(entity); } private long computePhash64(Path path) throws IOException { diff --git a/src/main/java/ru/soune/nocopy/service/file/FileEntityService.java b/src/main/java/ru/soune/nocopy/service/file/FileEntityService.java index 0a17c22..0cdf62c 100644 --- a/src/main/java/ru/soune/nocopy/service/file/FileEntityService.java +++ b/src/main/java/ru/soune/nocopy/service/file/FileEntityService.java @@ -9,8 +9,10 @@ import ru.soune.nocopy.dto.file.FileResponse; import ru.soune.nocopy.entity.file.FileEntity; import ru.soune.nocopy.entity.file.FileStatus; import ru.soune.nocopy.entity.file.FileUploadSession; +import ru.soune.nocopy.exception.DuplicateImageException; import ru.soune.nocopy.exception.FileEntityNotFoundException; import ru.soune.nocopy.repository.FileEntityRepository; +import ru.soune.nocopy.repository.SimilarImageProjection; import ru.soune.nocopy.service.FileSimilarityService; import ru.soune.nocopy.service.ImageHashService; @@ -20,6 +22,7 @@ import java.nio.file.Path; import java.nio.file.Paths; import java.time.LocalDateTime; import java.util.List; +import java.util.Map; import java.util.stream.Collectors; @Slf4j @@ -33,7 +36,7 @@ public class FileEntityService { private final FileSimilarityService fileSimilarityService; - @Transactional + @Transactional(noRollbackFor = DuplicateImageException.class) public FileEntity createFromUploadSession(FileUploadSession session, String checksum) { log.info("Creating FileEntity for upload session: {}", session.getUploadId()); @@ -44,6 +47,21 @@ public class FileEntityService { throw new IOException("File not found on disk: " + filePath); } + Map imageHash = Map.of(); + + if (session.getFileType().startsWith("image")) { + imageHash = imageHashService.calculateHash(filePath); + List duplicatedByHash = fileSimilarityService.findDuplicatedByHash( + imageHash.get("hi"), imageHash.get("low")); + + if (!duplicatedByHash.isEmpty()) { + SimilarImageProjection similarImageProjection = duplicatedByHash.get(0); + + throw new DuplicateImageException("Duplicate", similarImageProjection.getFileId(), + similarImageProjection.getUserId()); + } + } + long fileSize = Files.size(filePath); String originalName = session.getFileName(); String storedName = filePath.getFileName().toString(); @@ -63,8 +81,8 @@ public class FileEntityService { FileEntity saved = fileEntityRepository.save(fileEntity); - if (saved.getMimeType().startsWith("image")) { - imageHashService.create(saved, Path.of(saved.getFilePath())); + if (!imageHash.isEmpty()) { + imageHashService.create(saved, imageHash); } return saved; diff --git a/src/main/java/ru/soune/nocopy/service/file/FileUploadService.java b/src/main/java/ru/soune/nocopy/service/file/FileUploadService.java index b27dede..714ef21 100644 --- a/src/main/java/ru/soune/nocopy/service/file/FileUploadService.java +++ b/src/main/java/ru/soune/nocopy/service/file/FileUploadService.java @@ -17,7 +17,5 @@ public interface FileUploadService { void handleExpiredSession(FileUploadSession session); - public void retryFailedUpload(String uploadId); - public void cancelUpload(String uploadId); } diff --git a/src/main/java/ru/soune/nocopy/service/file/FileUploadServiceImpl.java b/src/main/java/ru/soune/nocopy/service/file/FileUploadServiceImpl.java index 81df025..587cdc1 100644 --- a/src/main/java/ru/soune/nocopy/service/file/FileUploadServiceImpl.java +++ b/src/main/java/ru/soune/nocopy/service/file/FileUploadServiceImpl.java @@ -10,12 +10,19 @@ import org.springframework.stereotype.Service; import org.springframework.transaction.annotation.Transactional; import org.springframework.web.multipart.MultipartFile; import ru.soune.nocopy.dto.file.UploadProgressResponse; +import ru.soune.nocopy.entity.file.FileEntity; +import ru.soune.nocopy.entity.file.FileStatus; import ru.soune.nocopy.entity.file.FileUploadSession; import ru.soune.nocopy.entity.file.UploadStatus; import ru.soune.nocopy.exception.ChunkSizeExceededException; +import ru.soune.nocopy.exception.DuplicateImageException; import ru.soune.nocopy.exception.FileUploadException; import ru.soune.nocopy.exception.UploadSessionNotFoundException; +import ru.soune.nocopy.repository.FileEntityRepository; import ru.soune.nocopy.repository.FileUploadSessionRepository; +import ru.soune.nocopy.repository.SimilarImageProjection; +import ru.soune.nocopy.service.FileSimilarityService; +import ru.soune.nocopy.service.ImageHashService; import java.io.*; import java.nio.file.*; @@ -23,10 +30,7 @@ import java.nio.file.attribute.PosixFilePermission; import java.security.MessageDigest; import java.security.NoSuchAlgorithmException; import java.time.LocalDateTime; -import java.util.Comparator; -import java.util.HashSet; -import java.util.Set; -import java.util.UUID; +import java.util.*; import java.util.concurrent.CompletableFuture; import java.util.stream.Stream; @@ -57,6 +61,15 @@ public class FileUploadServiceImpl implements FileUploadService { private Path storageRoot; + @Autowired + private ImageHashService imageHashService; + + @Autowired + private FileSimilarityService fileSimilarityService; + + @Autowired + private FileEntityRepository fileEntityRepository; + @Autowired private FileEntityService fileEntityService; @@ -129,10 +142,6 @@ public class FileUploadServiceImpl implements FileUploadService { @Transactional public UploadProgressResponse uploadChunk(String uploadId, Integer chunkNumber, MultipartFile chunkFile) { - - log.info("Processing chunk {} for session {}, size: {} bytes", - chunkNumber, uploadId, chunkFile.getSize()); - FileUploadSession session = sessionRepository.findById(uploadId) .orElseThrow(() -> new UploadSessionNotFoundException(uploadId)); @@ -173,12 +182,13 @@ public class FileUploadServiceImpl implements FileUploadService { @Async("fileUploadTaskExecutor") @Transactional - public void assembleFileAsync(FileUploadSession session) { + public void assembleFileAsync(FileUploadSession session) throws DuplicateImageException { try { assembleFile(session); log.info("File assembly completed successfully for session: {}", session.getUploadId()); - + } catch (DuplicateImageException e) { + throw new DuplicateImageException("DUBL", e.duplicateFileId(), e.userId()); } catch (Exception e) { log.error("Failed to assemble file for session {}: {}", session.getUploadId(), e.getMessage(), e); @@ -205,35 +215,6 @@ public class FileUploadServiceImpl implements FileUploadService { log.info("Upload cancelled: {}", uploadId); } - @Override - @Transactional - public void retryFailedUpload(String uploadId) { - FileUploadSession session = sessionRepository.findById(uploadId) - .orElseThrow(() -> new UploadSessionNotFoundException(uploadId)); - - if (session.getStatus() != UploadStatus.FAILED) { - throw new FileUploadException("Only failed uploads can be retried"); - } - - if (session.getRetryCount() >= maxRetryAttempts) { - throw new FileUploadException("Max retry attempts exceeded"); - } - - session.setStatus(UploadStatus.UPLOADING); - session.setRetryCount(session.getRetryCount() + 1); - session.setLastError(null); - - sessionRepository.save(session); - - if (session.getChunksUploaded().equals(session.getTotalChunks())) { - log.info("Retrying file assembly for session: {}", uploadId); - assembleFileAsync(session); - } - - log.info("Upload retry initiated for session: {} (attempt {})", - uploadId, session.getRetryCount()); - } - @Override public UploadProgressResponse getUploadProgress(String uploadId) { FileUploadSession session = sessionRepository.findById(uploadId) @@ -271,9 +252,42 @@ public class FileUploadServiceImpl implements FileUploadService { } } - private UploadProgressResponse processChunk(FileUploadSession session, - Integer chunkNumber, - MultipartFile chunkFile) { + @Async("fileUploadTaskExecutor") + @Transactional + public void completeFileProcessingAsync(FileUploadSession session) { + try { + Path filePath = Paths.get(session.getFilePath()); + String checksum = calculateChecksum(filePath); + + FileEntity fileEntity = FileEntity.builder() + .userId(session.getUserId()) + .originalFileName(session.getFileName()) + .storedFileName(filePath.getFileName().toString()) + .filePath(session.getFilePath()) + .fileSize(session.getFileSize()) + .mimeType(session.getFileType()) + .fileExtension(session.getExtension()) + .checksum(checksum) + .uploadSessionId(session.getUploadId()) + .status(FileStatus.ACTIVE) + .build(); + + FileEntity saved = fileEntityRepository.save(fileEntity); + + Map hash = imageHashService.calculateHash(filePath); + imageHashService.create(saved, hash); + + cleanupSessionFiles(session); + + log.info("File processing completed for session: {}", session.getUploadId()); + + } catch (Exception e) { + log.error("Failed to complete file processing for session {}: {}", + session.getUploadId(), e.getMessage(), e); + } + } + + private UploadProgressResponse processChunk(FileUploadSession session, Integer chunkNumber, MultipartFile chunkFile) { String chunkPath = null; try { @@ -291,17 +305,42 @@ public class FileUploadServiceImpl implements FileUploadService { log.debug("Chunk {} saved successfully. Uploaded: {}/{}", chunkNumber, session.getChunksUploaded(), session.getTotalChunks()); - if (session.getChunksUploaded().equals(session.getTotalChunks())) { + boolean isLastChunk = session.getChunksUploaded().equals(session.getTotalChunks()); + + if (isLastChunk) { log.info("All chunks uploaded for session {}. Starting assembly...", session.getUploadId()); - sessionRepository.save(session); - assembleFileAsync(session); + + String finalFilePath = assembleFileSynchronously(session); + + if (session.getFileType().startsWith("image")) { + checkForDuplicatesSynchronously(finalFilePath); + } + + session.setStatus(UploadStatus.COMPLETED); + session.setFilePath(finalFilePath); + + completeFileProcessingAsync(session); + } else { sessionRepository.save(session); } return UploadProgressResponse.fromSession(session); + } catch (DuplicateImageException e) { + if (chunkPath != null) { + cleanupFailedChunk(chunkPath); + } + + log.warn("Duplicate image found for session {}: {}", + session.getUploadId(), e.getMessage()); + + session.setStatus(UploadStatus.FAILED); + session.setLastError(e.getMessage()); + sessionRepository.save(session); + + throw e; } catch (Exception e) { if (chunkPath != null) { cleanupFailedChunk(chunkPath); @@ -318,6 +357,29 @@ public class FileUploadServiceImpl implements FileUploadService { } } + private String assembleFileSynchronously(FileUploadSession session) throws IOException { + Path finalFilePath = prepareFinalFile(session); + + validateAllChunksExist(session); + mergeChunksToFile(session, finalFilePath); + validateFinalFile(session, finalFilePath); + + return finalFilePath.toString(); + } + + private void checkForDuplicatesSynchronously(String filePath) + throws IOException { + Path path = Paths.get(filePath); + Map hash = imageHashService.calculateHash(path); + List duplicates = fileSimilarityService.findDuplicatedByHash( + hash.get("hi"), hash.get("low")); + + if (!duplicates.isEmpty()) { + throw new DuplicateImageException("Duplicate", duplicates.get(0).getFileId(), + duplicates.get(0).getUserId()); + } + } + private UploadProgressResponse handleExistingChunk(FileUploadSession session, Integer chunkNumber, MultipartFile chunkFile) throws IOException { @@ -325,25 +387,24 @@ public class FileUploadServiceImpl implements FileUploadService { Path chunkPath = Paths.get(existingPath); if (!Files.exists(chunkPath)) { - log.warn("Chunk file missing, re-uploading: {}", chunkPath); session.getChunkPaths().remove(chunkNumber); session.setChunksUploaded(session.getChunksUploaded() - 1); sessionRepository.save(session); + return processChunk(session, chunkNumber, chunkFile); } long existingSize = Files.size(chunkPath); + if (existingSize != chunkFile.getSize()) { - log.warn("Chunk size mismatch, re-uploading: {} != {}", - existingSize, chunkFile.getSize()); Files.deleteIfExists(chunkPath); session.getChunkPaths().remove(chunkNumber); session.setChunksUploaded(session.getChunksUploaded() - 1); sessionRepository.save(session); + return processChunk(session, chunkNumber, chunkFile); } - log.debug("Chunk {} already uploaded and valid", chunkNumber); return UploadProgressResponse.fromSession(session); } @@ -428,6 +489,8 @@ public class FileUploadServiceImpl implements FileUploadService { fileEntityService.createFromUploadSession(session, checksum); log.info("FileEntity successfully created for session: {}", session.getUploadId()); + } catch (DuplicateImageException e) { + throw new DuplicateImageException("Duplicate", e.duplicateFileId(), e.userId()); } catch (Exception e) { log.error("Failed to create FileEntity for session {} (file uploaded, but metadata not saved): {}", session.getUploadId(), e.getMessage()); From 60678ef889706c2d84fc13cfee83cca9f0344c47 Mon Sep 17 00:00:00 2001 From: vladp Date: Tue, 20 Jan 2026 21:55:42 +0700 Subject: [PATCH 26/27] Dev add support id --- README.md | 10 +++++ .../nocopy/dto/file/FileApiResponse.java | 41 ------------------- .../nocopy/dto/file/FileEntityResponse.java | 1 + .../soune/nocopy/entity/file/FileEntity.java | 12 +++--- .../service/file/FileEntityService.java | 2 +- 5 files changed, 18 insertions(+), 48 deletions(-) delete mode 100644 src/main/java/ru/soune/nocopy/dto/file/FileApiResponse.java diff --git a/README.md b/README.md index d133197..aa5e26c 100644 --- a/README.md +++ b/README.md @@ -66,3 +66,13 @@ GRANT ALL ON SEQUENCES TO ncp_db; Зайти в контейнер: docker exec -it {name} bash +------ + +Создать для сущности FileEntity sequence для икремента индетификатора + +CREATE SEQUENCE IF NOT EXISTS file_support_id_seq START 1; + +ALTER TABLE file_entities +ALTER COLUMN support_id +SET DEFAULT nextval('file_support_id_seq'); + diff --git a/src/main/java/ru/soune/nocopy/dto/file/FileApiResponse.java b/src/main/java/ru/soune/nocopy/dto/file/FileApiResponse.java deleted file mode 100644 index 9feb546..0000000 --- a/src/main/java/ru/soune/nocopy/dto/file/FileApiResponse.java +++ /dev/null @@ -1,41 +0,0 @@ -package ru.soune.nocopy.dto.file; - -import lombok.AllArgsConstructor; -import lombok.Builder; -import lombok.Data; -import lombok.NoArgsConstructor; - -@Data -@Builder -@NoArgsConstructor -@AllArgsConstructor -public class FileApiResponse { - - private boolean success; - private String message; - private T data; - private String error; - - public static FileApiResponse success(T data) { - return FileApiResponse.builder() - .success(true) - .message("Operation completed successfully") - .data(data) - .build(); - } - - public static FileApiResponse success(String message, T data) { - return FileApiResponse.builder() - .success(true) - .message(message) - .data(data) - .build(); - } - - public static FileApiResponse error(String error) { - return FileApiResponse.builder() - .success(false) - .error(error) - .build(); - } -} diff --git a/src/main/java/ru/soune/nocopy/dto/file/FileEntityResponse.java b/src/main/java/ru/soune/nocopy/dto/file/FileEntityResponse.java index 3e89725..cbfae88 100644 --- a/src/main/java/ru/soune/nocopy/dto/file/FileEntityResponse.java +++ b/src/main/java/ru/soune/nocopy/dto/file/FileEntityResponse.java @@ -30,4 +30,5 @@ public class FileEntityResponse { private String formattedSize; private String downloadUrl; private boolean existsOnDisk; + private Integer supportId; } diff --git a/src/main/java/ru/soune/nocopy/entity/file/FileEntity.java b/src/main/java/ru/soune/nocopy/entity/file/FileEntity.java index ffdd985..c39862a 100644 --- a/src/main/java/ru/soune/nocopy/entity/file/FileEntity.java +++ b/src/main/java/ru/soune/nocopy/entity/file/FileEntity.java @@ -2,10 +2,13 @@ package ru.soune.nocopy.entity.file; import jakarta.persistence.*; import lombok.*; +import org.hibernate.annotations.GenerationTime; import org.springframework.data.annotation.CreatedDate; import org.springframework.data.annotation.LastModifiedDate; import org.springframework.data.jpa.domain.support.AuditingEntityListener; +import java.sql.DriverManager; +import java.sql.SQLException; import java.time.LocalDateTime; @Data @@ -16,11 +19,13 @@ import java.time.LocalDateTime; @Table(name = "file_entities") @EntityListeners(AuditingEntityListener.class) public class FileEntity { - @Id @GeneratedValue(strategy = GenerationType.UUID) private String id; + @Column(name = "support_id", unique = true, updatable = false) + private Integer supportId; + @Column(name = "user_id", nullable = false) private Long userId; @@ -70,9 +75,4 @@ public class FileEntity { } this.updatedAt = LocalDateTime.now(); } - - @PreUpdate - public void preUpdate() { - this.updatedAt = LocalDateTime.now(); - } } diff --git a/src/main/java/ru/soune/nocopy/service/file/FileEntityService.java b/src/main/java/ru/soune/nocopy/service/file/FileEntityService.java index 617eeda..3eac369 100644 --- a/src/main/java/ru/soune/nocopy/service/file/FileEntityService.java +++ b/src/main/java/ru/soune/nocopy/service/file/FileEntityService.java @@ -17,7 +17,6 @@ import java.nio.file.Files; import java.nio.file.Path; import java.nio.file.Paths; import java.time.LocalDateTime; -import java.time.ZoneOffset; import java.util.List; import java.util.stream.Collectors; @@ -223,6 +222,7 @@ public class FileEntityService { .formattedSize(formatFileSize(fileEntity.getFileSize())) .downloadUrl("/api/v" + version + "/files/download/" + fileEntity.getId()) .existsOnDisk(existsOnDisk) + .supportId(fileEntity.getSupportId()) .build(); } From ad9cbd96f69fc3502cae64240e5644e01d949c82 Mon Sep 17 00:00:00 2001 From: vladp Date: Thu, 22 Jan 2026 11:24:37 +0700 Subject: [PATCH 27/27] NCBACK-34 update dev --- src/main/java/ru/soune/nocopy/controller/ApiController.java | 2 -- 1 file changed, 2 deletions(-) diff --git a/src/main/java/ru/soune/nocopy/controller/ApiController.java b/src/main/java/ru/soune/nocopy/controller/ApiController.java index d80b45c..c8261ed 100644 --- a/src/main/java/ru/soune/nocopy/controller/ApiController.java +++ b/src/main/java/ru/soune/nocopy/controller/ApiController.java @@ -19,7 +19,6 @@ import ru.soune.nocopy.dto.BaseResponse; import ru.soune.nocopy.dto.MessageCode; import ru.soune.nocopy.dto.RegAnswer; import ru.soune.nocopy.dto.file.*; -import ru.soune.nocopy.entity.AuthToken; import ru.soune.nocopy.entity.file.FileEntity; import ru.soune.nocopy.dto.file.ChunkUploadResponse; import ru.soune.nocopy.dto.file.CompleteUploadResponse; @@ -29,7 +28,6 @@ import ru.soune.nocopy.entity.file.FileStatus; import ru.soune.nocopy.entity.file.UploadStatus; import ru.soune.nocopy.exception.*; import ru.soune.nocopy.handler.*; -import ru.soune.nocopy.repository.AuthTokenRepository; import ru.soune.nocopy.repository.FileEntityRepository; import ru.soune.nocopy.service.FileSimilarityService; import ru.soune.nocopy.service.auth.AuthService;