From 3d679440f6b0607d32395d9f984b5ffab737713f Mon Sep 17 00:00:00 2001 From: vladp Date: Fri, 16 Jan 2026 14:30:45 +0700 Subject: [PATCH] NCBACK-34 add logic for clear does not exist files,add dto for similar file search controller --- .../nocopy/controller/ApiController.java | 121 +++++++++++------- .../java/ru/soune/nocopy/dto/MessageCode.java | 3 +- .../nocopy/dto/file/SimilarityFilter.java | 18 +++ .../soune/nocopy/entity/file/FileEntity.java | 5 + .../repository/FileEntityRepository.java | 3 + .../nocopy/service/FileSimilarityService.java | 64 ++++++++- .../file/NotExistFilesCleanupService.java | 35 +++++ 7 files changed, 202 insertions(+), 47 deletions(-) create mode 100644 src/main/java/ru/soune/nocopy/dto/file/SimilarityFilter.java create mode 100644 src/main/java/ru/soune/nocopy/service/file/NotExistFilesCleanupService.java diff --git a/src/main/java/ru/soune/nocopy/controller/ApiController.java b/src/main/java/ru/soune/nocopy/controller/ApiController.java index b966fd7..5c408fc 100644 --- a/src/main/java/ru/soune/nocopy/controller/ApiController.java +++ b/src/main/java/ru/soune/nocopy/controller/ApiController.java @@ -1,12 +1,12 @@ package ru.soune.nocopy.controller; -import jakarta.persistence.EntityManager; -import jakarta.persistence.PersistenceContext; import lombok.RequiredArgsConstructor; import lombok.extern.slf4j.Slf4j; -import org.springframework.beans.factory.annotation.Autowired; import org.springframework.core.io.Resource; import org.springframework.core.io.UrlResource; +import org.springframework.data.domain.Page; +import org.springframework.data.domain.Pageable; +import org.springframework.data.web.PageableDefault; import org.springframework.http.HttpHeaders; import org.springframework.http.MediaType; import org.springframework.http.ResponseEntity; @@ -28,7 +28,6 @@ import ru.soune.nocopy.handler.*; import ru.soune.nocopy.repository.AuthTokenRepository; import ru.soune.nocopy.repository.FileEntityRepository; import ru.soune.nocopy.service.FileSimilarityService; -import ru.soune.nocopy.service.ImageHashService; import ru.soune.nocopy.service.file.FileEntityService; import ru.soune.nocopy.service.file.FileUploadService; @@ -128,51 +127,39 @@ public class ApiController { } } - private ResponseEntity checkForDuplicates(String uploadId) throws IOException { - Optional uploadedFile = fileEntityRepository.findByUploadSessionId(uploadId); + @GetMapping("/v{version}/files/{fileId}/similar") + public ResponseEntity findSimilarFiles( + @PathVariable("version") int version, + @PathVariable String fileId, + @RequestParam(required = false) List similarityLevels, + @PageableDefault(size = 20, sort = "hammingDistance") Pageable pageable) { + SimilarityFilter filter = SimilarityFilter.builder() + .similarityLevels(similarityLevels) + .build(); + Page similarFiles = fileSimilarityService.findSimilarFiles(fileId, filter, pageable); - if (uploadedFile.isEmpty() || !uploadedFile.get().getMimeType().equals("image")) { - return null; + String messageDesc; + MessageCode success; + + if (similarFiles.isEmpty()) { + messageDesc = MessageCode.FILE_NOT_FOUND.getDescription(); + success = MessageCode.FILE_NOT_FOUND; + } else { + messageDesc = MessageCode.SIMILAR_FILES_FOUND.getDescription(); + success = MessageCode.SIMILAR_FILES_FOUND; } - FileEntity fileEntity = uploadedFile.get(); - List similarFiles = fileSimilarityService.findSimilarFiles(fileEntity.getId()); + Map responseData = new HashMap<>(); + responseData.put("content", similarFiles.getContent()); + responseData.put("page", similarFiles.getNumber()); + responseData.put("size", similarFiles.getSize()); + responseData.put("totalElements", similarFiles.getTotalElements()); + responseData.put("totalPages", similarFiles.getTotalPages()); + responseData.put("hasNext", similarFiles.hasNext()); + responseData.put("hasPrevious", similarFiles.hasPrevious()); - if (hasDuplicate(similarFiles)) { - return handleDuplicate(fileEntity, similarFiles); - } - - return null; - } - - private boolean hasDuplicate(List similarFiles) { - return similarFiles.stream().anyMatch(f -> f.getHammingDistance() <= 5); - } - - private ResponseEntity handleDuplicate(FileEntity fileEntity, List similarFiles) - throws IOException { - fileEntityService.deleteFromDisk(fileEntity); - - Optional originalFile = fileEntityRepository.findById(similarFiles.get(0).getFileId()); - - if (originalFile.isPresent()) { - Map duplicateInfo = Map.of( - "duplicate_file_id", originalFile.get().getId(), - "owner_user_id", String.valueOf(originalFile.get().getUserId())); - - return ResponseEntity.ok().body(new BaseResponse( - 20004, - MessageCode.DUPLICATE_FILE_UPLOAD.getCode(), - "Failed to upload chunk, duplicate", - duplicateInfo)); - } - - return null; - } - - @GetMapping("/files/{fileId}/similar") - public List getSimilarFiles(@PathVariable String fileId) { - return fileSimilarityService.findSimilarFiles(fileId); + return ResponseEntity.ok() + .body(new BaseResponse(20004, success.getCode(), messageDesc, responseData)); } @GetMapping("/v{version}/files/progress/{uploadId}") @@ -364,6 +351,50 @@ public class ApiController { } } + private ResponseEntity checkForDuplicates(String uploadId) throws IOException { + Optional uploadedFile = fileEntityRepository.findByUploadSessionId(uploadId); + + if (uploadedFile.isEmpty() || !uploadedFile.get().getMimeType().equals("image")) { + return null; + } + + FileEntity fileEntity = uploadedFile.get(); + List similarFiles = fileSimilarityService.findSimilarFiles(fileEntity.getId()); + + if (hasDuplicate(similarFiles)) { + return handleDuplicate(fileEntity, similarFiles); + } + + return null; + } + + private boolean hasDuplicate(List similarFiles) { + return similarFiles.stream().anyMatch(f -> f.getHammingDistance() <= 5); + } + + private ResponseEntity handleDuplicate(FileEntity fileEntity, List similarFiles) + throws IOException { + fileEntityService.deleteFromDisk(fileEntity); + + fileEntityService.markAsDeleted(fileEntity); + + Optional originalFile = fileEntityRepository.findById(similarFiles.get(0).getFileId()); + + if (originalFile.isPresent()) { + Map duplicateInfo = Map.of( + "duplicate_file_id", originalFile.get().getId(), + "owner_user_id", String.valueOf(originalFile.get().getUserId())); + + return ResponseEntity.ok().body(new BaseResponse( + 20004, + MessageCode.DUPLICATE_FILE_UPLOAD.getCode(), + "Failed to upload chunk, duplicate", + duplicateInfo)); + } + + return null; + } + private ResponseEntity buildSuccessResponse(String uploadId, Integer chunkNumber, MultipartFile chunk) { ChunkUploadResponse responseBody = ChunkUploadResponse.builder() .uploadId(uploadId) diff --git a/src/main/java/ru/soune/nocopy/dto/MessageCode.java b/src/main/java/ru/soune/nocopy/dto/MessageCode.java index 33faabc..3332a30 100644 --- a/src/main/java/ru/soune/nocopy/dto/MessageCode.java +++ b/src/main/java/ru/soune/nocopy/dto/MessageCode.java @@ -21,7 +21,8 @@ public enum MessageCode { AUTH_EMAIL_OR_TOKEN_NOT_FOUND(4, "Email or Token not found "), AUTH_TOKEN_MISMATCH(4, "Token mismatch"), FILE_NOT_FOUND(4, "File not found"), - AUTH_PASSWORD_NOT_MATCHES(2, "Password does not match"); + AUTH_PASSWORD_NOT_MATCHES(2, "Password does not match"), + SIMILAR_FILES_FOUND(0, "Similar files found"); private final Integer code; diff --git a/src/main/java/ru/soune/nocopy/dto/file/SimilarityFilter.java b/src/main/java/ru/soune/nocopy/dto/file/SimilarityFilter.java new file mode 100644 index 0000000..48a4433 --- /dev/null +++ b/src/main/java/ru/soune/nocopy/dto/file/SimilarityFilter.java @@ -0,0 +1,18 @@ +package ru.soune.nocopy.dto.file; + +import lombok.AllArgsConstructor; +import lombok.Builder; +import lombok.Data; +import lombok.NoArgsConstructor; + +import java.util.List; + +@Data +@Builder +@NoArgsConstructor +@AllArgsConstructor +public class SimilarityFilter { + private List similarityLevels; + private Integer page; + private Integer size; +} diff --git a/src/main/java/ru/soune/nocopy/entity/file/FileEntity.java b/src/main/java/ru/soune/nocopy/entity/file/FileEntity.java index ffdd985..e07b591 100644 --- a/src/main/java/ru/soune/nocopy/entity/file/FileEntity.java +++ b/src/main/java/ru/soune/nocopy/entity/file/FileEntity.java @@ -1,5 +1,6 @@ package ru.soune.nocopy.entity.file; +import com.fasterxml.jackson.annotation.JsonIgnore; import jakarta.persistence.*; import lombok.*; import org.springframework.data.annotation.CreatedDate; @@ -60,6 +61,10 @@ public class FileEntity { @Column(name = "updated_at") private LocalDateTime updatedAt; + @OneToOne(mappedBy = "file", cascade = CascadeType.ALL, orphanRemoval = true) + @JsonIgnore + private ImageHashEntity imageHash; + @PrePersist public void prePersist() { if (this.status == null) { diff --git a/src/main/java/ru/soune/nocopy/repository/FileEntityRepository.java b/src/main/java/ru/soune/nocopy/repository/FileEntityRepository.java index 3214e9d..e35ef2f 100644 --- a/src/main/java/ru/soune/nocopy/repository/FileEntityRepository.java +++ b/src/main/java/ru/soune/nocopy/repository/FileEntityRepository.java @@ -30,5 +30,8 @@ public interface FileEntityRepository extends JpaRepository @Query("SELECT f FROM FileEntity f WHERE f.userId = :userId AND f.originalFileName LIKE %:keyword%") List searchByFileName(@Param("userId") Long userId, @Param("keyword") String keyword); + @Query("SELECT f FROM FileEntity f WHERE f.status = :status") + List searchFileEntityByStatus(@Param("status") FileStatus status); + long countByUserId(Long userId); } diff --git a/src/main/java/ru/soune/nocopy/service/FileSimilarityService.java b/src/main/java/ru/soune/nocopy/service/FileSimilarityService.java index 2c02234..f8b7407 100644 --- a/src/main/java/ru/soune/nocopy/service/FileSimilarityService.java +++ b/src/main/java/ru/soune/nocopy/service/FileSimilarityService.java @@ -1,19 +1,25 @@ package ru.soune.nocopy.service; import lombok.RequiredArgsConstructor; +import org.springframework.data.domain.Page; +import org.springframework.data.domain.PageImpl; +import org.springframework.data.domain.Pageable; import org.springframework.stereotype.Service; import ru.soune.nocopy.dto.file.SimilarFileResponse; +import ru.soune.nocopy.dto.file.SimilarityFilter; import ru.soune.nocopy.repository.ImageHashRepository; import ru.soune.nocopy.repository.ImageSimilarityRepository; import ru.soune.nocopy.repository.SimilarImageProjection; +import java.util.Comparator; import java.util.List; +import java.util.stream.Collectors; @Service @RequiredArgsConstructor public class FileSimilarityService { - private final ImageSimilarityRepository repository; + private final ImageHashRepository hashRepository; public List findSimilarFiles(String fileId) { @@ -55,6 +61,62 @@ public class FileSimilarityService { .toList(); } + public Page findSimilarFiles(String fileId, SimilarityFilter filter, Pageable pageable) { + var imageHashEntity = hashRepository.findById(fileId) + .orElseThrow(() -> new RuntimeException("Hash not found")); + + Integer hash64Hi = imageHashEntity.getHash64Hi(); + Integer hash64Lo = imageHashEntity.getHash64Lo(); + + List candidates = repository.findCandidates(fileId); + + List similarityLevels = (filter != null && filter.getSimilarityLevels() != null) + ? filter.getSimilarityLevels() + : List.of("DUPLICATE", "SIMILAR", "DIFFERENT"); + + List allResults = candidates.stream() + .map(c -> createSimilarFileResponse(c, hash64Hi, hash64Lo)) + .filter(response -> similarityLevels.contains(response.getSimilarityLevel())) + .sorted(Comparator.comparingInt(SimilarFileResponse::getHammingDistance)) + .collect(Collectors.toList()); + + int total = allResults.size(); + int page = (pageable != null) ? pageable.getPageNumber() : 0; + int size = (pageable != null) ? pageable.getPageSize() : 20; + + int fromIndex = Math.min(page * size, total); + int toIndex = Math.min(fromIndex + size, total); + + List pageContent = allResults.subList(fromIndex, toIndex); + + return new PageImpl<>(pageContent, pageable, total); + } + + private SimilarFileResponse createSimilarFileResponse(SimilarImageProjection similarImageProjection, + Integer hash64Hi, Integer hash64Lo) { + Integer imageProjectionHash64Hi = similarImageProjection.getHash64Hi(); + Integer similarImageProjectionHash64Lo = similarImageProjection.getHash64Lo(); + + int hamming = hamming64(hash64Hi, hash64Lo, imageProjectionHash64Hi, similarImageProjectionHash64Lo); + + String level; + if (hamming <= 5) { + level = "DUPLICATE"; + } else if (hamming <= 12) { + level = "SIMILAR"; + } else { + level = "DIFFERENT"; + } + + return SimilarFileResponse.builder() + .fileId(similarImageProjection.getSimilarFileId()) + .originalFileName(similarImageProjection.getOriginalFileName()) + .fileSize(similarImageProjection.getFileSize()) + .hammingDistance(hamming) + .similarityLevel(level) + .build(); + } + private int hamming64(int aHi, int aLo, int bHi, int bLo) { return Integer.bitCount(aHi ^ bHi) + Integer.bitCount(aLo ^ bLo); diff --git a/src/main/java/ru/soune/nocopy/service/file/NotExistFilesCleanupService.java b/src/main/java/ru/soune/nocopy/service/file/NotExistFilesCleanupService.java new file mode 100644 index 0000000..1be7b9e --- /dev/null +++ b/src/main/java/ru/soune/nocopy/service/file/NotExistFilesCleanupService.java @@ -0,0 +1,35 @@ +package ru.soune.nocopy.service.file; + +import lombok.RequiredArgsConstructor; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.scheduling.annotation.Scheduled; +import org.springframework.stereotype.Service; +import org.springframework.transaction.annotation.Transactional; +import ru.soune.nocopy.entity.file.FileEntity; +import ru.soune.nocopy.entity.file.FileStatus; +import ru.soune.nocopy.repository.FileEntityRepository; + +import java.nio.file.Files; +import java.nio.file.Paths; +import java.util.List; + +@Service +@RequiredArgsConstructor +public class NotExistFilesCleanupService { + + @Autowired + private FileEntityRepository fileEntityRepository; + + @Transactional + @Scheduled(fixedDelay = 30000) + public void cleanUpNotExistFiles() { + List fileEntities = fileEntityRepository.searchFileEntityByStatus(FileStatus.DELETED); + + for (FileEntity fileEntity : fileEntities) { + boolean exists = Files.exists(Paths.get(fileEntity.getFilePath())); + if (!exists) { + fileEntityRepository.delete(fileEntity); + } + } + } +}