NCBACK-34 add logic for clear does not exist files,add dto for similar file search controller

src/main/java/ru/soune/nocopy/controller/ApiController.java

@@ -1,12 +1,12 @@
 package ru.soune.nocopy.controller;
 
-import jakarta.persistence.EntityManager;
-import jakarta.persistence.PersistenceContext;
 import lombok.RequiredArgsConstructor;
 import lombok.extern.slf4j.Slf4j;
-import org.springframework.beans.factory.annotation.Autowired;
 import org.springframework.core.io.Resource;
 import org.springframework.core.io.UrlResource;
+import org.springframework.data.domain.Page;
+import org.springframework.data.domain.Pageable;
+import org.springframework.data.web.PageableDefault;
 import org.springframework.http.HttpHeaders;
 import org.springframework.http.MediaType;
 import org.springframework.http.ResponseEntity;
@@ -28,7 +28,6 @@ import ru.soune.nocopy.handler.*;
 import ru.soune.nocopy.repository.AuthTokenRepository;
 import ru.soune.nocopy.repository.FileEntityRepository;
 import ru.soune.nocopy.service.FileSimilarityService;
-import ru.soune.nocopy.service.ImageHashService;
 import ru.soune.nocopy.service.file.FileEntityService;
 import ru.soune.nocopy.service.file.FileUploadService;
 
@@ -128,51 +127,39 @@ public class ApiController {
         }
     }
 
-    private ResponseEntity<BaseResponse> checkForDuplicates(String uploadId) throws IOException {
+    @GetMapping("/v{version}/files/{fileId}/similar")
-        Optional<FileEntity> uploadedFile = fileEntityRepository.findByUploadSessionId(uploadId);
+    public ResponseEntity<BaseResponse> findSimilarFiles(
+            @PathVariable("version") int version,
+            @PathVariable String fileId,
+            @RequestParam(required = false) List<String> similarityLevels,
+            @PageableDefault(size = 20, sort = "hammingDistance") Pageable pageable) {
+        SimilarityFilter filter = SimilarityFilter.builder()
+                .similarityLevels(similarityLevels)
+                .build();
+        Page<SimilarFileResponse> similarFiles = fileSimilarityService.findSimilarFiles(fileId, filter, pageable);
 
-        if (uploadedFile.isEmpty() || !uploadedFile.get().getMimeType().equals("image")) {
+        String messageDesc;
-            return null;
+        MessageCode success;
+
+        if (similarFiles.isEmpty()) {
+            messageDesc = MessageCode.FILE_NOT_FOUND.getDescription();
+            success = MessageCode.FILE_NOT_FOUND;
+        } else {
+            messageDesc = MessageCode.SIMILAR_FILES_FOUND.getDescription();
+            success = MessageCode.SIMILAR_FILES_FOUND;
         }
 
-        FileEntity fileEntity = uploadedFile.get();
+        Map<String, Object> responseData = new HashMap<>();
-        List<SimilarFileResponse> similarFiles = fileSimilarityService.findSimilarFiles(fileEntity.getId());
+        responseData.put("content", similarFiles.getContent());
+        responseData.put("page", similarFiles.getNumber());
+        responseData.put("size", similarFiles.getSize());
+        responseData.put("totalElements", similarFiles.getTotalElements());
+        responseData.put("totalPages", similarFiles.getTotalPages());
+        responseData.put("hasNext", similarFiles.hasNext());
+        responseData.put("hasPrevious", similarFiles.hasPrevious());
 
-        if (hasDuplicate(similarFiles)) {
+        return ResponseEntity.ok()
-            return handleDuplicate(fileEntity, similarFiles);
+                .body(new BaseResponse(20004, success.getCode(), messageDesc, responseData));
-        }
-
-        return null;
-    }
-
-    private boolean hasDuplicate(List<SimilarFileResponse> similarFiles) {
-        return similarFiles.stream().anyMatch(f -> f.getHammingDistance() <= 5);
-    }
-
-    private ResponseEntity<BaseResponse> handleDuplicate(FileEntity fileEntity, List<SimilarFileResponse> similarFiles)
-            throws IOException {
-        fileEntityService.deleteFromDisk(fileEntity);
-
-        Optional<FileEntity> originalFile = fileEntityRepository.findById(similarFiles.get(0).getFileId());
-
-        if (originalFile.isPresent()) {
-            Map<String, String> duplicateInfo = Map.of(
-                    "duplicate_file_id", originalFile.get().getId(),
-                    "owner_user_id", String.valueOf(originalFile.get().getUserId()));
-
-            return ResponseEntity.ok().body(new BaseResponse(
-                    20004,
-                    MessageCode.DUPLICATE_FILE_UPLOAD.getCode(),
-                    "Failed to upload chunk, duplicate",
-                    duplicateInfo));
-        }
-
-        return null;
-    }
-
-    @GetMapping("/files/{fileId}/similar")
-    public List<SimilarFileResponse> getSimilarFiles(@PathVariable String fileId) {
-        return fileSimilarityService.findSimilarFiles(fileId);
     }
 
     @GetMapping("/v{version}/files/progress/{uploadId}")
@@ -364,6 +351,50 @@ public class ApiController {
         }
     }
 
+    private ResponseEntity<BaseResponse> checkForDuplicates(String uploadId) throws IOException {
+        Optional<FileEntity> uploadedFile = fileEntityRepository.findByUploadSessionId(uploadId);
+
+        if (uploadedFile.isEmpty() || !uploadedFile.get().getMimeType().equals("image")) {
+            return null;
+        }
+
+        FileEntity fileEntity = uploadedFile.get();
+        List<SimilarFileResponse> similarFiles = fileSimilarityService.findSimilarFiles(fileEntity.getId());
+
+        if (hasDuplicate(similarFiles)) {
+            return handleDuplicate(fileEntity, similarFiles);
+        }
+
+        return null;
+    }
+
+    private boolean hasDuplicate(List<SimilarFileResponse> similarFiles) {
+        return similarFiles.stream().anyMatch(f -> f.getHammingDistance() <= 5);
+    }
+
+    private ResponseEntity<BaseResponse> handleDuplicate(FileEntity fileEntity, List<SimilarFileResponse> similarFiles)
+            throws IOException {
+        fileEntityService.deleteFromDisk(fileEntity);
+
+        fileEntityService.markAsDeleted(fileEntity);
+
+        Optional<FileEntity> originalFile = fileEntityRepository.findById(similarFiles.get(0).getFileId());
+
+        if (originalFile.isPresent()) {
+            Map<String, String> duplicateInfo = Map.of(
+                    "duplicate_file_id", originalFile.get().getId(),
+                    "owner_user_id", String.valueOf(originalFile.get().getUserId()));
+
+            return ResponseEntity.ok().body(new BaseResponse(
+                    20004,
+                    MessageCode.DUPLICATE_FILE_UPLOAD.getCode(),
+                    "Failed to upload chunk, duplicate",
+                    duplicateInfo));
+        }
+
+        return null;
+    }
+
     private ResponseEntity<BaseResponse> buildSuccessResponse(String uploadId, Integer chunkNumber, MultipartFile chunk) {
         ChunkUploadResponse responseBody = ChunkUploadResponse.builder()
                 .uploadId(uploadId)

src/main/java/ru/soune/nocopy/dto/MessageCode.java

@@ -21,7 +21,8 @@ public enum MessageCode {
     AUTH_EMAIL_OR_TOKEN_NOT_FOUND(4, "Email or Token not found "),
     AUTH_TOKEN_MISMATCH(4, "Token mismatch"),
     FILE_NOT_FOUND(4, "File not found"),
-    AUTH_PASSWORD_NOT_MATCHES(2, "Password does not match");
+    AUTH_PASSWORD_NOT_MATCHES(2, "Password does not match"),
+    SIMILAR_FILES_FOUND(0, "Similar files found");
 
     private final Integer code;
 

src/main/java/ru/soune/nocopy/dto/file/SimilarityFilter.java

@@ -0,0 +1,18 @@
+package ru.soune.nocopy.dto.file;
+
+import lombok.AllArgsConstructor;
+import lombok.Builder;
+import lombok.Data;
+import lombok.NoArgsConstructor;
+
+import java.util.List;
+
+@Data
+@Builder
+@NoArgsConstructor
+@AllArgsConstructor
+public class SimilarityFilter {
+    private List<String> similarityLevels;
+    private Integer page;
+    private Integer size;
+}

src/main/java/ru/soune/nocopy/entity/file/FileEntity.java

@@ -1,5 +1,6 @@
 package ru.soune.nocopy.entity.file;
 
+import com.fasterxml.jackson.annotation.JsonIgnore;
 import jakarta.persistence.*;
 import lombok.*;
 import org.springframework.data.annotation.CreatedDate;
@@ -60,6 +61,10 @@ public class FileEntity {
     @Column(name = "updated_at")
     private LocalDateTime updatedAt;
 
+    @OneToOne(mappedBy = "file", cascade = CascadeType.ALL, orphanRemoval = true)
+    @JsonIgnore
+    private ImageHashEntity imageHash;
+
     @PrePersist
     public void prePersist() {
         if (this.status == null) {

src/main/java/ru/soune/nocopy/repository/FileEntityRepository.java

@@ -30,5 +30,8 @@ public interface FileEntityRepository extends JpaRepository<FileEntity, String>
     @Query("SELECT f FROM FileEntity f WHERE f.userId = :userId AND f.originalFileName LIKE %:keyword%")
     List<FileEntity> searchByFileName(@Param("userId") Long userId, @Param("keyword") String keyword);
 
+    @Query("SELECT f FROM FileEntity f WHERE f.status = :status")
+    List<FileEntity> searchFileEntityByStatus(@Param("status") FileStatus status);
+
     long countByUserId(Long userId);
 }

src/main/java/ru/soune/nocopy/service/FileSimilarityService.java

@@ -1,19 +1,25 @@
 package ru.soune.nocopy.service;
 
 import lombok.RequiredArgsConstructor;
+import org.springframework.data.domain.Page;
+import org.springframework.data.domain.PageImpl;
+import org.springframework.data.domain.Pageable;
 import org.springframework.stereotype.Service;
 import ru.soune.nocopy.dto.file.SimilarFileResponse;
+import ru.soune.nocopy.dto.file.SimilarityFilter;
 import ru.soune.nocopy.repository.ImageHashRepository;
 import ru.soune.nocopy.repository.ImageSimilarityRepository;
 import ru.soune.nocopy.repository.SimilarImageProjection;
 
+import java.util.Comparator;
 import java.util.List;
+import java.util.stream.Collectors;
 
 @Service
 @RequiredArgsConstructor
 public class FileSimilarityService {
-
     private final ImageSimilarityRepository repository;
+
     private final ImageHashRepository hashRepository;
 
     public List<SimilarFileResponse> findSimilarFiles(String fileId) {
@@ -55,6 +61,62 @@ public class FileSimilarityService {
                 .toList();
     }
 
+    public Page<SimilarFileResponse> findSimilarFiles(String fileId, SimilarityFilter filter, Pageable pageable) {
+        var imageHashEntity = hashRepository.findById(fileId)
+                .orElseThrow(() -> new RuntimeException("Hash not found"));
+
+        Integer hash64Hi = imageHashEntity.getHash64Hi();
+        Integer hash64Lo = imageHashEntity.getHash64Lo();
+
+        List<SimilarImageProjection> candidates = repository.findCandidates(fileId);
+
+        List<String> similarityLevels = (filter != null && filter.getSimilarityLevels() != null)
+                ? filter.getSimilarityLevels()
+                : List.of("DUPLICATE", "SIMILAR", "DIFFERENT");
+
+        List<SimilarFileResponse> allResults = candidates.stream()
+                .map(c -> createSimilarFileResponse(c, hash64Hi, hash64Lo))
+                .filter(response -> similarityLevels.contains(response.getSimilarityLevel()))
+                .sorted(Comparator.comparingInt(SimilarFileResponse::getHammingDistance))
+                .collect(Collectors.toList());
+
+        int total = allResults.size();
+        int page = (pageable != null) ? pageable.getPageNumber() : 0;
+        int size = (pageable != null) ? pageable.getPageSize() : 20;
+
+        int fromIndex = Math.min(page * size, total);
+        int toIndex = Math.min(fromIndex + size, total);
+
+        List<SimilarFileResponse> pageContent = allResults.subList(fromIndex, toIndex);
+
+        return new PageImpl<>(pageContent, pageable, total);
+    }
+
+    private SimilarFileResponse createSimilarFileResponse(SimilarImageProjection similarImageProjection,
+                                                          Integer hash64Hi, Integer hash64Lo) {
+        Integer imageProjectionHash64Hi = similarImageProjection.getHash64Hi();
+        Integer similarImageProjectionHash64Lo = similarImageProjection.getHash64Lo();
+
+        int hamming = hamming64(hash64Hi, hash64Lo, imageProjectionHash64Hi, similarImageProjectionHash64Lo);
+
+        String level;
+        if (hamming <= 5) {
+            level = "DUPLICATE";
+        } else if (hamming <= 12) {
+            level = "SIMILAR";
+        } else {
+            level = "DIFFERENT";
+        }
+
+        return SimilarFileResponse.builder()
+                .fileId(similarImageProjection.getSimilarFileId())
+                .originalFileName(similarImageProjection.getOriginalFileName())
+                .fileSize(similarImageProjection.getFileSize())
+                .hammingDistance(hamming)
+                .similarityLevel(level)
+                .build();
+    }
+
     private int hamming64(int aHi, int aLo, int bHi, int bLo) {
         return Integer.bitCount(aHi ^ bHi)
                 + Integer.bitCount(aLo ^ bLo);

src/main/java/ru/soune/nocopy/service/file/NotExistFilesCleanupService.java

@@ -0,0 +1,35 @@
+package ru.soune.nocopy.service.file;
+
+import lombok.RequiredArgsConstructor;
+import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.scheduling.annotation.Scheduled;
+import org.springframework.stereotype.Service;
+import org.springframework.transaction.annotation.Transactional;
+import ru.soune.nocopy.entity.file.FileEntity;
+import ru.soune.nocopy.entity.file.FileStatus;
+import ru.soune.nocopy.repository.FileEntityRepository;
+
+import java.nio.file.Files;
+import java.nio.file.Paths;
+import java.util.List;
+
+@Service
+@RequiredArgsConstructor
+public class NotExistFilesCleanupService {
+
+    @Autowired
+    private FileEntityRepository fileEntityRepository;
+
+    @Transactional
+    @Scheduled(fixedDelay = 30000)
+    public void cleanUpNotExistFiles() {
+        List<FileEntity> fileEntities = fileEntityRepository.searchFileEntityByStatus(FileStatus.DELETED);
+
+        for (FileEntity fileEntity : fileEntities) {
+            boolean exists = Files.exists(Paths.get(fileEntity.getFilePath()));
+            if (!exists) {
+                fileEntityRepository.delete(fileEntity);
+            }
+        }
+    }
+}