diff --git a/src/main/java/ru/soune/nocopy/controller/ApiController.java b/src/main/java/ru/soune/nocopy/controller/ApiController.java index 5c408fc..bae2817 100644 --- a/src/main/java/ru/soune/nocopy/controller/ApiController.java +++ b/src/main/java/ru/soune/nocopy/controller/ApiController.java @@ -110,17 +110,13 @@ public class ApiController { return buildErrorResponse(uploadId, chunkNumber, "Valid chunk number is required"); } - UploadProgressResponse uploadProgressResponse = fileUploadService.uploadChunk(uploadId, chunkNumber, chunk); - - if (uploadProgressResponse.getStatus().equals(UploadStatus.COMPLETED)) { - ResponseEntity duplicateCheckResult = checkForDuplicates(uploadProgressResponse.getUploadId()); - if (duplicateCheckResult != null) { - return duplicateCheckResult; - } - } + fileUploadService.uploadChunk(uploadId, chunkNumber, chunk); return buildSuccessResponse(uploadId, chunkNumber, chunk); - + } catch (DuplicateImageException e) { + return ResponseEntity.ok().body(new BaseResponse(20004, MessageCode.DUPLICATE_FILE_UPLOAD.getCode(), + MessageCode.DUPLICATE_FILE_UPLOAD.getDescription(), + Map.of("duplicateOwnerId", e.userId(), "duplicateFileId", e.duplicateFileId()))); } catch (Exception e) { log.error("Error uploading chunk", e); return buildErrorResponse(uploadId, chunkNumber, "Failed to upload chunk: " + e.getMessage()); diff --git a/src/main/java/ru/soune/nocopy/exception/DuplicateImageException.java b/src/main/java/ru/soune/nocopy/exception/DuplicateImageException.java index a7593b8..3ac1325 100644 --- a/src/main/java/ru/soune/nocopy/exception/DuplicateImageException.java +++ b/src/main/java/ru/soune/nocopy/exception/DuplicateImageException.java @@ -1,7 +1,20 @@ package ru.soune.nocopy.exception; public class DuplicateImageException extends RuntimeException { - public DuplicateImageException(String message) { + private final String duplicateFileId; + private final Long userId; + + public DuplicateImageException(String message, String duplicateFileId, Long userId) { super(message); + this.duplicateFileId = duplicateFileId; + this.userId = userId; + } + + public String duplicateFileId() { + return duplicateFileId; + } + + public Long userId() { + return userId; } } diff --git a/src/main/java/ru/soune/nocopy/handler/FileUploadHandler.java b/src/main/java/ru/soune/nocopy/handler/FileUploadHandler.java index 8240403..2c75fb0 100644 --- a/src/main/java/ru/soune/nocopy/handler/FileUploadHandler.java +++ b/src/main/java/ru/soune/nocopy/handler/FileUploadHandler.java @@ -175,34 +175,6 @@ public class FileUploadHandler implements RequestHandler { } } - private BaseResponse handleRetryUpload(BaseRequest request, FileUploadRequest fileRequest) { - try { - fileUploadService.retryFailedUpload(fileRequest.getUploadId()); - ru.soune.nocopy.dto.file.UploadProgressResponse progress = - fileUploadService.getUploadProgress(fileRequest.getUploadId()); - - RetryUploadResponse response = RetryUploadResponse.builder() - .uploadId(progress.getUploadId()) - .message("Upload retry initiated") - .status(progress.getStatus().toString()) - .build(); - - return new BaseResponse(request.getMsgId(), MessageCode.SUCCESS.getCode(), "Upload retry initiated", - response); - - } catch (UploadSessionNotFoundException e) { - return new BaseResponse(request.getMsgId(), MessageCode.FILE_NOT_FOUND.getCode(), - "Upload session not found", null); - } catch (FileUploadException e) { - return new BaseResponse(request.getMsgId(), MessageCode.FILE_UPLOAD_ERROR.getCode(), e.getMessage(), - null); - } catch (Exception e) { - log.error("Error retrying upload", e); - return new BaseResponse(request.getMsgId(), MessageCode.FILE_UPLOAD_ERROR.getCode(), - "Failed to retry upload", null); - } - } - private BaseResponse handleGetChunkStatus(BaseRequest request, FileUploadRequest fileRequest) { try { FileUploadSession session = fileUploadSessionRepository.findById(fileRequest.getUploadId()) diff --git a/src/main/java/ru/soune/nocopy/repository/ImageSimilarityRepository.java b/src/main/java/ru/soune/nocopy/repository/ImageSimilarityRepository.java index eeabe3a..a6ef973 100644 --- a/src/main/java/ru/soune/nocopy/repository/ImageSimilarityRepository.java +++ b/src/main/java/ru/soune/nocopy/repository/ImageSimilarityRepository.java @@ -13,7 +13,8 @@ public interface ImageSimilarityRepository extends JpaRepository { @Query(value = """ - SELECT + + SELECT f.id AS similarFileId, f.original_file_name AS originalFileName, f.file_size AS fileSize, @@ -30,6 +31,24 @@ public interface ImageSimilarityRepository List findCandidates( @Param("fileId") String fileId ); + + @Query(value = """ + SELECT + h.hash64_hi AS hash64Hi, + h.hash64_lo AS hash64Lo, + h.file_id AS fileId, + f.user_id AS userId, + f.original_file_name AS originalFileName, + f.file_size AS fileSize, + f.stored_file_name AS similarFileId + FROM image_hashes h + JOIN file_entities f ON f.id = h.file_id + WHERE h.hash64_hi = :hash64Hi + AND h.hash64_lo = :hash64Lo + """, + nativeQuery = true) + List findExactDuplicates( + @Param("hash64Hi") Integer hash64_hi, + @Param("hash64Lo") Integer hash64_lo); } - diff --git a/src/main/java/ru/soune/nocopy/repository/SimilarImageProjection.java b/src/main/java/ru/soune/nocopy/repository/SimilarImageProjection.java index 7263394..ed2fdb4 100644 --- a/src/main/java/ru/soune/nocopy/repository/SimilarImageProjection.java +++ b/src/main/java/ru/soune/nocopy/repository/SimilarImageProjection.java @@ -1,14 +1,10 @@ package ru.soune.nocopy.repository; public interface SimilarImageProjection { - - String getSimilarFileId(); - - String getOriginalFileName(); - - Long getFileSize(); - Integer getHash64Hi(); - Integer getHash64Lo(); + String getFileId(); + Long getUserId(); + String getOriginalFileName(); + Long getFileSize(); } \ No newline at end of file diff --git a/src/main/java/ru/soune/nocopy/service/FileSimilarityService.java b/src/main/java/ru/soune/nocopy/service/FileSimilarityService.java index f8b7407..dea9334 100644 --- a/src/main/java/ru/soune/nocopy/service/FileSimilarityService.java +++ b/src/main/java/ru/soune/nocopy/service/FileSimilarityService.java @@ -11,6 +11,7 @@ import ru.soune.nocopy.repository.ImageHashRepository; import ru.soune.nocopy.repository.ImageSimilarityRepository; import ru.soune.nocopy.repository.SimilarImageProjection; +import java.util.ArrayList; import java.util.Comparator; import java.util.List; import java.util.stream.Collectors; @@ -49,7 +50,7 @@ public class FileSimilarityService { } return SimilarFileResponse.builder() - .fileId(c.getSimilarFileId()) + .fileId(c.getFileId()) .originalFileName(c.getOriginalFileName()) .fileSize(c.getFileSize()) .hammingDistance(hamming) @@ -61,6 +62,16 @@ public class FileSimilarityService { .toList(); } + public List findDuplicatedByHash(Integer hash64Hi, Integer hash64Lo) { + List duplicates = repository.findExactDuplicates(hash64Hi,hash64Lo); + + if (duplicates.isEmpty()) { + return new ArrayList<>(); + } + + return duplicates; + } + public Page findSimilarFiles(String fileId, SimilarityFilter filter, Pageable pageable) { var imageHashEntity = hashRepository.findById(fileId) .orElseThrow(() -> new RuntimeException("Hash not found")); @@ -109,7 +120,7 @@ public class FileSimilarityService { } return SimilarFileResponse.builder() - .fileId(similarImageProjection.getSimilarFileId()) + .fileId(similarImageProjection.getFileId()) .originalFileName(similarImageProjection.getOriginalFileName()) .fileSize(similarImageProjection.getFileSize()) .hammingDistance(hamming) diff --git a/src/main/java/ru/soune/nocopy/service/ImageHashService.java b/src/main/java/ru/soune/nocopy/service/ImageHashService.java index 992040b..b7b6882 100644 --- a/src/main/java/ru/soune/nocopy/service/ImageHashService.java +++ b/src/main/java/ru/soune/nocopy/service/ImageHashService.java @@ -16,6 +16,7 @@ import java.nio.file.Files; import java.nio.file.Path; import java.time.LocalDateTime; import java.util.Arrays; +import java.util.Map; @Service @RequiredArgsConstructor @@ -24,25 +25,24 @@ public class ImageHashService { private final ImageHashRepository repository; - public void create(FileEntity file, Path imagePath) { - try { - long hash64 = computePhash64(imagePath); + public Map calculateHash(Path imagePath) throws IOException { + long hash64 = computePhash64(imagePath); - int hi = high32(hash64); - int lo = low32(hash64); + return Map.of( + "hi", high32(hash64), + "low", low32(hash64)); + } - ImageHashEntity entity = ImageHashEntity.builder() - .file(file) - .hash64Hi(hi) - .hash64Lo(lo) - .hashAlgorithm("PHASH64") - .createdAt(LocalDateTime.now()) - .build(); + public void create(FileEntity file, Map stringIntegerMap) { + ImageHashEntity entity = ImageHashEntity.builder() + .file(file) + .hash64Hi(stringIntegerMap.get("hi")) + .hash64Lo(stringIntegerMap.get("low")) + .hashAlgorithm("PHASH64") + .createdAt(LocalDateTime.now()) + .build(); - repository.save(entity); - } catch (IOException e) { - throw new RuntimeException("Failed to compute image hash", e); - } + repository.save(entity); } private long computePhash64(Path path) throws IOException { diff --git a/src/main/java/ru/soune/nocopy/service/file/FileEntityService.java b/src/main/java/ru/soune/nocopy/service/file/FileEntityService.java index 0a17c22..0cdf62c 100644 --- a/src/main/java/ru/soune/nocopy/service/file/FileEntityService.java +++ b/src/main/java/ru/soune/nocopy/service/file/FileEntityService.java @@ -9,8 +9,10 @@ import ru.soune.nocopy.dto.file.FileResponse; import ru.soune.nocopy.entity.file.FileEntity; import ru.soune.nocopy.entity.file.FileStatus; import ru.soune.nocopy.entity.file.FileUploadSession; +import ru.soune.nocopy.exception.DuplicateImageException; import ru.soune.nocopy.exception.FileEntityNotFoundException; import ru.soune.nocopy.repository.FileEntityRepository; +import ru.soune.nocopy.repository.SimilarImageProjection; import ru.soune.nocopy.service.FileSimilarityService; import ru.soune.nocopy.service.ImageHashService; @@ -20,6 +22,7 @@ import java.nio.file.Path; import java.nio.file.Paths; import java.time.LocalDateTime; import java.util.List; +import java.util.Map; import java.util.stream.Collectors; @Slf4j @@ -33,7 +36,7 @@ public class FileEntityService { private final FileSimilarityService fileSimilarityService; - @Transactional + @Transactional(noRollbackFor = DuplicateImageException.class) public FileEntity createFromUploadSession(FileUploadSession session, String checksum) { log.info("Creating FileEntity for upload session: {}", session.getUploadId()); @@ -44,6 +47,21 @@ public class FileEntityService { throw new IOException("File not found on disk: " + filePath); } + Map imageHash = Map.of(); + + if (session.getFileType().startsWith("image")) { + imageHash = imageHashService.calculateHash(filePath); + List duplicatedByHash = fileSimilarityService.findDuplicatedByHash( + imageHash.get("hi"), imageHash.get("low")); + + if (!duplicatedByHash.isEmpty()) { + SimilarImageProjection similarImageProjection = duplicatedByHash.get(0); + + throw new DuplicateImageException("Duplicate", similarImageProjection.getFileId(), + similarImageProjection.getUserId()); + } + } + long fileSize = Files.size(filePath); String originalName = session.getFileName(); String storedName = filePath.getFileName().toString(); @@ -63,8 +81,8 @@ public class FileEntityService { FileEntity saved = fileEntityRepository.save(fileEntity); - if (saved.getMimeType().startsWith("image")) { - imageHashService.create(saved, Path.of(saved.getFilePath())); + if (!imageHash.isEmpty()) { + imageHashService.create(saved, imageHash); } return saved; diff --git a/src/main/java/ru/soune/nocopy/service/file/FileUploadService.java b/src/main/java/ru/soune/nocopy/service/file/FileUploadService.java index b27dede..714ef21 100644 --- a/src/main/java/ru/soune/nocopy/service/file/FileUploadService.java +++ b/src/main/java/ru/soune/nocopy/service/file/FileUploadService.java @@ -17,7 +17,5 @@ public interface FileUploadService { void handleExpiredSession(FileUploadSession session); - public void retryFailedUpload(String uploadId); - public void cancelUpload(String uploadId); } diff --git a/src/main/java/ru/soune/nocopy/service/file/FileUploadServiceImpl.java b/src/main/java/ru/soune/nocopy/service/file/FileUploadServiceImpl.java index 81df025..587cdc1 100644 --- a/src/main/java/ru/soune/nocopy/service/file/FileUploadServiceImpl.java +++ b/src/main/java/ru/soune/nocopy/service/file/FileUploadServiceImpl.java @@ -10,12 +10,19 @@ import org.springframework.stereotype.Service; import org.springframework.transaction.annotation.Transactional; import org.springframework.web.multipart.MultipartFile; import ru.soune.nocopy.dto.file.UploadProgressResponse; +import ru.soune.nocopy.entity.file.FileEntity; +import ru.soune.nocopy.entity.file.FileStatus; import ru.soune.nocopy.entity.file.FileUploadSession; import ru.soune.nocopy.entity.file.UploadStatus; import ru.soune.nocopy.exception.ChunkSizeExceededException; +import ru.soune.nocopy.exception.DuplicateImageException; import ru.soune.nocopy.exception.FileUploadException; import ru.soune.nocopy.exception.UploadSessionNotFoundException; +import ru.soune.nocopy.repository.FileEntityRepository; import ru.soune.nocopy.repository.FileUploadSessionRepository; +import ru.soune.nocopy.repository.SimilarImageProjection; +import ru.soune.nocopy.service.FileSimilarityService; +import ru.soune.nocopy.service.ImageHashService; import java.io.*; import java.nio.file.*; @@ -23,10 +30,7 @@ import java.nio.file.attribute.PosixFilePermission; import java.security.MessageDigest; import java.security.NoSuchAlgorithmException; import java.time.LocalDateTime; -import java.util.Comparator; -import java.util.HashSet; -import java.util.Set; -import java.util.UUID; +import java.util.*; import java.util.concurrent.CompletableFuture; import java.util.stream.Stream; @@ -57,6 +61,15 @@ public class FileUploadServiceImpl implements FileUploadService { private Path storageRoot; + @Autowired + private ImageHashService imageHashService; + + @Autowired + private FileSimilarityService fileSimilarityService; + + @Autowired + private FileEntityRepository fileEntityRepository; + @Autowired private FileEntityService fileEntityService; @@ -129,10 +142,6 @@ public class FileUploadServiceImpl implements FileUploadService { @Transactional public UploadProgressResponse uploadChunk(String uploadId, Integer chunkNumber, MultipartFile chunkFile) { - - log.info("Processing chunk {} for session {}, size: {} bytes", - chunkNumber, uploadId, chunkFile.getSize()); - FileUploadSession session = sessionRepository.findById(uploadId) .orElseThrow(() -> new UploadSessionNotFoundException(uploadId)); @@ -173,12 +182,13 @@ public class FileUploadServiceImpl implements FileUploadService { @Async("fileUploadTaskExecutor") @Transactional - public void assembleFileAsync(FileUploadSession session) { + public void assembleFileAsync(FileUploadSession session) throws DuplicateImageException { try { assembleFile(session); log.info("File assembly completed successfully for session: {}", session.getUploadId()); - + } catch (DuplicateImageException e) { + throw new DuplicateImageException("DUBL", e.duplicateFileId(), e.userId()); } catch (Exception e) { log.error("Failed to assemble file for session {}: {}", session.getUploadId(), e.getMessage(), e); @@ -205,35 +215,6 @@ public class FileUploadServiceImpl implements FileUploadService { log.info("Upload cancelled: {}", uploadId); } - @Override - @Transactional - public void retryFailedUpload(String uploadId) { - FileUploadSession session = sessionRepository.findById(uploadId) - .orElseThrow(() -> new UploadSessionNotFoundException(uploadId)); - - if (session.getStatus() != UploadStatus.FAILED) { - throw new FileUploadException("Only failed uploads can be retried"); - } - - if (session.getRetryCount() >= maxRetryAttempts) { - throw new FileUploadException("Max retry attempts exceeded"); - } - - session.setStatus(UploadStatus.UPLOADING); - session.setRetryCount(session.getRetryCount() + 1); - session.setLastError(null); - - sessionRepository.save(session); - - if (session.getChunksUploaded().equals(session.getTotalChunks())) { - log.info("Retrying file assembly for session: {}", uploadId); - assembleFileAsync(session); - } - - log.info("Upload retry initiated for session: {} (attempt {})", - uploadId, session.getRetryCount()); - } - @Override public UploadProgressResponse getUploadProgress(String uploadId) { FileUploadSession session = sessionRepository.findById(uploadId) @@ -271,9 +252,42 @@ public class FileUploadServiceImpl implements FileUploadService { } } - private UploadProgressResponse processChunk(FileUploadSession session, - Integer chunkNumber, - MultipartFile chunkFile) { + @Async("fileUploadTaskExecutor") + @Transactional + public void completeFileProcessingAsync(FileUploadSession session) { + try { + Path filePath = Paths.get(session.getFilePath()); + String checksum = calculateChecksum(filePath); + + FileEntity fileEntity = FileEntity.builder() + .userId(session.getUserId()) + .originalFileName(session.getFileName()) + .storedFileName(filePath.getFileName().toString()) + .filePath(session.getFilePath()) + .fileSize(session.getFileSize()) + .mimeType(session.getFileType()) + .fileExtension(session.getExtension()) + .checksum(checksum) + .uploadSessionId(session.getUploadId()) + .status(FileStatus.ACTIVE) + .build(); + + FileEntity saved = fileEntityRepository.save(fileEntity); + + Map hash = imageHashService.calculateHash(filePath); + imageHashService.create(saved, hash); + + cleanupSessionFiles(session); + + log.info("File processing completed for session: {}", session.getUploadId()); + + } catch (Exception e) { + log.error("Failed to complete file processing for session {}: {}", + session.getUploadId(), e.getMessage(), e); + } + } + + private UploadProgressResponse processChunk(FileUploadSession session, Integer chunkNumber, MultipartFile chunkFile) { String chunkPath = null; try { @@ -291,17 +305,42 @@ public class FileUploadServiceImpl implements FileUploadService { log.debug("Chunk {} saved successfully. Uploaded: {}/{}", chunkNumber, session.getChunksUploaded(), session.getTotalChunks()); - if (session.getChunksUploaded().equals(session.getTotalChunks())) { + boolean isLastChunk = session.getChunksUploaded().equals(session.getTotalChunks()); + + if (isLastChunk) { log.info("All chunks uploaded for session {}. Starting assembly...", session.getUploadId()); - sessionRepository.save(session); - assembleFileAsync(session); + + String finalFilePath = assembleFileSynchronously(session); + + if (session.getFileType().startsWith("image")) { + checkForDuplicatesSynchronously(finalFilePath); + } + + session.setStatus(UploadStatus.COMPLETED); + session.setFilePath(finalFilePath); + + completeFileProcessingAsync(session); + } else { sessionRepository.save(session); } return UploadProgressResponse.fromSession(session); + } catch (DuplicateImageException e) { + if (chunkPath != null) { + cleanupFailedChunk(chunkPath); + } + + log.warn("Duplicate image found for session {}: {}", + session.getUploadId(), e.getMessage()); + + session.setStatus(UploadStatus.FAILED); + session.setLastError(e.getMessage()); + sessionRepository.save(session); + + throw e; } catch (Exception e) { if (chunkPath != null) { cleanupFailedChunk(chunkPath); @@ -318,6 +357,29 @@ public class FileUploadServiceImpl implements FileUploadService { } } + private String assembleFileSynchronously(FileUploadSession session) throws IOException { + Path finalFilePath = prepareFinalFile(session); + + validateAllChunksExist(session); + mergeChunksToFile(session, finalFilePath); + validateFinalFile(session, finalFilePath); + + return finalFilePath.toString(); + } + + private void checkForDuplicatesSynchronously(String filePath) + throws IOException { + Path path = Paths.get(filePath); + Map hash = imageHashService.calculateHash(path); + List duplicates = fileSimilarityService.findDuplicatedByHash( + hash.get("hi"), hash.get("low")); + + if (!duplicates.isEmpty()) { + throw new DuplicateImageException("Duplicate", duplicates.get(0).getFileId(), + duplicates.get(0).getUserId()); + } + } + private UploadProgressResponse handleExistingChunk(FileUploadSession session, Integer chunkNumber, MultipartFile chunkFile) throws IOException { @@ -325,25 +387,24 @@ public class FileUploadServiceImpl implements FileUploadService { Path chunkPath = Paths.get(existingPath); if (!Files.exists(chunkPath)) { - log.warn("Chunk file missing, re-uploading: {}", chunkPath); session.getChunkPaths().remove(chunkNumber); session.setChunksUploaded(session.getChunksUploaded() - 1); sessionRepository.save(session); + return processChunk(session, chunkNumber, chunkFile); } long existingSize = Files.size(chunkPath); + if (existingSize != chunkFile.getSize()) { - log.warn("Chunk size mismatch, re-uploading: {} != {}", - existingSize, chunkFile.getSize()); Files.deleteIfExists(chunkPath); session.getChunkPaths().remove(chunkNumber); session.setChunksUploaded(session.getChunksUploaded() - 1); sessionRepository.save(session); + return processChunk(session, chunkNumber, chunkFile); } - log.debug("Chunk {} already uploaded and valid", chunkNumber); return UploadProgressResponse.fromSession(session); } @@ -428,6 +489,8 @@ public class FileUploadServiceImpl implements FileUploadService { fileEntityService.createFromUploadSession(session, checksum); log.info("FileEntity successfully created for session: {}", session.getUploadId()); + } catch (DuplicateImageException e) { + throw new DuplicateImageException("Duplicate", e.duplicateFileId(), e.userId()); } catch (Exception e) { log.error("Failed to create FileEntity for session {} (file uploaded, but metadata not saved): {}", session.getUploadId(), e.getMessage());