diff --git a/build.gradle b/build.gradle index 866c470..09c1b01 100644 --- a/build.gradle +++ b/build.gradle @@ -32,6 +32,8 @@ dependencies { implementation 'commons-validator:commons-validator:1.7' implementation group: 'org.springframework.boot', name: 'spring-boot-starter-mail', version: '4.0.1' + implementation 'org.flywaydb:flyway-core:9.22.0' + implementation 'com.fasterxml.jackson.dataformat:jackson-dataformat-xml:2.17.0' implementation 'tools.jackson.core:jackson-core:3.0.3' implementation 'jakarta.xml.bind:jakarta.xml.bind-api:4.0.0' diff --git a/referral/build.gradle.kts b/referral/build.gradle.kts new file mode 100644 index 0000000..7a4f37d --- /dev/null +++ b/referral/build.gradle.kts @@ -0,0 +1,28 @@ +plugins { + kotlin("jvm") version "2.1.10" +} + +group = "ru.soune" +version = "1.0.0" + +repositories { + mavenCentral() +} + +dependencies { + + implementation("org.jetbrains.kotlinx:kotlinx-coroutines-core:1.10.2") + implementation("org.jetbrains.kotlinx:kotlinx-coroutines-core-jvm:1.10.2") + + implementation("io.insert-koin:koin-core:4.1.1") + implementation("io.insert-koin:koin-core-jvm:4.1.1") + + testImplementation(kotlin("test")) +} + +tasks.test { + useJUnitPlatform() +} +kotlin { + jvmToolchain(21) +} \ No newline at end of file diff --git a/referral/src/main/kotlin/Main.kt b/referral/src/main/kotlin/Main.kt new file mode 100644 index 0000000..4c1e045 --- /dev/null +++ b/referral/src/main/kotlin/Main.kt @@ -0,0 +1,5 @@ +package ru.soune + +fun main() { + println("Hello World!") +} \ No newline at end of file diff --git a/settings.gradle b/settings.gradle index ab53a61..3d2f55e 100644 --- a/settings.gradle +++ b/settings.gradle @@ -1 +1,6 @@ +plugins { + id 'org.gradle.toolchains.foojay-resolver-convention' version '0.8.0' +} rootProject.name = 'no-copy' +include 'referral' + diff --git a/src/main/java/ru/soune/nocopy/controller/ApiController.java b/src/main/java/ru/soune/nocopy/controller/ApiController.java index a377eee..3ae0515 100644 --- a/src/main/java/ru/soune/nocopy/controller/ApiController.java +++ b/src/main/java/ru/soune/nocopy/controller/ApiController.java @@ -4,6 +4,9 @@ import lombok.RequiredArgsConstructor; import lombok.extern.slf4j.Slf4j; import org.springframework.core.io.Resource; import org.springframework.core.io.UrlResource; +import org.springframework.data.domain.Page; +import org.springframework.data.domain.Pageable; +import org.springframework.data.web.PageableDefault; import org.springframework.http.HttpHeaders; import org.springframework.http.MediaType; import org.springframework.http.ResponseEntity; @@ -23,6 +26,7 @@ import ru.soune.nocopy.entity.file.FileStatus; import ru.soune.nocopy.entity.file.UploadStatus; import ru.soune.nocopy.exception.*; import ru.soune.nocopy.handler.*; +import ru.soune.nocopy.repository.FileEntityRepository; import ru.soune.nocopy.service.register.AuthService; import ru.soune.nocopy.service.file.FileEntityService; import ru.soune.nocopy.service.file.FileUploadService; @@ -34,6 +38,7 @@ import java.nio.file.Paths; import java.util.HashMap; import java.util.List; import java.util.Map; +import java.util.Optional; import java.util.stream.Collectors; @Slf4j @@ -49,6 +54,10 @@ public class ApiController { private final AuthService authService; + private final FileSimilarityService fileSimilarityService; + + private final FileEntityRepository fileEntityRepository; + @PostMapping("/v{version}/data") public ResponseEntity handlePostRequest(@RequestBody BaseRequest request, @PathVariable("version") int version) { @@ -98,54 +107,63 @@ public class ApiController { @RequestParam(value = "chunk", required = false) MultipartFile chunk) { try { if (chunk == null || chunk.isEmpty()) { - return ResponseEntity.ok().body(new BaseResponse(20004, MessageCode.FILE_UPLOAD_ERROR.getCode(), - "Chunk file null or empty", ChunkUploadResponse.builder() - .uploadId(uploadId) - .chunkNumber(chunkNumber) - .build())); + return buildErrorResponse(uploadId, chunkNumber, "Chunk file null or empty"); } - if (uploadId == null || uploadId.isBlank()) { - return ResponseEntity.ok().body(new BaseResponse(20004, MessageCode.FILE_UPLOAD_ERROR.getCode(), - "Upload ID is required", ChunkUploadResponse.builder() - .uploadId(uploadId) - .chunkNumber(chunkNumber) - .build())); + return buildErrorResponse(uploadId, chunkNumber, "Upload ID is required"); } - if (chunkNumber == null || chunkNumber < 0) { - return ResponseEntity.ok().body(new BaseResponse(20004, MessageCode.FILE_UPLOAD_ERROR.getCode(), - "Valid chunk number is required", ChunkUploadResponse.builder() - .uploadId(uploadId) - .chunkNumber(chunkNumber) - .build())); + return buildErrorResponse(uploadId, chunkNumber, "Valid chunk number is required"); } fileUploadService.uploadChunk(uploadId, chunkNumber, chunk); - ChunkUploadResponse responseBody = ChunkUploadResponse.builder() - .uploadId(uploadId) - .chunkNumber(chunkNumber) - .chunkSize(chunk.getSize()) - .message("Chunk uploaded successfully") - .build(); - - return ResponseEntity.ok().body(new BaseResponse(20004, MessageCode.SUCCESS.getCode(), - "Chunk uploaded successfully", responseBody)); + return buildSuccessResponse(uploadId, chunkNumber, chunk); + } catch (DuplicateImageException e) { + return ResponseEntity.ok().body(new BaseResponse(20004, MessageCode.DUPLICATE_FILE_UPLOAD.getCode(), + MessageCode.DUPLICATE_FILE_UPLOAD.getDescription(), + Map.of("duplicateOwnerId", e.userId(), "duplicateFileId", e.duplicateFileId()))); } catch (Exception e) { log.error("Error uploading chunk", e); - - ChunkUploadResponse responseBody = ChunkUploadResponse.builder() - .uploadId(uploadId) - .chunkNumber(chunkNumber) - .build(); - - return ResponseEntity.ok().body(new BaseResponse( - 20004, MessageCode.FILE_UPLOAD_ERROR.getCode(), - "Failed to upload chunk: " + e.getMessage(), responseBody)); + return buildErrorResponse(uploadId, chunkNumber, "Failed to upload chunk: " + e.getMessage()); } } + @GetMapping("/v{version}/files/{fileId}/similar") + public ResponseEntity findSimilarFiles( + @PathVariable("version") int version, + @PathVariable String fileId, + @RequestParam(required = false) List similarityLevels, + @PageableDefault(size = 20, sort = "hammingDistance") Pageable pageable) { + SimilarityFilter filter = SimilarityFilter.builder() + .similarityLevels(similarityLevels) + .build(); + Page similarFiles = fileSimilarityService.findSimilarFiles(fileId, filter, pageable); + + String messageDesc; + MessageCode success; + + if (similarFiles.isEmpty()) { + messageDesc = MessageCode.FILE_NOT_FOUND.getDescription(); + success = MessageCode.FILE_NOT_FOUND; + } else { + messageDesc = MessageCode.SIMILAR_FILES_FOUND.getDescription(); + success = MessageCode.SIMILAR_FILES_FOUND; + } + + Map responseData = new HashMap<>(); + responseData.put("content", similarFiles.getContent()); + responseData.put("page", similarFiles.getNumber()); + responseData.put("size", similarFiles.getSize()); + responseData.put("totalElements", similarFiles.getTotalElements()); + responseData.put("totalPages", similarFiles.getTotalPages()); + responseData.put("hasNext", similarFiles.hasNext()); + responseData.put("hasPrevious", similarFiles.hasPrevious()); + + return ResponseEntity.ok() + .body(new BaseResponse(20004, success.getCode(), messageDesc, responseData)); + } + @GetMapping("/v{version}/files/progress/{uploadId}") public ResponseEntity getUploadProgress( @PathVariable("version") int version, @@ -169,7 +187,6 @@ public class ApiController { return ResponseEntity.ok().body(new BaseResponse( 20004, MessageCode.SUCCESS.getCode(), MessageCode.SUCCESS.getDescription(), responseBody)); - } catch (Exception e) { log.error("Error getting progress for upload: {}", uploadId, e); @@ -177,8 +194,7 @@ public class ApiController { .uploadId(uploadId) .build(); - return ResponseEntity.ok().body(new BaseResponse( - 20004, MessageCode.FILE_UPLOAD_ERROR.getCode(), + return ResponseEntity.ok().body(new BaseResponse(20004, MessageCode.FILE_UPLOAD_ERROR.getCode(), "Failed to get upload progress: " + e.getMessage(), responseBody)); } } @@ -337,6 +353,78 @@ public class ApiController { } } + private ResponseEntity checkForDuplicates(String uploadId) throws IOException { + Optional uploadedFile = fileEntityRepository.findByUploadSessionId(uploadId); + + if (uploadedFile.isEmpty() || !uploadedFile.get().getMimeType().equals("image")) { + return null; + } + + FileEntity fileEntity = uploadedFile.get(); + List similarFiles = fileSimilarityService.findSimilarFiles(fileEntity.getId()); + + if (hasDuplicate(similarFiles)) { + return handleDuplicate(fileEntity, similarFiles); + } + + return null; + } + + private boolean hasDuplicate(List similarFiles) { + return similarFiles.stream().anyMatch(f -> f.getHammingDistance() <= 5); + } + + private ResponseEntity handleDuplicate(FileEntity fileEntity, List similarFiles) + throws IOException { + fileEntityService.deleteFromDisk(fileEntity); + + fileEntityService.markAsDeleted(fileEntity); + + Optional originalFile = fileEntityRepository.findById(similarFiles.get(0).getFileId()); + + if (originalFile.isPresent()) { + Map duplicateInfo = Map.of( + "duplicate_file_id", originalFile.get().getId(), + "owner_user_id", String.valueOf(originalFile.get().getUserId())); + + return ResponseEntity.ok().body(new BaseResponse( + 20004, + MessageCode.DUPLICATE_FILE_UPLOAD.getCode(), + "Failed to upload chunk, duplicate", + duplicateInfo)); + } + + return null; + } + + private ResponseEntity buildSuccessResponse(String uploadId, Integer chunkNumber, MultipartFile chunk) { + ChunkUploadResponse responseBody = ChunkUploadResponse.builder() + .uploadId(uploadId) + .chunkNumber(chunkNumber) + .chunkSize(chunk.getSize()) + .message("Chunk uploaded successfully") + .build(); + + return ResponseEntity.ok().body(new BaseResponse( + 20000, + MessageCode.SUCCESS.getCode(), + "Chunk uploaded successfully", + responseBody)); + } + + private ResponseEntity buildErrorResponse(String uploadId, Integer chunkNumber, String errorMessage) { + ChunkUploadResponse responseBody = ChunkUploadResponse.builder() + .uploadId(uploadId) + .chunkNumber(chunkNumber) + .build(); + + return ResponseEntity.ok().body(new BaseResponse( + 20004, + MessageCode.FILE_UPLOAD_ERROR.getCode(), + errorMessage, + responseBody)); + } + private ResponseEntity createValidationErrorResponse(BindingResult bindingResult, Integer msgId) { List> fieldErrors = bindingResult.getFieldErrors() .stream() diff --git a/src/main/java/ru/soune/nocopy/dto/MessageCode.java b/src/main/java/ru/soune/nocopy/dto/MessageCode.java index 1f3cc27..8c5168d 100644 --- a/src/main/java/ru/soune/nocopy/dto/MessageCode.java +++ b/src/main/java/ru/soune/nocopy/dto/MessageCode.java @@ -9,6 +9,7 @@ public enum MessageCode { TOKEN_IS_ALIVE(2, "Token is alive"), INVALID_ACTION(2, "Invalid action"), FILE_UPLOAD_ERROR(2, "File upload error"), + DUPLICATE_FILE_UPLOAD(2, "Duplicate file upload"), FILE_DOWNLOAD_ERROR(2, "File download error"), USER_NOT_VERIFIED(2, "User not verified"), USER_NOT_FOUND(2, "User not found"), @@ -25,7 +26,8 @@ public enum MessageCode { AUTH_TOKEN_NOT_FOUND(4, "Token not found"), FILE_NOT_FOUND(4, "File not found"), AUTH_PASSWORD_NOT_MATCHES(2, "Password does not match"), - SEND_EMAIL_EXCEPTION(2, "Send email exception"),; + SEND_EMAIL_EXCEPTION(2, "Send email exception"), + SIMILAR_FILES_FOUND(0, "Similar files found"); private final Integer code; diff --git a/src/main/java/ru/soune/nocopy/dto/file/SimilarFileResponse.java b/src/main/java/ru/soune/nocopy/dto/file/SimilarFileResponse.java new file mode 100644 index 0000000..f7aae6e --- /dev/null +++ b/src/main/java/ru/soune/nocopy/dto/file/SimilarFileResponse.java @@ -0,0 +1,14 @@ +package ru.soune.nocopy.dto.file; + +import lombok.Builder; +import lombok.Value; + +@Value +@Builder +public class SimilarFileResponse { + String fileId; + String originalFileName; + Long fileSize; + Integer hammingDistance; + String similarityLevel; +} diff --git a/src/main/java/ru/soune/nocopy/dto/file/SimilarityFilter.java b/src/main/java/ru/soune/nocopy/dto/file/SimilarityFilter.java new file mode 100644 index 0000000..48a4433 --- /dev/null +++ b/src/main/java/ru/soune/nocopy/dto/file/SimilarityFilter.java @@ -0,0 +1,18 @@ +package ru.soune.nocopy.dto.file; + +import lombok.AllArgsConstructor; +import lombok.Builder; +import lombok.Data; +import lombok.NoArgsConstructor; + +import java.util.List; + +@Data +@Builder +@NoArgsConstructor +@AllArgsConstructor +public class SimilarityFilter { + private List similarityLevels; + private Integer page; + private Integer size; +} diff --git a/src/main/java/ru/soune/nocopy/entity/file/FileEntity.java b/src/main/java/ru/soune/nocopy/entity/file/FileEntity.java index c39862a..16e25a4 100644 --- a/src/main/java/ru/soune/nocopy/entity/file/FileEntity.java +++ b/src/main/java/ru/soune/nocopy/entity/file/FileEntity.java @@ -1,5 +1,6 @@ package ru.soune.nocopy.entity.file; +import com.fasterxml.jackson.annotation.JsonIgnore; import jakarta.persistence.*; import lombok.*; import org.hibernate.annotations.GenerationTime; @@ -65,6 +66,10 @@ public class FileEntity { @Column(name = "updated_at") private LocalDateTime updatedAt; + @OneToOne(mappedBy = "file", cascade = CascadeType.ALL, orphanRemoval = true) + @JsonIgnore + private ImageHashEntity imageHash; + @PrePersist public void prePersist() { if (this.status == null) { diff --git a/src/main/java/ru/soune/nocopy/entity/file/FileUploadSession.java b/src/main/java/ru/soune/nocopy/entity/file/FileUploadSession.java index 509d0f7..80dc12f 100644 --- a/src/main/java/ru/soune/nocopy/entity/file/FileUploadSession.java +++ b/src/main/java/ru/soune/nocopy/entity/file/FileUploadSession.java @@ -69,6 +69,7 @@ public class FileUploadSession { private String extension; @Column(name = "retry_count") + @Builder.Default private Integer retryCount = 0; @Column(name = "completed_at") @@ -81,6 +82,7 @@ public class FileUploadSession { ) @MapKeyColumn(name = "chunk_number") @Column(name = "chunk_path") + @Builder.Default private Map chunkPaths = new HashMap<>(); @PrePersist diff --git a/src/main/java/ru/soune/nocopy/entity/file/ImageHashEntity.java b/src/main/java/ru/soune/nocopy/entity/file/ImageHashEntity.java new file mode 100644 index 0000000..37f1b5a --- /dev/null +++ b/src/main/java/ru/soune/nocopy/entity/file/ImageHashEntity.java @@ -0,0 +1,39 @@ +package ru.soune.nocopy.entity.file; + +import jakarta.persistence.*; +import lombok.AllArgsConstructor; +import lombok.Builder; +import lombok.Data; +import lombok.NoArgsConstructor; + +import java.time.LocalDateTime; + +@Entity +@Table(name = "image_hashes") +@Data +@NoArgsConstructor +@AllArgsConstructor +@Builder +public class ImageHashEntity { + + @Id + @Column(name = "file_id") + private String fileId; + + @OneToOne(fetch = FetchType.LAZY) + @MapsId + @JoinColumn(name = "file_id") + private FileEntity file; + + @Column(name = "hash64_hi") + private Integer hash64Hi; + + @Column(name = "hash64_lo") + private Integer hash64Lo; + + @Column(name = "hash_algorithm", nullable = false) + private String hashAlgorithm; + + @Column(name = "created_at") + private LocalDateTime createdAt; +} diff --git a/src/main/java/ru/soune/nocopy/exception/DuplicateImageException.java b/src/main/java/ru/soune/nocopy/exception/DuplicateImageException.java new file mode 100644 index 0000000..3ac1325 --- /dev/null +++ b/src/main/java/ru/soune/nocopy/exception/DuplicateImageException.java @@ -0,0 +1,20 @@ +package ru.soune.nocopy.exception; + +public class DuplicateImageException extends RuntimeException { + private final String duplicateFileId; + private final Long userId; + + public DuplicateImageException(String message, String duplicateFileId, Long userId) { + super(message); + this.duplicateFileId = duplicateFileId; + this.userId = userId; + } + + public String duplicateFileId() { + return duplicateFileId; + } + + public Long userId() { + return userId; + } +} diff --git a/src/main/java/ru/soune/nocopy/handler/FileUploadHandler.java b/src/main/java/ru/soune/nocopy/handler/FileUploadHandler.java index c4ae60a..67755c0 100644 --- a/src/main/java/ru/soune/nocopy/handler/FileUploadHandler.java +++ b/src/main/java/ru/soune/nocopy/handler/FileUploadHandler.java @@ -177,40 +177,13 @@ public class FileUploadHandler implements RequestHandler { } } - private BaseResponse handleRetryUpload(BaseRequest request, FileUploadRequest fileRequest) { - try { - fileUploadService.retryFailedUpload(fileRequest.getUploadId()); - ru.soune.nocopy.dto.file.UploadProgressResponse progress = - fileUploadService.getUploadProgress(fileRequest.getUploadId()); - - RetryUploadResponse response = RetryUploadResponse.builder() - .uploadId(progress.getUploadId()) - .message("Upload retry initiated") - .status(progress.getStatus().toString()) - .build(); - - return new BaseResponse(request.getMsgId(), MessageCode.SUCCESS.getCode(), "Upload retry initiated", - response); - - } catch (UploadSessionNotFoundException e) { - return new BaseResponse(request.getMsgId(), MessageCode.FILE_NOT_FOUND.getCode(), - "Upload session not found", null); - } catch (FileUploadException e) { - return new BaseResponse(request.getMsgId(), MessageCode.FILE_UPLOAD_ERROR.getCode(), e.getMessage(), - null); - } catch (Exception e) { - log.error("Error retrying upload", e); - return new BaseResponse(request.getMsgId(), MessageCode.FILE_UPLOAD_ERROR.getCode(), - "Failed to retry upload", null); - } - } - private BaseResponse handleGetChunkStatus(BaseRequest request, FileUploadRequest fileRequest) { try { FileUploadSession session = fileUploadSessionRepository.findById(fileRequest.getUploadId()) .orElseThrow(() -> new UploadSessionNotFoundException(fileRequest.getUploadId())); Map chunkStatus = new HashMap<>(); + for (int i = 0; i < session.getTotalChunks(); i++) { chunkStatus.put("chunk_" + i, session.getChunkPaths().containsKey(i)); } @@ -225,7 +198,6 @@ public class FileUploadHandler implements RequestHandler { return new BaseResponse(request.getMsgId(), MessageCode.SUCCESS.getCode(), MessageCode.SUCCESS.getDescription(), response); - } catch (UploadSessionNotFoundException e) { return new BaseResponse(request.getMsgId(), MessageCode.FILE_NOT_FOUND.getCode(), "Upload session not found", null); diff --git a/src/main/java/ru/soune/nocopy/handler/GlobalExceptionHandler.java b/src/main/java/ru/soune/nocopy/handler/GlobalExceptionHandler.java index 071b9ae..7303fb5 100644 --- a/src/main/java/ru/soune/nocopy/handler/GlobalExceptionHandler.java +++ b/src/main/java/ru/soune/nocopy/handler/GlobalExceptionHandler.java @@ -5,15 +5,16 @@ import org.springframework.context.support.DefaultMessageSourceResolvable; import org.springframework.http.HttpStatus; import org.springframework.http.ResponseEntity; import org.springframework.web.bind.MethodArgumentNotValidException; -import org.springframework.web.bind.annotation.ExceptionHandler; -import org.springframework.web.bind.annotation.ResponseStatus; -import org.springframework.web.bind.annotation.RestControllerAdvice; +import org.springframework.web.bind.annotation.*; +import ru.soune.nocopy.dto.BaseResponse; +import ru.soune.nocopy.dto.MessageCode; import ru.soune.nocopy.exception.*; import java.util.Map; @RestControllerAdvice @AllArgsConstructor +@ControllerAdvice public class GlobalExceptionHandler { @ExceptionHandler(MethodArgumentNotValidException.class) @@ -81,4 +82,12 @@ public class GlobalExceptionHandler { "message" ,ex.getMessage() )); } + + @ExceptionHandler(DuplicateImageException.class) + @ResponseBody + public ResponseEntity handleDuplicateImage(DuplicateImageException e) { + return ResponseEntity.ok().body(new BaseResponse( + 20004, MessageCode.DUPLICATE_FILE_UPLOAD.getCode(), + "Duplicate image detected", e.getMessage())); + } } diff --git a/src/main/java/ru/soune/nocopy/repository/FileEntityRepository.java b/src/main/java/ru/soune/nocopy/repository/FileEntityRepository.java index 3214e9d..e35ef2f 100644 --- a/src/main/java/ru/soune/nocopy/repository/FileEntityRepository.java +++ b/src/main/java/ru/soune/nocopy/repository/FileEntityRepository.java @@ -30,5 +30,8 @@ public interface FileEntityRepository extends JpaRepository @Query("SELECT f FROM FileEntity f WHERE f.userId = :userId AND f.originalFileName LIKE %:keyword%") List searchByFileName(@Param("userId") Long userId, @Param("keyword") String keyword); + @Query("SELECT f FROM FileEntity f WHERE f.status = :status") + List searchFileEntityByStatus(@Param("status") FileStatus status); + long countByUserId(Long userId); } diff --git a/src/main/java/ru/soune/nocopy/repository/ImageHashRepository.java b/src/main/java/ru/soune/nocopy/repository/ImageHashRepository.java new file mode 100644 index 0000000..ee0aac2 --- /dev/null +++ b/src/main/java/ru/soune/nocopy/repository/ImageHashRepository.java @@ -0,0 +1,10 @@ +package ru.soune.nocopy.repository; + +import org.springframework.data.jpa.repository.JpaRepository; +import org.springframework.stereotype.Repository; +import ru.soune.nocopy.entity.file.ImageHashEntity; + +@Repository +public interface ImageHashRepository extends JpaRepository { + void deleteByFileId(String fileId); +} diff --git a/src/main/java/ru/soune/nocopy/repository/ImageSimilarityRepository.java b/src/main/java/ru/soune/nocopy/repository/ImageSimilarityRepository.java new file mode 100644 index 0000000..a6ef973 --- /dev/null +++ b/src/main/java/ru/soune/nocopy/repository/ImageSimilarityRepository.java @@ -0,0 +1,54 @@ +package ru.soune.nocopy.repository; + +import org.springframework.data.jpa.repository.JpaRepository; +import org.springframework.data.jpa.repository.Query; +import org.springframework.data.repository.query.Param; +import org.springframework.stereotype.Repository; +import ru.soune.nocopy.entity.file.FileEntity; + +import java.util.List; + +@Repository +public interface ImageSimilarityRepository + extends JpaRepository { + + @Query(value = """ + + SELECT + f.id AS similarFileId, + f.original_file_name AS originalFileName, + f.file_size AS fileSize, + h.hash64_hi AS hash64Hi, + h.hash64_lo AS hash64Lo + FROM image_hashes ref + JOIN image_hashes h + ON ref.file_id <> h.file_id + JOIN file_entities f + ON f.id = h.file_id + WHERE ref.file_id = :fileId + """, + nativeQuery = true) + List findCandidates( + @Param("fileId") String fileId + ); + + @Query(value = """ + SELECT + h.hash64_hi AS hash64Hi, + h.hash64_lo AS hash64Lo, + h.file_id AS fileId, + f.user_id AS userId, + f.original_file_name AS originalFileName, + f.file_size AS fileSize, + f.stored_file_name AS similarFileId + FROM image_hashes h + JOIN file_entities f ON f.id = h.file_id + WHERE h.hash64_hi = :hash64Hi + AND h.hash64_lo = :hash64Lo + """, + nativeQuery = true) + List findExactDuplicates( + @Param("hash64Hi") Integer hash64_hi, + @Param("hash64Lo") Integer hash64_lo); +} + diff --git a/src/main/java/ru/soune/nocopy/repository/SimilarImageProjection.java b/src/main/java/ru/soune/nocopy/repository/SimilarImageProjection.java new file mode 100644 index 0000000..cac73fb --- /dev/null +++ b/src/main/java/ru/soune/nocopy/repository/SimilarImageProjection.java @@ -0,0 +1,10 @@ +package ru.soune.nocopy.repository; + +public interface SimilarImageProjection { + Integer getHash64Hi(); + Integer getHash64Lo(); + String getId(); + Long getUserId(); + String getOriginalFileName(); + Long getFileSize(); +} \ No newline at end of file diff --git a/src/main/java/ru/soune/nocopy/service/FileSimilarityService.java b/src/main/java/ru/soune/nocopy/service/FileSimilarityService.java new file mode 100644 index 0000000..9ec4051 --- /dev/null +++ b/src/main/java/ru/soune/nocopy/service/FileSimilarityService.java @@ -0,0 +1,135 @@ +package ru.soune.nocopy.service; + +import lombok.RequiredArgsConstructor; +import org.springframework.data.domain.Page; +import org.springframework.data.domain.PageImpl; +import org.springframework.data.domain.Pageable; +import org.springframework.stereotype.Service; +import ru.soune.nocopy.dto.file.SimilarFileResponse; +import ru.soune.nocopy.dto.file.SimilarityFilter; +import ru.soune.nocopy.repository.ImageHashRepository; +import ru.soune.nocopy.repository.ImageSimilarityRepository; +import ru.soune.nocopy.repository.SimilarImageProjection; + +import java.util.ArrayList; +import java.util.Comparator; +import java.util.List; +import java.util.stream.Collectors; + +@Service +@RequiredArgsConstructor +public class FileSimilarityService { + private final ImageSimilarityRepository repository; + + private final ImageHashRepository hashRepository; + + public List findSimilarFiles(String fileId) { + var imageHashEntity = hashRepository.findById(fileId) + .orElseThrow(() -> new RuntimeException("Hash not found")); + + Integer hash64Hi = imageHashEntity.getHash64Hi(); + Integer hash64Lo = imageHashEntity.getHash64Lo(); + + List candidates = + repository.findCandidates(fileId); + + return candidates.stream() + .map(c -> { + Integer cHi = c.getHash64Hi(); + Integer cLo = c.getHash64Lo(); + + int hamming = hamming64(hash64Hi, hash64Lo, cHi, cLo); + + String level; + if (hamming <= 5) { + level = "DUPLICATE"; + } else if (hamming <= 12) { + level = "SIMILAR"; + } else { + level = "DIFFERENT"; + } + + return SimilarFileResponse.builder() + .fileId(c.getId()) + .originalFileName(c.getOriginalFileName()) + .fileSize(c.getFileSize()) + .hammingDistance(hamming) + .similarityLevel(level) + .build(); + }) + .sorted((a, b) -> + Integer.compare(a.getHammingDistance(), b.getHammingDistance())) + .toList(); + } + + public List findDuplicatedByHash(Integer hash64Hi, Integer hash64Lo) { + List duplicates = repository.findExactDuplicates(hash64Hi,hash64Lo); + + if (duplicates.isEmpty()) { + return new ArrayList<>(); + } + + return duplicates; + } + + public Page findSimilarFiles(String fileId, SimilarityFilter filter, Pageable pageable) { + var imageHashEntity = hashRepository.findById(fileId) + .orElseThrow(() -> new RuntimeException("Hash not found")); + + Integer hash64Hi = imageHashEntity.getHash64Hi(); + Integer hash64Lo = imageHashEntity.getHash64Lo(); + + List candidates = repository.findCandidates(fileId); + + List similarityLevels = (filter != null && filter.getSimilarityLevels() != null) + ? filter.getSimilarityLevels() + : List.of("DUPLICATE", "SIMILAR", "DIFFERENT"); + + List allResults = candidates.stream() + .map(c -> createSimilarFileResponse(c, hash64Hi, hash64Lo)) + .filter(response -> similarityLevels.contains(response.getSimilarityLevel())) + .sorted(Comparator.comparingInt(SimilarFileResponse::getHammingDistance)) + .collect(Collectors.toList()); + + int total = allResults.size(); + int page = (pageable != null) ? pageable.getPageNumber() : 0; + int size = (pageable != null) ? pageable.getPageSize() : 20; + + int fromIndex = Math.min(page * size, total); + int toIndex = Math.min(fromIndex + size, total); + + List pageContent = allResults.subList(fromIndex, toIndex); + + return new PageImpl<>(pageContent, pageable, total); + } + + private SimilarFileResponse createSimilarFileResponse(SimilarImageProjection similarImageProjection, + Integer hash64Hi, Integer hash64Lo) { + Integer imageProjectionHash64Hi = similarImageProjection.getHash64Hi(); + Integer similarImageProjectionHash64Lo = similarImageProjection.getHash64Lo(); + + int hamming = hamming64(hash64Hi, hash64Lo, imageProjectionHash64Hi, similarImageProjectionHash64Lo); + + String level; + if (hamming <= 5) { + level = "DUPLICATE"; + } else if (hamming <= 12) { + level = "SIMILAR"; + } else { + level = "DIFFERENT"; + } + + return SimilarFileResponse.builder() + .fileId(similarImageProjection.getId()) + .originalFileName(similarImageProjection.getOriginalFileName()) + .fileSize(similarImageProjection.getFileSize()) + .hammingDistance(hamming) + .similarityLevel(level) + .build(); + } + + private int hamming64(int aHi, int aLo, int bHi, int bLo) { + return Integer.bitCount(aHi ^ bHi) + + Integer.bitCount(aLo ^ bLo); + } +} diff --git a/src/main/java/ru/soune/nocopy/service/ImageHashService.java b/src/main/java/ru/soune/nocopy/service/ImageHashService.java new file mode 100644 index 0000000..b7b6882 --- /dev/null +++ b/src/main/java/ru/soune/nocopy/service/ImageHashService.java @@ -0,0 +1,154 @@ +package ru.soune.nocopy.service; + +import lombok.RequiredArgsConstructor; +import lombok.extern.slf4j.Slf4j; +import org.springframework.stereotype.Service; +import ru.soune.nocopy.entity.file.FileEntity; +import ru.soune.nocopy.entity.file.ImageHashEntity; +import ru.soune.nocopy.repository.ImageHashRepository; + +import javax.imageio.ImageIO; +import java.awt.*; +import java.awt.image.BufferedImage; +import java.io.IOException; +import java.io.InputStream; +import java.nio.file.Files; +import java.nio.file.Path; +import java.time.LocalDateTime; +import java.util.Arrays; +import java.util.Map; + +@Service +@RequiredArgsConstructor +@Slf4j +public class ImageHashService { + + private final ImageHashRepository repository; + + public Map calculateHash(Path imagePath) throws IOException { + long hash64 = computePhash64(imagePath); + + return Map.of( + "hi", high32(hash64), + "low", low32(hash64)); + } + + public void create(FileEntity file, Map stringIntegerMap) { + ImageHashEntity entity = ImageHashEntity.builder() + .file(file) + .hash64Hi(stringIntegerMap.get("hi")) + .hash64Lo(stringIntegerMap.get("low")) + .hashAlgorithm("PHASH64") + .createdAt(LocalDateTime.now()) + .build(); + + repository.save(entity); + } + + private long computePhash64(Path path) throws IOException { + try (InputStream is = Files.newInputStream(path)) { + BufferedImage image = ImageIO.read(is); + if (image == null) throw new IOException("Cannot read image file: " + path); + + BufferedImage gray = toGrayScale(image); + BufferedImage resized = resizeImage(gray, 32, 32); + + double[][] dct = applyDCT(resized); + double[][] topLeft = extractTopLeft8x8(dct); + double median = calculateMedian(topLeft, true); + + return create64BitHashLong(topLeft, median); + } + } + + private BufferedImage toGrayScale(BufferedImage image) { + BufferedImage gray = new BufferedImage(image.getWidth(), image.getHeight(), BufferedImage.TYPE_BYTE_GRAY); + Graphics2D g = gray.createGraphics(); + g.drawImage(image, 0, 0, null); + g.dispose(); + return gray; + } + + private BufferedImage resizeImage(BufferedImage image, int width, int height) { + BufferedImage resized = new BufferedImage(width, height, BufferedImage.TYPE_BYTE_GRAY); + Graphics2D g = resized.createGraphics(); + g.setRenderingHint(RenderingHints.KEY_INTERPOLATION, RenderingHints.VALUE_INTERPOLATION_BILINEAR); + g.drawImage(image, 0, 0, width, height, null); + g.dispose(); + return resized; + } + + private double[][] applyDCT(BufferedImage image) { + int w = image.getWidth(); + int h = image.getHeight(); + double[][] pixels = new double[h][w]; + + for (int y = 0; y < h; y++) { + for (int x = 0; x < w; x++) { + int gray = image.getRGB(x, y) & 0xFF; + pixels[y][x] = gray; + } + } + + double[][] dct = new double[h][w]; + for (int u = 0; u < h; u++) { + double au = u == 0 ? 1.0 / Math.sqrt(h) : Math.sqrt(2.0 / h); + for (int v = 0; v < w; v++) { + double av = v == 0 ? 1.0 / Math.sqrt(w) : Math.sqrt(2.0 / w); + double sum = 0.0; + for (int x = 0; x < h; x++) { + for (int y = 0; y < w; y++) { + sum += pixels[x][y] * + Math.cos((2 * x + 1) * u * Math.PI / (2.0 * h)) * + Math.cos((2 * y + 1) * v * Math.PI / (2.0 * w)); + } + } + dct[u][v] = au * av * sum; + } + } + return dct; + } + + private double[][] extractTopLeft8x8(double[][] dct) { + double[][] block = new double[8][8]; + for (int i = 0; i < 8; i++) System.arraycopy(dct[i], 0, block[i], 0, 8); + return block; + } + + private double calculateMedian(double[][] block, boolean excludeDC) { + double[] vals = new double[63]; + int idx = 0; + for (int i = 0; i < 8; i++) { + for (int j = 0; j < 8; j++) { + if (i == 0 && j == 0 && excludeDC) continue; + vals[idx++] = block[i][j]; + } + } + Arrays.sort(vals); + return (vals.length % 2 == 0) ? + (vals[vals.length / 2 - 1] + vals[vals.length / 2]) / 2.0 : + vals[vals.length / 2]; + } + + private long create64BitHashLong(double[][] block, double median) { + long hash = 0L; + + for (int i = 0; i < 8; i++) { + for (int j = 0; j < 8; j++) { + if (i == 0 && j == 0) continue; + hash <<= 1; + if (block[i][j] > median) hash |= 1; + } + } + + return hash; + } + + private int high32(long hash64) { + return (int) (hash64 >>> 32); + } + + private int low32(long hash64) { + return (int) hash64; + } +} diff --git a/src/main/java/ru/soune/nocopy/service/file/FileEntityService.java b/src/main/java/ru/soune/nocopy/service/file/FileEntityService.java index 3eac369..1f3464e 100644 --- a/src/main/java/ru/soune/nocopy/service/file/FileEntityService.java +++ b/src/main/java/ru/soune/nocopy/service/file/FileEntityService.java @@ -9,8 +9,12 @@ import ru.soune.nocopy.dto.file.FileResponse; import ru.soune.nocopy.entity.file.FileEntity; import ru.soune.nocopy.entity.file.FileStatus; import ru.soune.nocopy.entity.file.FileUploadSession; +import ru.soune.nocopy.exception.DuplicateImageException; import ru.soune.nocopy.exception.FileEntityNotFoundException; import ru.soune.nocopy.repository.FileEntityRepository; +import ru.soune.nocopy.repository.SimilarImageProjection; +import ru.soune.nocopy.service.FileSimilarityService; +import ru.soune.nocopy.service.ImageHashService; import java.io.IOException; import java.nio.file.Files; @@ -18,6 +22,7 @@ import java.nio.file.Path; import java.nio.file.Paths; import java.time.LocalDateTime; import java.util.List; +import java.util.Map; import java.util.stream.Collectors; @Slf4j @@ -27,7 +32,11 @@ public class FileEntityService { private final FileEntityRepository fileEntityRepository; - @Transactional + private final ImageHashService imageHashService; + + private final FileSimilarityService fileSimilarityService; + + @Transactional(noRollbackFor = DuplicateImageException.class) public FileEntity createFromUploadSession(FileUploadSession session, String checksum) { log.info("Creating FileEntity for upload session: {}", session.getUploadId()); @@ -38,6 +47,21 @@ public class FileEntityService { throw new IOException("File not found on disk: " + filePath); } + Map imageHash = Map.of(); + + if (session.getFileType().startsWith("image")) { + imageHash = imageHashService.calculateHash(filePath); + List duplicatedByHash = fileSimilarityService.findDuplicatedByHash( + imageHash.get("hi"), imageHash.get("low")); + + if (!duplicatedByHash.isEmpty()) { + SimilarImageProjection similarImageProjection = duplicatedByHash.get(0); + + throw new DuplicateImageException("Duplicate", similarImageProjection.getFileId(), + similarImageProjection.getUserId()); + } + } + long fileSize = Files.size(filePath); String originalName = session.getFileName(); String storedName = filePath.getFileName().toString(); @@ -56,11 +80,12 @@ public class FileEntityService { .build(); FileEntity saved = fileEntityRepository.save(fileEntity); - log.info("FileEntity created successfully: {} (size: {} bytes)", - saved.getId(), fileSize); + + if (!imageHash.isEmpty()) { + imageHashService.create(saved, imageHash); + } return saved; - } catch (IOException e) { log.error("Failed to create FileEntity for session {}: {}", session.getUploadId(), e.getMessage(), e); @@ -68,15 +93,6 @@ public class FileEntityService { } } - - private String extractFileExtension(String fileName) { - int dotIndex = fileName.lastIndexOf('.'); - if (dotIndex > 0 && dotIndex < fileName.length() - 1) { - return fileName.substring(dotIndex + 1).toLowerCase(); - } - return ""; - } - @Transactional(readOnly = true) public FileEntityResponse getById(String fileId, int version) { FileEntity fileEntity = fileEntityRepository.findById(fileId) @@ -94,39 +110,6 @@ public class FileEntityService { return convertToResponse(fileEntity, version); } - @Transactional(readOnly = true) - public FileEntityResponse getByFilePath(String filePath, int version) { - FileEntity fileEntity = fileEntityRepository.findByFilePath(filePath) - .orElseThrow(() -> new FileEntityNotFoundException("Path: " + filePath)); - - return convertToResponse(fileEntity, version); - } - - - @Transactional(readOnly = true) - public FileResponse getAllUserFiles(Long userId, int version) { - List fileEntities = fileEntityRepository.findByUserIdAndStatus( - userId, FileStatus.ACTIVE); - - List files = fileEntities.stream() - .map(file -> convertToResponse(file, version)) - .collect(Collectors.toList()); - - long totalSize = fileEntities.stream() - .mapToLong(FileEntity::getFileSize) - .sum(); - - return FileResponse.builder() - .files(files) - .totalCount(files.size()) - .totalSize(totalSize) - .formattedTotalSize(formatFileSize(totalSize)) - .page(1) - .pageSize(files.size()) - .build(); - } - - @Transactional(readOnly = true) public FileResponse getUserFiles(Long userId, int page, int pageSize, int version) { List allFiles = fileEntityRepository.findByUserIdAndStatus( @@ -184,6 +167,8 @@ public class FileEntityService { Files.delete(path); + markAsDeleted(fileEntity); + return true; } diff --git a/src/main/java/ru/soune/nocopy/service/file/FileUploadService.java b/src/main/java/ru/soune/nocopy/service/file/FileUploadService.java index b27dede..714ef21 100644 --- a/src/main/java/ru/soune/nocopy/service/file/FileUploadService.java +++ b/src/main/java/ru/soune/nocopy/service/file/FileUploadService.java @@ -17,7 +17,5 @@ public interface FileUploadService { void handleExpiredSession(FileUploadSession session); - public void retryFailedUpload(String uploadId); - public void cancelUpload(String uploadId); } diff --git a/src/main/java/ru/soune/nocopy/service/file/FileUploadServiceImpl.java b/src/main/java/ru/soune/nocopy/service/file/FileUploadServiceImpl.java index 3e09822..587cdc1 100644 --- a/src/main/java/ru/soune/nocopy/service/file/FileUploadServiceImpl.java +++ b/src/main/java/ru/soune/nocopy/service/file/FileUploadServiceImpl.java @@ -10,12 +10,19 @@ import org.springframework.stereotype.Service; import org.springframework.transaction.annotation.Transactional; import org.springframework.web.multipart.MultipartFile; import ru.soune.nocopy.dto.file.UploadProgressResponse; +import ru.soune.nocopy.entity.file.FileEntity; +import ru.soune.nocopy.entity.file.FileStatus; import ru.soune.nocopy.entity.file.FileUploadSession; import ru.soune.nocopy.entity.file.UploadStatus; import ru.soune.nocopy.exception.ChunkSizeExceededException; +import ru.soune.nocopy.exception.DuplicateImageException; import ru.soune.nocopy.exception.FileUploadException; import ru.soune.nocopy.exception.UploadSessionNotFoundException; +import ru.soune.nocopy.repository.FileEntityRepository; import ru.soune.nocopy.repository.FileUploadSessionRepository; +import ru.soune.nocopy.repository.SimilarImageProjection; +import ru.soune.nocopy.service.FileSimilarityService; +import ru.soune.nocopy.service.ImageHashService; import java.io.*; import java.nio.file.*; @@ -23,10 +30,7 @@ import java.nio.file.attribute.PosixFilePermission; import java.security.MessageDigest; import java.security.NoSuchAlgorithmException; import java.time.LocalDateTime; -import java.util.Comparator; -import java.util.HashSet; -import java.util.Set; -import java.util.UUID; +import java.util.*; import java.util.concurrent.CompletableFuture; import java.util.stream.Stream; @@ -57,6 +61,15 @@ public class FileUploadServiceImpl implements FileUploadService { private Path storageRoot; + @Autowired + private ImageHashService imageHashService; + + @Autowired + private FileSimilarityService fileSimilarityService; + + @Autowired + private FileEntityRepository fileEntityRepository; + @Autowired private FileEntityService fileEntityService; @@ -129,10 +142,6 @@ public class FileUploadServiceImpl implements FileUploadService { @Transactional public UploadProgressResponse uploadChunk(String uploadId, Integer chunkNumber, MultipartFile chunkFile) { - - log.info("Processing chunk {} for session {}, size: {} bytes", - chunkNumber, uploadId, chunkFile.getSize()); - FileUploadSession session = sessionRepository.findById(uploadId) .orElseThrow(() -> new UploadSessionNotFoundException(uploadId)); @@ -173,12 +182,13 @@ public class FileUploadServiceImpl implements FileUploadService { @Async("fileUploadTaskExecutor") @Transactional - public void assembleFileAsync(FileUploadSession session) { + public void assembleFileAsync(FileUploadSession session) throws DuplicateImageException { try { assembleFile(session); log.info("File assembly completed successfully for session: {}", session.getUploadId()); - + } catch (DuplicateImageException e) { + throw new DuplicateImageException("DUBL", e.duplicateFileId(), e.userId()); } catch (Exception e) { log.error("Failed to assemble file for session {}: {}", session.getUploadId(), e.getMessage(), e); @@ -205,35 +215,6 @@ public class FileUploadServiceImpl implements FileUploadService { log.info("Upload cancelled: {}", uploadId); } - @Override - @Transactional - public void retryFailedUpload(String uploadId) { - FileUploadSession session = sessionRepository.findById(uploadId) - .orElseThrow(() -> new UploadSessionNotFoundException(uploadId)); - - if (session.getStatus() != UploadStatus.FAILED) { - throw new FileUploadException("Only failed uploads can be retried"); - } - - if (session.getRetryCount() >= maxRetryAttempts) { - throw new FileUploadException("Max retry attempts exceeded"); - } - - session.setStatus(UploadStatus.UPLOADING); - session.setRetryCount(session.getRetryCount() + 1); - session.setLastError(null); - - sessionRepository.save(session); - - if (session.getChunksUploaded().equals(session.getTotalChunks())) { - log.info("Retrying file assembly for session: {}", uploadId); - assembleFileAsync(session); - } - - log.info("Upload retry initiated for session: {} (attempt {})", - uploadId, session.getRetryCount()); - } - @Override public UploadProgressResponse getUploadProgress(String uploadId) { FileUploadSession session = sessionRepository.findById(uploadId) @@ -271,9 +252,42 @@ public class FileUploadServiceImpl implements FileUploadService { } } - private UploadProgressResponse processChunk(FileUploadSession session, - Integer chunkNumber, - MultipartFile chunkFile) { + @Async("fileUploadTaskExecutor") + @Transactional + public void completeFileProcessingAsync(FileUploadSession session) { + try { + Path filePath = Paths.get(session.getFilePath()); + String checksum = calculateChecksum(filePath); + + FileEntity fileEntity = FileEntity.builder() + .userId(session.getUserId()) + .originalFileName(session.getFileName()) + .storedFileName(filePath.getFileName().toString()) + .filePath(session.getFilePath()) + .fileSize(session.getFileSize()) + .mimeType(session.getFileType()) + .fileExtension(session.getExtension()) + .checksum(checksum) + .uploadSessionId(session.getUploadId()) + .status(FileStatus.ACTIVE) + .build(); + + FileEntity saved = fileEntityRepository.save(fileEntity); + + Map hash = imageHashService.calculateHash(filePath); + imageHashService.create(saved, hash); + + cleanupSessionFiles(session); + + log.info("File processing completed for session: {}", session.getUploadId()); + + } catch (Exception e) { + log.error("Failed to complete file processing for session {}: {}", + session.getUploadId(), e.getMessage(), e); + } + } + + private UploadProgressResponse processChunk(FileUploadSession session, Integer chunkNumber, MultipartFile chunkFile) { String chunkPath = null; try { @@ -291,17 +305,42 @@ public class FileUploadServiceImpl implements FileUploadService { log.debug("Chunk {} saved successfully. Uploaded: {}/{}", chunkNumber, session.getChunksUploaded(), session.getTotalChunks()); - if (session.getChunksUploaded().equals(session.getTotalChunks())) { + boolean isLastChunk = session.getChunksUploaded().equals(session.getTotalChunks()); + + if (isLastChunk) { log.info("All chunks uploaded for session {}. Starting assembly...", session.getUploadId()); - sessionRepository.save(session); - assembleFileAsync(session); + + String finalFilePath = assembleFileSynchronously(session); + + if (session.getFileType().startsWith("image")) { + checkForDuplicatesSynchronously(finalFilePath); + } + + session.setStatus(UploadStatus.COMPLETED); + session.setFilePath(finalFilePath); + + completeFileProcessingAsync(session); + } else { sessionRepository.save(session); } return UploadProgressResponse.fromSession(session); + } catch (DuplicateImageException e) { + if (chunkPath != null) { + cleanupFailedChunk(chunkPath); + } + + log.warn("Duplicate image found for session {}: {}", + session.getUploadId(), e.getMessage()); + + session.setStatus(UploadStatus.FAILED); + session.setLastError(e.getMessage()); + sessionRepository.save(session); + + throw e; } catch (Exception e) { if (chunkPath != null) { cleanupFailedChunk(chunkPath); @@ -318,6 +357,29 @@ public class FileUploadServiceImpl implements FileUploadService { } } + private String assembleFileSynchronously(FileUploadSession session) throws IOException { + Path finalFilePath = prepareFinalFile(session); + + validateAllChunksExist(session); + mergeChunksToFile(session, finalFilePath); + validateFinalFile(session, finalFilePath); + + return finalFilePath.toString(); + } + + private void checkForDuplicatesSynchronously(String filePath) + throws IOException { + Path path = Paths.get(filePath); + Map hash = imageHashService.calculateHash(path); + List duplicates = fileSimilarityService.findDuplicatedByHash( + hash.get("hi"), hash.get("low")); + + if (!duplicates.isEmpty()) { + throw new DuplicateImageException("Duplicate", duplicates.get(0).getFileId(), + duplicates.get(0).getUserId()); + } + } + private UploadProgressResponse handleExistingChunk(FileUploadSession session, Integer chunkNumber, MultipartFile chunkFile) throws IOException { @@ -325,25 +387,24 @@ public class FileUploadServiceImpl implements FileUploadService { Path chunkPath = Paths.get(existingPath); if (!Files.exists(chunkPath)) { - log.warn("Chunk file missing, re-uploading: {}", chunkPath); session.getChunkPaths().remove(chunkNumber); session.setChunksUploaded(session.getChunksUploaded() - 1); sessionRepository.save(session); + return processChunk(session, chunkNumber, chunkFile); } long existingSize = Files.size(chunkPath); + if (existingSize != chunkFile.getSize()) { - log.warn("Chunk size mismatch, re-uploading: {} != {}", - existingSize, chunkFile.getSize()); Files.deleteIfExists(chunkPath); session.getChunkPaths().remove(chunkNumber); session.setChunksUploaded(session.getChunksUploaded() - 1); sessionRepository.save(session); + return processChunk(session, chunkNumber, chunkFile); } - log.debug("Chunk {} already uploaded and valid", chunkNumber); return UploadProgressResponse.fromSession(session); } @@ -402,7 +463,7 @@ public class FileUploadServiceImpl implements FileUploadService { session.getUploadId(), session.getFileName()); Path finalFilePath = null; - String checksum = null; + String checksum; try { finalFilePath = prepareFinalFile(session); @@ -428,6 +489,8 @@ public class FileUploadServiceImpl implements FileUploadService { fileEntityService.createFromUploadSession(session, checksum); log.info("FileEntity successfully created for session: {}", session.getUploadId()); + } catch (DuplicateImageException e) { + throw new DuplicateImageException("Duplicate", e.duplicateFileId(), e.userId()); } catch (Exception e) { log.error("Failed to create FileEntity for session {} (file uploaded, but metadata not saved): {}", session.getUploadId(), e.getMessage()); @@ -591,44 +654,10 @@ public class FileUploadServiceImpl implements FileUploadService { private void handleAssemblyFailure(FileUploadSession session, Exception e) { session.setStatus(UploadStatus.FAILED); session.setLastError(e.getMessage()); + sessionRepository.save(session); - - log.error("File assembly failed for session {}: {}", - session.getUploadId(), e.getMessage()); } - - -// @Override -// @Transactional -// public void cleanupExpiredSessions() { -// log.info("Starting cleanup of expired upload sessions"); -// -// LocalDateTime expiryThreshold = LocalDateTime.now().minusHours(sessionExpiryHours); -// -// try { -// var expiredSessions = sessionRepository.findByStatusInAndCreatedAtBefore( -// Set.of(UploadStatus.INITIATED, UploadStatus.UPLOADING, UploadStatus.FAILED), -// expiryThreshold -// ); -// -// int cleanedCount = 0; -// for (FileUploadSession session : expiredSessions) { -// session.setStatus(UploadStatus.FAILED); -// session.setLastError("Session expired during cleanup"); -// sessionRepository.save(session); -// -// CompletableFuture.runAsync(() -> cleanupSessionFiles(session)); -// cleanedCount++; -// } -// -// log.info("Cleaned up {} expired upload sessions", cleanedCount); -// -// } catch (Exception e) { -// log.error("Error during session cleanup", e); -// } -// } - private Path getChunkDirectory(Long userId, String uploadId) { return storageRoot.resolve("temp") .resolve("user") diff --git a/src/main/java/ru/soune/nocopy/service/file/NotExistFilesCleanupService.java b/src/main/java/ru/soune/nocopy/service/file/NotExistFilesCleanupService.java new file mode 100644 index 0000000..1be7b9e --- /dev/null +++ b/src/main/java/ru/soune/nocopy/service/file/NotExistFilesCleanupService.java @@ -0,0 +1,35 @@ +package ru.soune.nocopy.service.file; + +import lombok.RequiredArgsConstructor; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.scheduling.annotation.Scheduled; +import org.springframework.stereotype.Service; +import org.springframework.transaction.annotation.Transactional; +import ru.soune.nocopy.entity.file.FileEntity; +import ru.soune.nocopy.entity.file.FileStatus; +import ru.soune.nocopy.repository.FileEntityRepository; + +import java.nio.file.Files; +import java.nio.file.Paths; +import java.util.List; + +@Service +@RequiredArgsConstructor +public class NotExistFilesCleanupService { + + @Autowired + private FileEntityRepository fileEntityRepository; + + @Transactional + @Scheduled(fixedDelay = 30000) + public void cleanUpNotExistFiles() { + List fileEntities = fileEntityRepository.searchFileEntityByStatus(FileStatus.DELETED); + + for (FileEntity fileEntity : fileEntities) { + boolean exists = Files.exists(Paths.get(fileEntity.getFilePath())); + if (!exists) { + fileEntityRepository.delete(fileEntity); + } + } + } +} diff --git a/src/main/resources/application.yaml b/src/main/resources/application.yaml index 171cca5..0a3f966 100644 --- a/src/main/resources/application.yaml +++ b/src/main/resources/application.yaml @@ -21,6 +21,10 @@ spring: # mail.smtp.starttls.enable: false # mail.debug: true + flyway: + enabled: true + baseline-on-migrate: true + datasource: url: jdbc:postgresql://${POSTGRES_HOST}:${POSTGRES_PORT}/${POSTGRES_DB} username: ${POSTGRES_USER} diff --git a/src/main/resources/db/migration/V1__add_image_hash_prefix_index.sql b/src/main/resources/db/migration/V1__add_image_hash_prefix_index.sql new file mode 100644 index 0000000..2700671 --- /dev/null +++ b/src/main/resources/db/migration/V1__add_image_hash_prefix_index.sql @@ -0,0 +1,2 @@ +CREATE INDEX idx_image_hash_prefix + ON image_hashes ((hash_value >> 48)); \ No newline at end of file