diff --git a/build.gradle b/build.gradle index 7e7901d..c9e4f45 100644 --- a/build.gradle +++ b/build.gradle @@ -21,6 +21,9 @@ configurations { repositories { mavenCentral() + flatDir { + dirs 'libs' + } } dependencies { @@ -51,6 +54,9 @@ dependencies { testImplementation 'org.springframework.boot:spring-boot-starter-test' testImplementation 'org.mockito:mockito-core:5.3.1' testRuntimeOnly 'org.junit.platform:junit-platform-launcher' + + implementation name: 'testlib-fat-0.2.1-all' + } tasks.named('test') { diff --git a/libs/testlib-fat-0.2.1-all.jar b/libs/testlib-fat-0.2.1-all.jar new file mode 100644 index 0000000..c911c08 Binary files /dev/null and b/libs/testlib-fat-0.2.1-all.jar differ diff --git a/src/main/java/ru/soune/nocopy/configuration/ApplicationConfig.java b/src/main/java/ru/soune/nocopy/configuration/ApplicationConfig.java index 85ae0b8..cce0f0d 100644 --- a/src/main/java/ru/soune/nocopy/configuration/ApplicationConfig.java +++ b/src/main/java/ru/soune/nocopy/configuration/ApplicationConfig.java @@ -1,17 +1,49 @@ package ru.soune.nocopy.configuration; +import com.vrt.AudioFilePathProvider; +import com.vrt.fileprotection.FileProtector; +import com.vrt.fileprotection.audio.AudioLocalSearch; +import com.vrt.fileprotection.image.ImageLocalSearch; +import com.vrt.fileprotection.image.ImageUniqueCheck; +import com.vrt.fileprotection.image.phash.PerceptualHashHelper; +import lombok.AllArgsConstructor; import org.springframework.boot.autoconfigure.EnableAutoConfiguration; import org.springframework.context.annotation.Bean; import org.springframework.context.annotation.Configuration; import org.springframework.security.crypto.bcrypt.BCryptPasswordEncoder; import org.springframework.security.crypto.password.PasswordEncoder; +import java.util.Collections; + @Configuration @EnableAutoConfiguration +@AllArgsConstructor public class ApplicationConfig { @Bean PasswordEncoder passwordEncoder() { return new BCryptPasswordEncoder(); } + + @Bean + public com.vrt. + NoCopyFileService noCopyFileService( + FileProtector.FileProvider fileProvider, + FileProtector.ProcessingListener processingListener, + ImageUniqueCheck imageUniqueCheck, + ImageLocalSearch imageLocalSearch, + AudioLocalSearch audioLocalSearch, + AudioFilePathProvider audioFilePathProvider) { + + return new com.vrt.NoCopyFileService( + Collections.emptyList(), + fileProvider, + processingListener, + imageUniqueCheck, + imageLocalSearch, + audioLocalSearch, + audioFilePathProvider + ); + } } + diff --git a/src/main/java/ru/soune/nocopy/configuration/file/NoCopyInitializer.java b/src/main/java/ru/soune/nocopy/configuration/file/NoCopyInitializer.java new file mode 100644 index 0000000..39eef84 --- /dev/null +++ b/src/main/java/ru/soune/nocopy/configuration/file/NoCopyInitializer.java @@ -0,0 +1,19 @@ +package ru.soune.nocopy.configuration.file; + +import lombok.AllArgsConstructor; +import org.springframework.boot.context.event.ApplicationReadyEvent; +import org.springframework.context.event.EventListener; +import org.springframework.stereotype.Component; +import ru.soune.nocopy.service.file.FileProcessingOrchestrator; + +@Component +@AllArgsConstructor +public class NoCopyInitializer { + + private final FileProcessingOrchestrator orchestrator; + + @EventListener(ApplicationReadyEvent.class) + public void initializeOnStartup() { + orchestrator.initializeProcessingQueue(); + } +} \ No newline at end of file diff --git a/src/main/java/ru/soune/nocopy/controller/ApiController.java b/src/main/java/ru/soune/nocopy/controller/ApiController.java index c8261ed..21b2667 100644 --- a/src/main/java/ru/soune/nocopy/controller/ApiController.java +++ b/src/main/java/ru/soune/nocopy/controller/ApiController.java @@ -1,5 +1,8 @@ package ru.soune.nocopy.controller; +import com.vrt.NoCopyFileService; +import com.vrt.fileprotection.FileProtector; +import com.vrt.fileprotection.NoCopyCheckResult; import lombok.RequiredArgsConstructor; import lombok.extern.slf4j.Slf4j; import org.springframework.core.io.Resource; @@ -25,6 +28,7 @@ import ru.soune.nocopy.dto.file.CompleteUploadResponse; import ru.soune.nocopy.dto.file.FileEntityResponse; import ru.soune.nocopy.dto.file.UploadProgress; import ru.soune.nocopy.entity.file.FileStatus; +import ru.soune.nocopy.entity.file.ProtectionStatus; import ru.soune.nocopy.entity.file.UploadStatus; import ru.soune.nocopy.exception.*; import ru.soune.nocopy.handler.*; @@ -33,7 +37,9 @@ import ru.soune.nocopy.service.FileSimilarityService; import ru.soune.nocopy.service.auth.AuthService; import ru.soune.nocopy.service.file.FileEntityService; import ru.soune.nocopy.service.file.FileUploadService; +import ru.soune.nocopy.util.FileUtil; +import java.io.File; import java.io.IOException; import java.nio.file.Files; import java.nio.file.Path; @@ -61,6 +67,10 @@ public class ApiController { private final FileEntityRepository fileEntityRepository; + private final NoCopyFileService noCopyFileService; + + private final FileUtil fileUtil; + @PostMapping("/v{version}/data") public ResponseEntity handlePostRequest(@RequestBody BaseRequest request, @PathVariable("version") int version) { @@ -138,7 +148,7 @@ public class ApiController { SimilarityFilter filter = SimilarityFilter.builder() .similarityLevels(similarityLevels) .build(); - Page similarFiles = fileSimilarityService.findSimilarFiles(fileId, filter, pageable); + Page similarFiles = fileSimilarityService.findSimilarFiles(fileId, filter, pageable); String messageDesc; MessageCode success; @@ -194,7 +204,8 @@ public class ApiController { .uploadId(uploadId) .build(); - return ResponseEntity.ok().body(new BaseResponse(20004, MessageCode.FILE_UPLOAD_ERROR.getCode(), + return ResponseEntity.ok().body(new BaseResponse( + 20004, MessageCode.FILE_UPLOAD_ERROR.getCode(), "Failed to get upload progress: " + e.getMessage(), responseBody)); } } @@ -353,6 +364,43 @@ public class ApiController { } } + @GetMapping("/protect/{fileId}") + public ResponseEntity protect( @PathVariable(required = false) String fileId) { + Optional optionalFileEntity = fileEntityRepository.findById(fileId); + + if (!optionalFileEntity.isPresent()) { + return ResponseEntity.notFound().build(); + } + FileEntity fileEntity = optionalFileEntity.get(); + FileProtector.FileInfo fileInfo = fileUtil.createFileInfo(fileEntity); + + noCopyFileService.addFile(fileInfo); + + fileEntity.setProtectionStatus(ProtectionStatus.PROCESSING); + fileEntityRepository.save(fileEntity); + + return ResponseEntity.ok().build(); + } + + @GetMapping("/check/{fileId}/{type}") + public ResponseEntity check(@PathVariable(required = false) String fileId, + @PathVariable(required = false) String type) { + Optional optionalFileEntity = fileEntityRepository.findById(fileId); + if (!optionalFileEntity.isPresent()) { + return ResponseEntity.notFound().build(); + } + + FileEntity fileEntity = optionalFileEntity.get(); + + Path path = Paths.get(fileEntity.getProtectedFilePath()); + File file = path.toFile(); + + NoCopyCheckResult noCopyCheckResult = noCopyFileService.checkFile(file, + FileProtector.Type.valueOf(type.toUpperCase())); + + return ResponseEntity.ok().body(noCopyCheckResult); + } + private ResponseEntity checkForDuplicates(String uploadId) throws IOException { Optional uploadedFile = fileEntityRepository.findByUploadSessionId(uploadId); @@ -361,7 +409,7 @@ public class ApiController { } FileEntity fileEntity = uploadedFile.get(); - List similarFiles = fileSimilarityService.findSimilarFiles(fileEntity.getId()); + List similarFiles = fileSimilarityService.findSimilarFiles(fileEntity.getId()); if (hasDuplicate(similarFiles)) { return handleDuplicate(fileEntity, similarFiles); @@ -370,11 +418,11 @@ public class ApiController { return null; } - private boolean hasDuplicate(List similarFiles) { + private boolean hasDuplicate(List similarFiles) { return similarFiles.stream().anyMatch(f -> f.getHammingDistance() <= 5); } - private ResponseEntity handleDuplicate(FileEntity fileEntity, List similarFiles) + private ResponseEntity handleDuplicate(FileEntity fileEntity, List similarFiles) throws IOException { fileEntityService.deleteFromDisk(fileEntity); @@ -460,16 +508,4 @@ public class ApiController { } return contentType; } - - private boolean isPreviewSupported(String mimeType) { - if (mimeType == null) { - return false; - } - - return mimeType.startsWith("image") || - mimeType.startsWith("text") || - mimeType.equals("pdf") || - mimeType.startsWith("video") || - mimeType.startsWith("audio"); - } } \ No newline at end of file diff --git a/src/main/java/ru/soune/nocopy/dto/file/SimilarFileResponse.java b/src/main/java/ru/soune/nocopy/dto/file/SimilarFileDTO.java similarity index 82% rename from src/main/java/ru/soune/nocopy/dto/file/SimilarFileResponse.java rename to src/main/java/ru/soune/nocopy/dto/file/SimilarFileDTO.java index f7aae6e..45fb75c 100644 --- a/src/main/java/ru/soune/nocopy/dto/file/SimilarFileResponse.java +++ b/src/main/java/ru/soune/nocopy/dto/file/SimilarFileDTO.java @@ -5,10 +5,11 @@ import lombok.Value; @Value @Builder -public class SimilarFileResponse { +public class SimilarFileDTO { String fileId; String originalFileName; Long fileSize; Integer hammingDistance; String similarityLevel; + Long ownerId; } diff --git a/src/main/java/ru/soune/nocopy/entity/ImageProtection.java b/src/main/java/ru/soune/nocopy/entity/ImageProtection.java deleted file mode 100644 index eef9e95..0000000 --- a/src/main/java/ru/soune/nocopy/entity/ImageProtection.java +++ /dev/null @@ -1,49 +0,0 @@ -package ru.soune.nocopy.entity; - -import jakarta.persistence.*; -import lombok.*; -import org.springframework.data.annotation.CreatedDate; -import org.springframework.data.jpa.domain.support.AuditingEntityListener; - -import java.time.LocalDateTime; - -@Entity -@Table(name = "image_protection") -@Getter -@Setter -@NoArgsConstructor -@AllArgsConstructor -@ToString -@EntityListeners(AuditingEntityListener.class) -public class ImageProtection { - - @Id - @GeneratedValue(strategy = GenerationType.IDENTITY) - private Long protectionId; - - @ManyToOne(fetch = FetchType.LAZY) - @JoinColumn(name = "user_id", nullable = false) - @ToString.Exclude - private User user; - - @ManyToOne(fetch = FetchType.LAZY) - @JoinColumn(name = "content_id", nullable = false) - @ToString.Exclude - private UserContent content; - - @Column(name = "protection_method", nullable = false, length = 50) - private String protectionMethod; - - @Column(name = "protection_level", nullable = false) - private Integer protectionLevel; - - @Column(name = "is_active", nullable = false) - private Boolean isActive = true; - - @CreatedDate - @Column(name = "applied_at", nullable = false, updatable = false) - private LocalDateTime appliedAt; - - @Column(name = "metadata", columnDefinition = "JSON") - private String metadata; -} diff --git a/src/main/java/ru/soune/nocopy/entity/User.java b/src/main/java/ru/soune/nocopy/entity/User.java index 0841027..b893637 100644 --- a/src/main/java/ru/soune/nocopy/entity/User.java +++ b/src/main/java/ru/soune/nocopy/entity/User.java @@ -84,10 +84,4 @@ public class User { @OneToMany(mappedBy = "user", cascade = CascadeType.ALL, orphanRemoval = true) @JsonIgnore @ToString.Exclude - private List violations = new ArrayList<>(); - - @OneToMany(mappedBy = "user", cascade = CascadeType.ALL, orphanRemoval = true) - @JsonIgnore - @ToString.Exclude - private List imageProtections = new ArrayList<>(); -} + private List violations = new ArrayList<>();} diff --git a/src/main/java/ru/soune/nocopy/entity/file/FileEntity.java b/src/main/java/ru/soune/nocopy/entity/file/FileEntity.java index 16e25a4..dd193af 100644 --- a/src/main/java/ru/soune/nocopy/entity/file/FileEntity.java +++ b/src/main/java/ru/soune/nocopy/entity/file/FileEntity.java @@ -66,6 +66,20 @@ public class FileEntity { @Column(name = "updated_at") private LocalDateTime updatedAt; + @Column(name = "protected_file_path") + private String protectedFilePath; + + @Column(name = "protection_status") + @Enumerated(EnumType.STRING) + private ProtectionStatus protectionStatus; + + @Column(name = "protected_at") + private LocalDateTime protectedAt; + + @Column(name = "signature") + @JsonIgnore + private String signature; + @OneToOne(mappedBy = "file", cascade = CascadeType.ALL, orphanRemoval = true) @JsonIgnore private ImageHashEntity imageHash; @@ -75,9 +89,19 @@ public class FileEntity { if (this.status == null) { this.status = FileStatus.ACTIVE; } + + if (this.protectionStatus == null) { + this.protectionStatus = ProtectionStatus.NOT_PROTECTED; + } + if (this.createdAt == null) { this.createdAt = LocalDateTime.now(); } this.updatedAt = LocalDateTime.now(); } + + @PreUpdate + public void preUpdate() { + this.updatedAt = LocalDateTime.now(); + } } diff --git a/src/main/java/ru/soune/nocopy/entity/file/FileType.java b/src/main/java/ru/soune/nocopy/entity/file/FileType.java index 8844013..5c17f24 100644 --- a/src/main/java/ru/soune/nocopy/entity/file/FileType.java +++ b/src/main/java/ru/soune/nocopy/entity/file/FileType.java @@ -10,7 +10,7 @@ public enum FileType { IMAGE("image", Arrays.asList("jpg", "jpeg", "png", "gif", "bmp", "webp", "jfif")), VIDEO("video", Arrays.asList("mp4", "avi", "mov", "wmv", "flv", "mkv", "webm", "m4v", "mpg", "mpeg", "3gp", "3g2", "f4v", "m2ts", "mts", "vob", "ogv", "divx")), - AUDIO("audio", Arrays.asList("mp3", "wav", "flac")); + AUDIO("audio", List.of("wav")); private final String displayName; private final List allowedExtensions; diff --git a/src/main/java/ru/soune/nocopy/entity/file/ImageHashEntity.java b/src/main/java/ru/soune/nocopy/entity/file/ImageHashEntity.java index 37f1b5a..f18a572 100644 --- a/src/main/java/ru/soune/nocopy/entity/file/ImageHashEntity.java +++ b/src/main/java/ru/soune/nocopy/entity/file/ImageHashEntity.java @@ -26,10 +26,10 @@ public class ImageHashEntity { private FileEntity file; @Column(name = "hash64_hi") - private Integer hash64Hi; + private Long hash64Hi; @Column(name = "hash64_lo") - private Integer hash64Lo; + private Long hash64Lo; @Column(name = "hash_algorithm", nullable = false) private String hashAlgorithm; diff --git a/src/main/java/ru/soune/nocopy/entity/file/ProtectionStatus.java b/src/main/java/ru/soune/nocopy/entity/file/ProtectionStatus.java new file mode 100644 index 0000000..18fc726 --- /dev/null +++ b/src/main/java/ru/soune/nocopy/entity/file/ProtectionStatus.java @@ -0,0 +1,9 @@ +package ru.soune.nocopy.entity.file; + +public enum ProtectionStatus { + NOT_PROTECTED, + PROCESSING, + PROTECTED, + FAILED, + FAILED_SAVE +} diff --git a/src/main/java/ru/soune/nocopy/handler/validator/RegRequestValidator.java b/src/main/java/ru/soune/nocopy/handler/validator/RegRequestValidator.java index e01df8e..22e4b62 100644 --- a/src/main/java/ru/soune/nocopy/handler/validator/RegRequestValidator.java +++ b/src/main/java/ru/soune/nocopy/handler/validator/RegRequestValidator.java @@ -1,14 +1,11 @@ package ru.soune.nocopy.handler.validator; -import org.apache.commons.validator.routines.DomainValidator; import org.apache.commons.validator.routines.EmailValidator; import org.springframework.stereotype.Component; import org.springframework.validation.Errors; import org.springframework.validation.Validator; import ru.soune.nocopy.dto.RegRequest; -import java.net.InetAddress; -import java.net.UnknownHostException; import java.util.*; @Component @@ -141,15 +138,6 @@ public class RegRequestValidator implements Validator { errors.rejectValue("password", "password.contains.spaces", "Password cannot contain spaces"); } - -// recomment if need complexity -// checkPasswordComplexity(password, errors); - -// reccoment if need check simply standart password -// checkCommonPasswords(password, errors); - -// reccoment if need check simply standart password -// checkForSequences(password, errors); } private void checkPasswordComplexity(String password, Errors errors) { diff --git a/src/main/java/ru/soune/nocopy/repository/FileEntityRepository.java b/src/main/java/ru/soune/nocopy/repository/FileEntityRepository.java index e35ef2f..26d6df6 100644 --- a/src/main/java/ru/soune/nocopy/repository/FileEntityRepository.java +++ b/src/main/java/ru/soune/nocopy/repository/FileEntityRepository.java @@ -6,14 +6,15 @@ import org.springframework.data.repository.query.Param; import org.springframework.stereotype.Repository; import ru.soune.nocopy.entity.file.FileEntity; import ru.soune.nocopy.entity.file.FileStatus; +import ru.soune.nocopy.entity.file.ProtectionStatus; import java.util.List; import java.util.Optional; @Repository public interface FileEntityRepository extends JpaRepository { - List findByUserId(Long userId); + Optional findByUserIdAndChecksum(Long userId, String imageHash); List findByUserIdAndStatus(Long userId, FileStatus status); @@ -22,16 +23,28 @@ public interface FileEntityRepository extends JpaRepository Optional findByUploadSessionId(String uploadSessionId); + List findByProtectionStatus(ProtectionStatus protectionStatus); + boolean existsByFilePath(String filePath); @Query("SELECT SUM(f.fileSize) FROM FileEntity f WHERE f.userId = :userId AND f.status = 'ACTIVE'") Long getTotalSizeByUserId(@Param("userId") Long userId); + @Query("SELECT f FROM FileEntity f WHERE f.protectionStatus = null AND f.status = 'ACTIVE' OR " + + "f.protectionStatus = 'NOT_PROTECTED' AND f.status = 'ACTIVE'") + List findAllActiveFilesAndNotProtected(); + @Query("SELECT f FROM FileEntity f WHERE f.userId = :userId AND f.originalFileName LIKE %:keyword%") List searchByFileName(@Param("userId") Long userId, @Param("keyword") String keyword); @Query("SELECT f FROM FileEntity f WHERE f.status = :status") List searchFileEntityByStatus(@Param("status") FileStatus status); + @Query("SELECT f FROM FileEntity f WHERE f.id = :fileId") + FileEntity findByFileId(@Param("fileId") String fileId); + + @Query("SELECT f FROM FileEntity f WHERE f.signature = :signature") + FileEntity findBySignature(@Param("signature") String signature); + long countByUserId(Long userId); } diff --git a/src/main/java/ru/soune/nocopy/repository/ImageHashRepository.java b/src/main/java/ru/soune/nocopy/repository/ImageHashRepository.java index ee0aac2..48b4378 100644 --- a/src/main/java/ru/soune/nocopy/repository/ImageHashRepository.java +++ b/src/main/java/ru/soune/nocopy/repository/ImageHashRepository.java @@ -4,7 +4,11 @@ import org.springframework.data.jpa.repository.JpaRepository; import org.springframework.stereotype.Repository; import ru.soune.nocopy.entity.file.ImageHashEntity; +import java.util.List; + @Repository public interface ImageHashRepository extends JpaRepository { void deleteByFileId(String fileId); + List findByFileId(String fileId); + ImageHashEntity findByHash64HiAndHash64Lo(Long hash64Hi, Long hash64Lo); } diff --git a/src/main/java/ru/soune/nocopy/repository/ImageSimilarityRepository.java b/src/main/java/ru/soune/nocopy/repository/ImageSimilarityRepository.java index a6ef973..cd57871 100644 --- a/src/main/java/ru/soune/nocopy/repository/ImageSimilarityRepository.java +++ b/src/main/java/ru/soune/nocopy/repository/ImageSimilarityRepository.java @@ -48,7 +48,7 @@ public interface ImageSimilarityRepository """, nativeQuery = true) List findExactDuplicates( - @Param("hash64Hi") Integer hash64_hi, - @Param("hash64Lo") Integer hash64_lo); + @Param("hash64Hi") Long hash64_hi, + @Param("hash64Lo") Long hash64_lo); } diff --git a/src/main/java/ru/soune/nocopy/repository/SimilarImageProjection.java b/src/main/java/ru/soune/nocopy/repository/SimilarImageProjection.java index cac73fb..4bd444e 100644 --- a/src/main/java/ru/soune/nocopy/repository/SimilarImageProjection.java +++ b/src/main/java/ru/soune/nocopy/repository/SimilarImageProjection.java @@ -1,8 +1,8 @@ package ru.soune.nocopy.repository; public interface SimilarImageProjection { - Integer getHash64Hi(); - Integer getHash64Lo(); + Long getHash64Hi(); + Long getHash64Lo(); String getId(); Long getUserId(); String getOriginalFileName(); diff --git a/src/main/java/ru/soune/nocopy/service/FileSimilarityService.java b/src/main/java/ru/soune/nocopy/service/FileSimilarityService.java index 9ec4051..b882ab2 100644 --- a/src/main/java/ru/soune/nocopy/service/FileSimilarityService.java +++ b/src/main/java/ru/soune/nocopy/service/FileSimilarityService.java @@ -5,12 +5,14 @@ import org.springframework.data.domain.Page; import org.springframework.data.domain.PageImpl; import org.springframework.data.domain.Pageable; import org.springframework.stereotype.Service; -import ru.soune.nocopy.dto.file.SimilarFileResponse; +import ru.soune.nocopy.dto.file.SimilarFileDTO; import ru.soune.nocopy.dto.file.SimilarityFilter; import ru.soune.nocopy.repository.ImageHashRepository; import ru.soune.nocopy.repository.ImageSimilarityRepository; import ru.soune.nocopy.repository.SimilarImageProjection; +import ru.soune.nocopy.util.FileUtil; +import java.util.AbstractMap; import java.util.ArrayList; import java.util.Comparator; import java.util.List; @@ -19,26 +21,28 @@ import java.util.stream.Collectors; @Service @RequiredArgsConstructor public class FileSimilarityService { + private final ImageSimilarityRepository repository; private final ImageHashRepository hashRepository; - public List findSimilarFiles(String fileId) { + private final FileUtil fileUtil; + + public List findSimilarFiles(String fileId) { var imageHashEntity = hashRepository.findById(fileId) .orElseThrow(() -> new RuntimeException("Hash not found")); - Integer hash64Hi = imageHashEntity.getHash64Hi(); - Integer hash64Lo = imageHashEntity.getHash64Lo(); + Long hash64Hi = imageHashEntity.getHash64Hi(); + Long hash64Lo = imageHashEntity.getHash64Lo(); - List candidates = - repository.findCandidates(fileId); + List candidates = repository.findCandidates(fileId); return candidates.stream() .map(c -> { - Integer cHi = c.getHash64Hi(); - Integer cLo = c.getHash64Lo(); + Long cHi = c.getHash64Hi(); + Long cLo = c.getHash64Lo(); - int hamming = hamming64(hash64Hi, hash64Lo, cHi, cLo); + int hamming = fileUtil.hamming64(hash64Hi, hash64Lo, cHi, cLo); String level; if (hamming <= 5) { @@ -49,7 +53,7 @@ public class FileSimilarityService { level = "DIFFERENT"; } - return SimilarFileResponse.builder() + return SimilarFileDTO.builder() .fileId(c.getId()) .originalFileName(c.getOriginalFileName()) .fileSize(c.getFileSize()) @@ -62,7 +66,53 @@ public class FileSimilarityService { .toList(); } - public List findDuplicatedByHash(Integer hash64Hi, Integer hash64Lo) { + public List findDuplicateByHammingDistance(String fileId, int hammingDistance, + int duplicate, int similar) { + List candidates = repository.findCandidates(fileId); + + var imageHashEntity = hashRepository.findById(fileId) + .orElseThrow(() -> new RuntimeException("Hash not found")); + + Long hash64Hi = imageHashEntity.getHash64Hi(); + Long hash64Lo = imageHashEntity.getHash64Lo(); + + return candidates.stream() + .map(c -> { + Long cHi = c.getHash64Hi(); + Long cLo = c.getHash64Lo(); + + int hamming = fileUtil.hamming64(hash64Hi, hash64Lo, cHi, cLo); + + return new AbstractMap.SimpleEntry<>(c, hamming); + }) + .filter(entry -> entry.getValue() <= hammingDistance) + .sorted((a, b) -> Integer.compare(a.getValue(), b.getValue())) + .map(entry -> { + SimilarImageProjection similarImageProjection = entry.getKey(); + int hamming = entry.getValue(); + + String level; + if (hamming <= duplicate) { + level = "DUPLICATE"; + } else if (hamming <= similar) { + level = "SIMILAR"; + } else { + level = "DIFFERENT"; + } + + return SimilarFileDTO.builder() + .fileId(similarImageProjection.getFileId()) + .originalFileName(similarImageProjection.getOriginalFileName()) + .fileSize(similarImageProjection.getFileSize()) + .hammingDistance(hamming) + .similarityLevel(level) + .ownerId(similarImageProjection.getUserId()) + .build(); + }) + .toList(); + } + + public List findDuplicatedByHash(Long hash64Hi, Long hash64Lo) { List duplicates = repository.findExactDuplicates(hash64Hi,hash64Lo); if (duplicates.isEmpty()) { @@ -72,12 +122,12 @@ public class FileSimilarityService { return duplicates; } - public Page findSimilarFiles(String fileId, SimilarityFilter filter, Pageable pageable) { + public Page findSimilarFiles(String fileId, SimilarityFilter filter, Pageable pageable) { var imageHashEntity = hashRepository.findById(fileId) .orElseThrow(() -> new RuntimeException("Hash not found")); - Integer hash64Hi = imageHashEntity.getHash64Hi(); - Integer hash64Lo = imageHashEntity.getHash64Lo(); + Long hash64Hi = imageHashEntity.getHash64Hi(); + Long hash64Lo = imageHashEntity.getHash64Lo(); List candidates = repository.findCandidates(fileId); @@ -85,10 +135,10 @@ public class FileSimilarityService { ? filter.getSimilarityLevels() : List.of("DUPLICATE", "SIMILAR", "DIFFERENT"); - List allResults = candidates.stream() + List allResults = candidates.stream() .map(c -> createSimilarFileResponse(c, hash64Hi, hash64Lo)) .filter(response -> similarityLevels.contains(response.getSimilarityLevel())) - .sorted(Comparator.comparingInt(SimilarFileResponse::getHammingDistance)) + .sorted(Comparator.comparingInt(SimilarFileDTO::getHammingDistance)) .collect(Collectors.toList()); int total = allResults.size(); @@ -98,17 +148,17 @@ public class FileSimilarityService { int fromIndex = Math.min(page * size, total); int toIndex = Math.min(fromIndex + size, total); - List pageContent = allResults.subList(fromIndex, toIndex); + List pageContent = allResults.subList(fromIndex, toIndex); return new PageImpl<>(pageContent, pageable, total); } - private SimilarFileResponse createSimilarFileResponse(SimilarImageProjection similarImageProjection, - Integer hash64Hi, Integer hash64Lo) { - Integer imageProjectionHash64Hi = similarImageProjection.getHash64Hi(); - Integer similarImageProjectionHash64Lo = similarImageProjection.getHash64Lo(); + private SimilarFileDTO createSimilarFileResponse(SimilarImageProjection similarImageProjection, + Long hash64Hi, Long hash64Lo) { + Long imageProjectionHash64Hi = similarImageProjection.getHash64Hi(); + Long similarImageProjectionHash64Lo = similarImageProjection.getHash64Lo(); - int hamming = hamming64(hash64Hi, hash64Lo, imageProjectionHash64Hi, similarImageProjectionHash64Lo); + int hamming = fileUtil.hamming64(hash64Hi, hash64Lo, imageProjectionHash64Hi, similarImageProjectionHash64Lo); String level; if (hamming <= 5) { @@ -119,7 +169,7 @@ public class FileSimilarityService { level = "DIFFERENT"; } - return SimilarFileResponse.builder() + return SimilarFileDTO.builder() .fileId(similarImageProjection.getId()) .originalFileName(similarImageProjection.getOriginalFileName()) .fileSize(similarImageProjection.getFileSize()) @@ -127,9 +177,4 @@ public class FileSimilarityService { .similarityLevel(level) .build(); } - - private int hamming64(int aHi, int aLo, int bHi, int bLo) { - return Integer.bitCount(aHi ^ bHi) - + Integer.bitCount(aLo ^ bLo); - } } diff --git a/src/main/java/ru/soune/nocopy/service/ImageHashService.java b/src/main/java/ru/soune/nocopy/service/ImageHashService.java index b7b6882..672bf78 100644 --- a/src/main/java/ru/soune/nocopy/service/ImageHashService.java +++ b/src/main/java/ru/soune/nocopy/service/ImageHashService.java @@ -1,5 +1,7 @@ package ru.soune.nocopy.service; +import com.vrt.fileprotection.image.phash.PHash; +import com.vrt.fileprotection.image.phash.PerceptualHashHelper; import lombok.RequiredArgsConstructor; import lombok.extern.slf4j.Slf4j; import org.springframework.stereotype.Service; @@ -7,15 +9,10 @@ import ru.soune.nocopy.entity.file.FileEntity; import ru.soune.nocopy.entity.file.ImageHashEntity; import ru.soune.nocopy.repository.ImageHashRepository; -import javax.imageio.ImageIO; -import java.awt.*; -import java.awt.image.BufferedImage; +import java.io.File; import java.io.IOException; -import java.io.InputStream; -import java.nio.file.Files; import java.nio.file.Path; import java.time.LocalDateTime; -import java.util.Arrays; import java.util.Map; @Service @@ -25,15 +22,19 @@ public class ImageHashService { private final ImageHashRepository repository; - public Map calculateHash(Path imagePath) throws IOException { - long hash64 = computePhash64(imagePath); + public Map calculateHash(Path imagePath) throws IOException { + File file = imagePath.toFile(); + PHash pHash = PerceptualHashHelper.INSTANCE.generateDCTPerceptualHash(file); + Long firstPart = pHash.getFirstPart(); + Long secondPart = pHash.getSecondPart(); + Map hash = Map.of( + "hi", firstPart, + "low", secondPart); - return Map.of( - "hi", high32(hash64), - "low", low32(hash64)); + return hash; } - public void create(FileEntity file, Map stringIntegerMap) { + public void create(FileEntity file, Map stringIntegerMap) { ImageHashEntity entity = ImageHashEntity.builder() .file(file) .hash64Hi(stringIntegerMap.get("hi")) @@ -44,111 +45,4 @@ public class ImageHashService { repository.save(entity); } - - private long computePhash64(Path path) throws IOException { - try (InputStream is = Files.newInputStream(path)) { - BufferedImage image = ImageIO.read(is); - if (image == null) throw new IOException("Cannot read image file: " + path); - - BufferedImage gray = toGrayScale(image); - BufferedImage resized = resizeImage(gray, 32, 32); - - double[][] dct = applyDCT(resized); - double[][] topLeft = extractTopLeft8x8(dct); - double median = calculateMedian(topLeft, true); - - return create64BitHashLong(topLeft, median); - } - } - - private BufferedImage toGrayScale(BufferedImage image) { - BufferedImage gray = new BufferedImage(image.getWidth(), image.getHeight(), BufferedImage.TYPE_BYTE_GRAY); - Graphics2D g = gray.createGraphics(); - g.drawImage(image, 0, 0, null); - g.dispose(); - return gray; - } - - private BufferedImage resizeImage(BufferedImage image, int width, int height) { - BufferedImage resized = new BufferedImage(width, height, BufferedImage.TYPE_BYTE_GRAY); - Graphics2D g = resized.createGraphics(); - g.setRenderingHint(RenderingHints.KEY_INTERPOLATION, RenderingHints.VALUE_INTERPOLATION_BILINEAR); - g.drawImage(image, 0, 0, width, height, null); - g.dispose(); - return resized; - } - - private double[][] applyDCT(BufferedImage image) { - int w = image.getWidth(); - int h = image.getHeight(); - double[][] pixels = new double[h][w]; - - for (int y = 0; y < h; y++) { - for (int x = 0; x < w; x++) { - int gray = image.getRGB(x, y) & 0xFF; - pixels[y][x] = gray; - } - } - - double[][] dct = new double[h][w]; - for (int u = 0; u < h; u++) { - double au = u == 0 ? 1.0 / Math.sqrt(h) : Math.sqrt(2.0 / h); - for (int v = 0; v < w; v++) { - double av = v == 0 ? 1.0 / Math.sqrt(w) : Math.sqrt(2.0 / w); - double sum = 0.0; - for (int x = 0; x < h; x++) { - for (int y = 0; y < w; y++) { - sum += pixels[x][y] * - Math.cos((2 * x + 1) * u * Math.PI / (2.0 * h)) * - Math.cos((2 * y + 1) * v * Math.PI / (2.0 * w)); - } - } - dct[u][v] = au * av * sum; - } - } - return dct; - } - - private double[][] extractTopLeft8x8(double[][] dct) { - double[][] block = new double[8][8]; - for (int i = 0; i < 8; i++) System.arraycopy(dct[i], 0, block[i], 0, 8); - return block; - } - - private double calculateMedian(double[][] block, boolean excludeDC) { - double[] vals = new double[63]; - int idx = 0; - for (int i = 0; i < 8; i++) { - for (int j = 0; j < 8; j++) { - if (i == 0 && j == 0 && excludeDC) continue; - vals[idx++] = block[i][j]; - } - } - Arrays.sort(vals); - return (vals.length % 2 == 0) ? - (vals[vals.length / 2 - 1] + vals[vals.length / 2]) / 2.0 : - vals[vals.length / 2]; - } - - private long create64BitHashLong(double[][] block, double median) { - long hash = 0L; - - for (int i = 0; i < 8; i++) { - for (int j = 0; j < 8; j++) { - if (i == 0 && j == 0) continue; - hash <<= 1; - if (block[i][j] > median) hash |= 1; - } - } - - return hash; - } - - private int high32(long hash64) { - return (int) (hash64 >>> 32); - } - - private int low32(long hash64) { - return (int) hash64; - } } diff --git a/src/main/java/ru/soune/nocopy/service/YandexSearchService.java b/src/main/java/ru/soune/nocopy/service/YandexSearchService.java index d0f4710..9a8e9b2 100644 --- a/src/main/java/ru/soune/nocopy/service/YandexSearchService.java +++ b/src/main/java/ru/soune/nocopy/service/YandexSearchService.java @@ -12,7 +12,6 @@ import ru.soune.nocopy.dto.BaseResponse; import ru.soune.nocopy.dto.MessageCode; import ru.soune.nocopy.dto.file.YandexSearchResponse; import ru.soune.nocopy.entity.file.FileEntity; -import ru.soune.nocopy.exception.FileEntityNotFoundException; import ru.soune.nocopy.exception.NotValidFieldException; import ru.soune.nocopy.repository.FileEntityRepository; diff --git a/src/main/java/ru/soune/nocopy/service/file/FileEntityService.java b/src/main/java/ru/soune/nocopy/service/file/FileEntityService.java index bfc8123..ac0f35d 100644 --- a/src/main/java/ru/soune/nocopy/service/file/FileEntityService.java +++ b/src/main/java/ru/soune/nocopy/service/file/FileEntityService.java @@ -9,6 +9,7 @@ import ru.soune.nocopy.dto.file.FileResponse; import ru.soune.nocopy.entity.file.FileEntity; import ru.soune.nocopy.entity.file.FileStatus; import ru.soune.nocopy.entity.file.FileUploadSession; +import ru.soune.nocopy.entity.file.ProtectionStatus; import ru.soune.nocopy.exception.DuplicateImageException; import ru.soune.nocopy.exception.FileEntityNotFoundException; import ru.soune.nocopy.repository.FileEntityRepository; @@ -16,6 +17,7 @@ import ru.soune.nocopy.repository.SimilarImageProjection; import ru.soune.nocopy.service.FileSimilarityService; import ru.soune.nocopy.service.ImageHashService; +import java.io.File; import java.io.IOException; import java.nio.file.Files; import java.nio.file.Path; @@ -47,7 +49,7 @@ public class FileEntityService { throw new IOException("File not found on disk: " + filePath); } - Map imageHash = Map.of(); + Map imageHash = Map.of(); if (session.getFileType().startsWith("image")) { imageHash = imageHashService.calculateHash(filePath); @@ -110,6 +112,39 @@ public class FileEntityService { return convertToResponse(fileEntity, version); } + @Transactional(readOnly = true) + public FileEntityResponse getByFilePath(String filePath, int version) { + FileEntity fileEntity = fileEntityRepository.findByFilePath(filePath) + .orElseThrow(() -> new FileEntityNotFoundException("Path: " + filePath)); + + return convertToResponse(fileEntity, version); + } + + + @Transactional(readOnly = true) + public FileResponse getAllUserFiles(Long userId, int version) { + List fileEntities = fileEntityRepository.findByUserIdAndStatus( + userId, FileStatus.ACTIVE); + + List files = fileEntities.stream() + .map(file -> convertToResponse(file, version)) + .collect(Collectors.toList()); + + long totalSize = fileEntities.stream() + .mapToLong(FileEntity::getFileSize) + .sum(); + + return FileResponse.builder() + .files(files) + .totalCount(files.size()) + .totalSize(totalSize) + .formattedTotalSize(formatFileSize(totalSize)) + .page(1) + .pageSize(files.size()) + .build(); + } + + @Transactional(readOnly = true) public FileResponse getUserFiles(Long userId, int page, int pageSize, int version) { List allFiles = fileEntityRepository.findByUserIdAndStatus( @@ -178,6 +213,95 @@ public class FileEntityService { return totalSize != null ? totalSize : 0L; } + public void changeStatus(ProtectionStatus newStatus, String fileId) { + FileEntity fileEntity = fileEntityRepository.findById(fileId) + .orElseThrow(() -> new FileEntityNotFoundException(fileId)); + fileEntity.setProtectionStatus(newStatus); + + fileEntityRepository.save(fileEntity); + } + + @Transactional + public void writeProtectedFile(String id, byte[] data, String fileExt) throws IOException { + FileEntity fileEntity = fileEntityRepository.findById(id) + .orElseThrow(() -> new RuntimeException("File not found: " + id)); + + String extension = determineFileExtension(fileExt, fileEntity); + Path protectedFilePath = prepareProtectedPath(fileEntity, extension); + + if (Files.exists(protectedFilePath)) { + Files.delete(protectedFilePath); + } + + Files.write(protectedFilePath, data); + + fileEntity.setProtectedFilePath(protectedFilePath.toString()); + fileEntity.setProtectedAt(LocalDateTime.now()); + fileEntity.setUpdatedAt(LocalDateTime.now()); + fileEntity.setFileExtension(extension); + fileEntity.setProtectionStatus(ProtectionStatus.PROTECTED); + + fileEntityRepository.save(fileEntity); + } + + public FileEntity findBySignature(String signature) { + return fileEntityRepository.findBySignature(signature); + } + + public File getFileById(String id) { + try { + FileEntity fileEntity = fileEntityRepository.findById(id).orElseThrow(() -> + new RuntimeException("File not found: " + id)); + + File file = new File(fileEntity.getFilePath()); + + if (!file.exists()) { + throw new RuntimeException("File not found on disk: " + fileEntity.getFilePath()); + } + + return file; + } catch (Exception e) { + log.error("Error getting file: {}", id, e); + return null; + } + } + + public Path prepareProtectedPath(FileEntity fileEntity, String extension) throws IOException { + Path originalPath = Paths.get(fileEntity.getFilePath()); + + String pathStr = originalPath.toString(); + pathStr = pathStr.replaceFirst("/uploads/uploads/", "/uploads/protected/"); + + Path protectedPath = Paths.get(pathStr); + String fileName = protectedPath.getFileName().toString(); + + if (extension != null) { + String nameWithoutExt = fileName; + int lastDotIndex = fileName.lastIndexOf('.'); + if (lastDotIndex > 0) { + nameWithoutExt = fileName.substring(0, lastDotIndex); + } + fileName = nameWithoutExt + "." + extension; + + protectedPath = protectedPath.getParent().resolve(fileName); + } + + Files.createDirectories(protectedPath.getParent()); + + return protectedPath; + } + + public void updateSignature(String signature, String fileId) throws IOException { + FileEntity fileEntity = fileEntityRepository.findByFileId(fileId); + fileEntity.setSignature(signature); + fileEntity.setProtectionStatus(ProtectionStatus.PROTECTED); + fileEntity.setUpdatedAt(LocalDateTime.now()); + fileEntity.setProtectedAt(LocalDateTime.now()); + fileEntity.setProtectedFilePath(prepareProtectedPath(fileEntity, fileEntity.getFileExtension()).toString()); + + fileEntityRepository.save(fileEntity); + } + private boolean checkFileExistsOnDisk(String filePath) { try { return Files.exists(Paths.get(filePath)); @@ -187,6 +311,14 @@ public class FileEntityService { } } + private String determineFileExtension(String fileExt, FileEntity fileEntity) { + if (fileExt != null && !fileExt.trim().isEmpty()) { + return fileExt; + } else { + return fileEntity.getFileExtension(); + } + } + private FileEntityResponse convertToResponse(FileEntity fileEntity, int version) { boolean existsOnDisk = checkFileExistsOnDisk(fileEntity.getFilePath()); diff --git a/src/main/java/ru/soune/nocopy/service/file/FileProcessingOrchestrator.java b/src/main/java/ru/soune/nocopy/service/file/FileProcessingOrchestrator.java new file mode 100644 index 0000000..5c4df44 --- /dev/null +++ b/src/main/java/ru/soune/nocopy/service/file/FileProcessingOrchestrator.java @@ -0,0 +1,53 @@ +package ru.soune.nocopy.service.file; + +import com.vrt.NoCopyFileService; +import com.vrt.fileprotection.FileProtector; +import lombok.RequiredArgsConstructor; +import lombok.extern.slf4j.Slf4j; +import org.springframework.scheduling.annotation.Scheduled; +import org.springframework.stereotype.Service; +import ru.soune.nocopy.entity.file.FileEntity; +import ru.soune.nocopy.entity.file.ProtectionStatus; +import ru.soune.nocopy.repository.FileEntityRepository; +import ru.soune.nocopy.util.FileUtil; + +import java.util.List; + +@Slf4j +@Service +@RequiredArgsConstructor +public class FileProcessingOrchestrator { + + private final NoCopyFileService noCopyFileService; + + private final FileEntityRepository fileRepository; + + private final FileUtil fileUtil; + + public void initializeProcessingQueue() { + List filesToProtect = fileRepository.findByProtectionStatus(ProtectionStatus.NOT_PROTECTED); + + for (FileEntity fileEntity : filesToProtect) { + try { + FileProtector.FileInfo fileInfo = fileUtil.createFileInfo(fileEntity); + + noCopyFileService.addFile(fileInfo); + + log.info("Add to query: {}", fileEntity.getOriginalFileName()); + } catch (Exception e) { + log.error("Fail add to query: {}", fileEntity.getId(), e); + } + } + } + + @Scheduled(fixedDelay = 120000) + public void checkNewFilesForProtection() { + List newFiles = fileRepository.findAllActiveFilesAndNotProtected(); + + for (FileEntity fileEntity : newFiles) { + FileProtector.FileInfo fileInfo = fileUtil.createFileInfo(fileEntity); + + noCopyFileService.addFile(fileInfo); + } + } +} diff --git a/src/main/java/ru/soune/nocopy/service/file/impl/AudioFilePathProviderImpl.java b/src/main/java/ru/soune/nocopy/service/file/impl/AudioFilePathProviderImpl.java new file mode 100644 index 0000000..45b8058 --- /dev/null +++ b/src/main/java/ru/soune/nocopy/service/file/impl/AudioFilePathProviderImpl.java @@ -0,0 +1,44 @@ +package ru.soune.nocopy.service.file.impl; + +import com.vrt.AudioFilePathProvider; +import com.vrt.fileprotection.FileProtector; +import lombok.RequiredArgsConstructor; +import lombok.extern.slf4j.Slf4j; +import org.jetbrains.annotations.NotNull; +import org.springframework.stereotype.Service; +import ru.soune.nocopy.entity.file.FileEntity; +import ru.soune.nocopy.repository.FileEntityRepository; +import ru.soune.nocopy.service.file.FileEntityService; + +import java.io.IOException; +import java.util.Optional; + +@Slf4j +@Service +@RequiredArgsConstructor +public class AudioFilePathProviderImpl implements AudioFilePathProvider { + + private final FileEntityService fileEntityService; + + private final FileEntityRepository fileEntityRepository; + + @Override + public @NotNull String providePath(@NotNull FileProtector.FileInfo fileInfo) { + String filePath; + Optional optionalFileEntity = fileEntityRepository.findById(fileInfo.getId()); + + if (optionalFileEntity.isEmpty()) { + throw new NullPointerException(); + } + + FileEntity fileEntity = optionalFileEntity.get(); + + try { + filePath = fileEntityService.prepareProtectedPath(fileEntity, fileEntity.getFileExtension()).toString(); + } catch (IOException e) { + throw new RuntimeException(e); + } + + return filePath; + } +} diff --git a/src/main/java/ru/soune/nocopy/service/file/impl/AudioLocalSearchImpl.java b/src/main/java/ru/soune/nocopy/service/file/impl/AudioLocalSearchImpl.java new file mode 100644 index 0000000..6ac96bb --- /dev/null +++ b/src/main/java/ru/soune/nocopy/service/file/impl/AudioLocalSearchImpl.java @@ -0,0 +1,42 @@ +package ru.soune.nocopy.service.file.impl; + +import com.vrt.fileprotection.FileProtector; +import com.vrt.fileprotection.audio.AudioLocalSearch; +import lombok.RequiredArgsConstructor; +import lombok.extern.slf4j.Slf4j; +import org.jetbrains.annotations.NotNull; +import org.jetbrains.annotations.Nullable; +import org.springframework.stereotype.Service; +import ru.soune.nocopy.entity.file.FileEntity; +import ru.soune.nocopy.service.file.FileEntityService; +import ru.soune.nocopy.util.FileUtil; + +import java.io.IOException; + +@Slf4j +@Service +@RequiredArgsConstructor +public class AudioLocalSearchImpl implements AudioLocalSearch { + + private final FileEntityService fileEntityService; + + private final FileUtil fileUtil; + + @Override + public @Nullable FileProtector.FileInfo findBySignature(@NotNull String signature) { + FileEntity fileEntity = fileEntityService.findBySignature(signature); + + return fileUtil.createFileInfo(fileEntity); + } + + @Override + public void updateSignature(@NotNull String signature, @NotNull FileProtector.FileInfo fileInfo) { + try { + fileEntityService.updateSignature(signature, fileInfo.getId()); + } catch (IOException e) { + throw new RuntimeException(e); + } + } + + +} diff --git a/src/main/java/ru/soune/nocopy/service/file/FileUploadServiceImpl.java b/src/main/java/ru/soune/nocopy/service/file/impl/FileUploadServiceImpl.java similarity index 96% rename from src/main/java/ru/soune/nocopy/service/file/FileUploadServiceImpl.java rename to src/main/java/ru/soune/nocopy/service/file/impl/FileUploadServiceImpl.java index 05b5dcd..bccf5ed 100644 --- a/src/main/java/ru/soune/nocopy/service/file/FileUploadServiceImpl.java +++ b/src/main/java/ru/soune/nocopy/service/file/impl/FileUploadServiceImpl.java @@ -1,5 +1,7 @@ -package ru.soune.nocopy.service.file; +package ru.soune.nocopy.service.file.impl; +import com.vrt.NoCopyFileService; +import com.vrt.fileprotection.FileProtector; import jakarta.annotation.PostConstruct; import lombok.RequiredArgsConstructor; import lombok.extern.slf4j.Slf4j; @@ -23,6 +25,9 @@ import ru.soune.nocopy.repository.FileUploadSessionRepository; import ru.soune.nocopy.repository.SimilarImageProjection; import ru.soune.nocopy.service.FileSimilarityService; import ru.soune.nocopy.service.ImageHashService; +import ru.soune.nocopy.service.file.FileEntityService; +import ru.soune.nocopy.service.file.FileUploadService; +import ru.soune.nocopy.util.FileUtil; import java.io.*; import java.nio.file.*; @@ -73,6 +78,10 @@ public class FileUploadServiceImpl implements FileUploadService { @Autowired private FileEntityService fileEntityService; + private final NoCopyFileService noCopyFileService; + + private final FileUtil fileUtil; + @PostConstruct public void init() { try { @@ -274,11 +283,16 @@ public class FileUploadServiceImpl implements FileUploadService { FileEntity saved = fileEntityRepository.save(fileEntity); - Map hash = imageHashService.calculateHash(filePath); - imageHashService.create(saved, hash); + if (session.getFileType().equals("image")) { + Map hash = imageHashService.calculateHash(filePath); + + imageHashService.create(saved, hash); + } cleanupSessionFiles(session); + noCopyFileService.addFile(fileUtil.createFileInfo(fileEntity)); + log.info("File processing completed for session: {}", session.getUploadId()); } catch (Exception e) { @@ -370,7 +384,8 @@ public class FileUploadServiceImpl implements FileUploadService { private void checkForDuplicatesSynchronously(String filePath) throws IOException { Path path = Paths.get(filePath); - Map hash = imageHashService.calculateHash(path); + Map hash = imageHashService.calculateHash(path); + List duplicates = fileSimilarityService.findDuplicatedByHash( hash.get("hi"), hash.get("low")); @@ -482,11 +497,17 @@ public class FileUploadServiceImpl implements FileUploadService { session.setChecksum(checksum); session.setStatus(UploadStatus.COMPLETED); session.setCompletedAt(LocalDateTime.now()); + sessionRepository.save(session); log.info("Upload session updated to COMPLETED: {}", session.getUploadId()); try { - fileEntityService.createFromUploadSession(session, checksum); + FileEntity fileEntity = fileEntityService.createFromUploadSession(session, checksum); + + FileProtector.FileInfo fileInfo = fileUtil.createFileInfo(fileEntity); + + noCopyFileService.addFile(fileInfo); + log.info("FileEntity successfully created for session: {}", session.getUploadId()); } catch (DuplicateImageException e) { diff --git a/src/main/java/ru/soune/nocopy/service/file/impl/ImageLocalSearchImpl.java b/src/main/java/ru/soune/nocopy/service/file/impl/ImageLocalSearchImpl.java new file mode 100644 index 0000000..79b8963 --- /dev/null +++ b/src/main/java/ru/soune/nocopy/service/file/impl/ImageLocalSearchImpl.java @@ -0,0 +1,54 @@ +package ru.soune.nocopy.service.file.impl; + +import com.vrt.fileprotection.FileProtector; +import com.vrt.fileprotection.image.ImageLocalSearch; +import com.vrt.fileprotection.image.phash.PHash; +import lombok.RequiredArgsConstructor; +import lombok.extern.slf4j.Slf4j; +import org.jetbrains.annotations.NotNull; +import org.jetbrains.annotations.Nullable; +import org.springframework.stereotype.Service; +import ru.soune.nocopy.dto.file.SimilarFileDTO; +import ru.soune.nocopy.entity.file.FileEntity; +import ru.soune.nocopy.entity.file.ImageHashEntity; +import ru.soune.nocopy.repository.FileEntityRepository; +import ru.soune.nocopy.repository.ImageHashRepository; +import ru.soune.nocopy.service.FileSimilarityService; +import ru.soune.nocopy.util.FileUtil; + +import java.util.List; +@Slf4j +@Service +@RequiredArgsConstructor +public class ImageLocalSearchImpl implements ImageLocalSearch { + + private final FileSimilarityService fileSimilarityService; + + private final ImageHashRepository imageHashRepository; + + private final FileUtil fileUtil; + + private final FileEntityRepository fileEntityRepository; + + @Override + public @NotNull List find(@NotNull FileProtector.FileInfo fileInfo) { + List duplicateByHammingDistance = + fileSimilarityService.findDuplicateByHammingDistance(fileInfo.getId(), 10, + 5, 10); + + return duplicateByHammingDistance.stream() + .map(fileUtil::convertToResult) + .toList(); + } + + @Override + public @Nullable FileProtector.FileInfo findByPHash(@NotNull PHash pHash) { + ImageHashEntity imageHashEntity = + imageHashRepository.findByHash64HiAndHash64Lo(pHash.getFirstPart(), + pHash.getSecondPart()); + FileEntity file = fileEntityRepository.findByFileId(imageHashEntity.getFileId()); + + return new FileProtector.FileInfo(FileProtector.Type.IMAGE, imageHashEntity.getFileId(), + String.valueOf(file.getUserId())); + } +} diff --git a/src/main/java/ru/soune/nocopy/service/file/impl/ImageUniqueCheckImpl.java b/src/main/java/ru/soune/nocopy/service/file/impl/ImageUniqueCheckImpl.java new file mode 100644 index 0000000..c883a4d --- /dev/null +++ b/src/main/java/ru/soune/nocopy/service/file/impl/ImageUniqueCheckImpl.java @@ -0,0 +1,34 @@ +package ru.soune.nocopy.service.file.impl; + +import com.vrt.fileprotection.FileProtector; +import com.vrt.fileprotection.OperationResult; +import com.vrt.fileprotection.image.ImageUniqueCheck; +import lombok.RequiredArgsConstructor; +import lombok.extern.slf4j.Slf4j; +import org.jetbrains.annotations.NotNull; +import org.springframework.stereotype.Service; +import ru.soune.nocopy.dto.file.SimilarFileDTO; +import ru.soune.nocopy.service.FileSimilarityService; + +import java.util.List; + +@Slf4j +@Service +@RequiredArgsConstructor +public class ImageUniqueCheckImpl implements ImageUniqueCheck { + + private final FileSimilarityService fileSimilarityService; + + @Override + public @NotNull OperationResult check(@NotNull FileProtector.FileInfo fileInfo) { + String fileId = fileInfo.getId(); + List duplicateByHammingDistance = + fileSimilarityService.findDuplicateByHammingDistance(fileId, 3, 3, 4); + + if (duplicateByHammingDistance == null || duplicateByHammingDistance.isEmpty()) { + return OperationResult.Companion.success(); + } else { + return OperationResult.Companion.failure("Duplicate file"); + } + } +} diff --git a/src/main/java/ru/soune/nocopy/service/file/impl/NoCopyProcessingListenerImpl.java b/src/main/java/ru/soune/nocopy/service/file/impl/NoCopyProcessingListenerImpl.java new file mode 100644 index 0000000..257cf65 --- /dev/null +++ b/src/main/java/ru/soune/nocopy/service/file/impl/NoCopyProcessingListenerImpl.java @@ -0,0 +1,36 @@ +package ru.soune.nocopy.service.file.impl; + +import com.vrt.fileprotection.FileProtector; +import lombok.AllArgsConstructor; +import lombok.extern.slf4j.Slf4j; +import org.springframework.stereotype.Component; +import ru.soune.nocopy.entity.file.ProtectionStatus; +import ru.soune.nocopy.service.file.FileEntityService; + +@Slf4j +@Component +@AllArgsConstructor +public class NoCopyProcessingListenerImpl implements FileProtector.ProcessingListener { + + private final FileEntityService fileEntityService; + + @Override + public void onStartProcessing(FileProtector.FileInfo fileInfo) { + fileEntityService.changeStatus(ProtectionStatus.PROCESSING, fileInfo.getId()); + } + + @Override + public void onProcessingFailed(FileProtector.FileInfo fileInfo) { + fileEntityService.changeStatus(ProtectionStatus.FAILED, fileInfo.getId()); + } + + @Override + public void onSavingFailed(FileProtector.FileInfo fileInfo) { + fileEntityService.changeStatus(ProtectionStatus.FAILED_SAVE, fileInfo.getId()); + } + + @Override + public void onFinish(FileProtector.FileInfo fileInfo) { + fileEntityService.changeStatus(ProtectionStatus.PROTECTED, fileInfo.getId());; + } +} diff --git a/src/main/java/ru/soune/nocopy/service/file/impl/ProtectionFileProviderImpl.java b/src/main/java/ru/soune/nocopy/service/file/impl/ProtectionFileProviderImpl.java new file mode 100644 index 0000000..ffb4883 --- /dev/null +++ b/src/main/java/ru/soune/nocopy/service/file/impl/ProtectionFileProviderImpl.java @@ -0,0 +1,108 @@ +package ru.soune.nocopy.service.file.impl; + +import com.vrt.fileprotection.FileProtector; +import com.vrt.fileprotection.OperationResult; +import lombok.RequiredArgsConstructor; +import lombok.extern.slf4j.Slf4j; +import org.jetbrains.annotations.NotNull; +import org.jetbrains.annotations.Nullable; +import org.springframework.stereotype.Component; +import ru.soune.nocopy.service.file.FileEntityService; + +import java.io.File; +import java.io.IOException; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.Paths; + +@Slf4j +@Component +@RequiredArgsConstructor +public class ProtectionFileProviderImpl implements FileProtector.FileProvider { + + private static final Path SIGNATURE_FILE_PATH = Paths.get("/data/uploads/signature"); + + private final FileEntityService fileEntityService; + + @Nullable + @Override + public File getImageFile(@NotNull String id) { + return fileEntityService.getFileById(id); + } + + @Nullable + @Override + public File getVideoFile(@NotNull String id) { + return fileEntityService.getFileById(id); + } + + @Nullable + @Override + public File getAudioFile(@NotNull String id) { + return fileEntityService.getFileById(id); + } + + @Override + public @Nullable File getSignature() { + File signatureFile = SIGNATURE_FILE_PATH.toFile(); + + if (signatureFile.exists() && signatureFile.isFile() && signatureFile.length() > 0) { + return signatureFile; + } + + return null; + } + + @Override + public @NotNull OperationResult writeSignature(@NotNull byte[] bytes) { + try { + Path directory = SIGNATURE_FILE_PATH.getParent(); + + if (directory != null && !Files.exists(directory)) { + Files.createDirectories(directory); + } + + Files.write(SIGNATURE_FILE_PATH, bytes); + + return OperationResult.Companion.success(); + } catch (IOException e) { + return OperationResult.Companion.failure("Not saved signature file"); + } + } + + @NotNull + @Override + public com.vrt.fileprotection.OperationResult writeAudioFile(@NotNull String id, @NotNull byte[] data, @Nullable String fileExt) { + try { + fileEntityService.writeProtectedFile(id, data, fileExt); + return OperationResult.Companion.success(); + } catch (Exception e) { + log.error("Failed to create protected file: {}", id, e); + return OperationResult.Companion.failure("Failed to create protected file"); + } + } + + @NotNull + @Override + public com.vrt.fileprotection.OperationResult writeVideoFile(@NotNull String id, @NotNull byte[] data, @Nullable String fileExt) { + try { + fileEntityService.writeProtectedFile(id, data, fileExt); + return OperationResult.Companion.success(); + } catch (Exception e) { + log.error("Failed to create protected file: {}", id, e); + return OperationResult.Companion.failure("Failed to create protected file"); + } + } + + @NotNull + @Override + public com.vrt.fileprotection.OperationResult writeImageFile(@NotNull String id, @NotNull byte[] data, @Nullable String fileExt) { + try { + fileEntityService.writeProtectedFile(id, data, fileExt); + return OperationResult.Companion.success(); + } catch (Exception e) { + log.error("Failed to create protected file: {}", id, e); + return OperationResult.Companion.failure("Failed to create protected file"); + } + } +} diff --git a/src/main/java/ru/soune/nocopy/util/FileUtil.java b/src/main/java/ru/soune/nocopy/util/FileUtil.java new file mode 100644 index 0000000..192e1d3 --- /dev/null +++ b/src/main/java/ru/soune/nocopy/util/FileUtil.java @@ -0,0 +1,48 @@ +package ru.soune.nocopy.util; + +import com.vrt.fileprotection.FileProtector; +import com.vrt.fileprotection.image.ImageLocalSearch; +import com.vrt.fileprotection.image.ImageScore; +import org.springframework.stereotype.Service; +import ru.soune.nocopy.dto.file.SimilarFileDTO; +import ru.soune.nocopy.entity.file.FileEntity; + +@Service +public class FileUtil { + public ImageLocalSearch.Result convertToResult(SimilarFileDTO response) { + FileProtector.FileInfo fileInfo = + new FileProtector.FileInfo(FileProtector.Type.IMAGE, response.getFileId(), + String.valueOf(response.getOwnerId())); + ImageScore imageScore = new ImageScore(ImageScore.Rate.valueOf(response.getSimilarityLevel()), + response.getHammingDistance()); + + return new ImageLocalSearch.Result(fileInfo, imageScore); + } + + public FileProtector.FileInfo createFileInfo(FileEntity fileEntity) { + FileProtector.Type type = determineFileType(fileEntity.getMimeType()); + + return new FileProtector.FileInfo(type, fileEntity.getId(), String.valueOf(fileEntity.getUserId())); + } + + public int hamming64(long aHi, long aLo, long bHi, long bLo) { + return Long.bitCount(aHi ^ bHi) + + Long.bitCount(aLo ^ bLo); + } + + private FileProtector.Type determineFileType(String mimeType) { + if (mimeType == null) { + return FileProtector.Type.IMAGE; + } + + if (mimeType.startsWith("image")) { + return FileProtector.Type.IMAGE; + } else if (mimeType.startsWith("video")) { + return FileProtector.Type.VIDEO; + } else if (mimeType.startsWith("audio")) { + return FileProtector.Type.AUDIO; + } else { + return FileProtector.Type.IMAGE; + } + } +}