package ru.soune.nocopy.service; import com.vrt.NoCopyFileService; import com.vrt.fileprotection.FileProtector; import com.vrt.fileprotection.NoCopyCheckResult; import com.vrt.fileprotection.audio.AudioCheckResult; import com.vrt.fileprotection.documents.DocumentCheckResult; import com.vrt.fileprotection.image.phash.PHash; import com.vrt.fileprotection.image.phash.PerceptualHashHelper; import lombok.RequiredArgsConstructor; import lombok.extern.slf4j.Slf4j; import org.hibernate.validator.internal.util.stereotypes.Lazy; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.beans.factory.annotation.Value; import org.springframework.data.domain.Page; import org.springframework.data.domain.PageImpl; import org.springframework.data.domain.Pageable; import org.springframework.stereotype.Service; import ru.soune.nocopy.dto.file.SimilarFileDTO; import ru.soune.nocopy.dto.file.SimilarityFilter; import ru.soune.nocopy.entity.file.FileEntity; import ru.soune.nocopy.entity.file.ImageHashEntity; import ru.soune.nocopy.exception.DuplicateImageException; import ru.soune.nocopy.repository.*; import ru.soune.nocopy.util.FileUtil; import java.io.FileInputStream; import java.io.InputStream; import java.nio.file.Path; import java.nio.file.Paths; import java.security.MessageDigest; import java.util.*; import java.util.stream.Collectors; @Service @RequiredArgsConstructor @Slf4j public class FileSimilarityService { private final ImageSimilarityRepository repository; private final ImageHashRepository hashRepository; private final FileUtil fileUtil; private final AuthTokenRepository authTokenRepository; private final FileEntityRepository fileEntityRepository; @Value("${server.baseurl}") private String baseUrl; public List findSimilarFiles(String fileId) { var imageHashEntity = hashRepository.findById(fileId) .orElseThrow(() -> new RuntimeException("Hash not found")); Long hash64Hi = imageHashEntity.getHash64Hi(); Long hash64Lo = imageHashEntity.getHash64Lo(); List candidates = repository.findCandidates(fileId); return candidates.stream() .map(c -> { Long cHi = c.getHash64Hi(); Long cLo = c.getHash64Lo(); int hamming = fileUtil.hamming64(hash64Hi, hash64Lo, cHi, cLo); String level; if (hamming <= 5) { level = "DUPLICATE"; } else if (hamming <= 12) { level = "SIMILAR"; } else { level = "DIFFERENT"; } return SimilarFileDTO.builder() .fileId(c.getId()) .ownerId(c.getUserId()) .originalFileName(c.getOriginalFileName()) .fileSize(c.getFileSize()) .hammingDistance(hamming) .similarityLevel(level) .build(); }) .sorted((a, b) -> Integer.compare(a.getHammingDistance(), b.getHammingDistance())) .toList(); } public List findDuplicateByHammingDistance(String fileId, int hammingDistance, int duplicate, int similar) { List candidates = repository.findCandidates(fileId); Optional hashOptional = hashRepository.findById(fileId); if (hashOptional.isEmpty()) { return Collections.emptyList(); } var imageHashEntity = hashOptional.get(); Long hash64Hi = imageHashEntity.getHash64Hi(); Long hash64Lo = imageHashEntity.getHash64Lo(); return candidates.stream() .map(c -> { Long cHi = c.getHash64Hi(); Long cLo = c.getHash64Lo(); int hamming = fileUtil.hamming64(hash64Hi, hash64Lo, cHi, cLo); return new AbstractMap.SimpleEntry<>(c, hamming); }) .filter(entry -> entry.getValue() <= hammingDistance) .sorted((a, b) -> Integer.compare(a.getValue(), b.getValue())) .map(entry -> { SimilarImageProjection similarImageProjection = entry.getKey(); int hamming = entry.getValue(); String level; if (hamming <= duplicate) { level = "DUPLICATE"; } else if (hamming <= similar) { level = "SIMILAR"; } else { level = "DIFFERENT"; } return SimilarFileDTO.builder() .fileId(similarImageProjection.getId()) .originalFileName(similarImageProjection.getOriginalFileName()) .fileSize(similarImageProjection.getFileSize()) .hammingDistance(hamming) .similarityLevel(level) .ownerId(similarImageProjection.getUserId()) .build(); }) .toList(); } public List findDuplicatedByHash(Long hash64Hi, Long hash64Lo) { List duplicates = repository.findExactDuplicates(hash64Hi,hash64Lo); if (duplicates.isEmpty()) { return new ArrayList<>(); } return duplicates; } public void hasDuplicatesByHash(String path, Long userId, String mimeType) throws Exception { String hash = calculateFileHash(path); List fileEntityList = fileEntityRepository.findByUserIdAndMimeType(userId, mimeType); for (FileEntity file : fileEntityList) { if (calculateFileHash(file.getFilePath()).equals(hash)) { throw new DuplicateImageException("Duplicate", file.getId(), file.getUserId()); } } } @Autowired @Lazy private NoCopyFileService noCopyFileService; public Page findSimilarFiles(String fileId, SimilarityFilter filter, Pageable pageable, String authToken) throws Exception { FileEntity fileEntity = fileEntityRepository.findByFileId(fileId); String mimeType = fileEntity.getMimeType(); List allResults = new ArrayList<>(); if (mimeType.equals("image")) { var imageHashEntity = hashRepository.findById(fileId) .orElseThrow(() -> new RuntimeException("Hash not found")); Long hash64Hi = imageHashEntity.getHash64Hi(); Long hash64Lo = imageHashEntity.getHash64Lo(); List candidates = authToken.equals("all") ? repository.findCandidates(fileId): repository.findCandidatesFromUserFiles(fileId, authTokenRepository.findUserIdByToken(authToken)); List similarityLevels = (filter != null && filter.getSimilarityLevels() != null) ? filter.getSimilarityLevels() : List.of("DUPLICATE", "SIMILAR", "DIFFERENT"); allResults = candidates.stream() .map(c -> createSimilarFileResponse(c, hash64Hi, hash64Lo)) .filter(response -> similarityLevels.contains(response.getSimilarityLevel())) .sorted(Comparator.comparingInt(SimilarFileDTO::getHammingDistance)) .collect(Collectors.toList()); } else { // Long userId = fileEntity.getUserId(); // List fileEntityList = fileEntityRepository.findByUserIdAndMimeType(userId, mimeType); // NoCopyCheckResult noCopyCheckResult; // // for (FileEntity file : fileEntityList) { // Path path = Paths.get(file.getFilePath()); // if (mimeType.equals("document")) { // noCopyCheckResult = (DocumentCheckResult) noCopyFileService.checkFile( // path.toFile(), FileProtector.Type.valueOf(file.getMimeType().toUpperCase())); // } else { // noCopyCheckResult = (AudioCheckResult) noCopyFileService.checkFile( // path.toFile(), FileProtector.Type.valueOf(file.getMimeType().toUpperCase())); // } // // switch (noCopyCheckResult) { // case DocumentCheckResult.Success success -> { // FileProtector.FileInfo info = success.getInfo(); // allResults.add(buildDTO(file)); // } // case DocumentCheckResult.Failed failed -> { // String message = failed.getMessage(); // } // default -> throw new IllegalStateException("Unexpected result"); // } // } Long userId = fileEntity.getUserId(); List fileEntityList = fileEntityRepository.findByUserIdAndMimeType(userId, mimeType); List errors = new ArrayList<>(); for (FileEntity file : fileEntityList) { try { Path path = Paths.get(file.getFilePath()); FileProtector.Type fileType = file.getMimeType().equals("document") ? FileProtector.Type.DOC: FileProtector.Type.AUDIO; NoCopyCheckResult noCopyCheckResult = noCopyFileService.checkFile(path.toFile(), fileType); switch (noCopyCheckResult) { case DocumentCheckResult.Success success -> { FileProtector.FileInfo info = success.getInfo(); allResults.add(buildDTO(file)); log.debug("File processed successfully: {}", file.getFilePath()); } case AudioCheckResult.Success success -> { FileProtector.FileInfo info = success.getInfo(); allResults.add(buildDTO(file)); log.debug("File processed successfully: {}", file.getFilePath()); } case DocumentCheckResult.Failed failed -> { String errorMessage = String.format("Document check failed for %s: %s", file.getFilePath(), failed.getMessage()); log.warn(errorMessage); errors.add(errorMessage); } case AudioCheckResult.Failed failed -> { String errorMessage = String.format("Audio check failed for %s: %s", file.getFilePath(), failed.getMessage()); log.warn(errorMessage); errors.add(errorMessage); } default -> throw new IllegalStateException("Unexpected result type: " + noCopyCheckResult.getClass().getSimpleName()); } } catch (Exception e) { String errorMessage = String.format("Error processing file %s: %s", file.getFilePath(), e.getMessage()); log.error(errorMessage, e); errors.add(errorMessage); } } } int total = allResults.size(); int page = (pageable != null) ? pageable.getPageNumber() : 0; int size = (pageable != null) ? pageable.getPageSize() : 20; int fromIndex = Math.min(page * size, total); int toIndex = Math.min(fromIndex + size, total); List pageContent = allResults.subList(fromIndex, toIndex); return new PageImpl<>(pageContent, pageable, total); } private String calculateFileHash(String path) throws Exception { MessageDigest digest = MessageDigest.getInstance("SHA-256"); try (InputStream is = new FileInputStream(path)) { byte[] buffer = new byte[8192]; int read; while ((read = is.read(buffer)) > 0) { digest.update(buffer, 0, read); } } byte[] hashBytes = digest.digest(); return Base64.getEncoder().encodeToString(hashBytes); } private SimilarFileDTO createSimilarFileResponse(SimilarImageProjection similarImageProjection, Long hash64Hi, Long hash64Lo) { Long imageProjectionHash64Hi = similarImageProjection.getHash64Hi(); Long similarImageProjectionHash64Lo = similarImageProjection.getHash64Lo(); int hamming = PerceptualHashHelper.INSTANCE.hammingDistance(new PHash(hash64Hi, hash64Lo), new PHash(imageProjectionHash64Hi, similarImageProjectionHash64Lo)); String level; if (hamming <= 6) { level = "DUPLICATE"; } else if (hamming <= 12) { level = "SIMILAR"; } else { level = "DIFFERENT"; } return SimilarFileDTO.builder() .fileId(similarImageProjection.getId()) .ownerId(similarImageProjection.getUserId()) .originalFileName(similarImageProjection.getOriginalFileName().replace("." + similarImageProjection.getExtension(), "") + "_nocopy_protected" + "." + similarImageProjection.getExtension()) .fileSize(similarImageProjection.getFileSize()) .hammingDistance(hamming) .similarityLevel(level) .supportId(similarImageProjection.getSupportId()) .uploadDate(similarImageProjection.getUploadDate()) .status(similarImageProjection.getProtectionStatus()) .url(baseUrl + "/api/files/protected/" + similarImageProjection.getId()) .build(); } private SimilarFileDTO buildDTO(FileEntity fileEntity) { return SimilarFileDTO.builder() .fileId(fileEntity.getId()) .ownerId(fileEntity.getUserId()) .originalFileName(fileEntity.getOriginalFileName().replace("." + fileEntity.getFileExtension(), "") + "_nocopy_protected" + "." + fileEntity.getFileExtension()) .fileSize(fileEntity.getFileSize()) .uploadDate(fileEntity.getCreatedAt()) .status(fileEntity.getProtectionStatus()) .build(); } }