backdev/no-copy
1
0
Fork 0
Code Issues Pull Requests 5 Actions Packages Projects Releases Wiki Activity

dev add check hash for all types,exclude video
Test Workflow / test (push) Successful in 4s
Details

Browse Source
This commit is contained in:
vladp
2026-02-16 16:41:29 +07:00
parent d74571b7cb
commit 31544084ca
2 changed files with 83 additions and 19 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/main/java/ru/soune/nocopy/service/FileSimilarityService.java
+80 -19
View File
@@ -11,13 +11,14 @@ import org.springframework.data.domain.Pageable;
import org.springframework.stereotype.Service;
import ru.soune.nocopy.dto.file.SimilarFileDTO;
import ru.soune.nocopy.dto.file.SimilarityFilter;
import ru.soune.nocopy.entity.file.FileEntity;
import ru.soune.nocopy.entity.file.ImageHashEntity;
import ru.soune.nocopy.repository.AuthTokenRepository;
import ru.soune.nocopy.repository.ImageHashRepository;
import ru.soune.nocopy.repository.ImageSimilarityRepository;
import ru.soune.nocopy.repository.SimilarImageProjection;
import ru.soune.nocopy.repository.*;
import ru.soune.nocopy.util.FileUtil;
import java.io.FileInputStream;
import java.io.InputStream;
import java.security.MessageDigest;
import java.util.*;
import java.util.stream.Collectors;
@@ -137,25 +138,43 @@ public class FileSimilarityService {
return duplicates;
}
public Page<SimilarFileDTO> findSimilarFiles(String fileId, SimilarityFilter filter, Pageable pageable, String authToken) {
var imageHashEntity = hashRepository.findById(fileId)
.orElseThrow(() -> new RuntimeException("Hash not found"));
private final FileEntityRepository fileEntityRepository;
Long hash64Hi = imageHashEntity.getHash64Hi();
Long hash64Lo = imageHashEntity.getHash64Lo();
public Page<SimilarFileDTO> findSimilarFiles(String fileId, SimilarityFilter filter, Pageable pageable,
String authToken) throws Exception {
FileEntity fileEntity = fileEntityRepository.findByFileId(fileId);
String mimeType = fileEntity.getMimeType();
List<SimilarFileDTO> allResults = new ArrayList<>();
List<SimilarImageProjection> candidates = authToken.equals("all") ? repository.findCandidates(fileId):
repository.findCandidatesFromUserFiles(fileId, authTokenRepository.findUserIdByToken(authToken));
if (mimeType.equals("image")) {
var imageHashEntity = hashRepository.findById(fileId)
.orElseThrow(() -> new RuntimeException("Hash not found"));
List<String> similarityLevels = (filter != null && filter.getSimilarityLevels() != null)
? filter.getSimilarityLevels()
: List.of("DUPLICATE", "SIMILAR", "DIFFERENT");
Long hash64Hi = imageHashEntity.getHash64Hi();
Long hash64Lo = imageHashEntity.getHash64Lo();
List<SimilarImageProjection> candidates = authToken.equals("all") ? repository.findCandidates(fileId):
repository.findCandidatesFromUserFiles(fileId, authTokenRepository.findUserIdByToken(authToken));
List<SimilarFileDTO> allResults = candidates.stream()
.map(c -> createSimilarFileResponse(c, hash64Hi, hash64Lo))
.filter(response -> similarityLevels.contains(response.getSimilarityLevel()))
.sorted(Comparator.comparingInt(SimilarFileDTO::getHammingDistance))
.collect(Collectors.toList());
List<String> similarityLevels = (filter != null && filter.getSimilarityLevels() != null)
? filter.getSimilarityLevels()
: List.of("DUPLICATE", "SIMILAR", "DIFFERENT");
allResults = candidates.stream()
.map(c -> createSimilarFileResponse(c, hash64Hi, hash64Lo))
.filter(response -> similarityLevels.contains(response.getSimilarityLevel()))
.sorted(Comparator.comparingInt(SimilarFileDTO::getHammingDistance))
.collect(Collectors.toList());
} else {
String hash = calculateFileHash(fileEntity);
Long userId = fileEntity.getUserId();
List<FileEntity> fileEntityList = fileEntityRepository.findByUserIdAndMimeType(userId, mimeType);
for (FileEntity file : fileEntityList) {
if (calculateFileHash(file).equals(hash)) {
allResults.add(buildDTO(file));
}
}
}
int total = allResults.size();
int page = (pageable != null) ? pageable.getPageNumber() : 0;
@@ -169,6 +188,35 @@ public class FileSimilarityService {
return new PageImpl<>(pageContent, pageable, total);
}
private String calculateFileHash(FileEntity fileEntity) throws Exception {
MessageDigest digest = MessageDigest.getInstance("SHA-256");
// if (fileEntity.getStatus() == FileStatus.CLOUD) {
// File tempFile = cloudStorageService.readFileFromStorageByPath(fileEntity.getFilePath());
// try (InputStream is = new FileInputStream(tempFile)) {
// byte[] buffer = new byte[8192];
// int read;
// while ((read = is.read(buffer)) > 0) {
// digest.update(buffer, 0, read);
// }
// } finally {
// tempFile.delete();
// }
// } else {
try (InputStream is = new FileInputStream(fileEntity.getFilePath())) {
byte[] buffer = new byte[8192];
int read;
while ((read = is.read(buffer)) > 0) {
digest.update(buffer, 0, read);
}
// }
}
byte[] hashBytes = digest.digest();
return Base64.getEncoder().encodeToString(hashBytes);
}
private SimilarFileDTO createSimilarFileResponse(SimilarImageProjection similarImageProjection,
Long hash64Hi, Long hash64Lo) {
Long imageProjectionHash64Hi = similarImageProjection.getHash64Hi();
@@ -198,4 +246,17 @@ public class FileSimilarityService {
.url(baseUrl + "/api/files/protected/" + similarImageProjection.getId())
.build();
}
private SimilarFileDTO buildDTO(FileEntity fileEntity) {
return SimilarFileDTO.builder()
.fileId(fileEntity.getId())
.ownerId(fileEntity.getUserId())
.originalFileName(fileEntity.getOriginalFileName())
.fileSize(fileEntity.getFileSize())
.supportId(Long.valueOf(fileEntity.getSupportId()))
.uploadDate(fileEntity.getCreatedAt())
.status(fileEntity.getProtectionStatus())
.url(baseUrl + "/api/files/protected/" + fileEntity.getId())
.build();
}
}