NCBACK-34 add logic to find similar image in db
Test Workflow / test (push) Successful in 3s

This commit is contained in:
vladp
2026-01-13 22:40:52 +07:00
parent 344ec075d4
commit 8ac73ff045
18 changed files with 394 additions and 188 deletions
+2
View File
@@ -31,6 +31,8 @@ dependencies {
implementation 'org.mapstruct:mapstruct:1.5.5.Final'
implementation 'commons-validator:commons-validator:1.7'
implementation 'org.flywaydb:flyway-core:9.22.0'
implementation 'com.fasterxml.jackson.dataformat:jackson-dataformat-xml:2.17.0'
implementation 'tools.jackson.core:jackson-core:3.0.3'
implementation 'jakarta.xml.bind:jakarta.xml.bind-api:4.0.0'
@@ -15,16 +15,14 @@ import ru.soune.nocopy.dto.BaseRequest;
import ru.soune.nocopy.dto.BaseResponse;
import ru.soune.nocopy.dto.MessageCode;
import ru.soune.nocopy.dto.RegAnswer;
import ru.soune.nocopy.dto.file.ChunkUploadResponse;
import ru.soune.nocopy.dto.file.CompleteUploadResponse;
import ru.soune.nocopy.dto.file.FileEntityResponse;
import ru.soune.nocopy.dto.file.UploadProgress;
import ru.soune.nocopy.dto.file.*;
import ru.soune.nocopy.entity.AuthToken;
import ru.soune.nocopy.entity.file.FileStatus;
import ru.soune.nocopy.entity.file.UploadStatus;
import ru.soune.nocopy.exception.*;
import ru.soune.nocopy.handler.*;
import ru.soune.nocopy.repository.AuthTokenRepository;
import ru.soune.nocopy.service.FileSimilarityService;
import ru.soune.nocopy.service.file.FileEntityService;
import ru.soune.nocopy.service.file.FileUploadService;
@@ -50,6 +48,8 @@ public class ApiController {
private final AuthTokenRepository authTokenRepository;
private final FileSimilarityService fileSimilarityService;
@PostMapping("/v{version}/data")
public ResponseEntity<?> handlePostRequest(@RequestBody BaseRequest request,
@PathVariable("version") int version) {
@@ -127,6 +127,16 @@ public class ApiController {
return ResponseEntity.ok().body(new BaseResponse(20004, MessageCode.SUCCESS.getCode(),
"Chunk uploaded successfully", responseBody));
} catch (DuplicateImageException e) {
log.error("Error uploading chunk", e);
ChunkUploadResponse responseBody = ChunkUploadResponse.builder()
.uploadId(uploadId)
.chunkNumber(chunkNumber)
.build();
return ResponseEntity.ok().body(new BaseResponse(
20004, MessageCode.DUPLICATE_FILE_UPLOAD.getCode(),
"Failed to upload chunk: " + e.getMessage(), responseBody));
} catch (Exception e) {
log.error("Error uploading chunk", e);
@@ -141,6 +151,11 @@ public class ApiController {
}
}
@GetMapping("/files/{fileId}/similar")
public List<SimilarFileResponse> getSimilarFiles(@PathVariable String fileId) {
return fileSimilarityService.findSimilarFiles(fileId);
}
@GetMapping("/v{version}/files/progress/{uploadId}")
public ResponseEntity<BaseResponse> getUploadProgress(
@PathVariable("version") int version,
@@ -8,6 +8,7 @@ public enum MessageCode {
INVALID_TOKEN(2, "Invalid token"),
INVALID_ACTION(2, "Invalid action"),
FILE_UPLOAD_ERROR(2, "File upload error"),
DUPLICATE_FILE_UPLOAD(2, "Duplicate file upload"),
FILE_DOWNLOAD_ERROR(2, "File download error"),
FILE_DOWNLOAD_ERROR_NOT_CORRECT_FIELD(2, "Not correct field"),
IMAGE_FOUND_ERROR(2, "Image found error"),
@@ -0,0 +1,14 @@
package ru.soune.nocopy.dto.file;
import lombok.Builder;
import lombok.Value;
@Value
@Builder
public class SimilarFileResponse {
String fileId;
String originalFileName;
Long fileSize;
Integer hammingDistance;
String similarityLevel;
}
@@ -69,6 +69,7 @@ public class FileUploadSession {
private String extension;
@Column(name = "retry_count")
@Builder.Default
private Integer retryCount = 0;
@Column(name = "completed_at")
@@ -81,6 +82,7 @@ public class FileUploadSession {
)
@MapKeyColumn(name = "chunk_number")
@Column(name = "chunk_path")
@Builder.Default
private Map<Integer, String> chunkPaths = new HashMap<>();
@PrePersist
@@ -0,0 +1,39 @@
package ru.soune.nocopy.entity.file;
import jakarta.persistence.*;
import lombok.AllArgsConstructor;
import lombok.Builder;
import lombok.Data;
import lombok.NoArgsConstructor;
import java.time.LocalDateTime;
@Entity
@Table(name = "image_hashes")
@Data
@NoArgsConstructor
@AllArgsConstructor
@Builder
public class ImageHashEntity {
@Id
@Column(name = "file_id")
private String fileId;
@OneToOne(fetch = FetchType.LAZY)
@MapsId
@JoinColumn(name = "file_id")
private FileEntity file;
@Column(name = "hash64_hi")
private Integer hash64Hi;
@Column(name = "hash64_lo")
private Integer hash64Lo;
@Column(name = "hash_algorithm", nullable = false)
private String hashAlgorithm;
@Column(name = "created_at")
private LocalDateTime createdAt;
}
@@ -0,0 +1,7 @@
package ru.soune.nocopy.exception;
public class DuplicateImageException extends RuntimeException {
public DuplicateImageException(String message) {
super(message);
}
}
@@ -209,6 +209,7 @@ public class FileUploadHandler implements RequestHandler {
.orElseThrow(() -> new UploadSessionNotFoundException(fileRequest.getUploadId()));
Map<String, Boolean> chunkStatus = new HashMap<>();
for (int i = 0; i < session.getTotalChunks(); i++) {
chunkStatus.put("chunk_" + i, session.getChunkPaths().containsKey(i));
}
@@ -223,7 +224,6 @@ public class FileUploadHandler implements RequestHandler {
return new BaseResponse(request.getMsgId(), MessageCode.SUCCESS.getCode(),
MessageCode.SUCCESS.getDescription(), response);
} catch (UploadSessionNotFoundException e) {
return new BaseResponse(request.getMsgId(), MessageCode.FILE_NOT_FOUND.getCode(),
"Upload session not found", null);
@@ -0,0 +1,9 @@
package ru.soune.nocopy.repository;
import org.springframework.data.jpa.repository.JpaRepository;
import org.springframework.stereotype.Repository;
import ru.soune.nocopy.entity.file.ImageHashEntity;
@Repository
public interface ImageHashRepository extends JpaRepository<ImageHashEntity, String> {
}
@@ -0,0 +1,35 @@
package ru.soune.nocopy.repository;
import org.springframework.data.jpa.repository.JpaRepository;
import org.springframework.data.jpa.repository.Query;
import org.springframework.data.repository.query.Param;
import org.springframework.stereotype.Repository;
import ru.soune.nocopy.entity.file.FileEntity;
import java.util.List;
@Repository
public interface ImageSimilarityRepository
extends JpaRepository<FileEntity, String> {
@Query(value = """
SELECT
f.id AS similarFileId,
f.original_file_name AS originalFileName,
f.file_size AS fileSize,
h.hash64_hi AS hash64Hi,
h.hash64_lo AS hash64Lo
FROM image_hashes ref
JOIN image_hashes h
ON ref.file_id <> h.file_id
JOIN file_entities f
ON f.id = h.file_id
WHERE ref.file_id = :fileId
""",
nativeQuery = true)
List<SimilarImageProjection> findCandidates(
@Param("fileId") String fileId
);
}
@@ -0,0 +1,14 @@
package ru.soune.nocopy.repository;
public interface SimilarImageProjection {
String getSimilarFileId();
String getOriginalFileName();
Long getFileSize();
Integer getHash64Hi();
Integer getHash64Lo();
}
@@ -0,0 +1,62 @@
package ru.soune.nocopy.service;
import lombok.RequiredArgsConstructor;
import org.springframework.stereotype.Service;
import ru.soune.nocopy.dto.file.SimilarFileResponse;
import ru.soune.nocopy.repository.ImageHashRepository;
import ru.soune.nocopy.repository.ImageSimilarityRepository;
import ru.soune.nocopy.repository.SimilarImageProjection;
import java.util.List;
@Service
@RequiredArgsConstructor
public class FileSimilarityService {
private final ImageSimilarityRepository repository;
private final ImageHashRepository hashRepository;
public List<SimilarFileResponse> findSimilarFiles(String fileId) {
var imageHashEntity = hashRepository.findById(fileId)
.orElseThrow(() -> new RuntimeException("Hash not found"));
Integer hash64Hi = imageHashEntity.getHash64Hi();
Integer hash64Lo = imageHashEntity.getHash64Lo();
List<SimilarImageProjection> candidates =
repository.findCandidates(fileId);
return candidates.stream()
.map(c -> {
Integer cHi = c.getHash64Hi();
Integer cLo = c.getHash64Lo();
int hamming = hamming64(hash64Hi, hash64Lo, cHi, cLo);
String level;
if (hamming <= 5) {
level = "DUPLICATE";
} else if (hamming <= 12) {
level = "SIMILAR";
} else {
level = "DIFFERENT";
}
return SimilarFileResponse.builder()
.fileId(c.getSimilarFileId())
.originalFileName(c.getOriginalFileName())
.fileSize(c.getFileSize())
.hammingDistance(hamming)
.similarityLevel(level)
.build();
})
.sorted((a, b) ->
Integer.compare(a.getHammingDistance(), b.getHammingDistance()))
.toList();
}
private int hamming64(int aHi, int aLo, int bHi, int bLo) {
return Integer.bitCount(aHi ^ bHi)
+ Integer.bitCount(aLo ^ bLo);
}
}
@@ -0,0 +1,154 @@
package ru.soune.nocopy.service;
import lombok.RequiredArgsConstructor;
import lombok.extern.slf4j.Slf4j;
import org.springframework.stereotype.Service;
import ru.soune.nocopy.entity.file.FileEntity;
import ru.soune.nocopy.entity.file.ImageHashEntity;
import ru.soune.nocopy.repository.ImageHashRepository;
import javax.imageio.ImageIO;
import java.awt.*;
import java.awt.image.BufferedImage;
import java.io.IOException;
import java.io.InputStream;
import java.nio.file.Files;
import java.nio.file.Path;
import java.time.LocalDateTime;
import java.util.Arrays;
@Service
@RequiredArgsConstructor
@Slf4j
public class ImageHashService {
private final ImageHashRepository repository;
public void create(FileEntity file, Path imagePath) {
try {
long hash64 = computePhash64(imagePath);
int hi = high32(hash64);
int lo = low32(hash64);
ImageHashEntity entity = ImageHashEntity.builder()
.file(file)
.hash64Hi(hi)
.hash64Lo(lo)
.hashAlgorithm("PHASH64")
.createdAt(LocalDateTime.now())
.build();
repository.save(entity);
} catch (IOException e) {
throw new RuntimeException("Failed to compute image hash", e);
}
}
private long computePhash64(Path path) throws IOException {
try (InputStream is = Files.newInputStream(path)) {
BufferedImage image = ImageIO.read(is);
if (image == null) throw new IOException("Cannot read image file: " + path);
BufferedImage gray = toGrayScale(image);
BufferedImage resized = resizeImage(gray, 32, 32);
double[][] dct = applyDCT(resized);
double[][] topLeft = extractTopLeft8x8(dct);
double median = calculateMedian(topLeft, true);
return create64BitHashLong(topLeft, median);
}
}
private BufferedImage toGrayScale(BufferedImage image) {
BufferedImage gray = new BufferedImage(image.getWidth(), image.getHeight(), BufferedImage.TYPE_BYTE_GRAY);
Graphics2D g = gray.createGraphics();
g.drawImage(image, 0, 0, null);
g.dispose();
return gray;
}
private BufferedImage resizeImage(BufferedImage image, int width, int height) {
BufferedImage resized = new BufferedImage(width, height, BufferedImage.TYPE_BYTE_GRAY);
Graphics2D g = resized.createGraphics();
g.setRenderingHint(RenderingHints.KEY_INTERPOLATION, RenderingHints.VALUE_INTERPOLATION_BILINEAR);
g.drawImage(image, 0, 0, width, height, null);
g.dispose();
return resized;
}
private double[][] applyDCT(BufferedImage image) {
int w = image.getWidth();
int h = image.getHeight();
double[][] pixels = new double[h][w];
for (int y = 0; y < h; y++) {
for (int x = 0; x < w; x++) {
int gray = image.getRGB(x, y) & 0xFF;
pixels[y][x] = gray;
}
}
double[][] dct = new double[h][w];
for (int u = 0; u < h; u++) {
double au = u == 0 ? 1.0 / Math.sqrt(h) : Math.sqrt(2.0 / h);
for (int v = 0; v < w; v++) {
double av = v == 0 ? 1.0 / Math.sqrt(w) : Math.sqrt(2.0 / w);
double sum = 0.0;
for (int x = 0; x < h; x++) {
for (int y = 0; y < w; y++) {
sum += pixels[x][y] *
Math.cos((2 * x + 1) * u * Math.PI / (2.0 * h)) *
Math.cos((2 * y + 1) * v * Math.PI / (2.0 * w));
}
}
dct[u][v] = au * av * sum;
}
}
return dct;
}
private double[][] extractTopLeft8x8(double[][] dct) {
double[][] block = new double[8][8];
for (int i = 0; i < 8; i++) System.arraycopy(dct[i], 0, block[i], 0, 8);
return block;
}
private double calculateMedian(double[][] block, boolean excludeDC) {
double[] vals = new double[63];
int idx = 0;
for (int i = 0; i < 8; i++) {
for (int j = 0; j < 8; j++) {
if (i == 0 && j == 0 && excludeDC) continue;
vals[idx++] = block[i][j];
}
}
Arrays.sort(vals);
return (vals.length % 2 == 0) ?
(vals[vals.length / 2 - 1] + vals[vals.length / 2]) / 2.0 :
vals[vals.length / 2];
}
private long create64BitHashLong(double[][] block, double median) {
long hash = 0L;
for (int i = 0; i < 8; i++) {
for (int j = 0; j < 8; j++) {
if (i == 0 && j == 0) continue;
hash <<= 1;
if (block[i][j] > median) hash |= 1;
}
}
return hash;
}
private int high32(long hash64) {
return (int) (hash64 >>> 32);
}
private int low32(long hash64) {
return (int) hash64;
}
}
@@ -9,15 +9,17 @@ import ru.soune.nocopy.dto.file.FileResponse;
import ru.soune.nocopy.entity.file.FileEntity;
import ru.soune.nocopy.entity.file.FileStatus;
import ru.soune.nocopy.entity.file.FileUploadSession;
import ru.soune.nocopy.exception.DuplicateImageException;
import ru.soune.nocopy.exception.FileEntityNotFoundException;
import ru.soune.nocopy.repository.FileEntityRepository;
import ru.soune.nocopy.service.FileSimilarityService;
import ru.soune.nocopy.service.ImageHashService;
import java.io.IOException;
import java.nio.file.Files;
import java.nio.file.Path;
import java.nio.file.Paths;
import java.time.LocalDateTime;
import java.time.ZoneOffset;
import java.util.List;
import java.util.stream.Collectors;
@@ -28,6 +30,10 @@ public class FileEntityService {
private final FileEntityRepository fileEntityRepository;
private final ImageHashService imageHashService;
private final FileSimilarityService fileSimilarityService;
@Transactional
public FileEntity createFromUploadSession(FileUploadSession session, String checksum) {
log.info("Creating FileEntity for upload session: {}", session.getUploadId());
@@ -57,11 +63,28 @@ public class FileEntityService {
.build();
FileEntity saved = fileEntityRepository.save(fileEntity);
log.info("FileEntity created successfully: {} (size: {} bytes)",
saved.getId(), fileSize);
if (saved.getMimeType().startsWith("image")) {
imageHashService.create(saved, Path.of(saved.getFilePath()));
var similarFiles = fileSimilarityService.findSimilarFiles(saved.getId());
boolean hasDuplicate = similarFiles.stream()
.anyMatch(f -> f.getHammingDistance() <= 5);
if (hasDuplicate) {
log.warn("Duplicate image detected for upload session {}. Rolling back.",
session.getUploadId());
throw new DuplicateImageException("Duplicate image detected for upload session " + session.getUploadId());
}
similarFiles.stream()
.filter(f -> f.getHammingDistance() > 5)
.forEach(f -> log.info("Similar image found: {} (distance {})",
f.getFileId(), f.getHammingDistance()));
}
return saved;
} catch (IOException e) {
log.error("Failed to create FileEntity for session {}: {}",
session.getUploadId(), e.getMessage(), e);
@@ -69,15 +92,6 @@ public class FileEntityService {
}
}
private String extractFileExtension(String fileName) {
int dotIndex = fileName.lastIndexOf('.');
if (dotIndex > 0 && dotIndex < fileName.length() - 1) {
return fileName.substring(dotIndex + 1).toLowerCase();
}
return "";
}
@Transactional(readOnly = true)
public FileEntityResponse getById(String fileId, int version) {
FileEntity fileEntity = fileEntityRepository.findById(fileId)
@@ -95,39 +109,6 @@ public class FileEntityService {
return convertToResponse(fileEntity, version);
}
@Transactional(readOnly = true)
public FileEntityResponse getByFilePath(String filePath, int version) {
FileEntity fileEntity = fileEntityRepository.findByFilePath(filePath)
.orElseThrow(() -> new FileEntityNotFoundException("Path: " + filePath));
return convertToResponse(fileEntity, version);
}
@Transactional(readOnly = true)
public FileResponse getAllUserFiles(Long userId, int version) {
List<FileEntity> fileEntities = fileEntityRepository.findByUserIdAndStatus(
userId, FileStatus.ACTIVE);
List<FileEntityResponse> files = fileEntities.stream()
.map(file -> convertToResponse(file, version))
.collect(Collectors.toList());
long totalSize = fileEntities.stream()
.mapToLong(FileEntity::getFileSize)
.sum();
return FileResponse.builder()
.files(files)
.totalCount(files.size())
.totalSize(totalSize)
.formattedTotalSize(formatFileSize(totalSize))
.page(1)
.pageSize(files.size())
.build();
}
@Transactional(readOnly = true)
public FileResponse getUserFiles(Long userId, int page, int pageSize, int version) {
List<FileEntity> allFiles = fileEntityRepository.findByUserIdAndStatus(
@@ -402,7 +402,7 @@ public class FileUploadServiceImpl implements FileUploadService {
session.getUploadId(), session.getFileName());
Path finalFilePath = null;
String checksum = null;
String checksum;
try {
finalFilePath = prepareFinalFile(session);
@@ -591,44 +591,10 @@ public class FileUploadServiceImpl implements FileUploadService {
private void handleAssemblyFailure(FileUploadSession session, Exception e) {
session.setStatus(UploadStatus.FAILED);
session.setLastError(e.getMessage());
sessionRepository.save(session);
log.error("File assembly failed for session {}: {}",
session.getUploadId(), e.getMessage());
}
// @Override
// @Transactional
// public void cleanupExpiredSessions() {
// log.info("Starting cleanup of expired upload sessions");
//
// LocalDateTime expiryThreshold = LocalDateTime.now().minusHours(sessionExpiryHours);
//
// try {
// var expiredSessions = sessionRepository.findByStatusInAndCreatedAtBefore(
// Set.of(UploadStatus.INITIATED, UploadStatus.UPLOADING, UploadStatus.FAILED),
// expiryThreshold
// );
//
// int cleanedCount = 0;
// for (FileUploadSession session : expiredSessions) {
// session.setStatus(UploadStatus.FAILED);
// session.setLastError("Session expired during cleanup");
// sessionRepository.save(session);
//
// CompletableFuture.runAsync(() -> cleanupSessionFiles(session));
// cleanedCount++;
// }
//
// log.info("Cleaned up {} expired upload sessions", cleanedCount);
//
// } catch (Exception e) {
// log.error("Error during session cleanup", e);
// }
// }
private Path getChunkDirectory(Long userId, String uploadId) {
return storageRoot.resolve("temp")
.resolve("user")
+4
View File
@@ -1,4 +1,8 @@
spring:
flyway:
enabled: true
baseline-on-migrate: true
datasource:
url: jdbc:postgresql://${POSTGRES_HOST}:${POSTGRES_PORT}/${POSTGRES_DB}
username: ${POSTGRES_USER}
@@ -0,0 +1,2 @@
CREATE INDEX idx_image_hash_prefix
ON image_hashes ((hash_value >> 48));
@@ -1,101 +0,0 @@
package ru.soune.nocopy.service;
import org.junit.jupiter.api.BeforeEach;
import org.junit.jupiter.api.Test;
import org.mockito.ArgumentCaptor;
import org.mockito.InjectMocks;
import org.mockito.Mock;
import org.mockito.MockitoAnnotations;
import org.springframework.context.MessageSource;
import org.springframework.security.crypto.password.PasswordEncoder;
import ru.soune.nocopy.dto.LoginRequest;
import ru.soune.nocopy.entity.AuthToken;
import ru.soune.nocopy.entity.User;
import ru.soune.nocopy.exception.NotValidationPasswordException;
import ru.soune.nocopy.exception.UserNotFoundException;
import ru.soune.nocopy.repository.AuthTokenRepository;
import ru.soune.nocopy.repository.UserRepository;
import java.util.Optional;
import static org.junit.jupiter.api.Assertions.*;
import static org.mockito.ArgumentMatchers.any;
import static org.mockito.Mockito.verify;
import static org.mockito.Mockito.when;
public class AuthServiceTest {
@Mock
private UserRepository userRepository;
@Mock
private AuthTokenRepository authTokenRepository;
@Mock
private PasswordEncoder passwordEncoder;
@Mock
private MessageSource messageSource;
@InjectMocks
private AuthService authService;
@BeforeEach
void setUp() {
MockitoAnnotations.openMocks(this);
}
@Test
void login_success() {
LoginRequest request = new LoginRequest("test@mail.com", "password");
User user = new User();
user.setId(1L);
user.setEmail("test@mail.com");
user.setPassword("encoded_pass");
when(userRepository.findByEmail("test@mail.com"))
.thenReturn(Optional.of(user));
when(passwordEncoder.matches("password", "encoded_pass"))
.thenReturn(true);
when(authTokenRepository.save(any(AuthToken.class)))
.thenAnswer(invocation -> invocation.getArgument(0));
AuthToken token = authService.login(request);
assertNotNull(token);
assertNotNull(token.getToken());
// assertEquals(user.getTokens().get(0), token.getUser());
ArgumentCaptor<AuthToken> captor = ArgumentCaptor.forClass(AuthToken.class);
verify(authTokenRepository).save(captor.capture());
// assertEquals(user, captor.getValue().getUser());
assertNotNull(captor.getValue().getToken());
}
@Test
void login_userNotFound() {
LoginRequest request = new LoginRequest("missing@mail.com", "pwd");
when(userRepository.findByEmail("missing@mail.com")).thenReturn(Optional.empty());
assertThrows(UserNotFoundException.class, () -> authService.login(request));
}
@Test
void login_invalidPassword() {
LoginRequest request = new LoginRequest("test@mail.com", "wrong");
User user = new User();
user.setEmail("test@mail.com");
user.setPassword("encoded_pass");
when(userRepository.findByEmail("test@mail.com")).thenReturn(Optional.of(user));
when(passwordEncoder.matches("wrong", "encoded_pass")).thenReturn(false);
assertThrows(NotValidationPasswordException.class, () -> authService.login(request));
}
}