+1
-1
@@ -59,7 +59,7 @@ dependencies {
|
|||||||
testImplementation 'org.mockito:mockito-core:5.3.1'
|
testImplementation 'org.mockito:mockito-core:5.3.1'
|
||||||
testRuntimeOnly 'org.junit.platform:junit-platform-launcher'
|
testRuntimeOnly 'org.junit.platform:junit-platform-launcher'
|
||||||
|
|
||||||
implementation name: 'testlib-fat-0.3.0-all'
|
implementation name: 'testlib-fat-0.3.1-all'
|
||||||
|
|
||||||
implementation group: 'com.squareup.okhttp3', name: 'okhttp', version: '4.12.0'
|
implementation group: 'com.squareup.okhttp3', name: 'okhttp', version: '4.12.0'
|
||||||
|
|
||||||
|
|||||||
Binary file not shown.
@@ -4,16 +4,12 @@ import lombok.AllArgsConstructor;
|
|||||||
import org.springframework.boot.context.event.ApplicationReadyEvent;
|
import org.springframework.boot.context.event.ApplicationReadyEvent;
|
||||||
import org.springframework.context.event.EventListener;
|
import org.springframework.context.event.EventListener;
|
||||||
import org.springframework.stereotype.Component;
|
import org.springframework.stereotype.Component;
|
||||||
import ru.soune.nocopy.service.file.FileProcessingOrchestrator;
|
|
||||||
|
|
||||||
@Component
|
@Component
|
||||||
@AllArgsConstructor
|
@AllArgsConstructor
|
||||||
public class NoCopyInitializer {
|
public class NoCopyInitializer {
|
||||||
|
|
||||||
private final FileProcessingOrchestrator orchestrator;
|
|
||||||
|
|
||||||
@EventListener(ApplicationReadyEvent.class)
|
@EventListener(ApplicationReadyEvent.class)
|
||||||
public void initializeOnStartup() {
|
public void initializeOnStartup() {
|
||||||
orchestrator.initializeProcessingQueue();
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -28,4 +28,7 @@ public class FileUploadRequest {
|
|||||||
|
|
||||||
@JsonProperty("token")
|
@JsonProperty("token")
|
||||||
private String token;
|
private String token;
|
||||||
|
|
||||||
|
@JsonProperty("convertTo")
|
||||||
|
private String convertTo;
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -75,6 +75,9 @@ public class FileUploadSession {
|
|||||||
@Column(name = "completed_at")
|
@Column(name = "completed_at")
|
||||||
private LocalDateTime completedAt;
|
private LocalDateTime completedAt;
|
||||||
|
|
||||||
|
@Column(name = "convert_to")
|
||||||
|
String convertTo;
|
||||||
|
|
||||||
@ElementCollection
|
@ElementCollection
|
||||||
@CollectionTable(
|
@CollectionTable(
|
||||||
name = "uploaded_chunks",
|
name = "uploaded_chunks",
|
||||||
|
|||||||
@@ -111,6 +111,7 @@ public class FileUploadHandler implements RequestHandler {
|
|||||||
fileRequest.getFileName(),
|
fileRequest.getFileName(),
|
||||||
fileRequest.getFileType(),
|
fileRequest.getFileType(),
|
||||||
fileRequest.getExtension(),
|
fileRequest.getExtension(),
|
||||||
|
fileRequest.getConvertTo(),
|
||||||
fileRequest.getFileSize());
|
fileRequest.getFileSize());
|
||||||
|
|
||||||
InitFileResponse fileResponse = InitFileResponse.builder()
|
InitFileResponse fileResponse = InitFileResponse.builder()
|
||||||
|
|||||||
@@ -27,4 +27,11 @@ public interface FileUploadSessionRepository extends JpaRepository<FileUploadSes
|
|||||||
List<FileUploadSession> findExpiredSessions(
|
List<FileUploadSession> findExpiredSessions(
|
||||||
@Param("now") LocalDateTime now,
|
@Param("now") LocalDateTime now,
|
||||||
@Param("statuses") Set<UploadStatus> statuses);
|
@Param("statuses") Set<UploadStatus> statuses);
|
||||||
|
|
||||||
|
|
||||||
|
@Query("SELECT s FROM FileUploadSession s WHERE s.filePath = :filePath AND s.status = :status AND s.userId = :userId")
|
||||||
|
FileUploadSession findByFilePathAndStatus(
|
||||||
|
@Param("filePath") String filePath,
|
||||||
|
@Param("status") String status,
|
||||||
|
@Param("userId") Long userId);
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -1,55 +0,0 @@
|
|||||||
package ru.soune.nocopy.service.file;
|
|
||||||
|
|
||||||
import com.vrt.NoCopyFileService;
|
|
||||||
import com.vrt.fileprotection.FileProtector;
|
|
||||||
import lombok.RequiredArgsConstructor;
|
|
||||||
import lombok.extern.slf4j.Slf4j;
|
|
||||||
import org.springframework.context.annotation.Lazy;
|
|
||||||
import org.springframework.scheduling.annotation.Scheduled;
|
|
||||||
import org.springframework.stereotype.Service;
|
|
||||||
import ru.soune.nocopy.entity.file.FileEntity;
|
|
||||||
import ru.soune.nocopy.entity.file.ProtectionStatus;
|
|
||||||
import ru.soune.nocopy.repository.FileEntityRepository;
|
|
||||||
import ru.soune.nocopy.util.FileUtil;
|
|
||||||
|
|
||||||
import java.util.List;
|
|
||||||
|
|
||||||
@Slf4j
|
|
||||||
@Service
|
|
||||||
@RequiredArgsConstructor
|
|
||||||
public class FileProcessingOrchestrator {
|
|
||||||
|
|
||||||
@Lazy
|
|
||||||
private final NoCopyFileService noCopyFileService;
|
|
||||||
|
|
||||||
private final FileEntityRepository fileRepository;
|
|
||||||
|
|
||||||
private final FileUtil fileUtil;
|
|
||||||
|
|
||||||
public void initializeProcessingQueue() {
|
|
||||||
List<FileEntity> filesToProtect = fileRepository.findByProtectionStatus(ProtectionStatus.NOT_PROTECTED);
|
|
||||||
|
|
||||||
for (FileEntity fileEntity : filesToProtect) {
|
|
||||||
try {
|
|
||||||
FileProtector.FileInfo fileInfo = fileUtil.createFileInfo(fileEntity);
|
|
||||||
|
|
||||||
noCopyFileService.addFile(fileInfo);
|
|
||||||
|
|
||||||
log.info("Add to query: {}", fileEntity.getOriginalFileName());
|
|
||||||
} catch (Exception e) {
|
|
||||||
log.error("Fail add to query: {}", fileEntity.getId(), e);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
@Scheduled(fixedDelay = 120000)
|
|
||||||
public void checkNewFilesForProtection() {
|
|
||||||
List<FileEntity> newFiles = fileRepository.findAllActiveFilesAndNotProtected();
|
|
||||||
|
|
||||||
for (FileEntity fileEntity : newFiles) {
|
|
||||||
FileProtector.FileInfo fileInfo = fileUtil.createFileInfo(fileEntity);
|
|
||||||
|
|
||||||
noCopyFileService.addFile(fileInfo);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
@@ -9,7 +9,7 @@ import java.io.IOException;
|
|||||||
|
|
||||||
public interface FileUploadService {
|
public interface FileUploadService {
|
||||||
FileUploadSession initUpload(Long userId, String fileName,
|
FileUploadSession initUpload(Long userId, String fileName,
|
||||||
String fileType, String extension, long fileSize);
|
String fileType, String extension, String convertTo, long fileSize);
|
||||||
|
|
||||||
void handleExpiredSession(FileUploadSession session);
|
void handleExpiredSession(FileUploadSession session);
|
||||||
|
|
||||||
|
|||||||
@@ -28,7 +28,7 @@ public class AudioLocalSearchImpl implements AudioLocalSearch {
|
|||||||
|
|
||||||
if (fileEntity == null) return null;
|
if (fileEntity == null) return null;
|
||||||
|
|
||||||
return fileUtil.createFileInfo(fileEntity);
|
return fileUtil.createFileInfo(fileEntity, null);
|
||||||
}
|
}
|
||||||
|
|
||||||
@Override
|
@Override
|
||||||
|
|||||||
@@ -23,7 +23,7 @@ public class DocumentLocalSearchImpl implements DocumentLocalSearch {
|
|||||||
@Override
|
@Override
|
||||||
public @Nullable FileProtector.FileInfo findByIds(@NotNull String ownerId, @NotNull String fileId) {
|
public @Nullable FileProtector.FileInfo findByIds(@NotNull String ownerId, @NotNull String fileId) {
|
||||||
FileEntity fileEntity = fileEntityRepository.findByIdAndUserId(fileId, Long.valueOf(ownerId));
|
FileEntity fileEntity = fileEntityRepository.findByIdAndUserId(fileId, Long.valueOf(ownerId));
|
||||||
FileProtector.FileInfo fileInfo = fileUtil.createFileInfo(fileEntity);
|
FileProtector.FileInfo fileInfo = fileUtil.createFileInfo(fileEntity, null);
|
||||||
log.info("fileInfo: {}", fileInfo);
|
log.info("fileInfo: {}", fileInfo);
|
||||||
|
|
||||||
return fileInfo;
|
return fileInfo;
|
||||||
|
|||||||
@@ -107,7 +107,7 @@ public class FileUploadServiceImpl implements FileUploadService {
|
|||||||
@Override
|
@Override
|
||||||
@Transactional
|
@Transactional
|
||||||
public FileUploadSession initUpload(Long userId, String fileName,
|
public FileUploadSession initUpload(Long userId, String fileName,
|
||||||
String fileType, String extension, long fileSize) {
|
String fileType, String extension, String convertTo, long fileSize) {
|
||||||
log.info("Initializing upload for user {}: {} ({} bytes, type: {})",
|
log.info("Initializing upload for user {}: {} ({} bytes, type: {})",
|
||||||
userId, fileName, fileSize, fileType);
|
userId, fileName, fileSize, fileType);
|
||||||
|
|
||||||
@@ -138,6 +138,7 @@ public class FileUploadServiceImpl implements FileUploadService {
|
|||||||
.status(UploadStatus.INITIATED)
|
.status(UploadStatus.INITIATED)
|
||||||
.expiresAt(LocalDateTime.now().plusHours(sessionExpiryHours))
|
.expiresAt(LocalDateTime.now().plusHours(sessionExpiryHours))
|
||||||
.retryCount(0)
|
.retryCount(0)
|
||||||
|
.convertTo(convertTo)
|
||||||
.build();
|
.build();
|
||||||
|
|
||||||
FileUploadSession savedSession = sessionRepository.save(session);
|
FileUploadSession savedSession = sessionRepository.save(session);
|
||||||
@@ -248,7 +249,7 @@ public class FileUploadServiceImpl implements FileUploadService {
|
|||||||
cleanupSessionFiles(session);
|
cleanupSessionFiles(session);
|
||||||
|
|
||||||
if (status != FileStatus.TEMP) {
|
if (status != FileStatus.TEMP) {
|
||||||
FileProtector.FileInfo fileInfo = fileUtil.createFileInfo(fileEntity);
|
FileProtector.FileInfo fileInfo = fileUtil.createFileInfo(fileEntity, session.getConvertTo());
|
||||||
|
|
||||||
noCopyFileService.addFile(fileInfo);
|
noCopyFileService.addFile(fileInfo);
|
||||||
}
|
}
|
||||||
@@ -615,76 +616,6 @@ public class FileUploadServiceImpl implements FileUploadService {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
private void assembleFile(FileUploadSession session) throws IOException {
|
|
||||||
log.info("Starting file assembly for session: {} ({})",
|
|
||||||
session.getUploadId(), session.getFileName());
|
|
||||||
|
|
||||||
Path finalFilePath = null;
|
|
||||||
String checksum;
|
|
||||||
|
|
||||||
try {
|
|
||||||
finalFilePath = prepareFinalFile(session);
|
|
||||||
log.info("Final file path: {}", finalFilePath);
|
|
||||||
|
|
||||||
validateAllChunksExist(session);
|
|
||||||
|
|
||||||
mergeChunksToFile(session, finalFilePath);
|
|
||||||
|
|
||||||
validateFinalFile(session, finalFilePath);
|
|
||||||
|
|
||||||
checksum = calculateChecksum(finalFilePath);
|
|
||||||
log.debug("File checksum calculated: {}", checksum);
|
|
||||||
|
|
||||||
session.setFilePath(finalFilePath.toString());
|
|
||||||
session.setChecksum(checksum);
|
|
||||||
session.setStatus(UploadStatus.COMPLETED);
|
|
||||||
session.setCompletedAt(LocalDateTime.now());
|
|
||||||
|
|
||||||
sessionRepository.save(session);
|
|
||||||
log.info("Upload session updated to COMPLETED: {}", session.getUploadId());
|
|
||||||
|
|
||||||
try {
|
|
||||||
FileEntity fileEntity = fileEntityService.createFromUploadSession(session, checksum);
|
|
||||||
|
|
||||||
FileProtector.FileInfo fileInfo = fileUtil.createFileInfo(fileEntity);
|
|
||||||
|
|
||||||
noCopyFileService.addFile(fileInfo);
|
|
||||||
|
|
||||||
log.info("FileEntity successfully created for session: {}",
|
|
||||||
session.getUploadId());
|
|
||||||
} catch (DuplicateImageException e) {
|
|
||||||
throw new DuplicateImageException("Duplicate", e.duplicateFileId(), e.userId());
|
|
||||||
} catch (Exception e) {
|
|
||||||
log.error("Failed to create FileEntity for session {} (file uploaded, but metadata not saved): {}",
|
|
||||||
session.getUploadId(), e.getMessage());
|
|
||||||
}
|
|
||||||
|
|
||||||
cleanupSessionFiles(session);
|
|
||||||
|
|
||||||
log.info("File assembly completed successfully: {} -> {} ({} bytes)",
|
|
||||||
session.getFileName(), finalFilePath, session.getFileSize());
|
|
||||||
|
|
||||||
} catch (Exception e) {
|
|
||||||
log.error("File assembly failed for session {}: {}",
|
|
||||||
session.getUploadId(), e.getMessage(), e);
|
|
||||||
|
|
||||||
if (finalFilePath != null) {
|
|
||||||
try {
|
|
||||||
Files.deleteIfExists(finalFilePath);
|
|
||||||
log.debug("Cleaned up partial file: {}", finalFilePath);
|
|
||||||
} catch (IOException ioException) {
|
|
||||||
log.warn("Failed to cleanup partial file: {}", finalFilePath, ioException);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
session.setStatus(UploadStatus.FAILED);
|
|
||||||
session.setLastError(e.getMessage());
|
|
||||||
sessionRepository.save(session);
|
|
||||||
|
|
||||||
throw new IOException("File assembly failed: " + e.getMessage(), e);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
private Path prepareFinalFile(FileUploadSession session) throws IOException {
|
private Path prepareFinalFile(FileUploadSession session) throws IOException {
|
||||||
Path userUploadsDir = storageRoot.resolve("uploads")
|
Path userUploadsDir = storageRoot.resolve("uploads")
|
||||||
.resolve(String.valueOf(session.getUserId()))
|
.resolve(String.valueOf(session.getUserId()))
|
||||||
|
|||||||
@@ -51,6 +51,6 @@ public class ImageLocalSearchImpl implements ImageLocalSearch {
|
|||||||
FileEntity file = fileEntityRepository.findByFileId(imageHashEntity.getFileId());
|
FileEntity file = fileEntityRepository.findByFileId(imageHashEntity.getFileId());
|
||||||
|
|
||||||
return new FileProtector.FileInfo(FileProtector.Type.IMAGE, imageHashEntity.getFileId(),
|
return new FileProtector.FileInfo(FileProtector.Type.IMAGE, imageHashEntity.getFileId(),
|
||||||
String.valueOf(file.getUserId()));
|
String.valueOf(file.getUserId()), null);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -7,7 +7,11 @@ import lombok.extern.slf4j.Slf4j;
|
|||||||
import org.jetbrains.annotations.NotNull;
|
import org.jetbrains.annotations.NotNull;
|
||||||
import org.jetbrains.annotations.Nullable;
|
import org.jetbrains.annotations.Nullable;
|
||||||
import org.springframework.stereotype.Component;
|
import org.springframework.stereotype.Component;
|
||||||
|
import ru.soune.nocopy.entity.file.FileEntity;
|
||||||
|
import ru.soune.nocopy.repository.FileEntityRepository;
|
||||||
|
import ru.soune.nocopy.repository.FileUploadSessionRepository;
|
||||||
import ru.soune.nocopy.service.file.FileEntityService;
|
import ru.soune.nocopy.service.file.FileEntityService;
|
||||||
|
import ru.soune.nocopy.service.file.FileUploadService;
|
||||||
|
|
||||||
import java.io.File;
|
import java.io.File;
|
||||||
import java.io.IOException;
|
import java.io.IOException;
|
||||||
@@ -24,6 +28,10 @@ public class ProtectionFileProviderImpl implements FileProtector.FileProvider {
|
|||||||
|
|
||||||
private final FileEntityService fileEntityService;
|
private final FileEntityService fileEntityService;
|
||||||
|
|
||||||
|
private final FileUploadSessionRepository fileUploadSessionRepository;
|
||||||
|
|
||||||
|
private final FileEntityRepository fileEntityRepository;
|
||||||
|
|
||||||
@Nullable
|
@Nullable
|
||||||
@Override
|
@Override
|
||||||
public File getImageFile(@NotNull String id) {
|
public File getImageFile(@NotNull String id) {
|
||||||
|
|||||||
@@ -13,17 +13,17 @@ public class FileUtil {
|
|||||||
public ImageLocalSearch.Result convertToResult(SimilarFileDTO response) {
|
public ImageLocalSearch.Result convertToResult(SimilarFileDTO response) {
|
||||||
FileProtector.FileInfo fileInfo =
|
FileProtector.FileInfo fileInfo =
|
||||||
new FileProtector.FileInfo(FileProtector.Type.IMAGE, response.getFileId(),
|
new FileProtector.FileInfo(FileProtector.Type.IMAGE, response.getFileId(),
|
||||||
String.valueOf(response.getOwnerId()));
|
String.valueOf(response.getOwnerId()), null);
|
||||||
ImageScore imageScore = new ImageScore(ImageScore.Rate.valueOf(response.getSimilarityLevel()),
|
ImageScore imageScore = new ImageScore(ImageScore.Rate.valueOf(response.getSimilarityLevel()),
|
||||||
response.getHammingDistance());
|
response.getHammingDistance());
|
||||||
|
|
||||||
return new ImageLocalSearch.Result(fileInfo, imageScore);
|
return new ImageLocalSearch.Result(fileInfo, imageScore);
|
||||||
}
|
}
|
||||||
|
|
||||||
public FileProtector.FileInfo createFileInfo(FileEntity fileEntity) {
|
public FileProtector.FileInfo createFileInfo(FileEntity fileEntity, String convertTo) {
|
||||||
FileProtector.Type type = determineFileType(fileEntity.getMimeType());
|
FileProtector.Type type = determineFileType(fileEntity.getMimeType());
|
||||||
|
|
||||||
return new FileProtector.FileInfo(type, fileEntity.getId(), String.valueOf(fileEntity.getUserId()));
|
return new FileProtector.FileInfo(type, fileEntity.getId(), String.valueOf(fileEntity.getUserId()), convertTo);
|
||||||
}
|
}
|
||||||
|
|
||||||
public int hamming64(long aHi, long aLo, long bHi, long bLo) {
|
public int hamming64(long aHi, long aLo, long bHi, long bLo) {
|
||||||
|
|||||||
Reference in New Issue
Block a user