dev conver files
Test Workflow / test (push) Successful in 4s

This commit is contained in:
vladp
2026-02-25 19:29:19 +07:00
parent 694681fccc
commit 3bb6667c2f
15 changed files with 33 additions and 139 deletions
+1 -1
View File
@@ -59,7 +59,7 @@ dependencies {
testImplementation 'org.mockito:mockito-core:5.3.1'
testRuntimeOnly 'org.junit.platform:junit-platform-launcher'
implementation name: 'testlib-fat-0.3.0-all'
implementation name: 'testlib-fat-0.3.1-all'
implementation group: 'com.squareup.okhttp3', name: 'okhttp', version: '4.12.0'
@@ -4,16 +4,12 @@ import lombok.AllArgsConstructor;
import org.springframework.boot.context.event.ApplicationReadyEvent;
import org.springframework.context.event.EventListener;
import org.springframework.stereotype.Component;
import ru.soune.nocopy.service.file.FileProcessingOrchestrator;
@Component
@AllArgsConstructor
public class NoCopyInitializer {
private final FileProcessingOrchestrator orchestrator;
@EventListener(ApplicationReadyEvent.class)
public void initializeOnStartup() {
orchestrator.initializeProcessingQueue();
}
}
@@ -28,4 +28,7 @@ public class FileUploadRequest {
@JsonProperty("token")
private String token;
@JsonProperty("convertTo")
private String convertTo;
}
@@ -75,6 +75,9 @@ public class FileUploadSession {
@Column(name = "completed_at")
private LocalDateTime completedAt;
@Column(name = "convert_to")
String convertTo;
@ElementCollection
@CollectionTable(
name = "uploaded_chunks",
@@ -111,6 +111,7 @@ public class FileUploadHandler implements RequestHandler {
fileRequest.getFileName(),
fileRequest.getFileType(),
fileRequest.getExtension(),
fileRequest.getConvertTo(),
fileRequest.getFileSize());
InitFileResponse fileResponse = InitFileResponse.builder()
@@ -27,4 +27,11 @@ public interface FileUploadSessionRepository extends JpaRepository<FileUploadSes
List<FileUploadSession> findExpiredSessions(
@Param("now") LocalDateTime now,
@Param("statuses") Set<UploadStatus> statuses);
@Query("SELECT s FROM FileUploadSession s WHERE s.filePath = :filePath AND s.status = :status AND s.userId = :userId")
FileUploadSession findByFilePathAndStatus(
@Param("filePath") String filePath,
@Param("status") String status,
@Param("userId") Long userId);
}
@@ -1,55 +0,0 @@
package ru.soune.nocopy.service.file;
import com.vrt.NoCopyFileService;
import com.vrt.fileprotection.FileProtector;
import lombok.RequiredArgsConstructor;
import lombok.extern.slf4j.Slf4j;
import org.springframework.context.annotation.Lazy;
import org.springframework.scheduling.annotation.Scheduled;
import org.springframework.stereotype.Service;
import ru.soune.nocopy.entity.file.FileEntity;
import ru.soune.nocopy.entity.file.ProtectionStatus;
import ru.soune.nocopy.repository.FileEntityRepository;
import ru.soune.nocopy.util.FileUtil;
import java.util.List;
@Slf4j
@Service
@RequiredArgsConstructor
public class FileProcessingOrchestrator {
@Lazy
private final NoCopyFileService noCopyFileService;
private final FileEntityRepository fileRepository;
private final FileUtil fileUtil;
public void initializeProcessingQueue() {
List<FileEntity> filesToProtect = fileRepository.findByProtectionStatus(ProtectionStatus.NOT_PROTECTED);
for (FileEntity fileEntity : filesToProtect) {
try {
FileProtector.FileInfo fileInfo = fileUtil.createFileInfo(fileEntity);
noCopyFileService.addFile(fileInfo);
log.info("Add to query: {}", fileEntity.getOriginalFileName());
} catch (Exception e) {
log.error("Fail add to query: {}", fileEntity.getId(), e);
}
}
}
@Scheduled(fixedDelay = 120000)
public void checkNewFilesForProtection() {
List<FileEntity> newFiles = fileRepository.findAllActiveFilesAndNotProtected();
for (FileEntity fileEntity : newFiles) {
FileProtector.FileInfo fileInfo = fileUtil.createFileInfo(fileEntity);
noCopyFileService.addFile(fileInfo);
}
}
}
@@ -9,7 +9,7 @@ import java.io.IOException;
public interface FileUploadService {
FileUploadSession initUpload(Long userId, String fileName,
String fileType, String extension, long fileSize);
String fileType, String extension, String convertTo, long fileSize);
void handleExpiredSession(FileUploadSession session);
@@ -28,7 +28,7 @@ public class AudioLocalSearchImpl implements AudioLocalSearch {
if (fileEntity == null) return null;
return fileUtil.createFileInfo(fileEntity);
return fileUtil.createFileInfo(fileEntity, null);
}
@Override
@@ -23,7 +23,7 @@ public class DocumentLocalSearchImpl implements DocumentLocalSearch {
@Override
public @Nullable FileProtector.FileInfo findByIds(@NotNull String ownerId, @NotNull String fileId) {
FileEntity fileEntity = fileEntityRepository.findByIdAndUserId(fileId, Long.valueOf(ownerId));
FileProtector.FileInfo fileInfo = fileUtil.createFileInfo(fileEntity);
FileProtector.FileInfo fileInfo = fileUtil.createFileInfo(fileEntity, null);
log.info("fileInfo: {}", fileInfo);
return fileInfo;
@@ -107,7 +107,7 @@ public class FileUploadServiceImpl implements FileUploadService {
@Override
@Transactional
public FileUploadSession initUpload(Long userId, String fileName,
String fileType, String extension, long fileSize) {
String fileType, String extension, String convertTo, long fileSize) {
log.info("Initializing upload for user {}: {} ({} bytes, type: {})",
userId, fileName, fileSize, fileType);
@@ -138,6 +138,7 @@ public class FileUploadServiceImpl implements FileUploadService {
.status(UploadStatus.INITIATED)
.expiresAt(LocalDateTime.now().plusHours(sessionExpiryHours))
.retryCount(0)
.convertTo(convertTo)
.build();
FileUploadSession savedSession = sessionRepository.save(session);
@@ -248,7 +249,7 @@ public class FileUploadServiceImpl implements FileUploadService {
cleanupSessionFiles(session);
if (status != FileStatus.TEMP) {
FileProtector.FileInfo fileInfo = fileUtil.createFileInfo(fileEntity);
FileProtector.FileInfo fileInfo = fileUtil.createFileInfo(fileEntity, session.getConvertTo());
noCopyFileService.addFile(fileInfo);
}
@@ -615,76 +616,6 @@ public class FileUploadServiceImpl implements FileUploadService {
}
}
private void assembleFile(FileUploadSession session) throws IOException {
log.info("Starting file assembly for session: {} ({})",
session.getUploadId(), session.getFileName());
Path finalFilePath = null;
String checksum;
try {
finalFilePath = prepareFinalFile(session);
log.info("Final file path: {}", finalFilePath);
validateAllChunksExist(session);
mergeChunksToFile(session, finalFilePath);
validateFinalFile(session, finalFilePath);
checksum = calculateChecksum(finalFilePath);
log.debug("File checksum calculated: {}", checksum);
session.setFilePath(finalFilePath.toString());
session.setChecksum(checksum);
session.setStatus(UploadStatus.COMPLETED);
session.setCompletedAt(LocalDateTime.now());
sessionRepository.save(session);
log.info("Upload session updated to COMPLETED: {}", session.getUploadId());
try {
FileEntity fileEntity = fileEntityService.createFromUploadSession(session, checksum);
FileProtector.FileInfo fileInfo = fileUtil.createFileInfo(fileEntity);
noCopyFileService.addFile(fileInfo);
log.info("FileEntity successfully created for session: {}",
session.getUploadId());
} catch (DuplicateImageException e) {
throw new DuplicateImageException("Duplicate", e.duplicateFileId(), e.userId());
} catch (Exception e) {
log.error("Failed to create FileEntity for session {} (file uploaded, but metadata not saved): {}",
session.getUploadId(), e.getMessage());
}
cleanupSessionFiles(session);
log.info("File assembly completed successfully: {} -> {} ({} bytes)",
session.getFileName(), finalFilePath, session.getFileSize());
} catch (Exception e) {
log.error("File assembly failed for session {}: {}",
session.getUploadId(), e.getMessage(), e);
if (finalFilePath != null) {
try {
Files.deleteIfExists(finalFilePath);
log.debug("Cleaned up partial file: {}", finalFilePath);
} catch (IOException ioException) {
log.warn("Failed to cleanup partial file: {}", finalFilePath, ioException);
}
}
session.setStatus(UploadStatus.FAILED);
session.setLastError(e.getMessage());
sessionRepository.save(session);
throw new IOException("File assembly failed: " + e.getMessage(), e);
}
}
private Path prepareFinalFile(FileUploadSession session) throws IOException {
Path userUploadsDir = storageRoot.resolve("uploads")
.resolve(String.valueOf(session.getUserId()))
@@ -51,6 +51,6 @@ public class ImageLocalSearchImpl implements ImageLocalSearch {
FileEntity file = fileEntityRepository.findByFileId(imageHashEntity.getFileId());
return new FileProtector.FileInfo(FileProtector.Type.IMAGE, imageHashEntity.getFileId(),
String.valueOf(file.getUserId()));
String.valueOf(file.getUserId()), null);
}
}
@@ -7,7 +7,11 @@ import lombok.extern.slf4j.Slf4j;
import org.jetbrains.annotations.NotNull;
import org.jetbrains.annotations.Nullable;
import org.springframework.stereotype.Component;
import ru.soune.nocopy.entity.file.FileEntity;
import ru.soune.nocopy.repository.FileEntityRepository;
import ru.soune.nocopy.repository.FileUploadSessionRepository;
import ru.soune.nocopy.service.file.FileEntityService;
import ru.soune.nocopy.service.file.FileUploadService;
import java.io.File;
import java.io.IOException;
@@ -24,6 +28,10 @@ public class ProtectionFileProviderImpl implements FileProtector.FileProvider {
private final FileEntityService fileEntityService;
private final FileUploadSessionRepository fileUploadSessionRepository;
private final FileEntityRepository fileEntityRepository;
@Nullable
@Override
public File getImageFile(@NotNull String id) {
@@ -13,17 +13,17 @@ public class FileUtil {
public ImageLocalSearch.Result convertToResult(SimilarFileDTO response) {
FileProtector.FileInfo fileInfo =
new FileProtector.FileInfo(FileProtector.Type.IMAGE, response.getFileId(),
String.valueOf(response.getOwnerId()));
String.valueOf(response.getOwnerId()), null);
ImageScore imageScore = new ImageScore(ImageScore.Rate.valueOf(response.getSimilarityLevel()),
response.getHammingDistance());
return new ImageLocalSearch.Result(fileInfo, imageScore);
}
public FileProtector.FileInfo createFileInfo(FileEntity fileEntity) {
public FileProtector.FileInfo createFileInfo(FileEntity fileEntity, String convertTo) {
FileProtector.Type type = determineFileType(fileEntity.getMimeType());
return new FileProtector.FileInfo(type, fileEntity.getId(), String.valueOf(fileEntity.getUserId()));
return new FileProtector.FileInfo(type, fileEntity.getId(), String.valueOf(fileEntity.getUserId()), convertTo);
}
public int hamming64(long aHi, long aLo, long bHi, long bLo) {