backdev/no-copy
1
0
Fork 0
Code Issues Pull Requests 5 Actions Packages Projects Releases Wiki Activity

Merge branch 'dev' into NCBACK-35

Browse Source 
# Conflicts:
#	README.md
#	src/main/java/ru/soune/nocopy/controller/ApiController.java
#	src/main/java/ru/soune/nocopy/handler/FileEntityHandler.java
#	src/main/java/ru/soune/nocopy/service/file/FileCleanupService.java
#	src/main/java/ru/soune/nocopy/service/file/FileEntityService.java
#	src/main/java/ru/soune/nocopy/service/file/impl/FileUploadServiceImpl.java
#	src/main/java/ru/soune/nocopy/service/file/impl/ProtectionFileProviderImpl.java
This commit is contained in:
vladp
2026-03-31 14:24:03 +07:00
parent daef30ec11 ad4850414a
commit d7874aa37a
229 changed files with 10416 additions and 757 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/main/java/ru/soune/nocopy/service/file/impl/AudioLocalSearchImpl.java
+3 -1
View File
@@ -26,7 +26,9 @@ public class AudioLocalSearchImpl implements AudioLocalSearch {
public @Nullable FileProtector.FileInfo findBySignature(@NotNull String signature) {
FileEntity fileEntity = fileEntityService.findBySignature(signature);
return fileUtil.createFileInfo(fileEntity);
if (fileEntity == null) return null;
return fileUtil.createFileInfo(fileEntity, null);
}
@Override
src/main/java/ru/soune/nocopy/service/file/impl/DocumentLocalSearchImpl.java
+1 -1
View File
@@ -23,7 +23,7 @@ public class DocumentLocalSearchImpl implements DocumentLocalSearch {
@Override
public @Nullable FileProtector.FileInfo findByIds(@NotNull String ownerId, @NotNull String fileId) {
FileEntity fileEntity = fileEntityRepository.findByIdAndUserId(fileId, Long.valueOf(ownerId));
FileProtector.FileInfo fileInfo = fileUtil.createFileInfo(fileEntity);
FileProtector.FileInfo fileInfo = fileUtil.createFileInfo(fileEntity, null);
log.info("fileInfo: {}", fileInfo);
return fileInfo;
src/main/java/ru/soune/nocopy/service/file/impl/FileUploadServiceImpl.java
+141 -99
View File
@@ -2,27 +2,29 @@ package ru.soune.nocopy.service.file.impl;
import com.vrt.NoCopyFileService;
import com.vrt.fileprotection.FileProtector;
import com.vrt.fileprotection.NoCopyCheckResult;
import com.vrt.fileprotection.audio.AudioCheckResult;
import com.vrt.fileprotection.documents.DocumentCheckResult;
import jakarta.annotation.PostConstruct;
import jakarta.persistence.EntityManager;
import lombok.RequiredArgsConstructor;
import lombok.extern.slf4j.Slf4j;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.context.annotation.Lazy;
import org.springframework.scheduling.annotation.Async;
import org.springframework.stereotype.Service;
import org.springframework.transaction.annotation.Transactional;
import org.springframework.web.multipart.MultipartFile;
import ru.soune.nocopy.dto.file.UploadProgressResponse;
import ru.soune.nocopy.entity.file.*;
import ru.soune.nocopy.exception.ChunkSizeExceededException;
import ru.soune.nocopy.exception.DuplicateImageException;
import ru.soune.nocopy.exception.FileUploadException;
import ru.soune.nocopy.exception.UploadSessionNotFoundException;
import ru.soune.nocopy.exception.*;
import ru.soune.nocopy.repository.FileEntityRepository;
import ru.soune.nocopy.repository.FileUploadSessionRepository;
import ru.soune.nocopy.repository.SimilarImageProjection;
import ru.soune.nocopy.service.FileSimilarityService;
import ru.soune.nocopy.service.ImageHashService;
import ru.soune.nocopy.service.cost.CostService;
import ru.soune.nocopy.service.file.FileEntityService;
import ru.soune.nocopy.service.file.FileUploadService;
import ru.soune.nocopy.service.file.cloud.CloudStorageService;
@@ -59,9 +61,6 @@ public class FileUploadServiceImpl implements FileUploadService {
@Value("${file.storage.max-retry-attempts:3}")
private int maxRetryAttempts;
@Value("${file.storage.chunk-timeout-ms:300000}") // 5 минут
private long chunkTimeoutMs;
@Value("${file.storage.session-expiry-hours:24}")
private int sessionExpiryHours;
@@ -79,6 +78,9 @@ public class FileUploadServiceImpl implements FileUploadService {
@Autowired
private FileEntityService fileEntityService;
private final CostService costService;
@Lazy
private final NoCopyFileService noCopyFileService;
private final FileUtil fileUtil;
@@ -112,7 +114,7 @@ public class FileUploadServiceImpl implements FileUploadService {
@Override
@Transactional
public FileUploadSession initUpload(Long userId, String fileName,
String fileType, String extension, long fileSize) {
String fileType, String extension, String convertTo, long fileSize) {
log.info("Initializing upload for user {}: {} ({} bytes, type: {})",
userId, fileName, fileSize, fileType);
@@ -143,6 +145,7 @@ public class FileUploadServiceImpl implements FileUploadService {
.status(UploadStatus.INITIATED)
.expiresAt(LocalDateTime.now().plusHours(sessionExpiryHours))
.retryCount(0)
.convertTo(convertTo)
.build();
FileUploadSession savedSession = sessionRepository.save(session);
@@ -252,6 +255,12 @@ public class FileUploadServiceImpl implements FileUploadService {
cleanupSessionFiles(session);
if (status != FileStatus.TEMP) {
FileProtector.FileInfo fileInfo = fileUtil.createFileInfo(fileEntity, session.getConvertTo());
noCopyFileService.addFile(fileInfo);
}
log.info("File processing completed for session: {}", session.getUploadId());
} catch (Exception e) {
log.error("Failed to complete file processing for session {}: {}",
@@ -313,21 +322,58 @@ public class FileUploadServiceImpl implements FileUploadService {
chunkNumber, session.getChunksUploaded(), session.getTotalChunks());
boolean isLastChunk = session.getChunksUploaded().equals(session.getTotalChunks());
//TODO CHECK
if (isLastChunk) {
log.info("All chunks uploaded for session {}. Starting assembly...",
session.getUploadId());
String finalFilePath = assembleFileSynchronously(session);
if (session.getFileType().startsWith("image") && findSimilar == 0) {
checkForDuplicatesSynchronously(finalFilePath);
if (findSimilar == 0) {
if (session.getFileType().startsWith("image")) {
checkForDuplicatesSynchronously(finalFilePath);
} else if (session.getFileType().startsWith("audio") || session.getFileType().startsWith("document")) {
fileSimilarityService.hasDuplicatesByHash(finalFilePath ,session.getFileType(),
session.getUserId());
File file = new File(finalFilePath);
if (session.getFileType().startsWith("audio") && !isWavFile(file)) {
session.setStatus(UploadStatus.FAILED);
session.setLastError("wav file not have RIFF header");
sessionRepository.save(session);
throw new FileFormatException("Failed to upload chunk: .wav file not have RIFF header or " +
"another problems with .wav file");
}
FileProtector.Type type = "document".equals(session.getFileType()) ?
FileProtector.Type.DOC : FileProtector.Type.valueOf(session.getFileType().toUpperCase());
NoCopyCheckResult checkResult = noCopyFileService.checkFile(file, type);
switch (checkResult) {
case DocumentCheckResult.Success success -> {
FileProtector.FileInfo info = success.getInfo();
throw new DuplicateImageException("Duplicate", info.getId(),
Long.valueOf(info.getOwnerId()));
}
case AudioCheckResult.Success success -> {
FileProtector.FileInfo info = success.getInfo();
throw new DuplicateImageException("Duplicate", info.getId(),
Long.valueOf(info.getOwnerId()));
}
case DocumentCheckResult.Failed failed ->
log.warn("Document check failed: {}", failed.getMessage());
case AudioCheckResult.Failed failed ->
log.warn("Audio check failed: {}", failed.getMessage());
default -> log.warn("Unexpected result type: {}", checkResult.getClass().getSimpleName());
}
}
}
session.setStatus(UploadStatus.COMPLETED);
session.setFilePath(finalFilePath);
FileStatus status = findSimilar == 0 ? FileStatus.ACTIVE: FileStatus.TEMP;
FileStatus status = findSimilar == 0 ? FileStatus.MODERATION: FileStatus.TEMP;
if (status == FileStatus.TEMP) {
fileEntityService.clearTempFiles(session.getUserId());
@@ -351,7 +397,7 @@ public class FileUploadServiceImpl implements FileUploadService {
if (status != FileStatus.TEMP) {
String fileType = session.getFileType();
tariffInfoService.writeOffTokens(session.getUserId(), protectCost(fileType));
tariffInfoService.writeOffTokens(session.getUserId(), costService.protectFileCost(fileType));
}
} else {
sessionRepository.save(session);
@@ -388,21 +434,87 @@ public class FileUploadServiceImpl implements FileUploadService {
}
}
private int protectCost(String fileType) {
int cost = 0;
switch (fileType) {
case "video":
return TariffConstants.VIDEO_TOKEN_VALUE;
case "image":
return TariffConstants.IMAGE_TOKEN_VALUE;
case "audio":
return TariffConstants.AUDIO_TOKEN_VALUE;
case "document":
return TariffConstants.PDF_TOKEN_VALUE;
//TODO убрать когда научимся защищать другие типы
public boolean isWavFile(File file) {
if (file == null || !file.exists() || file.length() < 12) {
return false;
}
return cost;
try (FileInputStream fis = new FileInputStream(file)) {
byte[] header = new byte[4];
int bytesRead = fis.read(header);
if (bytesRead < 4) {
return false;
}
return header[0] == 82 && header[1] == 73 &&
header[2] == 70 && header[3] == 70;
} catch (IOException e) {
e.printStackTrace();
return false;
}
}
public static boolean isValidWav(File file) {
if (file == null || !file.exists() || file.length() < 44) {
return false;
}
try (FileInputStream fis = new FileInputStream(file)) {
byte[] header = new byte[44];
if (fis.read(header) < 44) {
return false;
}
if (header[0] != 'R' || header[1] != 'I' ||
header[2] != 'F' || header[3] != 'F') {
return false;
}
if (header[8] != 'W' || header[9] != 'A' ||
header[10] != 'V' || header[11] != 'E') {
return false;
}
if (header[12] != 'f' || header[13] != 'm' ||
header[14] != 't' || header[15] != ' ') {
return false;
}
if (header[36] != 'd' || header[37] != 'a' ||
header[38] != 't' || header[39] != 'a') {
return false;
}
int audioFormat = (header[20] & 0xFF) | (header[21] & 0xFF) << 8;
if (audioFormat != 1 && audioFormat != 3) {
return false;
}
int numChannels = (header[22] & 0xFF) | (header[23] & 0xFF) << 8;
if (numChannels < 1 || numChannels > 8) {
return false;
}
int sampleRate = (header[24] & 0xFF) | (header[25] & 0xFF) << 8 |
(header[26] & 0xFF) << 16 | (header[27] & 0xFF) << 24;
if (sampleRate < 8000 || sampleRate > 192000) {
return false;
}
int bitsPerSample = (header[34] & 0xFF) | (header[35] & 0xFF) << 8;
if (bitsPerSample < 8 || bitsPerSample > 32) {
return false;
}
return true;
} catch (IOException e) {
e.printStackTrace();
return false;
}
}
private String assembleFileSynchronously(FileUploadSession session) throws IOException {
@@ -510,76 +622,6 @@ public class FileUploadServiceImpl implements FileUploadService {
}
}
private void assembleFile(FileUploadSession session) throws IOException {
log.info("Starting file assembly for session: {} ({})",
session.getUploadId(), session.getFileName());
Path finalFilePath = null;
String checksum;
try {
finalFilePath = prepareFinalFile(session);
log.info("Final file path: {}", finalFilePath);
validateAllChunksExist(session);
mergeChunksToFile(session, finalFilePath);
validateFinalFile(session, finalFilePath);
checksum = calculateChecksum(finalFilePath);
log.debug("File checksum calculated: {}", checksum);
session.setFilePath(finalFilePath.toString());
session.setChecksum(checksum);
session.setStatus(UploadStatus.COMPLETED);
session.setCompletedAt(LocalDateTime.now());
sessionRepository.save(session);
log.info("Upload session updated to COMPLETED: {}", session.getUploadId());
try {
FileEntity fileEntity = fileEntityService.createFromUploadSession(session, checksum);
FileProtector.FileInfo fileInfo = fileUtil.createFileInfo(fileEntity);
noCopyFileService.addFile(fileInfo);
log.info("FileEntity successfully created for session: {}",
session.getUploadId());
} catch (DuplicateImageException e) {
throw new DuplicateImageException("Duplicate", e.duplicateFileId(), e.userId());
} catch (Exception e) {
log.error("Failed to create FileEntity for session {} (file uploaded, but metadata not saved): {}",
session.getUploadId(), e.getMessage());
}
cleanupSessionFiles(session);
log.info("File assembly completed successfully: {} -> {} ({} bytes)",
session.getFileName(), finalFilePath, session.getFileSize());
} catch (Exception e) {
log.error("File assembly failed for session {}: {}",
session.getUploadId(), e.getMessage(), e);
if (finalFilePath != null) {
try {
Files.deleteIfExists(finalFilePath);
log.debug("Cleaned up partial file: {}", finalFilePath);
} catch (IOException ioException) {
log.warn("Failed to cleanup partial file: {}", finalFilePath, ioException);
}
}
session.setStatus(UploadStatus.FAILED);
session.setLastError(e.getMessage());
sessionRepository.save(session);
throw new IOException("File assembly failed: " + e.getMessage(), e);
}
}
private Path prepareFinalFile(FileUploadSession session) throws IOException {
Path userUploadsDir = storageRoot.resolve("uploads")
.resolve(String.valueOf(session.getUserId()))
src/main/java/ru/soune/nocopy/service/file/impl/ImageLocalSearchImpl.java
+5 -3
View File
@@ -44,11 +44,13 @@ public class ImageLocalSearchImpl implements ImageLocalSearch {
@Override
public @Nullable FileProtector.FileInfo findByPHash(@NotNull PHash pHash) {
ImageHashEntity imageHashEntity =
imageHashRepository.findByHash64HiAndHash64Lo(pHash.getFirstPart(),
pHash.getSecondPart());
imageHashRepository.findByHash64HiAndHash64Lo(pHash.getFirstPart(), pHash.getSecondPart());
if (imageHashEntity == null) return null;
FileEntity file = fileEntityRepository.findByFileId(imageHashEntity.getFileId());
return new FileProtector.FileInfo(FileProtector.Type.IMAGE, imageHashEntity.getFileId(),
String.valueOf(file.getUserId()));
String.valueOf(file.getUserId()), null);
}
}
src/main/java/ru/soune/nocopy/service/file/impl/NoCopyProcessingListenerImpl.java
+4 -4
View File
@@ -16,21 +16,21 @@ public class NoCopyProcessingListenerImpl implements FileProtector.ProcessingLis
@Override
public void onStartProcessing(FileProtector.FileInfo fileInfo) {
fileEntityService.changeStatus(ProtectionStatus.PROCESSING, fileInfo.getId());
fileEntityService.changeProtectionStatus(ProtectionStatus.PROCESSING, fileInfo.getId());
}
@Override
public void onProcessingFailed(FileProtector.FileInfo fileInfo) {
fileEntityService.changeStatus(ProtectionStatus.FAILED, fileInfo.getId());
fileEntityService.changeProtectionStatus(ProtectionStatus.FAILED, fileInfo.getId());
}
@Override
public void onSavingFailed(FileProtector.FileInfo fileInfo) {
fileEntityService.changeStatus(ProtectionStatus.FAILED_SAVE, fileInfo.getId());
fileEntityService.changeProtectionStatus(ProtectionStatus.FAILED_SAVE, fileInfo.getId());
}
@Override
public void onFinish(FileProtector.FileInfo fileInfo) {
fileEntityService.changeStatus(ProtectionStatus.PROTECTED, fileInfo.getId());;
fileEntityService.changeProtectionStatus(ProtectionStatus.PROTECTED, fileInfo.getId());;
}
}
src/main/java/ru/soune/nocopy/service/file/impl/ProtectionFileProviderImpl.java
+8
View File
@@ -9,7 +9,11 @@ import org.jetbrains.annotations.Nullable;
import org.springframework.stereotype.Component;
import ru.soune.nocopy.entity.file.FileEntity;
import ru.soune.nocopy.repository.FileEntityRepository;
import ru.soune.nocopy.repository.FileUploadSessionRepository;
import ru.soune.nocopy.entity.file.FileEntity;
import ru.soune.nocopy.repository.FileEntityRepository;
import ru.soune.nocopy.service.file.FileEntityService;
import ru.soune.nocopy.service.file.FileUploadService;
import ru.soune.nocopy.service.file.cloud.CloudStorageService;
import java.io.File;
@@ -27,6 +31,10 @@ public class ProtectionFileProviderImpl implements FileProtector.FileProvider {
private final FileEntityService fileEntityService;
private final FileUploadSessionRepository fileUploadSessionRepository;
private final FileEntityRepository fileEntityRepository;
private final CloudStorageService cloudStorageService;
private final FileEntityRepository fileEntityRepository;