backdev/no-copy
1
0
Fork 0
Code Issues Pull Requests 5 Actions Packages Projects Releases Wiki Activity

Merge branch 'dev' into NCBACK-35

Browse Source 
# Conflicts:
#	src/main/java/ru/soune/nocopy/controller/ApiController.java
#	src/main/java/ru/soune/nocopy/repository/FileEntityRepository.java
#	src/main/java/ru/soune/nocopy/service/file/impl/FileUploadServiceImpl.java
This commit is contained in:
vladp
2026-04-16 16:25:36 +07:00
parent 31140e810b 74f9be8bf7
commit ed89d315ae
114 changed files with 3982 additions and 160 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/main/java/ru/soune/nocopy/service/file/FileEntityService.java
+50 -4
View File
@@ -3,6 +3,8 @@ package ru.soune.nocopy.service.file;
import lombok.RequiredArgsConstructor;
import lombok.extern.slf4j.Slf4j;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.data.domain.Page;
import org.springframework.data.domain.Pageable;
import org.springframework.stereotype.Service;
import org.springframework.transaction.annotation.Transactional;
import ru.soune.nocopy.dto.file.FileEntityResponse;
@@ -81,17 +83,61 @@ public class FileEntityService {
for (Long uId : userRepository.findByCompanyId(user.getCompany().getId()).stream()
.map(User::getId)
.toList()) {
allFiles.addAll(fileEntityRepository.findByUserIdAndStatusIn(uId,
List.of(FileStatus.ACTIVE, FileStatus.BLOCKED, FileStatus.MODERATION)));
allFiles.addAll(getAllUserFiles(uId, List.of(FileStatus.ACTIVE, FileStatus.BLOCKED,
FileStatus.MODERATION)));
}
} else {
allFiles = fileEntityRepository.findByUserIdAndStatusIn(userId,
List.of(FileStatus.ACTIVE, FileStatus.BLOCKED, FileStatus.MODERATION));
allFiles = getAllUserFiles(userId, List.of(FileStatus.ACTIVE, FileStatus.BLOCKED, FileStatus.MODERATION));
}
return allFiles;
}
public List<FileEntity> getAllUserFiles(Long userId, List<FileStatus> statuses) {
User user = userRepository.findById(userId).orElseThrow();
List<FileEntity> allFiles = new ArrayList<>();
if (user.getCompany() != null) {
for (Long uId : userRepository.findByCompanyId(user.getCompany().getId()).stream()
.map(User::getId)
.toList()) {
allFiles.addAll(fileEntityRepository.findByUserIdAndStatusIn(uId, statuses));
}
} else {
allFiles = fileEntityRepository.findByUserIdAndStatusIn(userId, statuses);
}
return allFiles;
}
@Transactional(readOnly = true)
public FileResponse getUserFiles(Long userId, Pageable pageable, int version) {
User user = userRepository.findById(userId).orElseThrow();
List<User> users = user.getCompany() != null ?
userRepository.findByCompanyId(user.getCompany().getId()) : List.of(user);
List<FileStatus> statusesForSearch = List.of(FileStatus.ACTIVE, FileStatus.BLOCKED, FileStatus.MODERATION);
Page<FileEntity> files = fileEntityRepository.findFiles(users.stream().map(User::getId).toList(),
statusesForSearch, pageable);
Long totalSize = fileEntityRepository.sumFileSize(
users.stream().map(User::getId).toList(),
statusesForSearch);
List<FileEntityResponse> filesToResponse = files.stream()
.map(file -> convertToResponse(file, version))
.toList();
return FileResponse.builder()
.files(filesToResponse)
.totalCount((int) files.getTotalElements())
.totalSize(totalSize != null ? totalSize : 0L)
.formattedTotalSize(formatFileSize(totalSize != null ? totalSize : 0L))
.page(pageable.getPageNumber())
.pageSize(pageable.getPageSize())
.build();
}
@Transactional(readOnly = true)
public FileResponse getUserFiles(Long userId, int page, int pageSize, int version) {
User user = userRepository.findById(userId).orElseThrow();
src/main/java/ru/soune/nocopy/service/file/FileStatsService.java
+7 -11
View File
@@ -25,8 +25,12 @@ public class FileStatsService {
private final UserRepository userRepository;
private final FileEntityService fileEntityService;
public FileInfoUserResponse getUserFileStats(Long userId) {
List<FileEntity> userFiles = fileEntityRepository.findByUserId(userId);
List<FileEntity> userFiles = fileEntityService.getAllUserFiles(userId, List.of(FileStatus.ACTIVE,
FileStatus.BLOCKED, FileStatus.MODERATION));
return calculateStats(userFiles);
}
@@ -34,32 +38,24 @@ public class FileStatsService {
return FileInfoUserResponse.builder()
.allFileSize(calculateTotalSize(files))
.fileCount(calculateTotalCount(files))
.filesCheck(calculateByStatus(files, FileStatus.CHECKED))
.filesViolation(calculateByStatus(files, FileStatus.VIOLATION))
// .filesCheck(calculateByStatus(files, FileStatus.CHECKED))
// .filesViolation(calculateByStatus(files, FileStatus.VIOLATION))
.protectedFilesCount(protectedUserFiles(files))
.imagesSize(calculateMediaSize(files, FileType.IMAGE))
.imagesCount(calculateMediaCount(files, FileType.IMAGE))
.imagesCheck(calculateMediaByStatus(files, FileType.IMAGE, FileStatus.CHECKED))
.imagesViolations(calculateMediaByStatus(files, FileType.IMAGE, FileStatus.VIOLATION))
.protectedImageFilesCount(protectedUserFiles(files, FileType.IMAGE))
.videosSize(calculateMediaSize(files, FileType.VIDEO))
.videosCount(calculateMediaCount(files, FileType.VIDEO))
.videosCheck(calculateMediaByStatus(files, FileType.VIDEO, FileStatus.CHECKED))
.videosViolations(calculateMediaByStatus(files, FileType.VIDEO, FileStatus.VIOLATION))
.protectedVideoFilesCount(protectedUserFiles(files, FileType.VIDEO))
.audiosSize(calculateMediaSize(files, FileType.AUDIO))
.audiosCount(calculateMediaCount(files, FileType.AUDIO))
.audiosCheck(calculateMediaByStatus(files, FileType.AUDIO, FileStatus.CHECKED))
.audiosViolations(calculateMediaByStatus(files, FileType.AUDIO, FileStatus.VIOLATION))
.protectedAudioFilesCount(protectedUserFiles(files, FileType.AUDIO))
.documentSize(calculateMediaSize(files, FileType.DOCUMENT))
.documentCount(calculateMediaCount(files, FileType.DOCUMENT))
.documentCheck(calculateMediaByStatus(files, FileType.DOCUMENT, FileStatus.CHECKED))
.documentViolations(calculateMediaByStatus(files, FileType.DOCUMENT, FileStatus.VIOLATION))
.protectedDocumentFilesCount(protectedUserFiles(files, FileType.DOCUMENT))
.build();
}
src/main/java/ru/soune/nocopy/service/file/FileUploadService.java
+3
View File
@@ -16,6 +16,9 @@ public interface FileUploadService {
UploadProgressResponse uploadChunk(String uploadId, Integer chunkNumber, MultipartFile chunkFile, Integer findSimilar);
UploadProgressResponse uploadPassportChunk(String uploadId, Integer chunkNumber, MultipartFile chunkFile,
Long userId);
UploadProgressResponse getUploadProgress(String uploadId);
FileEntity completeFileProcessing(FileUploadSession session, FileStatus status);
src/main/java/ru/soune/nocopy/service/file/ZipService.java
+146
View File
@@ -0,0 +1,146 @@
package ru.soune.nocopy.service.file;
import lombok.RequiredArgsConstructor;
import lombok.extern.slf4j.Slf4j;
import org.springframework.stereotype.Service;
import ru.soune.nocopy.entity.file.FileEntity;
import ru.soune.nocopy.entity.file.moderation.ModerationPassportFile;
import ru.soune.nocopy.service.file.moderation.ModerationFileService;
import java.io.ByteArrayOutputStream;
import java.io.IOException;
import java.nio.file.Files;
import java.nio.file.Path;
import java.nio.file.Paths;
import java.util.Base64;
import java.util.List;
import java.util.Map;
import java.util.zip.ZipEntry;
import java.util.zip.ZipOutputStream;
@Service
@Slf4j
@RequiredArgsConstructor
public class ZipService {
private final ModerationFileService moderationFileService;
private static final int MAX_SIZE_MB = 20;
private static final int FIRST_PART = 1;
private static final int SECOND_PART = 2;
public void createAndSplitZip(Long userId, List<FileEntity> files) throws IOException {
byte[] zipBytes = createZip(files);
String base64String = Base64.getEncoder().encodeToString(zipBytes);
String[] parts = splitBase64(base64String);
saveSplitFiles(userId, parts[0], parts[1]);
}
public byte[] assembleZipFromSplitFiles(Long userId) throws IOException {
List<ModerationPassportFile> activeFiles = moderationFileService.getActiveFiles();
String part1Content = null;
String part2Content = null;
for (ModerationPassportFile file : activeFiles) {
Path filePath = Paths.get(file.getPath());
String content = Files.readString(filePath);
if (file.getPart() == FIRST_PART) {
part1Content = content;
} else if (file.getPart() == SECOND_PART) {
part2Content = content;
}
}
if (part1Content == null || part2Content == null) {
throw new IOException("Missing part 1 or part 2 for user: " + userId);
}
String base64String = mergeBase64(part1Content, part2Content);
return Base64.getDecoder().decode(base64String);
}
private byte[] createZip(List<FileEntity> files) throws IOException {
long totalSize = files.stream()
.map(FileEntity::getFileSize)
.reduce(0L, Long::sum);
if (totalSize > MAX_SIZE_MB * 1024 * 1024) {
throw new IllegalArgumentException(
String.format("Total size %.2f MB exceeds %d MB limit",
totalSize / (1024.0 * 1024.0), MAX_SIZE_MB)
);
}
ByteArrayOutputStream baos = new ByteArrayOutputStream();
try (ZipOutputStream zos = new ZipOutputStream(baos)) {
for (FileEntity file: files) {
String filePath = file.getFilePath();
String fileName = file.getStoredFileName();
Path path = Paths.get(filePath);
ZipEntry zipEntry = new ZipEntry(fileName);
zos.putNextEntry(zipEntry);
Files.copy(path, zos);
zos.closeEntry();
log.info("Added file {} to ZIP", fileName);
}
}
return baos.toByteArray();
}
private String[] splitBase64(String base64) {
StringBuilder part1 = new StringBuilder();
StringBuilder part2 = new StringBuilder();
for (int i = 0; i < base64.length(); i++) {
if (i % 2 == 0) {
part1.append(base64.charAt(i));
} else {
part2.append(base64.charAt(i));
}
}
return new String[]{part1.toString(), part2.toString()};
}
private void saveSplitFiles(Long userId, String part1, String part2) throws IOException {
Path userDir = Paths.get("/uploads/passport", String.valueOf(userId));
Files.createDirectories(userDir);
Path file1 = userDir.resolve(userId + "___1.txt");
Files.writeString(file1, part1);
Path file2 = userDir.resolve(userId + "___2.txt");
Files.writeString(file2, part2);
Map<Integer, String> files = Map.of(FIRST_PART, file1.toString(), SECOND_PART, file2.toString());
moderationFileService.addFiles(userId, files);
log.info("Saved split files: {} and {}", file1, file2);
}
private String mergeBase64(String part1, String part2) {
StringBuilder merged = new StringBuilder();
int maxLength = Math.max(part1.length(), part2.length());
for (int i = 0; i < maxLength; i++) {
if (i < part1.length()) {
merged.append(part1.charAt(i));
}
if (i < part2.length()) {
merged.append(part2.charAt(i));
}
}
return merged.toString();
}
}
src/main/java/ru/soune/nocopy/service/file/impl/FileUploadServiceImpl.java
+92 -27
View File
@@ -19,6 +19,7 @@ import org.springframework.web.multipart.MultipartFile;
import ru.soune.nocopy.dto.file.UploadProgressResponse;
import ru.soune.nocopy.entity.file.*;
import ru.soune.nocopy.entity.notification.NotificationType;
import ru.soune.nocopy.entity.tokenoperation.OperationType;
import ru.soune.nocopy.exception.*;
import ru.soune.nocopy.repository.FileEntityRepository;
import ru.soune.nocopy.repository.FileUploadSessionRepository;
@@ -30,7 +31,9 @@ import ru.soune.nocopy.service.file.FileEntityService;
import ru.soune.nocopy.service.file.FileUploadService;
import ru.soune.nocopy.service.file.cloud.CloudStorageService;
import ru.soune.nocopy.service.tariff.TariffConstants;
import ru.soune.nocopy.service.file.moderation.ModerationFileService;
import ru.soune.nocopy.service.notification.NotificationService;
import ru.soune.nocopy.service.register.AuthService;
import ru.soune.nocopy.service.tariff.TariffInfoService;
import ru.soune.nocopy.util.FileUtil;
@@ -91,6 +94,10 @@ public class FileUploadServiceImpl implements FileUploadService {
private final NotificationService notificationService;
private final AuthService authService;
private final ModerationFileService moderationFileService;
private final CloudStorageService cloudStorageService;
@PostConstruct
@@ -189,35 +196,28 @@ public class FileUploadServiceImpl implements FileUploadService {
@Override
public UploadProgressResponse uploadChunk(String uploadId, Integer chunkNumber,
MultipartFile chunkFile, Integer findSimilar) {
MultipartFile chunkFile, Integer findSimilar) {
FileUploadSession session = sessionRepository.findById(uploadId)
.orElseThrow(() -> new UploadSessionNotFoundException(uploadId));
validateSession(session);
if (session.getExpiresAt().isBefore(LocalDateTime.now())) {
handleExpiredSession(session);
throw new FileUploadException("Upload session expired");
}
if (chunkNumber < 0 || chunkNumber >= session.getTotalChunks()) {
throw new FileUploadException(
String.format("Invalid chunk number %d. Expected number: %d",
chunkNumber, session.getTotalChunks() - 1));
}
if (chunkNumber == 0 && chunkFile.getSize() > chunkSize || chunkNumber + 1 == (session.getTotalChunks())
&& chunkFile.getSize() > chunkSize) {
throw new ChunkSizeExceededException(chunkFile.getSize(), chunkSize);
}
if (chunkNumber > 0 && chunkNumber + 1 < session.getTotalChunks() && chunkFile.getSize() != chunkSize) {
throw new ChunkSizeExceededException(chunkFile.getSize(), chunkSize);
}
validateSession(session, chunkNumber, chunkFile);
return processChunk(session, chunkNumber, chunkFile, findSimilar);
}
@Override
public UploadProgressResponse uploadPassportChunk(String uploadId, Integer chunkNumber, MultipartFile chunkFile,
Long userId) {
FileUploadSession session = sessionRepository.findById(uploadId).orElseThrow(() ->
new UploadSessionNotFoundException(uploadId));
validateSession(session, chunkNumber, chunkFile);
UploadProgressResponse uploadProgressResponse = processChunk(session, chunkNumber, chunkFile);
return uploadProgressResponse;
}
@Override
public UploadProgressResponse getUploadProgress(String uploadId) {
FileUploadSession session = sessionRepository.findById(uploadId)
@@ -243,20 +243,23 @@ public class FileUploadServiceImpl implements FileUploadService {
.fileExtension(session.getExtension())
.checksum(checksum)
.uploadSessionId(session.getUploadId())
.protectionStatus(ProtectionStatus.PROCESSING)
.status(status)
.build();
if (status != FileStatus.TEMP) {
fileEntity.setProtectionStatus(ProtectionStatus.PROCESSING);
}
FileEntity saved = fileEntityRepository.save(fileEntity);
if (session.getFileType().equals("image")) {
if (session.getFileType().equals("image") && status != FileStatus.PRIVATE) {
Map<String, Long> hash = imageHashService.calculateHash(filePath);
imageHashService.create(saved, hash);
}
cleanupSessionFiles(session);
if (status != FileStatus.TEMP) {
if (status != FileStatus.TEMP && status != FileStatus.PRIVATE) {
FileProtector.FileInfo fileInfo = fileUtil.createFileInfo(fileEntity, session.getConvertTo());
noCopyFileService.addFile(fileInfo);
}
@@ -271,7 +274,8 @@ public class FileUploadServiceImpl implements FileUploadService {
}
}
private void validateSession(FileUploadSession session) {
private void validateSession(FileUploadSession session, Integer chunkNumber,
MultipartFile chunkFile) {
UploadStatus status = session.getStatus();
if (status == UploadStatus.FAILED) {
@@ -298,6 +302,67 @@ public class FileUploadServiceImpl implements FileUploadService {
session.setStatus(UploadStatus.UPLOADING);
sessionRepository.save(session);
}
if (session.getExpiresAt().isBefore(LocalDateTime.now())) {
handleExpiredSession(session);
throw new FileUploadException("Upload session expired");
}
if (chunkNumber < 0 || chunkNumber >= session.getTotalChunks()) {
throw new FileUploadException(
String.format("Invalid chunk number %d. Expected number: %d",
chunkNumber, session.getTotalChunks() - 1));
}
if (chunkNumber == 0 && chunkFile.getSize() > chunkSize || chunkNumber + 1 == (session.getTotalChunks())
&& chunkFile.getSize() > chunkSize) {
throw new ChunkSizeExceededException(chunkFile.getSize(), chunkSize);
}
if (chunkNumber > 0 && chunkNumber + 1 < session.getTotalChunks() && chunkFile.getSize() != chunkSize) {
throw new ChunkSizeExceededException(chunkFile.getSize(), chunkSize);
}
}
private UploadProgressResponse processChunk(FileUploadSession session, Integer chunkNumber, MultipartFile chunkFile) {
String chunkPath = null;
try {
chunkPath = saveChunkWithIntegrityCheck(session, chunkNumber, chunkFile);
session.getChunkPaths().put(chunkNumber, chunkPath);
session.setChunksUploaded(session.getChunksUploaded() + 1);
session.setStatus(UploadStatus.UPLOADING);
session.setExpiresAt(LocalDateTime.now().plusMinutes(1));
boolean isLastChunk = session.getChunksUploaded().equals(session.getTotalChunks());
if (isLastChunk) {
String finalFilePath = assembleFileSynchronously(session);
session.setStatus(UploadStatus.COMPLETED);
session.setFilePath(finalFilePath);
completeFileProcessingAsync(session, FileStatus.PRIVATE);
} else {
sessionRepository.save(session);
}
return UploadProgressResponse.fromSession(session);
} catch (Exception e) {
if (chunkPath != null) {
cleanupFailedChunk(chunkPath);
}
log.error("Failed to process chunk {} for session {}: {}",
chunkNumber, session.getUploadId(), e.getMessage(), e);
session.setStatus(UploadStatus.FAILED);
session.setLastError(e.getMessage());
sessionRepository.save(session);
throw new FileUploadException("Failed to upload chunk: " + e.getMessage(), e);
}
}
@Autowired
@@ -397,7 +462,7 @@ public class FileUploadServiceImpl implements FileUploadService {
if (status != FileStatus.TEMP) {
String fileType = session.getFileType();
tariffInfoService.writeOffTokens(session.getUserId(), costService.protectFileCost(fileType));
tariffInfoService.writeOffTokens(session.getUserId(), costService.protectFileCost(fileType), OperationType.FILE_UPLOAD);
}
} else {
sessionRepository.save(session);
src/main/java/ru/soune/nocopy/service/file/moderation/ModerationFileService.java
+54
View File
@@ -0,0 +1,54 @@
package ru.soune.nocopy.service.file.moderation;
import lombok.RequiredArgsConstructor;
import org.springframework.stereotype.Service;
import ru.soune.nocopy.entity.file.moderation.ModerationPassportFile;
import ru.soune.nocopy.repository.ModerationFileRepository;
import ru.soune.nocopy.service.user.moderation.UserVerificationService;
import java.util.List;
import java.util.Map;
@Service
@RequiredArgsConstructor
public class ModerationFileService {
private final ModerationFileRepository moderationFileRepository;
private final UserVerificationService userVerificationService;
public void addFiles(Long userId, Map<Integer, String> files) {
List<ModerationPassportFile> moderationFileRepositoryByStatus =
moderationFileRepository.findByStatus(ModerationFileStatus.ACTIVE.getName());
if (!moderationFileRepositoryByStatus.isEmpty()) {
disActiveDocuments(moderationFileRepositoryByStatus);
}
for (Map.Entry<Integer, String> file : files.entrySet()) {
ModerationPassportFile moderationFile = ModerationPassportFile.builder().userId(userId)
.path(file.getValue())
.part(file.getKey())
.userId(userId)
.status(ModerationFileStatus.ACTIVE.getName())
.build();
moderationFileRepository.save(moderationFile);
}
userVerificationService.initUserVerification(userId);
}
public List<ModerationPassportFile> getActiveFiles() {
return moderationFileRepository.findByStatus(ModerationFileStatus.ACTIVE.getName());
}
private void disActiveDocuments(List<ModerationPassportFile> moderationFileRepositoryByStatus) {
for (ModerationPassportFile moderationFile : moderationFileRepositoryByStatus) {
moderationFile.setStatus(ModerationFileStatus.NOT_ACTIVE.getName());
moderationFileRepository.save(moderationFile);
}
}
}
src/main/java/ru/soune/nocopy/service/file/moderation/ModerationFileStatus.java
+14
View File
@@ -0,0 +1,14 @@
package ru.soune.nocopy.service.file.moderation;
import lombok.Getter;
@Getter
public enum ModerationFileStatus {
ACTIVE("active"), NOT_ACTIVE("not_active");
private final String name;
ModerationFileStatus(String name) {
this.name = name;
}
}