backdev/no-copy
1
0
Fork 0
Code Issues Pull Requests 5 Actions Packages Projects Releases Wiki Activity

Merge branch 'dev' into NCBACK-3
Test Workflow / test (push) Successful in 5s
Details

Browse Source 
# Conflicts:
#	docker-compose.yaml
#	src/main/java/ru/soune/nocopy/dto/MessageCode.java
This commit is contained in:
vladp
2026-01-09 12:34:24 +07:00
parent ffb147d716 63baaf7c80
commit edd508e5d8
21 changed files with 745 additions and 432 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/main/java/ru/soune/nocopy/service/file/FileEntityService.java
+18 -9
View File
@@ -17,6 +17,7 @@ import java.nio.file.Files;
import java.nio.file.Path;
import java.nio.file.Paths;
import java.time.LocalDateTime;
import java.time.ZoneOffset;
import java.util.List;
import java.util.stream.Collectors;
@@ -40,7 +41,6 @@ public class FileEntityService {
long fileSize = Files.size(filePath);
String originalName = session.getFileName();
String extension = extractFileExtension(originalName);
String storedName = filePath.getFileName().toString();
FileEntity fileEntity = FileEntity.builder()
@@ -50,7 +50,7 @@ public class FileEntityService {
.filePath(session.getFilePath())
.fileSize(fileSize)
.mimeType(session.getFileType())
.fileExtension(extension)
.fileExtension(session.getExtension())
.checksum(checksum)
.uploadSessionId(session.getUploadId())
.status(FileStatus.ACTIVE)
@@ -168,15 +168,24 @@ public class FileEntityService {
}
@Transactional
public void markAsDeleted(String fileId) {
FileEntity fileEntity = fileEntityRepository.findById(fileId)
.orElseThrow(() -> new FileEntityNotFoundException(fileId));
public void markAsDeleted(FileEntity fileEntity) {
fileEntity.setStatus(FileStatus.DELETED);
fileEntity.setUpdatedAt(LocalDateTime.now());
fileEntityRepository.save(fileEntity);
log.info("FileEntity marked as deleted: {}", fileId);
fileEntityRepository.save(fileEntity);
}
@Transactional
public boolean deleteFromDisk(FileEntity fileEntity) throws IOException {
Path path = Paths.get(fileEntity.getFilePath());
if (!Files.exists(path)) {
return true;
}
Files.delete(path);
return true;
}
@Transactional(readOnly = true)
@@ -217,7 +226,7 @@ public class FileEntityService {
.build();
}
private String formatFileSize(long size) {
public String formatFileSize(long size) {
if (size < 1024) {
return size + " B";
} else if (size < 1024 * 1024) {
src/main/java/ru/soune/nocopy/service/file/FileStatsService.java
+93
View File
@@ -0,0 +1,93 @@
package ru.soune.nocopy.service.file;
import lombok.RequiredArgsConstructor;
import org.springframework.stereotype.Service;
import ru.soune.nocopy.dto.file.FileInfoUserResponse;
import ru.soune.nocopy.entity.file.FileEntity;
import ru.soune.nocopy.entity.file.FileStatus;
import ru.soune.nocopy.entity.file.FileType;
import ru.soune.nocopy.repository.FileEntityRepository;
import java.util.List;
@Service
@RequiredArgsConstructor
public class FileStatsService {
private final FileEntityRepository fileEntityRepository;
public FileInfoUserResponse getUserFileStats(Long userId) {
List<FileEntity> userFiles = fileEntityRepository.findByUserId(userId);
return calculateStats(userFiles);
}
public FileInfoUserResponse calculateStats(List<FileEntity> files) {
return FileInfoUserResponse.builder()
.allFileSize(calculateTotalSize(files))
.fileCount(calculateTotalCount(files))
.filesCheck(calculateByStatus(files, FileStatus.CHECKED))
.filesViolation(calculateByStatus(files, FileStatus.VIOLATION))
.imagesSize(calculateMediaSize(files, FileType.IMAGE))
.imagesCount(calculateMediaCount(files, FileType.IMAGE))
.imagesCheck(calculateMediaByStatus(files, FileType.IMAGE, FileStatus.CHECKED))
.imagesViolations(calculateMediaByStatus(files, FileType.IMAGE, FileStatus.VIOLATION))
.videosSize(calculateMediaSize(files, FileType.VIDEO))
.videosCount(calculateMediaCount(files, FileType.VIDEO))
.videosCheck(calculateMediaByStatus(files, FileType.VIDEO, FileStatus.CHECKED))
.videosViolations(calculateMediaByStatus(files, FileType.VIDEO, FileStatus.VIOLATION))
.audiosSize(calculateMediaSize(files, FileType.AUDIO))
.audiosCount(calculateMediaCount(files, FileType.AUDIO))
.audiosCheck(calculateMediaByStatus(files, FileType.AUDIO, FileStatus.CHECKED))
.audiosViolations(calculateMediaByStatus(files, FileType.AUDIO, FileStatus.VIOLATION))
.build();
}
private Long calculateTotalSize(List<FileEntity> files) {
return files.stream()
.filter(file -> file.getStatus() != FileStatus.DELETED)
.mapToLong(FileEntity::getFileSize)
.sum();
}
private Integer calculateTotalCount(List<FileEntity> files) {
return (int) files.stream()
.filter(file -> file.getStatus() != FileStatus.DELETED)
.count();
}
private Integer calculateByStatus(List<FileEntity> files, FileStatus status) {
return (int) files.stream()
.filter(file -> file.getStatus() == status)
.count();
}
private Long calculateMediaSize(List<FileEntity> files, FileType fileType) {
return files.stream()
.filter(file -> file.getStatus() != FileStatus.DELETED)
.filter(file -> isFileType(file, fileType))
.mapToLong(FileEntity::getFileSize)
.sum();
}
private Integer calculateMediaCount(List<FileEntity> files, FileType fileType) {
return (int) files.stream()
.filter(file -> file.getStatus() != FileStatus.DELETED)
.filter(file -> isFileType(file, fileType))
.count();
}
private Integer calculateMediaByStatus(List<FileEntity> files, FileType fileType, FileStatus status) {
return (int) files.stream()
.filter(file -> isFileType(file, fileType))
.filter(file -> file.getStatus() == status)
.count();
}
private boolean isFileType(FileEntity file, FileType fileType) {
if (file.getFileExtension() == null) return false;
String extension = file.getFileExtension().toLowerCase().replace(".", "");
return fileType.supportsExtension(extension);
}
}
src/main/java/ru/soune/nocopy/service/file/FileUploadService.java
+1 -1
View File
@@ -15,7 +15,7 @@ public interface FileUploadService {
UploadProgressResponse getUploadProgress(String uploadId);
void cleanupExpiredSessions();
void handleExpiredSession(FileUploadSession session);
public void retryFailedUpload(String uploadId);
src/main/java/ru/soune/nocopy/service/file/FileUploadServiceImpl.java
+120 -146
View File
@@ -102,12 +102,14 @@ public class FileUploadServiceImpl implements FileUploadService {
int totalChunks = (int) Math.ceil((double) fileSize / chunkSize);
log.debug("File will be split into {} chunks (chunk size: {} bytes)",
totalChunks, chunkSize);
Long chunkSize = totalChunks == 1 ? fileSize : 1000000L;
FileUploadSession session = FileUploadSession.builder()
.userId(userId)
.extension(extension)
.fileName(fileName)
.fileType(fileType)
.chunkSize(chunkSize)
.fileSize(fileSize)
.totalChunks(totalChunks)
.chunksUploaded(0)
@@ -143,17 +145,103 @@ public class FileUploadServiceImpl implements FileUploadService {
if (chunkNumber < 0 || chunkNumber >= session.getTotalChunks()) {
throw new FileUploadException(
String.format("Invalid chunk number %d. Expected 0-%d",
String.format("Invalid chunk number %d. Expected number: %d",
chunkNumber, session.getTotalChunks() - 1));
}
if (chunkFile.getSize() > chunkSize) {
if (chunkNumber == 0 && chunkFile.getSize() > chunkSize || chunkNumber + 1 == (session.getTotalChunks())
&& chunkFile.getSize() > chunkSize) {
throw new ChunkSizeExceededException(chunkFile.getSize(), chunkSize);
}
if (chunkNumber > 0 && chunkNumber + 1 < session.getTotalChunks() && chunkFile.getSize() != chunkSize) {
throw new ChunkSizeExceededException(chunkFile.getSize(), chunkSize);
}
return processChunk(session, chunkNumber, chunkFile);
}
@Override
@Transactional
public void handleExpiredSession(FileUploadSession session) {
session.setStatus(UploadStatus.FAILED);
session.setLastError("Upload session expired");
sessionRepository.save(session);
CompletableFuture.runAsync(() -> cleanupSessionFiles(session));
}
@Async("fileUploadTaskExecutor")
@Transactional
public void assembleFileAsync(FileUploadSession session) {
try {
assembleFile(session);
log.info("File assembly completed successfully for session: {}",
session.getUploadId());
} catch (Exception e) {
log.error("Failed to assemble file for session {}: {}",
session.getUploadId(), e.getMessage(), e);
handleAssemblyFailure(session, e);
}
}
@Override
@Transactional
public void cancelUpload(String uploadId) {
FileUploadSession session = sessionRepository.findById(uploadId)
.orElseThrow(() -> new UploadSessionNotFoundException(uploadId));
if (session.getStatus() == UploadStatus.COMPLETED || session.getStatus() == UploadStatus.CANCELLED) {
throw new FileUploadException("Cannot cancel completed or cancelled upload");
}
session.setStatus(UploadStatus.CANCELLED);
sessionRepository.save(session);
CompletableFuture.runAsync(() -> cleanupSessionFiles(session));
log.info("Upload cancelled: {}", uploadId);
}
@Override
@Transactional
public void retryFailedUpload(String uploadId) {
FileUploadSession session = sessionRepository.findById(uploadId)
.orElseThrow(() -> new UploadSessionNotFoundException(uploadId));
if (session.getStatus() != UploadStatus.FAILED) {
throw new FileUploadException("Only failed uploads can be retried");
}
if (session.getRetryCount() >= maxRetryAttempts) {
throw new FileUploadException("Max retry attempts exceeded");
}
session.setStatus(UploadStatus.UPLOADING);
session.setRetryCount(session.getRetryCount() + 1);
session.setLastError(null);
sessionRepository.save(session);
if (session.getChunksUploaded().equals(session.getTotalChunks())) {
log.info("Retrying file assembly for session: {}", uploadId);
assembleFileAsync(session);
}
log.info("Upload retry initiated for session: {} (attempt {})",
uploadId, session.getRetryCount());
}
@Override
public UploadProgressResponse getUploadProgress(String uploadId) {
FileUploadSession session = sessionRepository.findById(uploadId)
.orElseThrow(() -> new UploadSessionNotFoundException(uploadId));
return UploadProgressResponse.fromSession(session);
}
private void validateSession(FileUploadSession session) {
UploadStatus status = session.getStatus();
@@ -183,14 +271,6 @@ public class FileUploadServiceImpl implements FileUploadService {
}
}
private void handleExpiredSession(FileUploadSession session) {
session.setStatus(UploadStatus.FAILED);
session.setLastError("Upload session expired");
sessionRepository.save(session);
CompletableFuture.runAsync(() -> cleanupSessionFiles(session));
}
private UploadProgressResponse processChunk(FileUploadSession session,
Integer chunkNumber,
MultipartFile chunkFile) {
@@ -206,6 +286,7 @@ public class FileUploadServiceImpl implements FileUploadService {
session.getChunkPaths().put(chunkNumber, chunkPath);
session.setChunksUploaded(session.getChunksUploaded() + 1);
session.setStatus(UploadStatus.UPLOADING);
session.setExpiresAt(LocalDateTime.now().plusMinutes(1));
log.debug("Chunk {} saved successfully. Uploaded: {}/{}",
chunkNumber, session.getChunksUploaded(), session.getTotalChunks());
@@ -316,52 +397,6 @@ public class FileUploadServiceImpl implements FileUploadService {
}
}
@Async("fileUploadTaskExecutor")
@Transactional
public void assembleFileAsync(FileUploadSession session) {
try {
assembleFile(session);
log.info("File assembly completed successfully for session: {}",
session.getUploadId());
} catch (Exception e) {
log.error("Failed to assemble file for session {}: {}",
session.getUploadId(), e.getMessage(), e);
handleAssemblyFailure(session, e);
}
}
// private void assembleFile(FileUploadSession session) throws IOException {
// log.info("Starting file assembly for session: {} ({})",
// session.getUploadId(), session.getFileName());
//
// Path finalFilePath = null;
//
// try {
// finalFilePath = prepareFinalFile(session);
//
// validateAllChunksExist(session);
//
// mergeChunksToFile(session, finalFilePath);
//
// validateFinalFile(session, finalFilePath);
//
// updateSessionOnSuccess(session, finalFilePath);
//
// cleanupSessionFiles(session);
//
// log.info("File assembly completed: {} -> {} ({} bytes)",
// session.getFileName(), finalFilePath, session.getFileSize());
//
// } catch (Exception e) {
// if (finalFilePath != null) {
// Files.deleteIfExists(finalFilePath);
// }
// throw e;
// }
// }
private void assembleFile(FileUploadSession session) throws IOException {
log.info("Starting file assembly for session: {} ({})",
session.getUploadId(), session.getFileName());
@@ -526,14 +561,6 @@ public class FileUploadServiceImpl implements FileUploadService {
log.debug("Final file checksum: {}", checksum);
}
private void updateSessionOnSuccess(FileUploadSession session, Path finalPath) {
session.setFilePath(finalPath.toString());
session.setChecksum(calculateChecksum(finalPath));
session.setStatus(UploadStatus.COMPLETED);
session.setCompletedAt(LocalDateTime.now());
sessionRepository.save(session);
}
private void cleanupSessionFiles(FileUploadSession session) {
try {
Path chunkDir = getChunkDirectory(session.getUserId(), session.getUploadId());
@@ -570,90 +597,37 @@ public class FileUploadServiceImpl implements FileUploadService {
session.getUploadId(), e.getMessage());
}
@Override
@Transactional
public void cancelUpload(String uploadId) {
FileUploadSession session = sessionRepository.findById(uploadId)
.orElseThrow(() -> new UploadSessionNotFoundException(uploadId));
if (session.getStatus() == UploadStatus.COMPLETED || session.getStatus() == UploadStatus.CANCELLED) {
throw new FileUploadException("Cannot cancel completed or cancelled upload");
}
session.setStatus(UploadStatus.CANCELLED);
sessionRepository.save(session);
CompletableFuture.runAsync(() -> cleanupSessionFiles(session));
log.info("Upload cancelled: {}", uploadId);
}
@Override
@Transactional
public void retryFailedUpload(String uploadId) {
FileUploadSession session = sessionRepository.findById(uploadId)
.orElseThrow(() -> new UploadSessionNotFoundException(uploadId));
if (session.getStatus() != UploadStatus.FAILED) {
throw new FileUploadException("Only failed uploads can be retried");
}
if (session.getRetryCount() >= maxRetryAttempts) {
throw new FileUploadException("Max retry attempts exceeded");
}
session.setStatus(UploadStatus.UPLOADING);
session.setRetryCount(session.getRetryCount() + 1);
session.setLastError(null);
sessionRepository.save(session);
if (session.getChunksUploaded().equals(session.getTotalChunks())) {
log.info("Retrying file assembly for session: {}", uploadId);
assembleFileAsync(session);
}
log.info("Upload retry initiated for session: {} (attempt {})",
uploadId, session.getRetryCount());
}
@Override
public UploadProgressResponse getUploadProgress(String uploadId) {
FileUploadSession session = sessionRepository.findById(uploadId)
.orElseThrow(() -> new UploadSessionNotFoundException(uploadId));
return UploadProgressResponse.fromSession(session);
}
@Override
@Transactional
public void cleanupExpiredSessions() {
log.info("Starting cleanup of expired upload sessions");
LocalDateTime expiryThreshold = LocalDateTime.now().minusHours(sessionExpiryHours);
try {
var expiredSessions = sessionRepository.findByStatusInAndCreatedAtBefore(
Set.of(UploadStatus.INITIATED, UploadStatus.UPLOADING, UploadStatus.FAILED),
expiryThreshold
);
int cleanedCount = 0;
for (FileUploadSession session : expiredSessions) {
session.setStatus(UploadStatus.FAILED);
session.setLastError("Session expired during cleanup");
sessionRepository.save(session);
CompletableFuture.runAsync(() -> cleanupSessionFiles(session));
cleanedCount++;
}
log.info("Cleaned up {} expired upload sessions", cleanedCount);
} catch (Exception e) {
log.error("Error during session cleanup", e);
}
}
// @Override
// @Transactional
// public void cleanupExpiredSessions() {
// log.info("Starting cleanup of expired upload sessions");
//
// LocalDateTime expiryThreshold = LocalDateTime.now().minusHours(sessionExpiryHours);
//
// try {
// var expiredSessions = sessionRepository.findByStatusInAndCreatedAtBefore(
// Set.of(UploadStatus.INITIATED, UploadStatus.UPLOADING, UploadStatus.FAILED),
// expiryThreshold
// );
//
// int cleanedCount = 0;
// for (FileUploadSession session : expiredSessions) {
// session.setStatus(UploadStatus.FAILED);
// session.setLastError("Session expired during cleanup");
// sessionRepository.save(session);
//
// CompletableFuture.runAsync(() -> cleanupSessionFiles(session));
// cleanedCount++;
// }
//
// log.info("Cleaned up {} expired upload sessions", cleanedCount);
//
// } catch (Exception e) {
// log.error("Error during session cleanup", e);
// }
// }
private Path getChunkDirectory(Long userId, String uploadId) {
return storageRoot.resolve("temp")
src/main/java/ru/soune/nocopy/service/file/UploadSessionCleanupService.java
+35
View File
@@ -0,0 +1,35 @@
package ru.soune.nocopy.service.file;
import lombok.RequiredArgsConstructor;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.scheduling.annotation.Scheduled;
import org.springframework.stereotype.Service;
import org.springframework.transaction.annotation.Transactional;
import ru.soune.nocopy.entity.file.FileUploadSession;
import ru.soune.nocopy.entity.file.UploadStatus;
import ru.soune.nocopy.repository.FileUploadSessionRepository;
import java.time.LocalDateTime;
import java.util.List;
import java.util.Set;
@Service
@RequiredArgsConstructor
public class UploadSessionCleanupService {
@Autowired
private FileUploadSessionRepository fileUploadSessionRepository;
@Autowired
private FileUploadService fileUploadService;
@Transactional
@Scheduled(fixedDelay = 30000)
public void cleanupSessions() {
List<FileUploadSession> expiredSessions = fileUploadSessionRepository.findExpiredSessions(
LocalDateTime.now(), Set.of(UploadStatus.INITIATED, UploadStatus.UPLOADING));
for (FileUploadSession session : expiredSessions) {
fileUploadService.handleExpiredSession(session);
}
}
}