add bone for file system

2025-12-15 19:49:09 +07:00
parent 07aad9f51c
commit 4ddf2f5dde
24 changed files with 871 additions and 7 deletions
@@ -5,7 +5,7 @@ import org.springframework.stereotype.Service;
 import org.springframework.transaction.annotation.Transactional;
 import ru.soune.no_copy.dto.UserContentRequest;
 import ru.soune.no_copy.dto.UserContentUpdateRequest;
-import ru.soune.no_copy.entity.FileType;
+import ru.soune.no_copy.entity.file.FileType;
 import ru.soune.no_copy.entity.User;
 import ru.soune.no_copy.entity.UserContent;
 import ru.soune.no_copy.exception.ContentNotFoundException;
@@ -0,0 +1,80 @@
+package ru.soune.no_copy.service.file;
+
+import org.springframework.beans.factory.annotation.Value;
+import org.springframework.scheduling.annotation.Scheduled;
+import org.springframework.stereotype.Component;
+import lombok.RequiredArgsConstructor;
+import lombok.extern.slf4j.Slf4j;
+
+import java.io.IOException;
+import java.nio.file.*;
+import java.nio.file.attribute.BasicFileAttributes;
+import java.time.Instant;
+import java.time.LocalDateTime;
+import java.time.ZoneId;
+
+@Slf4j
+@Component
+@RequiredArgsConstructor
+public class FileCleanupService {
+
+    @Value("${file.storage.base-path}")
+    private String basePath;
+
+    @Value("${file.storage.temp-ttl-hours}")
+    private int tempTtlHours;
+
+    @Scheduled(cron = "0 0 3 * * *")
+    public void cleanupExpiredFiles() {
+        log.info("Starting cleanup of expired temporary files");
+
+        Path tempDir = Paths.get(basePath, "temp");
+        if (!Files.exists(tempDir)) {
+            return;
+        }
+
+        try {
+            Files.walkFileTree(tempDir, new SimpleFileVisitor<Path>() {
+                @Override
+                public FileVisitResult visitFile(Path file, BasicFileAttributes attrs)
+                        throws IOException {
+
+                    LocalDateTime fileTime = LocalDateTime.ofInstant(
+                            Instant.ofEpochMilli(attrs.lastModifiedTime().toMillis()),
+                            ZoneId.systemDefault()
+                    );
+
+                    LocalDateTime cutoffTime = LocalDateTime.now()
+                            .minusHours(tempTtlHours);
+
+                    if (fileTime.isBefore(cutoffTime)) {
+                        Files.delete(file);
+                        log.debug("Deleted expired file: {}", file);
+                    }
+
+                    return FileVisitResult.CONTINUE;
+                }
+
+                @Override
+                public FileVisitResult postVisitDirectory(Path dir, IOException exc)
+                        throws IOException {
+
+                    if (exc == null) {
+                        if (Files.list(dir).count() == 0 &&
+                                !dir.equals(tempDir)) {
+                            Files.delete(dir);
+                            log.debug("Deleted empty directory: {}", dir);
+                        }
+                    }
+
+                    return FileVisitResult.CONTINUE;
+                }
+            });
+
+            log.info("Cleanup completed successfully");
+
+        } catch (IOException e) {
+            log.error("Error during cleanup", e);
+        }
+    }
+}
@@ -0,0 +1,17 @@
+package ru.soune.no_copy.service.file;
+
+import org.springframework.web.multipart.MultipartFile;
+import ru.soune.no_copy.dto.file.UploadProgressResponse;
+import ru.soune.no_copy.entity.file.FileUploadSession;
+
+public interface FileUploadService {
+    FileUploadSession initUpload(Long userId, String fileName,
+                                 String fileType, long fileSize);
+
+    UploadProgressResponse uploadChunk(String uploadId, Integer chunkNumber,
+                                       MultipartFile chunkFile);
+
+    UploadProgressResponse getUploadProgress(String uploadId);
+
+    void cleanupExpiredSessions();
+}
@@ -0,0 +1,364 @@
+package ru.soune.no_copy.service.file;
+
+import jakarta.annotation.PostConstruct;
+import lombok.RequiredArgsConstructor;
+import lombok.extern.slf4j.Slf4j;
+import org.springframework.beans.factory.annotation.Value;
+import org.springframework.scheduling.annotation.Async;
+import org.springframework.stereotype.Service;
+import org.springframework.transaction.annotation.Transactional;
+import org.springframework.web.multipart.MultipartFile;
+import ru.soune.no_copy.dto.file.UploadProgressResponse;
+import ru.soune.no_copy.entity.file.FileUploadSession;
+import ru.soune.no_copy.entity.file.UploadStatus;
+import ru.soune.no_copy.exception.ChunkSizeExceededException;
+import ru.soune.no_copy.exception.FileUploadException;
+import ru.soune.no_copy.exception.UploadSessionNotFoundException;
+import ru.soune.no_copy.repository.FileUploadSessionRepository;
+
+import java.io.*;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.nio.file.Paths;
+import java.nio.file.StandardOpenOption;
+import java.security.MessageDigest;
+import java.security.NoSuchAlgorithmException;
+import java.time.LocalDateTime;
+import java.util.Comparator;
+import java.util.stream.Stream;
+
+@Slf4j
+@Service
+@RequiredArgsConstructor
+public class FileUploadServiceImpl implements FileUploadService {
+
+    private final FileUploadSessionRepository sessionRepository;
+
+    @Value("${file.storage.base-path}")
+    private String basePath;
+
+    @Value("${file.storage.chunk-size}")
+    private int chunkSize;
+
+    @Value("${file.storage.max-file-size}")
+    private long maxFileSize;
+
+    @PostConstruct
+    public void init() {
+        try {
+            Path storageRoot = Paths.get(basePath).toAbsolutePath().normalize();
+
+            log.info("Storage will be at: {}", storageRoot);
+
+            if (!Files.exists(storageRoot)) {
+                Files.createDirectories(storageRoot);
+                log.info("Created: {}", storageRoot);
+            }
+
+            Files.createDirectories(storageRoot.resolve("temp"));
+            Files.createDirectories(storageRoot.resolve("user"));
+
+        } catch (IOException e) {
+            log.error("CANNOT CREATE STORAGE!", e);
+            throw new RuntimeException("Storage failed", e);
+        }
+    }
+
+    @Override
+    @Transactional
+    public FileUploadSession initUpload(Long userId, String fileName,
+                                        String fileType, long fileSize) {
+        if (fileSize > maxFileSize) {
+            throw new FileUploadException(
+                    String.format("File size %d exceeds maximum allowed size %d",
+                            fileSize, maxFileSize));
+        }
+
+        int totalChunks = (int) Math.ceil((double) fileSize / chunkSize);
+
+        FileUploadSession session = FileUploadSession.builder()
+                .userId(userId)
+                .fileName(fileName)
+                .fileType(fileType)
+                .fileSize(fileSize)
+                .totalChunks(totalChunks)
+                .chunksUploaded(0)
+                .status(UploadStatus.INITIATED)
+                .build();
+
+        return sessionRepository.save(session);
+    }
+
+    @Override
+    @Transactional
+    public UploadProgressResponse uploadChunk(String uploadId, Integer chunkNumber,
+                                              MultipartFile chunkFile) {
+
+        log.info("Uploading chunk {} for session {}, chunk size: {} bytes",
+                chunkNumber, uploadId, chunkFile.getSize());
+
+        FileUploadSession session = sessionRepository.findById(uploadId)
+                .orElseThrow(() -> new UploadSessionNotFoundException(uploadId));
+
+        if (session.getChunkPaths().containsKey(chunkNumber)) {
+            log.info("Chunk {} already uploaded for session {}", chunkNumber, uploadId);
+            return UploadProgressResponse.fromSession(session);
+        }
+
+        if (chunkFile.getSize() > chunkSize) {
+            throw new ChunkSizeExceededException(chunkFile.getSize(), chunkSize);
+        }
+
+        try {
+            log.debug("Chunk {} actual size: {} bytes", chunkNumber, chunkFile.getSize());
+
+            String chunkPath = saveChunkToDisk(session, chunkNumber, chunkFile);
+
+            session.getChunkPaths().put(chunkNumber, chunkPath);
+            session.setChunksUploaded(session.getChunksUploaded() + 1);
+
+            log.info("Chunk {} uploaded successfully. Total uploaded: {}/{}",
+                    chunkNumber, session.getChunksUploaded(), session.getTotalChunks());
+
+            if (session.getChunksUploaded().equals(session.getTotalChunks())) {
+                session.setStatus(UploadStatus.UPLOADING);
+                sessionRepository.save(session);
+
+                log.info("All chunks uploaded for session {}. Starting assembly...", uploadId);
+
+                assembleFileAsync(session);
+            } else {
+                sessionRepository.save(session);
+            }
+
+            return UploadProgressResponse.fromSession(session);
+
+        } catch (IOException e) {
+            log.error("Failed to save chunk {}", chunkNumber, e);
+            session.setStatus(UploadStatus.FAILED);
+            sessionRepository.save(session);
+            throw new RuntimeException("Failed to save chunk: " + e.getMessage(), e);
+        }
+    }
+
+    private String saveChunkToDisk(FileUploadSession session,
+                                   Integer chunkNumber,
+                                   MultipartFile chunkFile) throws IOException {
+
+        Long userId = session.getUserId();
+        String uploadId = session.getUploadId();
+
+        Path chunkDir = Paths.get(basePath, "temp",
+                "user", String.valueOf(userId), uploadId);
+        Files.createDirectories(chunkDir);
+
+        String chunkFileName = String.format("chunk_%03d.tmp", chunkNumber);
+        Path chunkPath = chunkDir.resolve(chunkFileName);
+
+        try (InputStream inputStream = chunkFile.getInputStream();
+             OutputStream outputStream = Files.newOutputStream(chunkPath,
+                     StandardOpenOption.CREATE, StandardOpenOption.TRUNCATE_EXISTING)) {
+
+            byte[] buffer = new byte[8192];
+            int bytesRead;
+            long totalBytes = 0;
+
+            while ((bytesRead = inputStream.read(buffer)) != -1) {
+                outputStream.write(buffer, 0, bytesRead);
+                totalBytes += bytesRead;
+            }
+
+            log.debug("Chunk {} saved to {}: {} bytes written",
+                    chunkNumber, chunkPath, totalBytes);
+
+        } catch (Exception e) {
+            log.error("Error saving chunk {} to {}", chunkNumber, chunkPath, e);
+            throw e;
+        }
+
+        long savedSize = Files.size(chunkPath);
+        log.debug("Chunk {} saved size: {} bytes", chunkNumber, savedSize);
+
+        return chunkPath.toString();
+    }
+
+    @Async("fileUploadTaskExecutor")
+    @Transactional
+    public void assembleFileAsync(FileUploadSession session) {
+        try {
+            assembleFile(session);
+            log.info("File assembly completed for session {}", session.getUploadId());
+
+        } catch (Exception e) {
+            log.error("Failed to assemble file for session {}",
+                    session.getUploadId(), e);
+
+            session.setStatus(UploadStatus.FAILED);
+            sessionRepository.save(session);
+        }
+    }
+
+    private void assembleFile(FileUploadSession session) throws IOException {
+        Long userId = session.getUserId();
+        String uploadId = session.getUploadId();
+
+        log.info("Assembling file for session {}: {} ({} chunks)",
+                uploadId, session.getFileName(), session.getTotalChunks());
+
+        Path finalDir = createUserDirectory(userId, session.getFileType());
+
+        String safeFileName = generateSafeFileName(session.getFileName());
+        Path finalFilePath = finalDir.resolve(safeFileName);
+
+        log.debug("Final file path: {}", finalFilePath);
+
+        try (OutputStream outputStream = new BufferedOutputStream(
+                Files.newOutputStream(finalFilePath, StandardOpenOption.CREATE))) {
+
+            Path chunkDir = getChunkDirectory(userId, uploadId);
+            long totalAssembled = 0;
+
+            for (int i = 0; i < session.getTotalChunks(); i++) {
+                Path chunkPath = chunkDir.resolve(String.format("chunk_%03d.tmp", i));
+
+                if (!Files.exists(chunkPath)) {
+                    throw new IOException("Missing chunk: " + chunkPath);
+                }
+
+                byte[] chunkData = Files.readAllBytes(chunkPath);
+                outputStream.write(chunkData);
+                totalAssembled += chunkData.length;
+
+                log.debug("Added chunk {}: {} bytes (total: {})",
+                        i, chunkData.length, totalAssembled);
+            }
+
+            log.info("File assembled: {} bytes total", totalAssembled);
+        }
+
+        try {
+            validateFileIntegrity(finalFilePath, session.getFileSize());
+        } catch (IOException e) {
+            log.warn("File size mismatch (temporarily ignored): {}", e.getMessage());
+        }
+
+        session.setFilePath(finalFilePath.toString());
+        session.setChecksum(calculateChecksum(finalFilePath));
+        session.setStatus(UploadStatus.COMPLETED);
+        session.setCompletedAt(LocalDateTime.now());
+        sessionRepository.save(session);
+
+        log.info("File assembled successfully: {} -> {}",
+                session.getFileName(), finalFilePath);
+
+        try {
+            cleanupChunks(getChunkDirectory(userId, uploadId));
+        } catch (IOException e) {
+            log.warn("Failed to cleanup chunks: {}", e.getMessage());
+        }
+    }
+
+    private Path createUserDirectory(Long userId, String fileType) throws IOException {
+        Path userDir = Paths.get(basePath, "user", String.valueOf(userId), extractCategory(fileType));
+
+        Files.createDirectories(userDir);
+
+        return userDir;
+    }
+
+    private String extractCategory(String fileType) {
+        if (fileType.startsWith("image/")) {
+            return "images";
+        } else if (fileType.startsWith("video/")) {
+            return "videos";
+        } else if (fileType.startsWith("audio/")) {
+            return "audio";
+        } else if (fileType.contains("pdf")) {
+            return "documents";
+        } else if (fileType.contains("text")) {
+            return "text";
+        } else {
+            return "other";
+        }
+    }
+
+    private String generateSafeFileName(String originalName) {
+        String safeName = originalName.replaceAll("[^a-zA-Z0-9\\.\\-_]", "_");
+
+        String timestamp = String.valueOf(System.currentTimeMillis());
+        int dotIndex = safeName.lastIndexOf('.');
+
+        if (dotIndex > 0) {
+            String name = safeName.substring(0, dotIndex);
+            String extension = safeName.substring(dotIndex);
+            return name + "_" + timestamp + extension;
+        } else {
+            return safeName + "_" + timestamp;
+        }
+    }
+
+    private void validateFileIntegrity(Path filePath, long expectedSize) throws IOException {
+        long actualSize = Files.size(filePath);
+        log.info("File integrity check: expected={}, actual={}", expectedSize, actualSize);
+
+        if (actualSize != expectedSize) {
+            throw new IOException(
+                    String.format("File size mismatch: expected %d, got %d",
+                            expectedSize, actualSize));
+        }
+    }
+
+    private String calculateChecksum(Path filePath) throws IOException {
+        try {
+            MessageDigest md = MessageDigest.getInstance("MD5");
+            byte[] buffer = new byte[8192];
+
+            try (InputStream inputStream = Files.newInputStream(filePath)) {
+                int bytesRead;
+                while ((bytesRead = inputStream.read(buffer)) != -1) {
+                    md.update(buffer, 0, bytesRead);
+                }
+            }
+
+            byte[] digest = md.digest();
+            StringBuilder sb = new StringBuilder();
+            for (byte b : digest) {
+                sb.append(String.format("%02x", b));
+            }
+
+            return sb.toString();
+
+        } catch (NoSuchAlgorithmException e) {
+            log.warn("MD5 algorithm not available, skipping checksum", e);
+            return "N/A";
+        }
+    }
+
+    private void cleanupChunks(Path chunkDir) throws IOException {
+        if (Files.exists(chunkDir)) {
+            try (Stream<Path> walk = Files.walk(chunkDir)) {
+                walk.sorted(Comparator.reverseOrder())
+                        .map(Path::toFile)
+                        .forEach(File::delete);
+                log.info("Cleaned up chunk directory: {}", chunkDir);
+            }
+        }
+    }
+
+    private Path getChunkDirectory(Long userId, String uploadId) {
+        return Paths.get(basePath, "temp", "user", String.valueOf(userId), uploadId);
+    }
+
+    @Override
+    public UploadProgressResponse getUploadProgress(String uploadId) {
+        FileUploadSession session = sessionRepository.findById(uploadId)
+                .orElseThrow(() -> new UploadSessionNotFoundException(uploadId));
+
+        return UploadProgressResponse.fromSession(session);
+    }
+
+    @Override
+    public void cleanupExpiredSessions() {
+        log.info("Cleanup not implemented yet");
+    }
+}