NCBACK-35 add worked logic for save protected files

src/main/java/ru/soune/nocopy/controller/ApiController.java

@@ -74,6 +74,8 @@ public class ApiController {
 
     private final AuthTokenRepository authTokenRepository;
 
+    private final CloudStorageService cloudStorageService;
+
     @PostMapping("/v{version}/data")
     public ResponseEntity<?> handlePostRequest(@RequestBody BaseRequest request,
                                                @PathVariable("version") int version) {
@@ -292,8 +294,6 @@ public class ApiController {
         }
     }
 
-    private final CloudStorageService cloudStorageService;
-
     @GetMapping("/v{version}/files/download/{fileId}")
     public ResponseEntity<?> downloadFile(
             @PathVariable(required = false) String fileId,
@@ -441,33 +441,6 @@ public class ApiController {
         return ResponseEntity.ok().body(fileTypeStats);
     }
 
-    private boolean hasDuplicate(List<SimilarFileDTO> similarFiles) {
-        return similarFiles.stream().anyMatch(f -> f.getHammingDistance() <= 5);
-    }
-
-    private ResponseEntity<BaseResponse> handleDuplicate(FileEntity fileEntity, List<SimilarFileDTO> similarFiles)
-            throws IOException {
-        fileEntityService.deleteFromDisk(fileEntity);
-
-        fileEntityService.softDeleteFileWithHash(fileEntity);
-
-        Optional<FileEntity> originalFile = fileEntityRepository.findById(similarFiles.get(0).getFileId());
-
-        if (originalFile.isPresent()) {
-            Map<String, String> duplicateInfo = Map.of(
-                    "duplicate_file_id", originalFile.get().getId(),
-                    "owner_user_id", String.valueOf(originalFile.get().getUserId()));
-
-            return ResponseEntity.ok().body(new BaseResponse(
-                    20004,
-                    MessageCode.DUPLICATE_FILE_UPLOAD.getCode(),
-                    "Failed to upload chunk, duplicate",
-                    duplicateInfo));
-        }
-
-        return null;
-    }
-
     private ResponseEntity<BaseResponse> buildSuccessResponse(String uploadId, Integer chunkNumber, MultipartFile chunk,
                                                               String fileId) {
         ChunkUploadResponse responseBody = ChunkUploadResponse.builder()

src/main/java/ru/soune/nocopy/controller/PhotoController.java

@@ -1,86 +0,0 @@
-package ru.soune.nocopy.controller;
-
-import io.swagger.v3.oas.annotations.responses.ApiResponse;
-import io.swagger.v3.oas.annotations.responses.ApiResponses;
-import io.swagger.v3.oas.annotations.tags.Tag;
-import org.springframework.http.HttpHeaders;
-import org.springframework.http.ResponseEntity;
-import org.springframework.web.bind.annotation.*;
-import org.springframework.web.multipart.MultipartFile;
-import software.amazon.awssdk.auth.credentials.AwsBasicCredentials;
-import software.amazon.awssdk.auth.credentials.AwsCredentials;
-import software.amazon.awssdk.auth.credentials.StaticCredentialsProvider;
-import software.amazon.awssdk.core.sync.RequestBody;
-import software.amazon.awssdk.http.apache.ApacheHttpClient;
-import software.amazon.awssdk.regions.Region;
-import software.amazon.awssdk.services.s3.S3Client;
-import software.amazon.awssdk.services.s3.model.GetObjectRequest;
-import software.amazon.awssdk.services.s3.model.PutObjectRequest;
-
-import java.io.IOException;
-import java.net.URI;
-
-import static org.springframework.http.MediaType.MULTIPART_FORM_DATA_VALUE;
-
-@RestController
-@RequestMapping("/photos")
-@Tag(name = "Photos")
-@ApiResponses(@ApiResponse(responseCode = "200", useReturnTypeSchema = true))
-public class PhotoController {
-
-    private static final String KEY_ID = "YCAJEmHZcWxzf4oGWvETG3mms";
-    private static final String SECRET_KEY = "YCPAPZ32XC-LWb8xE0b_xTcBD8NZUCHpSOfu8LUG";
-    private static final String REGION = "ru-central1";
-    private static final String S3_ENDPOINT = "https://storage.yandexcloud.net";
-
-    private static final String BUCKET = "ncp-test-storage";
-
-    private final S3Client s3Client;
-
-    public PhotoController() {
-        AwsCredentials credentials = AwsBasicCredentials.create(KEY_ID, SECRET_KEY);
-
-        s3Client = S3Client.builder()
-                .httpClient(ApacheHttpClient.create())
-                .region(Region.of(REGION))
-                .endpointOverride(URI.create(S3_ENDPOINT))
-                .credentialsProvider(StaticCredentialsProvider.create(credentials))
-                .build();
-    }
-
-
-    @PutMapping(consumes = MULTIPART_FORM_DATA_VALUE)
-    public String uploadFile(@RequestParam MultipartFile photo) throws IOException {
-
-        String key = "photos/" + photo.getOriginalFilename();
-        PutObjectRequest putObjectRequest = PutObjectRequest.builder()
-                .bucket(BUCKET)
-                .key(key)
-                .contentType(photo.getContentType())
-                .build();
-
-        s3Client.putObject(putObjectRequest, RequestBody.fromBytes(photo.getBytes()));
-
-        return key;
-    }
-
-    @GetMapping
-    public ResponseEntity<byte[]> downloadFile(@RequestParam String key) throws IOException {
-        GetObjectRequest objectRequest = GetObjectRequest.builder()
-                .bucket(BUCKET)
-                .key(key)
-                .build();
-
-        var inputStream = s3Client.getObject(objectRequest);
-        byte[] data = inputStream.readAllBytes();
-
-        var headers = new HttpHeaders();
-        headers.add(HttpHeaders.CONTENT_DISPOSITION, "inline");
-        headers.add(HttpHeaders.CONTENT_TYPE, inputStream.response().contentType());
-
-        return ResponseEntity.ok()
-                .headers(headers)
-                .body(data);
-    }
-}
-

src/main/java/ru/soune/nocopy/handler/FileEntityHandler.java

@@ -302,7 +302,7 @@ public class FileEntityHandler implements RequestHandler {
             int fullDelete = fileRequest.getFullDelete() == null ? 0 : fileRequest.getFullDelete();
 
             if (fileEntity.getStatus().equals(FileStatus.DELETED) || fullDelete == 1) {
-                fileEntityService.deleteFromDisk(fileEntity);
+                fileEntityService.deleteFromStorageDisk(fileEntity);
                 response = DeleteFileResponse.builder()
                         .fileId(fileRequest.getFileId())
                         .message("File deleted from disk")

src/main/java/ru/soune/nocopy/service/file/FileCleanupService.java

@@ -8,6 +8,7 @@ import lombok.extern.slf4j.Slf4j;
 import ru.soune.nocopy.entity.user.ProtectedFileCheck;
 import ru.soune.nocopy.repository.ProtectedFileCheckRepository;
 
+import java.io.File;
 import java.io.IOException;
 import java.nio.file.*;
 import java.nio.file.attribute.BasicFileAttributes;
@@ -15,7 +16,10 @@ import java.time.Instant;
 import java.time.LocalDate;
 import java.time.LocalDateTime;
 import java.time.ZoneId;
+import java.time.temporal.ChronoUnit;
 import java.util.List;
+import java.util.stream.Collectors;
+import java.util.stream.Stream;
 
 @Slf4j
 @Component
@@ -35,6 +39,7 @@ public class FileCleanupService {
         log.info("Starting cleanup of expired temporary files");
 
         cleanupOldPeriods();
+        cleanupTempFilesNightly();
 
         Path tempDir = Paths.get(basePath, "temp");
         if (!Files.exists(tempDir)) {
@@ -98,4 +103,50 @@ public class FileCleanupService {
         protectedFileCheckRepository.saveAll(checks);
         log.info("Cleaned up {} old protection usage records", checks.size());
     }
+
+    private void cleanupTempFilesNightly() {
+        log.info("Starting nightly cleanup of temporary files");
+
+        String tempDir = System.getProperty("java.io.tmpdir");
+        Path tempPath = Paths.get(tempDir);
+
+        if (!Files.exists(tempPath)) {
+            return;
+        }
+
+        Instant cutoffTime = Instant.now().minus(24, ChronoUnit.HOURS); // 24 часа
+        int deletedCount = 0;
+
+        try (Stream<Path> files = Files.list(tempPath)) {
+            List<Path> tempFiles = files
+                    .filter(path -> {
+                        String filename = path.getFileName().toString();
+                        return filename.contains("_") &&
+                                !filename.endsWith(".tmp") &&
+                                Files.isRegularFile(path);
+                    })
+                    .collect(Collectors.toList());
+
+            for (Path filePath : tempFiles) {
+                try {
+                    File file = filePath.toFile();
+                    Instant lastModified = Instant.ofEpochMilli(file.lastModified());
+
+                    if (lastModified.isBefore(cutoffTime)) {
+                        boolean deleted = Files.deleteIfExists(filePath);
+                        if (deleted) {
+                            deletedCount++;
+                        }
+                    }
+                } catch (IOException e) {
+                    log.error("Error deleting file: {}", filePath, e);
+                }
+            }
+
+            log.info("Nightly cleanup deleted {} temporary files", deletedCount);
+
+        } catch (IOException e) {
+            log.error("Failed to list temp directory", e);
+        }
+    }
 }

src/main/java/ru/soune/nocopy/service/file/FileEntityService.java

@@ -18,8 +18,8 @@ import ru.soune.nocopy.repository.SimilarImageProjection;
 import ru.soune.nocopy.repository.UserRepository;
 import ru.soune.nocopy.service.FileSimilarityService;
 import ru.soune.nocopy.service.ImageHashService;
+import ru.soune.nocopy.service.file.cloud.CloudStorageService;
 
-import java.io.File;
 import java.io.IOException;
 import java.nio.file.Files;
 import java.nio.file.Path;
@@ -43,6 +43,8 @@ public class FileEntityService {
 
     private final UserRepository userRepository;
 
+    private final CloudStorageService cloudStorageService;
+
     @Transactional(noRollbackFor = DuplicateImageException.class)
     public FileEntity createFromUploadSession(FileUploadSession session, String checksum) {
         log.info("Creating FileEntity for upload session: {}", session.getUploadId());
@@ -56,7 +58,7 @@ public class FileEntityService {
 
             Map<String, Long> imageHash = Map.of();
 
-            if (session.getFileType().startsWith("image")) {
+            if (!session.getFileType().startsWith("video")) {
                 imageHash = imageHashService.calculateHash(filePath);
                 List<SimilarImageProjection> duplicatedByHash = fileSimilarityService.findDuplicatedByHash(
                         imageHash.get("hi"), imageHash.get("low"));
@@ -117,38 +119,6 @@ public class FileEntityService {
         return convertToResponse(fileEntity, version);
     }
 
-    @Transactional(readOnly = true)
-    public FileEntityResponse getByFilePath(String filePath, int version) {
-        FileEntity fileEntity = fileEntityRepository.findByFilePath(filePath)
-                .orElseThrow(() -> new FileEntityNotFoundException("Path: " + filePath));
-
-        return convertToResponse(fileEntity, version);
-    }
-
-
-    @Transactional(readOnly = true)
-    public FileResponse getAllUserFiles(Long userId, int version) {
-        List<FileEntity> fileEntities = fileEntityRepository.findByUserIdAndStatus(
-                userId, FileStatus.ACTIVE);
-
-        List<FileEntityResponse> files = fileEntities.stream()
-                .map(file -> convertToResponse(file, version))
-                .collect(Collectors.toList());
-
-        long totalSize = fileEntities.stream()
-                .mapToLong(FileEntity::getFileSize)
-                .sum();
-
-        return FileResponse.builder()
-                .files(files)
-                .totalCount(files.size())
-                .totalSize(totalSize)
-                .formattedTotalSize(formatFileSize(totalSize))
-                .page(1)
-                .pageSize(files.size())
-                .build();
-    }
-
     @Transactional(readOnly = true)
     public FileResponse getUserFiles(Long userId, int page, int pageSize, int version) {
         User user = userRepository.findById(userId).orElseThrow();
@@ -200,19 +170,6 @@ public class FileEntityService {
                 .build();
     }
 
-    @Transactional
-    public FileEntity markAsDeleted(FileEntity fileEntity) throws IOException {
-        fileEntity.setStatus(FileStatus.DELETED);
-        fileEntity.setUpdatedAt(LocalDateTime.now());
-        fileEntity.setProtectionStatus(ProtectionStatus.NOT_PROTECTED);
-
-        Path path = Paths.get(fileEntity.getProtectedFilePath());
-        Files.deleteIfExists(path);
-        fileEntity.setProtectedFilePath("");
-
-        return fileEntityRepository.save(fileEntity);
-    }
-
     public void softDeleteFileWithHash(FileEntity fileEntity) throws IOException {
         if (fileEntity.getImageHash() != null) {
             fileEntity.setImageHash(null);
@@ -229,14 +186,9 @@ public class FileEntityService {
         fileEntityRepository.save(fileEntity);
     }
 
-    public void deleteFromDisk(FileEntity fileEntity) throws IOException {
+    public void deleteFromStorageDisk(FileEntity fileEntity) {
-        Path path = Paths.get(fileEntity.getFilePath());
+        cloudStorageService.deleteFileFromStorage(fileEntity.getFilePath());
-
+        cloudStorageService.deleteFileFromStorage(fileEntity.getProtectedFilePath());
-        if (!Files.exists(path)) {
-            return;
-        }
-
-        Files.delete(path);
 
         fileEntityRepository.delete(fileEntity);
     }
@@ -282,11 +234,11 @@ public class FileEntityService {
         return fileEntityRepository.save(fileEntity);
     }
 
-    public void clearTempFiles(long userId) throws IOException {
+    public void clearTempFiles(long userId) {
         List<FileEntity> fileByUserIdAndStatus = fileEntityRepository.findFileByUserIdAndStatus(userId, FileStatus.TEMP);
 
         for (FileEntity fileEntity : fileByUserIdAndStatus) {
-            deleteFromDisk(fileEntity);
+            deleteFromStorageDisk(fileEntity);
         }
     }
 
@@ -298,24 +250,6 @@ public class FileEntityService {
         return fileEntityRepository.findFileIdByFilePath(filePath);
     }
 
-    public File getFileById(String id) {
-        try {
-            FileEntity fileEntity = fileEntityRepository.findById(id).orElseThrow(() ->
-                    new RuntimeException("File not found: " + id));
-
-            File file = new File(fileEntity.getFilePath());
-
-            if (!file.exists()) {
-                throw new RuntimeException("File not found on disk: " + fileEntity.getFilePath());
-            }
-
-            return file;
-        } catch (Exception e) {
-            log.error("Error getting file: {}", id, e);
-            return null;
-        }
-    }
-
     public Path prepareProtectedPath(FileEntity fileEntity, String extension) throws IOException {
         Path originalPath = Paths.get(fileEntity.getFilePath());
 
@@ -352,6 +286,18 @@ public class FileEntityService {
         fileEntityRepository.save(fileEntity);
     }
 
+    public String formatFileSize(long size) {
+        if (size < 1024) {
+            return size + " B";
+        } else if (size < 1024 * 1024) {
+            return String.format("%.1f KB", size / 1024.0);
+        } else if (size < 1024 * 1024 * 1024) {
+            return String.format("%.1f MB", size / (1024.0 * 1024.0));
+        } else {
+            return String.format("%.1f GB", size / (1024.0 * 1024.0 * 1024.0));
+        }
+    }
+
     private boolean checkFileExistsOnDisk(String filePath) {
         try {
             return Files.exists(Paths.get(filePath));
@@ -404,16 +350,4 @@ public class FileEntityService {
                 .protectedFilePath(fileEntity.getProtectedFilePath())
                 .build();
     }
-
-    public String formatFileSize(long size) {
-        if (size < 1024) {
-            return size + " B";
-        } else if (size < 1024 * 1024) {
-            return String.format("%.1f KB", size / 1024.0);
-        } else if (size < 1024 * 1024 * 1024) {
-            return String.format("%.1f MB", size / (1024.0 * 1024.0));
-        } else {
-            return String.format("%.1f GB", size / (1024.0 * 1024.0 * 1024.0));
-        }
-    }
 }

src/main/java/ru/soune/nocopy/service/file/cloud/CloudStorageService.java

@@ -55,7 +55,7 @@ public class CloudStorageService {
 
         String mimeTypeFromFile = mimeType.equals("document") ? "application" : mimeType;
         String contentType = mimeTypeFromFile + "/" + extension;
-        String filePath = "files/" + sendToCloudFilePath;
+        String filePath = "files" + sendToCloudFilePath;
 
         PutObjectRequest putObjectRequest = PutObjectRequest.builder()
                 .bucket(bucket)
@@ -75,7 +75,7 @@ public class CloudStorageService {
     public File readFileFromStorageByPath(String filePath) throws IOException {
         GetObjectRequest objectRequest = GetObjectRequest.builder()
                 .bucket(bucket)
-                .key("files/" + filePath)
+                .key("files" + filePath)
                 .build();
         AwsCredentials credentials = AwsBasicCredentials.create(key, secretKey);
 
@@ -94,4 +94,39 @@ public class CloudStorageService {
 
         return tempFile;
     }
+
+    public void deleteFileFromStorage(String filePath) {
+        if (filePath == null || filePath.isEmpty()) {
+            log.warn("Attempted to delete file with empty path");
+            return;
+        }
+
+        AwsCredentials credentials = AwsBasicCredentials.create(key, secretKey);
+
+        S3Client s3Client = S3Client.builder()
+                .httpClient(ApacheHttpClient.create())
+                .region(Region.of(region))
+                .endpointOverride(URI.create(s3endpoint))
+                .credentialsProvider(StaticCredentialsProvider.create(credentials))
+                .build();
+
+        String cloudKey = "files" + filePath;
+
+        try {
+            try {
+                s3Client.headObject(builder -> builder.bucket(bucket).key(cloudKey).build());
+            } catch (Exception e) {
+                log.warn("File does not exist in cloud storage: {}", cloudKey);
+                return;
+            }
+
+            s3Client.deleteObject(builder -> builder.bucket(bucket).key(cloudKey).build());
+
+            log.info("File successfully deleted from cloud storage: {}", cloudKey);
+
+        } catch (Exception e) {
+            log.error("Failed to delete file from cloud storage: {}", cloudKey, e);
+            throw new RuntimeException("Failed to delete file from cloud storage: " + cloudKey, e);
+        }
+    }
 }

src/main/java/ru/soune/nocopy/service/file/impl/FileUploadServiceImpl.java

@@ -181,7 +181,6 @@ public class FileUploadServiceImpl implements FileUploadService {
     }
 
     @Override
-    @Transactional
     public UploadProgressResponse uploadChunk(String uploadId, Integer chunkNumber,
                             MultipartFile chunkFile, Integer findSimilar) {
         FileUploadSession session = sessionRepository.findById(uploadId)
@@ -336,7 +335,6 @@ public class FileUploadServiceImpl implements FileUploadService {
 
                 FileEntity fileEntity = completeFileProcessingAsync(session, status);
 
-                //TODO CHECK FOR PROTECTED
                 fileEntityRepository.flush();
 
                 entityManager.clear();

src/main/resources/application.yaml

@@ -105,7 +105,7 @@ yandex:
     secret-key: "YCPAPZ32XC-LWb8xE0b_xTcBD8NZUCHpSOfu8LUG"
     region: "ru-central1"
     s3-endpoint: "https://storage.yandexcloud.net"
-    bucket: "ncp-test-storage"
+    bucket: "no-copy-storage"
 
 
 searchapi: