NCBACK-35 add cloud for save when download last chunk and after protected

README.md

@@ -150,4 +150,14 @@ INSERT INTO referral_levels (id, name, min_invitees, reward_percentage) VALUES
 ('bronze', 'BRONZE', 0, 15),
 ('silver', 'SILVER', 6, 18),
 ('gold', 'GOLD', 16, 22),
-('platinum', 'PLATINUM', 50, 25);
+('platinum', 'PLATINUM', 50, 25);
+
+------
+
+Настройка Яндекс Клауд
+
+1. YCAJEpWAtaVkVGX0sH6_EupEg - индетификатор ключа
+2. YCMmVykfXrZ_nfU13Vo4yoCVGa70DnTlBgF1pUzO - секретный ключ
+
+   private static final String KEY_ID = "YCAJEmHZcWxzf4oGWvETG3mms";
+   private static final String SECRET_KEY = "YCPAPZ32XC-LWb8xE0b_xTcBD8NZUCHpSOfu8LUG";

build.gradle

@@ -65,6 +65,12 @@ dependencies {
 
 	implementation project(':referral')
 
+	//cloud
+	implementation("software.amazon.awssdk:aws-sdk-java:2.29.33")
+	implementation("software.amazon.awssdk:apache-client:2.29.33")
+
+	implementation("org.springdoc:springdoc-openapi-starter-webmvc-ui:2.7.0")
+	implementation("org.springframework.boot:spring-boot-starter-web")
 }
 
 tasks.named('test') {

src/main/java/ru/soune/nocopy/controller/ApiController.java

@@ -32,6 +32,7 @@ import ru.soune.nocopy.repository.AuthTokenRepository;
 import ru.soune.nocopy.repository.FileEntityRepository;
 import ru.soune.nocopy.service.FileSimilarityService;
 import ru.soune.nocopy.service.file.ProtectionsLimitService;
+import ru.soune.nocopy.service.file.cloud.CloudStorageService;
 import ru.soune.nocopy.service.register.AuthService;
 import ru.soune.nocopy.service.file.FileEntityService;
 import ru.soune.nocopy.service.file.FileUploadService;
@@ -291,6 +292,8 @@ public class ApiController {
         }
     }
 
+    private final CloudStorageService cloudStorageService;
+
     @GetMapping("/v{version}/files/download/{fileId}")
     public ResponseEntity<?> downloadFile(
             @PathVariable(required = false) String fileId,
@@ -342,29 +345,29 @@ public class ApiController {
                         errorData));
             }
 
-
             Path filePath = Paths.get(entityResponse.getProtectedFilePath());
-//            Resource resource = new UrlResource(filePath.toUri());
+            File file = cloudStorageService.readFileFromStorageByPath(
-            FileSystemResource resource = new FileSystemResource(filePath);
+                    entityResponse.getProtectedFilePath());
             long fileSize = Files.size(filePath);
 
-            if (!resource.exists()) {
+            if (!file.exists()) {
                 Map<String, Object> errorData = new HashMap<>();
-                errorData.put("resource", resource.exists());
+                errorData.put("file", file.exists());
 
                 return ResponseEntity.ok().body(new BaseResponse(20004,
                         MessageCode.FILE_DOWNLOAD_ERROR.getCode(),
                         MessageCode.FILE_DOWNLOAD_ERROR.getDescription(),
                         errorData));
             }
-                String contentType = determineContentType(filePath);
+
+            String contentType = determineContentType(filePath);
 
             return ResponseEntity.ok()
                     .contentLength(fileSize)
                     .contentType(MediaType.parseMediaType(contentType))
                     .header(HttpHeaders.CONTENT_DISPOSITION,
                             "attachment; filename=\"" + entityResponse.getFileName() + "\"")
-                    .body(resource);
+                    .body(file);
         } catch (FileEntityNotFoundException e) {
             Map<String, Object> errorData = new HashMap<>();
             errorData.put("fileId", fileId);

src/main/java/ru/soune/nocopy/controller/PhotoController.java

@@ -0,0 +1,86 @@
+package ru.soune.nocopy.controller;
+
+import io.swagger.v3.oas.annotations.responses.ApiResponse;
+import io.swagger.v3.oas.annotations.responses.ApiResponses;
+import io.swagger.v3.oas.annotations.tags.Tag;
+import org.springframework.http.HttpHeaders;
+import org.springframework.http.ResponseEntity;
+import org.springframework.web.bind.annotation.*;
+import org.springframework.web.multipart.MultipartFile;
+import software.amazon.awssdk.auth.credentials.AwsBasicCredentials;
+import software.amazon.awssdk.auth.credentials.AwsCredentials;
+import software.amazon.awssdk.auth.credentials.StaticCredentialsProvider;
+import software.amazon.awssdk.core.sync.RequestBody;
+import software.amazon.awssdk.http.apache.ApacheHttpClient;
+import software.amazon.awssdk.regions.Region;
+import software.amazon.awssdk.services.s3.S3Client;
+import software.amazon.awssdk.services.s3.model.GetObjectRequest;
+import software.amazon.awssdk.services.s3.model.PutObjectRequest;
+
+import java.io.IOException;
+import java.net.URI;
+
+import static org.springframework.http.MediaType.MULTIPART_FORM_DATA_VALUE;
+
+@RestController
+@RequestMapping("/photos")
+@Tag(name = "Photos")
+@ApiResponses(@ApiResponse(responseCode = "200", useReturnTypeSchema = true))
+public class PhotoController {
+
+    private static final String KEY_ID = "YCAJEmHZcWxzf4oGWvETG3mms";
+    private static final String SECRET_KEY = "YCPAPZ32XC-LWb8xE0b_xTcBD8NZUCHpSOfu8LUG";
+    private static final String REGION = "ru-central1";
+    private static final String S3_ENDPOINT = "https://storage.yandexcloud.net";
+
+    private static final String BUCKET = "ncp-test-storage";
+
+    private final S3Client s3Client;
+
+    public PhotoController() {
+        AwsCredentials credentials = AwsBasicCredentials.create(KEY_ID, SECRET_KEY);
+
+        s3Client = S3Client.builder()
+                .httpClient(ApacheHttpClient.create())
+                .region(Region.of(REGION))
+                .endpointOverride(URI.create(S3_ENDPOINT))
+                .credentialsProvider(StaticCredentialsProvider.create(credentials))
+                .build();
+    }
+
+
+    @PutMapping(consumes = MULTIPART_FORM_DATA_VALUE)
+    public String uploadFile(@RequestParam MultipartFile photo) throws IOException {
+
+        String key = "photos/" + photo.getOriginalFilename();
+        PutObjectRequest putObjectRequest = PutObjectRequest.builder()
+                .bucket(BUCKET)
+                .key(key)
+                .contentType(photo.getContentType())
+                .build();
+
+        s3Client.putObject(putObjectRequest, RequestBody.fromBytes(photo.getBytes()));
+
+        return key;
+    }
+
+    @GetMapping
+    public ResponseEntity<byte[]> downloadFile(@RequestParam String key) throws IOException {
+        GetObjectRequest objectRequest = GetObjectRequest.builder()
+                .bucket(BUCKET)
+                .key(key)
+                .build();
+
+        var inputStream = s3Client.getObject(objectRequest);
+        byte[] data = inputStream.readAllBytes();
+
+        var headers = new HttpHeaders();
+        headers.add(HttpHeaders.CONTENT_DISPOSITION, "inline");
+        headers.add(HttpHeaders.CONTENT_TYPE, inputStream.response().contentType());
+
+        return ResponseEntity.ok()
+                .headers(headers)
+                .body(data);
+    }
+}
+

src/main/java/ru/soune/nocopy/service/file/FileEntityService.java

@@ -241,6 +241,10 @@ public class FileEntityService {
         fileEntityRepository.delete(fileEntity);
     }
 
+    public void deleteFromDisk(String deleteFilePath) throws IOException {
+        Files.deleteIfExists(Paths.get(deleteFilePath));
+    }
+
     @Transactional(readOnly = true)
     public long getUserStorageUsed(Long userId) {
         Long totalSize = fileEntityRepository.getTotalSizeByUserId(userId);
@@ -256,12 +260,11 @@ public class FileEntityService {
     }
 
     @Transactional
-    public void writeProtectedFile(String id, byte[] data, String fileExt) throws IOException {
+    public FileEntity writeProtectedFile(String id, byte[] data, String fileExt) throws IOException {
         FileEntity fileEntity = fileEntityRepository.findById(id)
                 .orElseThrow(() -> new RuntimeException("File not found: " + id));
 
         String extension = determineFileExtension(fileExt, fileEntity);
-        log.info("EXTENSION: {}", extension);
         Path protectedFilePath = prepareProtectedPath(fileEntity, extension);
 
         if (Files.exists(protectedFilePath)) {
@@ -276,7 +279,7 @@ public class FileEntityService {
         fileEntity.setFileExtension(extension);
         fileEntity.setProtectionStatus(ProtectionStatus.PROTECTED);
 
-        fileEntityRepository.save(fileEntity);
+        return fileEntityRepository.save(fileEntity);
     }
 
     public void clearTempFiles(long userId) throws IOException {

src/main/java/ru/soune/nocopy/service/file/FileUploadService.java

@@ -2,6 +2,7 @@ package ru.soune.nocopy.service.file;
 
 import org.springframework.web.multipart.MultipartFile;
 import ru.soune.nocopy.dto.file.UploadProgressResponse;
+import ru.soune.nocopy.entity.file.FileEntity;
 import ru.soune.nocopy.entity.file.FileStatus;
 import ru.soune.nocopy.entity.file.FileUploadSession;
 
@@ -19,5 +20,5 @@ public interface FileUploadService {
 
     UploadProgressResponse getUploadProgress(String uploadId);
 
-    void completeFileProcessingAsync(FileUploadSession session, FileStatus status);
+    FileEntity completeFileProcessingAsync(FileUploadSession session, FileStatus status);
 }

src/main/java/ru/soune/nocopy/service/file/cloud/CloudStorageService.java

@@ -0,0 +1,97 @@
+package ru.soune.nocopy.service.file.cloud;
+
+import lombok.RequiredArgsConstructor;
+import lombok.extern.slf4j.Slf4j;
+import org.springframework.beans.factory.annotation.Value;
+import org.springframework.stereotype.Service;
+import ru.soune.nocopy.service.file.FileEntityService;
+import software.amazon.awssdk.auth.credentials.AwsBasicCredentials;
+import software.amazon.awssdk.auth.credentials.AwsCredentials;
+import software.amazon.awssdk.auth.credentials.StaticCredentialsProvider;
+import software.amazon.awssdk.core.sync.RequestBody;
+import software.amazon.awssdk.http.apache.ApacheHttpClient;
+import software.amazon.awssdk.regions.Region;
+import software.amazon.awssdk.services.s3.S3Client;
+import software.amazon.awssdk.services.s3.model.GetObjectRequest;
+import software.amazon.awssdk.services.s3.model.PutObjectRequest;
+
+import java.io.File;
+import java.io.IOException;
+import java.net.URI;
+import java.nio.file.Files;
+import java.nio.file.Paths;
+
+@Service
+@Slf4j
+@RequiredArgsConstructor
+public class CloudStorageService {
+
+    private final FileEntityService fileEntityService;
+
+    @Value("${yandex.cloud.key}")
+    private String key;
+
+    @Value("${yandex.cloud.secret-key}")
+    private String secretKey;
+
+    @Value("${yandex.cloud.region}")
+    private String region;
+
+    @Value("${yandex.cloud.s3-endpoint}")
+    private String s3endpoint;
+
+    @Value("${yandex.cloud.bucket}")
+    private String bucket;
+
+    public void saveFilesToStorage(String mimeType, String extension, String sendToCloudFilePath) {
+        AwsCredentials credentials = AwsBasicCredentials.create(key, secretKey);
+
+        S3Client s3Client = S3Client.builder()
+                .httpClient(ApacheHttpClient.create())
+                .region(Region.of(region))
+                .endpointOverride(URI.create(s3endpoint))
+                .credentialsProvider(StaticCredentialsProvider.create(credentials))
+                .build();
+
+        String mimeTypeFromFile = mimeType.equals("document") ? "application" : mimeType;
+        String contentType = mimeTypeFromFile + "/" + extension;
+        String filePath = "files/" + sendToCloudFilePath;
+
+        PutObjectRequest putObjectRequest = PutObjectRequest.builder()
+                .bucket(bucket)
+                .key(filePath)
+                .contentType(contentType)
+                .build();
+
+        try {
+            s3Client.putObject(putObjectRequest,
+                    RequestBody.fromBytes(Files.readAllBytes(Paths.get(sendToCloudFilePath))));
+            fileEntityService.deleteFromDisk(sendToCloudFilePath);
+        } catch (IOException e) {
+            log.error("Failed to read file: {}", filePath, e);
+        }
+    }
+
+    public File readFileFromStorageByPath(String filePath) throws IOException {
+        GetObjectRequest objectRequest = GetObjectRequest.builder()
+                .bucket(bucket)
+                .key("files/" + filePath)
+                .build();
+        AwsCredentials credentials = AwsBasicCredentials.create(key, secretKey);
+
+        S3Client s3Client = S3Client.builder()
+                .httpClient(ApacheHttpClient.create())
+                .region(Region.of(region))
+                .endpointOverride(URI.create(s3endpoint))
+                .credentialsProvider(StaticCredentialsProvider.create(credentials))
+                .build();
+
+        String tempDir = System.getProperty("java.io.tmpdir");
+        String fileName = filePath.replace("/", "_");
+        File tempFile = new File(tempDir, fileName);
+
+        s3Client.getObject(objectRequest, tempFile.toPath());
+
+        return tempFile;
+    }
+}

src/main/java/ru/soune/nocopy/service/file/impl/FileUploadServiceImpl.java

@@ -3,6 +3,7 @@ package ru.soune.nocopy.service.file.impl;
 import com.vrt.NoCopyFileService;
 import com.vrt.fileprotection.FileProtector;
 import jakarta.annotation.PostConstruct;
+import jakarta.persistence.EntityManager;
 import lombok.RequiredArgsConstructor;
 import lombok.extern.slf4j.Slf4j;
 import org.springframework.beans.factory.annotation.Autowired;
@@ -24,6 +25,7 @@ import ru.soune.nocopy.service.FileSimilarityService;
 import ru.soune.nocopy.service.ImageHashService;
 import ru.soune.nocopy.service.file.FileEntityService;
 import ru.soune.nocopy.service.file.FileUploadService;
+import ru.soune.nocopy.service.file.cloud.CloudStorageService;
 import ru.soune.nocopy.service.tariff.TariffConstants;
 import ru.soune.nocopy.service.tariff.TariffInfoService;
 import ru.soune.nocopy.util.FileUtil;
@@ -83,6 +85,8 @@ public class FileUploadServiceImpl implements FileUploadService {
 
     private final TariffInfoService tariffInfoService;
 
+    private final CloudStorageService cloudStorageService;
+
     @PostConstruct
     public void init() {
         try {
@@ -219,7 +223,8 @@ public class FileUploadServiceImpl implements FileUploadService {
     @Async("fileUploadTaskExecutor")
     @Transactional
     @Override
-    public void completeFileProcessingAsync(FileUploadSession session, FileStatus status) {
+    public FileEntity completeFileProcessingAsync(FileUploadSession session, FileStatus status) {
+        FileEntity saved = null;
         try {
             Path filePath = Paths.get(session.getFilePath());
             String checksum = calculateChecksum(filePath);
@@ -238,7 +243,7 @@ public class FileUploadServiceImpl implements FileUploadService {
                     .status(status)
                     .build();
 
-            FileEntity saved = fileEntityRepository.save(fileEntity);
+            saved = fileEntityRepository.save(fileEntity);
 
             if (session.getFileType().equals("image")) {
                 Map<String, Long> hash = imageHashService.calculateHash(filePath);
@@ -248,18 +253,13 @@ public class FileUploadServiceImpl implements FileUploadService {
 
             cleanupSessionFiles(session);
 
-            if (status != FileStatus.TEMP) {
-                FileProtector.FileInfo fileInfo = fileUtil.createFileInfo(fileEntity);
-
-                noCopyFileService.addFile(fileInfo);
-            }
-
             log.info("File processing completed for session: {}", session.getUploadId());
-
         } catch (Exception e) {
             log.error("Failed to complete file processing for session {}: {}",
                     session.getUploadId(), e.getMessage(), e);
         }
+
+        return saved;
     }
 
     private void validateSession(FileUploadSession session) {
@@ -291,6 +291,9 @@ public class FileUploadServiceImpl implements FileUploadService {
         }
     }
 
+    @Autowired
+    private EntityManager entityManager;
+
     private UploadProgressResponse processChunk(FileUploadSession session, Integer chunkNumber, MultipartFile chunkFile,
                                                 Integer findSimilar) {
         String chunkPath = null;
@@ -331,8 +334,21 @@ public class FileUploadServiceImpl implements FileUploadService {
                     fileEntityService.clearTempFiles(session.getUserId());
                 }
 
-                completeFileProcessingAsync(session, status);
+                FileEntity fileEntity = completeFileProcessingAsync(session, status);
 
+                //TODO CHECK FOR PROTECTED
+                fileEntityRepository.flush();
+
+                entityManager.clear();
+
+                cloudStorageService.saveFilesToStorage(fileEntity.getMimeType(), fileEntity.getFileExtension(),
+                        fileEntity.getFilePath());
+
+                if (status != FileStatus.TEMP) {
+                    FileProtector.FileInfo fileInfo = fileUtil.createFileInfo(fileEntity);
+
+                    noCopyFileService.addFile(fileInfo);
+                }
 
                 if (status != FileStatus.TEMP) {
                     String fileType = session.getFileType();

src/main/java/ru/soune/nocopy/service/file/impl/ProtectionFileProviderImpl.java

@@ -7,7 +7,10 @@ import lombok.extern.slf4j.Slf4j;
 import org.jetbrains.annotations.NotNull;
 import org.jetbrains.annotations.Nullable;
 import org.springframework.stereotype.Component;
+import ru.soune.nocopy.entity.file.FileEntity;
+import ru.soune.nocopy.repository.FileEntityRepository;
 import ru.soune.nocopy.service.file.FileEntityService;
+import ru.soune.nocopy.service.file.cloud.CloudStorageService;
 
 import java.io.File;
 import java.io.IOException;
@@ -24,27 +27,55 @@ public class ProtectionFileProviderImpl implements FileProtector.FileProvider {
 
     private final FileEntityService fileEntityService;
 
+    private final CloudStorageService cloudStorageService;
+
+    private final FileEntityRepository fileEntityRepository;
+
     @Nullable
     @Override
     public File getImageFile(@NotNull String id) {
-        return fileEntityService.getFileById(id);
+        FileEntity fileEntity = fileEntityRepository.findById(id).orElseThrow(() ->
+                new RuntimeException("File not found: " + id));
+        try {
+            return cloudStorageService.readFileFromStorageByPath(fileEntity.getFilePath());
+        } catch (IOException e) {
+            throw new RuntimeException(e);
+        }
     }
 
     @Nullable
     @Override
     public File getVideoFile(@NotNull String id) {
-        return fileEntityService.getFileById(id);
+        FileEntity fileEntity = fileEntityRepository.findById(id).orElseThrow(() ->
+                new RuntimeException("File not found: " + id));
+        try {
+            return cloudStorageService.readFileFromStorageByPath(fileEntity.getFilePath());
+        } catch (IOException e) {
+            throw new RuntimeException(e);
+        }
     }
 
     @Nullable
     @Override
     public File getAudioFile(@NotNull String id) {
-        return fileEntityService.getFileById(id);
+        FileEntity fileEntity = fileEntityRepository.findById(id).orElseThrow(() ->
+                new RuntimeException("File not found: " + id));
+        try {
+            return cloudStorageService.readFileFromStorageByPath(fileEntity.getFilePath());
+        } catch (IOException e) {
+            throw new RuntimeException(e);
+        }
     }
 
     @Override
     public @Nullable File getDocument(@NotNull String id) {
-        return fileEntityService.getFileById(id);
+        FileEntity fileEntity = fileEntityRepository.findById(id).orElseThrow(() ->
+                new RuntimeException("File not found: " + id));
+        try {
+            return cloudStorageService.readFileFromStorageByPath(fileEntity.getFilePath());
+        } catch (IOException e) {
+            throw new RuntimeException(e);
+        }
     }
 
     @Override
@@ -79,7 +110,11 @@ public class ProtectionFileProviderImpl implements FileProtector.FileProvider {
     @Override
     public com.vrt.fileprotection.OperationResult writeAudioFile(@NotNull String id, @NotNull byte[] data, @Nullable String fileExt) {
         try {
-            fileEntityService.writeProtectedFile(id, data, fileExt);
+            FileEntity fileEntity = fileEntityService.writeProtectedFile(id, data, fileExt);
+
+            cloudStorageService.saveFilesToStorage(fileEntity.getMimeType(), fileEntity.getFileExtension(),
+                    fileEntity.getProtectedFilePath());
+
             return OperationResult.Companion.success();
         } catch (Exception e) {
             log.error("Failed to create protected file: {}", id, e);
@@ -91,7 +126,11 @@ public class ProtectionFileProviderImpl implements FileProtector.FileProvider {
     @Override
     public com.vrt.fileprotection.OperationResult writeVideoFile(@NotNull String id, @NotNull byte[] data, @Nullable String fileExt) {
         try {
-            fileEntityService.writeProtectedFile(id, data, fileExt);
+            FileEntity fileEntity = fileEntityService.writeProtectedFile(id, data, fileExt);
+
+            cloudStorageService.saveFilesToStorage(fileEntity.getMimeType(), fileEntity.getFileExtension(),
+                    fileEntity.getProtectedFilePath());
+
             return OperationResult.Companion.success();
         } catch (Exception e) {
             log.error("Failed to create protected file: {}", id, e);
@@ -103,7 +142,11 @@ public class ProtectionFileProviderImpl implements FileProtector.FileProvider {
     @Override
     public com.vrt.fileprotection.OperationResult writeImageFile(@NotNull String id, @NotNull byte[] data, @Nullable String fileExt) {
         try {
-            fileEntityService.writeProtectedFile(id, data, fileExt);
+            FileEntity fileEntity = fileEntityService.writeProtectedFile(id, data, fileExt);
+
+            cloudStorageService.saveFilesToStorage(fileEntity.getMimeType(), fileEntity.getFileExtension(),
+                    fileEntity.getProtectedFilePath());
+
             return OperationResult.Companion.success();
         } catch (Exception e) {
             log.error("Failed to create protected file: {}", id, e);
@@ -114,7 +157,10 @@ public class ProtectionFileProviderImpl implements FileProtector.FileProvider {
     @Override
     public @NotNull OperationResult writeDocumentFile(@NotNull String id, @NotNull byte[] data) {
         try {
-            fileEntityService.writeProtectedFile(id, data, null);
+            FileEntity fileEntity = fileEntityService.writeProtectedFile(id, data, null);
+
+            cloudStorageService.saveFilesToStorage(fileEntity.getMimeType(), fileEntity.getFileExtension(),
+                    fileEntity.getProtectedFilePath());
             return OperationResult.Companion.success();
         } catch (Exception e) {
             log.error("Failed to create protected file: {}", id, e);

src/main/resources/application.yaml

@@ -100,6 +100,13 @@ yandex:
   api-key: ${YANDEX_API_KEY:AQVNyaVaUmgUb1GMCtf5zSEqFxy0woXrcMOOB43q}
   folder-id: ${YANDEX_FOLDER_ID:b1gokpdbm6qfpsou8pcd}
   search-url: ${YANDEX_SEARCH_URL:https://searchapi.api.cloud.yandex.net/v2/image/search_by_image}
+  cloud:
+    key: "YCAJEmHZcWxzf4oGWvETG3mms"
+    secret-key: "YCPAPZ32XC-LWb8xE0b_xTcBD8NZUCHpSOfu8LUG"
+    region: "ru-central1"
+    s3-endpoint: "https://storage.yandexcloud.net"
+    bucket: "ncp-test-storage"
+
 
 searchapi:
   api-key: ${SEARCHAPI_API_KEY:rdwNzCr2eybYVbU1KrGTJAgm}