From 33c0275a37c71137ea13cc3d0865c8f641c2fc2a Mon Sep 17 00:00:00 2001 From: backdev-1 Date: Thu, 23 Apr 2026 00:38:40 +0700 Subject: [PATCH] dev add cold storage --- .../repository/ModerationFileRepository.java | 2 + .../soune/nocopy/service/file/ZipService.java | 38 +++++---- .../file/cloud/CloudStorageService.java | 77 +++++++++++++++++++ .../moderation/ModerationFileService.java | 7 ++ src/main/resources/application.yaml | 1 + 5 files changed, 112 insertions(+), 13 deletions(-) diff --git a/src/main/java/ru/soune/nocopy/repository/ModerationFileRepository.java b/src/main/java/ru/soune/nocopy/repository/ModerationFileRepository.java index 6ad4167..39bd934 100644 --- a/src/main/java/ru/soune/nocopy/repository/ModerationFileRepository.java +++ b/src/main/java/ru/soune/nocopy/repository/ModerationFileRepository.java @@ -10,4 +10,6 @@ import java.util.List; public interface ModerationFileRepository extends CrudRepository { public List findByStatus(String status); + + public List findByUserIdAndStatus(Long userId, String status); } diff --git a/src/main/java/ru/soune/nocopy/service/file/ZipService.java b/src/main/java/ru/soune/nocopy/service/file/ZipService.java index abc1a07..772bfbd 100644 --- a/src/main/java/ru/soune/nocopy/service/file/ZipService.java +++ b/src/main/java/ru/soune/nocopy/service/file/ZipService.java @@ -2,9 +2,11 @@ package ru.soune.nocopy.service.file; import lombok.RequiredArgsConstructor; import lombok.extern.slf4j.Slf4j; +import org.springframework.beans.factory.annotation.Value; import org.springframework.stereotype.Service; import ru.soune.nocopy.entity.file.FileEntity; import ru.soune.nocopy.entity.file.moderation.ModerationPassportFile; +import ru.soune.nocopy.service.file.cloud.CloudStorageService; import ru.soune.nocopy.service.file.moderation.ModerationFileService; import java.io.ByteArrayOutputStream; @@ -25,12 +27,17 @@ public class ZipService { private final ModerationFileService moderationFileService; + private final CloudStorageService cloudStorageService; + private static final int MAX_SIZE_MB = 20; private static final int FIRST_PART = 1; private static final int SECOND_PART = 2; + @Value("${yandex.cloud.bucket-cold}") + private String bucketCold; + public void createAndSplitZip(Long userId, List files) throws IOException { byte[] zipBytes = createZip(files); String base64String = Base64.getEncoder().encodeToString(zipBytes); @@ -39,29 +46,29 @@ public class ZipService { } public byte[] assembleZipFromSplitFiles(Long userId) throws IOException { - List activeFiles = moderationFileService.getActiveFiles(); + List activeFiles = moderationFileService.getActiveFilesByUserId(userId); - String part1Content = null; - String part2Content = null; + String part1Key = null; + String part2Key = null; for (ModerationPassportFile file : activeFiles) { - Path filePath = Paths.get(file.getPath()); - String content = Files.readString(filePath); - if (file.getPart() == FIRST_PART) { - part1Content = content; + part1Key = file.getPath(); } else if (file.getPart() == SECOND_PART) { - part2Content = content; + part2Key = file.getPath(); } } - if (part1Content == null || part2Content == null) { + if (part1Key == null || part2Key == null) { throw new IOException("Missing part 1 or part 2 for user: " + userId); } - String base64String = mergeBase64(part1Content, part2Content); + String part1Content = cloudStorageService.downloadPartFromBucket(bucketCold, part1Key); + String part2Content = cloudStorageService.downloadPartFromBucket(bucketCold, part2Key); - return Base64.getDecoder().decode(base64String); + String fullBase64 = part1Content + part2Content; + + return Base64.getDecoder().decode(fullBase64); } private byte[] createZip(List files) throws IOException { @@ -120,11 +127,16 @@ public class ZipService { Files.writeString(file1, part1); Path file2 = userDir.resolve(userId + "___2.txt"); Files.writeString(file2, part2); - Map files = Map.of(FIRST_PART, file1.toString(), SECOND_PART, file2.toString()); + + Map cloudKeys = cloudStorageService.saveSplitPartsToColdStorage(userId, file1, file2); + + Map files = Map.of( + FIRST_PART, cloudKeys.get("part1"), + SECOND_PART, cloudKeys.get("part2")); moderationFileService.addFiles(userId, files); - log.info("Saved split files: {} and {}", file1, file2); + log.info("Saved split files to cold storage for userId: {}", userId); } private String mergeBase64(String part1, String part2) { diff --git a/src/main/java/ru/soune/nocopy/service/file/cloud/CloudStorageService.java b/src/main/java/ru/soune/nocopy/service/file/cloud/CloudStorageService.java index 6aee80e..9449fae 100644 --- a/src/main/java/ru/soune/nocopy/service/file/cloud/CloudStorageService.java +++ b/src/main/java/ru/soune/nocopy/service/file/cloud/CloudStorageService.java @@ -23,8 +23,11 @@ import java.io.File; import java.io.IOException; import java.io.InputStream; import java.net.URI; +import java.nio.charset.StandardCharsets; import java.nio.file.Files; +import java.nio.file.Path; import java.nio.file.Paths; +import java.util.Map; import java.util.UUID; @Service @@ -49,6 +52,9 @@ public class CloudStorageService { @Value("${yandex.cloud.bucket}") private String bucket; + @Value("${yandex.cloud.bucket-cold}") + private String bucketCold; + public void saveFilesToStorage(String mimeType, String extension, String sendToCloudFilePath) { AwsCredentials credentials = AwsBasicCredentials.create(key, secretKey); S3Client s3Client = S3Client.builder() @@ -77,6 +83,77 @@ public class CloudStorageService { } } + public Map saveSplitPartsToColdStorage(Long userId, Path file1Path, Path file2Path) { + AwsCredentials credentials = AwsBasicCredentials.create(key, secretKey); + S3Client s3Client = S3Client.builder() + .httpClient(ApacheHttpClient.create()) + .region(Region.of(region)) + .endpointOverride(URI.create(s3endpoint)) + .credentialsProvider(StaticCredentialsProvider.create(credentials)) + .build(); + + String part1Key = uploadFileToBucket(s3Client, bucketCold, file1Path, userId, "1"); + String part2Key = uploadFileToBucket(s3Client, bucketCold, file2Path, userId, "2"); + + try { + Files.deleteIfExists(file1Path); + Files.deleteIfExists(file2Path); + log.info("Deleted local split files for userId: {}", userId); + } catch (IOException e) { + log.error("Failed to delete local split files for userId: {}", userId, e); + } + + return Map.of("part1", part1Key, "part2", part2Key); + } + + public String downloadPartFromBucket(String bucket, String cloudKey) { + AwsCredentials credentials = AwsBasicCredentials.create(key, secretKey); + S3Client s3Client = S3Client.builder() + .httpClient(ApacheHttpClient.create()) + .region(Region.of(region)) + .endpointOverride(URI.create(s3endpoint)) + .credentialsProvider(StaticCredentialsProvider.create(credentials)) + .build(); + + try { + GetObjectRequest getObjectRequest = GetObjectRequest.builder() + .bucket(bucket) + .key(cloudKey) + .build(); + + ResponseBytes objectBytes = s3Client.getObjectAsBytes(getObjectRequest); + String content = new String(objectBytes.asByteArray(), StandardCharsets.UTF_8); + + log.info("Downloaded from cold bucket, key: {}, size: {} chars", cloudKey, content.length()); + return content; + + } catch (Exception e) { + log.error("Failed to download from cold bucket, key: {}", cloudKey, e); + throw new RuntimeException("Failed to download file from cold storage", e); + } + } + + private String uploadFileToBucket(S3Client s3Client, String bucket, Path filePath, Long userId, String partSuffix) { + String cloudKey = String.format("passport/%d/part_%s.txt", userId, partSuffix); + + try { + byte[] bytes = Files.readAllBytes(filePath); + + PutObjectRequest putObjectRequest = PutObjectRequest.builder() + .bucket(bucket) + .key(cloudKey) + .contentType("text/plain") + .build(); + + s3Client.putObject(putObjectRequest, RequestBody.fromBytes(bytes)); + log.info("Uploaded part {} to cold bucket, key: {}", partSuffix, cloudKey); + return cloudKey; + } catch (IOException e) { + log.error("Failed to read file for upload: {}", filePath, e); + throw new RuntimeException("Failed to upload split part to cold storage", e); + } + } + public File readFileFromStorageByPath(String filePath) throws IOException { GetObjectRequest objectRequest = GetObjectRequest.builder() .bucket(bucket) diff --git a/src/main/java/ru/soune/nocopy/service/file/moderation/ModerationFileService.java b/src/main/java/ru/soune/nocopy/service/file/moderation/ModerationFileService.java index 6a859a0..47528f6 100644 --- a/src/main/java/ru/soune/nocopy/service/file/moderation/ModerationFileService.java +++ b/src/main/java/ru/soune/nocopy/service/file/moderation/ModerationFileService.java @@ -4,6 +4,7 @@ import lombok.RequiredArgsConstructor; import org.springframework.stereotype.Service; import ru.soune.nocopy.entity.file.moderation.ModerationPassportFile; import ru.soune.nocopy.repository.ModerationFileRepository; +import ru.soune.nocopy.service.file.cloud.CloudStorageService; import ru.soune.nocopy.service.user.moderation.UserVerificationService; import java.util.List; @@ -17,6 +18,8 @@ public class ModerationFileService { private final UserVerificationService userVerificationService; + private final CloudStorageService cloudStorageService; + public void addFiles(Long userId, Map files) { List moderationFileRepositoryByStatus = moderationFileRepository.findByStatus(ModerationFileStatus.ACTIVE.getName()); @@ -44,6 +47,10 @@ public class ModerationFileService { return moderationFileRepository.findByStatus(ModerationFileStatus.ACTIVE.getName()); } + public List getActiveFilesByUserId(Long userId) { + return moderationFileRepository.findByUserIdAndStatus(userId, ModerationFileStatus.ACTIVE.getName()); + } + private void disActiveDocuments(List moderationFileRepositoryByStatus) { for (ModerationPassportFile moderationFile : moderationFileRepositoryByStatus) { moderationFile.setStatus(ModerationFileStatus.NOT_ACTIVE.getName()); diff --git a/src/main/resources/application.yaml b/src/main/resources/application.yaml index 6bc410a..9863947 100644 --- a/src/main/resources/application.yaml +++ b/src/main/resources/application.yaml @@ -108,6 +108,7 @@ yandex: region: "ru-central1" s3-endpoint: "https://storage.yandexcloud.net" bucket: "no-copy-storage" + bucket-cold: "no-copy-storage-cold" searchapi: