diff --git a/README.md b/README.md index ffac0f8..fe72727 100644 --- a/README.md +++ b/README.md @@ -323,6 +323,7 @@ server_name dev-workspace.not-copy.com; ssl_certificate_key /etc/letsencrypt/live/dev-workspace.not-copy.com/privkey.pem; location /api/ { + proxy_pass http://localhost:3001; proxy_set_header Host $host; proxy_set_header X-Real-IP $remote_addr; @@ -489,4 +490,14 @@ killall 3proxy # Остановка прокси На основном сервере: bash curl -x http://193.46.217.94:3128 https://api.ipify.org # Проверка прокси -telnet 193.46.217.94 3128 # Проверка порта \ No newline at end of file +telnet 193.46.217.94 3128 # Проверка порта + +------ + +Настройка Яндекс Клауд + +1. YCAJEpWAtaVkVGX0sH6_EupEg - индетификатор ключа +2. YCMmVykfXrZ_nfU13Vo4yoCVGa70DnTlBgF1pUzO - секретный ключ + + private static final String KEY_ID = "YCAJEmHZcWxzf4oGWvETG3mms"; + private static final String SECRET_KEY = "YCPAPZ32XC-LWb8xE0b_xTcBD8NZUCHpSOfu8LUG"; \ No newline at end of file diff --git a/build.gradle b/build.gradle index c5286a8..3bc912f 100644 --- a/build.gradle +++ b/build.gradle @@ -66,6 +66,12 @@ dependencies { implementation project(':referral') + //cloud + implementation("software.amazon.awssdk:aws-sdk-java:2.29.33") + implementation("software.amazon.awssdk:apache-client:2.29.33") + + implementation("org.springdoc:springdoc-openapi-starter-webmvc-ui:2.7.0") + implementation("org.springframework.boot:spring-boot-starter-web") } tasks.named('test') { diff --git a/src/main/java/ru/soune/nocopy/controller/ApiController.java b/src/main/java/ru/soune/nocopy/controller/ApiController.java index 0fee9ba..aed2408 100644 --- a/src/main/java/ru/soune/nocopy/controller/ApiController.java +++ b/src/main/java/ru/soune/nocopy/controller/ApiController.java @@ -1,5 +1,7 @@ package ru.soune.nocopy.controller; +import com.google.api.client.util.IOUtils; +import com.google.common.io.ByteStreams; import com.vrt.NoCopyFileService; import com.vrt.fileprotection.FileProtector; import com.vrt.fileprotection.NoCopyCheckResult; @@ -8,16 +10,20 @@ import com.vrt.fileprotection.documents.DocumentCheckResult; import lombok.RequiredArgsConstructor; import lombok.extern.slf4j.Slf4j; import org.springframework.core.io.FileSystemResource; +import org.springframework.core.io.InputStreamResource; import org.springframework.data.domain.Page; import org.springframework.data.domain.Pageable; import org.springframework.data.web.PageableDefault; import org.springframework.http.HttpHeaders; +import org.springframework.http.HttpStatus; import org.springframework.http.MediaType; import org.springframework.http.ResponseEntity; import org.springframework.validation.BindingResult; import org.springframework.validation.FieldError; import org.springframework.web.bind.annotation.*; import org.springframework.web.multipart.MultipartFile; +import org.springframework.web.server.ResponseStatusException; +import org.springframework.web.servlet.mvc.method.annotation.StreamingResponseBody; import ru.soune.nocopy.dto.BaseRequest; import ru.soune.nocopy.dto.BaseResponse; import ru.soune.nocopy.dto.MessageCode; @@ -36,12 +42,17 @@ import ru.soune.nocopy.repository.FileEntityRepository; import ru.soune.nocopy.repository.UserRepository; import ru.soune.nocopy.service.FileSimilarityService; import ru.soune.nocopy.service.file.*; +import ru.soune.nocopy.service.file.CheckCounterService; +import ru.soune.nocopy.service.file.ProtectionsLimitService; +import ru.soune.nocopy.service.file.cloud.CloudStorageService; import ru.soune.nocopy.service.register.AuthService; import ru.soune.nocopy.service.user.moderation.UserVerificationService; import ru.soune.nocopy.util.FileUtil; +import software.amazon.awssdk.services.s3.model.NoSuchKeyException; import java.io.File; import java.io.IOException; +import java.io.InputStream; import java.nio.file.Files; import java.nio.file.Path; import java.nio.file.Paths; @@ -65,6 +76,8 @@ public class ApiController { private final FileEntityRepository fileEntityRepository; + private final NoCopyFileService noCopyFileService; + private final FileUtil fileUtil; private final ProtectionsLimitService protectionsLimitService; @@ -75,6 +88,8 @@ public class ApiController { private final CheckCounterService checkCounterService; + private final CloudStorageService cloudStorageService; + private final ZipService zipService; private final UserVerificationService userVerificationService; @@ -159,7 +174,7 @@ public class ApiController { MessageCode.DUPLICATE_FILE_UPLOAD.getDescription(), duplicateData )); - }catch (FileFormatException e){ + } catch (FileFormatException e){ return buildErrorResponse(uploadId, chunkNumber, e.getMessage()); } catch (Exception e) { log.error("Error uploading chunk", e); @@ -205,8 +220,6 @@ public class ApiController { } } - private final NoCopyFileService noCopyFileService; - @GetMapping("/check/{fileId}") public ResponseEntity check(@PathVariable String fileId) { FileEntity fileEntity = fileEntityRepository.findByFileId(fileId); @@ -489,39 +502,35 @@ public class ApiController { MessageCode.FILE_DELETE.getDescription(), errorData)); } +//TODO - if (!entityResponse.isExistsOnDisk()) { - Map errorData = new HashMap<>(); - errorData.put("onDisk", entityResponse.isExistsOnDisk()); +// if (!entityResponse.isExistsOnDisk()) { +// Map errorData = new HashMap<>(); +// errorData.put("onDisk", entityResponse.isExistsOnDisk()); +// +// return ResponseEntity.ok().body(new BaseResponse(20004, +// MessageCode.FILE_NOT_EXIST.getCode(), +// MessageCode.FILE_NOT_EXIST.getDescription(), +// errorData)); +// } - return ResponseEntity.ok().body(new BaseResponse(20004, - MessageCode.FILE_NOT_EXIST.getCode(), - MessageCode.FILE_NOT_EXIST.getDescription(), - errorData)); - } +// if (!file.exists()) { +// Map errorData = new HashMap<>(); +// errorData.put("file", file.exists()); +// +// return ResponseEntity.ok().body(new BaseResponse(20004, +// MessageCode.FILE_DOWNLOAD_ERROR.getCode(), +// MessageCode.FILE_DOWNLOAD_ERROR.getDescription(), +// errorData)); +// } - - Path filePath = Paths.get(entityResponse.getProtectedFilePath()); - FileSystemResource resource = new FileSystemResource(filePath); - long fileSize = Files.size(filePath); - - if (!resource.exists()) { - Map errorData = new HashMap<>(); - errorData.put("resource", resource.exists()); - - return ResponseEntity.ok().body(new BaseResponse(20004, - MessageCode.FILE_DOWNLOAD_ERROR.getCode(), - MessageCode.FILE_DOWNLOAD_ERROR.getDescription(), - errorData)); - } - String contentType = determineContentType(filePath); + byte[] bytes = cloudStorageService.readFileFromStorageBytes(entityResponse.getProtectedFilePath()); return ResponseEntity.ok() - .contentLength(fileSize) - .contentType(MediaType.parseMediaType(contentType)) + .contentType(MediaType.APPLICATION_OCTET_STREAM) .header(HttpHeaders.CONTENT_DISPOSITION, "attachment; filename=\"" + entityResponse.getFileName() + "\"") - .body(resource); + .body(bytes); } catch (FileEntityNotFoundException e) { Map errorData = new HashMap<>(); errorData.put("fileId", fileId); @@ -538,16 +547,16 @@ public class ApiController { MessageCode.AUTH_TOKEN_NOT_FOUND.getCode(), MessageCode.AUTH_TOKEN_NOT_FOUND.getDescription(), errorData)); - } catch (IOException e) { - Map errorData = new HashMap<>(); - errorData.put("token", tokenHeader); - errorData.put("fileId", fileId); - errorData.put("version", version); - - return ResponseEntity.ok().body(new BaseResponse(20004, - MessageCode.FILE_DOWNLOAD_ERROR_NOT_CORRECT_FIELD.getCode(), - MessageCode.FILE_DOWNLOAD_ERROR_NOT_CORRECT_FIELD.getDescription(), - errorData)); +// } catch (IOException e) { +// Map errorData = new HashMap<>(); +// errorData.put("token", tokenHeader); +// errorData.put("fileId", fileId); +// errorData.put("version", version); +// +// return ResponseEntity.ok().body(new BaseResponse(20004, +// MessageCode.FILE_DOWNLOAD_ERROR_NOT_CORRECT_FIELD.getCode(), +// MessageCode.FILE_DOWNLOAD_ERROR_NOT_CORRECT_FIELD.getDescription(), +// errorData)); } } @@ -564,33 +573,6 @@ public class ApiController { return ResponseEntity.ok().body(fileTypeStats); } - private boolean hasDuplicate(List similarFiles) { - return similarFiles.stream().anyMatch(f -> f.getHammingDistance() <= 5); - } - - private ResponseEntity handleDuplicate(FileEntity fileEntity, List similarFiles) - throws IOException { - fileEntityService.deleteFromDisk(fileEntity); - - fileEntityService.softDeleteFileWithHash(fileEntity); - - Optional originalFile = fileEntityRepository.findById(similarFiles.get(0).getFileId()); - - if (originalFile.isPresent()) { - Map duplicateInfo = Map.of( - "duplicate_file_id", originalFile.get().getId(), - "owner_user_id", String.valueOf(originalFile.get().getUserId())); - - return ResponseEntity.ok().body(new BaseResponse( - 20004, - MessageCode.DUPLICATE_FILE_UPLOAD.getCode(), - "Failed to upload chunk, duplicate", - duplicateInfo)); - } - - return null; - } - private ResponseEntity buildSuccessResponse(String uploadId, Integer chunkNumber, MultipartFile chunk, String fileId) { ChunkUploadResponse responseBody = ChunkUploadResponse.builder() @@ -647,13 +629,4 @@ public class ApiController { return errorDetail; } - - - private String determineContentType(Path filePath) throws IOException { - String contentType = Files.probeContentType(filePath); - if (contentType == null) { - contentType = "application/octet-stream"; - } - return contentType; - } } \ No newline at end of file diff --git a/src/main/java/ru/soune/nocopy/controller/FileController.java b/src/main/java/ru/soune/nocopy/controller/FileController.java index 38cf43e..56830a0 100644 --- a/src/main/java/ru/soune/nocopy/controller/FileController.java +++ b/src/main/java/ru/soune/nocopy/controller/FileController.java @@ -1,12 +1,16 @@ package ru.soune.nocopy.controller; +import com.google.api.client.util.IOUtils; +import com.google.common.io.ByteStreams; import com.vrt.NoCopyFileService; import com.vrt.fileprotection.FileProtector; import lombok.AllArgsConstructor; import lombok.extern.slf4j.Slf4j; +import org.springframework.core.io.InputStreamResource; import org.springframework.core.io.Resource; import org.springframework.http.*; import org.springframework.web.bind.annotation.*; +import org.springframework.web.servlet.mvc.method.annotation.StreamingResponseBody; import ru.soune.nocopy.dto.file.CheckStatus; import ru.soune.nocopy.entity.file.FileEntity; import ru.soune.nocopy.entity.file.ProtectionStatus; @@ -18,13 +22,14 @@ import ru.soune.nocopy.service.file.CheckCounterService; import ru.soune.nocopy.service.file.FileStorageService; import ru.soune.nocopy.service.file.ImageResizeService; import ru.soune.nocopy.service.file.ZipService; +import ru.soune.nocopy.service.file.cloud.CloudStorageService; import ru.soune.nocopy.util.FileUtil; +import software.amazon.awssdk.services.s3.model.NoSuchKeyException; -import java.io.FileNotFoundException; +import java.io.*; import java.net.URLEncoder; import java.nio.charset.StandardCharsets; -import java.io.IOException; import java.nio.file.Files; import java.nio.file.Path; import java.nio.file.Paths; @@ -51,19 +56,13 @@ public class FileController { private ZipService zipService; + private CloudStorageService cloudStorageService; + @GetMapping("/public/{fileId}") - public ResponseEntity getPublicFile(@PathVariable String fileId) { + public ResponseEntity getPublicFile(@PathVariable String fileId) { try { FileEntity fileEntity = fileRepository.findById(fileId) - .orElseThrow(() -> new FileNotFoundException("Not found: " + fileId)); - - Path filePath = Paths.get(fileEntity.getFilePath()); - if (!Files.exists(filePath)) { - log.error("Not on disk: {}", fileEntity.getFilePath()); - return ResponseEntity.notFound().build(); - } - - Resource resource = fileStorageService.loadFileAsResource(fileEntity.getFilePath()); + .orElseThrow(() -> new FileNotFoundException("Not found in DB: " + fileId)); MediaType mediaType = getMediaType(fileEntity); @@ -71,18 +70,28 @@ public class FileController { .filename(fileEntity.getOriginalFileName(), StandardCharsets.UTF_8) .build(); + StreamingResponseBody responseBody = outputStream -> { + try (InputStream inputStream = cloudStorageService.readFileFromStorage(fileEntity.getFilePath())) { + ByteStreams.copy(inputStream, outputStream); + } catch (NoSuchKeyException e) { + log.error("File disappeared from S3 during streaming: {}", fileId, e); + throw new IOException("File not found in storage", e); + } + }; + return ResponseEntity.ok() .contentType(mediaType) - .contentLength(Files.size(filePath)) .header(HttpHeaders.CONTENT_DISPOSITION, contentDisposition.toString()) .header(HttpHeaders.CACHE_CONTROL, "public, max-age=3600") - .body(resource); - + .body(responseBody); } catch (FileNotFoundException e) { - log.warn("Not on disk: {}", fileId); + log.warn("File not found in DB: {}", fileId); return ResponseEntity.notFound().build(); - } catch (IOException e) { - log.error("Read file exception: {}", fileId, e); + } catch (NoSuchKeyException e) { + log.warn("File not found in S3: {}", fileId); + return ResponseEntity.status(HttpStatus.NOT_FOUND).build(); + } catch (Exception e) { + log.error("Error reading file: {}", fileId, e); return ResponseEntity.status(HttpStatus.INTERNAL_SERVER_ERROR).build(); } } @@ -113,9 +122,14 @@ public class FileController { FileEntity fileEntity = fileRepository.findById(fileId) .orElseThrow(() -> new RuntimeException("File not found")); - String path = imageResizeService.getPath(fileEntity, type); + String filePath = type.equals("thumbnail") ? fileEntity.getThumbnailPath(): fileEntity.getFilePath(); - Resource resource = fileStorageService.loadFileAsResource(path); +// InputStream file = cloudStorageService.readFileFromStorage(filePath); + +// if (file == null || !file.exists()) { +// log.error("File not found in storage: {}", fileEntity.getFilePath()); +// return ResponseEntity.notFound().build(); +// } if (fileEntity.getProtectionStatus() == ProtectionStatus.NOT_PROTECTED || fileEntity.getProtectionStatus() == ProtectionStatus.PROCESSING) { @@ -128,6 +142,8 @@ public class FileController { String encodedFileName = URLEncoder.encode(fileName, StandardCharsets.UTF_8) .replace("+", "%20"); + InputStreamResource resource = new InputStreamResource(cloudStorageService.readFileFromStorage(filePath)); + return ResponseEntity.ok() .contentType(new MediaType(fileEntity.getMimeType(), fileEntity.getFileExtension())) .header(HttpHeaders.CONTENT_DISPOSITION, diff --git a/src/main/java/ru/soune/nocopy/handler/FileEntityHandler.java b/src/main/java/ru/soune/nocopy/handler/FileEntityHandler.java index 40cb18a..5c04a99 100644 --- a/src/main/java/ru/soune/nocopy/handler/FileEntityHandler.java +++ b/src/main/java/ru/soune/nocopy/handler/FileEntityHandler.java @@ -18,6 +18,7 @@ import ru.soune.nocopy.entity.file.moderation.ModerationLog; import ru.soune.nocopy.exception.*; import ru.soune.nocopy.repository.FileAppealRepository; import ru.soune.nocopy.repository.FileEntityRepository; +import ru.soune.nocopy.service.file.cloud.CloudStorageService; import ru.soune.nocopy.repository.ModerationLogRepository; import ru.soune.nocopy.service.file.moderation.ModerationService; import ru.soune.nocopy.service.register.AuthService; @@ -43,6 +44,8 @@ public class FileEntityHandler implements RequestHandler { private final FileEntityRepository fileEntityRepository; + private final CloudStorageService cloudStorageService; + private final ModerationService moderationService; private final FileAppealRepository fileAppealRepository; @@ -700,7 +703,7 @@ public class FileEntityHandler implements RequestHandler { int fullDelete = fileRequest.getFullDelete() == null ? 0 : fileRequest.getFullDelete(); if (fileEntity.getStatus().equals(FileStatus.DELETED) || fullDelete == 1) { - fileEntityService.deleteFromDisk(fileEntity); + cloudStorageService.deleteFromStorageDisk(fileEntity); response = DeleteFileResponse.builder() .fileId(fileRequest.getFileId()) .message("File deleted from disk") diff --git a/src/main/java/ru/soune/nocopy/repository/FileEntityRepository.java b/src/main/java/ru/soune/nocopy/repository/FileEntityRepository.java index 2476b5f..9f6f2c9 100644 --- a/src/main/java/ru/soune/nocopy/repository/FileEntityRepository.java +++ b/src/main/java/ru/soune/nocopy/repository/FileEntityRepository.java @@ -60,12 +60,6 @@ public interface FileEntityRepository extends JpaRepository List findFileByUserIdAndStatus(Long userId, FileStatus status); - @Query("SELECT f FROM FileEntity f WHERE f.userId = :userId AND f.mimeType = :mimeType") - List findByUserIdAndMimeType(Long userId, String mimeType); - - @Query("SELECT f FROM FileEntity f WHERE f.mimeType = :mimeType") - List findByMimeType(String mimeType); - @Query("SELECT f FROM FileEntity f WHERE f.mimeType = :mimeType AND f.userId IN :userIds") List findByMimeTypeAndUserIds(String mimeType, List userIds); diff --git a/src/main/java/ru/soune/nocopy/service/FileSimilarityService.java b/src/main/java/ru/soune/nocopy/service/FileSimilarityService.java index cb7334c..b6a0ca4 100644 --- a/src/main/java/ru/soune/nocopy/service/FileSimilarityService.java +++ b/src/main/java/ru/soune/nocopy/service/FileSimilarityService.java @@ -17,6 +17,7 @@ import ru.soune.nocopy.entity.file.ImageHashEntity; import ru.soune.nocopy.entity.user.User; import ru.soune.nocopy.exception.DuplicateImageException; import ru.soune.nocopy.repository.*; +import ru.soune.nocopy.service.file.cloud.CloudStorageService; import ru.soune.nocopy.util.FileUtil; import java.io.File; @@ -41,6 +42,8 @@ public class FileSimilarityService { private final FileEntityRepository fileEntityRepository; + private final CloudStorageService cloudStorageService; + @Value("${server.baseurl}") private String baseUrl; @@ -114,7 +117,7 @@ public class FileSimilarityService { } public FileEntity findDuplicateByHash(String path, String mimeType, Long userId) throws Exception { - String hash = calculateFileHash(path); + String hash = calculateFileHash(path, false); User user = userRepository.findById(userId).orElseThrow(); Company company = user.getCompany(); @@ -129,15 +132,15 @@ public class FileSimilarityService { List fileEntityList = fileEntityRepository.findByMimeTypeAndUserIds(mimeType, userIds); for (FileEntity file : fileEntityList) { - if (file.getProtectedFilePath() == null) continue; - File existingFile = new File(file.getProtectedFilePath()); - File newFile = new File(path); +// if (file.getProtectedFilePath() == null) continue; +// File existingFile = cloudStorageService.readFileFromStorageByPath(file.getFilePath()); +// File newFile = new File(path); - if (existingFile.length() == newFile.length()) { - if (calculateFileHash(file.getProtectedFilePath()).equals(hash)) { +// if (existingFile.length() == newFile.length()) { + if (calculateFileHash(file.getFilePath(), true).equals(hash)) { return file; } - } +// } } return null; @@ -196,14 +199,19 @@ public class FileSimilarityService { } - private String calculateFileHash(String path) throws Exception { + private String calculateFileHash(String path, boolean cloud) throws Exception { MessageDigest digest = MessageDigest.getInstance("SHA-256"); - try (InputStream is = new FileInputStream(path)) { - byte[] buffer = new byte[8192]; - int read; - while ((read = is.read(buffer)) > 0) { - digest.update(buffer, 0, read); + if (cloud) { + byte[] data = cloudStorageService.readFileFromStorageBytes(path); + digest.update(data); + } else { + try (InputStream is = new FileInputStream(path)) { + byte[] buffer = new byte[8192]; + int read; + while ((read = is.read(buffer)) > 0) { + digest.update(buffer, 0, read); + } } } diff --git a/src/main/java/ru/soune/nocopy/service/ImageHashService.java b/src/main/java/ru/soune/nocopy/service/ImageHashService.java index 0d1bbd6..44d0b54 100644 --- a/src/main/java/ru/soune/nocopy/service/ImageHashService.java +++ b/src/main/java/ru/soune/nocopy/service/ImageHashService.java @@ -8,9 +8,11 @@ import org.springframework.stereotype.Service; import ru.soune.nocopy.entity.file.FileEntity; import ru.soune.nocopy.entity.file.ImageHashEntity; import ru.soune.nocopy.repository.ImageHashRepository; +import ru.soune.nocopy.service.file.cloud.CloudStorageService; import java.io.File; import java.io.IOException; +import java.nio.file.Files; import java.nio.file.Path; import java.time.LocalDateTime; import java.util.Map; @@ -22,18 +24,50 @@ public class ImageHashService { private final ImageHashRepository repository; - public Map calculateHash(Path imagePath) throws IOException { - File file = imagePath.toFile(); + private final CloudStorageService cloudStorageService; - PHash pHash = PerceptualHashHelper.INSTANCE.generateDCTPerceptualHash(file); +// public Map calculateHash(Path imagePath) throws IOException { +//// File file = cloudStorageService.readFileFromStorageByPath(imagePath.toString()); +// File file = imagePath.toFile(); +// if (file == null) { +// file = cloudStorageService.readFileFromStorageByPath(imagePath.toString()); +// } +// +// PHash pHash = PerceptualHashHelper.INSTANCE.generateDCTPerceptualHash(file); +// +// Long firstPart = pHash.getFirstPart(); +// Long secondPart = pHash.getSecondPart(); +// Map hash = Map.of( +// "hi", firstPart, +// "low", secondPart); +// +// return hash; +// } + + public Map calculateHash(Path imagePath) throws IOException { + PHash pHash; + + if (imagePath.toFile().exists()) { + pHash = PerceptualHashHelper.INSTANCE.generateDCTPerceptualHash(imagePath.toFile()); + } else { + byte[] data = cloudStorageService.readFileFromStorageBytes(imagePath.toString()); + + Path tempFile = Files.createTempFile("perceptual_hash_", ".tmp"); + try { + Files.write(tempFile, data); + pHash = PerceptualHashHelper.INSTANCE.generateDCTPerceptualHash(tempFile.toFile()); + } finally { + Files.deleteIfExists(tempFile); + } + } Long firstPart = pHash.getFirstPart(); Long secondPart = pHash.getSecondPart(); - Map hash = Map.of( - "hi", firstPart, - "low", secondPart); - return hash; + return Map.of( + "hi", firstPart, + "low", secondPart + ); } public void create(FileEntity file, Map stringIntegerMap) { diff --git a/src/main/java/ru/soune/nocopy/service/file/FileCleanupService.java b/src/main/java/ru/soune/nocopy/service/file/FileCleanupService.java index 7d90acb..10b12a9 100644 --- a/src/main/java/ru/soune/nocopy/service/file/FileCleanupService.java +++ b/src/main/java/ru/soune/nocopy/service/file/FileCleanupService.java @@ -8,6 +8,7 @@ import lombok.extern.slf4j.Slf4j; import ru.soune.nocopy.entity.user.ProtectedFileCheck; import ru.soune.nocopy.repository.ProtectedFileCheckRepository; +import java.io.File; import java.io.IOException; import java.nio.file.*; import java.nio.file.attribute.BasicFileAttributes; @@ -15,7 +16,10 @@ import java.time.Instant; import java.time.LocalDate; import java.time.LocalDateTime; import java.time.ZoneId; +import java.time.temporal.ChronoUnit; import java.util.List; +import java.util.stream.Collectors; +import java.util.stream.Stream; @Slf4j @Component @@ -34,6 +38,8 @@ public class FileCleanupService { public void cleanupExpiredFiles() { log.info("Starting cleanup of expired temporary files"); + cleanupTempFilesNightly(); + Path tempDir = Paths.get(basePath, "temp"); if (!Files.exists(tempDir)) { return; @@ -83,4 +89,50 @@ public class FileCleanupService { log.error("Error during cleanup", e); } } + + private void cleanupTempFilesNightly() { + log.info("Starting nightly cleanup of temporary files"); + + String tempDir = System.getProperty("java.io.tmpdir"); + Path tempPath = Paths.get(tempDir); + + if (!Files.exists(tempPath)) { + return; + } + + Instant cutoffTime = Instant.now().minus(24, ChronoUnit.HOURS); // 24 часа + int deletedCount = 0; + + try (Stream files = Files.list(tempPath)) { + List tempFiles = files + .filter(path -> { + String filename = path.getFileName().toString(); + return filename.contains("_") && + !filename.endsWith(".tmp") && + Files.isRegularFile(path); + }) + .collect(Collectors.toList()); + + for (Path filePath : tempFiles) { + try { + File file = filePath.toFile(); + Instant lastModified = Instant.ofEpochMilli(file.lastModified()); + + if (lastModified.isBefore(cutoffTime)) { + boolean deleted = Files.deleteIfExists(filePath); + if (deleted) { + deletedCount++; + } + } + } catch (IOException e) { + log.error("Error deleting file: {}", filePath, e); + } + } + + log.info("Nightly cleanup deleted {} temporary files", deletedCount); + + } catch (IOException e) { + log.error("Failed to list temp directory", e); + } + } } diff --git a/src/main/java/ru/soune/nocopy/service/file/FileEntityService.java b/src/main/java/ru/soune/nocopy/service/file/FileEntityService.java index 0c1ed48..0228914 100644 --- a/src/main/java/ru/soune/nocopy/service/file/FileEntityService.java +++ b/src/main/java/ru/soune/nocopy/service/file/FileEntityService.java @@ -19,12 +19,15 @@ import ru.soune.nocopy.exception.FileEntityNotFoundException; import ru.soune.nocopy.repository.FileEntityRepository; import ru.soune.nocopy.repository.FileMonitoringRepository; import ru.soune.nocopy.repository.UserRepository; +import ru.soune.nocopy.service.file.cloud.CloudStorageService; -import java.io.File; import java.io.IOException; +import java.nio.ByteBuffer; +import java.nio.channels.FileChannel; import java.nio.file.Files; import java.nio.file.Path; import java.nio.file.Paths; +import java.nio.file.StandardOpenOption; import java.time.LocalDateTime; import java.util.ArrayList; import java.util.List; @@ -42,6 +45,8 @@ public class FileEntityService { private final ImageResizeService imageResizeService; + private final CloudStorageService cloudStorageService; + @Value("${server.baseurl}") private String baseUrl; @@ -202,16 +207,8 @@ public class FileEntityService { fileEntityRepository.save(fileEntity); } - public void deleteFromDisk(FileEntity fileEntity) throws IOException { - Path path = Paths.get(fileEntity.getFilePath()); - - if (!Files.exists(path)) { - return; - } - - Files.delete(path); - - fileEntityRepository.delete(fileEntity); + public void deleteFromDisk(String deleteFilePath) throws IOException { + Files.deleteIfExists(Paths.get(deleteFilePath)); } @Transactional(readOnly = true) @@ -237,18 +234,27 @@ public class FileEntityService { } @Transactional - public void writeProtectedFile(String id, byte[] data, String fileExt) throws IOException { + public FileEntity writeProtectedFile(String id, byte[] data, String fileExt) throws IOException { FileEntity fileEntity = fileEntityRepository.findById(id) .orElseThrow(() -> new RuntimeException("File not found: " + id)); String extension = determineFileExtension(fileExt, fileEntity); Path protectedFilePath = prepareProtectedPath(fileEntity, extension); - if (Files.exists(protectedFilePath)) { - Files.delete(protectedFilePath); - } + Files.createDirectories(protectedFilePath.getParent()); - Files.write(protectedFilePath, data); + Files.deleteIfExists(protectedFilePath); + + try (FileChannel channel = FileChannel.open(protectedFilePath, + StandardOpenOption.CREATE, + StandardOpenOption.WRITE)) { + + ByteBuffer buffer = ByteBuffer.wrap(data); + while (buffer.hasRemaining()) { + channel.write(buffer); + } + channel.force(true); + } if (imageResizeService.isImage(extension)) { imageResizeService.generateSizes(fileEntity, data); @@ -260,14 +266,14 @@ public class FileEntityService { fileEntity.setFileExtension(extension); fileEntity.setProtectionStatus(ProtectionStatus.PROTECTED); - fileEntityRepository.save(fileEntity); + return fileEntityRepository.save(fileEntity); } - public void clearTempFiles(long userId) throws IOException { + public void clearTempFiles(long userId) { List fileByUserIdAndStatus = fileEntityRepository.findFileByUserIdAndStatus(userId, FileStatus.TEMP); for (FileEntity fileEntity : fileByUserIdAndStatus) { - deleteFromDisk(fileEntity); + cloudStorageService.deleteFromStorageDisk(fileEntity); } } @@ -279,24 +285,6 @@ public class FileEntityService { return fileEntityRepository.findFileIdByFilePath(filePath); } - public File getFileById(String id) { - try { - FileEntity fileEntity = fileEntityRepository.findById(id).orElseThrow(() -> - new RuntimeException("File not found: " + id)); - - File file = new File(fileEntity.getFilePath()); - - if (!file.exists()) { - throw new RuntimeException("File not found on disk: " + fileEntity.getFilePath()); - } - - return file; - } catch (Exception e) { - log.error("Error getting file: {}", id, e); - return null; - } - } - public Path prepareProtectedPath(FileEntity fileEntity, String extension) throws IOException { Path originalPath = Paths.get(fileEntity.getFilePath()); @@ -322,7 +310,7 @@ public class FileEntityService { return protectedPath; } - public void updateSignature(String signature, String fileId) throws IOException { + public FileEntity updateSignature(String signature, String fileId) throws IOException { FileEntity fileEntity = fileEntityRepository.findByFileId(fileId); fileEntity.setSignature(signature); fileEntity.setProtectionStatus(ProtectionStatus.PROTECTED); @@ -330,18 +318,30 @@ public class FileEntityService { fileEntity.setProtectedAt(LocalDateTime.now()); fileEntity.setProtectedFilePath(prepareProtectedPath(fileEntity, fileEntity.getFileExtension()).toString()); - fileEntityRepository.save(fileEntity); + return fileEntityRepository.save(fileEntity); } - private boolean checkFileExistsOnDisk(String filePath) { - try { - return Files.exists(Paths.get(filePath)); - } catch (Exception e) { - log.warn("Error checking file existence: {}", filePath, e); - return false; + public String formatFileSize(long size) { + if (size < 1024) { + return size + " B"; + } else if (size < 1024 * 1024) { + return String.format("%.1f KB", size / 1024.0); + } else if (size < 1024 * 1024 * 1024) { + return String.format("%.1f MB", size / (1024.0 * 1024.0)); + } else { + return String.format("%.1f GB", size / (1024.0 * 1024.0 * 1024.0)); } } +// private boolean checkFileExistsOnDisk(String filePath) { +// try { +// return Files.exists(Paths.get(filePath)); +// } catch (Exception e) { +// log.warn("Error checking file existence: {}", filePath, e); +// return false; +// } +// } + private String determineFileExtension(String fileExt, FileEntity fileEntity) { if (fileExt != null) { return fileExt; @@ -353,7 +353,7 @@ public class FileEntityService { private final FileMonitoringRepository fileMonitoringRepository; private FileEntityResponse convertToResponse(FileEntity fileEntity, int version) { - boolean existsOnDisk = checkFileExistsOnDisk(fileEntity.getFilePath()); +// boolean existsOnDisk = checkFileExistsOnDisk(fileEntity.getFilePath()); User user = userRepository.findById(fileEntity.getUserId()).get(); String fileName = fileEntity.getOriginalFileName(); String name = fileName; @@ -385,7 +385,7 @@ public class FileEntityService { .updatedAt(fileEntity.getUpdatedAt()) .formattedSize(formatFileSize(fileEntity.getFileSize())) .downloadUrl("/api/v" + version + "/files/download/" + fileEntity.getId()) - .existsOnDisk(existsOnDisk) +// .existsOnDisk(existsOnDisk) .supportId(fileEntity.getSupportId()) .protectStatus(fileEntity.getProtectionStatus().toString()) .fileName(name + "_nocopy_protected" + "." + fileEntity.getFileExtension()) @@ -400,16 +400,4 @@ public class FileEntityService { .thumbnailFileUrl(baseUrl + "/api/files/protected/" + fileEntity.getId() + "/thumbnail") .build(); } - - public String formatFileSize(long size) { - if (size < 1024) { - return size + " B"; - } else if (size < 1024 * 1024) { - return String.format("%.1f KB", size / 1024.0); - } else if (size < 1024 * 1024 * 1024) { - return String.format("%.1f MB", size / (1024.0 * 1024.0)); - } else { - return String.format("%.1f GB", size / (1024.0 * 1024.0 * 1024.0)); - } - } } \ No newline at end of file diff --git a/src/main/java/ru/soune/nocopy/service/file/FileUploadService.java b/src/main/java/ru/soune/nocopy/service/file/FileUploadService.java index 88eb125..ac7bc06 100644 --- a/src/main/java/ru/soune/nocopy/service/file/FileUploadService.java +++ b/src/main/java/ru/soune/nocopy/service/file/FileUploadService.java @@ -2,11 +2,10 @@ package ru.soune.nocopy.service.file; import org.springframework.web.multipart.MultipartFile; import ru.soune.nocopy.dto.file.UploadProgressResponse; +import ru.soune.nocopy.entity.file.FileEntity; import ru.soune.nocopy.entity.file.FileStatus; import ru.soune.nocopy.entity.file.FileUploadSession; -import java.io.IOException; - public interface FileUploadService { FileUploadSession initUpload(Long userId, String fileName, String fileType, String extension, String convertTo, long fileSize); @@ -22,5 +21,5 @@ public interface FileUploadService { UploadProgressResponse getUploadProgress(String uploadId); - void completeFileProcessingAsync(FileUploadSession session, FileStatus status); + FileEntity completeFileProcessing(FileUploadSession session, FileStatus status); } diff --git a/src/main/java/ru/soune/nocopy/service/file/ImageResizeService.java b/src/main/java/ru/soune/nocopy/service/file/ImageResizeService.java index 57d6216..34d4c00 100644 --- a/src/main/java/ru/soune/nocopy/service/file/ImageResizeService.java +++ b/src/main/java/ru/soune/nocopy/service/file/ImageResizeService.java @@ -1,10 +1,12 @@ package ru.soune.nocopy.service.file; +import lombok.RequiredArgsConstructor; import lombok.extern.slf4j.Slf4j; import org.springframework.beans.factory.annotation.Value; import org.springframework.stereotype.Service; import org.springframework.transaction.annotation.Transactional; import ru.soune.nocopy.entity.file.FileEntity; +import ru.soune.nocopy.service.file.cloud.CloudStorageService; import javax.imageio.ImageIO; import java.awt.*; @@ -18,9 +20,13 @@ import java.util.List; @Service @Slf4j +@RequiredArgsConstructor public class ImageResizeService { + private final CloudStorageService cloudStorageService; + private static final int THUMBNAIL_SIZE = 150; + private static final int MEDIUM_SIZE = 600; @Value("${file.storage.base-path}") @@ -50,11 +56,15 @@ public class ImageResizeService { if (original.getWidth() > THUMBNAIL_SIZE) { String thumbPath = resizeAndSave(original, baseName + "_thumb", THUMBNAIL_SIZE); file.setThumbnailPath(thumbPath); + cloudStorageService.saveFilesToStorage(file.getMimeType(), file.getFileExtension(), + thumbPath); } if (original.getWidth() > MEDIUM_SIZE) { String mediumPath = resizeAndSave(original, baseName + "_medium", MEDIUM_SIZE); file.setMediumPath(mediumPath); + cloudStorageService.saveFilesToStorage(file.getMimeType(), file.getFileExtension(), + mediumPath); } } } diff --git a/src/main/java/ru/soune/nocopy/service/file/cloud/CloudStorageService.java b/src/main/java/ru/soune/nocopy/service/file/cloud/CloudStorageService.java new file mode 100644 index 0000000..6aee80e --- /dev/null +++ b/src/main/java/ru/soune/nocopy/service/file/cloud/CloudStorageService.java @@ -0,0 +1,194 @@ +package ru.soune.nocopy.service.file.cloud; + +import lombok.RequiredArgsConstructor; +import lombok.extern.slf4j.Slf4j; +import org.springframework.beans.factory.annotation.Value; +import org.springframework.stereotype.Service; +import ru.soune.nocopy.entity.file.FileEntity; +import ru.soune.nocopy.repository.FileEntityRepository; +import software.amazon.awssdk.auth.credentials.AwsBasicCredentials; +import software.amazon.awssdk.auth.credentials.AwsCredentials; +import software.amazon.awssdk.auth.credentials.StaticCredentialsProvider; +import software.amazon.awssdk.core.ResponseBytes; +import software.amazon.awssdk.core.sync.RequestBody; +import software.amazon.awssdk.core.sync.ResponseTransformer; +import software.amazon.awssdk.http.apache.ApacheHttpClient; +import software.amazon.awssdk.regions.Region; +import software.amazon.awssdk.services.s3.S3Client; +import software.amazon.awssdk.services.s3.model.GetObjectRequest; +import software.amazon.awssdk.services.s3.model.GetObjectResponse; +import software.amazon.awssdk.services.s3.model.PutObjectRequest; + +import java.io.File; +import java.io.IOException; +import java.io.InputStream; +import java.net.URI; +import java.nio.file.Files; +import java.nio.file.Paths; +import java.util.UUID; + +@Service +@Slf4j +@RequiredArgsConstructor +public class CloudStorageService { + + private final FileEntityRepository fileEntityRepository; + + @Value("${yandex.cloud.key}") + private String key; + + @Value("${yandex.cloud.secret-key}") + private String secretKey; + + @Value("${yandex.cloud.region}") + private String region; + + @Value("${yandex.cloud.s3-endpoint}") + private String s3endpoint; + + @Value("${yandex.cloud.bucket}") + private String bucket; + + public void saveFilesToStorage(String mimeType, String extension, String sendToCloudFilePath) { + AwsCredentials credentials = AwsBasicCredentials.create(key, secretKey); + S3Client s3Client = S3Client.builder() + .httpClient(ApacheHttpClient.create()) + .region(Region.of(region)) + .endpointOverride(URI.create(s3endpoint)) + .credentialsProvider(StaticCredentialsProvider.create(credentials)) + .build(); + + String mimeTypeFromFile = mimeType.equals("document") ? "application" : mimeType; + String contentType = mimeTypeFromFile + "/" + extension; + String filePath = "files" + sendToCloudFilePath; + + PutObjectRequest putObjectRequest = PutObjectRequest.builder() + .bucket(bucket) + .key(filePath) + .contentType(contentType) + .build(); + try { + byte[] bytes = Files.readAllBytes(Paths.get(sendToCloudFilePath)); + s3Client.putObject(putObjectRequest, + RequestBody.fromBytes(bytes)); + Files.deleteIfExists(Paths.get(sendToCloudFilePath)); + } catch (IOException e) { + log.error("Failed to read file: {}", filePath, e); + } + } + + public File readFileFromStorageByPath(String filePath) throws IOException { + GetObjectRequest objectRequest = GetObjectRequest.builder() + .bucket(bucket) + .key("files" + filePath) + .build(); + AwsCredentials credentials = AwsBasicCredentials.create(key, secretKey); + + S3Client s3Client = S3Client.builder() + .httpClient(ApacheHttpClient.create()) + .region(Region.of(region)) + .endpointOverride(URI.create(s3endpoint)) + .credentialsProvider(StaticCredentialsProvider.create(credentials)) + .build(); + + String tempDir = System.getProperty("java.io.tmpdir"); + + String fileName = System.currentTimeMillis() + "_" + UUID.randomUUID() + "_" + + filePath.replace("/", "_"); + File tempFile = new File(tempDir, fileName); + + s3Client.getObject(objectRequest, tempFile.toPath()); + + tempFile.deleteOnExit(); + + return tempFile; + } + + public byte[] readFileFromStorageBytes(String filePath) { + AwsCredentials credentials = AwsBasicCredentials.create(key, secretKey); + + S3Client s3Client = S3Client.builder() + .httpClient(ApacheHttpClient.create()) + .region(Region.of(region)) + .endpointOverride(URI.create(s3endpoint)) + .credentialsProvider(StaticCredentialsProvider.create(credentials)) + .build(); + + GetObjectRequest objectRequest = GetObjectRequest.builder() + .bucket(bucket) + .key("files" + filePath) + .build(); + + try { + ResponseBytes responseBytes = s3Client.getObjectAsBytes(objectRequest); + return responseBytes.asByteArray(); + } catch (Exception e) { + log.error("Failed to read file from S3: {}", filePath, e); + throw new RuntimeException("Failed to read file from S3", e); + } + } + + + public InputStream readFileFromStorage(String filePath) { + GetObjectRequest objectRequest = GetObjectRequest.builder() + .bucket(bucket) + .key("files" + filePath) + .build(); + + AwsCredentials credentials = AwsBasicCredentials.create(key, secretKey); + + S3Client s3Client = S3Client.builder() + .httpClient(ApacheHttpClient.create()) + .region(Region.of(region)) + .endpointOverride(URI.create(s3endpoint)) + .credentialsProvider(StaticCredentialsProvider.create(credentials)) + .build(); + + return s3Client.getObject(objectRequest, ResponseTransformer.toInputStream()); + } + + public void deleteFileFromStorage(String filePath) { + if (filePath == null || filePath.isEmpty()) { + log.warn("Attempted to delete file with empty path"); + return; + } + + AwsCredentials credentials = AwsBasicCredentials.create(key, secretKey); + + S3Client s3Client = S3Client.builder() + .httpClient(ApacheHttpClient.create()) + .region(Region.of(region)) + .endpointOverride(URI.create(s3endpoint)) + .credentialsProvider(StaticCredentialsProvider.create(credentials)) + .build(); + + String cloudKey = "files" + filePath; + + try { + try { + s3Client.headObject(builder -> builder.bucket(bucket).key(cloudKey).build()); + } catch (Exception e) { + log.warn("File does not exist in cloud storage: {}", cloudKey); + return; + } + + s3Client.deleteObject(builder -> builder.bucket(bucket).key(cloudKey).build()); + + log.info("File successfully deleted from cloud storage: {}", cloudKey); + + } catch (Exception e) { + log.error("Failed to delete file from cloud storage: {}", cloudKey, e); + throw new RuntimeException("Failed to delete file from cloud storage: " + cloudKey, e); + } + } + + public void deleteFromStorageDisk(FileEntity fileEntity) { + deleteFileFromStorage(fileEntity.getFilePath()); + + if (fileEntity.getProtectedFilePath() != null) { + deleteFileFromStorage(fileEntity.getProtectedFilePath()); + } + + fileEntityRepository.delete(fileEntity); + } +} diff --git a/src/main/java/ru/soune/nocopy/service/file/impl/AudioLocalSearchImpl.java b/src/main/java/ru/soune/nocopy/service/file/impl/AudioLocalSearchImpl.java index 7d2e137..948e744 100644 --- a/src/main/java/ru/soune/nocopy/service/file/impl/AudioLocalSearchImpl.java +++ b/src/main/java/ru/soune/nocopy/service/file/impl/AudioLocalSearchImpl.java @@ -9,6 +9,7 @@ import org.jetbrains.annotations.Nullable; import org.springframework.stereotype.Service; import ru.soune.nocopy.entity.file.FileEntity; import ru.soune.nocopy.service.file.FileEntityService; +import ru.soune.nocopy.service.file.cloud.CloudStorageService; import ru.soune.nocopy.util.FileUtil; import java.io.IOException; @@ -22,6 +23,8 @@ public class AudioLocalSearchImpl implements AudioLocalSearch { private final FileUtil fileUtil; + private final CloudStorageService cloudStorageService; + @Override public @Nullable FileProtector.FileInfo findBySignature(@NotNull String signature) { FileEntity fileEntity = fileEntityService.findBySignature(signature); @@ -34,7 +37,10 @@ public class AudioLocalSearchImpl implements AudioLocalSearch { @Override public void updateSignature(@NotNull String signature, @NotNull FileProtector.FileInfo fileInfo) { try { - fileEntityService.updateSignature(signature, fileInfo.getId()); + FileEntity fileEntity = fileEntityService.updateSignature(signature, fileInfo.getId()); + + cloudStorageService.saveFilesToStorage(fileEntity.getMimeType(), fileEntity.getFileExtension(), + fileEntity.getProtectedFilePath()); } catch (IOException e) { throw new RuntimeException(e); } diff --git a/src/main/java/ru/soune/nocopy/service/file/impl/DocumentLocalSearchImpl.java b/src/main/java/ru/soune/nocopy/service/file/impl/DocumentLocalSearchImpl.java index b7151d1..e5709f5 100644 --- a/src/main/java/ru/soune/nocopy/service/file/impl/DocumentLocalSearchImpl.java +++ b/src/main/java/ru/soune/nocopy/service/file/impl/DocumentLocalSearchImpl.java @@ -25,6 +25,23 @@ public class DocumentLocalSearchImpl implements DocumentLocalSearch { FileEntity fileEntity = fileEntityRepository.findByIdAndUserId(fileId, Long.valueOf(ownerId)); FileProtector.FileInfo fileInfo = fileUtil.createFileInfo(fileEntity, null); log.info("fileInfo: {}", fileInfo); + log.info("fileInfo: {}", fileInfo); + log.info("fileInfo: {}", fileInfo); + log.info("fileInfo: {}", fileInfo); + log.info("fileInfo: {}", fileInfo); + log.info("fileInfo: {}", fileInfo); + log.info("fileInfo: {}", fileInfo); + log.info("fileInfo: {}", fileInfo); + log.info("fileInfo: {}", fileInfo); + log.info("fileInfo: {}", fileInfo); + log.info("fileInfo: {}", fileInfo); + log.info("fileInfo: {}", fileInfo); + log.info("fileInfo: {}", fileInfo); + log.info("fileInfo: {}", fileInfo); + log.info("fileInfo: {}", fileInfo); + log.info("fileInfo: {}", fileInfo); + log.info("fileInfo: {}", fileInfo); + log.info("fileInfo: {}", fileInfo); return fileInfo; } diff --git a/src/main/java/ru/soune/nocopy/service/file/impl/FileUploadServiceImpl.java b/src/main/java/ru/soune/nocopy/service/file/impl/FileUploadServiceImpl.java index 53d1b1a..cdb6b40 100644 --- a/src/main/java/ru/soune/nocopy/service/file/impl/FileUploadServiceImpl.java +++ b/src/main/java/ru/soune/nocopy/service/file/impl/FileUploadServiceImpl.java @@ -6,6 +6,7 @@ import com.vrt.fileprotection.NoCopyCheckResult; import com.vrt.fileprotection.audio.AudioCheckResult; import com.vrt.fileprotection.documents.DocumentCheckResult; import jakarta.annotation.PostConstruct; +import jakarta.persistence.EntityManager; import lombok.RequiredArgsConstructor; import lombok.extern.slf4j.Slf4j; import org.springframework.beans.factory.annotation.Autowired; @@ -28,6 +29,8 @@ import ru.soune.nocopy.service.ImageHashService; import ru.soune.nocopy.service.cost.CostService; import ru.soune.nocopy.service.file.FileEntityService; import ru.soune.nocopy.service.file.FileUploadService; +import ru.soune.nocopy.service.file.cloud.CloudStorageService; +import ru.soune.nocopy.service.tariff.TariffConstants; import ru.soune.nocopy.service.file.moderation.ModerationFileService; import ru.soune.nocopy.service.notification.NotificationService; import ru.soune.nocopy.service.register.AuthService; @@ -95,6 +98,8 @@ public class FileUploadServiceImpl implements FileUploadService { private final ModerationFileService moderationFileService; + private final CloudStorageService cloudStorageService; + @PostConstruct public void init() { try { @@ -190,7 +195,6 @@ public class FileUploadServiceImpl implements FileUploadService { } @Override - @Transactional public UploadProgressResponse uploadChunk(String uploadId, Integer chunkNumber, MultipartFile chunkFile, Integer findSimilar) { FileUploadSession session = sessionRepository.findById(uploadId) @@ -222,10 +226,9 @@ public class FileUploadServiceImpl implements FileUploadService { return UploadProgressResponse.fromSession(session); } - @Async("fileUploadTaskExecutor") @Transactional @Override - public void completeFileProcessingAsync(FileUploadSession session, FileStatus status) { + public FileEntity completeFileProcessing(FileUploadSession session, FileStatus status) { try { Path filePath = Paths.get(session.getFilePath()); String checksum = calculateChecksum(filePath); @@ -251,22 +254,17 @@ public class FileUploadServiceImpl implements FileUploadService { if (session.getFileType().equals("image") && status != FileStatus.PRIVATE) { Map hash = imageHashService.calculateHash(filePath); - imageHashService.create(saved, hash); } cleanupSessionFiles(session); - - if (status != FileStatus.TEMP && status != FileStatus.PRIVATE) { - FileProtector.FileInfo fileInfo = fileUtil.createFileInfo(fileEntity, session.getConvertTo()); - - noCopyFileService.addFile(fileInfo); - } - log.info("File processing completed for session: {}", session.getUploadId()); + + return saved; } catch (Exception e) { log.error("Failed to complete file processing for session {}: {}", session.getUploadId(), e.getMessage(), e); + throw new FileUploadException("Failed to complete file processing", e); } } @@ -339,7 +337,7 @@ public class FileUploadServiceImpl implements FileUploadService { session.setStatus(UploadStatus.COMPLETED); session.setFilePath(finalFilePath); - completeFileProcessingAsync(session, FileStatus.PRIVATE); + completeFileProcessing(session, FileStatus.PRIVATE); } else { sessionRepository.save(session); } @@ -381,7 +379,7 @@ public class FileUploadServiceImpl implements FileUploadService { chunkNumber, session.getChunksUploaded(), session.getTotalChunks()); boolean isLastChunk = session.getChunksUploaded().equals(session.getTotalChunks()); - + //TODO CHECK if (isLastChunk) { String finalFilePath = assembleFileSynchronously(session); @@ -393,6 +391,7 @@ public class FileUploadServiceImpl implements FileUploadService { session.getUserId()); File file = new File(finalFilePath); +// File file = cloudStorageService.readFileFromStorageByPath(finalFilePath); if (session.getFileType().startsWith("audio") && !isWavFile(file)) { session.setStatus(UploadStatus.FAILED); @@ -432,20 +431,29 @@ public class FileUploadServiceImpl implements FileUploadService { session.setStatus(UploadStatus.COMPLETED); session.setFilePath(finalFilePath); - notificationService.addNotification(NotificationType.FILE_ADDED_TO_SYSTEM, session.getUserId()); - FileStatus status = findSimilar == 0 ? FileStatus.MODERATION: FileStatus.TEMP; if (status == FileStatus.TEMP) { fileEntityService.clearTempFiles(session.getUserId()); } - completeFileProcessingAsync(session, status); + FileEntity fileEntity = completeFileProcessing(session, status); + + cloudStorageService.saveFilesToStorage(fileEntity.getMimeType(), fileEntity.getFileExtension(), + fileEntity.getFilePath()); if (status != FileStatus.TEMP) { String fileType = session.getFileType(); - tariffInfoService.writeOffTokens(session.getUserId(), costService.protectFileCost(fileType), OperationType.FILE_UPLOAD); + tariffInfoService.writeOffTokens(session.getUserId(), costService.protectFileCost(fileType), + OperationType.FILE_UPLOAD); + notificationService.addNotification(NotificationType.FILE_ADDED_TO_SYSTEM, session.getUserId()); + } + + if (status != FileStatus.TEMP) { + FileProtector.FileInfo fileInfo = fileUtil.createFileInfo(fileEntity, session.getConvertTo()); + + noCopyFileService.addFile(fileInfo); } } else { sessionRepository.save(session); @@ -580,6 +588,7 @@ public class FileUploadServiceImpl implements FileUploadService { private void checkForDuplicatesSynchronously(String filePath) throws IOException , DuplicateImageException { Path path = Paths.get(filePath); + Map hash = imageHashService.calculateHash(path); List duplicates = fileSimilarityService.findDuplicatedByHash( diff --git a/src/main/java/ru/soune/nocopy/service/file/impl/ProtectionFileProviderImpl.java b/src/main/java/ru/soune/nocopy/service/file/impl/ProtectionFileProviderImpl.java index 304db80..bd07d25 100644 --- a/src/main/java/ru/soune/nocopy/service/file/impl/ProtectionFileProviderImpl.java +++ b/src/main/java/ru/soune/nocopy/service/file/impl/ProtectionFileProviderImpl.java @@ -11,7 +11,7 @@ import ru.soune.nocopy.entity.file.FileEntity; import ru.soune.nocopy.repository.FileEntityRepository; import ru.soune.nocopy.repository.FileUploadSessionRepository; import ru.soune.nocopy.service.file.FileEntityService; -import ru.soune.nocopy.service.file.FileUploadService; +import ru.soune.nocopy.service.file.cloud.CloudStorageService; import java.io.File; import java.io.IOException; @@ -32,27 +32,53 @@ public class ProtectionFileProviderImpl implements FileProtector.FileProvider { private final FileEntityRepository fileEntityRepository; + private final CloudStorageService cloudStorageService; + @Nullable @Override public File getImageFile(@NotNull String id) { - return fileEntityService.getFileById(id); + FileEntity fileEntity = fileEntityRepository.findById(id).orElseThrow(() -> + new RuntimeException("File not found: " + id)); + try { + return cloudStorageService.readFileFromStorageByPath(fileEntity.getFilePath()); + } catch (IOException e) { + throw new RuntimeException(e); + } } @Nullable @Override public File getVideoFile(@NotNull String id) { - return fileEntityService.getFileById(id); + FileEntity fileEntity = fileEntityRepository.findById(id).orElseThrow(() -> + new RuntimeException("File not found: " + id)); + try { + return cloudStorageService.readFileFromStorageByPath(fileEntity.getFilePath()); + } catch (IOException e) { + throw new RuntimeException(e); + } } @Nullable @Override public File getAudioFile(@NotNull String id) { - return fileEntityService.getFileById(id); + FileEntity fileEntity = fileEntityRepository.findById(id).orElseThrow(() -> + new RuntimeException("File not found: " + id)); + try { + return cloudStorageService.readFileFromStorageByPath(fileEntity.getFilePath()); + } catch (IOException e) { + throw new RuntimeException(e); + } } @Override public @Nullable File getDocument(@NotNull String id) { - return fileEntityService.getFileById(id); + FileEntity fileEntity = fileEntityRepository.findById(id).orElseThrow(() -> + new RuntimeException("File not found: " + id)); + try { + return cloudStorageService.readFileFromStorageByPath(fileEntity.getFilePath()); + } catch (IOException e) { + throw new RuntimeException(e); + } } @Override @@ -87,7 +113,20 @@ public class ProtectionFileProviderImpl implements FileProtector.FileProvider { @Override public com.vrt.fileprotection.OperationResult writeAudioFile(@NotNull String id, @NotNull byte[] data, @Nullable String fileExt) { try { - fileEntityService.writeProtectedFile(id, data, fileExt); + FileEntity fileEntity = fileEntityService.writeProtectedFile(id, data, fileExt); + + Path filePath = Paths.get(fileEntity.getProtectedFilePath()); + + if (!Files.exists(filePath)) { + log.error("File not found after write: {}", filePath); + return OperationResult.Companion.failure("File not found"); + } + + log.info("File ready for upload: {} (size: {} bytes)", filePath, Files.size(filePath)); + + cloudStorageService.saveFilesToStorage(fileEntity.getMimeType(), fileEntity.getFileExtension(), + fileEntity.getProtectedFilePath()); + return OperationResult.Companion.success(); } catch (Exception e) { log.error("Failed to create protected file: {}", id, e); @@ -99,7 +138,11 @@ public class ProtectionFileProviderImpl implements FileProtector.FileProvider { @Override public com.vrt.fileprotection.OperationResult writeVideoFile(@NotNull String id, @NotNull byte[] data, @Nullable String fileExt) { try { - fileEntityService.writeProtectedFile(id, data, fileExt); + FileEntity fileEntity = fileEntityService.writeProtectedFile(id, data, fileExt); + + cloudStorageService.saveFilesToStorage(fileEntity.getMimeType(), fileEntity.getFileExtension(), + fileEntity.getProtectedFilePath()); + return OperationResult.Companion.success(); } catch (Exception e) { log.error("Failed to create protected file: {}", id, e); @@ -111,7 +154,11 @@ public class ProtectionFileProviderImpl implements FileProtector.FileProvider { @Override public com.vrt.fileprotection.OperationResult writeImageFile(@NotNull String id, @NotNull byte[] data, @Nullable String fileExt) { try { - fileEntityService.writeProtectedFile(id, data, fileExt); + FileEntity fileEntity = fileEntityService.writeProtectedFile(id, data, fileExt); + + cloudStorageService.saveFilesToStorage(fileEntity.getMimeType(), fileEntity.getFileExtension(), + fileEntity.getProtectedFilePath()); + return OperationResult.Companion.success(); } catch (Exception e) { log.error("Failed to create protected file: {}", id, e); @@ -122,7 +169,10 @@ public class ProtectionFileProviderImpl implements FileProtector.FileProvider { @Override public @NotNull OperationResult writeDocumentFile(@NotNull String id, @NotNull byte[] data) { try { - fileEntityService.writeProtectedFile(id, data, null); + FileEntity fileEntity = fileEntityService.writeProtectedFile(id, data, null); + + cloudStorageService.saveFilesToStorage(fileEntity.getMimeType(), fileEntity.getFileExtension(), + fileEntity.getProtectedFilePath()); return OperationResult.Companion.success(); } catch (Exception e) { log.error("Failed to create protected file: {}", id, e); diff --git a/src/main/resources/application.yaml b/src/main/resources/application.yaml index 668dda5..6bc410a 100644 --- a/src/main/resources/application.yaml +++ b/src/main/resources/application.yaml @@ -102,6 +102,13 @@ yandex: api-key: ${YANDEX_API_KEY:AQVNyaVaUmgUb1GMCtf5zSEqFxy0woXrcMOOB43q} folder-id: ${YANDEX_FOLDER_ID:b1gokpdbm6qfpsou8pcd} search-url: ${YANDEX_SEARCH_URL:https://searchapi.api.cloud.yandex.net/v2/image/search_by_image} + cloud: + key: "YCAJEmHZcWxzf4oGWvETG3mms" + secret-key: "YCPAPZ32XC-LWb8xE0b_xTcBD8NZUCHpSOfu8LUG" + region: "ru-central1" + s3-endpoint: "https://storage.yandexcloud.net" + bucket: "no-copy-storage" + searchapi: api-key: ${SEARCHAPI_API_KEY:5jyYZC8jSaxhZTwjMUhwtAXi}