diff --git a/src/main/java/ru/soune/nocopy/controller/ApiController.java b/src/main/java/ru/soune/nocopy/controller/ApiController.java index d267848..535302b 100644 --- a/src/main/java/ru/soune/nocopy/controller/ApiController.java +++ b/src/main/java/ru/soune/nocopy/controller/ApiController.java @@ -74,6 +74,8 @@ public class ApiController { private final AuthTokenRepository authTokenRepository; + private final CloudStorageService cloudStorageService; + @PostMapping("/v{version}/data") public ResponseEntity handlePostRequest(@RequestBody BaseRequest request, @PathVariable("version") int version) { @@ -292,8 +294,6 @@ public class ApiController { } } - private final CloudStorageService cloudStorageService; - @GetMapping("/v{version}/files/download/{fileId}") public ResponseEntity downloadFile( @PathVariable(required = false) String fileId, @@ -441,33 +441,6 @@ public class ApiController { return ResponseEntity.ok().body(fileTypeStats); } - private boolean hasDuplicate(List similarFiles) { - return similarFiles.stream().anyMatch(f -> f.getHammingDistance() <= 5); - } - - private ResponseEntity handleDuplicate(FileEntity fileEntity, List similarFiles) - throws IOException { - fileEntityService.deleteFromDisk(fileEntity); - - fileEntityService.softDeleteFileWithHash(fileEntity); - - Optional originalFile = fileEntityRepository.findById(similarFiles.get(0).getFileId()); - - if (originalFile.isPresent()) { - Map duplicateInfo = Map.of( - "duplicate_file_id", originalFile.get().getId(), - "owner_user_id", String.valueOf(originalFile.get().getUserId())); - - return ResponseEntity.ok().body(new BaseResponse( - 20004, - MessageCode.DUPLICATE_FILE_UPLOAD.getCode(), - "Failed to upload chunk, duplicate", - duplicateInfo)); - } - - return null; - } - private ResponseEntity buildSuccessResponse(String uploadId, Integer chunkNumber, MultipartFile chunk, String fileId) { ChunkUploadResponse responseBody = ChunkUploadResponse.builder() diff --git a/src/main/java/ru/soune/nocopy/controller/PhotoController.java b/src/main/java/ru/soune/nocopy/controller/PhotoController.java deleted file mode 100644 index 7a3b856..0000000 --- a/src/main/java/ru/soune/nocopy/controller/PhotoController.java +++ /dev/null @@ -1,86 +0,0 @@ -package ru.soune.nocopy.controller; - -import io.swagger.v3.oas.annotations.responses.ApiResponse; -import io.swagger.v3.oas.annotations.responses.ApiResponses; -import io.swagger.v3.oas.annotations.tags.Tag; -import org.springframework.http.HttpHeaders; -import org.springframework.http.ResponseEntity; -import org.springframework.web.bind.annotation.*; -import org.springframework.web.multipart.MultipartFile; -import software.amazon.awssdk.auth.credentials.AwsBasicCredentials; -import software.amazon.awssdk.auth.credentials.AwsCredentials; -import software.amazon.awssdk.auth.credentials.StaticCredentialsProvider; -import software.amazon.awssdk.core.sync.RequestBody; -import software.amazon.awssdk.http.apache.ApacheHttpClient; -import software.amazon.awssdk.regions.Region; -import software.amazon.awssdk.services.s3.S3Client; -import software.amazon.awssdk.services.s3.model.GetObjectRequest; -import software.amazon.awssdk.services.s3.model.PutObjectRequest; - -import java.io.IOException; -import java.net.URI; - -import static org.springframework.http.MediaType.MULTIPART_FORM_DATA_VALUE; - -@RestController -@RequestMapping("/photos") -@Tag(name = "Photos") -@ApiResponses(@ApiResponse(responseCode = "200", useReturnTypeSchema = true)) -public class PhotoController { - - private static final String KEY_ID = "YCAJEmHZcWxzf4oGWvETG3mms"; - private static final String SECRET_KEY = "YCPAPZ32XC-LWb8xE0b_xTcBD8NZUCHpSOfu8LUG"; - private static final String REGION = "ru-central1"; - private static final String S3_ENDPOINT = "https://storage.yandexcloud.net"; - - private static final String BUCKET = "ncp-test-storage"; - - private final S3Client s3Client; - - public PhotoController() { - AwsCredentials credentials = AwsBasicCredentials.create(KEY_ID, SECRET_KEY); - - s3Client = S3Client.builder() - .httpClient(ApacheHttpClient.create()) - .region(Region.of(REGION)) - .endpointOverride(URI.create(S3_ENDPOINT)) - .credentialsProvider(StaticCredentialsProvider.create(credentials)) - .build(); - } - - - @PutMapping(consumes = MULTIPART_FORM_DATA_VALUE) - public String uploadFile(@RequestParam MultipartFile photo) throws IOException { - - String key = "photos/" + photo.getOriginalFilename(); - PutObjectRequest putObjectRequest = PutObjectRequest.builder() - .bucket(BUCKET) - .key(key) - .contentType(photo.getContentType()) - .build(); - - s3Client.putObject(putObjectRequest, RequestBody.fromBytes(photo.getBytes())); - - return key; - } - - @GetMapping - public ResponseEntity downloadFile(@RequestParam String key) throws IOException { - GetObjectRequest objectRequest = GetObjectRequest.builder() - .bucket(BUCKET) - .key(key) - .build(); - - var inputStream = s3Client.getObject(objectRequest); - byte[] data = inputStream.readAllBytes(); - - var headers = new HttpHeaders(); - headers.add(HttpHeaders.CONTENT_DISPOSITION, "inline"); - headers.add(HttpHeaders.CONTENT_TYPE, inputStream.response().contentType()); - - return ResponseEntity.ok() - .headers(headers) - .body(data); - } -} - diff --git a/src/main/java/ru/soune/nocopy/handler/FileEntityHandler.java b/src/main/java/ru/soune/nocopy/handler/FileEntityHandler.java index 3a1f56e..4d9c867 100644 --- a/src/main/java/ru/soune/nocopy/handler/FileEntityHandler.java +++ b/src/main/java/ru/soune/nocopy/handler/FileEntityHandler.java @@ -302,7 +302,7 @@ public class FileEntityHandler implements RequestHandler { int fullDelete = fileRequest.getFullDelete() == null ? 0 : fileRequest.getFullDelete(); if (fileEntity.getStatus().equals(FileStatus.DELETED) || fullDelete == 1) { - fileEntityService.deleteFromDisk(fileEntity); + fileEntityService.deleteFromStorageDisk(fileEntity); response = DeleteFileResponse.builder() .fileId(fileRequest.getFileId()) .message("File deleted from disk") diff --git a/src/main/java/ru/soune/nocopy/service/file/FileCleanupService.java b/src/main/java/ru/soune/nocopy/service/file/FileCleanupService.java index 06e91eb..3ed37c7 100644 --- a/src/main/java/ru/soune/nocopy/service/file/FileCleanupService.java +++ b/src/main/java/ru/soune/nocopy/service/file/FileCleanupService.java @@ -8,6 +8,7 @@ import lombok.extern.slf4j.Slf4j; import ru.soune.nocopy.entity.user.ProtectedFileCheck; import ru.soune.nocopy.repository.ProtectedFileCheckRepository; +import java.io.File; import java.io.IOException; import java.nio.file.*; import java.nio.file.attribute.BasicFileAttributes; @@ -15,7 +16,10 @@ import java.time.Instant; import java.time.LocalDate; import java.time.LocalDateTime; import java.time.ZoneId; +import java.time.temporal.ChronoUnit; import java.util.List; +import java.util.stream.Collectors; +import java.util.stream.Stream; @Slf4j @Component @@ -35,6 +39,7 @@ public class FileCleanupService { log.info("Starting cleanup of expired temporary files"); cleanupOldPeriods(); + cleanupTempFilesNightly(); Path tempDir = Paths.get(basePath, "temp"); if (!Files.exists(tempDir)) { @@ -98,4 +103,50 @@ public class FileCleanupService { protectedFileCheckRepository.saveAll(checks); log.info("Cleaned up {} old protection usage records", checks.size()); } + + private void cleanupTempFilesNightly() { + log.info("Starting nightly cleanup of temporary files"); + + String tempDir = System.getProperty("java.io.tmpdir"); + Path tempPath = Paths.get(tempDir); + + if (!Files.exists(tempPath)) { + return; + } + + Instant cutoffTime = Instant.now().minus(24, ChronoUnit.HOURS); // 24 часа + int deletedCount = 0; + + try (Stream files = Files.list(tempPath)) { + List tempFiles = files + .filter(path -> { + String filename = path.getFileName().toString(); + return filename.contains("_") && + !filename.endsWith(".tmp") && + Files.isRegularFile(path); + }) + .collect(Collectors.toList()); + + for (Path filePath : tempFiles) { + try { + File file = filePath.toFile(); + Instant lastModified = Instant.ofEpochMilli(file.lastModified()); + + if (lastModified.isBefore(cutoffTime)) { + boolean deleted = Files.deleteIfExists(filePath); + if (deleted) { + deletedCount++; + } + } + } catch (IOException e) { + log.error("Error deleting file: {}", filePath, e); + } + } + + log.info("Nightly cleanup deleted {} temporary files", deletedCount); + + } catch (IOException e) { + log.error("Failed to list temp directory", e); + } + } } diff --git a/src/main/java/ru/soune/nocopy/service/file/FileEntityService.java b/src/main/java/ru/soune/nocopy/service/file/FileEntityService.java index 33267b4..7a6a4cb 100644 --- a/src/main/java/ru/soune/nocopy/service/file/FileEntityService.java +++ b/src/main/java/ru/soune/nocopy/service/file/FileEntityService.java @@ -18,8 +18,8 @@ import ru.soune.nocopy.repository.SimilarImageProjection; import ru.soune.nocopy.repository.UserRepository; import ru.soune.nocopy.service.FileSimilarityService; import ru.soune.nocopy.service.ImageHashService; +import ru.soune.nocopy.service.file.cloud.CloudStorageService; -import java.io.File; import java.io.IOException; import java.nio.file.Files; import java.nio.file.Path; @@ -43,6 +43,8 @@ public class FileEntityService { private final UserRepository userRepository; + private final CloudStorageService cloudStorageService; + @Transactional(noRollbackFor = DuplicateImageException.class) public FileEntity createFromUploadSession(FileUploadSession session, String checksum) { log.info("Creating FileEntity for upload session: {}", session.getUploadId()); @@ -56,7 +58,7 @@ public class FileEntityService { Map imageHash = Map.of(); - if (session.getFileType().startsWith("image")) { + if (!session.getFileType().startsWith("video")) { imageHash = imageHashService.calculateHash(filePath); List duplicatedByHash = fileSimilarityService.findDuplicatedByHash( imageHash.get("hi"), imageHash.get("low")); @@ -117,38 +119,6 @@ public class FileEntityService { return convertToResponse(fileEntity, version); } - @Transactional(readOnly = true) - public FileEntityResponse getByFilePath(String filePath, int version) { - FileEntity fileEntity = fileEntityRepository.findByFilePath(filePath) - .orElseThrow(() -> new FileEntityNotFoundException("Path: " + filePath)); - - return convertToResponse(fileEntity, version); - } - - - @Transactional(readOnly = true) - public FileResponse getAllUserFiles(Long userId, int version) { - List fileEntities = fileEntityRepository.findByUserIdAndStatus( - userId, FileStatus.ACTIVE); - - List files = fileEntities.stream() - .map(file -> convertToResponse(file, version)) - .collect(Collectors.toList()); - - long totalSize = fileEntities.stream() - .mapToLong(FileEntity::getFileSize) - .sum(); - - return FileResponse.builder() - .files(files) - .totalCount(files.size()) - .totalSize(totalSize) - .formattedTotalSize(formatFileSize(totalSize)) - .page(1) - .pageSize(files.size()) - .build(); - } - @Transactional(readOnly = true) public FileResponse getUserFiles(Long userId, int page, int pageSize, int version) { User user = userRepository.findById(userId).orElseThrow(); @@ -200,19 +170,6 @@ public class FileEntityService { .build(); } - @Transactional - public FileEntity markAsDeleted(FileEntity fileEntity) throws IOException { - fileEntity.setStatus(FileStatus.DELETED); - fileEntity.setUpdatedAt(LocalDateTime.now()); - fileEntity.setProtectionStatus(ProtectionStatus.NOT_PROTECTED); - - Path path = Paths.get(fileEntity.getProtectedFilePath()); - Files.deleteIfExists(path); - fileEntity.setProtectedFilePath(""); - - return fileEntityRepository.save(fileEntity); - } - public void softDeleteFileWithHash(FileEntity fileEntity) throws IOException { if (fileEntity.getImageHash() != null) { fileEntity.setImageHash(null); @@ -229,14 +186,9 @@ public class FileEntityService { fileEntityRepository.save(fileEntity); } - public void deleteFromDisk(FileEntity fileEntity) throws IOException { - Path path = Paths.get(fileEntity.getFilePath()); - - if (!Files.exists(path)) { - return; - } - - Files.delete(path); + public void deleteFromStorageDisk(FileEntity fileEntity) { + cloudStorageService.deleteFileFromStorage(fileEntity.getFilePath()); + cloudStorageService.deleteFileFromStorage(fileEntity.getProtectedFilePath()); fileEntityRepository.delete(fileEntity); } @@ -282,11 +234,11 @@ public class FileEntityService { return fileEntityRepository.save(fileEntity); } - public void clearTempFiles(long userId) throws IOException { + public void clearTempFiles(long userId) { List fileByUserIdAndStatus = fileEntityRepository.findFileByUserIdAndStatus(userId, FileStatus.TEMP); for (FileEntity fileEntity : fileByUserIdAndStatus) { - deleteFromDisk(fileEntity); + deleteFromStorageDisk(fileEntity); } } @@ -298,24 +250,6 @@ public class FileEntityService { return fileEntityRepository.findFileIdByFilePath(filePath); } - public File getFileById(String id) { - try { - FileEntity fileEntity = fileEntityRepository.findById(id).orElseThrow(() -> - new RuntimeException("File not found: " + id)); - - File file = new File(fileEntity.getFilePath()); - - if (!file.exists()) { - throw new RuntimeException("File not found on disk: " + fileEntity.getFilePath()); - } - - return file; - } catch (Exception e) { - log.error("Error getting file: {}", id, e); - return null; - } - } - public Path prepareProtectedPath(FileEntity fileEntity, String extension) throws IOException { Path originalPath = Paths.get(fileEntity.getFilePath()); @@ -352,6 +286,18 @@ public class FileEntityService { fileEntityRepository.save(fileEntity); } + public String formatFileSize(long size) { + if (size < 1024) { + return size + " B"; + } else if (size < 1024 * 1024) { + return String.format("%.1f KB", size / 1024.0); + } else if (size < 1024 * 1024 * 1024) { + return String.format("%.1f MB", size / (1024.0 * 1024.0)); + } else { + return String.format("%.1f GB", size / (1024.0 * 1024.0 * 1024.0)); + } + } + private boolean checkFileExistsOnDisk(String filePath) { try { return Files.exists(Paths.get(filePath)); @@ -404,16 +350,4 @@ public class FileEntityService { .protectedFilePath(fileEntity.getProtectedFilePath()) .build(); } - - public String formatFileSize(long size) { - if (size < 1024) { - return size + " B"; - } else if (size < 1024 * 1024) { - return String.format("%.1f KB", size / 1024.0); - } else if (size < 1024 * 1024 * 1024) { - return String.format("%.1f MB", size / (1024.0 * 1024.0)); - } else { - return String.format("%.1f GB", size / (1024.0 * 1024.0 * 1024.0)); - } - } } \ No newline at end of file diff --git a/src/main/java/ru/soune/nocopy/service/file/cloud/CloudStorageService.java b/src/main/java/ru/soune/nocopy/service/file/cloud/CloudStorageService.java index 8b848d5..bbe90d0 100644 --- a/src/main/java/ru/soune/nocopy/service/file/cloud/CloudStorageService.java +++ b/src/main/java/ru/soune/nocopy/service/file/cloud/CloudStorageService.java @@ -55,7 +55,7 @@ public class CloudStorageService { String mimeTypeFromFile = mimeType.equals("document") ? "application" : mimeType; String contentType = mimeTypeFromFile + "/" + extension; - String filePath = "files/" + sendToCloudFilePath; + String filePath = "files" + sendToCloudFilePath; PutObjectRequest putObjectRequest = PutObjectRequest.builder() .bucket(bucket) @@ -75,7 +75,7 @@ public class CloudStorageService { public File readFileFromStorageByPath(String filePath) throws IOException { GetObjectRequest objectRequest = GetObjectRequest.builder() .bucket(bucket) - .key("files/" + filePath) + .key("files" + filePath) .build(); AwsCredentials credentials = AwsBasicCredentials.create(key, secretKey); @@ -94,4 +94,39 @@ public class CloudStorageService { return tempFile; } + + public void deleteFileFromStorage(String filePath) { + if (filePath == null || filePath.isEmpty()) { + log.warn("Attempted to delete file with empty path"); + return; + } + + AwsCredentials credentials = AwsBasicCredentials.create(key, secretKey); + + S3Client s3Client = S3Client.builder() + .httpClient(ApacheHttpClient.create()) + .region(Region.of(region)) + .endpointOverride(URI.create(s3endpoint)) + .credentialsProvider(StaticCredentialsProvider.create(credentials)) + .build(); + + String cloudKey = "files" + filePath; + + try { + try { + s3Client.headObject(builder -> builder.bucket(bucket).key(cloudKey).build()); + } catch (Exception e) { + log.warn("File does not exist in cloud storage: {}", cloudKey); + return; + } + + s3Client.deleteObject(builder -> builder.bucket(bucket).key(cloudKey).build()); + + log.info("File successfully deleted from cloud storage: {}", cloudKey); + + } catch (Exception e) { + log.error("Failed to delete file from cloud storage: {}", cloudKey, e); + throw new RuntimeException("Failed to delete file from cloud storage: " + cloudKey, e); + } + } } diff --git a/src/main/java/ru/soune/nocopy/service/file/impl/FileUploadServiceImpl.java b/src/main/java/ru/soune/nocopy/service/file/impl/FileUploadServiceImpl.java index 0f1b889..c1e3948 100644 --- a/src/main/java/ru/soune/nocopy/service/file/impl/FileUploadServiceImpl.java +++ b/src/main/java/ru/soune/nocopy/service/file/impl/FileUploadServiceImpl.java @@ -181,7 +181,6 @@ public class FileUploadServiceImpl implements FileUploadService { } @Override - @Transactional public UploadProgressResponse uploadChunk(String uploadId, Integer chunkNumber, MultipartFile chunkFile, Integer findSimilar) { FileUploadSession session = sessionRepository.findById(uploadId) @@ -336,7 +335,6 @@ public class FileUploadServiceImpl implements FileUploadService { FileEntity fileEntity = completeFileProcessingAsync(session, status); - //TODO CHECK FOR PROTECTED fileEntityRepository.flush(); entityManager.clear(); diff --git a/src/main/resources/application.yaml b/src/main/resources/application.yaml index 88623a9..7f8823c 100644 --- a/src/main/resources/application.yaml +++ b/src/main/resources/application.yaml @@ -105,7 +105,7 @@ yandex: secret-key: "YCPAPZ32XC-LWb8xE0b_xTcBD8NZUCHpSOfu8LUG" region: "ru-central1" s3-endpoint: "https://storage.yandexcloud.net" - bucket: "ncp-test-storage" + bucket: "no-copy-storage" searchapi: