backdev/no-copy
1
0
Fork 0
Code Issues Pull Requests 5 Actions Packages Projects Releases Wiki Activity

NCBACK-35 add worked logic for save protected files
Test Workflow / test (push) Successful in 4s
Details

Browse Source
This commit is contained in:
vladp
2026-02-16 15:04:58 +07:00
parent 2985d84b04
commit a25b33fe13
8 changed files with 113 additions and 208 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/main/java/ru/soune/nocopy/controller/ApiController.java
+2 -29
View File
@@ -74,6 +74,8 @@ public class ApiController {
private final AuthTokenRepository authTokenRepository;
private final CloudStorageService cloudStorageService;
@PostMapping("/v{version}/data")
public ResponseEntity<?> handlePostRequest(@RequestBody BaseRequest request,
@PathVariable("version") int version) {
@@ -292,8 +294,6 @@ public class ApiController {
}
}
private final CloudStorageService cloudStorageService;
@GetMapping("/v{version}/files/download/{fileId}")
public ResponseEntity<?> downloadFile(
@PathVariable(required = false) String fileId,
@@ -441,33 +441,6 @@ public class ApiController {
return ResponseEntity.ok().body(fileTypeStats);
}
private boolean hasDuplicate(List<SimilarFileDTO> similarFiles) {
return similarFiles.stream().anyMatch(f -> f.getHammingDistance() <= 5);
}
private ResponseEntity<BaseResponse> handleDuplicate(FileEntity fileEntity, List<SimilarFileDTO> similarFiles)
throws IOException {
fileEntityService.deleteFromDisk(fileEntity);
fileEntityService.softDeleteFileWithHash(fileEntity);
Optional<FileEntity> originalFile = fileEntityRepository.findById(similarFiles.get(0).getFileId());
if (originalFile.isPresent()) {
Map<String, String> duplicateInfo = Map.of(
"duplicate_file_id", originalFile.get().getId(),
"owner_user_id", String.valueOf(originalFile.get().getUserId()));
return ResponseEntity.ok().body(new BaseResponse(
20004,
MessageCode.DUPLICATE_FILE_UPLOAD.getCode(),
"Failed to upload chunk, duplicate",
duplicateInfo));
}
return null;
}
private ResponseEntity<BaseResponse> buildSuccessResponse(String uploadId, Integer chunkNumber, MultipartFile chunk,
String fileId) {
ChunkUploadResponse responseBody = ChunkUploadResponse.builder()
src/main/java/ru/soune/nocopy/controller/PhotoController.java
-86
View File
@@ -1,86 +0,0 @@
package ru.soune.nocopy.controller;
import io.swagger.v3.oas.annotations.responses.ApiResponse;
import io.swagger.v3.oas.annotations.responses.ApiResponses;
import io.swagger.v3.oas.annotations.tags.Tag;
import org.springframework.http.HttpHeaders;
import org.springframework.http.ResponseEntity;
import org.springframework.web.bind.annotation.*;
import org.springframework.web.multipart.MultipartFile;
import software.amazon.awssdk.auth.credentials.AwsBasicCredentials;
import software.amazon.awssdk.auth.credentials.AwsCredentials;
import software.amazon.awssdk.auth.credentials.StaticCredentialsProvider;
import software.amazon.awssdk.core.sync.RequestBody;
import software.amazon.awssdk.http.apache.ApacheHttpClient;
import software.amazon.awssdk.regions.Region;
import software.amazon.awssdk.services.s3.S3Client;
import software.amazon.awssdk.services.s3.model.GetObjectRequest;
import software.amazon.awssdk.services.s3.model.PutObjectRequest;
import java.io.IOException;
import java.net.URI;
import static org.springframework.http.MediaType.MULTIPART_FORM_DATA_VALUE;
@RestController
@RequestMapping("/photos")
@Tag(name = "Photos")
@ApiResponses(@ApiResponse(responseCode = "200", useReturnTypeSchema = true))
public class PhotoController {
private static final String KEY_ID = "YCAJEmHZcWxzf4oGWvETG3mms";
private static final String SECRET_KEY = "YCPAPZ32XC-LWb8xE0b_xTcBD8NZUCHpSOfu8LUG";
private static final String REGION = "ru-central1";
private static final String S3_ENDPOINT = "https://storage.yandexcloud.net";
private static final String BUCKET = "ncp-test-storage";
private final S3Client s3Client;
public PhotoController() {
AwsCredentials credentials = AwsBasicCredentials.create(KEY_ID, SECRET_KEY);
s3Client = S3Client.builder()
.httpClient(ApacheHttpClient.create())
.region(Region.of(REGION))
.endpointOverride(URI.create(S3_ENDPOINT))
.credentialsProvider(StaticCredentialsProvider.create(credentials))
.build();
}
@PutMapping(consumes = MULTIPART_FORM_DATA_VALUE)
public String uploadFile(@RequestParam MultipartFile photo) throws IOException {
String key = "photos/" + photo.getOriginalFilename();
PutObjectRequest putObjectRequest = PutObjectRequest.builder()
.bucket(BUCKET)
.key(key)
.contentType(photo.getContentType())
.build();
s3Client.putObject(putObjectRequest, RequestBody.fromBytes(photo.getBytes()));
return key;
}
@GetMapping
public ResponseEntity<byte[]> downloadFile(@RequestParam String key) throws IOException {
GetObjectRequest objectRequest = GetObjectRequest.builder()
.bucket(BUCKET)
.key(key)
.build();
var inputStream = s3Client.getObject(objectRequest);
byte[] data = inputStream.readAllBytes();
var headers = new HttpHeaders();
headers.add(HttpHeaders.CONTENT_DISPOSITION, "inline");
headers.add(HttpHeaders.CONTENT_TYPE, inputStream.response().contentType());
return ResponseEntity.ok()
.headers(headers)
.body(data);
}
}
src/main/java/ru/soune/nocopy/handler/FileEntityHandler.java
+1 -1
View File
@@ -302,7 +302,7 @@ public class FileEntityHandler implements RequestHandler {
int fullDelete = fileRequest.getFullDelete() == null ? 0 : fileRequest.getFullDelete();
if (fileEntity.getStatus().equals(FileStatus.DELETED) || fullDelete == 1) {
fileEntityService.deleteFromDisk(fileEntity);
fileEntityService.deleteFromStorageDisk(fileEntity);
response = DeleteFileResponse.builder()
.fileId(fileRequest.getFileId())
.message("File deleted from disk")
src/main/java/ru/soune/nocopy/service/file/FileCleanupService.java
+51
View File
@@ -8,6 +8,7 @@ import lombok.extern.slf4j.Slf4j;
import ru.soune.nocopy.entity.user.ProtectedFileCheck;
import ru.soune.nocopy.repository.ProtectedFileCheckRepository;
import java.io.File;
import java.io.IOException;
import java.nio.file.*;
import java.nio.file.attribute.BasicFileAttributes;
@@ -15,7 +16,10 @@ import java.time.Instant;
import java.time.LocalDate;
import java.time.LocalDateTime;
import java.time.ZoneId;
import java.time.temporal.ChronoUnit;
import java.util.List;
import java.util.stream.Collectors;
import java.util.stream.Stream;
@Slf4j
@Component
@@ -35,6 +39,7 @@ public class FileCleanupService {
log.info("Starting cleanup of expired temporary files");
cleanupOldPeriods();
cleanupTempFilesNightly();
Path tempDir = Paths.get(basePath, "temp");
if (!Files.exists(tempDir)) {
@@ -98,4 +103,50 @@ public class FileCleanupService {
protectedFileCheckRepository.saveAll(checks);
log.info("Cleaned up {} old protection usage records", checks.size());
}
private void cleanupTempFilesNightly() {
log.info("Starting nightly cleanup of temporary files");
String tempDir = System.getProperty("java.io.tmpdir");
Path tempPath = Paths.get(tempDir);
if (!Files.exists(tempPath)) {
return;
}
Instant cutoffTime = Instant.now().minus(24, ChronoUnit.HOURS); // 24 часа
int deletedCount = 0;
try (Stream<Path> files = Files.list(tempPath)) {
List<Path> tempFiles = files
.filter(path -> {
String filename = path.getFileName().toString();
return filename.contains("_") &&
!filename.endsWith(".tmp") &&
Files.isRegularFile(path);
})
.collect(Collectors.toList());
for (Path filePath : tempFiles) {
try {
File file = filePath.toFile();
Instant lastModified = Instant.ofEpochMilli(file.lastModified());
if (lastModified.isBefore(cutoffTime)) {
boolean deleted = Files.deleteIfExists(filePath);
if (deleted) {
deletedCount++;
}
}
} catch (IOException e) {
log.error("Error deleting file: {}", filePath, e);
}
}
log.info("Nightly cleanup deleted {} temporary files", deletedCount);
} catch (IOException e) {
log.error("Failed to list temp directory", e);
}
}
}
src/main/java/ru/soune/nocopy/service/file/FileEntityService.java
+21 -87
View File
@@ -18,8 +18,8 @@ import ru.soune.nocopy.repository.SimilarImageProjection;
import ru.soune.nocopy.repository.UserRepository;
import ru.soune.nocopy.service.FileSimilarityService;
import ru.soune.nocopy.service.ImageHashService;
import ru.soune.nocopy.service.file.cloud.CloudStorageService;
import java.io.File;
import java.io.IOException;
import java.nio.file.Files;
import java.nio.file.Path;
@@ -43,6 +43,8 @@ public class FileEntityService {
private final UserRepository userRepository;
private final CloudStorageService cloudStorageService;
@Transactional(noRollbackFor = DuplicateImageException.class)
public FileEntity createFromUploadSession(FileUploadSession session, String checksum) {
log.info("Creating FileEntity for upload session: {}", session.getUploadId());
@@ -56,7 +58,7 @@ public class FileEntityService {
Map<String, Long> imageHash = Map.of();
if (session.getFileType().startsWith("image")) {
if (!session.getFileType().startsWith("video")) {
imageHash = imageHashService.calculateHash(filePath);
List<SimilarImageProjection> duplicatedByHash = fileSimilarityService.findDuplicatedByHash(
imageHash.get("hi"), imageHash.get("low"));
@@ -117,38 +119,6 @@ public class FileEntityService {
return convertToResponse(fileEntity, version);
}
@Transactional(readOnly = true)
public FileEntityResponse getByFilePath(String filePath, int version) {
FileEntity fileEntity = fileEntityRepository.findByFilePath(filePath)
.orElseThrow(() -> new FileEntityNotFoundException("Path: " + filePath));
return convertToResponse(fileEntity, version);
}
@Transactional(readOnly = true)
public FileResponse getAllUserFiles(Long userId, int version) {
List<FileEntity> fileEntities = fileEntityRepository.findByUserIdAndStatus(
userId, FileStatus.ACTIVE);
List<FileEntityResponse> files = fileEntities.stream()
.map(file -> convertToResponse(file, version))
.collect(Collectors.toList());
long totalSize = fileEntities.stream()
.mapToLong(FileEntity::getFileSize)
.sum();
return FileResponse.builder()
.files(files)
.totalCount(files.size())
.totalSize(totalSize)
.formattedTotalSize(formatFileSize(totalSize))
.page(1)
.pageSize(files.size())
.build();
}
@Transactional(readOnly = true)
public FileResponse getUserFiles(Long userId, int page, int pageSize, int version) {
User user = userRepository.findById(userId).orElseThrow();
@@ -200,19 +170,6 @@ public class FileEntityService {
.build();
}
@Transactional
public FileEntity markAsDeleted(FileEntity fileEntity) throws IOException {
fileEntity.setStatus(FileStatus.DELETED);
fileEntity.setUpdatedAt(LocalDateTime.now());
fileEntity.setProtectionStatus(ProtectionStatus.NOT_PROTECTED);
Path path = Paths.get(fileEntity.getProtectedFilePath());
Files.deleteIfExists(path);
fileEntity.setProtectedFilePath("");
return fileEntityRepository.save(fileEntity);
}
public void softDeleteFileWithHash(FileEntity fileEntity) throws IOException {
if (fileEntity.getImageHash() != null) {
fileEntity.setImageHash(null);
@@ -229,14 +186,9 @@ public class FileEntityService {
fileEntityRepository.save(fileEntity);
}
public void deleteFromDisk(FileEntity fileEntity) throws IOException {
Path path = Paths.get(fileEntity.getFilePath());
if (!Files.exists(path)) {
return;
}
Files.delete(path);
public void deleteFromStorageDisk(FileEntity fileEntity) {
cloudStorageService.deleteFileFromStorage(fileEntity.getFilePath());
cloudStorageService.deleteFileFromStorage(fileEntity.getProtectedFilePath());
fileEntityRepository.delete(fileEntity);
}
@@ -282,11 +234,11 @@ public class FileEntityService {
return fileEntityRepository.save(fileEntity);
}
public void clearTempFiles(long userId) throws IOException {
public void clearTempFiles(long userId) {
List<FileEntity> fileByUserIdAndStatus = fileEntityRepository.findFileByUserIdAndStatus(userId, FileStatus.TEMP);
for (FileEntity fileEntity : fileByUserIdAndStatus) {
deleteFromDisk(fileEntity);
deleteFromStorageDisk(fileEntity);
}
}
@@ -298,24 +250,6 @@ public class FileEntityService {
return fileEntityRepository.findFileIdByFilePath(filePath);
}
public File getFileById(String id) {
try {
FileEntity fileEntity = fileEntityRepository.findById(id).orElseThrow(() ->
new RuntimeException("File not found: " + id));
File file = new File(fileEntity.getFilePath());
if (!file.exists()) {
throw new RuntimeException("File not found on disk: " + fileEntity.getFilePath());
}
return file;
} catch (Exception e) {
log.error("Error getting file: {}", id, e);
return null;
}
}
public Path prepareProtectedPath(FileEntity fileEntity, String extension) throws IOException {
Path originalPath = Paths.get(fileEntity.getFilePath());
@@ -352,6 +286,18 @@ public class FileEntityService {
fileEntityRepository.save(fileEntity);
}
public String formatFileSize(long size) {
if (size < 1024) {
return size + " B";
} else if (size < 1024 * 1024) {
return String.format("%.1f KB", size / 1024.0);
} else if (size < 1024 * 1024 * 1024) {
return String.format("%.1f MB", size / (1024.0 * 1024.0));
} else {
return String.format("%.1f GB", size / (1024.0 * 1024.0 * 1024.0));
}
}
private boolean checkFileExistsOnDisk(String filePath) {
try {
return Files.exists(Paths.get(filePath));
@@ -404,16 +350,4 @@ public class FileEntityService {
.protectedFilePath(fileEntity.getProtectedFilePath())
.build();
}
public String formatFileSize(long size) {
if (size < 1024) {
return size + " B";
} else if (size < 1024 * 1024) {
return String.format("%.1f KB", size / 1024.0);
} else if (size < 1024 * 1024 * 1024) {
return String.format("%.1f MB", size / (1024.0 * 1024.0));
} else {
return String.format("%.1f GB", size / (1024.0 * 1024.0 * 1024.0));
}
}
}
src/main/java/ru/soune/nocopy/service/file/cloud/CloudStorageService.java
+37 -2
View File
@@ -55,7 +55,7 @@ public class CloudStorageService {
String mimeTypeFromFile = mimeType.equals("document") ? "application" : mimeType;
String contentType = mimeTypeFromFile + "/" + extension;
String filePath = "files/" + sendToCloudFilePath;
String filePath = "files" + sendToCloudFilePath;
PutObjectRequest putObjectRequest = PutObjectRequest.builder()
.bucket(bucket)
@@ -75,7 +75,7 @@ public class CloudStorageService {
public File readFileFromStorageByPath(String filePath) throws IOException {
GetObjectRequest objectRequest = GetObjectRequest.builder()
.bucket(bucket)
.key("files/" + filePath)
.key("files" + filePath)
.build();
AwsCredentials credentials = AwsBasicCredentials.create(key, secretKey);
@@ -94,4 +94,39 @@ public class CloudStorageService {
return tempFile;
}
public void deleteFileFromStorage(String filePath) {
if (filePath == null || filePath.isEmpty()) {
log.warn("Attempted to delete file with empty path");
return;
}
AwsCredentials credentials = AwsBasicCredentials.create(key, secretKey);
S3Client s3Client = S3Client.builder()
.httpClient(ApacheHttpClient.create())
.region(Region.of(region))
.endpointOverride(URI.create(s3endpoint))
.credentialsProvider(StaticCredentialsProvider.create(credentials))
.build();
String cloudKey = "files" + filePath;
try {
try {
s3Client.headObject(builder -> builder.bucket(bucket).key(cloudKey).build());
} catch (Exception e) {
log.warn("File does not exist in cloud storage: {}", cloudKey);
return;
}
s3Client.deleteObject(builder -> builder.bucket(bucket).key(cloudKey).build());
log.info("File successfully deleted from cloud storage: {}", cloudKey);
} catch (Exception e) {
log.error("Failed to delete file from cloud storage: {}", cloudKey, e);
throw new RuntimeException("Failed to delete file from cloud storage: " + cloudKey, e);
}
}
}
src/main/java/ru/soune/nocopy/service/file/impl/FileUploadServiceImpl.java
-2
View File
@@ -181,7 +181,6 @@ public class FileUploadServiceImpl implements FileUploadService {
}
@Override
@Transactional
public UploadProgressResponse uploadChunk(String uploadId, Integer chunkNumber,
MultipartFile chunkFile, Integer findSimilar) {
FileUploadSession session = sessionRepository.findById(uploadId)
@@ -336,7 +335,6 @@ public class FileUploadServiceImpl implements FileUploadService {
FileEntity fileEntity = completeFileProcessingAsync(session, status);
//TODO CHECK FOR PROTECTED
fileEntityRepository.flush();
entityManager.clear();
src/main/resources/application.yaml
+1 -1
View File
@@ -105,7 +105,7 @@ yandex:
secret-key: "YCPAPZ32XC-LWb8xE0b_xTcBD8NZUCHpSOfu8LUG"
region: "ru-central1"
s3-endpoint: "https://storage.yandexcloud.net"
bucket: "ncp-test-storage"
bucket: "no-copy-storage"
searchapi: