backdev/no-copy
1
0
Fork 0
Code Issues Pull Requests 5 Actions Packages Projects Releases Wiki Activity

Merge branch 'NCBACK-35' into dev

Browse Source
This commit is contained in:
backdev
2026-04-22 16:03:28 +07:00
parent eb223c2ecb 4dd6c4a3c0
commit cbcd16d376
18 changed files with 589 additions and 212 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/main/java/ru/soune/nocopy/service/FileSimilarityService.java
+21 -13
View File
@@ -17,6 +17,7 @@ import ru.soune.nocopy.entity.file.ImageHashEntity;
import ru.soune.nocopy.entity.user.User;
import ru.soune.nocopy.exception.DuplicateImageException;
import ru.soune.nocopy.repository.*;
import ru.soune.nocopy.service.file.cloud.CloudStorageService;
import ru.soune.nocopy.util.FileUtil;
import java.io.File;
@@ -41,6 +42,8 @@ public class FileSimilarityService {
private final FileEntityRepository fileEntityRepository;
private final CloudStorageService cloudStorageService;
@Value("${server.baseurl}")
private String baseUrl;
@@ -114,7 +117,7 @@ public class FileSimilarityService {
}
public FileEntity findDuplicateByHash(String path, String mimeType, Long userId) throws Exception {
String hash = calculateFileHash(path);
String hash = calculateFileHash(path, false);
User user = userRepository.findById(userId).orElseThrow();
Company company = user.getCompany();
@@ -129,15 +132,15 @@ public class FileSimilarityService {
List<FileEntity> fileEntityList = fileEntityRepository.findByMimeTypeAndUserIds(mimeType, userIds);
for (FileEntity file : fileEntityList) {
if (file.getProtectedFilePath() == null) continue;
File existingFile = new File(file.getProtectedFilePath());
File newFile = new File(path);
// if (file.getProtectedFilePath() == null) continue;
// File existingFile = cloudStorageService.readFileFromStorageByPath(file.getFilePath());
// File newFile = new File(path);
if (existingFile.length() == newFile.length()) {
if (calculateFileHash(file.getProtectedFilePath()).equals(hash)) {
// if (existingFile.length() == newFile.length()) {
if (calculateFileHash(file.getFilePath(), true).equals(hash)) {
return file;
}
}
// }
}
return null;
@@ -196,14 +199,19 @@ public class FileSimilarityService {
}
private String calculateFileHash(String path) throws Exception {
private String calculateFileHash(String path, boolean cloud) throws Exception {
MessageDigest digest = MessageDigest.getInstance("SHA-256");
try (InputStream is = new FileInputStream(path)) {
byte[] buffer = new byte[8192];
int read;
while ((read = is.read(buffer)) > 0) {
digest.update(buffer, 0, read);
if (cloud) {
byte[] data = cloudStorageService.readFileFromStorageBytes(path);
digest.update(data);
} else {
try (InputStream is = new FileInputStream(path)) {
byte[] buffer = new byte[8192];
int read;
while ((read = is.read(buffer)) > 0) {
digest.update(buffer, 0, read);
}
}
}
src/main/java/ru/soune/nocopy/service/ImageHashService.java
+41 -7
View File
@@ -8,9 +8,11 @@ import org.springframework.stereotype.Service;
import ru.soune.nocopy.entity.file.FileEntity;
import ru.soune.nocopy.entity.file.ImageHashEntity;
import ru.soune.nocopy.repository.ImageHashRepository;
import ru.soune.nocopy.service.file.cloud.CloudStorageService;
import java.io.File;
import java.io.IOException;
import java.nio.file.Files;
import java.nio.file.Path;
import java.time.LocalDateTime;
import java.util.Map;
@@ -22,18 +24,50 @@ public class ImageHashService {
private final ImageHashRepository repository;
public Map<String, Long> calculateHash(Path imagePath) throws IOException {
File file = imagePath.toFile();
private final CloudStorageService cloudStorageService;
PHash pHash = PerceptualHashHelper.INSTANCE.generateDCTPerceptualHash(file);
// public Map<String, Long> calculateHash(Path imagePath) throws IOException {
//// File file = cloudStorageService.readFileFromStorageByPath(imagePath.toString());
// File file = imagePath.toFile();
// if (file == null) {
// file = cloudStorageService.readFileFromStorageByPath(imagePath.toString());
// }
//
// PHash pHash = PerceptualHashHelper.INSTANCE.generateDCTPerceptualHash(file);
//
// Long firstPart = pHash.getFirstPart();
// Long secondPart = pHash.getSecondPart();
// Map<String, Long> hash = Map.of(
// "hi", firstPart,
// "low", secondPart);
//
// return hash;
// }
public Map<String, Long> calculateHash(Path imagePath) throws IOException {
PHash pHash;
if (imagePath.toFile().exists()) {
pHash = PerceptualHashHelper.INSTANCE.generateDCTPerceptualHash(imagePath.toFile());
} else {
byte[] data = cloudStorageService.readFileFromStorageBytes(imagePath.toString());
Path tempFile = Files.createTempFile("perceptual_hash_", ".tmp");
try {
Files.write(tempFile, data);
pHash = PerceptualHashHelper.INSTANCE.generateDCTPerceptualHash(tempFile.toFile());
} finally {
Files.deleteIfExists(tempFile);
}
}
Long firstPart = pHash.getFirstPart();
Long secondPart = pHash.getSecondPart();
Map<String, Long> hash = Map.of(
"hi", firstPart,
"low", secondPart);
return hash;
return Map.of(
"hi", firstPart,
"low", secondPart
);
}
public void create(FileEntity file, Map<String, Long> stringIntegerMap) {
src/main/java/ru/soune/nocopy/service/file/FileCleanupService.java
+52
View File
@@ -8,6 +8,7 @@ import lombok.extern.slf4j.Slf4j;
import ru.soune.nocopy.entity.user.ProtectedFileCheck;
import ru.soune.nocopy.repository.ProtectedFileCheckRepository;
import java.io.File;
import java.io.IOException;
import java.nio.file.*;
import java.nio.file.attribute.BasicFileAttributes;
@@ -15,7 +16,10 @@ import java.time.Instant;
import java.time.LocalDate;
import java.time.LocalDateTime;
import java.time.ZoneId;
import java.time.temporal.ChronoUnit;
import java.util.List;
import java.util.stream.Collectors;
import java.util.stream.Stream;
@Slf4j
@Component
@@ -34,6 +38,8 @@ public class FileCleanupService {
public void cleanupExpiredFiles() {
log.info("Starting cleanup of expired temporary files");
cleanupTempFilesNightly();
Path tempDir = Paths.get(basePath, "temp");
if (!Files.exists(tempDir)) {
return;
@@ -83,4 +89,50 @@ public class FileCleanupService {
log.error("Error during cleanup", e);
}
}
private void cleanupTempFilesNightly() {
log.info("Starting nightly cleanup of temporary files");
String tempDir = System.getProperty("java.io.tmpdir");
Path tempPath = Paths.get(tempDir);
if (!Files.exists(tempPath)) {
return;
}
Instant cutoffTime = Instant.now().minus(24, ChronoUnit.HOURS); // 24 часа
int deletedCount = 0;
try (Stream<Path> files = Files.list(tempPath)) {
List<Path> tempFiles = files
.filter(path -> {
String filename = path.getFileName().toString();
return filename.contains("_") &&
!filename.endsWith(".tmp") &&
Files.isRegularFile(path);
})
.collect(Collectors.toList());
for (Path filePath : tempFiles) {
try {
File file = filePath.toFile();
Instant lastModified = Instant.ofEpochMilli(file.lastModified());
if (lastModified.isBefore(cutoffTime)) {
boolean deleted = Files.deleteIfExists(filePath);
if (deleted) {
deletedCount++;
}
}
} catch (IOException e) {
log.error("Error deleting file: {}", filePath, e);
}
}
log.info("Nightly cleanup deleted {} temporary files", deletedCount);
} catch (IOException e) {
log.error("Failed to list temp directory", e);
}
}
}
src/main/java/ru/soune/nocopy/service/file/FileEntityService.java
+47 -59
View File
@@ -19,12 +19,15 @@ import ru.soune.nocopy.exception.FileEntityNotFoundException;
import ru.soune.nocopy.repository.FileEntityRepository;
import ru.soune.nocopy.repository.FileMonitoringRepository;
import ru.soune.nocopy.repository.UserRepository;
import ru.soune.nocopy.service.file.cloud.CloudStorageService;
import java.io.File;
import java.io.IOException;
import java.nio.ByteBuffer;
import java.nio.channels.FileChannel;
import java.nio.file.Files;
import java.nio.file.Path;
import java.nio.file.Paths;
import java.nio.file.StandardOpenOption;
import java.time.LocalDateTime;
import java.util.ArrayList;
import java.util.List;
@@ -42,6 +45,8 @@ public class FileEntityService {
private final ImageResizeService imageResizeService;
private final CloudStorageService cloudStorageService;
@Value("${server.baseurl}")
private String baseUrl;
@@ -202,16 +207,8 @@ public class FileEntityService {
fileEntityRepository.save(fileEntity);
}
public void deleteFromDisk(FileEntity fileEntity) throws IOException {
Path path = Paths.get(fileEntity.getFilePath());
if (!Files.exists(path)) {
return;
}
Files.delete(path);
fileEntityRepository.delete(fileEntity);
public void deleteFromDisk(String deleteFilePath) throws IOException {
Files.deleteIfExists(Paths.get(deleteFilePath));
}
@Transactional(readOnly = true)
@@ -237,18 +234,27 @@ public class FileEntityService {
}
@Transactional
public void writeProtectedFile(String id, byte[] data, String fileExt) throws IOException {
public FileEntity writeProtectedFile(String id, byte[] data, String fileExt) throws IOException {
FileEntity fileEntity = fileEntityRepository.findById(id)
.orElseThrow(() -> new RuntimeException("File not found: " + id));
String extension = determineFileExtension(fileExt, fileEntity);
Path protectedFilePath = prepareProtectedPath(fileEntity, extension);
if (Files.exists(protectedFilePath)) {
Files.delete(protectedFilePath);
}
Files.createDirectories(protectedFilePath.getParent());
Files.write(protectedFilePath, data);
Files.deleteIfExists(protectedFilePath);
try (FileChannel channel = FileChannel.open(protectedFilePath,
StandardOpenOption.CREATE,
StandardOpenOption.WRITE)) {
ByteBuffer buffer = ByteBuffer.wrap(data);
while (buffer.hasRemaining()) {
channel.write(buffer);
}
channel.force(true);
}
if (imageResizeService.isImage(extension)) {
imageResizeService.generateSizes(fileEntity, data);
@@ -260,14 +266,14 @@ public class FileEntityService {
fileEntity.setFileExtension(extension);
fileEntity.setProtectionStatus(ProtectionStatus.PROTECTED);
fileEntityRepository.save(fileEntity);
return fileEntityRepository.save(fileEntity);
}
public void clearTempFiles(long userId) throws IOException {
public void clearTempFiles(long userId) {
List<FileEntity> fileByUserIdAndStatus = fileEntityRepository.findFileByUserIdAndStatus(userId, FileStatus.TEMP);
for (FileEntity fileEntity : fileByUserIdAndStatus) {
deleteFromDisk(fileEntity);
cloudStorageService.deleteFromStorageDisk(fileEntity);
}
}
@@ -279,24 +285,6 @@ public class FileEntityService {
return fileEntityRepository.findFileIdByFilePath(filePath);
}
public File getFileById(String id) {
try {
FileEntity fileEntity = fileEntityRepository.findById(id).orElseThrow(() ->
new RuntimeException("File not found: " + id));
File file = new File(fileEntity.getFilePath());
if (!file.exists()) {
throw new RuntimeException("File not found on disk: " + fileEntity.getFilePath());
}
return file;
} catch (Exception e) {
log.error("Error getting file: {}", id, e);
return null;
}
}
public Path prepareProtectedPath(FileEntity fileEntity, String extension) throws IOException {
Path originalPath = Paths.get(fileEntity.getFilePath());
@@ -322,7 +310,7 @@ public class FileEntityService {
return protectedPath;
}
public void updateSignature(String signature, String fileId) throws IOException {
public FileEntity updateSignature(String signature, String fileId) throws IOException {
FileEntity fileEntity = fileEntityRepository.findByFileId(fileId);
fileEntity.setSignature(signature);
fileEntity.setProtectionStatus(ProtectionStatus.PROTECTED);
@@ -330,18 +318,30 @@ public class FileEntityService {
fileEntity.setProtectedAt(LocalDateTime.now());
fileEntity.setProtectedFilePath(prepareProtectedPath(fileEntity, fileEntity.getFileExtension()).toString());
fileEntityRepository.save(fileEntity);
return fileEntityRepository.save(fileEntity);
}
private boolean checkFileExistsOnDisk(String filePath) {
try {
return Files.exists(Paths.get(filePath));
} catch (Exception e) {
log.warn("Error checking file existence: {}", filePath, e);
return false;
public String formatFileSize(long size) {
if (size < 1024) {
return size + " B";
} else if (size < 1024 * 1024) {
return String.format("%.1f KB", size / 1024.0);
} else if (size < 1024 * 1024 * 1024) {
return String.format("%.1f MB", size / (1024.0 * 1024.0));
} else {
return String.format("%.1f GB", size / (1024.0 * 1024.0 * 1024.0));
}
}
// private boolean checkFileExistsOnDisk(String filePath) {
// try {
// return Files.exists(Paths.get(filePath));
// } catch (Exception e) {
// log.warn("Error checking file existence: {}", filePath, e);
// return false;
// }
// }
private String determineFileExtension(String fileExt, FileEntity fileEntity) {
if (fileExt != null) {
return fileExt;
@@ -353,7 +353,7 @@ public class FileEntityService {
private final FileMonitoringRepository fileMonitoringRepository;
private FileEntityResponse convertToResponse(FileEntity fileEntity, int version) {
boolean existsOnDisk = checkFileExistsOnDisk(fileEntity.getFilePath());
// boolean existsOnDisk = checkFileExistsOnDisk(fileEntity.getFilePath());
User user = userRepository.findById(fileEntity.getUserId()).get();
String fileName = fileEntity.getOriginalFileName();
String name = fileName;
@@ -385,7 +385,7 @@ public class FileEntityService {
.updatedAt(fileEntity.getUpdatedAt())
.formattedSize(formatFileSize(fileEntity.getFileSize()))
.downloadUrl("/api/v" + version + "/files/download/" + fileEntity.getId())
.existsOnDisk(existsOnDisk)
// .existsOnDisk(existsOnDisk)
.supportId(fileEntity.getSupportId())
.protectStatus(fileEntity.getProtectionStatus().toString())
.fileName(name + "_nocopy_protected" + "." + fileEntity.getFileExtension())
@@ -400,16 +400,4 @@ public class FileEntityService {
.thumbnailFileUrl(baseUrl + "/api/files/protected/" + fileEntity.getId() + "/thumbnail")
.build();
}
public String formatFileSize(long size) {
if (size < 1024) {
return size + " B";
} else if (size < 1024 * 1024) {
return String.format("%.1f KB", size / 1024.0);
} else if (size < 1024 * 1024 * 1024) {
return String.format("%.1f MB", size / (1024.0 * 1024.0));
} else {
return String.format("%.1f GB", size / (1024.0 * 1024.0 * 1024.0));
}
}
}
src/main/java/ru/soune/nocopy/service/file/FileUploadService.java
+2 -3
View File
@@ -2,11 +2,10 @@ package ru.soune.nocopy.service.file;
import org.springframework.web.multipart.MultipartFile;
import ru.soune.nocopy.dto.file.UploadProgressResponse;
import ru.soune.nocopy.entity.file.FileEntity;
import ru.soune.nocopy.entity.file.FileStatus;
import ru.soune.nocopy.entity.file.FileUploadSession;
import java.io.IOException;
public interface FileUploadService {
FileUploadSession initUpload(Long userId, String fileName,
String fileType, String extension, String convertTo, long fileSize);
@@ -22,5 +21,5 @@ public interface FileUploadService {
UploadProgressResponse getUploadProgress(String uploadId);
void completeFileProcessingAsync(FileUploadSession session, FileStatus status);
FileEntity completeFileProcessing(FileUploadSession session, FileStatus status);
}
src/main/java/ru/soune/nocopy/service/file/ImageResizeService.java
+10
View File
@@ -1,10 +1,12 @@
package ru.soune.nocopy.service.file;
import lombok.RequiredArgsConstructor;
import lombok.extern.slf4j.Slf4j;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.stereotype.Service;
import org.springframework.transaction.annotation.Transactional;
import ru.soune.nocopy.entity.file.FileEntity;
import ru.soune.nocopy.service.file.cloud.CloudStorageService;
import javax.imageio.ImageIO;
import java.awt.*;
@@ -18,9 +20,13 @@ import java.util.List;
@Service
@Slf4j
@RequiredArgsConstructor
public class ImageResizeService {
private final CloudStorageService cloudStorageService;
private static final int THUMBNAIL_SIZE = 150;
private static final int MEDIUM_SIZE = 600;
@Value("${file.storage.base-path}")
@@ -50,11 +56,15 @@ public class ImageResizeService {
if (original.getWidth() > THUMBNAIL_SIZE) {
String thumbPath = resizeAndSave(original, baseName + "_thumb", THUMBNAIL_SIZE);
file.setThumbnailPath(thumbPath);
cloudStorageService.saveFilesToStorage(file.getMimeType(), file.getFileExtension(),
thumbPath);
}
if (original.getWidth() > MEDIUM_SIZE) {
String mediumPath = resizeAndSave(original, baseName + "_medium", MEDIUM_SIZE);
file.setMediumPath(mediumPath);
cloudStorageService.saveFilesToStorage(file.getMimeType(), file.getFileExtension(),
mediumPath);
}
}
}
src/main/java/ru/soune/nocopy/service/file/cloud/CloudStorageService.java
+194
View File
@@ -0,0 +1,194 @@
package ru.soune.nocopy.service.file.cloud;
import lombok.RequiredArgsConstructor;
import lombok.extern.slf4j.Slf4j;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.stereotype.Service;
import ru.soune.nocopy.entity.file.FileEntity;
import ru.soune.nocopy.repository.FileEntityRepository;
import software.amazon.awssdk.auth.credentials.AwsBasicCredentials;
import software.amazon.awssdk.auth.credentials.AwsCredentials;
import software.amazon.awssdk.auth.credentials.StaticCredentialsProvider;
import software.amazon.awssdk.core.ResponseBytes;
import software.amazon.awssdk.core.sync.RequestBody;
import software.amazon.awssdk.core.sync.ResponseTransformer;
import software.amazon.awssdk.http.apache.ApacheHttpClient;
import software.amazon.awssdk.regions.Region;
import software.amazon.awssdk.services.s3.S3Client;
import software.amazon.awssdk.services.s3.model.GetObjectRequest;
import software.amazon.awssdk.services.s3.model.GetObjectResponse;
import software.amazon.awssdk.services.s3.model.PutObjectRequest;
import java.io.File;
import java.io.IOException;
import java.io.InputStream;
import java.net.URI;
import java.nio.file.Files;
import java.nio.file.Paths;
import java.util.UUID;
@Service
@Slf4j
@RequiredArgsConstructor
public class CloudStorageService {
private final FileEntityRepository fileEntityRepository;
@Value("${yandex.cloud.key}")
private String key;
@Value("${yandex.cloud.secret-key}")
private String secretKey;
@Value("${yandex.cloud.region}")
private String region;
@Value("${yandex.cloud.s3-endpoint}")
private String s3endpoint;
@Value("${yandex.cloud.bucket}")
private String bucket;
public void saveFilesToStorage(String mimeType, String extension, String sendToCloudFilePath) {
AwsCredentials credentials = AwsBasicCredentials.create(key, secretKey);
S3Client s3Client = S3Client.builder()
.httpClient(ApacheHttpClient.create())
.region(Region.of(region))
.endpointOverride(URI.create(s3endpoint))
.credentialsProvider(StaticCredentialsProvider.create(credentials))
.build();
String mimeTypeFromFile = mimeType.equals("document") ? "application" : mimeType;
String contentType = mimeTypeFromFile + "/" + extension;
String filePath = "files" + sendToCloudFilePath;
PutObjectRequest putObjectRequest = PutObjectRequest.builder()
.bucket(bucket)
.key(filePath)
.contentType(contentType)
.build();
try {
byte[] bytes = Files.readAllBytes(Paths.get(sendToCloudFilePath));
s3Client.putObject(putObjectRequest,
RequestBody.fromBytes(bytes));
Files.deleteIfExists(Paths.get(sendToCloudFilePath));
} catch (IOException e) {
log.error("Failed to read file: {}", filePath, e);
}
}
public File readFileFromStorageByPath(String filePath) throws IOException {
GetObjectRequest objectRequest = GetObjectRequest.builder()
.bucket(bucket)
.key("files" + filePath)
.build();
AwsCredentials credentials = AwsBasicCredentials.create(key, secretKey);
S3Client s3Client = S3Client.builder()
.httpClient(ApacheHttpClient.create())
.region(Region.of(region))
.endpointOverride(URI.create(s3endpoint))
.credentialsProvider(StaticCredentialsProvider.create(credentials))
.build();
String tempDir = System.getProperty("java.io.tmpdir");
String fileName = System.currentTimeMillis() + "_" + UUID.randomUUID() + "_" +
filePath.replace("/", "_");
File tempFile = new File(tempDir, fileName);
s3Client.getObject(objectRequest, tempFile.toPath());
tempFile.deleteOnExit();
return tempFile;
}
public byte[] readFileFromStorageBytes(String filePath) {
AwsCredentials credentials = AwsBasicCredentials.create(key, secretKey);
S3Client s3Client = S3Client.builder()
.httpClient(ApacheHttpClient.create())
.region(Region.of(region))
.endpointOverride(URI.create(s3endpoint))
.credentialsProvider(StaticCredentialsProvider.create(credentials))
.build();
GetObjectRequest objectRequest = GetObjectRequest.builder()
.bucket(bucket)
.key("files" + filePath)
.build();
try {
ResponseBytes<GetObjectResponse> responseBytes = s3Client.getObjectAsBytes(objectRequest);
return responseBytes.asByteArray();
} catch (Exception e) {
log.error("Failed to read file from S3: {}", filePath, e);
throw new RuntimeException("Failed to read file from S3", e);
}
}
public InputStream readFileFromStorage(String filePath) {
GetObjectRequest objectRequest = GetObjectRequest.builder()
.bucket(bucket)
.key("files" + filePath)
.build();
AwsCredentials credentials = AwsBasicCredentials.create(key, secretKey);
S3Client s3Client = S3Client.builder()
.httpClient(ApacheHttpClient.create())
.region(Region.of(region))
.endpointOverride(URI.create(s3endpoint))
.credentialsProvider(StaticCredentialsProvider.create(credentials))
.build();
return s3Client.getObject(objectRequest, ResponseTransformer.toInputStream());
}
public void deleteFileFromStorage(String filePath) {
if (filePath == null || filePath.isEmpty()) {
log.warn("Attempted to delete file with empty path");
return;
}
AwsCredentials credentials = AwsBasicCredentials.create(key, secretKey);
S3Client s3Client = S3Client.builder()
.httpClient(ApacheHttpClient.create())
.region(Region.of(region))
.endpointOverride(URI.create(s3endpoint))
.credentialsProvider(StaticCredentialsProvider.create(credentials))
.build();
String cloudKey = "files" + filePath;
try {
try {
s3Client.headObject(builder -> builder.bucket(bucket).key(cloudKey).build());
} catch (Exception e) {
log.warn("File does not exist in cloud storage: {}", cloudKey);
return;
}
s3Client.deleteObject(builder -> builder.bucket(bucket).key(cloudKey).build());
log.info("File successfully deleted from cloud storage: {}", cloudKey);
} catch (Exception e) {
log.error("Failed to delete file from cloud storage: {}", cloudKey, e);
throw new RuntimeException("Failed to delete file from cloud storage: " + cloudKey, e);
}
}
public void deleteFromStorageDisk(FileEntity fileEntity) {
deleteFileFromStorage(fileEntity.getFilePath());
if (fileEntity.getProtectedFilePath() != null) {
deleteFileFromStorage(fileEntity.getProtectedFilePath());
}
fileEntityRepository.delete(fileEntity);
}
}
src/main/java/ru/soune/nocopy/service/file/impl/AudioLocalSearchImpl.java
+7 -1
View File
@@ -9,6 +9,7 @@ import org.jetbrains.annotations.Nullable;
import org.springframework.stereotype.Service;
import ru.soune.nocopy.entity.file.FileEntity;
import ru.soune.nocopy.service.file.FileEntityService;
import ru.soune.nocopy.service.file.cloud.CloudStorageService;
import ru.soune.nocopy.util.FileUtil;
import java.io.IOException;
@@ -22,6 +23,8 @@ public class AudioLocalSearchImpl implements AudioLocalSearch {
private final FileUtil fileUtil;
private final CloudStorageService cloudStorageService;
@Override
public @Nullable FileProtector.FileInfo findBySignature(@NotNull String signature) {
FileEntity fileEntity = fileEntityService.findBySignature(signature);
@@ -34,7 +37,10 @@ public class AudioLocalSearchImpl implements AudioLocalSearch {
@Override
public void updateSignature(@NotNull String signature, @NotNull FileProtector.FileInfo fileInfo) {
try {
fileEntityService.updateSignature(signature, fileInfo.getId());
FileEntity fileEntity = fileEntityService.updateSignature(signature, fileInfo.getId());
cloudStorageService.saveFilesToStorage(fileEntity.getMimeType(), fileEntity.getFileExtension(),
fileEntity.getProtectedFilePath());
} catch (IOException e) {
throw new RuntimeException(e);
}
src/main/java/ru/soune/nocopy/service/file/impl/DocumentLocalSearchImpl.java
+17
View File
@@ -25,6 +25,23 @@ public class DocumentLocalSearchImpl implements DocumentLocalSearch {
FileEntity fileEntity = fileEntityRepository.findByIdAndUserId(fileId, Long.valueOf(ownerId));
FileProtector.FileInfo fileInfo = fileUtil.createFileInfo(fileEntity, null);
log.info("fileInfo: {}", fileInfo);
log.info("fileInfo: {}", fileInfo);
log.info("fileInfo: {}", fileInfo);
log.info("fileInfo: {}", fileInfo);
log.info("fileInfo: {}", fileInfo);
log.info("fileInfo: {}", fileInfo);
log.info("fileInfo: {}", fileInfo);
log.info("fileInfo: {}", fileInfo);
log.info("fileInfo: {}", fileInfo);
log.info("fileInfo: {}", fileInfo);
log.info("fileInfo: {}", fileInfo);
log.info("fileInfo: {}", fileInfo);
log.info("fileInfo: {}", fileInfo);
log.info("fileInfo: {}", fileInfo);
log.info("fileInfo: {}", fileInfo);
log.info("fileInfo: {}", fileInfo);
log.info("fileInfo: {}", fileInfo);
log.info("fileInfo: {}", fileInfo);
return fileInfo;
}
src/main/java/ru/soune/nocopy/service/file/impl/FileUploadServiceImpl.java
+26 -17
View File
@@ -6,6 +6,7 @@ import com.vrt.fileprotection.NoCopyCheckResult;
import com.vrt.fileprotection.audio.AudioCheckResult;
import com.vrt.fileprotection.documents.DocumentCheckResult;
import jakarta.annotation.PostConstruct;
import jakarta.persistence.EntityManager;
import lombok.RequiredArgsConstructor;
import lombok.extern.slf4j.Slf4j;
import org.springframework.beans.factory.annotation.Autowired;
@@ -28,6 +29,8 @@ import ru.soune.nocopy.service.ImageHashService;
import ru.soune.nocopy.service.cost.CostService;
import ru.soune.nocopy.service.file.FileEntityService;
import ru.soune.nocopy.service.file.FileUploadService;
import ru.soune.nocopy.service.file.cloud.CloudStorageService;
import ru.soune.nocopy.service.tariff.TariffConstants;
import ru.soune.nocopy.service.file.moderation.ModerationFileService;
import ru.soune.nocopy.service.notification.NotificationService;
import ru.soune.nocopy.service.register.AuthService;
@@ -95,6 +98,8 @@ public class FileUploadServiceImpl implements FileUploadService {
private final ModerationFileService moderationFileService;
private final CloudStorageService cloudStorageService;
@PostConstruct
public void init() {
try {
@@ -190,7 +195,6 @@ public class FileUploadServiceImpl implements FileUploadService {
}
@Override
@Transactional
public UploadProgressResponse uploadChunk(String uploadId, Integer chunkNumber,
MultipartFile chunkFile, Integer findSimilar) {
FileUploadSession session = sessionRepository.findById(uploadId)
@@ -222,10 +226,9 @@ public class FileUploadServiceImpl implements FileUploadService {
return UploadProgressResponse.fromSession(session);
}
@Async("fileUploadTaskExecutor")
@Transactional
@Override
public void completeFileProcessingAsync(FileUploadSession session, FileStatus status) {
public FileEntity completeFileProcessing(FileUploadSession session, FileStatus status) {
try {
Path filePath = Paths.get(session.getFilePath());
String checksum = calculateChecksum(filePath);
@@ -251,22 +254,17 @@ public class FileUploadServiceImpl implements FileUploadService {
if (session.getFileType().equals("image") && status != FileStatus.PRIVATE) {
Map<String, Long> hash = imageHashService.calculateHash(filePath);
imageHashService.create(saved, hash);
}
cleanupSessionFiles(session);
if (status != FileStatus.TEMP && status != FileStatus.PRIVATE) {
FileProtector.FileInfo fileInfo = fileUtil.createFileInfo(fileEntity, session.getConvertTo());
noCopyFileService.addFile(fileInfo);
}
log.info("File processing completed for session: {}", session.getUploadId());
return saved;
} catch (Exception e) {
log.error("Failed to complete file processing for session {}: {}",
session.getUploadId(), e.getMessage(), e);
throw new FileUploadException("Failed to complete file processing", e);
}
}
@@ -339,7 +337,7 @@ public class FileUploadServiceImpl implements FileUploadService {
session.setStatus(UploadStatus.COMPLETED);
session.setFilePath(finalFilePath);
completeFileProcessingAsync(session, FileStatus.PRIVATE);
completeFileProcessing(session, FileStatus.PRIVATE);
} else {
sessionRepository.save(session);
}
@@ -381,7 +379,7 @@ public class FileUploadServiceImpl implements FileUploadService {
chunkNumber, session.getChunksUploaded(), session.getTotalChunks());
boolean isLastChunk = session.getChunksUploaded().equals(session.getTotalChunks());
//TODO CHECK
if (isLastChunk) {
String finalFilePath = assembleFileSynchronously(session);
@@ -393,6 +391,7 @@ public class FileUploadServiceImpl implements FileUploadService {
session.getUserId());
File file = new File(finalFilePath);
// File file = cloudStorageService.readFileFromStorageByPath(finalFilePath);
if (session.getFileType().startsWith("audio") && !isWavFile(file)) {
session.setStatus(UploadStatus.FAILED);
@@ -432,20 +431,29 @@ public class FileUploadServiceImpl implements FileUploadService {
session.setStatus(UploadStatus.COMPLETED);
session.setFilePath(finalFilePath);
notificationService.addNotification(NotificationType.FILE_ADDED_TO_SYSTEM, session.getUserId());
FileStatus status = findSimilar == 0 ? FileStatus.MODERATION: FileStatus.TEMP;
if (status == FileStatus.TEMP) {
fileEntityService.clearTempFiles(session.getUserId());
}
completeFileProcessingAsync(session, status);
FileEntity fileEntity = completeFileProcessing(session, status);
cloudStorageService.saveFilesToStorage(fileEntity.getMimeType(), fileEntity.getFileExtension(),
fileEntity.getFilePath());
if (status != FileStatus.TEMP) {
String fileType = session.getFileType();
tariffInfoService.writeOffTokens(session.getUserId(), costService.protectFileCost(fileType), OperationType.FILE_UPLOAD);
tariffInfoService.writeOffTokens(session.getUserId(), costService.protectFileCost(fileType),
OperationType.FILE_UPLOAD);
notificationService.addNotification(NotificationType.FILE_ADDED_TO_SYSTEM, session.getUserId());
}
if (status != FileStatus.TEMP) {
FileProtector.FileInfo fileInfo = fileUtil.createFileInfo(fileEntity, session.getConvertTo());
noCopyFileService.addFile(fileInfo);
}
} else {
sessionRepository.save(session);
@@ -580,6 +588,7 @@ public class FileUploadServiceImpl implements FileUploadService {
private void checkForDuplicatesSynchronously(String filePath)
throws IOException , DuplicateImageException {
Path path = Paths.get(filePath);
Map<String, Long> hash = imageHashService.calculateHash(path);
List<SimilarImageProjection> duplicates = fileSimilarityService.findDuplicatedByHash(
src/main/java/ru/soune/nocopy/service/file/impl/ProtectionFileProviderImpl.java
+59 -9
View File
@@ -11,7 +11,7 @@ import ru.soune.nocopy.entity.file.FileEntity;
import ru.soune.nocopy.repository.FileEntityRepository;
import ru.soune.nocopy.repository.FileUploadSessionRepository;
import ru.soune.nocopy.service.file.FileEntityService;
import ru.soune.nocopy.service.file.FileUploadService;
import ru.soune.nocopy.service.file.cloud.CloudStorageService;
import java.io.File;
import java.io.IOException;
@@ -32,27 +32,53 @@ public class ProtectionFileProviderImpl implements FileProtector.FileProvider {
private final FileEntityRepository fileEntityRepository;
private final CloudStorageService cloudStorageService;
@Nullable
@Override
public File getImageFile(@NotNull String id) {
return fileEntityService.getFileById(id);
FileEntity fileEntity = fileEntityRepository.findById(id).orElseThrow(() ->
new RuntimeException("File not found: " + id));
try {
return cloudStorageService.readFileFromStorageByPath(fileEntity.getFilePath());
} catch (IOException e) {
throw new RuntimeException(e);
}
}
@Nullable
@Override
public File getVideoFile(@NotNull String id) {
return fileEntityService.getFileById(id);
FileEntity fileEntity = fileEntityRepository.findById(id).orElseThrow(() ->
new RuntimeException("File not found: " + id));
try {
return cloudStorageService.readFileFromStorageByPath(fileEntity.getFilePath());
} catch (IOException e) {
throw new RuntimeException(e);
}
}
@Nullable
@Override
public File getAudioFile(@NotNull String id) {
return fileEntityService.getFileById(id);
FileEntity fileEntity = fileEntityRepository.findById(id).orElseThrow(() ->
new RuntimeException("File not found: " + id));
try {
return cloudStorageService.readFileFromStorageByPath(fileEntity.getFilePath());
} catch (IOException e) {
throw new RuntimeException(e);
}
}
@Override
public @Nullable File getDocument(@NotNull String id) {
return fileEntityService.getFileById(id);
FileEntity fileEntity = fileEntityRepository.findById(id).orElseThrow(() ->
new RuntimeException("File not found: " + id));
try {
return cloudStorageService.readFileFromStorageByPath(fileEntity.getFilePath());
} catch (IOException e) {
throw new RuntimeException(e);
}
}
@Override
@@ -87,7 +113,20 @@ public class ProtectionFileProviderImpl implements FileProtector.FileProvider {
@Override
public com.vrt.fileprotection.OperationResult writeAudioFile(@NotNull String id, @NotNull byte[] data, @Nullable String fileExt) {
try {
fileEntityService.writeProtectedFile(id, data, fileExt);
FileEntity fileEntity = fileEntityService.writeProtectedFile(id, data, fileExt);
Path filePath = Paths.get(fileEntity.getProtectedFilePath());
if (!Files.exists(filePath)) {
log.error("File not found after write: {}", filePath);
return OperationResult.Companion.failure("File not found");
}
log.info("File ready for upload: {} (size: {} bytes)", filePath, Files.size(filePath));
cloudStorageService.saveFilesToStorage(fileEntity.getMimeType(), fileEntity.getFileExtension(),
fileEntity.getProtectedFilePath());
return OperationResult.Companion.success();
} catch (Exception e) {
log.error("Failed to create protected file: {}", id, e);
@@ -99,7 +138,11 @@ public class ProtectionFileProviderImpl implements FileProtector.FileProvider {
@Override
public com.vrt.fileprotection.OperationResult writeVideoFile(@NotNull String id, @NotNull byte[] data, @Nullable String fileExt) {
try {
fileEntityService.writeProtectedFile(id, data, fileExt);
FileEntity fileEntity = fileEntityService.writeProtectedFile(id, data, fileExt);
cloudStorageService.saveFilesToStorage(fileEntity.getMimeType(), fileEntity.getFileExtension(),
fileEntity.getProtectedFilePath());
return OperationResult.Companion.success();
} catch (Exception e) {
log.error("Failed to create protected file: {}", id, e);
@@ -111,7 +154,11 @@ public class ProtectionFileProviderImpl implements FileProtector.FileProvider {
@Override
public com.vrt.fileprotection.OperationResult writeImageFile(@NotNull String id, @NotNull byte[] data, @Nullable String fileExt) {
try {
fileEntityService.writeProtectedFile(id, data, fileExt);
FileEntity fileEntity = fileEntityService.writeProtectedFile(id, data, fileExt);
cloudStorageService.saveFilesToStorage(fileEntity.getMimeType(), fileEntity.getFileExtension(),
fileEntity.getProtectedFilePath());
return OperationResult.Companion.success();
} catch (Exception e) {
log.error("Failed to create protected file: {}", id, e);
@@ -122,7 +169,10 @@ public class ProtectionFileProviderImpl implements FileProtector.FileProvider {
@Override
public @NotNull OperationResult writeDocumentFile(@NotNull String id, @NotNull byte[] data) {
try {
fileEntityService.writeProtectedFile(id, data, null);
FileEntity fileEntity = fileEntityService.writeProtectedFile(id, data, null);
cloudStorageService.saveFilesToStorage(fileEntity.getMimeType(), fileEntity.getFileExtension(),
fileEntity.getProtectedFilePath());
return OperationResult.Companion.success();
} catch (Exception e) {
log.error("Failed to create protected file: {}", id, e);