5 Commits
Author SHA1 Message Date
backdev eb223c2ecb dev add new action
Test Workflow / test (push) Has been cancelled
2026-04-22 12:38:05 +07:00
backdev bb96e3ac93 dev add new action
Test Workflow / test (push) Has been cancelled
2026-04-22 12:16:37 +07:00
backdev 572255c066 dev add monitoring statistic
Test Workflow / test (push) Has been cancelled
2026-04-20 16:15:08 +07:00
backdev d01ea6e2bd dev fix save to storage
Test Workflow / test (push) Has been cancelled
2026-04-20 15:35:24 +07:00
backdev d1a91fec1a dev add user_id for find all complaints
Test Workflow / test (push) Has been cancelled
2026-04-17 22:00:33 +07:00
37 changed files with 458 additions and 600 deletions
-11
View File
@@ -323,7 +323,6 @@ server_name dev-workspace.not-copy.com;
ssl_certificate_key /etc/letsencrypt/live/dev-workspace.not-copy.com/privkey.pem; ssl_certificate_key /etc/letsencrypt/live/dev-workspace.not-copy.com/privkey.pem;
location /api/ { location /api/ {
proxy_pass http://localhost:3001; proxy_pass http://localhost:3001;
proxy_set_header Host $host; proxy_set_header Host $host;
proxy_set_header X-Real-IP $remote_addr; proxy_set_header X-Real-IP $remote_addr;
@@ -491,13 +490,3 @@ killall 3proxy # Остановка прокси
bash bash
curl -x http://193.46.217.94:3128 https://api.ipify.org # Проверка прокси curl -x http://193.46.217.94:3128 https://api.ipify.org # Проверка прокси
telnet 193.46.217.94 3128 # Проверка порта telnet 193.46.217.94 3128 # Проверка порта
------
Настройка Яндекс Клауд
1. YCAJEpWAtaVkVGX0sH6_EupEg - индетификатор ключа
2. YCMmVykfXrZ_nfU13Vo4yoCVGa70DnTlBgF1pUzO - секретный ключ
private static final String KEY_ID = "YCAJEmHZcWxzf4oGWvETG3mms";
private static final String SECRET_KEY = "YCPAPZ32XC-LWb8xE0b_xTcBD8NZUCHpSOfu8LUG";
-6
View File
@@ -66,12 +66,6 @@ dependencies {
implementation project(':referral') implementation project(':referral')
//cloud
implementation("software.amazon.awssdk:aws-sdk-java:2.29.33")
implementation("software.amazon.awssdk:apache-client:2.29.33")
implementation("org.springdoc:springdoc-openapi-starter-webmvc-ui:2.7.0")
implementation("org.springframework.boot:spring-boot-starter-web")
} }
tasks.named('test') { tasks.named('test') {
@@ -46,7 +46,8 @@ public class HandlerConfig {
StatisticViolationHandler statisticViolationHandler, StatisticViolationHandler statisticViolationHandler,
StatisticSubscriberHandler statisticSubscriberHandler, StatisticSubscriberHandler statisticSubscriberHandler,
StatisticTokenHandler statisticTokenHandler, StatisticTokenHandler statisticTokenHandler,
StatisticIncomeHandler statisticIncomeHandler StatisticIncomeHandler statisticIncomeHandler,
MonitoringStatisticHandler monitoringStatisticHandler
) { ) {
Map<Integer, RequestHandler> map = new HashMap<>(); Map<Integer, RequestHandler> map = new HashMap<>();
@@ -85,6 +86,7 @@ public class HandlerConfig {
map.put(30024, statisticSubscriberHandler); map.put(30024, statisticSubscriberHandler);
map.put(30025, statisticTokenHandler); map.put(30025, statisticTokenHandler);
map.put(30026, statisticIncomeHandler); map.put(30026, statisticIncomeHandler);
map.put(30027, monitoringStatisticHandler);
return map; return map;
} }
@@ -1,7 +1,5 @@
package ru.soune.nocopy.controller; package ru.soune.nocopy.controller;
import com.google.api.client.util.IOUtils;
import com.google.common.io.ByteStreams;
import com.vrt.NoCopyFileService; import com.vrt.NoCopyFileService;
import com.vrt.fileprotection.FileProtector; import com.vrt.fileprotection.FileProtector;
import com.vrt.fileprotection.NoCopyCheckResult; import com.vrt.fileprotection.NoCopyCheckResult;
@@ -10,20 +8,16 @@ import com.vrt.fileprotection.documents.DocumentCheckResult;
import lombok.RequiredArgsConstructor; import lombok.RequiredArgsConstructor;
import lombok.extern.slf4j.Slf4j; import lombok.extern.slf4j.Slf4j;
import org.springframework.core.io.FileSystemResource; import org.springframework.core.io.FileSystemResource;
import org.springframework.core.io.InputStreamResource;
import org.springframework.data.domain.Page; import org.springframework.data.domain.Page;
import org.springframework.data.domain.Pageable; import org.springframework.data.domain.Pageable;
import org.springframework.data.web.PageableDefault; import org.springframework.data.web.PageableDefault;
import org.springframework.http.HttpHeaders; import org.springframework.http.HttpHeaders;
import org.springframework.http.HttpStatus;
import org.springframework.http.MediaType; import org.springframework.http.MediaType;
import org.springframework.http.ResponseEntity; import org.springframework.http.ResponseEntity;
import org.springframework.validation.BindingResult; import org.springframework.validation.BindingResult;
import org.springframework.validation.FieldError; import org.springframework.validation.FieldError;
import org.springframework.web.bind.annotation.*; import org.springframework.web.bind.annotation.*;
import org.springframework.web.multipart.MultipartFile; import org.springframework.web.multipart.MultipartFile;
import org.springframework.web.server.ResponseStatusException;
import org.springframework.web.servlet.mvc.method.annotation.StreamingResponseBody;
import ru.soune.nocopy.dto.BaseRequest; import ru.soune.nocopy.dto.BaseRequest;
import ru.soune.nocopy.dto.BaseResponse; import ru.soune.nocopy.dto.BaseResponse;
import ru.soune.nocopy.dto.MessageCode; import ru.soune.nocopy.dto.MessageCode;
@@ -42,17 +36,12 @@ import ru.soune.nocopy.repository.FileEntityRepository;
import ru.soune.nocopy.repository.UserRepository; import ru.soune.nocopy.repository.UserRepository;
import ru.soune.nocopy.service.FileSimilarityService; import ru.soune.nocopy.service.FileSimilarityService;
import ru.soune.nocopy.service.file.*; import ru.soune.nocopy.service.file.*;
import ru.soune.nocopy.service.file.CheckCounterService;
import ru.soune.nocopy.service.file.ProtectionsLimitService;
import ru.soune.nocopy.service.file.cloud.CloudStorageService;
import ru.soune.nocopy.service.register.AuthService; import ru.soune.nocopy.service.register.AuthService;
import ru.soune.nocopy.service.user.moderation.UserVerificationService; import ru.soune.nocopy.service.user.moderation.UserVerificationService;
import ru.soune.nocopy.util.FileUtil; import ru.soune.nocopy.util.FileUtil;
import software.amazon.awssdk.services.s3.model.NoSuchKeyException;
import java.io.File; import java.io.File;
import java.io.IOException; import java.io.IOException;
import java.io.InputStream;
import java.nio.file.Files; import java.nio.file.Files;
import java.nio.file.Path; import java.nio.file.Path;
import java.nio.file.Paths; import java.nio.file.Paths;
@@ -76,8 +65,6 @@ public class ApiController {
private final FileEntityRepository fileEntityRepository; private final FileEntityRepository fileEntityRepository;
private final NoCopyFileService noCopyFileService;
private final FileUtil fileUtil; private final FileUtil fileUtil;
private final ProtectionsLimitService protectionsLimitService; private final ProtectionsLimitService protectionsLimitService;
@@ -88,8 +75,6 @@ public class ApiController {
private final CheckCounterService checkCounterService; private final CheckCounterService checkCounterService;
private final CloudStorageService cloudStorageService;
private final ZipService zipService; private final ZipService zipService;
private final UserVerificationService userVerificationService; private final UserVerificationService userVerificationService;
@@ -220,6 +205,8 @@ public class ApiController {
} }
} }
private final NoCopyFileService noCopyFileService;
@GetMapping("/check/{fileId}") @GetMapping("/check/{fileId}")
public ResponseEntity<BaseResponse> check(@PathVariable String fileId) { public ResponseEntity<BaseResponse> check(@PathVariable String fileId) {
FileEntity fileEntity = fileEntityRepository.findByFileId(fileId); FileEntity fileEntity = fileEntityRepository.findByFileId(fileId);
@@ -502,35 +489,39 @@ public class ApiController {
MessageCode.FILE_DELETE.getDescription(), MessageCode.FILE_DELETE.getDescription(),
errorData)); errorData));
} }
//TODO
// if (!entityResponse.isExistsOnDisk()) { if (!entityResponse.isExistsOnDisk()) {
// Map<String, Object> errorData = new HashMap<>(); Map<String, Object> errorData = new HashMap<>();
// errorData.put("onDisk", entityResponse.isExistsOnDisk()); errorData.put("onDisk", entityResponse.isExistsOnDisk());
//
// return ResponseEntity.ok().body(new BaseResponse(20004,
// MessageCode.FILE_NOT_EXIST.getCode(),
// MessageCode.FILE_NOT_EXIST.getDescription(),
// errorData));
// }
// if (!file.exists()) { return ResponseEntity.ok().body(new BaseResponse(20004,
// Map<String, Object> errorData = new HashMap<>(); MessageCode.FILE_NOT_EXIST.getCode(),
// errorData.put("file", file.exists()); MessageCode.FILE_NOT_EXIST.getDescription(),
// errorData));
// return ResponseEntity.ok().body(new BaseResponse(20004, }
// MessageCode.FILE_DOWNLOAD_ERROR.getCode(),
// MessageCode.FILE_DOWNLOAD_ERROR.getDescription(),
// errorData));
// }
byte[] bytes = cloudStorageService.readFileFromStorageBytes(entityResponse.getProtectedFilePath());
Path filePath = Paths.get(entityResponse.getProtectedFilePath());
FileSystemResource resource = new FileSystemResource(filePath);
long fileSize = Files.size(filePath);
if (!resource.exists()) {
Map<String, Object> errorData = new HashMap<>();
errorData.put("resource", resource.exists());
return ResponseEntity.ok().body(new BaseResponse(20004,
MessageCode.FILE_DOWNLOAD_ERROR.getCode(),
MessageCode.FILE_DOWNLOAD_ERROR.getDescription(),
errorData));
}
String contentType = determineContentType(filePath);
return ResponseEntity.ok() return ResponseEntity.ok()
.contentType(MediaType.APPLICATION_OCTET_STREAM) .contentLength(fileSize)
.contentType(MediaType.parseMediaType(contentType))
.header(HttpHeaders.CONTENT_DISPOSITION, .header(HttpHeaders.CONTENT_DISPOSITION,
"attachment; filename=\"" + entityResponse.getFileName() + "\"") "attachment; filename=\"" + entityResponse.getFileName() + "\"")
.body(bytes); .body(resource);
} catch (FileEntityNotFoundException e) { } catch (FileEntityNotFoundException e) {
Map<String, Object> errorData = new HashMap<>(); Map<String, Object> errorData = new HashMap<>();
errorData.put("fileId", fileId); errorData.put("fileId", fileId);
@@ -547,16 +538,16 @@ public class ApiController {
MessageCode.AUTH_TOKEN_NOT_FOUND.getCode(), MessageCode.AUTH_TOKEN_NOT_FOUND.getCode(),
MessageCode.AUTH_TOKEN_NOT_FOUND.getDescription(), MessageCode.AUTH_TOKEN_NOT_FOUND.getDescription(),
errorData)); errorData));
// } catch (IOException e) { } catch (IOException e) {
// Map<String, Object> errorData = new HashMap<>(); Map<String, Object> errorData = new HashMap<>();
// errorData.put("token", tokenHeader); errorData.put("token", tokenHeader);
// errorData.put("fileId", fileId); errorData.put("fileId", fileId);
// errorData.put("version", version); errorData.put("version", version);
//
// return ResponseEntity.ok().body(new BaseResponse(20004, return ResponseEntity.ok().body(new BaseResponse(20004,
// MessageCode.FILE_DOWNLOAD_ERROR_NOT_CORRECT_FIELD.getCode(), MessageCode.FILE_DOWNLOAD_ERROR_NOT_CORRECT_FIELD.getCode(),
// MessageCode.FILE_DOWNLOAD_ERROR_NOT_CORRECT_FIELD.getDescription(), MessageCode.FILE_DOWNLOAD_ERROR_NOT_CORRECT_FIELD.getDescription(),
// errorData)); errorData));
} }
} }
@@ -573,6 +564,33 @@ public class ApiController {
return ResponseEntity.ok().body(fileTypeStats); return ResponseEntity.ok().body(fileTypeStats);
} }
private boolean hasDuplicate(List<SimilarFileDTO> similarFiles) {
return similarFiles.stream().anyMatch(f -> f.getHammingDistance() <= 5);
}
private ResponseEntity<BaseResponse> handleDuplicate(FileEntity fileEntity, List<SimilarFileDTO> similarFiles)
throws IOException {
fileEntityService.deleteFromDisk(fileEntity);
fileEntityService.softDeleteFileWithHash(fileEntity);
Optional<FileEntity> originalFile = fileEntityRepository.findById(similarFiles.get(0).getFileId());
if (originalFile.isPresent()) {
Map<String, String> duplicateInfo = Map.of(
"duplicate_file_id", originalFile.get().getId(),
"owner_user_id", String.valueOf(originalFile.get().getUserId()));
return ResponseEntity.ok().body(new BaseResponse(
20004,
MessageCode.DUPLICATE_FILE_UPLOAD.getCode(),
"Failed to upload chunk, duplicate",
duplicateInfo));
}
return null;
}
private ResponseEntity<BaseResponse> buildSuccessResponse(String uploadId, Integer chunkNumber, MultipartFile chunk, private ResponseEntity<BaseResponse> buildSuccessResponse(String uploadId, Integer chunkNumber, MultipartFile chunk,
String fileId) { String fileId) {
ChunkUploadResponse responseBody = ChunkUploadResponse.builder() ChunkUploadResponse responseBody = ChunkUploadResponse.builder()
@@ -629,4 +647,13 @@ public class ApiController {
return errorDetail; return errorDetail;
} }
private String determineContentType(Path filePath) throws IOException {
String contentType = Files.probeContentType(filePath);
if (contentType == null) {
contentType = "application/octet-stream";
}
return contentType;
}
} }
@@ -1,16 +1,12 @@
package ru.soune.nocopy.controller; package ru.soune.nocopy.controller;
import com.google.api.client.util.IOUtils;
import com.google.common.io.ByteStreams;
import com.vrt.NoCopyFileService; import com.vrt.NoCopyFileService;
import com.vrt.fileprotection.FileProtector; import com.vrt.fileprotection.FileProtector;
import lombok.AllArgsConstructor; import lombok.AllArgsConstructor;
import lombok.extern.slf4j.Slf4j; import lombok.extern.slf4j.Slf4j;
import org.springframework.core.io.InputStreamResource;
import org.springframework.core.io.Resource; import org.springframework.core.io.Resource;
import org.springframework.http.*; import org.springframework.http.*;
import org.springframework.web.bind.annotation.*; import org.springframework.web.bind.annotation.*;
import org.springframework.web.servlet.mvc.method.annotation.StreamingResponseBody;
import ru.soune.nocopy.dto.file.CheckStatus; import ru.soune.nocopy.dto.file.CheckStatus;
import ru.soune.nocopy.entity.file.FileEntity; import ru.soune.nocopy.entity.file.FileEntity;
import ru.soune.nocopy.entity.file.ProtectionStatus; import ru.soune.nocopy.entity.file.ProtectionStatus;
@@ -22,14 +18,13 @@ import ru.soune.nocopy.service.file.CheckCounterService;
import ru.soune.nocopy.service.file.FileStorageService; import ru.soune.nocopy.service.file.FileStorageService;
import ru.soune.nocopy.service.file.ImageResizeService; import ru.soune.nocopy.service.file.ImageResizeService;
import ru.soune.nocopy.service.file.ZipService; import ru.soune.nocopy.service.file.ZipService;
import ru.soune.nocopy.service.file.cloud.CloudStorageService;
import ru.soune.nocopy.util.FileUtil; import ru.soune.nocopy.util.FileUtil;
import software.amazon.awssdk.services.s3.model.NoSuchKeyException;
import java.io.*; import java.io.FileNotFoundException;
import java.net.URLEncoder; import java.net.URLEncoder;
import java.nio.charset.StandardCharsets; import java.nio.charset.StandardCharsets;
import java.io.IOException;
import java.nio.file.Files; import java.nio.file.Files;
import java.nio.file.Path; import java.nio.file.Path;
import java.nio.file.Paths; import java.nio.file.Paths;
@@ -56,13 +51,19 @@ public class FileController {
private ZipService zipService; private ZipService zipService;
private CloudStorageService cloudStorageService;
@GetMapping("/public/{fileId}") @GetMapping("/public/{fileId}")
public ResponseEntity<StreamingResponseBody> getPublicFile(@PathVariable String fileId) { public ResponseEntity<Resource> getPublicFile(@PathVariable String fileId) {
try { try {
FileEntity fileEntity = fileRepository.findById(fileId) FileEntity fileEntity = fileRepository.findById(fileId)
.orElseThrow(() -> new FileNotFoundException("Not found in DB: " + fileId)); .orElseThrow(() -> new FileNotFoundException("Not found: " + fileId));
Path filePath = Paths.get(fileEntity.getFilePath());
if (!Files.exists(filePath)) {
log.error("Not on disk: {}", fileEntity.getFilePath());
return ResponseEntity.notFound().build();
}
Resource resource = fileStorageService.loadFileAsResource(fileEntity.getFilePath());
MediaType mediaType = getMediaType(fileEntity); MediaType mediaType = getMediaType(fileEntity);
@@ -70,28 +71,18 @@ public class FileController {
.filename(fileEntity.getOriginalFileName(), StandardCharsets.UTF_8) .filename(fileEntity.getOriginalFileName(), StandardCharsets.UTF_8)
.build(); .build();
StreamingResponseBody responseBody = outputStream -> {
try (InputStream inputStream = cloudStorageService.readFileFromStorage(fileEntity.getFilePath())) {
ByteStreams.copy(inputStream, outputStream);
} catch (NoSuchKeyException e) {
log.error("File disappeared from S3 during streaming: {}", fileId, e);
throw new IOException("File not found in storage", e);
}
};
return ResponseEntity.ok() return ResponseEntity.ok()
.contentType(mediaType) .contentType(mediaType)
.contentLength(Files.size(filePath))
.header(HttpHeaders.CONTENT_DISPOSITION, contentDisposition.toString()) .header(HttpHeaders.CONTENT_DISPOSITION, contentDisposition.toString())
.header(HttpHeaders.CACHE_CONTROL, "public, max-age=3600") .header(HttpHeaders.CACHE_CONTROL, "public, max-age=3600")
.body(responseBody); .body(resource);
} catch (FileNotFoundException e) { } catch (FileNotFoundException e) {
log.warn("File not found in DB: {}", fileId); log.warn("Not on disk: {}", fileId);
return ResponseEntity.notFound().build(); return ResponseEntity.notFound().build();
} catch (NoSuchKeyException e) { } catch (IOException e) {
log.warn("File not found in S3: {}", fileId); log.error("Read file exception: {}", fileId, e);
return ResponseEntity.status(HttpStatus.NOT_FOUND).build();
} catch (Exception e) {
log.error("Error reading file: {}", fileId, e);
return ResponseEntity.status(HttpStatus.INTERNAL_SERVER_ERROR).build(); return ResponseEntity.status(HttpStatus.INTERNAL_SERVER_ERROR).build();
} }
} }
@@ -122,14 +113,9 @@ public class FileController {
FileEntity fileEntity = fileRepository.findById(fileId) FileEntity fileEntity = fileRepository.findById(fileId)
.orElseThrow(() -> new RuntimeException("File not found")); .orElseThrow(() -> new RuntimeException("File not found"));
String filePath = type.equals("thumbnail") ? fileEntity.getThumbnailPath(): fileEntity.getFilePath(); String path = imageResizeService.getPath(fileEntity, type);
// InputStream file = cloudStorageService.readFileFromStorage(filePath); Resource resource = fileStorageService.loadFileAsResource(path);
// if (file == null || !file.exists()) {
// log.error("File not found in storage: {}", fileEntity.getFilePath());
// return ResponseEntity.notFound().build();
// }
if (fileEntity.getProtectionStatus() == ProtectionStatus.NOT_PROTECTED || if (fileEntity.getProtectionStatus() == ProtectionStatus.NOT_PROTECTED ||
fileEntity.getProtectionStatus() == ProtectionStatus.PROCESSING) { fileEntity.getProtectionStatus() == ProtectionStatus.PROCESSING) {
@@ -142,8 +128,6 @@ public class FileController {
String encodedFileName = URLEncoder.encode(fileName, StandardCharsets.UTF_8) String encodedFileName = URLEncoder.encode(fileName, StandardCharsets.UTF_8)
.replace("+", "%20"); .replace("+", "%20");
InputStreamResource resource = new InputStreamResource(cloudStorageService.readFileFromStorage(filePath));
return ResponseEntity.ok() return ResponseEntity.ok()
.contentType(new MediaType(fileEntity.getMimeType(), fileEntity.getFileExtension())) .contentType(new MediaType(fileEntity.getMimeType(), fileEntity.getFileExtension()))
.header(HttpHeaders.CONTENT_DISPOSITION, .header(HttpHeaders.CONTENT_DISPOSITION,
@@ -35,4 +35,7 @@ public class ComplaintRequest {
@JsonProperty("sort_direction") @JsonProperty("sort_direction")
private String sortDirection; private String sortDirection;
@JsonProperty("token")
private String token;
} }
@@ -36,4 +36,7 @@ public class ComplaintResponse {
@JsonProperty("not_moderated") @JsonProperty("not_moderated")
private Boolean notModerated; private Boolean notModerated;
@JsonProperty("file_id")
private String fileId;
} }
@@ -1,5 +1,6 @@
package ru.soune.nocopy.dto.complaint; package ru.soune.nocopy.dto.complaint;
import com.fasterxml.jackson.annotation.JsonProperty;
import lombok.Builder; import lombok.Builder;
import lombok.Data; import lombok.Data;
import ru.soune.nocopy.entity.complaint.LawCase; import ru.soune.nocopy.entity.complaint.LawCase;
@@ -14,19 +15,43 @@ import java.util.List;
@Builder @Builder
public class LawCaseResponse { public class LawCaseResponse {
private Long id; private Long id;
private String name; private String name;
private String description; private String description;
private BigDecimal amount; private BigDecimal amount;
private LawCasePriority priority; private LawCasePriority priority;
private LawCaseType type; private LawCaseType type;
private String lawyer; private String lawyer;
@JsonProperty("created_at")
private LocalDateTime createdAt; private LocalDateTime createdAt;
@JsonProperty("updated_at")
private LocalDateTime updatedAt; private LocalDateTime updatedAt;
@JsonProperty("page_number")
private int pageNumber; private int pageNumber;
@JsonProperty("page_size")
private int pageSize; private int pageSize;
@JsonProperty("total_elements")
private long totalElements; private long totalElements;
@JsonProperty("total_pages")
private int totalPages; private int totalPages;
@JsonProperty("violation_id")
private long violationId; private long violationId;
@JsonProperty("file_id")
private String fileId;
private List<LawCaseResponse> content; private List<LawCaseResponse> content;
public static LawCaseResponse fromEntity(LawCase lawCase) { public static LawCaseResponse fromEntity(LawCase lawCase) {
@@ -0,0 +1,8 @@
package ru.soune.nocopy.dto.statistic;
import lombok.Data;
@Data
public class MonitoringStatisticRequest {
private String authToken;
}
@@ -0,0 +1,20 @@
package ru.soune.nocopy.dto.statistic;
import com.fasterxml.jackson.annotation.JsonProperty;
import lombok.Builder;
import lombok.Data;
@Data
@Builder
public class MonitoringStatisticResponse {
@JsonProperty("day")
private Long everyDay;
@JsonProperty("month")
private Long everyMonth;
@JsonProperty("weekly")
private Long everyWeekly;
}
@@ -1,12 +1,22 @@
package ru.soune.nocopy.dto.violation; package ru.soune.nocopy.dto.violation;
import com.fasterxml.jackson.annotation.JsonProperty;
import lombok.Data; import lombok.Data;
@Data @Data
public class ViolationNotionRequest { public class ViolationNotionRequest {
@JsonProperty("token")
private String token; private String token;
@JsonProperty("action")
private String action; private String action;
@JsonProperty("violation_id")
private Long violationId; private Long violationId;
@JsonProperty("notion_id")
private Long notionId; private Long notionId;
@JsonProperty("message")
private String message; private String message;
} }
@@ -13,8 +13,14 @@ import ru.soune.nocopy.dto.MessageCode;
import ru.soune.nocopy.dto.complaint.ComplaintRequest; import ru.soune.nocopy.dto.complaint.ComplaintRequest;
import ru.soune.nocopy.dto.complaint.ComplaintResponse; import ru.soune.nocopy.dto.complaint.ComplaintResponse;
import ru.soune.nocopy.entity.complaint.ComplaintStatus; import ru.soune.nocopy.entity.complaint.ComplaintStatus;
import ru.soune.nocopy.entity.user.User;
import ru.soune.nocopy.repository.UserRepository;
import ru.soune.nocopy.service.complaint.ComplaintEntityService; import ru.soune.nocopy.service.complaint.ComplaintEntityService;
import ru.soune.nocopy.service.register.AuthService;
import ru.soune.nocopy.service.user.UserService;
import java.util.HashMap; import java.util.HashMap;
import java.util.List;
import java.util.Map; import java.util.Map;
@Slf4j @Slf4j
@@ -25,11 +31,31 @@ public class ComplaintEntityHandler implements RequestHandler {
private final ObjectMapper objectMapper; private final ObjectMapper objectMapper;
private final UserRepository userRepository;
private final UserService userService;
private final AuthService authService;
@Override @Override
public BaseResponse handle(BaseRequest request) throws Exception { public BaseResponse handle(BaseRequest request) throws Exception {
Integer msgId = request.getMsgId(); Integer msgId = request.getMsgId();
ComplaintRequest complaintRequest = objectMapper.convertValue(request.getMessageBody(), ComplaintRequest.class); ComplaintRequest complaintRequest = objectMapper.convertValue(request.getMessageBody(), ComplaintRequest.class);
String action = complaintRequest.getAction(); String action = complaintRequest.getAction();
String token = complaintRequest.getToken();
Long userId = authService.useUserAuthToken(token);
User user = userRepository.findById(userId).orElse(null);
if (user == null) {
return new BaseResponse(
request.getMsgId(),
MessageCode.USER_NOT_FOUND.getCode(),
MessageCode.USER_NOT_FOUND.getDescription(),
Map.of("token: ", token));
}
List<String> emails = user.getCompany() == null ? List.of(user.getEmail()):
userService.getUsersCompanyEmails(user.getCompany().getId());
try { try {
switch (action.toLowerCase()) { switch (action.toLowerCase()) {
@@ -38,7 +64,7 @@ public class ComplaintEntityHandler implements RequestHandler {
case "get": case "get":
return handleGet(msgId, complaintRequest); return handleGet(msgId, complaintRequest);
case "get_all": case "get_all":
return handleGetAll(msgId, complaintRequest); return handleGetAll(msgId, complaintRequest, emails);
case "get_by_violation": case "get_by_violation":
return handleGetByViolation(msgId, complaintRequest); return handleGetByViolation(msgId, complaintRequest);
case "update_status": case "update_status":
@@ -74,14 +100,15 @@ public class ComplaintEntityHandler implements RequestHandler {
return successResponse(msgId, "Complaint retrieved successfully", response); return successResponse(msgId, "Complaint retrieved successfully", response);
} }
private BaseResponse handleGetAll(Integer msgId, ComplaintRequest req) { private BaseResponse handleGetAll(Integer msgId, ComplaintRequest req, List<String> emails) {
Pageable pageable = PageRequest.of( Pageable pageable = PageRequest.of(
req.getPage() != null ? req.getPage() : 0, req.getPage() != null ? req.getPage() : 0,
req.getSize() != null ? req.getSize() : 5, req.getSize() != null ? req.getSize() : 5,
Sort.by(Sort.Direction.fromString(req.getSortDirection() != null ? req.getSortDirection() : "desc"), Sort.by(Sort.Direction.fromString(req.getSortDirection() != null ? req.getSortDirection() : "desc"),
req.getSortBy() != null ? req.getSortBy() : "updatedAt")); req.getSortBy() != null ? req.getSortBy() : "updatedAt"));
var page = complaintService.getAllComplaints(pageable); var page = complaintService.getAllComplaintsByEmail(pageable, emails);
Map<String, Object> data = new HashMap<>(); Map<String, Object> data = new HashMap<>();
data.put("content", page.getContent()); data.put("content", page.getContent());
data.put("page", page.getNumber()); data.put("page", page.getNumber());
@@ -18,7 +18,6 @@ import ru.soune.nocopy.entity.file.moderation.ModerationLog;
import ru.soune.nocopy.exception.*; import ru.soune.nocopy.exception.*;
import ru.soune.nocopy.repository.FileAppealRepository; import ru.soune.nocopy.repository.FileAppealRepository;
import ru.soune.nocopy.repository.FileEntityRepository; import ru.soune.nocopy.repository.FileEntityRepository;
import ru.soune.nocopy.service.file.cloud.CloudStorageService;
import ru.soune.nocopy.repository.ModerationLogRepository; import ru.soune.nocopy.repository.ModerationLogRepository;
import ru.soune.nocopy.service.file.moderation.ModerationService; import ru.soune.nocopy.service.file.moderation.ModerationService;
import ru.soune.nocopy.service.register.AuthService; import ru.soune.nocopy.service.register.AuthService;
@@ -44,8 +43,6 @@ public class FileEntityHandler implements RequestHandler {
private final FileEntityRepository fileEntityRepository; private final FileEntityRepository fileEntityRepository;
private final CloudStorageService cloudStorageService;
private final ModerationService moderationService; private final ModerationService moderationService;
private final FileAppealRepository fileAppealRepository; private final FileAppealRepository fileAppealRepository;
@@ -703,7 +700,7 @@ public class FileEntityHandler implements RequestHandler {
int fullDelete = fileRequest.getFullDelete() == null ? 0 : fileRequest.getFullDelete(); int fullDelete = fileRequest.getFullDelete() == null ? 0 : fileRequest.getFullDelete();
if (fileEntity.getStatus().equals(FileStatus.DELETED) || fullDelete == 1) { if (fileEntity.getStatus().equals(FileStatus.DELETED) || fullDelete == 1) {
cloudStorageService.deleteFromStorageDisk(fileEntity); fileEntityService.deleteFromDisk(fileEntity);
response = DeleteFileResponse.builder() response = DeleteFileResponse.builder()
.fileId(fileRequest.getFileId()) .fileId(fileRequest.getFileId())
.message("File deleted from disk") .message("File deleted from disk")
@@ -6,12 +6,21 @@ import lombok.extern.slf4j.Slf4j;
import org.springframework.stereotype.Component; import org.springframework.stereotype.Component;
import ru.soune.nocopy.dto.BaseRequest; import ru.soune.nocopy.dto.BaseRequest;
import ru.soune.nocopy.dto.BaseResponse; import ru.soune.nocopy.dto.BaseResponse;
import ru.soune.nocopy.dto.MessageCode;
import ru.soune.nocopy.dto.search.GlobalSearchStatisticsRequest; import ru.soune.nocopy.dto.search.GlobalSearchStatisticsRequest;
import ru.soune.nocopy.dto.search.GlobalSearchStatisticsResponse; import ru.soune.nocopy.dto.search.GlobalSearchStatisticsResponse;
import ru.soune.nocopy.entity.company.Company;
import ru.soune.nocopy.entity.user.AuthToken; import ru.soune.nocopy.entity.user.AuthToken;
import ru.soune.nocopy.entity.user.User;
import ru.soune.nocopy.repository.AuthTokenRepository; import ru.soune.nocopy.repository.AuthTokenRepository;
import ru.soune.nocopy.repository.UserRepository;
import ru.soune.nocopy.service.CompanyService;
import ru.soune.nocopy.service.search.GlobalSearchStatisticsService; import ru.soune.nocopy.service.search.GlobalSearchStatisticsService;
import ru.soune.nocopy.service.user.UserService;
import java.util.ArrayList;
import java.util.List;
import java.util.Map;
import java.util.Optional; import java.util.Optional;
@Slf4j @Slf4j
@@ -17,6 +17,7 @@ import ru.soune.nocopy.entity.complaint.LawCasePriority;
import ru.soune.nocopy.entity.complaint.LawCaseType; import ru.soune.nocopy.entity.complaint.LawCaseType;
import ru.soune.nocopy.entity.tokenoperation.OperationType; import ru.soune.nocopy.entity.tokenoperation.OperationType;
import ru.soune.nocopy.entity.user.User; import ru.soune.nocopy.entity.user.User;
import ru.soune.nocopy.entity.violation.Violation;
import ru.soune.nocopy.exception.NotValidFieldException; import ru.soune.nocopy.exception.NotValidFieldException;
import ru.soune.nocopy.exception.TariffNotFoundException; import ru.soune.nocopy.exception.TariffNotFoundException;
import ru.soune.nocopy.exception.UserNotFoundException; import ru.soune.nocopy.exception.UserNotFoundException;
@@ -25,8 +26,10 @@ import ru.soune.nocopy.service.complaint.LawCaseService;
import ru.soune.nocopy.service.register.AuthService; import ru.soune.nocopy.service.register.AuthService;
import ru.soune.nocopy.service.tariff.TariffConstants; import ru.soune.nocopy.service.tariff.TariffConstants;
import ru.soune.nocopy.service.tariff.TariffInfoService; import ru.soune.nocopy.service.tariff.TariffInfoService;
import ru.soune.nocopy.service.violation.ViolationService;
import java.io.IOException; import java.io.IOException;
import java.util.ArrayList;
import java.util.List; import java.util.List;
import java.util.Map; import java.util.Map;
import java.util.stream.Collectors; import java.util.stream.Collectors;
@@ -46,6 +49,8 @@ public class LawCaseHandler implements RequestHandler {
private final TariffInfoService tariffInfoService; private final TariffInfoService tariffInfoService;
private final ViolationService violationService;
@Override @Override
public BaseResponse handle(BaseRequest request) throws Exception { public BaseResponse handle(BaseRequest request) throws Exception {
@@ -84,9 +89,17 @@ public class LawCaseHandler implements RequestHandler {
lawCaseRequest.getSortBy(), lawCaseRequest.getSortBy(),
lawCaseRequest.getSortDir()); lawCaseRequest.getSortDir());
List<LawCaseResponse> responses = lawCases.getContent().stream() List<LawCaseResponse> responses = new ArrayList<>();
.map(LawCaseResponse::fromEntity) List<LawCase> content = lawCases.getContent();
.collect(Collectors.toList());
for (LawCase lawCase: content) {
LawCaseResponse lawCaseResponse = LawCaseResponse.fromEntity(lawCase);
Violation violation = violationService.getById(lawCase.getViolationId());
if (violation != null) lawCaseResponse.setFileId(violation.getFileEntity().getId());
responses.add(lawCaseResponse);
}
PaginatedResponse<LawCaseResponse> paginatedResponse = new PaginatedResponse<>( PaginatedResponse<LawCaseResponse> paginatedResponse = new PaginatedResponse<>(
responses, responses,
@@ -0,0 +1,72 @@
package ru.soune.nocopy.handler;
import com.fasterxml.jackson.databind.ObjectMapper;
import lombok.RequiredArgsConstructor;
import lombok.extern.slf4j.Slf4j;
import org.springframework.stereotype.Component;
import ru.soune.nocopy.dto.BaseRequest;
import ru.soune.nocopy.dto.BaseResponse;
import ru.soune.nocopy.dto.MessageCode;
import ru.soune.nocopy.dto.statistic.MonitoringStatisticRequest;
import ru.soune.nocopy.dto.statistic.MonitoringStatisticResponse;
import ru.soune.nocopy.entity.company.Company;
import ru.soune.nocopy.entity.monitoring.MonitoringType;
import ru.soune.nocopy.entity.user.User;
import ru.soune.nocopy.repository.FileMonitoringRepository;
import ru.soune.nocopy.repository.UserRepository;
import ru.soune.nocopy.service.register.AuthService;
import java.util.List;
@Slf4j
@Component
@RequiredArgsConstructor
public class MonitoringStatisticHandler implements RequestHandler {
private final ObjectMapper objectMapper;
private final AuthService authService;
private final FileMonitoringRepository fileMonitoringRepository;
private final UserRepository userRepository;
@Override
public BaseResponse handle(BaseRequest request) throws Exception {
MonitoringStatisticRequest monitoringStatisticRequest = objectMapper.convertValue(request.getMessageBody(),
MonitoringStatisticRequest.class);
try {
String authToken = monitoringStatisticRequest.getAuthToken();
Long userId = authService.useUserAuthToken(authToken);
User user = userRepository.findById(userId).orElseThrow();
Company company = user.getCompany();
List<Long> userIds = company == null ? List.of(userId) :
user.getCompany()
.getUsers()
.stream()
.map(User::getId).toList();
MonitoringStatisticResponse monitoringStatisticResponse = MonitoringStatisticResponse.builder()
.everyDay(fileMonitoringRepository.countByUserIdsAndMonitoringType(userIds,
MonitoringType.MONITORING_DAILY))
.everyMonth(fileMonitoringRepository.countByUserIdsAndMonitoringType(userIds,
MonitoringType.MONITORING_MONTHLY))
.everyWeekly(fileMonitoringRepository.countByUserIdsAndMonitoringType(userIds,
MonitoringType.MONITORING_WEEKLY))
.build();
return new BaseResponse(
request.getMsgId(),
MessageCode.SUCCESS.getCode(),
MessageCode.SUCCESS.getDescription(),
monitoringStatisticResponse);
} catch (Exception e) {
return new BaseResponse(
request.getMsgId(),
MessageCode.INVALID_JSON_BODY.getCode(),
MessageCode.INVALID_JSON_BODY.getDescription(),
null);
}
}
}
@@ -25,6 +25,7 @@ import java.time.format.DateTimeParseException;
import java.util.ArrayList; import java.util.ArrayList;
import java.util.List; import java.util.List;
import java.util.Map; import java.util.Map;
import java.util.Objects;
@Component @Component
@Slf4j @Slf4j
@@ -115,11 +116,21 @@ public class ViolationHandler implements RequestHandler {
violationRequest.getPage(), violationRequest.getSize(), violationRequest.getSortDirection() violationRequest.getPage(), violationRequest.getSize(), violationRequest.getSortDirection()
); );
} }
ViolationResponse.ViolationDto firstShowViolation = null;
List<Violation> content = violationPage.getContent(); List<Violation> content = violationPage.getContent();
List<ViolationResponse.ViolationDto> violationDtos = new ArrayList<>(); List<ViolationResponse.ViolationDto> violationDtos = new ArrayList<>();
if ("getByViolationId".equals(violationRequest.getAction())) {
Violation violation = violationService.getById(violationRequest.getViolationId());
if (violation != null) {
firstShowViolation = ViolationResponse.ViolationDto.fromEntity(violation);
violationDtos.add(firstShowViolation);
}
}
for (Violation violation : content) { for (Violation violation : content) {
if (firstShowViolation != null && Objects.equals(firstShowViolation.getId(), violation.getId())) continue;
ViolationResponse.ViolationDto violationDto = ViolationResponse.ViolationDto.fromEntity(violation); ViolationResponse.ViolationDto violationDto = ViolationResponse.ViolationDto.fromEntity(violation);
violationDto.setCountry(geoCountryService.getCountryName(violation.getPageUrl())); violationDto.setCountry(geoCountryService.getCountryName(violation.getPageUrl()));
violationDto.setCountryCode(geoCountryService.getCountryCode(violation.getPageUrl())); violationDto.setCountryCode(geoCountryService.getCountryCode(violation.getPageUrl()));
@@ -5,6 +5,7 @@ import org.springframework.data.domain.Pageable;
import org.springframework.data.jpa.repository.JpaRepository; import org.springframework.data.jpa.repository.JpaRepository;
import org.springframework.data.jpa.repository.Modifying; import org.springframework.data.jpa.repository.Modifying;
import org.springframework.data.jpa.repository.Query; import org.springframework.data.jpa.repository.Query;
import org.springframework.data.repository.query.Param;
import org.springframework.stereotype.Repository; import org.springframework.stereotype.Repository;
import org.springframework.transaction.annotation.Transactional; import org.springframework.transaction.annotation.Transactional;
import ru.soune.nocopy.entity.complaint.ComplaintEntity; import ru.soune.nocopy.entity.complaint.ComplaintEntity;
@@ -37,4 +38,7 @@ public interface ComplaintEntityRepository extends JpaRepository<ComplaintEntity
) )
""") """)
Long getComplaintsWithoutLawCaseCount(); Long getComplaintsWithoutLawCaseCount();
@Query("SELECT c FROM ComplaintEntity c WHERE c.email IN :emails")
Page<ComplaintEntity> getComplaintsByUserEmails(@Param("emails") List<String> emails, Pageable pageable);
} }
@@ -60,6 +60,12 @@ public interface FileEntityRepository extends JpaRepository<FileEntity, String>
List<FileEntity> findFileByUserIdAndStatus(Long userId, FileStatus status); List<FileEntity> findFileByUserIdAndStatus(Long userId, FileStatus status);
@Query("SELECT f FROM FileEntity f WHERE f.userId = :userId AND f.mimeType = :mimeType")
List<FileEntity> findByUserIdAndMimeType(Long userId, String mimeType);
@Query("SELECT f FROM FileEntity f WHERE f.mimeType = :mimeType")
List<FileEntity> findByMimeType(String mimeType);
@Query("SELECT f FROM FileEntity f WHERE f.mimeType = :mimeType AND f.userId IN :userIds") @Query("SELECT f FROM FileEntity f WHERE f.mimeType = :mimeType AND f.userId IN :userIds")
List<FileEntity> findByMimeTypeAndUserIds(String mimeType, List<Long> userIds); List<FileEntity> findByMimeTypeAndUserIds(String mimeType, List<Long> userIds);
@@ -22,4 +22,11 @@ public interface FileMonitoringRepository extends JpaRepository<FileMonitoringEn
List<FileMonitoringEntity> findReadyForRun(@Param("now") LocalDateTime now); List<FileMonitoringEntity> findReadyForRun(@Param("now") LocalDateTime now);
List<FileMonitoringEntity> findByMonitoringTypeAndIsActiveTrue(MonitoringType type); List<FileMonitoringEntity> findByMonitoringTypeAndIsActiveTrue(MonitoringType type);
@Query("SELECT COUNT(m) FROM FileMonitoringEntity m " +
"WHERE m.userId IN :userIds " +
"AND m.monitoringType = :monitoringType " +
"AND m.isActive = true")
long countByUserIdsAndMonitoringType(@Param("userIds") List<Long> userIds,
@Param("monitoringType") MonitoringType monitoringType);
} }
@@ -45,4 +45,7 @@ public interface UserRepository extends JpaRepository<User, Long> {
@Query("SELECT u FROM User u WHERE u.personalTariffInfo.id = :tariffInfoId") @Query("SELECT u FROM User u WHERE u.personalTariffInfo.id = :tariffInfoId")
Optional<User> findByPersonalTariffInfoId(@Param("tariffInfoId") String tariffInfoId); Optional<User> findByPersonalTariffInfoId(@Param("tariffInfoId") String tariffInfoId);
@Query("SELECT u.email FROM User u WHERE u.company = :company")
List<String> emailsByCompany(@Param("company") String company);
} }
@@ -17,7 +17,6 @@ import ru.soune.nocopy.entity.file.ImageHashEntity;
import ru.soune.nocopy.entity.user.User; import ru.soune.nocopy.entity.user.User;
import ru.soune.nocopy.exception.DuplicateImageException; import ru.soune.nocopy.exception.DuplicateImageException;
import ru.soune.nocopy.repository.*; import ru.soune.nocopy.repository.*;
import ru.soune.nocopy.service.file.cloud.CloudStorageService;
import ru.soune.nocopy.util.FileUtil; import ru.soune.nocopy.util.FileUtil;
import java.io.File; import java.io.File;
@@ -42,8 +41,6 @@ public class FileSimilarityService {
private final FileEntityRepository fileEntityRepository; private final FileEntityRepository fileEntityRepository;
private final CloudStorageService cloudStorageService;
@Value("${server.baseurl}") @Value("${server.baseurl}")
private String baseUrl; private String baseUrl;
@@ -117,7 +114,7 @@ public class FileSimilarityService {
} }
public FileEntity findDuplicateByHash(String path, String mimeType, Long userId) throws Exception { public FileEntity findDuplicateByHash(String path, String mimeType, Long userId) throws Exception {
String hash = calculateFileHash(path, false); String hash = calculateFileHash(path);
User user = userRepository.findById(userId).orElseThrow(); User user = userRepository.findById(userId).orElseThrow();
Company company = user.getCompany(); Company company = user.getCompany();
@@ -132,15 +129,15 @@ public class FileSimilarityService {
List<FileEntity> fileEntityList = fileEntityRepository.findByMimeTypeAndUserIds(mimeType, userIds); List<FileEntity> fileEntityList = fileEntityRepository.findByMimeTypeAndUserIds(mimeType, userIds);
for (FileEntity file : fileEntityList) { for (FileEntity file : fileEntityList) {
// if (file.getProtectedFilePath() == null) continue; if (file.getProtectedFilePath() == null) continue;
// File existingFile = cloudStorageService.readFileFromStorageByPath(file.getFilePath()); File existingFile = new File(file.getProtectedFilePath());
// File newFile = new File(path); File newFile = new File(path);
// if (existingFile.length() == newFile.length()) { if (existingFile.length() == newFile.length()) {
if (calculateFileHash(file.getFilePath(), true).equals(hash)) { if (calculateFileHash(file.getProtectedFilePath()).equals(hash)) {
return file; return file;
} }
// } }
} }
return null; return null;
@@ -199,13 +196,9 @@ public class FileSimilarityService {
} }
private String calculateFileHash(String path, boolean cloud) throws Exception { private String calculateFileHash(String path) throws Exception {
MessageDigest digest = MessageDigest.getInstance("SHA-256"); MessageDigest digest = MessageDigest.getInstance("SHA-256");
if (cloud) {
byte[] data = cloudStorageService.readFileFromStorageBytes(path);
digest.update(data);
} else {
try (InputStream is = new FileInputStream(path)) { try (InputStream is = new FileInputStream(path)) {
byte[] buffer = new byte[8192]; byte[] buffer = new byte[8192];
int read; int read;
@@ -213,7 +206,6 @@ public class FileSimilarityService {
digest.update(buffer, 0, read); digest.update(buffer, 0, read);
} }
} }
}
byte[] hashBytes = digest.digest(); byte[] hashBytes = digest.digest();
return Base64.getEncoder().encodeToString(hashBytes); return Base64.getEncoder().encodeToString(hashBytes);
@@ -8,11 +8,9 @@ import org.springframework.stereotype.Service;
import ru.soune.nocopy.entity.file.FileEntity; import ru.soune.nocopy.entity.file.FileEntity;
import ru.soune.nocopy.entity.file.ImageHashEntity; import ru.soune.nocopy.entity.file.ImageHashEntity;
import ru.soune.nocopy.repository.ImageHashRepository; import ru.soune.nocopy.repository.ImageHashRepository;
import ru.soune.nocopy.service.file.cloud.CloudStorageService;
import java.io.File; import java.io.File;
import java.io.IOException; import java.io.IOException;
import java.nio.file.Files;
import java.nio.file.Path; import java.nio.file.Path;
import java.time.LocalDateTime; import java.time.LocalDateTime;
import java.util.Map; import java.util.Map;
@@ -24,50 +22,18 @@ public class ImageHashService {
private final ImageHashRepository repository; private final ImageHashRepository repository;
private final CloudStorageService cloudStorageService;
// public Map<String, Long> calculateHash(Path imagePath) throws IOException {
//// File file = cloudStorageService.readFileFromStorageByPath(imagePath.toString());
// File file = imagePath.toFile();
// if (file == null) {
// file = cloudStorageService.readFileFromStorageByPath(imagePath.toString());
// }
//
// PHash pHash = PerceptualHashHelper.INSTANCE.generateDCTPerceptualHash(file);
//
// Long firstPart = pHash.getFirstPart();
// Long secondPart = pHash.getSecondPart();
// Map<String, Long> hash = Map.of(
// "hi", firstPart,
// "low", secondPart);
//
// return hash;
// }
public Map<String, Long> calculateHash(Path imagePath) throws IOException { public Map<String, Long> calculateHash(Path imagePath) throws IOException {
PHash pHash; File file = imagePath.toFile();
if (imagePath.toFile().exists()) { PHash pHash = PerceptualHashHelper.INSTANCE.generateDCTPerceptualHash(file);
pHash = PerceptualHashHelper.INSTANCE.generateDCTPerceptualHash(imagePath.toFile());
} else {
byte[] data = cloudStorageService.readFileFromStorageBytes(imagePath.toString());
Path tempFile = Files.createTempFile("perceptual_hash_", ".tmp");
try {
Files.write(tempFile, data);
pHash = PerceptualHashHelper.INSTANCE.generateDCTPerceptualHash(tempFile.toFile());
} finally {
Files.deleteIfExists(tempFile);
}
}
Long firstPart = pHash.getFirstPart(); Long firstPart = pHash.getFirstPart();
Long secondPart = pHash.getSecondPart(); Long secondPart = pHash.getSecondPart();
Map<String, Long> hash = Map.of(
return Map.of(
"hi", firstPart, "hi", firstPart,
"low", secondPart "low", secondPart);
);
return hash;
} }
public void create(FileEntity file, Map<String, Long> stringIntegerMap) { public void create(FileEntity file, Map<String, Long> stringIntegerMap) {
@@ -62,6 +62,12 @@ public class ComplaintEntityService {
return mapToResponse(complaintRepository.save(complaint)); return mapToResponse(complaintRepository.save(complaint));
} }
@Transactional(readOnly = true)
public Page<ComplaintResponse> getAllComplaintsByEmail(Pageable pageable, List<String> emails) {
return complaintRepository.getComplaintsByUserEmails(emails, pageable)
.map(this::mapToResponse);
}
@Transactional(readOnly = true) @Transactional(readOnly = true)
public ComplaintResponse getComplaintById(Long id) { public ComplaintResponse getComplaintById(Long id) {
return complaintRepository.findById(id) return complaintRepository.findById(id)
@@ -121,13 +127,15 @@ public class ComplaintEntityService {
} }
private ComplaintResponse mapToResponse(ComplaintEntity entity) { private ComplaintResponse mapToResponse(ComplaintEntity entity) {
Violation violation = entity.getViolation();
return ComplaintResponse.builder() return ComplaintResponse.builder()
.id(entity.getId()) .id(entity.getId())
.status(entity.getStatus() != null ? entity.getStatus().name() : null) .status(entity.getStatus() != null ? entity.getStatus().name() : null)
.complaintText(entity.getComplaintText()) .complaintText(entity.getComplaintText())
.email(entity.getEmail()) .email(entity.getEmail())
.violationId(entity.getViolation() != null ? entity.getViolation().getId() : null) .violationId(violation != null ? violation.getId() : null)
.fileId(violation != null ? violation.getFileEntity().getId() : null)
.createdAt(entity.getCreatedAt() != null ? entity.getCreatedAt().format(DATE_FORMATTER) : null) .createdAt(entity.getCreatedAt() != null ? entity.getCreatedAt().format(DATE_FORMATTER) : null)
.updatedAt(entity.getUpdatedAt() != null ? entity.getUpdatedAt().format(DATE_FORMATTER) : null) .updatedAt(entity.getUpdatedAt() != null ? entity.getUpdatedAt().format(DATE_FORMATTER) : null)
.notModerated(entity.getNot_moderated_file() != null ? entity.getNot_moderated_file() : null) .notModerated(entity.getNot_moderated_file() != null ? entity.getNot_moderated_file() : null)
@@ -8,7 +8,6 @@ import lombok.extern.slf4j.Slf4j;
import ru.soune.nocopy.entity.user.ProtectedFileCheck; import ru.soune.nocopy.entity.user.ProtectedFileCheck;
import ru.soune.nocopy.repository.ProtectedFileCheckRepository; import ru.soune.nocopy.repository.ProtectedFileCheckRepository;
import java.io.File;
import java.io.IOException; import java.io.IOException;
import java.nio.file.*; import java.nio.file.*;
import java.nio.file.attribute.BasicFileAttributes; import java.nio.file.attribute.BasicFileAttributes;
@@ -16,10 +15,7 @@ import java.time.Instant;
import java.time.LocalDate; import java.time.LocalDate;
import java.time.LocalDateTime; import java.time.LocalDateTime;
import java.time.ZoneId; import java.time.ZoneId;
import java.time.temporal.ChronoUnit;
import java.util.List; import java.util.List;
import java.util.stream.Collectors;
import java.util.stream.Stream;
@Slf4j @Slf4j
@Component @Component
@@ -38,8 +34,6 @@ public class FileCleanupService {
public void cleanupExpiredFiles() { public void cleanupExpiredFiles() {
log.info("Starting cleanup of expired temporary files"); log.info("Starting cleanup of expired temporary files");
cleanupTempFilesNightly();
Path tempDir = Paths.get(basePath, "temp"); Path tempDir = Paths.get(basePath, "temp");
if (!Files.exists(tempDir)) { if (!Files.exists(tempDir)) {
return; return;
@@ -89,50 +83,4 @@ public class FileCleanupService {
log.error("Error during cleanup", e); log.error("Error during cleanup", e);
} }
} }
private void cleanupTempFilesNightly() {
log.info("Starting nightly cleanup of temporary files");
String tempDir = System.getProperty("java.io.tmpdir");
Path tempPath = Paths.get(tempDir);
if (!Files.exists(tempPath)) {
return;
}
Instant cutoffTime = Instant.now().minus(24, ChronoUnit.HOURS); // 24 часа
int deletedCount = 0;
try (Stream<Path> files = Files.list(tempPath)) {
List<Path> tempFiles = files
.filter(path -> {
String filename = path.getFileName().toString();
return filename.contains("_") &&
!filename.endsWith(".tmp") &&
Files.isRegularFile(path);
})
.collect(Collectors.toList());
for (Path filePath : tempFiles) {
try {
File file = filePath.toFile();
Instant lastModified = Instant.ofEpochMilli(file.lastModified());
if (lastModified.isBefore(cutoffTime)) {
boolean deleted = Files.deleteIfExists(filePath);
if (deleted) {
deletedCount++;
}
}
} catch (IOException e) {
log.error("Error deleting file: {}", filePath, e);
}
}
log.info("Nightly cleanup deleted {} temporary files", deletedCount);
} catch (IOException e) {
log.error("Failed to list temp directory", e);
}
}
} }
@@ -19,15 +19,12 @@ import ru.soune.nocopy.exception.FileEntityNotFoundException;
import ru.soune.nocopy.repository.FileEntityRepository; import ru.soune.nocopy.repository.FileEntityRepository;
import ru.soune.nocopy.repository.FileMonitoringRepository; import ru.soune.nocopy.repository.FileMonitoringRepository;
import ru.soune.nocopy.repository.UserRepository; import ru.soune.nocopy.repository.UserRepository;
import ru.soune.nocopy.service.file.cloud.CloudStorageService;
import java.io.File;
import java.io.IOException; import java.io.IOException;
import java.nio.ByteBuffer;
import java.nio.channels.FileChannel;
import java.nio.file.Files; import java.nio.file.Files;
import java.nio.file.Path; import java.nio.file.Path;
import java.nio.file.Paths; import java.nio.file.Paths;
import java.nio.file.StandardOpenOption;
import java.time.LocalDateTime; import java.time.LocalDateTime;
import java.util.ArrayList; import java.util.ArrayList;
import java.util.List; import java.util.List;
@@ -45,8 +42,6 @@ public class FileEntityService {
private final ImageResizeService imageResizeService; private final ImageResizeService imageResizeService;
private final CloudStorageService cloudStorageService;
@Value("${server.baseurl}") @Value("${server.baseurl}")
private String baseUrl; private String baseUrl;
@@ -207,8 +202,16 @@ public class FileEntityService {
fileEntityRepository.save(fileEntity); fileEntityRepository.save(fileEntity);
} }
public void deleteFromDisk(String deleteFilePath) throws IOException { public void deleteFromDisk(FileEntity fileEntity) throws IOException {
Files.deleteIfExists(Paths.get(deleteFilePath)); Path path = Paths.get(fileEntity.getFilePath());
if (!Files.exists(path)) {
return;
}
Files.delete(path);
fileEntityRepository.delete(fileEntity);
} }
@Transactional(readOnly = true) @Transactional(readOnly = true)
@@ -234,28 +237,19 @@ public class FileEntityService {
} }
@Transactional @Transactional
public FileEntity writeProtectedFile(String id, byte[] data, String fileExt) throws IOException { public void writeProtectedFile(String id, byte[] data, String fileExt) throws IOException {
FileEntity fileEntity = fileEntityRepository.findById(id) FileEntity fileEntity = fileEntityRepository.findById(id)
.orElseThrow(() -> new RuntimeException("File not found: " + id)); .orElseThrow(() -> new RuntimeException("File not found: " + id));
String extension = determineFileExtension(fileExt, fileEntity); String extension = determineFileExtension(fileExt, fileEntity);
Path protectedFilePath = prepareProtectedPath(fileEntity, extension); Path protectedFilePath = prepareProtectedPath(fileEntity, extension);
Files.createDirectories(protectedFilePath.getParent()); if (Files.exists(protectedFilePath)) {
Files.delete(protectedFilePath);
Files.deleteIfExists(protectedFilePath);
try (FileChannel channel = FileChannel.open(protectedFilePath,
StandardOpenOption.CREATE,
StandardOpenOption.WRITE)) {
ByteBuffer buffer = ByteBuffer.wrap(data);
while (buffer.hasRemaining()) {
channel.write(buffer);
}
channel.force(true);
} }
Files.write(protectedFilePath, data);
if (imageResizeService.isImage(extension)) { if (imageResizeService.isImage(extension)) {
imageResizeService.generateSizes(fileEntity, data); imageResizeService.generateSizes(fileEntity, data);
} }
@@ -266,14 +260,14 @@ public class FileEntityService {
fileEntity.setFileExtension(extension); fileEntity.setFileExtension(extension);
fileEntity.setProtectionStatus(ProtectionStatus.PROTECTED); fileEntity.setProtectionStatus(ProtectionStatus.PROTECTED);
return fileEntityRepository.save(fileEntity); fileEntityRepository.save(fileEntity);
} }
public void clearTempFiles(long userId) { public void clearTempFiles(long userId) throws IOException {
List<FileEntity> fileByUserIdAndStatus = fileEntityRepository.findFileByUserIdAndStatus(userId, FileStatus.TEMP); List<FileEntity> fileByUserIdAndStatus = fileEntityRepository.findFileByUserIdAndStatus(userId, FileStatus.TEMP);
for (FileEntity fileEntity : fileByUserIdAndStatus) { for (FileEntity fileEntity : fileByUserIdAndStatus) {
cloudStorageService.deleteFromStorageDisk(fileEntity); deleteFromDisk(fileEntity);
} }
} }
@@ -285,6 +279,24 @@ public class FileEntityService {
return fileEntityRepository.findFileIdByFilePath(filePath); return fileEntityRepository.findFileIdByFilePath(filePath);
} }
public File getFileById(String id) {
try {
FileEntity fileEntity = fileEntityRepository.findById(id).orElseThrow(() ->
new RuntimeException("File not found: " + id));
File file = new File(fileEntity.getFilePath());
if (!file.exists()) {
throw new RuntimeException("File not found on disk: " + fileEntity.getFilePath());
}
return file;
} catch (Exception e) {
log.error("Error getting file: {}", id, e);
return null;
}
}
public Path prepareProtectedPath(FileEntity fileEntity, String extension) throws IOException { public Path prepareProtectedPath(FileEntity fileEntity, String extension) throws IOException {
Path originalPath = Paths.get(fileEntity.getFilePath()); Path originalPath = Paths.get(fileEntity.getFilePath());
@@ -310,7 +322,7 @@ public class FileEntityService {
return protectedPath; return protectedPath;
} }
public FileEntity updateSignature(String signature, String fileId) throws IOException { public void updateSignature(String signature, String fileId) throws IOException {
FileEntity fileEntity = fileEntityRepository.findByFileId(fileId); FileEntity fileEntity = fileEntityRepository.findByFileId(fileId);
fileEntity.setSignature(signature); fileEntity.setSignature(signature);
fileEntity.setProtectionStatus(ProtectionStatus.PROTECTED); fileEntity.setProtectionStatus(ProtectionStatus.PROTECTED);
@@ -318,30 +330,18 @@ public class FileEntityService {
fileEntity.setProtectedAt(LocalDateTime.now()); fileEntity.setProtectedAt(LocalDateTime.now());
fileEntity.setProtectedFilePath(prepareProtectedPath(fileEntity, fileEntity.getFileExtension()).toString()); fileEntity.setProtectedFilePath(prepareProtectedPath(fileEntity, fileEntity.getFileExtension()).toString());
return fileEntityRepository.save(fileEntity); fileEntityRepository.save(fileEntity);
} }
public String formatFileSize(long size) { private boolean checkFileExistsOnDisk(String filePath) {
if (size < 1024) { try {
return size + " B"; return Files.exists(Paths.get(filePath));
} else if (size < 1024 * 1024) { } catch (Exception e) {
return String.format("%.1f KB", size / 1024.0); log.warn("Error checking file existence: {}", filePath, e);
} else if (size < 1024 * 1024 * 1024) { return false;
return String.format("%.1f MB", size / (1024.0 * 1024.0));
} else {
return String.format("%.1f GB", size / (1024.0 * 1024.0 * 1024.0));
} }
} }
// private boolean checkFileExistsOnDisk(String filePath) {
// try {
// return Files.exists(Paths.get(filePath));
// } catch (Exception e) {
// log.warn("Error checking file existence: {}", filePath, e);
// return false;
// }
// }
private String determineFileExtension(String fileExt, FileEntity fileEntity) { private String determineFileExtension(String fileExt, FileEntity fileEntity) {
if (fileExt != null) { if (fileExt != null) {
return fileExt; return fileExt;
@@ -353,7 +353,7 @@ public class FileEntityService {
private final FileMonitoringRepository fileMonitoringRepository; private final FileMonitoringRepository fileMonitoringRepository;
private FileEntityResponse convertToResponse(FileEntity fileEntity, int version) { private FileEntityResponse convertToResponse(FileEntity fileEntity, int version) {
// boolean existsOnDisk = checkFileExistsOnDisk(fileEntity.getFilePath()); boolean existsOnDisk = checkFileExistsOnDisk(fileEntity.getFilePath());
User user = userRepository.findById(fileEntity.getUserId()).get(); User user = userRepository.findById(fileEntity.getUserId()).get();
String fileName = fileEntity.getOriginalFileName(); String fileName = fileEntity.getOriginalFileName();
String name = fileName; String name = fileName;
@@ -385,7 +385,7 @@ public class FileEntityService {
.updatedAt(fileEntity.getUpdatedAt()) .updatedAt(fileEntity.getUpdatedAt())
.formattedSize(formatFileSize(fileEntity.getFileSize())) .formattedSize(formatFileSize(fileEntity.getFileSize()))
.downloadUrl("/api/v" + version + "/files/download/" + fileEntity.getId()) .downloadUrl("/api/v" + version + "/files/download/" + fileEntity.getId())
// .existsOnDisk(existsOnDisk) .existsOnDisk(existsOnDisk)
.supportId(fileEntity.getSupportId()) .supportId(fileEntity.getSupportId())
.protectStatus(fileEntity.getProtectionStatus().toString()) .protectStatus(fileEntity.getProtectionStatus().toString())
.fileName(name + "_nocopy_protected" + "." + fileEntity.getFileExtension()) .fileName(name + "_nocopy_protected" + "." + fileEntity.getFileExtension())
@@ -400,4 +400,16 @@ public class FileEntityService {
.thumbnailFileUrl(baseUrl + "/api/files/protected/" + fileEntity.getId() + "/thumbnail") .thumbnailFileUrl(baseUrl + "/api/files/protected/" + fileEntity.getId() + "/thumbnail")
.build(); .build();
} }
public String formatFileSize(long size) {
if (size < 1024) {
return size + " B";
} else if (size < 1024 * 1024) {
return String.format("%.1f KB", size / 1024.0);
} else if (size < 1024 * 1024 * 1024) {
return String.format("%.1f MB", size / (1024.0 * 1024.0));
} else {
return String.format("%.1f GB", size / (1024.0 * 1024.0 * 1024.0));
}
}
} }
@@ -2,10 +2,11 @@ package ru.soune.nocopy.service.file;
import org.springframework.web.multipart.MultipartFile; import org.springframework.web.multipart.MultipartFile;
import ru.soune.nocopy.dto.file.UploadProgressResponse; import ru.soune.nocopy.dto.file.UploadProgressResponse;
import ru.soune.nocopy.entity.file.FileEntity;
import ru.soune.nocopy.entity.file.FileStatus; import ru.soune.nocopy.entity.file.FileStatus;
import ru.soune.nocopy.entity.file.FileUploadSession; import ru.soune.nocopy.entity.file.FileUploadSession;
import java.io.IOException;
public interface FileUploadService { public interface FileUploadService {
FileUploadSession initUpload(Long userId, String fileName, FileUploadSession initUpload(Long userId, String fileName,
String fileType, String extension, String convertTo, long fileSize); String fileType, String extension, String convertTo, long fileSize);
@@ -21,5 +22,5 @@ public interface FileUploadService {
UploadProgressResponse getUploadProgress(String uploadId); UploadProgressResponse getUploadProgress(String uploadId);
FileEntity completeFileProcessing(FileUploadSession session, FileStatus status); void completeFileProcessingAsync(FileUploadSession session, FileStatus status);
} }
@@ -1,12 +1,10 @@
package ru.soune.nocopy.service.file; package ru.soune.nocopy.service.file;
import lombok.RequiredArgsConstructor;
import lombok.extern.slf4j.Slf4j; import lombok.extern.slf4j.Slf4j;
import org.springframework.beans.factory.annotation.Value; import org.springframework.beans.factory.annotation.Value;
import org.springframework.stereotype.Service; import org.springframework.stereotype.Service;
import org.springframework.transaction.annotation.Transactional; import org.springframework.transaction.annotation.Transactional;
import ru.soune.nocopy.entity.file.FileEntity; import ru.soune.nocopy.entity.file.FileEntity;
import ru.soune.nocopy.service.file.cloud.CloudStorageService;
import javax.imageio.ImageIO; import javax.imageio.ImageIO;
import java.awt.*; import java.awt.*;
@@ -20,13 +18,9 @@ import java.util.List;
@Service @Service
@Slf4j @Slf4j
@RequiredArgsConstructor
public class ImageResizeService { public class ImageResizeService {
private final CloudStorageService cloudStorageService;
private static final int THUMBNAIL_SIZE = 150; private static final int THUMBNAIL_SIZE = 150;
private static final int MEDIUM_SIZE = 600; private static final int MEDIUM_SIZE = 600;
@Value("${file.storage.base-path}") @Value("${file.storage.base-path}")
@@ -56,15 +50,11 @@ public class ImageResizeService {
if (original.getWidth() > THUMBNAIL_SIZE) { if (original.getWidth() > THUMBNAIL_SIZE) {
String thumbPath = resizeAndSave(original, baseName + "_thumb", THUMBNAIL_SIZE); String thumbPath = resizeAndSave(original, baseName + "_thumb", THUMBNAIL_SIZE);
file.setThumbnailPath(thumbPath); file.setThumbnailPath(thumbPath);
cloudStorageService.saveFilesToStorage(file.getMimeType(), file.getFileExtension(),
thumbPath);
} }
if (original.getWidth() > MEDIUM_SIZE) { if (original.getWidth() > MEDIUM_SIZE) {
String mediumPath = resizeAndSave(original, baseName + "_medium", MEDIUM_SIZE); String mediumPath = resizeAndSave(original, baseName + "_medium", MEDIUM_SIZE);
file.setMediumPath(mediumPath); file.setMediumPath(mediumPath);
cloudStorageService.saveFilesToStorage(file.getMimeType(), file.getFileExtension(),
mediumPath);
} }
} }
} }
@@ -1,194 +0,0 @@
package ru.soune.nocopy.service.file.cloud;
import lombok.RequiredArgsConstructor;
import lombok.extern.slf4j.Slf4j;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.stereotype.Service;
import ru.soune.nocopy.entity.file.FileEntity;
import ru.soune.nocopy.repository.FileEntityRepository;
import software.amazon.awssdk.auth.credentials.AwsBasicCredentials;
import software.amazon.awssdk.auth.credentials.AwsCredentials;
import software.amazon.awssdk.auth.credentials.StaticCredentialsProvider;
import software.amazon.awssdk.core.ResponseBytes;
import software.amazon.awssdk.core.sync.RequestBody;
import software.amazon.awssdk.core.sync.ResponseTransformer;
import software.amazon.awssdk.http.apache.ApacheHttpClient;
import software.amazon.awssdk.regions.Region;
import software.amazon.awssdk.services.s3.S3Client;
import software.amazon.awssdk.services.s3.model.GetObjectRequest;
import software.amazon.awssdk.services.s3.model.GetObjectResponse;
import software.amazon.awssdk.services.s3.model.PutObjectRequest;
import java.io.File;
import java.io.IOException;
import java.io.InputStream;
import java.net.URI;
import java.nio.file.Files;
import java.nio.file.Paths;
import java.util.UUID;
@Service
@Slf4j
@RequiredArgsConstructor
public class CloudStorageService {
private final FileEntityRepository fileEntityRepository;
@Value("${yandex.cloud.key}")
private String key;
@Value("${yandex.cloud.secret-key}")
private String secretKey;
@Value("${yandex.cloud.region}")
private String region;
@Value("${yandex.cloud.s3-endpoint}")
private String s3endpoint;
@Value("${yandex.cloud.bucket}")
private String bucket;
public void saveFilesToStorage(String mimeType, String extension, String sendToCloudFilePath) {
AwsCredentials credentials = AwsBasicCredentials.create(key, secretKey);
S3Client s3Client = S3Client.builder()
.httpClient(ApacheHttpClient.create())
.region(Region.of(region))
.endpointOverride(URI.create(s3endpoint))
.credentialsProvider(StaticCredentialsProvider.create(credentials))
.build();
String mimeTypeFromFile = mimeType.equals("document") ? "application" : mimeType;
String contentType = mimeTypeFromFile + "/" + extension;
String filePath = "files" + sendToCloudFilePath;
PutObjectRequest putObjectRequest = PutObjectRequest.builder()
.bucket(bucket)
.key(filePath)
.contentType(contentType)
.build();
try {
byte[] bytes = Files.readAllBytes(Paths.get(sendToCloudFilePath));
s3Client.putObject(putObjectRequest,
RequestBody.fromBytes(bytes));
Files.deleteIfExists(Paths.get(sendToCloudFilePath));
} catch (IOException e) {
log.error("Failed to read file: {}", filePath, e);
}
}
public File readFileFromStorageByPath(String filePath) throws IOException {
GetObjectRequest objectRequest = GetObjectRequest.builder()
.bucket(bucket)
.key("files" + filePath)
.build();
AwsCredentials credentials = AwsBasicCredentials.create(key, secretKey);
S3Client s3Client = S3Client.builder()
.httpClient(ApacheHttpClient.create())
.region(Region.of(region))
.endpointOverride(URI.create(s3endpoint))
.credentialsProvider(StaticCredentialsProvider.create(credentials))
.build();
String tempDir = System.getProperty("java.io.tmpdir");
String fileName = System.currentTimeMillis() + "_" + UUID.randomUUID() + "_" +
filePath.replace("/", "_");
File tempFile = new File(tempDir, fileName);
s3Client.getObject(objectRequest, tempFile.toPath());
tempFile.deleteOnExit();
return tempFile;
}
public byte[] readFileFromStorageBytes(String filePath) {
AwsCredentials credentials = AwsBasicCredentials.create(key, secretKey);
S3Client s3Client = S3Client.builder()
.httpClient(ApacheHttpClient.create())
.region(Region.of(region))
.endpointOverride(URI.create(s3endpoint))
.credentialsProvider(StaticCredentialsProvider.create(credentials))
.build();
GetObjectRequest objectRequest = GetObjectRequest.builder()
.bucket(bucket)
.key("files" + filePath)
.build();
try {
ResponseBytes<GetObjectResponse> responseBytes = s3Client.getObjectAsBytes(objectRequest);
return responseBytes.asByteArray();
} catch (Exception e) {
log.error("Failed to read file from S3: {}", filePath, e);
throw new RuntimeException("Failed to read file from S3", e);
}
}
public InputStream readFileFromStorage(String filePath) {
GetObjectRequest objectRequest = GetObjectRequest.builder()
.bucket(bucket)
.key("files" + filePath)
.build();
AwsCredentials credentials = AwsBasicCredentials.create(key, secretKey);
S3Client s3Client = S3Client.builder()
.httpClient(ApacheHttpClient.create())
.region(Region.of(region))
.endpointOverride(URI.create(s3endpoint))
.credentialsProvider(StaticCredentialsProvider.create(credentials))
.build();
return s3Client.getObject(objectRequest, ResponseTransformer.toInputStream());
}
public void deleteFileFromStorage(String filePath) {
if (filePath == null || filePath.isEmpty()) {
log.warn("Attempted to delete file with empty path");
return;
}
AwsCredentials credentials = AwsBasicCredentials.create(key, secretKey);
S3Client s3Client = S3Client.builder()
.httpClient(ApacheHttpClient.create())
.region(Region.of(region))
.endpointOverride(URI.create(s3endpoint))
.credentialsProvider(StaticCredentialsProvider.create(credentials))
.build();
String cloudKey = "files" + filePath;
try {
try {
s3Client.headObject(builder -> builder.bucket(bucket).key(cloudKey).build());
} catch (Exception e) {
log.warn("File does not exist in cloud storage: {}", cloudKey);
return;
}
s3Client.deleteObject(builder -> builder.bucket(bucket).key(cloudKey).build());
log.info("File successfully deleted from cloud storage: {}", cloudKey);
} catch (Exception e) {
log.error("Failed to delete file from cloud storage: {}", cloudKey, e);
throw new RuntimeException("Failed to delete file from cloud storage: " + cloudKey, e);
}
}
public void deleteFromStorageDisk(FileEntity fileEntity) {
deleteFileFromStorage(fileEntity.getFilePath());
if (fileEntity.getProtectedFilePath() != null) {
deleteFileFromStorage(fileEntity.getProtectedFilePath());
}
fileEntityRepository.delete(fileEntity);
}
}
@@ -9,7 +9,6 @@ import org.jetbrains.annotations.Nullable;
import org.springframework.stereotype.Service; import org.springframework.stereotype.Service;
import ru.soune.nocopy.entity.file.FileEntity; import ru.soune.nocopy.entity.file.FileEntity;
import ru.soune.nocopy.service.file.FileEntityService; import ru.soune.nocopy.service.file.FileEntityService;
import ru.soune.nocopy.service.file.cloud.CloudStorageService;
import ru.soune.nocopy.util.FileUtil; import ru.soune.nocopy.util.FileUtil;
import java.io.IOException; import java.io.IOException;
@@ -23,8 +22,6 @@ public class AudioLocalSearchImpl implements AudioLocalSearch {
private final FileUtil fileUtil; private final FileUtil fileUtil;
private final CloudStorageService cloudStorageService;
@Override @Override
public @Nullable FileProtector.FileInfo findBySignature(@NotNull String signature) { public @Nullable FileProtector.FileInfo findBySignature(@NotNull String signature) {
FileEntity fileEntity = fileEntityService.findBySignature(signature); FileEntity fileEntity = fileEntityService.findBySignature(signature);
@@ -37,10 +34,7 @@ public class AudioLocalSearchImpl implements AudioLocalSearch {
@Override @Override
public void updateSignature(@NotNull String signature, @NotNull FileProtector.FileInfo fileInfo) { public void updateSignature(@NotNull String signature, @NotNull FileProtector.FileInfo fileInfo) {
try { try {
FileEntity fileEntity = fileEntityService.updateSignature(signature, fileInfo.getId()); fileEntityService.updateSignature(signature, fileInfo.getId());
cloudStorageService.saveFilesToStorage(fileEntity.getMimeType(), fileEntity.getFileExtension(),
fileEntity.getProtectedFilePath());
} catch (IOException e) { } catch (IOException e) {
throw new RuntimeException(e); throw new RuntimeException(e);
} }
@@ -25,23 +25,6 @@ public class DocumentLocalSearchImpl implements DocumentLocalSearch {
FileEntity fileEntity = fileEntityRepository.findByIdAndUserId(fileId, Long.valueOf(ownerId)); FileEntity fileEntity = fileEntityRepository.findByIdAndUserId(fileId, Long.valueOf(ownerId));
FileProtector.FileInfo fileInfo = fileUtil.createFileInfo(fileEntity, null); FileProtector.FileInfo fileInfo = fileUtil.createFileInfo(fileEntity, null);
log.info("fileInfo: {}", fileInfo); log.info("fileInfo: {}", fileInfo);
log.info("fileInfo: {}", fileInfo);
log.info("fileInfo: {}", fileInfo);
log.info("fileInfo: {}", fileInfo);
log.info("fileInfo: {}", fileInfo);
log.info("fileInfo: {}", fileInfo);
log.info("fileInfo: {}", fileInfo);
log.info("fileInfo: {}", fileInfo);
log.info("fileInfo: {}", fileInfo);
log.info("fileInfo: {}", fileInfo);
log.info("fileInfo: {}", fileInfo);
log.info("fileInfo: {}", fileInfo);
log.info("fileInfo: {}", fileInfo);
log.info("fileInfo: {}", fileInfo);
log.info("fileInfo: {}", fileInfo);
log.info("fileInfo: {}", fileInfo);
log.info("fileInfo: {}", fileInfo);
log.info("fileInfo: {}", fileInfo);
return fileInfo; return fileInfo;
} }
@@ -6,7 +6,6 @@ import com.vrt.fileprotection.NoCopyCheckResult;
import com.vrt.fileprotection.audio.AudioCheckResult; import com.vrt.fileprotection.audio.AudioCheckResult;
import com.vrt.fileprotection.documents.DocumentCheckResult; import com.vrt.fileprotection.documents.DocumentCheckResult;
import jakarta.annotation.PostConstruct; import jakarta.annotation.PostConstruct;
import jakarta.persistence.EntityManager;
import lombok.RequiredArgsConstructor; import lombok.RequiredArgsConstructor;
import lombok.extern.slf4j.Slf4j; import lombok.extern.slf4j.Slf4j;
import org.springframework.beans.factory.annotation.Autowired; import org.springframework.beans.factory.annotation.Autowired;
@@ -29,8 +28,6 @@ import ru.soune.nocopy.service.ImageHashService;
import ru.soune.nocopy.service.cost.CostService; import ru.soune.nocopy.service.cost.CostService;
import ru.soune.nocopy.service.file.FileEntityService; import ru.soune.nocopy.service.file.FileEntityService;
import ru.soune.nocopy.service.file.FileUploadService; import ru.soune.nocopy.service.file.FileUploadService;
import ru.soune.nocopy.service.file.cloud.CloudStorageService;
import ru.soune.nocopy.service.tariff.TariffConstants;
import ru.soune.nocopy.service.file.moderation.ModerationFileService; import ru.soune.nocopy.service.file.moderation.ModerationFileService;
import ru.soune.nocopy.service.notification.NotificationService; import ru.soune.nocopy.service.notification.NotificationService;
import ru.soune.nocopy.service.register.AuthService; import ru.soune.nocopy.service.register.AuthService;
@@ -98,8 +95,6 @@ public class FileUploadServiceImpl implements FileUploadService {
private final ModerationFileService moderationFileService; private final ModerationFileService moderationFileService;
private final CloudStorageService cloudStorageService;
@PostConstruct @PostConstruct
public void init() { public void init() {
try { try {
@@ -195,6 +190,7 @@ public class FileUploadServiceImpl implements FileUploadService {
} }
@Override @Override
@Transactional
public UploadProgressResponse uploadChunk(String uploadId, Integer chunkNumber, public UploadProgressResponse uploadChunk(String uploadId, Integer chunkNumber,
MultipartFile chunkFile, Integer findSimilar) { MultipartFile chunkFile, Integer findSimilar) {
FileUploadSession session = sessionRepository.findById(uploadId) FileUploadSession session = sessionRepository.findById(uploadId)
@@ -226,9 +222,10 @@ public class FileUploadServiceImpl implements FileUploadService {
return UploadProgressResponse.fromSession(session); return UploadProgressResponse.fromSession(session);
} }
@Async("fileUploadTaskExecutor")
@Transactional @Transactional
@Override @Override
public FileEntity completeFileProcessing(FileUploadSession session, FileStatus status) { public void completeFileProcessingAsync(FileUploadSession session, FileStatus status) {
try { try {
Path filePath = Paths.get(session.getFilePath()); Path filePath = Paths.get(session.getFilePath());
String checksum = calculateChecksum(filePath); String checksum = calculateChecksum(filePath);
@@ -254,17 +251,22 @@ public class FileUploadServiceImpl implements FileUploadService {
if (session.getFileType().equals("image") && status != FileStatus.PRIVATE) { if (session.getFileType().equals("image") && status != FileStatus.PRIVATE) {
Map<String, Long> hash = imageHashService.calculateHash(filePath); Map<String, Long> hash = imageHashService.calculateHash(filePath);
imageHashService.create(saved, hash); imageHashService.create(saved, hash);
} }
cleanupSessionFiles(session); cleanupSessionFiles(session);
log.info("File processing completed for session: {}", session.getUploadId());
return saved; if (status != FileStatus.TEMP && status != FileStatus.PRIVATE) {
FileProtector.FileInfo fileInfo = fileUtil.createFileInfo(fileEntity, session.getConvertTo());
noCopyFileService.addFile(fileInfo);
}
log.info("File processing completed for session: {}", session.getUploadId());
} catch (Exception e) { } catch (Exception e) {
log.error("Failed to complete file processing for session {}: {}", log.error("Failed to complete file processing for session {}: {}",
session.getUploadId(), e.getMessage(), e); session.getUploadId(), e.getMessage(), e);
throw new FileUploadException("Failed to complete file processing", e);
} }
} }
@@ -337,7 +339,7 @@ public class FileUploadServiceImpl implements FileUploadService {
session.setStatus(UploadStatus.COMPLETED); session.setStatus(UploadStatus.COMPLETED);
session.setFilePath(finalFilePath); session.setFilePath(finalFilePath);
completeFileProcessing(session, FileStatus.PRIVATE); completeFileProcessingAsync(session, FileStatus.PRIVATE);
} else { } else {
sessionRepository.save(session); sessionRepository.save(session);
} }
@@ -379,7 +381,7 @@ public class FileUploadServiceImpl implements FileUploadService {
chunkNumber, session.getChunksUploaded(), session.getTotalChunks()); chunkNumber, session.getChunksUploaded(), session.getTotalChunks());
boolean isLastChunk = session.getChunksUploaded().equals(session.getTotalChunks()); boolean isLastChunk = session.getChunksUploaded().equals(session.getTotalChunks());
//TODO CHECK
if (isLastChunk) { if (isLastChunk) {
String finalFilePath = assembleFileSynchronously(session); String finalFilePath = assembleFileSynchronously(session);
@@ -391,7 +393,6 @@ public class FileUploadServiceImpl implements FileUploadService {
session.getUserId()); session.getUserId());
File file = new File(finalFilePath); File file = new File(finalFilePath);
// File file = cloudStorageService.readFileFromStorageByPath(finalFilePath);
if (session.getFileType().startsWith("audio") && !isWavFile(file)) { if (session.getFileType().startsWith("audio") && !isWavFile(file)) {
session.setStatus(UploadStatus.FAILED); session.setStatus(UploadStatus.FAILED);
@@ -431,29 +432,20 @@ public class FileUploadServiceImpl implements FileUploadService {
session.setStatus(UploadStatus.COMPLETED); session.setStatus(UploadStatus.COMPLETED);
session.setFilePath(finalFilePath); session.setFilePath(finalFilePath);
notificationService.addNotification(NotificationType.FILE_ADDED_TO_SYSTEM, session.getUserId());
FileStatus status = findSimilar == 0 ? FileStatus.MODERATION: FileStatus.TEMP; FileStatus status = findSimilar == 0 ? FileStatus.MODERATION: FileStatus.TEMP;
if (status == FileStatus.TEMP) { if (status == FileStatus.TEMP) {
fileEntityService.clearTempFiles(session.getUserId()); fileEntityService.clearTempFiles(session.getUserId());
} }
FileEntity fileEntity = completeFileProcessing(session, status); completeFileProcessingAsync(session, status);
cloudStorageService.saveFilesToStorage(fileEntity.getMimeType(), fileEntity.getFileExtension(),
fileEntity.getFilePath());
if (status != FileStatus.TEMP) { if (status != FileStatus.TEMP) {
String fileType = session.getFileType(); String fileType = session.getFileType();
tariffInfoService.writeOffTokens(session.getUserId(), costService.protectFileCost(fileType), tariffInfoService.writeOffTokens(session.getUserId(), costService.protectFileCost(fileType), OperationType.FILE_UPLOAD);
OperationType.FILE_UPLOAD);
notificationService.addNotification(NotificationType.FILE_ADDED_TO_SYSTEM, session.getUserId());
}
if (status != FileStatus.TEMP) {
FileProtector.FileInfo fileInfo = fileUtil.createFileInfo(fileEntity, session.getConvertTo());
noCopyFileService.addFile(fileInfo);
} }
} else { } else {
sessionRepository.save(session); sessionRepository.save(session);
@@ -588,7 +580,6 @@ public class FileUploadServiceImpl implements FileUploadService {
private void checkForDuplicatesSynchronously(String filePath) private void checkForDuplicatesSynchronously(String filePath)
throws IOException , DuplicateImageException { throws IOException , DuplicateImageException {
Path path = Paths.get(filePath); Path path = Paths.get(filePath);
Map<String, Long> hash = imageHashService.calculateHash(path); Map<String, Long> hash = imageHashService.calculateHash(path);
List<SimilarImageProjection> duplicates = fileSimilarityService.findDuplicatedByHash( List<SimilarImageProjection> duplicates = fileSimilarityService.findDuplicatedByHash(
@@ -11,7 +11,7 @@ import ru.soune.nocopy.entity.file.FileEntity;
import ru.soune.nocopy.repository.FileEntityRepository; import ru.soune.nocopy.repository.FileEntityRepository;
import ru.soune.nocopy.repository.FileUploadSessionRepository; import ru.soune.nocopy.repository.FileUploadSessionRepository;
import ru.soune.nocopy.service.file.FileEntityService; import ru.soune.nocopy.service.file.FileEntityService;
import ru.soune.nocopy.service.file.cloud.CloudStorageService; import ru.soune.nocopy.service.file.FileUploadService;
import java.io.File; import java.io.File;
import java.io.IOException; import java.io.IOException;
@@ -32,53 +32,27 @@ public class ProtectionFileProviderImpl implements FileProtector.FileProvider {
private final FileEntityRepository fileEntityRepository; private final FileEntityRepository fileEntityRepository;
private final CloudStorageService cloudStorageService;
@Nullable @Nullable
@Override @Override
public File getImageFile(@NotNull String id) { public File getImageFile(@NotNull String id) {
FileEntity fileEntity = fileEntityRepository.findById(id).orElseThrow(() -> return fileEntityService.getFileById(id);
new RuntimeException("File not found: " + id));
try {
return cloudStorageService.readFileFromStorageByPath(fileEntity.getFilePath());
} catch (IOException e) {
throw new RuntimeException(e);
}
} }
@Nullable @Nullable
@Override @Override
public File getVideoFile(@NotNull String id) { public File getVideoFile(@NotNull String id) {
FileEntity fileEntity = fileEntityRepository.findById(id).orElseThrow(() -> return fileEntityService.getFileById(id);
new RuntimeException("File not found: " + id));
try {
return cloudStorageService.readFileFromStorageByPath(fileEntity.getFilePath());
} catch (IOException e) {
throw new RuntimeException(e);
}
} }
@Nullable @Nullable
@Override @Override
public File getAudioFile(@NotNull String id) { public File getAudioFile(@NotNull String id) {
FileEntity fileEntity = fileEntityRepository.findById(id).orElseThrow(() -> return fileEntityService.getFileById(id);
new RuntimeException("File not found: " + id));
try {
return cloudStorageService.readFileFromStorageByPath(fileEntity.getFilePath());
} catch (IOException e) {
throw new RuntimeException(e);
}
} }
@Override @Override
public @Nullable File getDocument(@NotNull String id) { public @Nullable File getDocument(@NotNull String id) {
FileEntity fileEntity = fileEntityRepository.findById(id).orElseThrow(() -> return fileEntityService.getFileById(id);
new RuntimeException("File not found: " + id));
try {
return cloudStorageService.readFileFromStorageByPath(fileEntity.getFilePath());
} catch (IOException e) {
throw new RuntimeException(e);
}
} }
@Override @Override
@@ -113,20 +87,7 @@ public class ProtectionFileProviderImpl implements FileProtector.FileProvider {
@Override @Override
public com.vrt.fileprotection.OperationResult writeAudioFile(@NotNull String id, @NotNull byte[] data, @Nullable String fileExt) { public com.vrt.fileprotection.OperationResult writeAudioFile(@NotNull String id, @NotNull byte[] data, @Nullable String fileExt) {
try { try {
FileEntity fileEntity = fileEntityService.writeProtectedFile(id, data, fileExt); fileEntityService.writeProtectedFile(id, data, fileExt);
Path filePath = Paths.get(fileEntity.getProtectedFilePath());
if (!Files.exists(filePath)) {
log.error("File not found after write: {}", filePath);
return OperationResult.Companion.failure("File not found");
}
log.info("File ready for upload: {} (size: {} bytes)", filePath, Files.size(filePath));
cloudStorageService.saveFilesToStorage(fileEntity.getMimeType(), fileEntity.getFileExtension(),
fileEntity.getProtectedFilePath());
return OperationResult.Companion.success(); return OperationResult.Companion.success();
} catch (Exception e) { } catch (Exception e) {
log.error("Failed to create protected file: {}", id, e); log.error("Failed to create protected file: {}", id, e);
@@ -138,11 +99,7 @@ public class ProtectionFileProviderImpl implements FileProtector.FileProvider {
@Override @Override
public com.vrt.fileprotection.OperationResult writeVideoFile(@NotNull String id, @NotNull byte[] data, @Nullable String fileExt) { public com.vrt.fileprotection.OperationResult writeVideoFile(@NotNull String id, @NotNull byte[] data, @Nullable String fileExt) {
try { try {
FileEntity fileEntity = fileEntityService.writeProtectedFile(id, data, fileExt); fileEntityService.writeProtectedFile(id, data, fileExt);
cloudStorageService.saveFilesToStorage(fileEntity.getMimeType(), fileEntity.getFileExtension(),
fileEntity.getProtectedFilePath());
return OperationResult.Companion.success(); return OperationResult.Companion.success();
} catch (Exception e) { } catch (Exception e) {
log.error("Failed to create protected file: {}", id, e); log.error("Failed to create protected file: {}", id, e);
@@ -154,11 +111,7 @@ public class ProtectionFileProviderImpl implements FileProtector.FileProvider {
@Override @Override
public com.vrt.fileprotection.OperationResult writeImageFile(@NotNull String id, @NotNull byte[] data, @Nullable String fileExt) { public com.vrt.fileprotection.OperationResult writeImageFile(@NotNull String id, @NotNull byte[] data, @Nullable String fileExt) {
try { try {
FileEntity fileEntity = fileEntityService.writeProtectedFile(id, data, fileExt); fileEntityService.writeProtectedFile(id, data, fileExt);
cloudStorageService.saveFilesToStorage(fileEntity.getMimeType(), fileEntity.getFileExtension(),
fileEntity.getProtectedFilePath());
return OperationResult.Companion.success(); return OperationResult.Companion.success();
} catch (Exception e) { } catch (Exception e) {
log.error("Failed to create protected file: {}", id, e); log.error("Failed to create protected file: {}", id, e);
@@ -169,10 +122,7 @@ public class ProtectionFileProviderImpl implements FileProtector.FileProvider {
@Override @Override
public @NotNull OperationResult writeDocumentFile(@NotNull String id, @NotNull byte[] data) { public @NotNull OperationResult writeDocumentFile(@NotNull String id, @NotNull byte[] data) {
try { try {
FileEntity fileEntity = fileEntityService.writeProtectedFile(id, data, null); fileEntityService.writeProtectedFile(id, data, null);
cloudStorageService.saveFilesToStorage(fileEntity.getMimeType(), fileEntity.getFileExtension(),
fileEntity.getProtectedFilePath());
return OperationResult.Companion.success(); return OperationResult.Companion.success();
} catch (Exception e) { } catch (Exception e) {
log.error("Failed to create protected file: {}", id, e); log.error("Failed to create protected file: {}", id, e);
@@ -44,13 +44,13 @@ public class GlobalSearchService {
task.setProcessedFiles(0); task.setProcessedFiles(0);
task.setCreatedAt(LocalDateTime.now()); task.setCreatedAt(LocalDateTime.now());
globalSearchTaskRepository.save(task); GlobalSearchTask save = globalSearchTaskRepository.save(task);
List<String> fileIds = filesToProcess.stream().map(FileEntity::getId).toList(); List<String> fileIds = filesToProcess.stream().map(FileEntity::getId).toList();
asyncProcessor.processFilesAsync(task.getTaskId(), fileIds, userId); asyncProcessor.processFilesAsync(save.getTaskId(), fileIds, userId);
return task.getTaskId(); return save.getTaskId();
} }
public List<FileEntity> getFilesToProcess(GlobalSearchStartRequest request, Long userId) { public List<FileEntity> getFilesToProcess(GlobalSearchStartRequest request, Long userId) {
@@ -13,6 +13,7 @@ import ru.soune.nocopy.exception.NotValidationPasswordException;
import ru.soune.nocopy.exception.UserNotFoundException; import ru.soune.nocopy.exception.UserNotFoundException;
import ru.soune.nocopy.repository.UserRepository; import ru.soune.nocopy.repository.UserRepository;
import java.util.List;
import java.util.Optional; import java.util.Optional;
@Service @Service
@@ -46,6 +47,11 @@ public class UserService {
return mapToDTO(updatedUser); return mapToDTO(updatedUser);
} }
@Transactional
public List<String> getUsersCompanyEmails(String company) {
return userRepository.emailsByCompany(company);
}
@Transactional @Transactional
public User activateAndVerifyUser(long userId) { public User activateAndVerifyUser(long userId) {
Optional<User> users = userRepository.findById(userId); Optional<User> users = userRepository.findById(userId);
@@ -56,6 +56,10 @@ public class ViolationService {
} }
} }
public Violation getById(Long violationId) {
return violationRepository.findById(violationId).orElse(null);
}
public List<FileViolationSummaryDTO> getFileViolationsSummary(Long userId) { public List<FileViolationSummaryDTO> getFileViolationsSummary(Long userId) {
List<FileEntity> userFiles = fileEntityService.getAllUserFiles(userId); List<FileEntity> userFiles = fileEntityService.getAllUserFiles(userId);
List<Violation> violations = violationRepository.findByFileEntityIn(userFiles); List<Violation> violations = violationRepository.findByFileEntityIn(userFiles);
-7
View File
@@ -102,13 +102,6 @@ yandex:
api-key: ${YANDEX_API_KEY:AQVNyaVaUmgUb1GMCtf5zSEqFxy0woXrcMOOB43q} api-key: ${YANDEX_API_KEY:AQVNyaVaUmgUb1GMCtf5zSEqFxy0woXrcMOOB43q}
folder-id: ${YANDEX_FOLDER_ID:b1gokpdbm6qfpsou8pcd} folder-id: ${YANDEX_FOLDER_ID:b1gokpdbm6qfpsou8pcd}
search-url: ${YANDEX_SEARCH_URL:https://searchapi.api.cloud.yandex.net/v2/image/search_by_image} search-url: ${YANDEX_SEARCH_URL:https://searchapi.api.cloud.yandex.net/v2/image/search_by_image}
cloud:
key: "YCAJEmHZcWxzf4oGWvETG3mms"
secret-key: "YCPAPZ32XC-LWb8xE0b_xTcBD8NZUCHpSOfu8LUG"
region: "ru-central1"
s3-endpoint: "https://storage.yandexcloud.net"
bucket: "no-copy-storage"
searchapi: searchapi:
api-key: ${SEARCHAPI_API_KEY:5jyYZC8jSaxhZTwjMUhwtAXi} api-key: ${SEARCHAPI_API_KEY:5jyYZC8jSaxhZTwjMUhwtAXi}