5 Commits
Author SHA1 Message Date
backdev eb223c2ecb dev add new action
Test Workflow / test (push) Has been cancelled
2026-04-22 12:38:05 +07:00
backdev bb96e3ac93 dev add new action
Test Workflow / test (push) Has been cancelled
2026-04-22 12:16:37 +07:00
backdev 572255c066 dev add monitoring statistic
Test Workflow / test (push) Has been cancelled
2026-04-20 16:15:08 +07:00
backdev d01ea6e2bd dev fix save to storage
Test Workflow / test (push) Has been cancelled
2026-04-20 15:35:24 +07:00
backdev d1a91fec1a dev add user_id for find all complaints
Test Workflow / test (push) Has been cancelled
2026-04-17 22:00:33 +07:00
37 changed files with 458 additions and 600 deletions
+1 -12
View File
@@ -323,7 +323,6 @@ server_name dev-workspace.not-copy.com;
ssl_certificate_key /etc/letsencrypt/live/dev-workspace.not-copy.com/privkey.pem;
location /api/ {
proxy_pass http://localhost:3001;
proxy_set_header Host $host;
proxy_set_header X-Real-IP $remote_addr;
@@ -490,14 +489,4 @@ killall 3proxy # Остановка прокси
На основном сервере:
bash
curl -x http://193.46.217.94:3128 https://api.ipify.org # Проверка прокси
telnet 193.46.217.94 3128 # Проверка порта
------
Настройка Яндекс Клауд
1. YCAJEpWAtaVkVGX0sH6_EupEg - индетификатор ключа
2. YCMmVykfXrZ_nfU13Vo4yoCVGa70DnTlBgF1pUzO - секретный ключ
private static final String KEY_ID = "YCAJEmHZcWxzf4oGWvETG3mms";
private static final String SECRET_KEY = "YCPAPZ32XC-LWb8xE0b_xTcBD8NZUCHpSOfu8LUG";
telnet 193.46.217.94 3128 # Проверка порта
-6
View File
@@ -66,12 +66,6 @@ dependencies {
implementation project(':referral')
//cloud
implementation("software.amazon.awssdk:aws-sdk-java:2.29.33")
implementation("software.amazon.awssdk:apache-client:2.29.33")
implementation("org.springdoc:springdoc-openapi-starter-webmvc-ui:2.7.0")
implementation("org.springframework.boot:spring-boot-starter-web")
}
tasks.named('test') {
@@ -46,7 +46,8 @@ public class HandlerConfig {
StatisticViolationHandler statisticViolationHandler,
StatisticSubscriberHandler statisticSubscriberHandler,
StatisticTokenHandler statisticTokenHandler,
StatisticIncomeHandler statisticIncomeHandler
StatisticIncomeHandler statisticIncomeHandler,
MonitoringStatisticHandler monitoringStatisticHandler
) {
Map<Integer, RequestHandler> map = new HashMap<>();
@@ -85,6 +86,7 @@ public class HandlerConfig {
map.put(30024, statisticSubscriberHandler);
map.put(30025, statisticTokenHandler);
map.put(30026, statisticIncomeHandler);
map.put(30027, monitoringStatisticHandler);
return map;
}
@@ -1,7 +1,5 @@
package ru.soune.nocopy.controller;
import com.google.api.client.util.IOUtils;
import com.google.common.io.ByteStreams;
import com.vrt.NoCopyFileService;
import com.vrt.fileprotection.FileProtector;
import com.vrt.fileprotection.NoCopyCheckResult;
@@ -10,20 +8,16 @@ import com.vrt.fileprotection.documents.DocumentCheckResult;
import lombok.RequiredArgsConstructor;
import lombok.extern.slf4j.Slf4j;
import org.springframework.core.io.FileSystemResource;
import org.springframework.core.io.InputStreamResource;
import org.springframework.data.domain.Page;
import org.springframework.data.domain.Pageable;
import org.springframework.data.web.PageableDefault;
import org.springframework.http.HttpHeaders;
import org.springframework.http.HttpStatus;
import org.springframework.http.MediaType;
import org.springframework.http.ResponseEntity;
import org.springframework.validation.BindingResult;
import org.springframework.validation.FieldError;
import org.springframework.web.bind.annotation.*;
import org.springframework.web.multipart.MultipartFile;
import org.springframework.web.server.ResponseStatusException;
import org.springframework.web.servlet.mvc.method.annotation.StreamingResponseBody;
import ru.soune.nocopy.dto.BaseRequest;
import ru.soune.nocopy.dto.BaseResponse;
import ru.soune.nocopy.dto.MessageCode;
@@ -42,17 +36,12 @@ import ru.soune.nocopy.repository.FileEntityRepository;
import ru.soune.nocopy.repository.UserRepository;
import ru.soune.nocopy.service.FileSimilarityService;
import ru.soune.nocopy.service.file.*;
import ru.soune.nocopy.service.file.CheckCounterService;
import ru.soune.nocopy.service.file.ProtectionsLimitService;
import ru.soune.nocopy.service.file.cloud.CloudStorageService;
import ru.soune.nocopy.service.register.AuthService;
import ru.soune.nocopy.service.user.moderation.UserVerificationService;
import ru.soune.nocopy.util.FileUtil;
import software.amazon.awssdk.services.s3.model.NoSuchKeyException;
import java.io.File;
import java.io.IOException;
import java.io.InputStream;
import java.nio.file.Files;
import java.nio.file.Path;
import java.nio.file.Paths;
@@ -76,8 +65,6 @@ public class ApiController {
private final FileEntityRepository fileEntityRepository;
private final NoCopyFileService noCopyFileService;
private final FileUtil fileUtil;
private final ProtectionsLimitService protectionsLimitService;
@@ -88,8 +75,6 @@ public class ApiController {
private final CheckCounterService checkCounterService;
private final CloudStorageService cloudStorageService;
private final ZipService zipService;
private final UserVerificationService userVerificationService;
@@ -174,7 +159,7 @@ public class ApiController {
MessageCode.DUPLICATE_FILE_UPLOAD.getDescription(),
duplicateData
));
} catch (FileFormatException e){
}catch (FileFormatException e){
return buildErrorResponse(uploadId, chunkNumber, e.getMessage());
} catch (Exception e) {
log.error("Error uploading chunk", e);
@@ -220,6 +205,8 @@ public class ApiController {
}
}
private final NoCopyFileService noCopyFileService;
@GetMapping("/check/{fileId}")
public ResponseEntity<BaseResponse> check(@PathVariable String fileId) {
FileEntity fileEntity = fileEntityRepository.findByFileId(fileId);
@@ -502,35 +489,39 @@ public class ApiController {
MessageCode.FILE_DELETE.getDescription(),
errorData));
}
//TODO
// if (!entityResponse.isExistsOnDisk()) {
// Map<String, Object> errorData = new HashMap<>();
// errorData.put("onDisk", entityResponse.isExistsOnDisk());
//
// return ResponseEntity.ok().body(new BaseResponse(20004,
// MessageCode.FILE_NOT_EXIST.getCode(),
// MessageCode.FILE_NOT_EXIST.getDescription(),
// errorData));
// }
if (!entityResponse.isExistsOnDisk()) {
Map<String, Object> errorData = new HashMap<>();
errorData.put("onDisk", entityResponse.isExistsOnDisk());
// if (!file.exists()) {
// Map<String, Object> errorData = new HashMap<>();
// errorData.put("file", file.exists());
//
// return ResponseEntity.ok().body(new BaseResponse(20004,
// MessageCode.FILE_DOWNLOAD_ERROR.getCode(),
// MessageCode.FILE_DOWNLOAD_ERROR.getDescription(),
// errorData));
// }
return ResponseEntity.ok().body(new BaseResponse(20004,
MessageCode.FILE_NOT_EXIST.getCode(),
MessageCode.FILE_NOT_EXIST.getDescription(),
errorData));
}
byte[] bytes = cloudStorageService.readFileFromStorageBytes(entityResponse.getProtectedFilePath());
Path filePath = Paths.get(entityResponse.getProtectedFilePath());
FileSystemResource resource = new FileSystemResource(filePath);
long fileSize = Files.size(filePath);
if (!resource.exists()) {
Map<String, Object> errorData = new HashMap<>();
errorData.put("resource", resource.exists());
return ResponseEntity.ok().body(new BaseResponse(20004,
MessageCode.FILE_DOWNLOAD_ERROR.getCode(),
MessageCode.FILE_DOWNLOAD_ERROR.getDescription(),
errorData));
}
String contentType = determineContentType(filePath);
return ResponseEntity.ok()
.contentType(MediaType.APPLICATION_OCTET_STREAM)
.contentLength(fileSize)
.contentType(MediaType.parseMediaType(contentType))
.header(HttpHeaders.CONTENT_DISPOSITION,
"attachment; filename=\"" + entityResponse.getFileName() + "\"")
.body(bytes);
.body(resource);
} catch (FileEntityNotFoundException e) {
Map<String, Object> errorData = new HashMap<>();
errorData.put("fileId", fileId);
@@ -547,16 +538,16 @@ public class ApiController {
MessageCode.AUTH_TOKEN_NOT_FOUND.getCode(),
MessageCode.AUTH_TOKEN_NOT_FOUND.getDescription(),
errorData));
// } catch (IOException e) {
// Map<String, Object> errorData = new HashMap<>();
// errorData.put("token", tokenHeader);
// errorData.put("fileId", fileId);
// errorData.put("version", version);
//
// return ResponseEntity.ok().body(new BaseResponse(20004,
// MessageCode.FILE_DOWNLOAD_ERROR_NOT_CORRECT_FIELD.getCode(),
// MessageCode.FILE_DOWNLOAD_ERROR_NOT_CORRECT_FIELD.getDescription(),
// errorData));
} catch (IOException e) {
Map<String, Object> errorData = new HashMap<>();
errorData.put("token", tokenHeader);
errorData.put("fileId", fileId);
errorData.put("version", version);
return ResponseEntity.ok().body(new BaseResponse(20004,
MessageCode.FILE_DOWNLOAD_ERROR_NOT_CORRECT_FIELD.getCode(),
MessageCode.FILE_DOWNLOAD_ERROR_NOT_CORRECT_FIELD.getDescription(),
errorData));
}
}
@@ -573,6 +564,33 @@ public class ApiController {
return ResponseEntity.ok().body(fileTypeStats);
}
private boolean hasDuplicate(List<SimilarFileDTO> similarFiles) {
return similarFiles.stream().anyMatch(f -> f.getHammingDistance() <= 5);
}
private ResponseEntity<BaseResponse> handleDuplicate(FileEntity fileEntity, List<SimilarFileDTO> similarFiles)
throws IOException {
fileEntityService.deleteFromDisk(fileEntity);
fileEntityService.softDeleteFileWithHash(fileEntity);
Optional<FileEntity> originalFile = fileEntityRepository.findById(similarFiles.get(0).getFileId());
if (originalFile.isPresent()) {
Map<String, String> duplicateInfo = Map.of(
"duplicate_file_id", originalFile.get().getId(),
"owner_user_id", String.valueOf(originalFile.get().getUserId()));
return ResponseEntity.ok().body(new BaseResponse(
20004,
MessageCode.DUPLICATE_FILE_UPLOAD.getCode(),
"Failed to upload chunk, duplicate",
duplicateInfo));
}
return null;
}
private ResponseEntity<BaseResponse> buildSuccessResponse(String uploadId, Integer chunkNumber, MultipartFile chunk,
String fileId) {
ChunkUploadResponse responseBody = ChunkUploadResponse.builder()
@@ -629,4 +647,13 @@ public class ApiController {
return errorDetail;
}
private String determineContentType(Path filePath) throws IOException {
String contentType = Files.probeContentType(filePath);
if (contentType == null) {
contentType = "application/octet-stream";
}
return contentType;
}
}
@@ -1,16 +1,12 @@
package ru.soune.nocopy.controller;
import com.google.api.client.util.IOUtils;
import com.google.common.io.ByteStreams;
import com.vrt.NoCopyFileService;
import com.vrt.fileprotection.FileProtector;
import lombok.AllArgsConstructor;
import lombok.extern.slf4j.Slf4j;
import org.springframework.core.io.InputStreamResource;
import org.springframework.core.io.Resource;
import org.springframework.http.*;
import org.springframework.web.bind.annotation.*;
import org.springframework.web.servlet.mvc.method.annotation.StreamingResponseBody;
import ru.soune.nocopy.dto.file.CheckStatus;
import ru.soune.nocopy.entity.file.FileEntity;
import ru.soune.nocopy.entity.file.ProtectionStatus;
@@ -22,14 +18,13 @@ import ru.soune.nocopy.service.file.CheckCounterService;
import ru.soune.nocopy.service.file.FileStorageService;
import ru.soune.nocopy.service.file.ImageResizeService;
import ru.soune.nocopy.service.file.ZipService;
import ru.soune.nocopy.service.file.cloud.CloudStorageService;
import ru.soune.nocopy.util.FileUtil;
import software.amazon.awssdk.services.s3.model.NoSuchKeyException;
import java.io.*;
import java.io.FileNotFoundException;
import java.net.URLEncoder;
import java.nio.charset.StandardCharsets;
import java.io.IOException;
import java.nio.file.Files;
import java.nio.file.Path;
import java.nio.file.Paths;
@@ -56,13 +51,19 @@ public class FileController {
private ZipService zipService;
private CloudStorageService cloudStorageService;
@GetMapping("/public/{fileId}")
public ResponseEntity<StreamingResponseBody> getPublicFile(@PathVariable String fileId) {
public ResponseEntity<Resource> getPublicFile(@PathVariable String fileId) {
try {
FileEntity fileEntity = fileRepository.findById(fileId)
.orElseThrow(() -> new FileNotFoundException("Not found in DB: " + fileId));
.orElseThrow(() -> new FileNotFoundException("Not found: " + fileId));
Path filePath = Paths.get(fileEntity.getFilePath());
if (!Files.exists(filePath)) {
log.error("Not on disk: {}", fileEntity.getFilePath());
return ResponseEntity.notFound().build();
}
Resource resource = fileStorageService.loadFileAsResource(fileEntity.getFilePath());
MediaType mediaType = getMediaType(fileEntity);
@@ -70,28 +71,18 @@ public class FileController {
.filename(fileEntity.getOriginalFileName(), StandardCharsets.UTF_8)
.build();
StreamingResponseBody responseBody = outputStream -> {
try (InputStream inputStream = cloudStorageService.readFileFromStorage(fileEntity.getFilePath())) {
ByteStreams.copy(inputStream, outputStream);
} catch (NoSuchKeyException e) {
log.error("File disappeared from S3 during streaming: {}", fileId, e);
throw new IOException("File not found in storage", e);
}
};
return ResponseEntity.ok()
.contentType(mediaType)
.contentLength(Files.size(filePath))
.header(HttpHeaders.CONTENT_DISPOSITION, contentDisposition.toString())
.header(HttpHeaders.CACHE_CONTROL, "public, max-age=3600")
.body(responseBody);
.body(resource);
} catch (FileNotFoundException e) {
log.warn("File not found in DB: {}", fileId);
log.warn("Not on disk: {}", fileId);
return ResponseEntity.notFound().build();
} catch (NoSuchKeyException e) {
log.warn("File not found in S3: {}", fileId);
return ResponseEntity.status(HttpStatus.NOT_FOUND).build();
} catch (Exception e) {
log.error("Error reading file: {}", fileId, e);
} catch (IOException e) {
log.error("Read file exception: {}", fileId, e);
return ResponseEntity.status(HttpStatus.INTERNAL_SERVER_ERROR).build();
}
}
@@ -122,14 +113,9 @@ public class FileController {
FileEntity fileEntity = fileRepository.findById(fileId)
.orElseThrow(() -> new RuntimeException("File not found"));
String filePath = type.equals("thumbnail") ? fileEntity.getThumbnailPath(): fileEntity.getFilePath();
String path = imageResizeService.getPath(fileEntity, type);
// InputStream file = cloudStorageService.readFileFromStorage(filePath);
// if (file == null || !file.exists()) {
// log.error("File not found in storage: {}", fileEntity.getFilePath());
// return ResponseEntity.notFound().build();
// }
Resource resource = fileStorageService.loadFileAsResource(path);
if (fileEntity.getProtectionStatus() == ProtectionStatus.NOT_PROTECTED ||
fileEntity.getProtectionStatus() == ProtectionStatus.PROCESSING) {
@@ -142,8 +128,6 @@ public class FileController {
String encodedFileName = URLEncoder.encode(fileName, StandardCharsets.UTF_8)
.replace("+", "%20");
InputStreamResource resource = new InputStreamResource(cloudStorageService.readFileFromStorage(filePath));
return ResponseEntity.ok()
.contentType(new MediaType(fileEntity.getMimeType(), fileEntity.getFileExtension()))
.header(HttpHeaders.CONTENT_DISPOSITION,
@@ -35,4 +35,7 @@ public class ComplaintRequest {
@JsonProperty("sort_direction")
private String sortDirection;
@JsonProperty("token")
private String token;
}
@@ -36,4 +36,7 @@ public class ComplaintResponse {
@JsonProperty("not_moderated")
private Boolean notModerated;
@JsonProperty("file_id")
private String fileId;
}
@@ -1,5 +1,6 @@
package ru.soune.nocopy.dto.complaint;
import com.fasterxml.jackson.annotation.JsonProperty;
import lombok.Builder;
import lombok.Data;
import ru.soune.nocopy.entity.complaint.LawCase;
@@ -14,19 +15,43 @@ import java.util.List;
@Builder
public class LawCaseResponse {
private Long id;
private String name;
private String description;
private BigDecimal amount;
private LawCasePriority priority;
private LawCaseType type;
private String lawyer;
@JsonProperty("created_at")
private LocalDateTime createdAt;
@JsonProperty("updated_at")
private LocalDateTime updatedAt;
@JsonProperty("page_number")
private int pageNumber;
@JsonProperty("page_size")
private int pageSize;
@JsonProperty("total_elements")
private long totalElements;
@JsonProperty("total_pages")
private int totalPages;
@JsonProperty("violation_id")
private long violationId;
@JsonProperty("file_id")
private String fileId;
private List<LawCaseResponse> content;
public static LawCaseResponse fromEntity(LawCase lawCase) {
@@ -0,0 +1,8 @@
package ru.soune.nocopy.dto.statistic;
import lombok.Data;
@Data
public class MonitoringStatisticRequest {
private String authToken;
}
@@ -0,0 +1,20 @@
package ru.soune.nocopy.dto.statistic;
import com.fasterxml.jackson.annotation.JsonProperty;
import lombok.Builder;
import lombok.Data;
@Data
@Builder
public class MonitoringStatisticResponse {
@JsonProperty("day")
private Long everyDay;
@JsonProperty("month")
private Long everyMonth;
@JsonProperty("weekly")
private Long everyWeekly;
}
@@ -1,12 +1,22 @@
package ru.soune.nocopy.dto.violation;
import com.fasterxml.jackson.annotation.JsonProperty;
import lombok.Data;
@Data
public class ViolationNotionRequest {
@JsonProperty("token")
private String token;
@JsonProperty("action")
private String action;
@JsonProperty("violation_id")
private Long violationId;
@JsonProperty("notion_id")
private Long notionId;
@JsonProperty("message")
private String message;
}
@@ -13,8 +13,14 @@ import ru.soune.nocopy.dto.MessageCode;
import ru.soune.nocopy.dto.complaint.ComplaintRequest;
import ru.soune.nocopy.dto.complaint.ComplaintResponse;
import ru.soune.nocopy.entity.complaint.ComplaintStatus;
import ru.soune.nocopy.entity.user.User;
import ru.soune.nocopy.repository.UserRepository;
import ru.soune.nocopy.service.complaint.ComplaintEntityService;
import ru.soune.nocopy.service.register.AuthService;
import ru.soune.nocopy.service.user.UserService;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
@Slf4j
@@ -25,11 +31,31 @@ public class ComplaintEntityHandler implements RequestHandler {
private final ObjectMapper objectMapper;
private final UserRepository userRepository;
private final UserService userService;
private final AuthService authService;
@Override
public BaseResponse handle(BaseRequest request) throws Exception {
Integer msgId = request.getMsgId();
ComplaintRequest complaintRequest = objectMapper.convertValue(request.getMessageBody(), ComplaintRequest.class);
String action = complaintRequest.getAction();
String token = complaintRequest.getToken();
Long userId = authService.useUserAuthToken(token);
User user = userRepository.findById(userId).orElse(null);
if (user == null) {
return new BaseResponse(
request.getMsgId(),
MessageCode.USER_NOT_FOUND.getCode(),
MessageCode.USER_NOT_FOUND.getDescription(),
Map.of("token: ", token));
}
List<String> emails = user.getCompany() == null ? List.of(user.getEmail()):
userService.getUsersCompanyEmails(user.getCompany().getId());
try {
switch (action.toLowerCase()) {
@@ -38,7 +64,7 @@ public class ComplaintEntityHandler implements RequestHandler {
case "get":
return handleGet(msgId, complaintRequest);
case "get_all":
return handleGetAll(msgId, complaintRequest);
return handleGetAll(msgId, complaintRequest, emails);
case "get_by_violation":
return handleGetByViolation(msgId, complaintRequest);
case "update_status":
@@ -74,14 +100,15 @@ public class ComplaintEntityHandler implements RequestHandler {
return successResponse(msgId, "Complaint retrieved successfully", response);
}
private BaseResponse handleGetAll(Integer msgId, ComplaintRequest req) {
private BaseResponse handleGetAll(Integer msgId, ComplaintRequest req, List<String> emails) {
Pageable pageable = PageRequest.of(
req.getPage() != null ? req.getPage() : 0,
req.getSize() != null ? req.getSize() : 5,
Sort.by(Sort.Direction.fromString(req.getSortDirection() != null ? req.getSortDirection() : "desc"),
req.getSortBy() != null ? req.getSortBy() : "updatedAt"));
var page = complaintService.getAllComplaints(pageable);
var page = complaintService.getAllComplaintsByEmail(pageable, emails);
Map<String, Object> data = new HashMap<>();
data.put("content", page.getContent());
data.put("page", page.getNumber());
@@ -18,7 +18,6 @@ import ru.soune.nocopy.entity.file.moderation.ModerationLog;
import ru.soune.nocopy.exception.*;
import ru.soune.nocopy.repository.FileAppealRepository;
import ru.soune.nocopy.repository.FileEntityRepository;
import ru.soune.nocopy.service.file.cloud.CloudStorageService;
import ru.soune.nocopy.repository.ModerationLogRepository;
import ru.soune.nocopy.service.file.moderation.ModerationService;
import ru.soune.nocopy.service.register.AuthService;
@@ -44,8 +43,6 @@ public class FileEntityHandler implements RequestHandler {
private final FileEntityRepository fileEntityRepository;
private final CloudStorageService cloudStorageService;
private final ModerationService moderationService;
private final FileAppealRepository fileAppealRepository;
@@ -703,7 +700,7 @@ public class FileEntityHandler implements RequestHandler {
int fullDelete = fileRequest.getFullDelete() == null ? 0 : fileRequest.getFullDelete();
if (fileEntity.getStatus().equals(FileStatus.DELETED) || fullDelete == 1) {
cloudStorageService.deleteFromStorageDisk(fileEntity);
fileEntityService.deleteFromDisk(fileEntity);
response = DeleteFileResponse.builder()
.fileId(fileRequest.getFileId())
.message("File deleted from disk")
@@ -6,12 +6,21 @@ import lombok.extern.slf4j.Slf4j;
import org.springframework.stereotype.Component;
import ru.soune.nocopy.dto.BaseRequest;
import ru.soune.nocopy.dto.BaseResponse;
import ru.soune.nocopy.dto.MessageCode;
import ru.soune.nocopy.dto.search.GlobalSearchStatisticsRequest;
import ru.soune.nocopy.dto.search.GlobalSearchStatisticsResponse;
import ru.soune.nocopy.entity.company.Company;
import ru.soune.nocopy.entity.user.AuthToken;
import ru.soune.nocopy.entity.user.User;
import ru.soune.nocopy.repository.AuthTokenRepository;
import ru.soune.nocopy.repository.UserRepository;
import ru.soune.nocopy.service.CompanyService;
import ru.soune.nocopy.service.search.GlobalSearchStatisticsService;
import ru.soune.nocopy.service.user.UserService;
import java.util.ArrayList;
import java.util.List;
import java.util.Map;
import java.util.Optional;
@Slf4j
@@ -17,6 +17,7 @@ import ru.soune.nocopy.entity.complaint.LawCasePriority;
import ru.soune.nocopy.entity.complaint.LawCaseType;
import ru.soune.nocopy.entity.tokenoperation.OperationType;
import ru.soune.nocopy.entity.user.User;
import ru.soune.nocopy.entity.violation.Violation;
import ru.soune.nocopy.exception.NotValidFieldException;
import ru.soune.nocopy.exception.TariffNotFoundException;
import ru.soune.nocopy.exception.UserNotFoundException;
@@ -25,8 +26,10 @@ import ru.soune.nocopy.service.complaint.LawCaseService;
import ru.soune.nocopy.service.register.AuthService;
import ru.soune.nocopy.service.tariff.TariffConstants;
import ru.soune.nocopy.service.tariff.TariffInfoService;
import ru.soune.nocopy.service.violation.ViolationService;
import java.io.IOException;
import java.util.ArrayList;
import java.util.List;
import java.util.Map;
import java.util.stream.Collectors;
@@ -46,6 +49,8 @@ public class LawCaseHandler implements RequestHandler {
private final TariffInfoService tariffInfoService;
private final ViolationService violationService;
@Override
public BaseResponse handle(BaseRequest request) throws Exception {
@@ -84,9 +89,17 @@ public class LawCaseHandler implements RequestHandler {
lawCaseRequest.getSortBy(),
lawCaseRequest.getSortDir());
List<LawCaseResponse> responses = lawCases.getContent().stream()
.map(LawCaseResponse::fromEntity)
.collect(Collectors.toList());
List<LawCaseResponse> responses = new ArrayList<>();
List<LawCase> content = lawCases.getContent();
for (LawCase lawCase: content) {
LawCaseResponse lawCaseResponse = LawCaseResponse.fromEntity(lawCase);
Violation violation = violationService.getById(lawCase.getViolationId());
if (violation != null) lawCaseResponse.setFileId(violation.getFileEntity().getId());
responses.add(lawCaseResponse);
}
PaginatedResponse<LawCaseResponse> paginatedResponse = new PaginatedResponse<>(
responses,
@@ -0,0 +1,72 @@
package ru.soune.nocopy.handler;
import com.fasterxml.jackson.databind.ObjectMapper;
import lombok.RequiredArgsConstructor;
import lombok.extern.slf4j.Slf4j;
import org.springframework.stereotype.Component;
import ru.soune.nocopy.dto.BaseRequest;
import ru.soune.nocopy.dto.BaseResponse;
import ru.soune.nocopy.dto.MessageCode;
import ru.soune.nocopy.dto.statistic.MonitoringStatisticRequest;
import ru.soune.nocopy.dto.statistic.MonitoringStatisticResponse;
import ru.soune.nocopy.entity.company.Company;
import ru.soune.nocopy.entity.monitoring.MonitoringType;
import ru.soune.nocopy.entity.user.User;
import ru.soune.nocopy.repository.FileMonitoringRepository;
import ru.soune.nocopy.repository.UserRepository;
import ru.soune.nocopy.service.register.AuthService;
import java.util.List;
@Slf4j
@Component
@RequiredArgsConstructor
public class MonitoringStatisticHandler implements RequestHandler {
private final ObjectMapper objectMapper;
private final AuthService authService;
private final FileMonitoringRepository fileMonitoringRepository;
private final UserRepository userRepository;
@Override
public BaseResponse handle(BaseRequest request) throws Exception {
MonitoringStatisticRequest monitoringStatisticRequest = objectMapper.convertValue(request.getMessageBody(),
MonitoringStatisticRequest.class);
try {
String authToken = monitoringStatisticRequest.getAuthToken();
Long userId = authService.useUserAuthToken(authToken);
User user = userRepository.findById(userId).orElseThrow();
Company company = user.getCompany();
List<Long> userIds = company == null ? List.of(userId) :
user.getCompany()
.getUsers()
.stream()
.map(User::getId).toList();
MonitoringStatisticResponse monitoringStatisticResponse = MonitoringStatisticResponse.builder()
.everyDay(fileMonitoringRepository.countByUserIdsAndMonitoringType(userIds,
MonitoringType.MONITORING_DAILY))
.everyMonth(fileMonitoringRepository.countByUserIdsAndMonitoringType(userIds,
MonitoringType.MONITORING_MONTHLY))
.everyWeekly(fileMonitoringRepository.countByUserIdsAndMonitoringType(userIds,
MonitoringType.MONITORING_WEEKLY))
.build();
return new BaseResponse(
request.getMsgId(),
MessageCode.SUCCESS.getCode(),
MessageCode.SUCCESS.getDescription(),
monitoringStatisticResponse);
} catch (Exception e) {
return new BaseResponse(
request.getMsgId(),
MessageCode.INVALID_JSON_BODY.getCode(),
MessageCode.INVALID_JSON_BODY.getDescription(),
null);
}
}
}
@@ -25,6 +25,7 @@ import java.time.format.DateTimeParseException;
import java.util.ArrayList;
import java.util.List;
import java.util.Map;
import java.util.Objects;
@Component
@Slf4j
@@ -115,11 +116,21 @@ public class ViolationHandler implements RequestHandler {
violationRequest.getPage(), violationRequest.getSize(), violationRequest.getSortDirection()
);
}
ViolationResponse.ViolationDto firstShowViolation = null;
List<Violation> content = violationPage.getContent();
List<ViolationResponse.ViolationDto> violationDtos = new ArrayList<>();
if ("getByViolationId".equals(violationRequest.getAction())) {
Violation violation = violationService.getById(violationRequest.getViolationId());
if (violation != null) {
firstShowViolation = ViolationResponse.ViolationDto.fromEntity(violation);
violationDtos.add(firstShowViolation);
}
}
for (Violation violation : content) {
if (firstShowViolation != null && Objects.equals(firstShowViolation.getId(), violation.getId())) continue;
ViolationResponse.ViolationDto violationDto = ViolationResponse.ViolationDto.fromEntity(violation);
violationDto.setCountry(geoCountryService.getCountryName(violation.getPageUrl()));
violationDto.setCountryCode(geoCountryService.getCountryCode(violation.getPageUrl()));
@@ -5,6 +5,7 @@ import org.springframework.data.domain.Pageable;
import org.springframework.data.jpa.repository.JpaRepository;
import org.springframework.data.jpa.repository.Modifying;
import org.springframework.data.jpa.repository.Query;
import org.springframework.data.repository.query.Param;
import org.springframework.stereotype.Repository;
import org.springframework.transaction.annotation.Transactional;
import ru.soune.nocopy.entity.complaint.ComplaintEntity;
@@ -37,4 +38,7 @@ public interface ComplaintEntityRepository extends JpaRepository<ComplaintEntity
)
""")
Long getComplaintsWithoutLawCaseCount();
@Query("SELECT c FROM ComplaintEntity c WHERE c.email IN :emails")
Page<ComplaintEntity> getComplaintsByUserEmails(@Param("emails") List<String> emails, Pageable pageable);
}
@@ -60,6 +60,12 @@ public interface FileEntityRepository extends JpaRepository<FileEntity, String>
List<FileEntity> findFileByUserIdAndStatus(Long userId, FileStatus status);
@Query("SELECT f FROM FileEntity f WHERE f.userId = :userId AND f.mimeType = :mimeType")
List<FileEntity> findByUserIdAndMimeType(Long userId, String mimeType);
@Query("SELECT f FROM FileEntity f WHERE f.mimeType = :mimeType")
List<FileEntity> findByMimeType(String mimeType);
@Query("SELECT f FROM FileEntity f WHERE f.mimeType = :mimeType AND f.userId IN :userIds")
List<FileEntity> findByMimeTypeAndUserIds(String mimeType, List<Long> userIds);
@@ -22,4 +22,11 @@ public interface FileMonitoringRepository extends JpaRepository<FileMonitoringEn
List<FileMonitoringEntity> findReadyForRun(@Param("now") LocalDateTime now);
List<FileMonitoringEntity> findByMonitoringTypeAndIsActiveTrue(MonitoringType type);
@Query("SELECT COUNT(m) FROM FileMonitoringEntity m " +
"WHERE m.userId IN :userIds " +
"AND m.monitoringType = :monitoringType " +
"AND m.isActive = true")
long countByUserIdsAndMonitoringType(@Param("userIds") List<Long> userIds,
@Param("monitoringType") MonitoringType monitoringType);
}
@@ -45,4 +45,7 @@ public interface UserRepository extends JpaRepository<User, Long> {
@Query("SELECT u FROM User u WHERE u.personalTariffInfo.id = :tariffInfoId")
Optional<User> findByPersonalTariffInfoId(@Param("tariffInfoId") String tariffInfoId);
@Query("SELECT u.email FROM User u WHERE u.company = :company")
List<String> emailsByCompany(@Param("company") String company);
}
@@ -17,7 +17,6 @@ import ru.soune.nocopy.entity.file.ImageHashEntity;
import ru.soune.nocopy.entity.user.User;
import ru.soune.nocopy.exception.DuplicateImageException;
import ru.soune.nocopy.repository.*;
import ru.soune.nocopy.service.file.cloud.CloudStorageService;
import ru.soune.nocopy.util.FileUtil;
import java.io.File;
@@ -42,8 +41,6 @@ public class FileSimilarityService {
private final FileEntityRepository fileEntityRepository;
private final CloudStorageService cloudStorageService;
@Value("${server.baseurl}")
private String baseUrl;
@@ -117,7 +114,7 @@ public class FileSimilarityService {
}
public FileEntity findDuplicateByHash(String path, String mimeType, Long userId) throws Exception {
String hash = calculateFileHash(path, false);
String hash = calculateFileHash(path);
User user = userRepository.findById(userId).orElseThrow();
Company company = user.getCompany();
@@ -132,15 +129,15 @@ public class FileSimilarityService {
List<FileEntity> fileEntityList = fileEntityRepository.findByMimeTypeAndUserIds(mimeType, userIds);
for (FileEntity file : fileEntityList) {
// if (file.getProtectedFilePath() == null) continue;
// File existingFile = cloudStorageService.readFileFromStorageByPath(file.getFilePath());
// File newFile = new File(path);
if (file.getProtectedFilePath() == null) continue;
File existingFile = new File(file.getProtectedFilePath());
File newFile = new File(path);
// if (existingFile.length() == newFile.length()) {
if (calculateFileHash(file.getFilePath(), true).equals(hash)) {
if (existingFile.length() == newFile.length()) {
if (calculateFileHash(file.getProtectedFilePath()).equals(hash)) {
return file;
}
// }
}
}
return null;
@@ -199,19 +196,14 @@ public class FileSimilarityService {
}
private String calculateFileHash(String path, boolean cloud) throws Exception {
private String calculateFileHash(String path) throws Exception {
MessageDigest digest = MessageDigest.getInstance("SHA-256");
if (cloud) {
byte[] data = cloudStorageService.readFileFromStorageBytes(path);
digest.update(data);
} else {
try (InputStream is = new FileInputStream(path)) {
byte[] buffer = new byte[8192];
int read;
while ((read = is.read(buffer)) > 0) {
digest.update(buffer, 0, read);
}
try (InputStream is = new FileInputStream(path)) {
byte[] buffer = new byte[8192];
int read;
while ((read = is.read(buffer)) > 0) {
digest.update(buffer, 0, read);
}
}
@@ -8,11 +8,9 @@ import org.springframework.stereotype.Service;
import ru.soune.nocopy.entity.file.FileEntity;
import ru.soune.nocopy.entity.file.ImageHashEntity;
import ru.soune.nocopy.repository.ImageHashRepository;
import ru.soune.nocopy.service.file.cloud.CloudStorageService;
import java.io.File;
import java.io.IOException;
import java.nio.file.Files;
import java.nio.file.Path;
import java.time.LocalDateTime;
import java.util.Map;
@@ -24,50 +22,18 @@ public class ImageHashService {
private final ImageHashRepository repository;
private final CloudStorageService cloudStorageService;
// public Map<String, Long> calculateHash(Path imagePath) throws IOException {
//// File file = cloudStorageService.readFileFromStorageByPath(imagePath.toString());
// File file = imagePath.toFile();
// if (file == null) {
// file = cloudStorageService.readFileFromStorageByPath(imagePath.toString());
// }
//
// PHash pHash = PerceptualHashHelper.INSTANCE.generateDCTPerceptualHash(file);
//
// Long firstPart = pHash.getFirstPart();
// Long secondPart = pHash.getSecondPart();
// Map<String, Long> hash = Map.of(
// "hi", firstPart,
// "low", secondPart);
//
// return hash;
// }
public Map<String, Long> calculateHash(Path imagePath) throws IOException {
PHash pHash;
File file = imagePath.toFile();
if (imagePath.toFile().exists()) {
pHash = PerceptualHashHelper.INSTANCE.generateDCTPerceptualHash(imagePath.toFile());
} else {
byte[] data = cloudStorageService.readFileFromStorageBytes(imagePath.toString());
Path tempFile = Files.createTempFile("perceptual_hash_", ".tmp");
try {
Files.write(tempFile, data);
pHash = PerceptualHashHelper.INSTANCE.generateDCTPerceptualHash(tempFile.toFile());
} finally {
Files.deleteIfExists(tempFile);
}
}
PHash pHash = PerceptualHashHelper.INSTANCE.generateDCTPerceptualHash(file);
Long firstPart = pHash.getFirstPart();
Long secondPart = pHash.getSecondPart();
return Map.of(
Map<String, Long> hash = Map.of(
"hi", firstPart,
"low", secondPart
);
"low", secondPart);
return hash;
}
public void create(FileEntity file, Map<String, Long> stringIntegerMap) {
@@ -62,6 +62,12 @@ public class ComplaintEntityService {
return mapToResponse(complaintRepository.save(complaint));
}
@Transactional(readOnly = true)
public Page<ComplaintResponse> getAllComplaintsByEmail(Pageable pageable, List<String> emails) {
return complaintRepository.getComplaintsByUserEmails(emails, pageable)
.map(this::mapToResponse);
}
@Transactional(readOnly = true)
public ComplaintResponse getComplaintById(Long id) {
return complaintRepository.findById(id)
@@ -121,13 +127,15 @@ public class ComplaintEntityService {
}
private ComplaintResponse mapToResponse(ComplaintEntity entity) {
Violation violation = entity.getViolation();
return ComplaintResponse.builder()
.id(entity.getId())
.status(entity.getStatus() != null ? entity.getStatus().name() : null)
.complaintText(entity.getComplaintText())
.email(entity.getEmail())
.violationId(entity.getViolation() != null ? entity.getViolation().getId() : null)
.violationId(violation != null ? violation.getId() : null)
.fileId(violation != null ? violation.getFileEntity().getId() : null)
.createdAt(entity.getCreatedAt() != null ? entity.getCreatedAt().format(DATE_FORMATTER) : null)
.updatedAt(entity.getUpdatedAt() != null ? entity.getUpdatedAt().format(DATE_FORMATTER) : null)
.notModerated(entity.getNot_moderated_file() != null ? entity.getNot_moderated_file() : null)
@@ -8,7 +8,6 @@ import lombok.extern.slf4j.Slf4j;
import ru.soune.nocopy.entity.user.ProtectedFileCheck;
import ru.soune.nocopy.repository.ProtectedFileCheckRepository;
import java.io.File;
import java.io.IOException;
import java.nio.file.*;
import java.nio.file.attribute.BasicFileAttributes;
@@ -16,10 +15,7 @@ import java.time.Instant;
import java.time.LocalDate;
import java.time.LocalDateTime;
import java.time.ZoneId;
import java.time.temporal.ChronoUnit;
import java.util.List;
import java.util.stream.Collectors;
import java.util.stream.Stream;
@Slf4j
@Component
@@ -38,8 +34,6 @@ public class FileCleanupService {
public void cleanupExpiredFiles() {
log.info("Starting cleanup of expired temporary files");
cleanupTempFilesNightly();
Path tempDir = Paths.get(basePath, "temp");
if (!Files.exists(tempDir)) {
return;
@@ -89,50 +83,4 @@ public class FileCleanupService {
log.error("Error during cleanup", e);
}
}
private void cleanupTempFilesNightly() {
log.info("Starting nightly cleanup of temporary files");
String tempDir = System.getProperty("java.io.tmpdir");
Path tempPath = Paths.get(tempDir);
if (!Files.exists(tempPath)) {
return;
}
Instant cutoffTime = Instant.now().minus(24, ChronoUnit.HOURS); // 24 часа
int deletedCount = 0;
try (Stream<Path> files = Files.list(tempPath)) {
List<Path> tempFiles = files
.filter(path -> {
String filename = path.getFileName().toString();
return filename.contains("_") &&
!filename.endsWith(".tmp") &&
Files.isRegularFile(path);
})
.collect(Collectors.toList());
for (Path filePath : tempFiles) {
try {
File file = filePath.toFile();
Instant lastModified = Instant.ofEpochMilli(file.lastModified());
if (lastModified.isBefore(cutoffTime)) {
boolean deleted = Files.deleteIfExists(filePath);
if (deleted) {
deletedCount++;
}
}
} catch (IOException e) {
log.error("Error deleting file: {}", filePath, e);
}
}
log.info("Nightly cleanup deleted {} temporary files", deletedCount);
} catch (IOException e) {
log.error("Failed to list temp directory", e);
}
}
}
@@ -19,15 +19,12 @@ import ru.soune.nocopy.exception.FileEntityNotFoundException;
import ru.soune.nocopy.repository.FileEntityRepository;
import ru.soune.nocopy.repository.FileMonitoringRepository;
import ru.soune.nocopy.repository.UserRepository;
import ru.soune.nocopy.service.file.cloud.CloudStorageService;
import java.io.File;
import java.io.IOException;
import java.nio.ByteBuffer;
import java.nio.channels.FileChannel;
import java.nio.file.Files;
import java.nio.file.Path;
import java.nio.file.Paths;
import java.nio.file.StandardOpenOption;
import java.time.LocalDateTime;
import java.util.ArrayList;
import java.util.List;
@@ -45,8 +42,6 @@ public class FileEntityService {
private final ImageResizeService imageResizeService;
private final CloudStorageService cloudStorageService;
@Value("${server.baseurl}")
private String baseUrl;
@@ -207,8 +202,16 @@ public class FileEntityService {
fileEntityRepository.save(fileEntity);
}
public void deleteFromDisk(String deleteFilePath) throws IOException {
Files.deleteIfExists(Paths.get(deleteFilePath));
public void deleteFromDisk(FileEntity fileEntity) throws IOException {
Path path = Paths.get(fileEntity.getFilePath());
if (!Files.exists(path)) {
return;
}
Files.delete(path);
fileEntityRepository.delete(fileEntity);
}
@Transactional(readOnly = true)
@@ -234,28 +237,19 @@ public class FileEntityService {
}
@Transactional
public FileEntity writeProtectedFile(String id, byte[] data, String fileExt) throws IOException {
public void writeProtectedFile(String id, byte[] data, String fileExt) throws IOException {
FileEntity fileEntity = fileEntityRepository.findById(id)
.orElseThrow(() -> new RuntimeException("File not found: " + id));
String extension = determineFileExtension(fileExt, fileEntity);
Path protectedFilePath = prepareProtectedPath(fileEntity, extension);
Files.createDirectories(protectedFilePath.getParent());
Files.deleteIfExists(protectedFilePath);
try (FileChannel channel = FileChannel.open(protectedFilePath,
StandardOpenOption.CREATE,
StandardOpenOption.WRITE)) {
ByteBuffer buffer = ByteBuffer.wrap(data);
while (buffer.hasRemaining()) {
channel.write(buffer);
}
channel.force(true);
if (Files.exists(protectedFilePath)) {
Files.delete(protectedFilePath);
}
Files.write(protectedFilePath, data);
if (imageResizeService.isImage(extension)) {
imageResizeService.generateSizes(fileEntity, data);
}
@@ -266,14 +260,14 @@ public class FileEntityService {
fileEntity.setFileExtension(extension);
fileEntity.setProtectionStatus(ProtectionStatus.PROTECTED);
return fileEntityRepository.save(fileEntity);
fileEntityRepository.save(fileEntity);
}
public void clearTempFiles(long userId) {
public void clearTempFiles(long userId) throws IOException {
List<FileEntity> fileByUserIdAndStatus = fileEntityRepository.findFileByUserIdAndStatus(userId, FileStatus.TEMP);
for (FileEntity fileEntity : fileByUserIdAndStatus) {
cloudStorageService.deleteFromStorageDisk(fileEntity);
deleteFromDisk(fileEntity);
}
}
@@ -285,6 +279,24 @@ public class FileEntityService {
return fileEntityRepository.findFileIdByFilePath(filePath);
}
public File getFileById(String id) {
try {
FileEntity fileEntity = fileEntityRepository.findById(id).orElseThrow(() ->
new RuntimeException("File not found: " + id));
File file = new File(fileEntity.getFilePath());
if (!file.exists()) {
throw new RuntimeException("File not found on disk: " + fileEntity.getFilePath());
}
return file;
} catch (Exception e) {
log.error("Error getting file: {}", id, e);
return null;
}
}
public Path prepareProtectedPath(FileEntity fileEntity, String extension) throws IOException {
Path originalPath = Paths.get(fileEntity.getFilePath());
@@ -310,7 +322,7 @@ public class FileEntityService {
return protectedPath;
}
public FileEntity updateSignature(String signature, String fileId) throws IOException {
public void updateSignature(String signature, String fileId) throws IOException {
FileEntity fileEntity = fileEntityRepository.findByFileId(fileId);
fileEntity.setSignature(signature);
fileEntity.setProtectionStatus(ProtectionStatus.PROTECTED);
@@ -318,30 +330,18 @@ public class FileEntityService {
fileEntity.setProtectedAt(LocalDateTime.now());
fileEntity.setProtectedFilePath(prepareProtectedPath(fileEntity, fileEntity.getFileExtension()).toString());
return fileEntityRepository.save(fileEntity);
fileEntityRepository.save(fileEntity);
}
public String formatFileSize(long size) {
if (size < 1024) {
return size + " B";
} else if (size < 1024 * 1024) {
return String.format("%.1f KB", size / 1024.0);
} else if (size < 1024 * 1024 * 1024) {
return String.format("%.1f MB", size / (1024.0 * 1024.0));
} else {
return String.format("%.1f GB", size / (1024.0 * 1024.0 * 1024.0));
private boolean checkFileExistsOnDisk(String filePath) {
try {
return Files.exists(Paths.get(filePath));
} catch (Exception e) {
log.warn("Error checking file existence: {}", filePath, e);
return false;
}
}
// private boolean checkFileExistsOnDisk(String filePath) {
// try {
// return Files.exists(Paths.get(filePath));
// } catch (Exception e) {
// log.warn("Error checking file existence: {}", filePath, e);
// return false;
// }
// }
private String determineFileExtension(String fileExt, FileEntity fileEntity) {
if (fileExt != null) {
return fileExt;
@@ -353,7 +353,7 @@ public class FileEntityService {
private final FileMonitoringRepository fileMonitoringRepository;
private FileEntityResponse convertToResponse(FileEntity fileEntity, int version) {
// boolean existsOnDisk = checkFileExistsOnDisk(fileEntity.getFilePath());
boolean existsOnDisk = checkFileExistsOnDisk(fileEntity.getFilePath());
User user = userRepository.findById(fileEntity.getUserId()).get();
String fileName = fileEntity.getOriginalFileName();
String name = fileName;
@@ -385,7 +385,7 @@ public class FileEntityService {
.updatedAt(fileEntity.getUpdatedAt())
.formattedSize(formatFileSize(fileEntity.getFileSize()))
.downloadUrl("/api/v" + version + "/files/download/" + fileEntity.getId())
// .existsOnDisk(existsOnDisk)
.existsOnDisk(existsOnDisk)
.supportId(fileEntity.getSupportId())
.protectStatus(fileEntity.getProtectionStatus().toString())
.fileName(name + "_nocopy_protected" + "." + fileEntity.getFileExtension())
@@ -400,4 +400,16 @@ public class FileEntityService {
.thumbnailFileUrl(baseUrl + "/api/files/protected/" + fileEntity.getId() + "/thumbnail")
.build();
}
public String formatFileSize(long size) {
if (size < 1024) {
return size + " B";
} else if (size < 1024 * 1024) {
return String.format("%.1f KB", size / 1024.0);
} else if (size < 1024 * 1024 * 1024) {
return String.format("%.1f MB", size / (1024.0 * 1024.0));
} else {
return String.format("%.1f GB", size / (1024.0 * 1024.0 * 1024.0));
}
}
}
@@ -2,10 +2,11 @@ package ru.soune.nocopy.service.file;
import org.springframework.web.multipart.MultipartFile;
import ru.soune.nocopy.dto.file.UploadProgressResponse;
import ru.soune.nocopy.entity.file.FileEntity;
import ru.soune.nocopy.entity.file.FileStatus;
import ru.soune.nocopy.entity.file.FileUploadSession;
import java.io.IOException;
public interface FileUploadService {
FileUploadSession initUpload(Long userId, String fileName,
String fileType, String extension, String convertTo, long fileSize);
@@ -21,5 +22,5 @@ public interface FileUploadService {
UploadProgressResponse getUploadProgress(String uploadId);
FileEntity completeFileProcessing(FileUploadSession session, FileStatus status);
void completeFileProcessingAsync(FileUploadSession session, FileStatus status);
}
@@ -1,12 +1,10 @@
package ru.soune.nocopy.service.file;
import lombok.RequiredArgsConstructor;
import lombok.extern.slf4j.Slf4j;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.stereotype.Service;
import org.springframework.transaction.annotation.Transactional;
import ru.soune.nocopy.entity.file.FileEntity;
import ru.soune.nocopy.service.file.cloud.CloudStorageService;
import javax.imageio.ImageIO;
import java.awt.*;
@@ -20,13 +18,9 @@ import java.util.List;
@Service
@Slf4j
@RequiredArgsConstructor
public class ImageResizeService {
private final CloudStorageService cloudStorageService;
private static final int THUMBNAIL_SIZE = 150;
private static final int MEDIUM_SIZE = 600;
@Value("${file.storage.base-path}")
@@ -56,15 +50,11 @@ public class ImageResizeService {
if (original.getWidth() > THUMBNAIL_SIZE) {
String thumbPath = resizeAndSave(original, baseName + "_thumb", THUMBNAIL_SIZE);
file.setThumbnailPath(thumbPath);
cloudStorageService.saveFilesToStorage(file.getMimeType(), file.getFileExtension(),
thumbPath);
}
if (original.getWidth() > MEDIUM_SIZE) {
String mediumPath = resizeAndSave(original, baseName + "_medium", MEDIUM_SIZE);
file.setMediumPath(mediumPath);
cloudStorageService.saveFilesToStorage(file.getMimeType(), file.getFileExtension(),
mediumPath);
}
}
}
@@ -1,194 +0,0 @@
package ru.soune.nocopy.service.file.cloud;
import lombok.RequiredArgsConstructor;
import lombok.extern.slf4j.Slf4j;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.stereotype.Service;
import ru.soune.nocopy.entity.file.FileEntity;
import ru.soune.nocopy.repository.FileEntityRepository;
import software.amazon.awssdk.auth.credentials.AwsBasicCredentials;
import software.amazon.awssdk.auth.credentials.AwsCredentials;
import software.amazon.awssdk.auth.credentials.StaticCredentialsProvider;
import software.amazon.awssdk.core.ResponseBytes;
import software.amazon.awssdk.core.sync.RequestBody;
import software.amazon.awssdk.core.sync.ResponseTransformer;
import software.amazon.awssdk.http.apache.ApacheHttpClient;
import software.amazon.awssdk.regions.Region;
import software.amazon.awssdk.services.s3.S3Client;
import software.amazon.awssdk.services.s3.model.GetObjectRequest;
import software.amazon.awssdk.services.s3.model.GetObjectResponse;
import software.amazon.awssdk.services.s3.model.PutObjectRequest;
import java.io.File;
import java.io.IOException;
import java.io.InputStream;
import java.net.URI;
import java.nio.file.Files;
import java.nio.file.Paths;
import java.util.UUID;
@Service
@Slf4j
@RequiredArgsConstructor
public class CloudStorageService {
private final FileEntityRepository fileEntityRepository;
@Value("${yandex.cloud.key}")
private String key;
@Value("${yandex.cloud.secret-key}")
private String secretKey;
@Value("${yandex.cloud.region}")
private String region;
@Value("${yandex.cloud.s3-endpoint}")
private String s3endpoint;
@Value("${yandex.cloud.bucket}")
private String bucket;
public void saveFilesToStorage(String mimeType, String extension, String sendToCloudFilePath) {
AwsCredentials credentials = AwsBasicCredentials.create(key, secretKey);
S3Client s3Client = S3Client.builder()
.httpClient(ApacheHttpClient.create())
.region(Region.of(region))
.endpointOverride(URI.create(s3endpoint))
.credentialsProvider(StaticCredentialsProvider.create(credentials))
.build();
String mimeTypeFromFile = mimeType.equals("document") ? "application" : mimeType;
String contentType = mimeTypeFromFile + "/" + extension;
String filePath = "files" + sendToCloudFilePath;
PutObjectRequest putObjectRequest = PutObjectRequest.builder()
.bucket(bucket)
.key(filePath)
.contentType(contentType)
.build();
try {
byte[] bytes = Files.readAllBytes(Paths.get(sendToCloudFilePath));
s3Client.putObject(putObjectRequest,
RequestBody.fromBytes(bytes));
Files.deleteIfExists(Paths.get(sendToCloudFilePath));
} catch (IOException e) {
log.error("Failed to read file: {}", filePath, e);
}
}
public File readFileFromStorageByPath(String filePath) throws IOException {
GetObjectRequest objectRequest = GetObjectRequest.builder()
.bucket(bucket)
.key("files" + filePath)
.build();
AwsCredentials credentials = AwsBasicCredentials.create(key, secretKey);
S3Client s3Client = S3Client.builder()
.httpClient(ApacheHttpClient.create())
.region(Region.of(region))
.endpointOverride(URI.create(s3endpoint))
.credentialsProvider(StaticCredentialsProvider.create(credentials))
.build();
String tempDir = System.getProperty("java.io.tmpdir");
String fileName = System.currentTimeMillis() + "_" + UUID.randomUUID() + "_" +
filePath.replace("/", "_");
File tempFile = new File(tempDir, fileName);
s3Client.getObject(objectRequest, tempFile.toPath());
tempFile.deleteOnExit();
return tempFile;
}
public byte[] readFileFromStorageBytes(String filePath) {
AwsCredentials credentials = AwsBasicCredentials.create(key, secretKey);
S3Client s3Client = S3Client.builder()
.httpClient(ApacheHttpClient.create())
.region(Region.of(region))
.endpointOverride(URI.create(s3endpoint))
.credentialsProvider(StaticCredentialsProvider.create(credentials))
.build();
GetObjectRequest objectRequest = GetObjectRequest.builder()
.bucket(bucket)
.key("files" + filePath)
.build();
try {
ResponseBytes<GetObjectResponse> responseBytes = s3Client.getObjectAsBytes(objectRequest);
return responseBytes.asByteArray();
} catch (Exception e) {
log.error("Failed to read file from S3: {}", filePath, e);
throw new RuntimeException("Failed to read file from S3", e);
}
}
public InputStream readFileFromStorage(String filePath) {
GetObjectRequest objectRequest = GetObjectRequest.builder()
.bucket(bucket)
.key("files" + filePath)
.build();
AwsCredentials credentials = AwsBasicCredentials.create(key, secretKey);
S3Client s3Client = S3Client.builder()
.httpClient(ApacheHttpClient.create())
.region(Region.of(region))
.endpointOverride(URI.create(s3endpoint))
.credentialsProvider(StaticCredentialsProvider.create(credentials))
.build();
return s3Client.getObject(objectRequest, ResponseTransformer.toInputStream());
}
public void deleteFileFromStorage(String filePath) {
if (filePath == null || filePath.isEmpty()) {
log.warn("Attempted to delete file with empty path");
return;
}
AwsCredentials credentials = AwsBasicCredentials.create(key, secretKey);
S3Client s3Client = S3Client.builder()
.httpClient(ApacheHttpClient.create())
.region(Region.of(region))
.endpointOverride(URI.create(s3endpoint))
.credentialsProvider(StaticCredentialsProvider.create(credentials))
.build();
String cloudKey = "files" + filePath;
try {
try {
s3Client.headObject(builder -> builder.bucket(bucket).key(cloudKey).build());
} catch (Exception e) {
log.warn("File does not exist in cloud storage: {}", cloudKey);
return;
}
s3Client.deleteObject(builder -> builder.bucket(bucket).key(cloudKey).build());
log.info("File successfully deleted from cloud storage: {}", cloudKey);
} catch (Exception e) {
log.error("Failed to delete file from cloud storage: {}", cloudKey, e);
throw new RuntimeException("Failed to delete file from cloud storage: " + cloudKey, e);
}
}
public void deleteFromStorageDisk(FileEntity fileEntity) {
deleteFileFromStorage(fileEntity.getFilePath());
if (fileEntity.getProtectedFilePath() != null) {
deleteFileFromStorage(fileEntity.getProtectedFilePath());
}
fileEntityRepository.delete(fileEntity);
}
}
@@ -9,7 +9,6 @@ import org.jetbrains.annotations.Nullable;
import org.springframework.stereotype.Service;
import ru.soune.nocopy.entity.file.FileEntity;
import ru.soune.nocopy.service.file.FileEntityService;
import ru.soune.nocopy.service.file.cloud.CloudStorageService;
import ru.soune.nocopy.util.FileUtil;
import java.io.IOException;
@@ -23,8 +22,6 @@ public class AudioLocalSearchImpl implements AudioLocalSearch {
private final FileUtil fileUtil;
private final CloudStorageService cloudStorageService;
@Override
public @Nullable FileProtector.FileInfo findBySignature(@NotNull String signature) {
FileEntity fileEntity = fileEntityService.findBySignature(signature);
@@ -37,10 +34,7 @@ public class AudioLocalSearchImpl implements AudioLocalSearch {
@Override
public void updateSignature(@NotNull String signature, @NotNull FileProtector.FileInfo fileInfo) {
try {
FileEntity fileEntity = fileEntityService.updateSignature(signature, fileInfo.getId());
cloudStorageService.saveFilesToStorage(fileEntity.getMimeType(), fileEntity.getFileExtension(),
fileEntity.getProtectedFilePath());
fileEntityService.updateSignature(signature, fileInfo.getId());
} catch (IOException e) {
throw new RuntimeException(e);
}
@@ -25,23 +25,6 @@ public class DocumentLocalSearchImpl implements DocumentLocalSearch {
FileEntity fileEntity = fileEntityRepository.findByIdAndUserId(fileId, Long.valueOf(ownerId));
FileProtector.FileInfo fileInfo = fileUtil.createFileInfo(fileEntity, null);
log.info("fileInfo: {}", fileInfo);
log.info("fileInfo: {}", fileInfo);
log.info("fileInfo: {}", fileInfo);
log.info("fileInfo: {}", fileInfo);
log.info("fileInfo: {}", fileInfo);
log.info("fileInfo: {}", fileInfo);
log.info("fileInfo: {}", fileInfo);
log.info("fileInfo: {}", fileInfo);
log.info("fileInfo: {}", fileInfo);
log.info("fileInfo: {}", fileInfo);
log.info("fileInfo: {}", fileInfo);
log.info("fileInfo: {}", fileInfo);
log.info("fileInfo: {}", fileInfo);
log.info("fileInfo: {}", fileInfo);
log.info("fileInfo: {}", fileInfo);
log.info("fileInfo: {}", fileInfo);
log.info("fileInfo: {}", fileInfo);
log.info("fileInfo: {}", fileInfo);
return fileInfo;
}
@@ -6,7 +6,6 @@ import com.vrt.fileprotection.NoCopyCheckResult;
import com.vrt.fileprotection.audio.AudioCheckResult;
import com.vrt.fileprotection.documents.DocumentCheckResult;
import jakarta.annotation.PostConstruct;
import jakarta.persistence.EntityManager;
import lombok.RequiredArgsConstructor;
import lombok.extern.slf4j.Slf4j;
import org.springframework.beans.factory.annotation.Autowired;
@@ -29,8 +28,6 @@ import ru.soune.nocopy.service.ImageHashService;
import ru.soune.nocopy.service.cost.CostService;
import ru.soune.nocopy.service.file.FileEntityService;
import ru.soune.nocopy.service.file.FileUploadService;
import ru.soune.nocopy.service.file.cloud.CloudStorageService;
import ru.soune.nocopy.service.tariff.TariffConstants;
import ru.soune.nocopy.service.file.moderation.ModerationFileService;
import ru.soune.nocopy.service.notification.NotificationService;
import ru.soune.nocopy.service.register.AuthService;
@@ -98,8 +95,6 @@ public class FileUploadServiceImpl implements FileUploadService {
private final ModerationFileService moderationFileService;
private final CloudStorageService cloudStorageService;
@PostConstruct
public void init() {
try {
@@ -195,6 +190,7 @@ public class FileUploadServiceImpl implements FileUploadService {
}
@Override
@Transactional
public UploadProgressResponse uploadChunk(String uploadId, Integer chunkNumber,
MultipartFile chunkFile, Integer findSimilar) {
FileUploadSession session = sessionRepository.findById(uploadId)
@@ -226,9 +222,10 @@ public class FileUploadServiceImpl implements FileUploadService {
return UploadProgressResponse.fromSession(session);
}
@Async("fileUploadTaskExecutor")
@Transactional
@Override
public FileEntity completeFileProcessing(FileUploadSession session, FileStatus status) {
public void completeFileProcessingAsync(FileUploadSession session, FileStatus status) {
try {
Path filePath = Paths.get(session.getFilePath());
String checksum = calculateChecksum(filePath);
@@ -254,17 +251,22 @@ public class FileUploadServiceImpl implements FileUploadService {
if (session.getFileType().equals("image") && status != FileStatus.PRIVATE) {
Map<String, Long> hash = imageHashService.calculateHash(filePath);
imageHashService.create(saved, hash);
}
cleanupSessionFiles(session);
log.info("File processing completed for session: {}", session.getUploadId());
return saved;
if (status != FileStatus.TEMP && status != FileStatus.PRIVATE) {
FileProtector.FileInfo fileInfo = fileUtil.createFileInfo(fileEntity, session.getConvertTo());
noCopyFileService.addFile(fileInfo);
}
log.info("File processing completed for session: {}", session.getUploadId());
} catch (Exception e) {
log.error("Failed to complete file processing for session {}: {}",
session.getUploadId(), e.getMessage(), e);
throw new FileUploadException("Failed to complete file processing", e);
}
}
@@ -337,7 +339,7 @@ public class FileUploadServiceImpl implements FileUploadService {
session.setStatus(UploadStatus.COMPLETED);
session.setFilePath(finalFilePath);
completeFileProcessing(session, FileStatus.PRIVATE);
completeFileProcessingAsync(session, FileStatus.PRIVATE);
} else {
sessionRepository.save(session);
}
@@ -379,7 +381,7 @@ public class FileUploadServiceImpl implements FileUploadService {
chunkNumber, session.getChunksUploaded(), session.getTotalChunks());
boolean isLastChunk = session.getChunksUploaded().equals(session.getTotalChunks());
//TODO CHECK
if (isLastChunk) {
String finalFilePath = assembleFileSynchronously(session);
@@ -391,7 +393,6 @@ public class FileUploadServiceImpl implements FileUploadService {
session.getUserId());
File file = new File(finalFilePath);
// File file = cloudStorageService.readFileFromStorageByPath(finalFilePath);
if (session.getFileType().startsWith("audio") && !isWavFile(file)) {
session.setStatus(UploadStatus.FAILED);
@@ -431,29 +432,20 @@ public class FileUploadServiceImpl implements FileUploadService {
session.setStatus(UploadStatus.COMPLETED);
session.setFilePath(finalFilePath);
notificationService.addNotification(NotificationType.FILE_ADDED_TO_SYSTEM, session.getUserId());
FileStatus status = findSimilar == 0 ? FileStatus.MODERATION: FileStatus.TEMP;
if (status == FileStatus.TEMP) {
fileEntityService.clearTempFiles(session.getUserId());
}
FileEntity fileEntity = completeFileProcessing(session, status);
cloudStorageService.saveFilesToStorage(fileEntity.getMimeType(), fileEntity.getFileExtension(),
fileEntity.getFilePath());
completeFileProcessingAsync(session, status);
if (status != FileStatus.TEMP) {
String fileType = session.getFileType();
tariffInfoService.writeOffTokens(session.getUserId(), costService.protectFileCost(fileType),
OperationType.FILE_UPLOAD);
notificationService.addNotification(NotificationType.FILE_ADDED_TO_SYSTEM, session.getUserId());
}
if (status != FileStatus.TEMP) {
FileProtector.FileInfo fileInfo = fileUtil.createFileInfo(fileEntity, session.getConvertTo());
noCopyFileService.addFile(fileInfo);
tariffInfoService.writeOffTokens(session.getUserId(), costService.protectFileCost(fileType), OperationType.FILE_UPLOAD);
}
} else {
sessionRepository.save(session);
@@ -588,7 +580,6 @@ public class FileUploadServiceImpl implements FileUploadService {
private void checkForDuplicatesSynchronously(String filePath)
throws IOException , DuplicateImageException {
Path path = Paths.get(filePath);
Map<String, Long> hash = imageHashService.calculateHash(path);
List<SimilarImageProjection> duplicates = fileSimilarityService.findDuplicatedByHash(
@@ -11,7 +11,7 @@ import ru.soune.nocopy.entity.file.FileEntity;
import ru.soune.nocopy.repository.FileEntityRepository;
import ru.soune.nocopy.repository.FileUploadSessionRepository;
import ru.soune.nocopy.service.file.FileEntityService;
import ru.soune.nocopy.service.file.cloud.CloudStorageService;
import ru.soune.nocopy.service.file.FileUploadService;
import java.io.File;
import java.io.IOException;
@@ -32,53 +32,27 @@ public class ProtectionFileProviderImpl implements FileProtector.FileProvider {
private final FileEntityRepository fileEntityRepository;
private final CloudStorageService cloudStorageService;
@Nullable
@Override
public File getImageFile(@NotNull String id) {
FileEntity fileEntity = fileEntityRepository.findById(id).orElseThrow(() ->
new RuntimeException("File not found: " + id));
try {
return cloudStorageService.readFileFromStorageByPath(fileEntity.getFilePath());
} catch (IOException e) {
throw new RuntimeException(e);
}
return fileEntityService.getFileById(id);
}
@Nullable
@Override
public File getVideoFile(@NotNull String id) {
FileEntity fileEntity = fileEntityRepository.findById(id).orElseThrow(() ->
new RuntimeException("File not found: " + id));
try {
return cloudStorageService.readFileFromStorageByPath(fileEntity.getFilePath());
} catch (IOException e) {
throw new RuntimeException(e);
}
return fileEntityService.getFileById(id);
}
@Nullable
@Override
public File getAudioFile(@NotNull String id) {
FileEntity fileEntity = fileEntityRepository.findById(id).orElseThrow(() ->
new RuntimeException("File not found: " + id));
try {
return cloudStorageService.readFileFromStorageByPath(fileEntity.getFilePath());
} catch (IOException e) {
throw new RuntimeException(e);
}
return fileEntityService.getFileById(id);
}
@Override
public @Nullable File getDocument(@NotNull String id) {
FileEntity fileEntity = fileEntityRepository.findById(id).orElseThrow(() ->
new RuntimeException("File not found: " + id));
try {
return cloudStorageService.readFileFromStorageByPath(fileEntity.getFilePath());
} catch (IOException e) {
throw new RuntimeException(e);
}
return fileEntityService.getFileById(id);
}
@Override
@@ -113,20 +87,7 @@ public class ProtectionFileProviderImpl implements FileProtector.FileProvider {
@Override
public com.vrt.fileprotection.OperationResult writeAudioFile(@NotNull String id, @NotNull byte[] data, @Nullable String fileExt) {
try {
FileEntity fileEntity = fileEntityService.writeProtectedFile(id, data, fileExt);
Path filePath = Paths.get(fileEntity.getProtectedFilePath());
if (!Files.exists(filePath)) {
log.error("File not found after write: {}", filePath);
return OperationResult.Companion.failure("File not found");
}
log.info("File ready for upload: {} (size: {} bytes)", filePath, Files.size(filePath));
cloudStorageService.saveFilesToStorage(fileEntity.getMimeType(), fileEntity.getFileExtension(),
fileEntity.getProtectedFilePath());
fileEntityService.writeProtectedFile(id, data, fileExt);
return OperationResult.Companion.success();
} catch (Exception e) {
log.error("Failed to create protected file: {}", id, e);
@@ -138,11 +99,7 @@ public class ProtectionFileProviderImpl implements FileProtector.FileProvider {
@Override
public com.vrt.fileprotection.OperationResult writeVideoFile(@NotNull String id, @NotNull byte[] data, @Nullable String fileExt) {
try {
FileEntity fileEntity = fileEntityService.writeProtectedFile(id, data, fileExt);
cloudStorageService.saveFilesToStorage(fileEntity.getMimeType(), fileEntity.getFileExtension(),
fileEntity.getProtectedFilePath());
fileEntityService.writeProtectedFile(id, data, fileExt);
return OperationResult.Companion.success();
} catch (Exception e) {
log.error("Failed to create protected file: {}", id, e);
@@ -154,11 +111,7 @@ public class ProtectionFileProviderImpl implements FileProtector.FileProvider {
@Override
public com.vrt.fileprotection.OperationResult writeImageFile(@NotNull String id, @NotNull byte[] data, @Nullable String fileExt) {
try {
FileEntity fileEntity = fileEntityService.writeProtectedFile(id, data, fileExt);
cloudStorageService.saveFilesToStorage(fileEntity.getMimeType(), fileEntity.getFileExtension(),
fileEntity.getProtectedFilePath());
fileEntityService.writeProtectedFile(id, data, fileExt);
return OperationResult.Companion.success();
} catch (Exception e) {
log.error("Failed to create protected file: {}", id, e);
@@ -169,10 +122,7 @@ public class ProtectionFileProviderImpl implements FileProtector.FileProvider {
@Override
public @NotNull OperationResult writeDocumentFile(@NotNull String id, @NotNull byte[] data) {
try {
FileEntity fileEntity = fileEntityService.writeProtectedFile(id, data, null);
cloudStorageService.saveFilesToStorage(fileEntity.getMimeType(), fileEntity.getFileExtension(),
fileEntity.getProtectedFilePath());
fileEntityService.writeProtectedFile(id, data, null);
return OperationResult.Companion.success();
} catch (Exception e) {
log.error("Failed to create protected file: {}", id, e);
@@ -44,13 +44,13 @@ public class GlobalSearchService {
task.setProcessedFiles(0);
task.setCreatedAt(LocalDateTime.now());
globalSearchTaskRepository.save(task);
GlobalSearchTask save = globalSearchTaskRepository.save(task);
List<String> fileIds = filesToProcess.stream().map(FileEntity::getId).toList();
asyncProcessor.processFilesAsync(task.getTaskId(), fileIds, userId);
asyncProcessor.processFilesAsync(save.getTaskId(), fileIds, userId);
return task.getTaskId();
return save.getTaskId();
}
public List<FileEntity> getFilesToProcess(GlobalSearchStartRequest request, Long userId) {
@@ -13,6 +13,7 @@ import ru.soune.nocopy.exception.NotValidationPasswordException;
import ru.soune.nocopy.exception.UserNotFoundException;
import ru.soune.nocopy.repository.UserRepository;
import java.util.List;
import java.util.Optional;
@Service
@@ -46,6 +47,11 @@ public class UserService {
return mapToDTO(updatedUser);
}
@Transactional
public List<String> getUsersCompanyEmails(String company) {
return userRepository.emailsByCompany(company);
}
@Transactional
public User activateAndVerifyUser(long userId) {
Optional<User> users = userRepository.findById(userId);
@@ -56,6 +56,10 @@ public class ViolationService {
}
}
public Violation getById(Long violationId) {
return violationRepository.findById(violationId).orElse(null);
}
public List<FileViolationSummaryDTO> getFileViolationsSummary(Long userId) {
List<FileEntity> userFiles = fileEntityService.getAllUserFiles(userId);
List<Violation> violations = violationRepository.findByFileEntityIn(userFiles);
-7
View File
@@ -102,13 +102,6 @@ yandex:
api-key: ${YANDEX_API_KEY:AQVNyaVaUmgUb1GMCtf5zSEqFxy0woXrcMOOB43q}
folder-id: ${YANDEX_FOLDER_ID:b1gokpdbm6qfpsou8pcd}
search-url: ${YANDEX_SEARCH_URL:https://searchapi.api.cloud.yandex.net/v2/image/search_by_image}
cloud:
key: "YCAJEmHZcWxzf4oGWvETG3mms"
secret-key: "YCPAPZ32XC-LWb8xE0b_xTcBD8NZUCHpSOfu8LUG"
region: "ru-central1"
s3-endpoint: "https://storage.yandexcloud.net"
bucket: "no-copy-storage"
searchapi:
api-key: ${SEARCHAPI_API_KEY:5jyYZC8jSaxhZTwjMUhwtAXi}