diff --git a/src/main/java/ru/soune/nocopy/configuration/HandlerConfig.java b/src/main/java/ru/soune/nocopy/configuration/HandlerConfig.java index 386bb20..e159080 100644 --- a/src/main/java/ru/soune/nocopy/configuration/HandlerConfig.java +++ b/src/main/java/ru/soune/nocopy/configuration/HandlerConfig.java @@ -27,9 +27,9 @@ public class HandlerConfig { ResetPasswordHandler resetPasswordHandler, DaDataHandler daDataHandler, PaymentHandler paymentHandler, - GetViolationsHandler getViolationsHandler, CostHandler costHandler, - MonitoringHandler monitoringHandler + MonitoringHandler monitoringHandler, + ViolationHandler violationHandler ) { Map map = new HashMap<>(); map.put(20001, login); @@ -47,9 +47,9 @@ public class HandlerConfig { map.put(30003, referralHandler); map.put(30004, daDataHandler); map.put(30005, paymentHandler); - map.put(30006, getViolationsHandler); map.put(30007, monitoringHandler); map.put(30008, costHandler); + map.put(30009, violationHandler); return map; } diff --git a/src/main/java/ru/soune/nocopy/controller/GlobalSearchController.java b/src/main/java/ru/soune/nocopy/controller/GlobalSearchController.java index cab9462..c388962 100644 --- a/src/main/java/ru/soune/nocopy/controller/GlobalSearchController.java +++ b/src/main/java/ru/soune/nocopy/controller/GlobalSearchController.java @@ -36,10 +36,6 @@ public class GlobalSearchController { private final GlobalSearchTaskRepository searchTaskRepository; - private final GlobalSearchResultRepository globalSearchResultRepository; - - private final ObjectMapper objectMapper; - @PostMapping("/start") public ResponseEntity startSearch( @RequestBody GlobalSearchStartRequest request, @@ -74,8 +70,7 @@ public class GlobalSearchController { } @GetMapping("/status/{taskId}") - public ResponseEntity getStatus( - @PathVariable String taskId, + public ResponseEntity getStatus(@PathVariable String taskId, @RequestHeader(value = "Authorization", required = false) String tokenHeader) { if (tokenHeader == null || tokenHeader.isBlank()) { @@ -93,38 +88,6 @@ public class GlobalSearchController { GlobalSearchTask task = taskOptional.orElseThrow(); - List results = globalSearchResultRepository.findByTaskIdOrderByIdAsc(taskId); - - List fileResults = results.stream() - .map(result -> { - GlobalSearchFileResult dto = new GlobalSearchFileResult(); - dto.setFileId(result.getFileId()); - dto.setFileName(result.getFileName()); - dto.setThumbnail(result.getThumbnail()); - dto.setFileStatus(result.getFileStatus()); - - try { - YandexSearchResponse searchResponse = objectMapper.readValue( - result.getSearchResults(), - YandexSearchResponse.class - ); - - dto.setImages(searchResponse.getImages()); - dto.setPage(searchResponse.getPage()); - dto.setPageSize(searchResponse.getPageSize()); - dto.setTotalResults(searchResponse.getTotalResults()); - dto.setTotalPages(searchResponse.getTotalPages()); - - } catch (Exception e) { - log.error("Failed to parse search results for file: {}", result.getFileId(), e); - dto.setImages(new ArrayList<>()); - dto.setTotalResults(0); - } - - return dto; - }) - .collect(Collectors.toList()); - int progress = task.getTotalFiles() > 0 ? task.getProcessedFiles() * 100 / task.getTotalFiles() : 0; @@ -133,7 +96,6 @@ public class GlobalSearchController { response.setTaskId(taskId); response.setStatus(task.getStatus()); response.setProgress(progress); - response.setResults(fileResults); return ResponseEntity.ok(response); } diff --git a/src/main/java/ru/soune/nocopy/dto/BaseResponse.java b/src/main/java/ru/soune/nocopy/dto/BaseResponse.java index 75f1543..6a4e3b0 100644 --- a/src/main/java/ru/soune/nocopy/dto/BaseResponse.java +++ b/src/main/java/ru/soune/nocopy/dto/BaseResponse.java @@ -3,10 +3,12 @@ package ru.soune.nocopy.dto; import com.fasterxml.jackson.annotation.JsonFormat; import com.fasterxml.jackson.annotation.JsonProperty; import lombok.AllArgsConstructor; +import lombok.Builder; import lombok.Data; @Data @AllArgsConstructor +@Builder @JsonFormat(with = JsonFormat.Feature.ACCEPT_CASE_INSENSITIVE_PROPERTIES) public class BaseResponse { @JsonProperty("msg_id") diff --git a/src/main/java/ru/soune/nocopy/dto/monitoring/ViolationResponse.java b/src/main/java/ru/soune/nocopy/dto/monitoring/ViolationResponse.java deleted file mode 100644 index 188b4b8..0000000 --- a/src/main/java/ru/soune/nocopy/dto/monitoring/ViolationResponse.java +++ /dev/null @@ -1,43 +0,0 @@ -package ru.soune.nocopy.dto.monitoring; - -import com.fasterxml.jackson.annotation.JsonProperty; -import lombok.Builder; -import lombok.Data; -import ru.soune.nocopy.entity.monitoring.SearchEngine; - -import java.time.LocalDateTime; - -@Data -@Builder -public class ViolationResponse { - - @JsonProperty("id") - private String id; - - @JsonProperty("file_id") - private String fileId; - - @JsonProperty("file_name") - private String fileName; - - @JsonProperty("found_url") - private String foundUrl; - - @JsonProperty("page_url") - private String pageUrl; - - @JsonProperty("page_title") - private String pageTitle; - - @JsonProperty("host") - private String host; - - @JsonProperty("search_engine") - private SearchEngine searchEngine; - - @JsonProperty("found_at") - private LocalDateTime foundAt; - - @JsonProperty("is_new") - private boolean isNew; -} diff --git a/src/main/java/ru/soune/nocopy/dto/violation/ViolationRequest.java b/src/main/java/ru/soune/nocopy/dto/violation/ViolationRequest.java new file mode 100644 index 0000000..97cdbd2 --- /dev/null +++ b/src/main/java/ru/soune/nocopy/dto/violation/ViolationRequest.java @@ -0,0 +1,36 @@ +package ru.soune.nocopy.dto.violation; + + +import com.fasterxml.jackson.annotation.JsonProperty; +import lombok.Data; + +@Data +public class ViolationRequest { + + @JsonProperty("file_id") + private String fileId; + + @JsonProperty("page") + private Integer page = 0; + + @JsonProperty("size") + private Integer size = 10; + + @JsonProperty("sort_direction") + private String sortDirection = "desc"; + + @JsonProperty("status") + private String status; + + @JsonProperty("start_date") + private String startDate; + + @JsonProperty("end_date") + private String endDate; + + @JsonProperty("group_by") + private String groupBy; + + @JsonProperty("action") + private String action; +} \ No newline at end of file diff --git a/src/main/java/ru/soune/nocopy/dto/violation/ViolationResponse.java b/src/main/java/ru/soune/nocopy/dto/violation/ViolationResponse.java new file mode 100644 index 0000000..94f1321 --- /dev/null +++ b/src/main/java/ru/soune/nocopy/dto/violation/ViolationResponse.java @@ -0,0 +1,77 @@ +package ru.soune.nocopy.dto.violation; + +import com.fasterxml.jackson.annotation.JsonProperty; +import lombok.Builder; +import lombok.Data; +import ru.soune.nocopy.entity.violation.Violation; + +import java.time.LocalDateTime; +import java.util.List; + +@Data +@Builder +public class ViolationResponse { + + @JsonProperty("violations") + private List violations; + + @JsonProperty("total_elements") + private long totalElements; + + @JsonProperty("total_pages") + private int totalPages; + + @JsonProperty("current_page") + private int currentPage; + + @JsonProperty("page_size") + private int pageSize; + + @JsonProperty("has_next") + private boolean hasNext; + + @JsonProperty("has_previous") + private boolean hasPrevious; + + @Data + @Builder + public static class ViolationDto { + + @JsonProperty("id") + private Long id; + + @JsonProperty("url") + private String url; + + @JsonProperty("page_url") + private String pageUrl; + + @JsonProperty("page_title") + private String pageTitle; + + @JsonProperty("host") + private String host; + + @JsonProperty("status") + private String status; + + @JsonProperty("created_date") + private LocalDateTime createdDate; + + @JsonProperty("file_id") + private String fileId; + + public static ViolationDto fromEntity(Violation violation) { + return ViolationDto.builder() + .id(violation.getId()) + .url(violation.getUrl()) + .pageUrl(violation.getPageUrl()) + .pageTitle(violation.getPageTitle()) + .host(violation.getHost()) + .status(violation.getStatus()) + .createdDate(violation.getCreatedDate()) + .fileId(violation.getFileEntity() != null ? violation.getFileEntity().getId() : null) + .build(); + } + } +} diff --git a/src/main/java/ru/soune/nocopy/entity/file/Violation.java b/src/main/java/ru/soune/nocopy/entity/file/Violation.java deleted file mode 100644 index c5317bc..0000000 --- a/src/main/java/ru/soune/nocopy/entity/file/Violation.java +++ /dev/null @@ -1,57 +0,0 @@ -package ru.soune.nocopy.entity.file; - -import jakarta.persistence.*; -import lombok.*; -import org.springframework.data.annotation.CreatedDate; -import org.springframework.data.jpa.domain.support.AuditingEntityListener; -import ru.soune.nocopy.entity.user.User; - -import java.time.LocalDateTime; - -@Entity -@Table(name = "violations") -@Getter -@Setter -@NoArgsConstructor -@AllArgsConstructor -@ToString -@EntityListeners(AuditingEntityListener.class) -public class Violation { - - @Id - @GeneratedValue(strategy = GenerationType.IDENTITY) - private Long violationId; - - @ManyToOne(fetch = FetchType.LAZY) - @JoinColumn(name = "user_id", nullable = false) - @ToString.Exclude - private User user; - - @Column(name = "violation_type", nullable = false, length = 50) - private String violationType; - - @Column(name = "description", columnDefinition = "TEXT") - private String description; - - @Column(name = "content_id") - private Long contentId; - - @Column(name = "status", nullable = false, length = 20) - private String status = "new"; - - @Column(name = "severity", length = 20) - private String severity; - - @Column(name = "resolved_at") - private LocalDateTime resolvedAt; - - @Column(name = "resolved_by") - private Long resolvedBy; - - @Column(name = "resolution_notes", columnDefinition = "TEXT") - private String resolutionNotes; - - @CreatedDate - @Column(name = "created_at", nullable = false, updatable = false) - private LocalDateTime createdAt; -} diff --git a/src/main/java/ru/soune/nocopy/entity/search/GlobalSearchResult.java b/src/main/java/ru/soune/nocopy/entity/search/GlobalSearchResult.java index ab8661f..bd0af95 100644 --- a/src/main/java/ru/soune/nocopy/entity/search/GlobalSearchResult.java +++ b/src/main/java/ru/soune/nocopy/entity/search/GlobalSearchResult.java @@ -23,10 +23,5 @@ public class GlobalSearchResult { private String fileName; - private String thumbnail; - private String fileStatus; - - @Column(length = 5000) - private String searchResults; } diff --git a/src/main/java/ru/soune/nocopy/entity/user/User.java b/src/main/java/ru/soune/nocopy/entity/user/User.java index cd8febd..0c58638 100644 --- a/src/main/java/ru/soune/nocopy/entity/user/User.java +++ b/src/main/java/ru/soune/nocopy/entity/user/User.java @@ -8,7 +8,6 @@ import org.hibernate.annotations.ColumnDefault; import org.springframework.data.annotation.CreatedDate; import org.springframework.data.jpa.domain.support.AuditingEntityListener; import ru.soune.nocopy.entity.company.Company; -import ru.soune.nocopy.entity.file.Violation; import ru.soune.nocopy.entity.tarif.TariffInfo; import ru.soune.nocopy.entity.tarif.TariffStatus; @@ -81,11 +80,6 @@ public class User { @ToString.Exclude private List userContents = new ArrayList<>(); - @OneToMany(mappedBy = "user", cascade = CascadeType.ALL, orphanRemoval = true) - @JsonIgnore - @ToString.Exclude - private List violations = new ArrayList<>(); - @OneToOne(mappedBy = "user", cascade = CascadeType.ALL, orphanRemoval = true) @JsonIgnore private ProtectedFileCheck protectedFileCheck; diff --git a/src/main/java/ru/soune/nocopy/entity/violation/Violation.java b/src/main/java/ru/soune/nocopy/entity/violation/Violation.java new file mode 100644 index 0000000..2c65045 --- /dev/null +++ b/src/main/java/ru/soune/nocopy/entity/violation/Violation.java @@ -0,0 +1,54 @@ +package ru.soune.nocopy.entity.violation; + +import com.fasterxml.jackson.annotation.JsonIgnore; +import jakarta.persistence.*; +import lombok.*; +import org.springframework.data.annotation.CreatedDate; +import ru.soune.nocopy.entity.file.FileEntity; +import ru.soune.nocopy.entity.search.GlobalSearchResult; + +import java.time.LocalDateTime; + +@Entity +@Table(name = "violation") +@Getter +@Setter +@NoArgsConstructor +@AllArgsConstructor +public class Violation { + + @Id + @GeneratedValue(strategy = GenerationType.IDENTITY) + private Long id; + + @Column(name = "url", nullable = false, unique = true) + private String url; + + @Column(name = "page_url") + private String pageUrl; + + @Column(name = "title") + private String pageTitle; + + @Column(name = "host") + private String host; + + @Column(name = "status") + private String status; + + @CreatedDate + @Column(name = "created_at", updatable = false) + private LocalDateTime createdDate; + + @ManyToOne(fetch = FetchType.LAZY) + @JoinColumn(name = "file") + @ToString.Exclude + @JsonIgnore + private FileEntity fileEntity; + + @ManyToOne(fetch = FetchType.LAZY) + @JoinColumn(name = "global_search_result") + @ToString.Exclude + @JsonIgnore + private GlobalSearchResult globalSearchResult; +} diff --git a/src/main/java/ru/soune/nocopy/exception/ViolationNotFoundException.java b/src/main/java/ru/soune/nocopy/exception/ViolationNotFoundException.java new file mode 100644 index 0000000..b088ed1 --- /dev/null +++ b/src/main/java/ru/soune/nocopy/exception/ViolationNotFoundException.java @@ -0,0 +1,7 @@ +package ru.soune.nocopy.exception; + +public class ViolationNotFoundException extends RuntimeException { + public ViolationNotFoundException(String message) { + super(message); + } +} diff --git a/src/main/java/ru/soune/nocopy/handler/GetViolationsHandler.java b/src/main/java/ru/soune/nocopy/handler/GetViolationsHandler.java deleted file mode 100644 index edd261e..0000000 --- a/src/main/java/ru/soune/nocopy/handler/GetViolationsHandler.java +++ /dev/null @@ -1,85 +0,0 @@ -package ru.soune.nocopy.handler; - -import com.fasterxml.jackson.databind.ObjectMapper; -import lombok.RequiredArgsConstructor; -import lombok.extern.slf4j.Slf4j; -import org.springframework.stereotype.Component; -import ru.soune.nocopy.dto.BaseRequest; -import ru.soune.nocopy.dto.BaseResponse; -import ru.soune.nocopy.dto.MessageCode; -import ru.soune.nocopy.dto.monitoring.GetViolationsRequest; -import ru.soune.nocopy.dto.monitoring.ViolationResponse; -import ru.soune.nocopy.entity.monitoring.FoundViolationEntity; -import ru.soune.nocopy.entity.user.User; -import ru.soune.nocopy.repository.FoundViolationRepository; -import ru.soune.nocopy.service.register.AuthService; - -import java.util.List; -import java.util.stream.Collectors; - -@Slf4j -@Component -@RequiredArgsConstructor -public class GetViolationsHandler implements RequestHandler { - private final ObjectMapper objectMapper; - - private final FoundViolationRepository violationRepository; - - private final AuthService authService; - - @Override - public BaseResponse handle(BaseRequest request) throws Exception { - GetViolationsRequest violationsRequest = objectMapper.convertValue( - request.getMessageBody(), GetViolationsRequest.class); - - String authToken = violationsRequest.getAuthToken(); - - //TODO add return for null token - if (authToken == null) { - return new BaseResponse(0, 0, "", new Object()); - } - - User user = authService.getAuthToken(authToken).getUser(); - List violations; - - if (violationsRequest.getFileId() != null) { - violations = violationRepository.findByFileIdOrderByFoundAtDesc( - violationsRequest.getFileId()); - } else { - violations = violationRepository.findByUserIdOrderByFoundAtDesc(user.getId()); - } - - if (violationsRequest.isMarkAsRead()) { - violations.stream() - .filter(FoundViolationEntity::isNew) - .forEach(v -> v.setNew(false)); - violationRepository.saveAll(violations); - } - - List response = violations.stream() - .map(this::mapToResponse) - .collect(Collectors.toList()); - - return new BaseResponse( - request.getMsgId(), - MessageCode.SUCCESS.getCode(), - "Violations retrieved", - response - ); - } - - private ViolationResponse mapToResponse(FoundViolationEntity entity) { - return ViolationResponse.builder() - .id(entity.getId()) - .fileId(entity.getFileId()) - .fileName(entity.getFileName()) - .foundUrl(entity.getFoundUrl()) - .pageUrl(entity.getPageUrl()) - .pageTitle(entity.getPageTitle()) - .host(entity.getHost()) - .searchEngine(entity.getSearchEngine()) - .foundAt(entity.getFoundAt()) - .isNew(entity.isNew()) - .build(); - } -} diff --git a/src/main/java/ru/soune/nocopy/handler/ViolationHandler.java b/src/main/java/ru/soune/nocopy/handler/ViolationHandler.java new file mode 100644 index 0000000..9989e50 --- /dev/null +++ b/src/main/java/ru/soune/nocopy/handler/ViolationHandler.java @@ -0,0 +1,139 @@ +package ru.soune.nocopy.handler; + +import com.fasterxml.jackson.databind.ObjectMapper; +import lombok.RequiredArgsConstructor; +import lombok.extern.slf4j.Slf4j; +import org.springframework.data.domain.Page; +import org.springframework.stereotype.Component; +import ru.soune.nocopy.dto.BaseRequest; +import ru.soune.nocopy.dto.BaseResponse; +import ru.soune.nocopy.dto.violation.ViolationRequest; +import ru.soune.nocopy.dto.violation.ViolationResponse; +import ru.soune.nocopy.entity.file.FileEntity; +import ru.soune.nocopy.entity.violation.Violation; +import ru.soune.nocopy.repository.FileEntityRepository; +import ru.soune.nocopy.service.violation.ViolationService; + +import java.io.FileNotFoundException; +import java.time.LocalDateTime; +import java.time.format.DateTimeFormatter; +import java.time.format.DateTimeParseException; +import java.util.List; +import java.util.Map; + +@Component +@Slf4j +@RequiredArgsConstructor +public class ViolationHandler implements RequestHandler { + private final ObjectMapper objectMapper; + + private final ViolationService violationService; + + private final FileEntityRepository fileRepository; + + private static final DateTimeFormatter DATE_FORMATTER = DateTimeFormatter.ofPattern("yyyy-MM-dd HH:mm:ss"); + + @Override + public BaseResponse handle(BaseRequest request) throws Exception { + ViolationRequest violationRequest = objectMapper.convertValue(request.getMessageBody(), ViolationRequest.class); + + validateRequest(violationRequest); + //TODO add response + FileEntity file = fileRepository.findById(violationRequest.getFileId()) + .orElseThrow(() -> new FileNotFoundException("File not found with id: " + violationRequest.getFileId())); + + LocalDateTime startDate = null; + LocalDateTime endDate = null; + + if (violationRequest.getStartDate() != null && violationRequest.getEndDate() != null) { + startDate = parseDate(violationRequest.getStartDate()); + endDate = parseDate(violationRequest.getEndDate()); + } + + if ("group".equalsIgnoreCase(violationRequest.getAction())) { + Map groupedData = violationService.getGroupedViolations( + file, + violationRequest.getGroupBy(), + violationRequest.getStatus(), + startDate, + endDate + ); + + return BaseResponse.builder() + .msgId(request.getMsgId()) + .messageBody(groupedData) + .build(); + } + // Пагинация + else { + Page violationPage; + + if (startDate != null && endDate != null) { + violationPage = violationService.getViolationsByFileAndDateRange( + file, startDate, endDate, + violationRequest.getPage(), violationRequest.getSize(), violationRequest.getSortDirection() + ); + } else if (violationRequest.getStatus() != null && !violationRequest.getStatus().isEmpty()) { + violationPage = violationService.getViolationsByFileAndStatus( + file, violationRequest.getStatus(), + violationRequest.getPage(), violationRequest.getSize(), violationRequest.getSortDirection() + ); + } else { + violationPage = violationService.getViolationsByFile( + file, + violationRequest.getPage(), violationRequest.getSize(), violationRequest.getSortDirection() + ); + } + + List violationDtos = violationPage.getContent() + .stream() + .map(ViolationResponse.ViolationDto::fromEntity) + .toList(); + + ViolationResponse response = ViolationResponse.builder() + .violations(violationDtos) + .totalElements(violationPage.getTotalElements()) + .totalPages(violationPage.getTotalPages()) + .currentPage(violationPage.getNumber()) + .pageSize(violationPage.getSize()) + .hasNext(violationPage.hasNext()) + .hasPrevious(violationPage.hasPrevious()) + .build(); + + return BaseResponse.builder() + .msgId(request.getMsgId()) + .messageBody(response) + .build(); + } + } + + //TODO add response exception + private void validateRequest(ViolationRequest request) { + if (request.getFileId() == null) { + return; + } + + if (request.getPage() < 0) { + request.setPage(0); + } + + if (request.getSize() <= 0 || request.getSize() > 100) { + request.setSize(10); + } + + if (!request.getSortDirection().equalsIgnoreCase("asc") && + !request.getSortDirection().equalsIgnoreCase("desc")) { + request.setSortDirection("desc"); + } + } + + //TODO add response exception + private LocalDateTime parseDate(String dateStr) { + try { + return LocalDateTime.parse(dateStr, DATE_FORMATTER); + } catch (DateTimeParseException e) { + log.error(e.getMessage()); + return null; + } + } +} diff --git a/src/main/java/ru/soune/nocopy/repository/ViolationRepository.java b/src/main/java/ru/soune/nocopy/repository/ViolationRepository.java new file mode 100644 index 0000000..b73ec54 --- /dev/null +++ b/src/main/java/ru/soune/nocopy/repository/ViolationRepository.java @@ -0,0 +1,39 @@ +package ru.soune.nocopy.repository; + +import org.springframework.data.domain.Page; +import org.springframework.data.domain.Pageable; +import org.springframework.data.jpa.repository.JpaRepository; +import org.springframework.stereotype.Repository; +import ru.soune.nocopy.entity.file.FileEntity; +import ru.soune.nocopy.entity.violation.Violation; + +import java.time.LocalDateTime; +import java.util.List; +import java.util.Optional; + +@Repository +public interface ViolationRepository extends JpaRepository { + + Page findByFileEntity(FileEntity file, Pageable pageable); + + Page findByFileEntityAndStatus(FileEntity file, String status, Pageable pageable); + + Page findByFileEntityAndCreatedDateBetween(FileEntity file, LocalDateTime startDate, + LocalDateTime endDate, Pageable pageable); + + List findByFileEntity(FileEntity file); + + List findByFileEntityAndStatus(FileEntity file, String status); + + List findByFileEntityAndCreatedDateBetween(FileEntity file, LocalDateTime startDate, + LocalDateTime endDate); + + List findByFileEntityAndStatusAndCreatedDateBetween(FileEntity file, String status, + LocalDateTime startDate, LocalDateTime endDate); + + boolean existsByUrl(String url); + + Optional findById(Long id); + + void deleteByFileEntity(FileEntity file); +} \ No newline at end of file diff --git a/src/main/java/ru/soune/nocopy/service/monitoring/MonitoringSearchService.java b/src/main/java/ru/soune/nocopy/service/monitoring/MonitoringSearchService.java index 9fa44d8..5eedfa5 100644 --- a/src/main/java/ru/soune/nocopy/service/monitoring/MonitoringSearchService.java +++ b/src/main/java/ru/soune/nocopy/service/monitoring/MonitoringSearchService.java @@ -3,32 +3,28 @@ package ru.soune.nocopy.service.monitoring; import jakarta.mail.MessagingException; import lombok.RequiredArgsConstructor; import lombok.extern.slf4j.Slf4j; -import org.springframework.scheduling.annotation.Async; import org.springframework.stereotype.Service; import org.springframework.transaction.annotation.Transactional; import ru.soune.nocopy.dto.file.YandexSearchResponse; import ru.soune.nocopy.dto.tarriff.TariffDTO; -import ru.soune.nocopy.entity.file.*; import ru.soune.nocopy.entity.monitoring.FileMonitoringEntity; -import ru.soune.nocopy.entity.monitoring.FoundViolationEntity; import ru.soune.nocopy.entity.monitoring.MonitoringType; -import ru.soune.nocopy.entity.monitoring.SearchEngine; +import ru.soune.nocopy.entity.search.GlobalSearchResult; import ru.soune.nocopy.entity.tarif.TariffInfo; import ru.soune.nocopy.entity.tarif.TariffType; import ru.soune.nocopy.entity.user.User; import ru.soune.nocopy.exception.TariffNotFoundException; import ru.soune.nocopy.repository.FileMonitoringRepository; -import ru.soune.nocopy.repository.FoundViolationRepository; import ru.soune.nocopy.repository.UserRepository; import ru.soune.nocopy.service.mail.EmailService; import ru.soune.nocopy.service.search.SearchImageService; import ru.soune.nocopy.service.tariff.TariffInfoService; import ru.soune.nocopy.service.tariff.TariffService; +import ru.soune.nocopy.service.violation.ViolationService; import java.io.IOException; import java.time.LocalDateTime; import java.util.List; -import java.util.UUID; import java.util.concurrent.TimeoutException; @Slf4j @@ -37,8 +33,6 @@ import java.util.concurrent.TimeoutException; public class MonitoringSearchService { private final SearchImageService searchImageService; - private final FoundViolationRepository violationRepository; - private final FileMonitoringRepository monitoringRepository; private final TariffInfoService tariffInfoService; @@ -49,18 +43,16 @@ public class MonitoringSearchService { private final UserRepository userRepository; + private final ViolationService violationService; + @Transactional public void processFileSearch(FileMonitoringEntity monitoring) throws MessagingException, IOException { - String searchSessionId = UUID.randomUUID().toString(); - log.info("Starting search session: {} for file: {}", searchSessionId, monitoring.getFile().getId()); - monitoring.setLastRun(LocalDateTime.now()); monitoring.setLastRunStatus("IN_PROGRESS"); monitoringRepository.save(monitoring); MonitoringType monitoringType = monitoring.getMonitoringType(); TariffDTO tariffMonitoring = tariffService.getTariffByType(TariffType.valueOf(monitoringType.name())); - try { tariffInfoService.writeOffTokens(monitoring.getUserId(), tariffMonitoring.getTokens()); @@ -70,7 +62,9 @@ public class MonitoringSearchService { List yandexImages = searchImageService.getAllImagesWithoutPagination(yandexResponse, "visual_matches"); - int savedCount = saveFoundImages(monitoring, yandexImages, SearchEngine.YANDEX, searchSessionId); + for (YandexSearchResponse.ImageResult imageResult : yandexImages) { + violationService.processViolation(imageResult, monitoring.getFile(), null); + } try { String googleResponse = searchImageService.searchReverseByPublicUrl( @@ -79,13 +73,16 @@ public class MonitoringSearchService { List googleImages = searchImageService.getAllImagesWithoutPagination(googleResponse, "exact_matches"); - savedCount += saveFoundImages(monitoring, googleImages, SearchEngine.GOOGLE, searchSessionId); + + for (YandexSearchResponse.ImageResult imageResult : googleImages) { + violationService.processViolation(imageResult, monitoring.getFile(), null); + } } catch (TimeoutException | IOException e) { - log.warn("Google search failed for session: {}", searchSessionId, e); + log.warn("Google search failed"); } - monitoring.setLastRunStatus("SUCCESS - Found " + savedCount + " violations"); + monitoring.setLastRunStatus("SUCCESS"); } catch (TariffNotFoundException e) { User user = userRepository.findById(monitoring.getUserId()).orElseThrow(); TariffInfo activeTariffInfo = user.getActiveTariffInfo(); @@ -97,7 +94,6 @@ public class MonitoringSearchService { monitoringRepository.save(monitoring); } catch (Exception e) { - log.error("Search failed for session: {}", searchSessionId, e); monitoring.setLastRunStatus("ERROR: " + e.getMessage()); } finally { updateNextRun(monitoring); @@ -105,42 +101,6 @@ public class MonitoringSearchService { } } - private int saveFoundImages(FileMonitoringEntity monitoring, - List images, - SearchEngine engine, - String sessionId) { - int savedCount = 0; - - for (YandexSearchResponse.ImageResult image : images) { - List existing = violationRepository.findDuplicate( - monitoring.getFile().getId(), image.getUrl()); - - if (existing.isEmpty()) { - FoundViolationEntity violation = FoundViolationEntity.builder() - .file(monitoring.getFile()) - .fileId(monitoring.getFile().getId()) - .fileName(monitoring.getFile().getOriginalFileName()) - .userId(monitoring.getUserId()) - .foundUrl(image.getUrl()) - .pageUrl(image.getPageUrl()) - .pageTitle(image.getPageTitle()) - .host(image.getHost()) - .imageWidth(image.getWidth()) - .imageHeight(image.getHeight()) - .searchEngine(engine) - .searchSessionId(sessionId) - .isNew(true) - .isNotified(false) - .build(); - - violationRepository.save(violation); - savedCount++; - } - } - - return savedCount; - } - private void updateNextRun(FileMonitoringEntity monitoring) { LocalDateTime nextRun = calculateNextRun(monitoring.getMonitoringType()); monitoring.setNextRun(nextRun); diff --git a/src/main/java/ru/soune/nocopy/service/search/GlobalSearchAsyncProcessor.java b/src/main/java/ru/soune/nocopy/service/search/GlobalSearchAsyncProcessor.java index cd55272..ab85231 100644 --- a/src/main/java/ru/soune/nocopy/service/search/GlobalSearchAsyncProcessor.java +++ b/src/main/java/ru/soune/nocopy/service/search/GlobalSearchAsyncProcessor.java @@ -1,6 +1,7 @@ package ru.soune.nocopy.service.search; import com.fasterxml.jackson.databind.ObjectMapper; +import com.vrt.fileprotection.image.ImageCheckResult; import lombok.RequiredArgsConstructor; import lombok.extern.slf4j.Slf4j; import org.springframework.scheduling.annotation.Async; @@ -11,10 +12,13 @@ import ru.soune.nocopy.entity.file.FileEntity; import ru.soune.nocopy.entity.search.GlobalSearchResult; import ru.soune.nocopy.entity.search.GlobalSearchTask; import ru.soune.nocopy.entity.search.SearchStatus; +import ru.soune.nocopy.entity.violation.Violation; import ru.soune.nocopy.repository.GlobalSearchResultRepository; import ru.soune.nocopy.repository.GlobalSearchTaskRepository; +import ru.soune.nocopy.repository.ViolationRepository; import ru.soune.nocopy.service.tariff.TariffConstants; import ru.soune.nocopy.service.tariff.TariffInfoService; +import ru.soune.nocopy.service.violation.ViolationService; import java.io.IOException; import java.time.LocalDateTime; @@ -38,6 +42,10 @@ public class GlobalSearchAsyncProcessor { private final GlobalSearchTaskRepository globalSearchTaskRepository; + private final ViolationRepository violationRepository; + + private final ViolationService violationService; + @Async @Transactional public void processFilesAsync(String taskId, List filesToProcess, Long userId) { @@ -74,7 +82,6 @@ public class GlobalSearchAsyncProcessor { result.setTaskId(taskId); result.setFileId(file.getId()); result.setFileName(file.getOriginalFileName()); - result.setThumbnail(file.getThumbnailPath()); List allUniqueImages = new ArrayList<>(); boolean hasTimeout = false; @@ -117,21 +124,10 @@ public class GlobalSearchAsyncProcessor { allUniqueImages.stream() .filter(img -> img.getUrl() != null) .collect(Collectors.toList()), - new ArrayList<>() - ); + new ArrayList<>()); - YandexSearchResponse searchResponse = new YandexSearchResponse(); - searchResponse.setImages(allUniqueImages); - searchResponse.setPage(1); - searchResponse.setPageSize(allUniqueImages.size()); - searchResponse.setTotalResults(allUniqueImages.size()); - searchResponse.setTotalPages(1); - - try { - result.setSearchResults(objectMapper.writeValueAsString(searchResponse)); - } catch (Exception e) { - log.error("Failed to serialize search results", e); - result.setSearchResults("{}"); + for (YandexSearchResponse.ImageResult imageResult : allUniqueImages) { + violationService.processViolation(imageResult, file, result); } if (allUniqueImages.isEmpty() && hasTimeout) { diff --git a/src/main/java/ru/soune/nocopy/service/violation/ViolationService.java b/src/main/java/ru/soune/nocopy/service/violation/ViolationService.java new file mode 100644 index 0000000..9d8660d --- /dev/null +++ b/src/main/java/ru/soune/nocopy/service/violation/ViolationService.java @@ -0,0 +1,113 @@ +package ru.soune.nocopy.service.violation; + +import lombok.AllArgsConstructor; +import org.springframework.data.domain.Page; +import org.springframework.data.domain.PageRequest; +import org.springframework.data.domain.Pageable; +import org.springframework.data.domain.Sort; +import org.springframework.stereotype.Service; +import ru.soune.nocopy.dto.file.YandexSearchResponse; +import ru.soune.nocopy.entity.file.FileEntity; +import ru.soune.nocopy.entity.search.GlobalSearchResult; +import ru.soune.nocopy.entity.violation.Violation; +import ru.soune.nocopy.exception.ViolationNotFoundException; +import ru.soune.nocopy.repository.ViolationRepository; + +import java.time.LocalDateTime; +import java.util.List; +import java.util.Map; +import java.util.stream.Collectors; + +@Service +@AllArgsConstructor +public class ViolationService { + + private final ViolationRepository violationRepository; + + public void processViolation(YandexSearchResponse.ImageResult imageResult, FileEntity file, + GlobalSearchResult result) { + String url = imageResult.getUrl(); + + if (!violationRepository.existsByUrl(url)) { + Violation violation = new Violation(); + + if (result != null) { + violation.setGlobalSearchResult(result); + } + + violation.setHost(imageResult.getHost()); + violation.setUrl(url); + violation.setPageUrl(imageResult.getPageUrl()); + violation.setPageTitle(imageResult.getPageTitle()); + violation.setFileEntity(file); + violation.setStatus("CREATED"); + + violationRepository.save(violation); + } + } + + public List violationsByFileId(FileEntity file) { + return violationRepository.findByFileEntity(file); + } + + public void changeStatusViolation(Long violationId, String status) { + Violation violation = violationRepository.findById(violationId) + .orElseThrow(() -> new ViolationNotFoundException("Violation not found with id: " + violationId)); + violation.setStatus(status); + violationRepository.save(violation); + } + + public Page getViolationsByFile(FileEntity file, int page, int size, String sortDirection) { + Sort sort = Sort.by(sortDirection.equalsIgnoreCase("desc") ? Sort.Direction.DESC : Sort.Direction.ASC, "createdDate"); + Pageable pageable = PageRequest.of(page, size, sort); + return violationRepository.findByFileEntity(file, pageable); + } + + public Page getViolationsByFileAndStatus(FileEntity file, String status, int page, int size, String sortDirection) { + Sort sort = Sort.by(sortDirection.equalsIgnoreCase("desc") ? Sort.Direction.DESC : Sort.Direction.ASC, "createdDate"); + Pageable pageable = PageRequest.of(page, size, sort); + return violationRepository.findByFileEntityAndStatus(file, status, pageable); + } + + public Page getViolationsByFileAndDateRange(FileEntity file, LocalDateTime startDate, LocalDateTime endDate, + int page, int size, String sortDirection) { + Sort sort = Sort.by(sortDirection.equalsIgnoreCase("desc") ? Sort.Direction.DESC : Sort.Direction.ASC, "createdDate"); + Pageable pageable = PageRequest.of(page, size, sort); + return violationRepository.findByFileEntityAndCreatedDateBetween(file, startDate, endDate, pageable); + } + + public Map getGroupedViolations(FileEntity file, String groupBy, String status, + LocalDateTime startDate, LocalDateTime endDate) { + + List violations; + + if (status != null && !status.isEmpty() && startDate != null && endDate != null) { + violations = violationRepository.findByFileEntityAndStatusAndCreatedDateBetween(file, status, startDate, endDate); + } else if (status != null && !status.isEmpty()) { + violations = violationRepository.findByFileEntityAndStatus(file, status); + } else if (startDate != null && endDate != null) { + violations = violationRepository.findByFileEntityAndCreatedDateBetween(file, startDate, endDate); + } else { + violations = violationRepository.findByFileEntity(file); + } + + return violations.stream() + .collect(Collectors.groupingBy( + v -> extractGroupKey(v.getUrl(), groupBy), + Collectors.counting() + )); + } + + private String extractGroupKey(String url, String groupBy) { + try { + String domain = url.replaceAll("https?://(www\\.)?", "").split("/")[0]; + if ("tld".equalsIgnoreCase(groupBy)) { + int lastDot = domain.lastIndexOf('.'); + return lastDot > 0 ? domain.substring(lastDot + 1) : domain; + } + return domain; + } catch (Exception e) { + return "unknown"; + } + } +} \ No newline at end of file