dev add violation notion
This commit is contained in:
@@ -0,0 +1,26 @@
|
|||||||
|
package ru.soune.nocopy.configuration.search;
|
||||||
|
|
||||||
|
import lombok.Data;
|
||||||
|
import org.springframework.boot.context.properties.ConfigurationProperties;
|
||||||
|
import org.springframework.stereotype.Component;
|
||||||
|
import java.util.Map;
|
||||||
|
|
||||||
|
@Data
|
||||||
|
@Component
|
||||||
|
@ConfigurationProperties(prefix = "search")
|
||||||
|
public class SearchProperties {
|
||||||
|
private Map<String, EngineConfig> engines;
|
||||||
|
private ProxyConfig proxy;
|
||||||
|
|
||||||
|
@Data
|
||||||
|
public static class EngineConfig {
|
||||||
|
private boolean enabled;
|
||||||
|
}
|
||||||
|
|
||||||
|
@Data
|
||||||
|
public static class ProxyConfig {
|
||||||
|
private boolean enabled;
|
||||||
|
private String host;
|
||||||
|
private int port;
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -0,0 +1,58 @@
|
|||||||
|
package ru.soune.nocopy.controller;
|
||||||
|
|
||||||
|
import lombok.RequiredArgsConstructor;
|
||||||
|
import lombok.extern.slf4j.Slf4j;
|
||||||
|
import org.springframework.http.ResponseEntity;
|
||||||
|
import org.springframework.web.bind.annotation.*;
|
||||||
|
import ru.soune.nocopy.configuration.search.SearchProperties;
|
||||||
|
import ru.soune.nocopy.dto.BaseResponse;
|
||||||
|
import ru.soune.nocopy.dto.search.config.SearchSettingsDto;
|
||||||
|
|
||||||
|
import java.util.Map;
|
||||||
|
|
||||||
|
|
||||||
|
@Slf4j
|
||||||
|
@RestController
|
||||||
|
@RequestMapping("/api/search/settings")
|
||||||
|
@RequiredArgsConstructor
|
||||||
|
public class SearchSettingsController {
|
||||||
|
|
||||||
|
private final SearchProperties searchProperties;
|
||||||
|
|
||||||
|
@GetMapping
|
||||||
|
public ResponseEntity<?> getSettings() {
|
||||||
|
SearchSettingsDto dto = new SearchSettingsDto();
|
||||||
|
|
||||||
|
dto.setEngines(Map.of(
|
||||||
|
"yandex", searchProperties.getEngines().get("yandex").isEnabled(),
|
||||||
|
"google", searchProperties.getEngines().get("google").isEnabled()));
|
||||||
|
|
||||||
|
dto.setProxyEnabled(searchProperties.getProxy().isEnabled());
|
||||||
|
|
||||||
|
return ResponseEntity.ok().body(BaseResponse.builder()
|
||||||
|
.messageDesc("Success")
|
||||||
|
.messageBody(dto)
|
||||||
|
.build());
|
||||||
|
}
|
||||||
|
|
||||||
|
@PutMapping
|
||||||
|
public ResponseEntity<?> updateSettings(@RequestBody SearchSettingsDto settings) {
|
||||||
|
if (settings.getEngines() != null) {
|
||||||
|
settings.getEngines().forEach((key, value) -> {
|
||||||
|
if (searchProperties.getEngines().containsKey(key)) {
|
||||||
|
searchProperties.getEngines().get(key).setEnabled(value);
|
||||||
|
}
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
|
if (settings.getProxyEnabled() != null) {
|
||||||
|
searchProperties.getProxy().setEnabled(settings.getProxyEnabled());
|
||||||
|
}
|
||||||
|
|
||||||
|
log.info("Settings updated: {}", settings);
|
||||||
|
|
||||||
|
return ResponseEntity.ok().body(BaseResponse.builder()
|
||||||
|
.messageDesc("Settings updated")
|
||||||
|
.build());
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -0,0 +1,11 @@
|
|||||||
|
package ru.soune.nocopy.dto.search.config;
|
||||||
|
|
||||||
|
|
||||||
|
import lombok.Data;
|
||||||
|
import java.util.Map;
|
||||||
|
|
||||||
|
@Data
|
||||||
|
public class SearchSettingsDto {
|
||||||
|
private Map<String, Boolean> engines;
|
||||||
|
private Boolean proxyEnabled;
|
||||||
|
}
|
||||||
@@ -4,6 +4,7 @@ import com.fasterxml.jackson.databind.ObjectMapper;
|
|||||||
import lombok.RequiredArgsConstructor;
|
import lombok.RequiredArgsConstructor;
|
||||||
import lombok.extern.slf4j.Slf4j;
|
import lombok.extern.slf4j.Slf4j;
|
||||||
import org.springframework.stereotype.Component;
|
import org.springframework.stereotype.Component;
|
||||||
|
import ru.soune.nocopy.configuration.search.SearchProperties;
|
||||||
import ru.soune.nocopy.dto.BaseRequest;
|
import ru.soune.nocopy.dto.BaseRequest;
|
||||||
import ru.soune.nocopy.dto.BaseResponse;
|
import ru.soune.nocopy.dto.BaseResponse;
|
||||||
import ru.soune.nocopy.dto.MessageCode;
|
import ru.soune.nocopy.dto.MessageCode;
|
||||||
@@ -36,6 +37,8 @@ public class ImageFoundRequestHandler implements RequestHandler {
|
|||||||
|
|
||||||
private final TariffInfoService tariffInfoService;
|
private final TariffInfoService tariffInfoService;
|
||||||
|
|
||||||
|
private final SearchProperties searchProperties;
|
||||||
|
|
||||||
@Override
|
@Override
|
||||||
public BaseResponse handle(BaseRequest request) throws Exception {
|
public BaseResponse handle(BaseRequest request) throws Exception {
|
||||||
ImageSearchRequest imageSearchRequest = objectMapper.convertValue(request.getMessageBody(),
|
ImageSearchRequest imageSearchRequest = objectMapper.convertValue(request.getMessageBody(),
|
||||||
@@ -47,32 +50,39 @@ public class ImageFoundRequestHandler implements RequestHandler {
|
|||||||
.orElseThrow(() -> {
|
.orElseThrow(() -> {
|
||||||
throw new NotValidFieldException("File not found", new BaseResponse(20007,
|
throw new NotValidFieldException("File not found", new BaseResponse(20007,
|
||||||
MessageCode.FILE_NOT_FOUND.getCode(), MessageCode.FILE_NOT_FOUND.getDescription(),
|
MessageCode.FILE_NOT_FOUND.getCode(), MessageCode.FILE_NOT_FOUND.getDescription(),
|
||||||
Map.of("fileId",fileId)));
|
Map.of("fileId", fileId)));});
|
||||||
});
|
|
||||||
|
|
||||||
String searchResponseYandex;
|
|
||||||
// String searchResponseGoogle;
|
|
||||||
List<YandexSearchResponse.ImageResult> allYandexImages = new ArrayList<>();
|
List<YandexSearchResponse.ImageResult> allYandexImages = new ArrayList<>();
|
||||||
// List<YandexSearchResponse.ImageResult> allGoogleImages = new ArrayList<>();
|
List<YandexSearchResponse.ImageResult> allGoogleImages = new ArrayList<>();
|
||||||
List<YandexSearchResponse.ImageResult> allUniqueImages;
|
|
||||||
boolean hasTimeout = false;
|
boolean hasTimeout = false;
|
||||||
//
|
|
||||||
// try {
|
|
||||||
// searchResponseGoogle = searchImageService.searchReverseByPublicUrl(fileEntity, "google_lens",
|
|
||||||
// "exact_matches");
|
|
||||||
// log.info("searchResponseGoogle OK,{}", searchResponseGoogle);
|
|
||||||
// allGoogleImages = searchImageService.getAllImagesWithoutPagination(
|
|
||||||
// searchResponseGoogle, "exact_matches");
|
|
||||||
//
|
|
||||||
// } catch (TimeoutException e) {
|
|
||||||
// log.warn("Google search timeout for file {}", fileId);
|
|
||||||
// hasTimeout = true;
|
|
||||||
// } catch (IOException e) {
|
|
||||||
// log.error("Google search failed for file {}", fileId, e);
|
|
||||||
// }
|
|
||||||
|
|
||||||
|
boolean useGoogle = searchProperties.getEngines().getOrDefault("google",
|
||||||
|
new SearchProperties.EngineConfig()).isEnabled();
|
||||||
|
boolean useYandex = searchProperties.getEngines().getOrDefault("yandex",
|
||||||
|
new SearchProperties.EngineConfig()).isEnabled();
|
||||||
|
|
||||||
|
log.info("Search settings: useGoogle={}, useYandex={}", useGoogle, useYandex);
|
||||||
|
|
||||||
|
if (useGoogle) {
|
||||||
try {
|
try {
|
||||||
searchResponseYandex = searchImageService.searchReverseByPublicUrl(fileEntity, "yandex_reverse_image",
|
String searchResponseGoogle = searchImageService.searchReverseByPublicUrl(fileEntity, "google_lens",
|
||||||
|
"exact_matches");
|
||||||
|
log.info("searchResponseGoogle OK");
|
||||||
|
allGoogleImages = searchImageService.getAllImagesWithoutPagination(
|
||||||
|
searchResponseGoogle, "exact_matches");
|
||||||
|
} catch (TimeoutException e) {
|
||||||
|
log.warn("Google search timeout for file {}", fileId);
|
||||||
|
hasTimeout = true;
|
||||||
|
} catch (IOException e) {
|
||||||
|
log.error("Google search failed for file {}", fileId, e);
|
||||||
|
}
|
||||||
|
} else {
|
||||||
|
log.info("Google search is disabled by settings");
|
||||||
|
}
|
||||||
|
|
||||||
|
if (useYandex) {
|
||||||
|
try {
|
||||||
|
String searchResponseYandex = searchImageService.searchReverseByPublicUrl(fileEntity, "yandex_reverse_image",
|
||||||
"visual_matches");
|
"visual_matches");
|
||||||
log.info("searchResponseYandex OK");
|
log.info("searchResponseYandex OK");
|
||||||
allYandexImages = searchImageService.getAllImagesWithoutPagination(
|
allYandexImages = searchImageService.getAllImagesWithoutPagination(
|
||||||
@@ -83,11 +93,29 @@ public class ImageFoundRequestHandler implements RequestHandler {
|
|||||||
} catch (IOException e) {
|
} catch (IOException e) {
|
||||||
log.error("Yandex search failed for file {}", fileId, e);
|
log.error("Yandex search failed for file {}", fileId, e);
|
||||||
}
|
}
|
||||||
|
} else {
|
||||||
|
log.info("Yandex search is disabled by settings");
|
||||||
|
}
|
||||||
|
|
||||||
// allUniqueImages = searchImageService.removeDuplicateUrls(allYandexImages, allGoogleImages);
|
if (allYandexImages.isEmpty() && allGoogleImages.isEmpty()) {
|
||||||
// allUniqueImages = allGoogleImages;
|
log.warn("No results from any search engine");
|
||||||
|
String errorMessage = useGoogle || useYandex ? "No results found" : "All search engines are disabled";
|
||||||
|
|
||||||
|
return new BaseResponse(request.getMsgId(), MessageCode.SUCCESS.getCode(),
|
||||||
|
errorMessage, new YandexSearchResponse());
|
||||||
|
}
|
||||||
|
|
||||||
|
List<YandexSearchResponse.ImageResult> allUniqueImages;
|
||||||
|
if (useGoogle && useYandex) {
|
||||||
|
allUniqueImages = searchImageService.removeDuplicateUrls(allYandexImages, allGoogleImages);
|
||||||
|
log.info("Merged results from both engines: {} unique images", allUniqueImages.size());
|
||||||
|
} else if (useGoogle) {
|
||||||
|
allUniqueImages = allGoogleImages;
|
||||||
|
log.info("Results only from Google: {} images", allUniqueImages.size());
|
||||||
|
} else {
|
||||||
allUniqueImages = allYandexImages;
|
allUniqueImages = allYandexImages;
|
||||||
log.info("allUniqueImages OK: {} images", allUniqueImages.size());
|
log.info("Results only from Yandex: {} images", allUniqueImages.size());
|
||||||
|
}
|
||||||
|
|
||||||
tariffInfoService.writeOffTokens(fileEntity.getUserId(), TariffConstants.TOKEN_VALUE_FOR_SEARCH);
|
tariffInfoService.writeOffTokens(fileEntity.getUserId(), TariffConstants.TOKEN_VALUE_FOR_SEARCH);
|
||||||
checkCounterService.incrementCheckCount(fileEntity.getUserId(), fileEntity.getMimeType());
|
checkCounterService.incrementCheckCount(fileEntity.getUserId(), fileEntity.getMimeType());
|
||||||
|
|||||||
@@ -34,8 +34,6 @@ public class FileCleanupService {
|
|||||||
public void cleanupExpiredFiles() {
|
public void cleanupExpiredFiles() {
|
||||||
log.info("Starting cleanup of expired temporary files");
|
log.info("Starting cleanup of expired temporary files");
|
||||||
|
|
||||||
// cleanupOldPeriods();
|
|
||||||
|
|
||||||
Path tempDir = Paths.get(basePath, "temp");
|
Path tempDir = Paths.get(basePath, "temp");
|
||||||
if (!Files.exists(tempDir)) {
|
if (!Files.exists(tempDir)) {
|
||||||
return;
|
return;
|
||||||
@@ -85,17 +83,4 @@ public class FileCleanupService {
|
|||||||
log.error("Error during cleanup", e);
|
log.error("Error during cleanup", e);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
private void cleanupOldPeriods() {
|
|
||||||
LocalDate thirtyDaysAgo = LocalDate.now().minusDays(30);
|
|
||||||
|
|
||||||
List<ProtectedFileCheck> checks = protectedFileCheckRepository.findByPeriodStartDateBefore(thirtyDaysAgo);
|
|
||||||
|
|
||||||
for (ProtectedFileCheck check : checks) {
|
|
||||||
check.cleanupOldData();
|
|
||||||
}
|
|
||||||
|
|
||||||
protectedFileCheckRepository.saveAll(checks);
|
|
||||||
log.info("Cleaned up {} old protection usage records", checks.size());
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -5,11 +5,11 @@ import lombok.RequiredArgsConstructor;
|
|||||||
import lombok.extern.slf4j.Slf4j;
|
import lombok.extern.slf4j.Slf4j;
|
||||||
import org.springframework.stereotype.Service;
|
import org.springframework.stereotype.Service;
|
||||||
import org.springframework.transaction.annotation.Transactional;
|
import org.springframework.transaction.annotation.Transactional;
|
||||||
|
import ru.soune.nocopy.configuration.search.SearchProperties;
|
||||||
import ru.soune.nocopy.dto.file.YandexSearchResponse;
|
import ru.soune.nocopy.dto.file.YandexSearchResponse;
|
||||||
import ru.soune.nocopy.dto.tarriff.TariffDTO;
|
import ru.soune.nocopy.dto.tarriff.TariffDTO;
|
||||||
import ru.soune.nocopy.entity.monitoring.FileMonitoringEntity;
|
import ru.soune.nocopy.entity.monitoring.FileMonitoringEntity;
|
||||||
import ru.soune.nocopy.entity.monitoring.MonitoringType;
|
import ru.soune.nocopy.entity.monitoring.MonitoringType;
|
||||||
import ru.soune.nocopy.entity.search.GlobalSearchResult;
|
|
||||||
import ru.soune.nocopy.entity.tarif.TariffInfo;
|
import ru.soune.nocopy.entity.tarif.TariffInfo;
|
||||||
import ru.soune.nocopy.entity.tarif.TariffType;
|
import ru.soune.nocopy.entity.tarif.TariffType;
|
||||||
import ru.soune.nocopy.entity.user.User;
|
import ru.soune.nocopy.entity.user.User;
|
||||||
@@ -24,6 +24,7 @@ import ru.soune.nocopy.service.violation.ViolationService;
|
|||||||
|
|
||||||
import java.io.IOException;
|
import java.io.IOException;
|
||||||
import java.time.LocalDateTime;
|
import java.time.LocalDateTime;
|
||||||
|
import java.util.ArrayList;
|
||||||
import java.util.List;
|
import java.util.List;
|
||||||
import java.util.concurrent.TimeoutException;
|
import java.util.concurrent.TimeoutException;
|
||||||
|
|
||||||
@@ -45,6 +46,8 @@ public class MonitoringSearchService {
|
|||||||
|
|
||||||
private final ViolationService violationService;
|
private final ViolationService violationService;
|
||||||
|
|
||||||
|
private final SearchProperties searchProperties;
|
||||||
|
|
||||||
@Transactional
|
@Transactional
|
||||||
public void processFileSearch(FileMonitoringEntity monitoring) throws MessagingException, IOException {
|
public void processFileSearch(FileMonitoringEntity monitoring) throws MessagingException, IOException {
|
||||||
monitoring.setLastRun(LocalDateTime.now());
|
monitoring.setLastRun(LocalDateTime.now());
|
||||||
@@ -53,36 +56,78 @@ public class MonitoringSearchService {
|
|||||||
|
|
||||||
MonitoringType monitoringType = monitoring.getMonitoringType();
|
MonitoringType monitoringType = monitoring.getMonitoringType();
|
||||||
TariffDTO tariffMonitoring = tariffService.getTariffByType(TariffType.valueOf(monitoringType.name()));
|
TariffDTO tariffMonitoring = tariffService.getTariffByType(TariffType.valueOf(monitoringType.name()));
|
||||||
|
|
||||||
try {
|
try {
|
||||||
tariffInfoService.writeOffTokens(monitoring.getUserId(), tariffMonitoring.getTokens());
|
tariffInfoService.writeOffTokens(monitoring.getUserId(), tariffMonitoring.getTokens());
|
||||||
|
|
||||||
|
boolean useYandex = searchProperties.getEngines().getOrDefault("yandex",
|
||||||
|
new SearchProperties.EngineConfig()).isEnabled();
|
||||||
|
boolean useGoogle = searchProperties.getEngines().getOrDefault("google",
|
||||||
|
new SearchProperties.EngineConfig()).isEnabled();
|
||||||
|
|
||||||
|
log.info("Monitoring search settings: useYandex={}, useGoogle={}", useYandex, useGoogle);
|
||||||
|
|
||||||
|
List<YandexSearchResponse.ImageResult> allResults = new ArrayList<>();
|
||||||
|
|
||||||
|
if (useYandex) {
|
||||||
|
try {
|
||||||
String yandexResponse = searchImageService.searchReverseByPublicUrl(
|
String yandexResponse = searchImageService.searchReverseByPublicUrl(
|
||||||
monitoring.getFile(), "yandex_reverse_image", "visual_matches");
|
monitoring.getFile(), "yandex_reverse_image", "visual_matches");
|
||||||
|
|
||||||
List<YandexSearchResponse.ImageResult> yandexImages =
|
List<YandexSearchResponse.ImageResult> yandexImages =
|
||||||
searchImageService.getAllImagesWithoutPagination(yandexResponse, "visual_matches");
|
searchImageService.getAllImagesWithoutPagination(yandexResponse, "visual_matches");
|
||||||
|
|
||||||
for (YandexSearchResponse.ImageResult imageResult : yandexImages) {
|
allResults.addAll(yandexImages);
|
||||||
|
log.info("Yandex search found {} results", yandexImages.size());
|
||||||
|
|
||||||
|
} catch (TimeoutException e) {
|
||||||
|
log.warn("Yandex search timeout for monitoring file {}", monitoring.getFile().getId());
|
||||||
|
} catch (IOException e) {
|
||||||
|
log.error("Yandex search failed for monitoring file {}", monitoring.getFile().getId(), e);
|
||||||
|
}
|
||||||
|
} else {
|
||||||
|
log.info("Yandex search is disabled by settings");
|
||||||
|
}
|
||||||
|
|
||||||
|
if (useGoogle) {
|
||||||
|
try {
|
||||||
|
String googleResponse = searchImageService.searchReverseByPublicUrl(
|
||||||
|
monitoring.getFile(), "google_lens", "exact_matches");
|
||||||
|
|
||||||
|
List<YandexSearchResponse.ImageResult> googleImages =
|
||||||
|
searchImageService.getAllImagesWithoutPagination(googleResponse, "exact_matches");
|
||||||
|
|
||||||
|
allResults.addAll(googleImages);
|
||||||
|
log.info("Google search found {} results", googleImages.size());
|
||||||
|
|
||||||
|
} catch (TimeoutException e) {
|
||||||
|
log.warn("Google search timeout for monitoring file {}", monitoring.getFile().getId());
|
||||||
|
} catch (IOException e) {
|
||||||
|
log.error("Google search failed for monitoring file {}", monitoring.getFile().getId(), e);
|
||||||
|
}
|
||||||
|
} else {
|
||||||
|
log.info("Google search is disabled by settings");
|
||||||
|
}
|
||||||
|
|
||||||
|
List<YandexSearchResponse.ImageResult> uniqueResults = allResults;
|
||||||
|
if (useYandex && useGoogle) {
|
||||||
|
uniqueResults = searchImageService.removeDuplicateUrls(
|
||||||
|
useYandex ? allResults : new ArrayList<>(),
|
||||||
|
useGoogle ? allResults : new ArrayList<>()
|
||||||
|
);
|
||||||
|
log.info("After deduplication: {} unique results", uniqueResults.size());
|
||||||
|
}
|
||||||
|
|
||||||
|
for (YandexSearchResponse.ImageResult imageResult : uniqueResults) {
|
||||||
violationService.processViolation(imageResult, monitoring.getFile(), null);
|
violationService.processViolation(imageResult, monitoring.getFile(), null);
|
||||||
}
|
}
|
||||||
|
|
||||||
// try {
|
|
||||||
// String googleResponse = searchImageService.searchReverseByPublicUrl(
|
|
||||||
// monitoring.getFile(), "google_lens", "exact_matches");
|
|
||||||
//
|
|
||||||
// List<YandexSearchResponse.ImageResult> googleImages =
|
|
||||||
// searchImageService.getAllImagesWithoutPagination(googleResponse, "exact_matches");
|
|
||||||
//
|
|
||||||
//
|
|
||||||
// for (YandexSearchResponse.ImageResult imageResult : googleImages) {
|
|
||||||
// violationService.processViolation(imageResult, monitoring.getFile(), null);
|
|
||||||
// }
|
|
||||||
//
|
|
||||||
// } catch (TimeoutException | IOException e) {
|
|
||||||
// log.warn("Google search failed");
|
|
||||||
// }
|
|
||||||
|
|
||||||
monitoring.setLastRunStatus("SUCCESS");
|
monitoring.setLastRunStatus("SUCCESS");
|
||||||
|
|
||||||
|
if (uniqueResults.isEmpty()) {
|
||||||
|
log.info("No results found for monitoring file {}", monitoring.getFile().getId());
|
||||||
|
}
|
||||||
|
|
||||||
} catch (TariffNotFoundException e) {
|
} catch (TariffNotFoundException e) {
|
||||||
User user = userRepository.findById(monitoring.getUserId()).orElseThrow();
|
User user = userRepository.findById(monitoring.getUserId()).orElseThrow();
|
||||||
TariffInfo activeTariffInfo = user.getActiveTariffInfo();
|
TariffInfo activeTariffInfo = user.getActiveTariffInfo();
|
||||||
@@ -91,10 +136,12 @@ public class MonitoringSearchService {
|
|||||||
emailService.sendTokensNotFoundEmail(user, currentTokens, tariffMonitoring.getTokens());
|
emailService.sendTokensNotFoundEmail(user, currentTokens, tariffMonitoring.getTokens());
|
||||||
monitoring.setLastRunStatus("ERROR: " + e.getMessage());
|
monitoring.setLastRunStatus("ERROR: " + e.getMessage());
|
||||||
updateNextRun(monitoring);
|
updateNextRun(monitoring);
|
||||||
|
|
||||||
monitoringRepository.save(monitoring);
|
monitoringRepository.save(monitoring);
|
||||||
|
|
||||||
} catch (Exception e) {
|
} catch (Exception e) {
|
||||||
|
log.error("Error processing monitoring search", e);
|
||||||
monitoring.setLastRunStatus("ERROR: " + e.getMessage());
|
monitoring.setLastRunStatus("ERROR: " + e.getMessage());
|
||||||
|
|
||||||
} finally {
|
} finally {
|
||||||
updateNextRun(monitoring);
|
updateNextRun(monitoring);
|
||||||
monitoringRepository.save(monitoring);
|
monitoringRepository.save(monitoring);
|
||||||
|
|||||||
@@ -1,21 +1,18 @@
|
|||||||
package ru.soune.nocopy.service.search;
|
package ru.soune.nocopy.service.search;
|
||||||
|
|
||||||
import com.fasterxml.jackson.databind.ObjectMapper;
|
|
||||||
import com.vrt.fileprotection.image.ImageCheckResult;
|
|
||||||
import lombok.RequiredArgsConstructor;
|
import lombok.RequiredArgsConstructor;
|
||||||
import lombok.extern.slf4j.Slf4j;
|
import lombok.extern.slf4j.Slf4j;
|
||||||
import org.springframework.scheduling.annotation.Async;
|
import org.springframework.scheduling.annotation.Async;
|
||||||
import org.springframework.stereotype.Component;
|
import org.springframework.stereotype.Component;
|
||||||
import org.springframework.transaction.annotation.Transactional;
|
import org.springframework.transaction.annotation.Transactional;
|
||||||
|
import ru.soune.nocopy.configuration.search.SearchProperties;
|
||||||
import ru.soune.nocopy.dto.file.YandexSearchResponse;
|
import ru.soune.nocopy.dto.file.YandexSearchResponse;
|
||||||
import ru.soune.nocopy.entity.file.FileEntity;
|
import ru.soune.nocopy.entity.file.FileEntity;
|
||||||
import ru.soune.nocopy.entity.search.GlobalSearchResult;
|
import ru.soune.nocopy.entity.search.GlobalSearchResult;
|
||||||
import ru.soune.nocopy.entity.search.GlobalSearchTask;
|
import ru.soune.nocopy.entity.search.GlobalSearchTask;
|
||||||
import ru.soune.nocopy.entity.search.SearchStatus;
|
import ru.soune.nocopy.entity.search.SearchStatus;
|
||||||
import ru.soune.nocopy.entity.violation.Violation;
|
|
||||||
import ru.soune.nocopy.repository.GlobalSearchResultRepository;
|
import ru.soune.nocopy.repository.GlobalSearchResultRepository;
|
||||||
import ru.soune.nocopy.repository.GlobalSearchTaskRepository;
|
import ru.soune.nocopy.repository.GlobalSearchTaskRepository;
|
||||||
import ru.soune.nocopy.repository.ViolationRepository;
|
|
||||||
import ru.soune.nocopy.service.tariff.TariffConstants;
|
import ru.soune.nocopy.service.tariff.TariffConstants;
|
||||||
import ru.soune.nocopy.service.tariff.TariffInfoService;
|
import ru.soune.nocopy.service.tariff.TariffInfoService;
|
||||||
import ru.soune.nocopy.service.violation.ViolationService;
|
import ru.soune.nocopy.service.violation.ViolationService;
|
||||||
@@ -36,16 +33,14 @@ public class GlobalSearchAsyncProcessor {
|
|||||||
|
|
||||||
private final SearchImageService searchImageService;
|
private final SearchImageService searchImageService;
|
||||||
|
|
||||||
private final ObjectMapper objectMapper;
|
|
||||||
|
|
||||||
private final TariffInfoService tariffInfoService;
|
private final TariffInfoService tariffInfoService;
|
||||||
|
|
||||||
private final GlobalSearchTaskRepository globalSearchTaskRepository;
|
private final GlobalSearchTaskRepository globalSearchTaskRepository;
|
||||||
|
|
||||||
private final ViolationRepository violationRepository;
|
|
||||||
|
|
||||||
private final ViolationService violationService;
|
private final ViolationService violationService;
|
||||||
|
|
||||||
|
private final SearchProperties searchProperties;
|
||||||
|
|
||||||
@Async
|
@Async
|
||||||
@Transactional
|
@Transactional
|
||||||
public void processFilesAsync(String taskId, List<FileEntity> filesToProcess, Long userId) {
|
public void processFilesAsync(String taskId, List<FileEntity> filesToProcess, Long userId) {
|
||||||
@@ -85,37 +80,49 @@ public class GlobalSearchAsyncProcessor {
|
|||||||
|
|
||||||
result = globalSearchResultRepository.save(result);
|
result = globalSearchResultRepository.save(result);
|
||||||
|
|
||||||
// List<YandexSearchResponse.ImageResult> googleImages = new ArrayList<>();
|
boolean useGoogle = searchProperties.getEngines().getOrDefault("google",
|
||||||
|
new SearchProperties.EngineConfig()).isEnabled();
|
||||||
|
boolean useYandex = searchProperties.getEngines().getOrDefault("yandex",
|
||||||
|
new SearchProperties.EngineConfig()).isEnabled();
|
||||||
|
|
||||||
|
log.info("Global search settings for file {}: useGoogle={}, useYandex={}",
|
||||||
|
file.getId(), useGoogle, useYandex);
|
||||||
|
|
||||||
|
List<YandexSearchResponse.ImageResult> googleImages = new ArrayList<>();
|
||||||
List<YandexSearchResponse.ImageResult> yandexImages = new ArrayList<>();
|
List<YandexSearchResponse.ImageResult> yandexImages = new ArrayList<>();
|
||||||
List<YandexSearchResponse.ImageResult> allUniqueImages = new ArrayList<>();
|
List<YandexSearchResponse.ImageResult> allResults = new ArrayList<>();
|
||||||
boolean hasTimeout = false;
|
boolean hasTimeout = false;
|
||||||
|
|
||||||
// try {
|
if (useGoogle) {
|
||||||
// String searchResponseGoogle = searchImageService.searchReverseByPublicUrl(
|
try {
|
||||||
// file, "google_lens", "exact_matches");
|
String searchResponseGoogle = searchImageService.searchReverseByPublicUrl(
|
||||||
//
|
file, "google_lens", "exact_matches");
|
||||||
// googleImages = searchImageService.getAllImagesWithoutPagination(searchResponseGoogle,
|
|
||||||
// "exact_matches");
|
|
||||||
//
|
|
||||||
// allUniqueImages.addAll(googleImages);
|
|
||||||
// log.info("Google search OK for file {}", file.getId());
|
|
||||||
//
|
|
||||||
// } catch (TimeoutException e) {
|
|
||||||
// log.warn("Google search timeout for file {}", file.getId());
|
|
||||||
// hasTimeout = true;
|
|
||||||
// } catch (IOException e) {
|
|
||||||
// log.error("Google search failed for file {}", file.getId(), e);
|
|
||||||
// }
|
|
||||||
|
|
||||||
|
googleImages = searchImageService.getAllImagesWithoutPagination(searchResponseGoogle,
|
||||||
|
"exact_matches");
|
||||||
|
|
||||||
|
allResults.addAll(googleImages);
|
||||||
|
log.info("Google search OK for file {}, found {} results", file.getId(), googleImages.size());
|
||||||
|
|
||||||
|
} catch (TimeoutException e) {
|
||||||
|
log.warn("Google search timeout for file {}", file.getId());
|
||||||
|
hasTimeout = true;
|
||||||
|
} catch (IOException e) {
|
||||||
|
log.error("Google search failed for file {}", file.getId(), e);
|
||||||
|
}
|
||||||
|
} else {
|
||||||
|
log.info("Google search is disabled by settings for file {}", file.getId());
|
||||||
|
}
|
||||||
|
|
||||||
|
if (useYandex) {
|
||||||
try {
|
try {
|
||||||
String searchResponseYandex = searchImageService.searchReverseByPublicUrl(
|
String searchResponseYandex = searchImageService.searchReverseByPublicUrl(
|
||||||
file, "yandex_reverse_image", "visual_matches");
|
file, "yandex_reverse_image", "visual_matches");
|
||||||
|
|
||||||
yandexImages =
|
yandexImages = searchImageService.getAllImagesWithoutPagination(searchResponseYandex, "visual_matches");
|
||||||
searchImageService.getAllImagesWithoutPagination(searchResponseYandex, "visual_matches");
|
|
||||||
|
|
||||||
allUniqueImages.addAll(yandexImages);
|
allResults.addAll(yandexImages);
|
||||||
log.info("Yandex search OK for file {}", file.getId());
|
log.info("Yandex search OK for file {}, found {} results", file.getId(), yandexImages.size());
|
||||||
|
|
||||||
} catch (TimeoutException e) {
|
} catch (TimeoutException e) {
|
||||||
log.warn("Yandex search timeout for file {}", file.getId());
|
log.warn("Yandex search timeout for file {}", file.getId());
|
||||||
@@ -123,14 +130,35 @@ public class GlobalSearchAsyncProcessor {
|
|||||||
} catch (IOException e) {
|
} catch (IOException e) {
|
||||||
log.error("Yandex search failed for file {}", file.getId(), e);
|
log.error("Yandex search failed for file {}", file.getId(), e);
|
||||||
}
|
}
|
||||||
|
} else {
|
||||||
|
log.info("Yandex search is disabled by settings for file {}", file.getId());
|
||||||
|
}
|
||||||
|
|
||||||
// allUniqueImages = searchImageService.removeDuplicateUrls(
|
List<YandexSearchResponse.ImageResult> allUniqueImages;
|
||||||
// allUniqueImages.stream()
|
if (useGoogle && useYandex) {
|
||||||
// .filter(img -> img.getUrl() != null)
|
allUniqueImages = searchImageService.removeDuplicateUrls(
|
||||||
// .collect(Collectors.toList()),
|
yandexImages.stream()
|
||||||
// new ArrayList<>());
|
.filter(img -> img.getUrl() != null)
|
||||||
|
.collect(Collectors.toList()),
|
||||||
allUniqueImages = yandexImages;
|
googleImages.stream()
|
||||||
|
.filter(img -> img.getUrl() != null)
|
||||||
|
.collect(Collectors.toList())
|
||||||
|
);
|
||||||
|
log.info("Merged results from both engines: {} unique images", allUniqueImages.size());
|
||||||
|
} else if (useGoogle) {
|
||||||
|
allUniqueImages = googleImages.stream()
|
||||||
|
.filter(img -> img.getUrl() != null)
|
||||||
|
.collect(Collectors.toList());
|
||||||
|
log.info("Results only from Google: {} images", allUniqueImages.size());
|
||||||
|
} else if (useYandex) {
|
||||||
|
allUniqueImages = yandexImages.stream()
|
||||||
|
.filter(img -> img.getUrl() != null)
|
||||||
|
.collect(Collectors.toList());
|
||||||
|
log.info("Results only from Yandex: {} images", allUniqueImages.size());
|
||||||
|
} else {
|
||||||
|
allUniqueImages = new ArrayList<>();
|
||||||
|
log.warn("All search engines are disabled for file {}", file.getId());
|
||||||
|
}
|
||||||
|
|
||||||
for (YandexSearchResponse.ImageResult imageResult : allUniqueImages) {
|
for (YandexSearchResponse.ImageResult imageResult : allUniqueImages) {
|
||||||
violationService.processViolation(imageResult, file, result);
|
violationService.processViolation(imageResult, file, result);
|
||||||
@@ -138,11 +166,16 @@ public class GlobalSearchAsyncProcessor {
|
|||||||
|
|
||||||
if (allUniqueImages.isEmpty() && hasTimeout) {
|
if (allUniqueImages.isEmpty() && hasTimeout) {
|
||||||
result.setFileStatus(SearchStatus.TIMEOUT.name());
|
result.setFileStatus(SearchStatus.TIMEOUT.name());
|
||||||
|
} else if (allUniqueImages.isEmpty() && (!useGoogle && !useYandex)) {
|
||||||
|
result.setFileStatus(SearchStatus.FAILED.name());
|
||||||
|
log.warn("No results for file {} because all engines are disabled", file.getId());
|
||||||
} else {
|
} else {
|
||||||
result.setFileStatus(SearchStatus.SUCCESS.name());
|
result.setFileStatus(SearchStatus.SUCCESS.name());
|
||||||
|
|
||||||
|
if (!allUniqueImages.isEmpty()) {
|
||||||
tariffInfoService.writeOffTokens(userId, TariffConstants.TOKEN_VALUE_FOR_SEARCH);
|
tariffInfoService.writeOffTokens(userId, TariffConstants.TOKEN_VALUE_FOR_SEARCH);
|
||||||
}
|
}
|
||||||
|
}
|
||||||
|
|
||||||
return result;
|
return result;
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -8,6 +8,7 @@ import lombok.extern.slf4j.Slf4j;
|
|||||||
import okhttp3.*;
|
import okhttp3.*;
|
||||||
import org.springframework.beans.factory.annotation.Value;
|
import org.springframework.beans.factory.annotation.Value;
|
||||||
import org.springframework.stereotype.Service;
|
import org.springframework.stereotype.Service;
|
||||||
|
import ru.soune.nocopy.configuration.search.SearchProperties;
|
||||||
import ru.soune.nocopy.dto.BaseResponse;
|
import ru.soune.nocopy.dto.BaseResponse;
|
||||||
import ru.soune.nocopy.dto.MessageCode;
|
import ru.soune.nocopy.dto.MessageCode;
|
||||||
import ru.soune.nocopy.dto.file.YandexSearchResponse;
|
import ru.soune.nocopy.dto.file.YandexSearchResponse;
|
||||||
@@ -23,40 +24,15 @@ import java.util.concurrent.*;
|
|||||||
|
|
||||||
@Slf4j
|
@Slf4j
|
||||||
@Service
|
@Service
|
||||||
|
|
||||||
public class SearchImageService {
|
public class SearchImageService {
|
||||||
|
|
||||||
private final ObjectMapper objectMapper;
|
private final ObjectMapper objectMapper;
|
||||||
|
|
||||||
private final OkHttpClient yandexHttpClient;
|
private final SearchProperties searchProperties;
|
||||||
|
|
||||||
private final OkHttpClient googleHttpClient;
|
|
||||||
|
|
||||||
public SearchImageService() {
|
|
||||||
Proxy proxy = new Proxy(Proxy.Type.HTTP,
|
|
||||||
new InetSocketAddress("193.46.217.94", 3128));
|
|
||||||
|
|
||||||
this.yandexHttpClient = new OkHttpClient.Builder()
|
|
||||||
// .proxy(proxy)
|
|
||||||
.connectTimeout(60, TimeUnit.SECONDS)
|
|
||||||
.writeTimeout(60, TimeUnit.SECONDS)
|
|
||||||
.readTimeout(60, TimeUnit.SECONDS)
|
|
||||||
.callTimeout(60, TimeUnit.SECONDS)
|
|
||||||
.followRedirects(true)
|
|
||||||
.followSslRedirects(true)
|
|
||||||
.retryOnConnectionFailure(true)
|
|
||||||
.build();
|
|
||||||
|
|
||||||
this.googleHttpClient = new OkHttpClient.Builder()
|
|
||||||
// .proxy(proxy)
|
|
||||||
.connectTimeout(35, TimeUnit.SECONDS)
|
|
||||||
.writeTimeout(35, TimeUnit.SECONDS)
|
|
||||||
.readTimeout(35, TimeUnit.SECONDS)
|
|
||||||
.callTimeout(35, TimeUnit.SECONDS)
|
|
||||||
.followRedirects(true)
|
|
||||||
.followSslRedirects(true)
|
|
||||||
.retryOnConnectionFailure(true)
|
|
||||||
.build();
|
|
||||||
|
|
||||||
|
public SearchImageService(SearchProperties searchProperties) {
|
||||||
|
this.searchProperties = searchProperties;
|
||||||
this.objectMapper = new ObjectMapper();
|
this.objectMapper = new ObjectMapper();
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -113,17 +89,34 @@ public class SearchImageService {
|
|||||||
return mimeType != null && mimeType.startsWith("image");
|
return mimeType != null && mimeType.startsWith("image");
|
||||||
}
|
}
|
||||||
|
|
||||||
private String callReverseImageApiByUrl(String imageUrl, String engine, String searchType) throws IOException {
|
private OkHttpClient createHttpClient(boolean useProxy) {
|
||||||
log.info("=== START SEARCH API CALL ===");
|
OkHttpClient.Builder builder = new OkHttpClient.Builder()
|
||||||
log.info("Engine: {}, SearchType: {}", engine, searchType);
|
.connectTimeout(35, TimeUnit.SECONDS)
|
||||||
log.info("ImageUrl: {}", imageUrl);
|
.writeTimeout(35, TimeUnit.SECONDS)
|
||||||
|
.readTimeout(35, TimeUnit.SECONDS)
|
||||||
|
.callTimeout(35, TimeUnit.SECONDS)
|
||||||
|
.followRedirects(true)
|
||||||
|
.followSslRedirects(true)
|
||||||
|
.retryOnConnectionFailure(true);
|
||||||
|
|
||||||
|
if (useProxy && searchProperties.getProxy().isEnabled()) {
|
||||||
|
Proxy proxy = new Proxy(Proxy.Type.HTTP,
|
||||||
|
new InetSocketAddress(searchProperties.getProxy().getHost(),
|
||||||
|
searchProperties.getProxy().getPort()));
|
||||||
|
builder.proxy(proxy);
|
||||||
|
}
|
||||||
|
|
||||||
|
return builder.build();
|
||||||
|
}
|
||||||
|
|
||||||
|
private String callReverseImageApiByUrl(String imageUrl, String engine, String searchType) throws IOException {
|
||||||
if (searchApiKey == null || searchApiKey.isBlank()) {
|
if (searchApiKey == null || searchApiKey.isBlank()) {
|
||||||
log.error("SearchAPI key not configured");
|
|
||||||
throw new IllegalStateException("SearchAPI key not configured");
|
throw new IllegalStateException("SearchAPI key not configured");
|
||||||
}
|
}
|
||||||
|
|
||||||
OkHttpClient client = engine.contains("yandex_reverse_image") ? yandexHttpClient : googleHttpClient;
|
boolean useProxy = searchProperties.getProxy().isEnabled();
|
||||||
|
|
||||||
|
OkHttpClient client = createHttpClient(useProxy);
|
||||||
|
|
||||||
HttpUrl url = HttpUrl.parse("https://www.searchapi.io/api/v1/search")
|
HttpUrl url = HttpUrl.parse("https://www.searchapi.io/api/v1/search")
|
||||||
.newBuilder()
|
.newBuilder()
|
||||||
@@ -134,8 +127,6 @@ public class SearchImageService {
|
|||||||
.addQueryParameter("t_", String.valueOf(System.currentTimeMillis()))
|
.addQueryParameter("t_", String.valueOf(System.currentTimeMillis()))
|
||||||
.build();
|
.build();
|
||||||
|
|
||||||
log.info("Request URL: {}", url);
|
|
||||||
|
|
||||||
Request request = new Request.Builder()
|
Request request = new Request.Builder()
|
||||||
.url(url)
|
.url(url)
|
||||||
.header("Accept", "application/json")
|
.header("Accept", "application/json")
|
||||||
@@ -149,27 +140,17 @@ public class SearchImageService {
|
|||||||
log.info("SearchAPI response code={}, duration={}ms, engine={}",
|
log.info("SearchAPI response code={}, duration={}ms, engine={}",
|
||||||
response.code(), duration, engine);
|
response.code(), duration, engine);
|
||||||
|
|
||||||
|
if (!response.isSuccessful()) {
|
||||||
|
String errorBody = response.body() != null ? response.body().string() : "null";
|
||||||
|
throw new IOException("API error " + response.code() + ": " + errorBody);
|
||||||
|
}
|
||||||
|
|
||||||
ResponseBody body = response.body();
|
ResponseBody body = response.body();
|
||||||
if (body == null) {
|
if (body == null) {
|
||||||
log.error("Response body is null");
|
|
||||||
throw new IOException("Empty response body");
|
throw new IOException("Empty response body");
|
||||||
}
|
}
|
||||||
|
|
||||||
String responseBody = body.string();
|
return body.string();
|
||||||
|
|
||||||
log.info("=== RESPONSE BODY LENGTH: {} ===", responseBody.length());
|
|
||||||
log.info("=== RESPONSE BODY START: {} ===",
|
|
||||||
responseBody.length() > 200 ? responseBody.substring(0, 200) : responseBody);
|
|
||||||
|
|
||||||
if (!response.isSuccessful()) {
|
|
||||||
log.error("Response not successful: {}", response.code());
|
|
||||||
throw new IOException("API error " + response.code() + ": " + responseBody);
|
|
||||||
}
|
|
||||||
|
|
||||||
return responseBody;
|
|
||||||
} catch (Exception e) {
|
|
||||||
log.error("Exception in search API call: {}", e.getMessage(), e);
|
|
||||||
throw e;
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
@@ -107,6 +107,17 @@ searchapi:
|
|||||||
api-key: ${SEARCHAPI_API_KEY:5jyYZC8jSaxhZTwjMUhwtAXi}
|
api-key: ${SEARCHAPI_API_KEY:5jyYZC8jSaxhZTwjMUhwtAXi}
|
||||||
reverse-image-url: "https://searchapi.io/api/v1/search"
|
reverse-image-url: "https://searchapi.io/api/v1/search"
|
||||||
|
|
||||||
|
search:
|
||||||
|
engines:
|
||||||
|
yandex:
|
||||||
|
enabled: true
|
||||||
|
google:
|
||||||
|
enabled: false
|
||||||
|
proxy:
|
||||||
|
enabled: true
|
||||||
|
host: "193.46.217.94"
|
||||||
|
port: 3128
|
||||||
|
|
||||||
yookassa:
|
yookassa:
|
||||||
secret-key: test_0Yns_0NHV5GJf6ypJ5HC4NSfnLO8SJkw-1PwrVWsDl4
|
secret-key: test_0Yns_0NHV5GJf6ypJ5HC4NSfnLO8SJkw-1PwrVWsDl4
|
||||||
shop-id: 1276731
|
shop-id: 1276731
|
||||||
|
|||||||
Reference in New Issue
Block a user