dev async global search

2026-03-06 17:28:52 +07:00
parent 52bc0e254c
commit 31c5d9aaab
14 changed files with 520 additions and 2 deletions
@@ -0,0 +1,147 @@
+package ru.soune.nocopy.service.search;
+
+import com.fasterxml.jackson.databind.ObjectMapper;
+import lombok.RequiredArgsConstructor;
+import lombok.extern.slf4j.Slf4j;
+import org.springframework.scheduling.annotation.Async;
+import org.springframework.stereotype.Component;
+import org.springframework.transaction.annotation.Transactional;
+import ru.soune.nocopy.dto.file.YandexSearchResponse;
+import ru.soune.nocopy.entity.file.FileEntity;
+import ru.soune.nocopy.entity.search.GlobalSearchResult;
+import ru.soune.nocopy.entity.search.GlobalSearchTask;
+import ru.soune.nocopy.entity.search.SearchStatus;
+import ru.soune.nocopy.repository.GlobalSearchResultRepository;
+import ru.soune.nocopy.repository.GlobalSearchTaskRepository;
+import ru.soune.nocopy.service.tariff.TariffConstants;
+import ru.soune.nocopy.service.tariff.TariffInfoService;
+
+import java.io.IOException;
+import java.time.LocalDateTime;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.concurrent.TimeoutException;
+import java.util.stream.Collectors;
+
+@Slf4j
+@Component
+@RequiredArgsConstructor
+public class GlobalSearchAsyncProcessor {
+
+    private final GlobalSearchResultRepository globalSearchResultRepository;
+
+    private final SearchImageService searchImageService;
+
+    private final ObjectMapper objectMapper;
+
+    private final TariffInfoService tariffInfoService;
+
+    private final GlobalSearchTaskRepository globalSearchTaskRepository;
+
+    @Async
+    @Transactional
+    public void processFilesAsync(String taskId, List<FileEntity> filesToProcess, Long userId) {
+        try {
+            for (int i = 0; i < filesToProcess.size(); i++) {
+                FileEntity file = filesToProcess.get(i);
+
+                GlobalSearchResult result = processFile(file, taskId, userId);
+                globalSearchResultRepository.save(result);
+
+                GlobalSearchTask task = globalSearchTaskRepository.findById(taskId).orElseThrow();
+                task.setProcessedFiles(i + 1);
+                task.setUpdatedAt(LocalDateTime.now());
+                globalSearchTaskRepository.save(task);
+            }
+
+            GlobalSearchTask task = globalSearchTaskRepository.findById(taskId).orElseThrow();
+            task.setStatus(SearchStatus.COMPLETED.name());
+            task.setUpdatedAt(LocalDateTime.now());
+            globalSearchTaskRepository.save(task);
+
+        } catch (Exception e) {
+            log.error("Global search failed for task: {}", taskId, e);
+
+            GlobalSearchTask task = globalSearchTaskRepository.findById(taskId).orElseThrow();
+            task.setStatus(SearchStatus.FAILED.name());
+            task.setUpdatedAt(LocalDateTime.now());
+            globalSearchTaskRepository.save(task);
+        }
+    }
+
+    private GlobalSearchResult processFile(FileEntity file, String taskId, Long userId) {
+        GlobalSearchResult result = new GlobalSearchResult();
+        result.setTaskId(taskId);
+        result.setFileId(file.getId());
+        result.setFileName(file.getOriginalFileName());
+        result.setThumbnail(file.getThumbnailPath());
+
+        List<YandexSearchResponse.ImageResult> allUniqueImages = new ArrayList<>();
+        boolean hasTimeout = false;
+
+        try {
+            String searchResponseGoogle = searchImageService.searchReverseByPublicUrl(
+                    file, "google_lens", "exact_matches");
+
+            List<YandexSearchResponse.ImageResult> googleImages =
+                    searchImageService.getAllImagesWithoutPagination(searchResponseGoogle, "exact_matches");
+
+            allUniqueImages.addAll(googleImages);
+            log.info("Google search OK for file {}", file.getId());
+
+        } catch (TimeoutException e) {
+            log.warn("Google search timeout for file {}", file.getId());
+            hasTimeout = true;
+        } catch (IOException e) {
+            log.error("Google search failed for file {}", file.getId(), e);
+        }
+
+        try {
+            String searchResponseYandex = searchImageService.searchReverseByPublicUrl(
+                    file, "yandex_reverse_image", "visual_matches");
+
+            List<YandexSearchResponse.ImageResult> yandexImages =
+                    searchImageService.getAllImagesWithoutPagination(searchResponseYandex, "visual_matches");
+
+            allUniqueImages.addAll(yandexImages);
+            log.info("Yandex search OK for file {}", file.getId());
+
+        } catch (TimeoutException e) {
+            log.warn("Yandex search timeout for file {}", file.getId());
+            hasTimeout = true;
+        } catch (IOException e) {
+            log.error("Yandex search failed for file {}", file.getId(), e);
+        }
+
+        allUniqueImages = searchImageService.removeDuplicateUrls(
+                allUniqueImages.stream()
+                        .filter(img -> img.getUrl() != null)
+                        .collect(Collectors.toList()),
+                new ArrayList<>()
+        );
+
+        YandexSearchResponse searchResponse = new YandexSearchResponse();
+        searchResponse.setImages(allUniqueImages);
+        searchResponse.setPage(1);
+        searchResponse.setPageSize(allUniqueImages.size());
+        searchResponse.setTotalResults(allUniqueImages.size());
+        searchResponse.setTotalPages(1);
+
+        try {
+            result.setSearchResults(objectMapper.writeValueAsString(searchResponse));
+        } catch (Exception e) {
+            log.error("Failed to serialize search results", e);
+            result.setSearchResults("{}");
+        }
+
+        if (allUniqueImages.isEmpty() && hasTimeout) {
+            result.setFileStatus(SearchStatus.TIMEOUT.name());
+        } else {
+            result.setFileStatus(SearchStatus.SUCCESS.name());
+
+            tariffInfoService.writeOffTokens(userId, TariffConstants.TOKEN_VALUE_FOR_SEARCH);
+        }
+
+        return result;
+    }
+}
@@ -0,0 +1,62 @@
+package ru.soune.nocopy.service.search;
+
+import lombok.RequiredArgsConstructor;
+import lombok.extern.slf4j.Slf4j;
+import org.springframework.stereotype.Service;
+import org.springframework.transaction.annotation.Transactional;
+import ru.soune.nocopy.dto.search.GlobalSearchStartRequest;
+import ru.soune.nocopy.entity.file.FileEntity;
+import ru.soune.nocopy.entity.monitoring.FileMonitoringEntity;
+import ru.soune.nocopy.entity.search.GlobalSearchTask;
+import ru.soune.nocopy.entity.search.SearchStatus;
+import ru.soune.nocopy.repository.FileEntityRepository;
+import ru.soune.nocopy.repository.FileMonitoringRepository;
+import ru.soune.nocopy.repository.GlobalSearchTaskRepository;
+
+import java.time.LocalDateTime;
+import java.util.ArrayList;
+import java.util.List;
+
+@Slf4j
+@Service
+@RequiredArgsConstructor
+public class GlobalSearchService {
+
+    private final FileEntityRepository fileEntityRepository;
+
+    private final FileMonitoringRepository fileMonitoringRepository;
+
+    private final GlobalSearchTaskRepository globalSearchTaskRepository;
+
+    private final GlobalSearchAsyncProcessor asyncProcessor;
+
+    @Transactional
+    public String startSearch(GlobalSearchStartRequest request, Long userId, List<FileEntity> filesToProcess) {
+        GlobalSearchTask task = new GlobalSearchTask();
+        task.setUserId(userId);
+        task.setSearchType(request.getSearchType());
+        task.setStatus(SearchStatus.PROCESSING.name());
+        task.setTotalFiles(filesToProcess.size());
+        task.setProcessedFiles(0);
+        task.setCreatedAt(LocalDateTime.now());
+
+        globalSearchTaskRepository.save(task);
+
+        asyncProcessor.processFilesAsync(task.getTaskId(), filesToProcess, userId);
+
+        return task.getTaskId();
+    }
+
+    public List<FileEntity> getFilesToProcess(GlobalSearchStartRequest request, Long userId) {
+        if ("monitoring".equals(request.getSearchType())) {
+            List<FileMonitoringEntity> monitoringFiles =
+                    fileMonitoringRepository.findByUserIdAndIsActiveTrue(userId);
+
+            return monitoringFiles.isEmpty() ? new ArrayList<>(): monitoringFiles.stream()
+                    .map(FileMonitoringEntity::getFile)
+                    .toList();
+        } else {
+            return fileEntityRepository.findAllById(request.getFileIds());
+        }
+    }
+}