dev async global search

src/main/java/ru/soune/nocopy/configuration/file/AsyncConfig.java

@@ -21,4 +21,15 @@ public class AsyncConfig {
         executor.initialize();
         return executor;
     }
+
+    @Bean(name = "taskExecutor")
+    public Executor taskExecutor() {
+        ThreadPoolTaskExecutor executor = new ThreadPoolTaskExecutor();
+        executor.setCorePoolSize(3);
+        executor.setMaxPoolSize(5);
+        executor.setQueueCapacity(50);
+        executor.setThreadNamePrefix("global-search-");
+        executor.initialize();
+        return executor;
+    }
 }

src/main/java/ru/soune/nocopy/controller/GlobalSearchController.java

@@ -0,0 +1,140 @@
+package ru.soune.nocopy.controller;
+
+import com.fasterxml.jackson.databind.ObjectMapper;
+import lombok.RequiredArgsConstructor;
+import lombok.extern.slf4j.Slf4j;
+import org.springframework.http.ResponseEntity;
+import org.springframework.web.bind.annotation.*;
+import ru.soune.nocopy.dto.file.YandexSearchResponse;
+import ru.soune.nocopy.dto.search.GlobalSearchFileResult;
+import ru.soune.nocopy.dto.search.GlobalSearchStartRequest;
+import ru.soune.nocopy.dto.search.GlobalSearchStartResponse;
+import ru.soune.nocopy.dto.search.GlobalSearchStatusResponse;
+import ru.soune.nocopy.entity.file.FileEntity;
+import ru.soune.nocopy.entity.search.GlobalSearchResult;
+import ru.soune.nocopy.entity.search.GlobalSearchTask;
+import ru.soune.nocopy.entity.search.SearchStatus;
+import ru.soune.nocopy.repository.GlobalSearchResultRepository;
+import ru.soune.nocopy.repository.GlobalSearchTaskRepository;
+import ru.soune.nocopy.service.register.AuthService;
+import ru.soune.nocopy.service.search.GlobalSearchService;
+
+import java.util.*;
+import java.util.stream.Collectors;
+
+@Slf4j
+@RestController
+@RequestMapping("/api/v1/global-search")
+@RequiredArgsConstructor
+public class GlobalSearchController {
+
+    private final AuthService authService;
+
+    private final GlobalSearchService globalSearchService;
+
+    private final GlobalSearchTaskRepository globalSearchTaskRepository;
+
+    private final GlobalSearchTaskRepository searchTaskRepository;
+
+    private final GlobalSearchResultRepository globalSearchResultRepository;
+
+    private final ObjectMapper objectMapper;
+
+    @PostMapping("/start")
+    public ResponseEntity<?> startSearch(
+            @RequestBody GlobalSearchStartRequest request,
+            @RequestHeader(value = "Authorization", required = false) String tokenHeader) {
+
+        if (tokenHeader == null || tokenHeader.isBlank()) {
+            Map<String, Object> errorData = new HashMap<>();
+            errorData.put("token", tokenHeader);
+
+            return ResponseEntity.ok().body(Map.of("error", errorData));
+        }
+
+        Long userId = authService.useUserAuthToken(tokenHeader);
+
+        List<FileEntity> filesToProcess = globalSearchService.getFilesToProcess(request, userId);
+
+        if (filesToProcess.isEmpty()) return ResponseEntity.ok().body("Files for search not found");
+
+        String taskId = globalSearchService.startSearch(request, userId, filesToProcess);
+
+        Optional<GlobalSearchTask> taskOptional = globalSearchTaskRepository.findById(taskId);
+
+        if (taskOptional.isEmpty()) return ResponseEntity.ok().body("Task not found");
+
+        GlobalSearchTask task = taskOptional.orElseThrow();
+        GlobalSearchStartResponse response = new GlobalSearchStartResponse();
+        response.setTaskId(taskId);
+        response.setStatus(SearchStatus.ACCEPTED.name());
+        response.setTotalFiles(task.getTotalFiles());
+
+        return ResponseEntity.ok(response);
+    }
+
+    @GetMapping("/status/{taskId}")
+    public ResponseEntity<?> getStatus(
+            @PathVariable String taskId,
+            @RequestHeader(value = "Authorization", required = false) String tokenHeader) {
+
+        if (tokenHeader == null || tokenHeader.isBlank()) {
+            Map<String, Object> errorData = new HashMap<>();
+            errorData.put("token", tokenHeader);
+
+            return ResponseEntity.ok().body(Map.of("error", errorData));
+        }
+
+        Long userId = authService.useUserAuthToken(tokenHeader);
+
+        Optional<GlobalSearchTask> taskOptional = searchTaskRepository.findByTaskIdAndUserId(taskId, userId);
+
+        if (taskOptional.isEmpty()) return ResponseEntity.ok().body("Task not found");
+
+        GlobalSearchTask task = taskOptional.orElseThrow();
+
+        List<GlobalSearchResult> results = globalSearchResultRepository.findByTaskIdOrderByIdAsc(taskId);
+
+        List<GlobalSearchFileResult> fileResults = results.stream()
+                .map(result -> {
+                    GlobalSearchFileResult dto = new GlobalSearchFileResult();
+                    dto.setFileId(result.getFileId());
+                    dto.setFileName(result.getFileName());
+                    dto.setThumbnail(result.getThumbnail());
+                    dto.setFileStatus(result.getFileStatus());
+
+                    try {
+                        YandexSearchResponse searchResponse = objectMapper.readValue(
+                                result.getSearchResults(),
+                                YandexSearchResponse.class
+                        );
+
+                        dto.setImages(searchResponse.getImages());
+                        dto.setPage(searchResponse.getPage());
+                        dto.setPageSize(searchResponse.getPageSize());
+                        dto.setTotalResults(searchResponse.getTotalResults());
+                        dto.setTotalPages(searchResponse.getTotalPages());
+
+                    } catch (Exception e) {
+                        log.error("Failed to parse search results for file: {}", result.getFileId(), e);
+                        dto.setImages(new ArrayList<>());
+                        dto.setTotalResults(0);
+                    }
+
+                    return dto;
+                })
+                .collect(Collectors.toList());
+
+        int progress = task.getTotalFiles() > 0
+                ? task.getProcessedFiles() * 100 / task.getTotalFiles()
+                : 0;
+
+        GlobalSearchStatusResponse response = new GlobalSearchStatusResponse();
+        response.setTaskId(taskId);
+        response.setStatus(task.getStatus());
+        response.setProgress(progress);
+        response.setResults(fileResults);
+
+        return ResponseEntity.ok(response);
+    }
+}

src/main/java/ru/soune/nocopy/dto/search/GlobalSearchFileResult.java

@@ -0,0 +1,19 @@
+package ru.soune.nocopy.dto.search;
+
+import lombok.Data;
+import ru.soune.nocopy.dto.file.YandexSearchResponse;
+
+import java.util.List;
+
+@Data
+public class GlobalSearchFileResult {
+    private String fileId;
+    private String fileName;
+    private String thumbnail;
+    private String fileStatus;
+    private List<YandexSearchResponse.ImageResult> images;
+    private int page;
+    private int pageSize;
+    private int totalResults;
+    private int totalPages;
+}

src/main/java/ru/soune/nocopy/dto/search/GlobalSearchStartRequest.java

@@ -0,0 +1,14 @@
+package ru.soune.nocopy.dto.search;
+
+import lombok.Builder;
+import lombok.Data;
+
+import java.util.List;
+
+@Builder
+@Data
+public class GlobalSearchStartRequest {
+    private String searchType;
+
+    private List<String> fileIds;
+}

src/main/java/ru/soune/nocopy/dto/search/GlobalSearchStartResponse.java

@@ -0,0 +1,12 @@
+package ru.soune.nocopy.dto.search;
+
+import lombok.Data;
+
+@Data
+public class GlobalSearchStartResponse {
+    private String taskId;
+
+    private String status;
+
+    private Integer totalFiles;
+}

src/main/java/ru/soune/nocopy/dto/search/GlobalSearchStatusResponse.java

@@ -0,0 +1,18 @@
+package ru.soune.nocopy.dto.search;
+
+import lombok.Data;
+
+import java.util.List;
+
+@Data
+public class GlobalSearchStatusResponse {
+    private String taskId;
+
+    private String status;
+
+    private String searchType;
+
+    private Integer progress;
+
+    private List<GlobalSearchFileResult> results;
+}

src/main/java/ru/soune/nocopy/entity/search/GlobalSearchResult.java

@@ -0,0 +1,32 @@
+package ru.soune.nocopy.entity.search;
+
+import jakarta.persistence.*;
+import lombok.Data;
+import lombok.Getter;
+import lombok.RequiredArgsConstructor;
+import lombok.Setter;
+
+@Entity
+@Table(name = "global_search_results")
+@Data
+@Getter @Setter
+@RequiredArgsConstructor
+public class GlobalSearchResult {
+
+    @Id
+    @GeneratedValue(strategy = GenerationType.IDENTITY)
+    private Long id;
+
+    private String taskId;
+
+    private String fileId;
+
+    private String fileName;
+
+    private String thumbnail;
+
+    private String fileStatus;
+
+    @Column(length = 5000)
+    private String searchResults;
+}

src/main/java/ru/soune/nocopy/entity/search/GlobalSearchTask.java

@@ -0,0 +1,34 @@
+package ru.soune.nocopy.entity.search;
+
+import jakarta.persistence.*;
+import lombok.Data;
+import lombok.Getter;
+import lombok.RequiredArgsConstructor;
+import lombok.Setter;
+
+import java.time.LocalDateTime;
+
+@Entity
+@Table(name = "global_search_tasks")
+@Data
+@Getter @Setter
+@RequiredArgsConstructor
+public class GlobalSearchTask {
+    @Id
+    @GeneratedValue(strategy = GenerationType.UUID)
+    private String taskId;
+
+    private Long userId;
+
+    private String searchType;
+
+    private String status;
+
+    private Integer totalFiles;
+
+    private Integer processedFiles;
+
+    private LocalDateTime createdAt;
+
+    private LocalDateTime updatedAt;
+}

src/main/java/ru/soune/nocopy/entity/search/SearchStatus.java

@@ -0,0 +1,5 @@
+package ru.soune.nocopy.entity.search;
+
+public enum SearchStatus {
+    PROCESSING, COMPLETED, ACCEPTED, FAILED, SUCCESS, TIMEOUT
+}

src/main/java/ru/soune/nocopy/handler/ImageFoundRequestHandler.java

@@ -90,10 +90,8 @@ public class ImageFoundRequestHandler implements RequestHandler {
         allUniqueImages = searchImageService.removeDuplicateUrls(allYandexImages, allGoogleImages);
         log.info("allUniqueImages OK: {} images", allUniqueImages.size());
 
-//        if (!allUniqueImages.isEmpty()) {
         tariffInfoService.writeOffTokens(fileEntity.getUserId(), TariffConstants.TOKEN_VALUE_FOR_SEARCH);
         checkCounterService.incrementCheckCount(fileEntity.getUserId(), fileEntity.getMimeType());
-//        }
 
         int page = imageSearchRequest.getPage() != null ? imageSearchRequest.getPage() : 1;
         int pageSize = 5;

src/main/java/ru/soune/nocopy/repository/GlobalSearchResultRepository.java

@@ -0,0 +1,13 @@
+package ru.soune.nocopy.repository;
+
+import org.springframework.data.jpa.repository.JpaRepository;
+import org.springframework.stereotype.Repository;
+import ru.soune.nocopy.entity.search.GlobalSearchResult;
+
+import java.util.List;
+
+@Repository
+public interface GlobalSearchResultRepository extends JpaRepository<GlobalSearchResult, Long> {
+    List<GlobalSearchResult> findByTaskIdOrderByIdAsc(String taskId);
+    void deleteByTaskId(String taskId);
+}

src/main/java/ru/soune/nocopy/repository/GlobalSearchTaskRepository.java

@@ -0,0 +1,13 @@
+package ru.soune.nocopy.repository;
+
+import org.springframework.data.jpa.repository.JpaRepository;
+import org.springframework.stereotype.Repository;
+import ru.soune.nocopy.entity.search.GlobalSearchTask;
+
+import java.util.Optional;
+
+@Repository
+public interface GlobalSearchTaskRepository extends JpaRepository<GlobalSearchTask, String> {
+    Optional<GlobalSearchTask> findByTaskIdAndUserId(String taskId, Long userId);
+}
+

src/main/java/ru/soune/nocopy/service/search/GlobalSearchAsyncProcessor.java

@@ -0,0 +1,147 @@
+package ru.soune.nocopy.service.search;
+
+import com.fasterxml.jackson.databind.ObjectMapper;
+import lombok.RequiredArgsConstructor;
+import lombok.extern.slf4j.Slf4j;
+import org.springframework.scheduling.annotation.Async;
+import org.springframework.stereotype.Component;
+import org.springframework.transaction.annotation.Transactional;
+import ru.soune.nocopy.dto.file.YandexSearchResponse;
+import ru.soune.nocopy.entity.file.FileEntity;
+import ru.soune.nocopy.entity.search.GlobalSearchResult;
+import ru.soune.nocopy.entity.search.GlobalSearchTask;
+import ru.soune.nocopy.entity.search.SearchStatus;
+import ru.soune.nocopy.repository.GlobalSearchResultRepository;
+import ru.soune.nocopy.repository.GlobalSearchTaskRepository;
+import ru.soune.nocopy.service.tariff.TariffConstants;
+import ru.soune.nocopy.service.tariff.TariffInfoService;
+
+import java.io.IOException;
+import java.time.LocalDateTime;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.concurrent.TimeoutException;
+import java.util.stream.Collectors;
+
+@Slf4j
+@Component
+@RequiredArgsConstructor
+public class GlobalSearchAsyncProcessor {
+
+    private final GlobalSearchResultRepository globalSearchResultRepository;
+
+    private final SearchImageService searchImageService;
+
+    private final ObjectMapper objectMapper;
+
+    private final TariffInfoService tariffInfoService;
+
+    private final GlobalSearchTaskRepository globalSearchTaskRepository;
+
+    @Async
+    @Transactional
+    public void processFilesAsync(String taskId, List<FileEntity> filesToProcess, Long userId) {
+        try {
+            for (int i = 0; i < filesToProcess.size(); i++) {
+                FileEntity file = filesToProcess.get(i);
+
+                GlobalSearchResult result = processFile(file, taskId, userId);
+                globalSearchResultRepository.save(result);
+
+                GlobalSearchTask task = globalSearchTaskRepository.findById(taskId).orElseThrow();
+                task.setProcessedFiles(i + 1);
+                task.setUpdatedAt(LocalDateTime.now());
+                globalSearchTaskRepository.save(task);
+            }
+
+            GlobalSearchTask task = globalSearchTaskRepository.findById(taskId).orElseThrow();
+            task.setStatus(SearchStatus.COMPLETED.name());
+            task.setUpdatedAt(LocalDateTime.now());
+            globalSearchTaskRepository.save(task);
+
+        } catch (Exception e) {
+            log.error("Global search failed for task: {}", taskId, e);
+
+            GlobalSearchTask task = globalSearchTaskRepository.findById(taskId).orElseThrow();
+            task.setStatus(SearchStatus.FAILED.name());
+            task.setUpdatedAt(LocalDateTime.now());
+            globalSearchTaskRepository.save(task);
+        }
+    }
+
+    private GlobalSearchResult processFile(FileEntity file, String taskId, Long userId) {
+        GlobalSearchResult result = new GlobalSearchResult();
+        result.setTaskId(taskId);
+        result.setFileId(file.getId());
+        result.setFileName(file.getOriginalFileName());
+        result.setThumbnail(file.getThumbnailPath());
+
+        List<YandexSearchResponse.ImageResult> allUniqueImages = new ArrayList<>();
+        boolean hasTimeout = false;
+
+        try {
+            String searchResponseGoogle = searchImageService.searchReverseByPublicUrl(
+                    file, "google_lens", "exact_matches");
+
+            List<YandexSearchResponse.ImageResult> googleImages =
+                    searchImageService.getAllImagesWithoutPagination(searchResponseGoogle, "exact_matches");
+
+            allUniqueImages.addAll(googleImages);
+            log.info("Google search OK for file {}", file.getId());
+
+        } catch (TimeoutException e) {
+            log.warn("Google search timeout for file {}", file.getId());
+            hasTimeout = true;
+        } catch (IOException e) {
+            log.error("Google search failed for file {}", file.getId(), e);
+        }
+
+        try {
+            String searchResponseYandex = searchImageService.searchReverseByPublicUrl(
+                    file, "yandex_reverse_image", "visual_matches");
+
+            List<YandexSearchResponse.ImageResult> yandexImages =
+                    searchImageService.getAllImagesWithoutPagination(searchResponseYandex, "visual_matches");
+
+            allUniqueImages.addAll(yandexImages);
+            log.info("Yandex search OK for file {}", file.getId());
+
+        } catch (TimeoutException e) {
+            log.warn("Yandex search timeout for file {}", file.getId());
+            hasTimeout = true;
+        } catch (IOException e) {
+            log.error("Yandex search failed for file {}", file.getId(), e);
+        }
+
+        allUniqueImages = searchImageService.removeDuplicateUrls(
+                allUniqueImages.stream()
+                        .filter(img -> img.getUrl() != null)
+                        .collect(Collectors.toList()),
+                new ArrayList<>()
+        );
+
+        YandexSearchResponse searchResponse = new YandexSearchResponse();
+        searchResponse.setImages(allUniqueImages);
+        searchResponse.setPage(1);
+        searchResponse.setPageSize(allUniqueImages.size());
+        searchResponse.setTotalResults(allUniqueImages.size());
+        searchResponse.setTotalPages(1);
+
+        try {
+            result.setSearchResults(objectMapper.writeValueAsString(searchResponse));
+        } catch (Exception e) {
+            log.error("Failed to serialize search results", e);
+            result.setSearchResults("{}");
+        }
+
+        if (allUniqueImages.isEmpty() && hasTimeout) {
+            result.setFileStatus(SearchStatus.TIMEOUT.name());
+        } else {
+            result.setFileStatus(SearchStatus.SUCCESS.name());
+
+            tariffInfoService.writeOffTokens(userId, TariffConstants.TOKEN_VALUE_FOR_SEARCH);
+        }
+
+        return result;
+    }
+}

src/main/java/ru/soune/nocopy/service/search/GlobalSearchService.java

@@ -0,0 +1,62 @@
+package ru.soune.nocopy.service.search;
+
+import lombok.RequiredArgsConstructor;
+import lombok.extern.slf4j.Slf4j;
+import org.springframework.stereotype.Service;
+import org.springframework.transaction.annotation.Transactional;
+import ru.soune.nocopy.dto.search.GlobalSearchStartRequest;
+import ru.soune.nocopy.entity.file.FileEntity;
+import ru.soune.nocopy.entity.monitoring.FileMonitoringEntity;
+import ru.soune.nocopy.entity.search.GlobalSearchTask;
+import ru.soune.nocopy.entity.search.SearchStatus;
+import ru.soune.nocopy.repository.FileEntityRepository;
+import ru.soune.nocopy.repository.FileMonitoringRepository;
+import ru.soune.nocopy.repository.GlobalSearchTaskRepository;
+
+import java.time.LocalDateTime;
+import java.util.ArrayList;
+import java.util.List;
+
+@Slf4j
+@Service
+@RequiredArgsConstructor
+public class GlobalSearchService {
+
+    private final FileEntityRepository fileEntityRepository;
+
+    private final FileMonitoringRepository fileMonitoringRepository;
+
+    private final GlobalSearchTaskRepository globalSearchTaskRepository;
+
+    private final GlobalSearchAsyncProcessor asyncProcessor;
+
+    @Transactional
+    public String startSearch(GlobalSearchStartRequest request, Long userId, List<FileEntity> filesToProcess) {
+        GlobalSearchTask task = new GlobalSearchTask();
+        task.setUserId(userId);
+        task.setSearchType(request.getSearchType());
+        task.setStatus(SearchStatus.PROCESSING.name());
+        task.setTotalFiles(filesToProcess.size());
+        task.setProcessedFiles(0);
+        task.setCreatedAt(LocalDateTime.now());
+
+        globalSearchTaskRepository.save(task);
+
+        asyncProcessor.processFilesAsync(task.getTaskId(), filesToProcess, userId);
+
+        return task.getTaskId();
+    }
+
+    public List<FileEntity> getFilesToProcess(GlobalSearchStartRequest request, Long userId) {
+        if ("monitoring".equals(request.getSearchType())) {
+            List<FileMonitoringEntity> monitoringFiles =
+                    fileMonitoringRepository.findByUserIdAndIsActiveTrue(userId);
+
+            return monitoringFiles.isEmpty() ? new ArrayList<>(): monitoringFiles.stream()
+                    .map(FileMonitoringEntity::getFile)
+                    .toList();
+        } else {
+            return fileEntityRepository.findAllById(request.getFileIds());
+        }
+    }
+}