dev change to visual_matches engine

2026-02-12 11:58:05 +07:00
parent 8a2be0d310
commit fb2ac0cf03
2 changed files with 140 additions and 15 deletions
@@ -1,5 +1,6 @@
 package ru.soune.nocopy.handler;

+import com.fasterxml.jackson.databind.JsonNode;
 import com.fasterxml.jackson.databind.ObjectMapper;
 import lombok.RequiredArgsConstructor;
 import lombok.extern.slf4j.Slf4j;
@@ -20,10 +21,7 @@ import ru.soune.nocopy.service.tariff.TariffConstants;
 import ru.soune.nocopy.service.tariff.TariffInfoService;

 import java.io.IOException;
-import java.util.ArrayList;
-import java.util.Collections;
-import java.util.List;
-import java.util.Map;
+import java.util.*;
 import java.util.concurrent.TimeoutException;

@Slf4j
@@ -90,22 +88,45 @@ public class ImageFoundRequestHandler implements RequestHandler {

        checkCounterService.incrementCheckCount(fileEntity.getUserId(), fileEntity.getMimeType());

-        int page = imageSearchRequest.getPage() != null ? imageSearchRequest.getPage() : 1;
-
-        YandexSearchResponse yandexSearchResponse = mapper.mapToYandexResponse(searchResponseYandex, page, 5,
+        List<YandexSearchResponse.ImageResult> allYandexImages = mapper.getAllImagesWithoutPagination(searchResponseYandex,
                "visual_matches");
-        YandexSearchResponse googleSearchResponse = mapper.mapToYandexResponse(searchResponseGoogle, page, 5,
+        List<YandexSearchResponse.ImageResult> allGoogleImages = mapper.getAllImagesWithoutPagination(searchResponseGoogle,
                "exact_matches");

-        int totalSize = yandexSearchResponse.getImages().size() + googleSearchResponse.getImages().size();
-        List<YandexSearchResponse.ImageResult> results = new ArrayList<>(totalSize);
-        results.addAll(yandexSearchResponse.getImages());
-        results.addAll(googleSearchResponse.getImages());
+        List<YandexSearchResponse.ImageResult> allUniqueImages = mapper.removeDuplicateUrls(allYandexImages, allGoogleImages);

-        yandexSearchResponse.setImages(results);
-        yandexSearchResponse.setTotalResults(totalSize);
+        int page = imageSearchRequest.getPage() != null ? imageSearchRequest.getPage() : 1;
+        int pageSize = 5;
+
+        List<YandexSearchResponse.ImageResult> pagedResults = mapper.paginateResults(allUniqueImages, page,
+                pageSize * 2);
+
+        YandexSearchResponse finalResponse = new YandexSearchResponse();
+        finalResponse.setImages(pagedResults);
+        finalResponse.setPage(page);
+        finalResponse.setPageSize(pageSize * 2);
+        finalResponse.setTotalResults(allUniqueImages.size());
+        finalResponse.setTotalPages((int) Math.ceil((double) allUniqueImages.size() / (pageSize * 2)));

        return new BaseResponse(request.getMsgId(), MessageCode.SUCCESS.getCode(),
-                MessageCode.SUCCESS.getDescription(), yandexSearchResponse);
+                MessageCode.SUCCESS.getDescription(), finalResponse);
+//        int page = imageSearchRequest.getPage() != null ? imageSearchRequest.getPage() : 1;
+
+//
+//        YandexSearchResponse yandexSearchResponse = mapper.mapToYandexResponse(searchResponseYandex, page, 5,
+//                "visual_matches");
+//        YandexSearchResponse googleSearchResponse = mapper.mapToYandexResponse(searchResponseGoogle, page, 5,
+//                "exact_matches");
+//
+//        int totalSize = yandexSearchResponse.getImages().size() + googleSearchResponse.getImages().size();
+//        List<YandexSearchResponse.ImageResult> results = new ArrayList<>(totalSize);
+//        results.addAll(yandexSearchResponse.getImages());
+//        results.addAll(googleSearchResponse.getImages());
+//
+//        yandexSearchResponse.setImages(results);
+//        yandexSearchResponse.setTotalResults(totalSize);
+//
+//        return new BaseResponse(request.getMsgId(), MessageCode.SUCCESS.getCode(),
+//                MessageCode.SUCCESS.getDescription(), yandexSearchResponse);
    }
 }
@@ -6,7 +6,9 @@ import ru.soune.nocopy.dto.file.YandexSearchResponse;

 import java.io.IOException;
 import java.util.ArrayList;
+import java.util.LinkedHashMap;
 import java.util.List;
+import java.util.Map;

 public class SearchApiToYandexResponseMapper {
    private final ObjectMapper objectMapper = new ObjectMapper();
@@ -125,6 +127,108 @@ public class SearchApiToYandexResponseMapper {
 //        return response;
 //    }

+    public List<YandexSearchResponse.ImageResult> getAllImagesWithoutPagination(String searchApiJson, String findType)
+            throws IOException {
+        JsonNode root = objectMapper.readTree(searchApiJson);
+        JsonNode matches = root.path(findType);
+
+        List<YandexSearchResponse.ImageResult> allImages = new ArrayList<>();
+
+        if (matches.isArray()) {
+            for (JsonNode match : matches) {
+                YandexSearchResponse.ImageResult result = mapImageResult(match);
+
+                if ("exact_matches".equals(findType)) {
+                    JsonNode thumbnail = match.path("thumbnail");
+                    if (!thumbnail.isMissingNode() && thumbnail.asText().startsWith("data:image")) {
+                        result.setUrl(thumbnail.asText());
+                    }
+
+                    JsonNode imageNode = match.path("image");
+                    if (!imageNode.isMissingNode()) {
+                        String directUrl = imageNode.path("link").asText(null);
+                        if (directUrl != null && !directUrl.isBlank()) {
+                            result.setUrl(directUrl);
+                        }
+                    }
+                }
+
+                if (result.getUrl() != null && !result.getUrl().isBlank()) {
+                    allImages.add(result);
+                }
+            }
+        }
+
+        return allImages;
+    }
+
+    public List<YandexSearchResponse.ImageResult> removeDuplicateUrls(
+            List<YandexSearchResponse.ImageResult> yandexImages,
+            List<YandexSearchResponse.ImageResult> googleImages) {
+
+        Map<String, YandexSearchResponse.ImageResult> uniqueByUrl = new LinkedHashMap<>();
+
+        for (YandexSearchResponse.ImageResult image : googleImages) {
+            String normalizedUrl = normalizeUrl(image.getUrl());
+            uniqueByUrl.putIfAbsent(normalizedUrl, image);
+        }
+
+        for (YandexSearchResponse.ImageResult image : yandexImages) {
+            String normalizedUrl = normalizeUrl(image.getUrl());
+            uniqueByUrl.putIfAbsent(normalizedUrl, image);
+        }
+
+        return new ArrayList<>(uniqueByUrl.values());
+    }
+
+    public List<YandexSearchResponse.ImageResult> paginateResults(
+            List<YandexSearchResponse.ImageResult> allResults,
+            int page,
+            int pageSize) {
+
+        if (allResults.isEmpty()) {
+            return new ArrayList<>();
+        }
+
+        int totalPages = (int) Math.ceil((double) allResults.size() / pageSize);
+
+        if (page > totalPages) {
+            return new ArrayList<>();
+        }
+
+        int startIndex = (page - 1) * pageSize;
+        int endIndex = Math.min(startIndex + pageSize, allResults.size());
+
+        return new ArrayList<>(allResults.subList(startIndex, endIndex));
+    }
+
+    private String normalizeUrl(String url) {
+        if (url == null) return null;
+
+        try {
+            if (url.startsWith("data:image")) {
+                return url.length() > 100 ? url.substring(0, 100) : url;
+            }
+
+            java.net.URI uri = new java.net.URI(url);
+            String host = uri.getHost();
+            if (host != null) {
+                host = host.toLowerCase().replaceFirst("^www\\.", "");
+            }
+
+            String path = uri.getPath();
+            if (path != null && path.length() > 1 && path.endsWith("/")) {
+                path = path.substring(0, path.length() - 1);
+            }
+
+            return (host != null ? host : "") + (path != null ? path : "");
+        } catch (Exception e) {
+            return url.toLowerCase()
+                    .replaceFirst("^(https?://)?(www\\.)?", "")
+                    .replaceFirst("/$", "");
+        }
+    }
+
    private YandexSearchResponse.ImageResult mapImageResult(JsonNode match) {

        YandexSearchResponse.ImageResult result =