From 920fc6796220bc69edb3563ab2defb4fcb8f0445 Mon Sep 17 00:00:00 2001 From: vladp Date: Thu, 12 Feb 2026 12:07:41 +0700 Subject: [PATCH] dev only unige result --- .../handler/ImageFoundRequestHandler.java | 34 +-- .../SearchApiToYandexResponseMapper.java | 266 ------------------ .../service/search/SearchImageService.java | 250 ++++++++-------- 3 files changed, 144 insertions(+), 406 deletions(-) delete mode 100644 src/main/java/ru/soune/nocopy/service/search/SearchApiToYandexResponseMapper.java diff --git a/src/main/java/ru/soune/nocopy/handler/ImageFoundRequestHandler.java b/src/main/java/ru/soune/nocopy/handler/ImageFoundRequestHandler.java index fc0b8c6..2d1f5f0 100644 --- a/src/main/java/ru/soune/nocopy/handler/ImageFoundRequestHandler.java +++ b/src/main/java/ru/soune/nocopy/handler/ImageFoundRequestHandler.java @@ -14,7 +14,6 @@ import ru.soune.nocopy.entity.file.FileEntity; import ru.soune.nocopy.exception.NotValidFieldException; import ru.soune.nocopy.repository.FileEntityRepository; import ru.soune.nocopy.service.file.CheckCounterService; -import ru.soune.nocopy.service.search.SearchApiToYandexResponseMapper; import ru.soune.nocopy.service.search.SearchImageService; import ru.soune.nocopy.service.search.GoogleVisionSearchService; import ru.soune.nocopy.service.tariff.TariffConstants; @@ -41,8 +40,6 @@ public class ImageFoundRequestHandler implements RequestHandler { private final TariffInfoService tariffInfoService; - private final SearchApiToYandexResponseMapper mapper = new SearchApiToYandexResponseMapper(); - @Override public BaseResponse handle(BaseRequest request) throws Exception { ImageSearchRequest imageSearchRequest = objectMapper.convertValue(request.getMessageBody(), @@ -88,17 +85,18 @@ public class ImageFoundRequestHandler implements RequestHandler { checkCounterService.incrementCheckCount(fileEntity.getUserId(), fileEntity.getMimeType()); - List allYandexImages = mapper.getAllImagesWithoutPagination(searchResponseYandex, - "visual_matches"); - List allGoogleImages = mapper.getAllImagesWithoutPagination(searchResponseGoogle, - "exact_matches"); + List allYandexImages = searchImageService.getAllImagesWithoutPagination( + searchResponseYandex, "visual_matches"); + List allGoogleImages = searchImageService.getAllImagesWithoutPagination( + searchResponseGoogle, "exact_matches"); - List allUniqueImages = mapper.removeDuplicateUrls(allYandexImages, allGoogleImages); + List allUniqueImages = searchImageService.removeDuplicateUrls(allYandexImages, + allGoogleImages); int page = imageSearchRequest.getPage() != null ? imageSearchRequest.getPage() : 1; int pageSize = 5; - List pagedResults = mapper.paginateResults(allUniqueImages, page, + List pagedResults = searchImageService.paginateResults(allUniqueImages, page, pageSize * 2); YandexSearchResponse finalResponse = new YandexSearchResponse(); @@ -110,23 +108,5 @@ public class ImageFoundRequestHandler implements RequestHandler { return new BaseResponse(request.getMsgId(), MessageCode.SUCCESS.getCode(), MessageCode.SUCCESS.getDescription(), finalResponse); -// int page = imageSearchRequest.getPage() != null ? imageSearchRequest.getPage() : 1; - -// -// YandexSearchResponse yandexSearchResponse = mapper.mapToYandexResponse(searchResponseYandex, page, 5, -// "visual_matches"); -// YandexSearchResponse googleSearchResponse = mapper.mapToYandexResponse(searchResponseGoogle, page, 5, -// "exact_matches"); -// -// int totalSize = yandexSearchResponse.getImages().size() + googleSearchResponse.getImages().size(); -// List results = new ArrayList<>(totalSize); -// results.addAll(yandexSearchResponse.getImages()); -// results.addAll(googleSearchResponse.getImages()); -// -// yandexSearchResponse.setImages(results); -// yandexSearchResponse.setTotalResults(totalSize); -// -// return new BaseResponse(request.getMsgId(), MessageCode.SUCCESS.getCode(), -// MessageCode.SUCCESS.getDescription(), yandexSearchResponse); } } diff --git a/src/main/java/ru/soune/nocopy/service/search/SearchApiToYandexResponseMapper.java b/src/main/java/ru/soune/nocopy/service/search/SearchApiToYandexResponseMapper.java deleted file mode 100644 index ab73acb..0000000 --- a/src/main/java/ru/soune/nocopy/service/search/SearchApiToYandexResponseMapper.java +++ /dev/null @@ -1,266 +0,0 @@ -package ru.soune.nocopy.service.search; - -import com.fasterxml.jackson.databind.JsonNode; -import com.fasterxml.jackson.databind.ObjectMapper; -import ru.soune.nocopy.dto.file.YandexSearchResponse; - -import java.io.IOException; -import java.util.ArrayList; -import java.util.LinkedHashMap; -import java.util.List; -import java.util.Map; - -public class SearchApiToYandexResponseMapper { - private final ObjectMapper objectMapper = new ObjectMapper(); - - public YandexSearchResponse mapToYandexResponse(String searchApiJson, int page, int pageSize, String findType) throws IOException { - JsonNode root = objectMapper.readTree(searchApiJson); - JsonNode matches = root.path(findType); - - YandexSearchResponse response = new YandexSearchResponse(); - List images = new ArrayList<>(); - - if (!matches.isArray() || matches.isEmpty()) { - response.setImages(images); - response.setPage(page); - response.setPageSize(pageSize); - response.setTotalResults(0); - response.setTotalPages(0); - return response; - } - - int totalResults = matches.size(); - int totalPages = (int) Math.ceil((double) totalResults / pageSize); - - if (page > totalPages) { - response.setImages(images); - response.setPage(page); - response.setPageSize(pageSize); - response.setTotalResults(totalResults); - response.setTotalPages(totalPages); - return response; - } - - int startIndex = (page - 1) * pageSize; - int endIndex = Math.min(startIndex + pageSize, totalResults); - - for (int i = startIndex; i < endIndex; i++) { - JsonNode match = matches.get(i); - YandexSearchResponse.ImageResult result = mapImageResult(match); - - if ("exact_matches".equals(findType)) { - JsonNode thumbnail = match.path("thumbnail"); - if (!thumbnail.isMissingNode() && thumbnail.asText().startsWith("data:image")) { - result.setUrl(thumbnail.asText()); - } - - JsonNode imageNode = match.path("image"); - if (!imageNode.isMissingNode()) { - String directUrl = imageNode.path("link").asText(null); - if (directUrl != null && !directUrl.isBlank()) { - result.setUrl(directUrl); - } - } - } - - if (result.getUrl() != null && !result.getUrl().isBlank()) { - images.add(result); - } - } - - response.setImages(images); - response.setPage(page); - response.setPageSize(pageSize); - response.setTotalResults(totalResults); - response.setTotalPages(totalPages); - - return response; - } - -// public YandexSearchResponse mapToYandexResponse(String searchApiJson, int page, int pageSize, String findType) throws IOException { -// -// JsonNode root = objectMapper.readTree(searchApiJson); -// JsonNode visualMatches = root.path(findType); -// -// YandexSearchResponse response = new YandexSearchResponse(); -// List images = new ArrayList<>(); -// -// if (!visualMatches.isArray() || visualMatches.isEmpty()) { -// response.setImages(images); -// response.setPage(page); -// response.setPageSize(pageSize); -// response.setTotalResults(0); -// response.setTotalPages(0); -// return response; -// } -// -// int totalResults = visualMatches.size(); -// int totalPages = (int) Math.ceil((double) totalResults / pageSize); -// -// if (page > totalPages) { -// response.setImages(images); -// response.setPage(page); -// response.setPageSize(pageSize); -// response.setTotalResults(totalResults); -// response.setTotalPages(totalPages); -// return response; -// } -// -// int startIndex = (page - 1) * pageSize; -// int endIndex = Math.min(startIndex + pageSize, totalResults); -// -// for (int i = startIndex; i < endIndex; i++) { -// JsonNode match = visualMatches.get(i); -// YandexSearchResponse.ImageResult result = mapImageResult(match); -// -// if (result.getUrl() != null && !result.getUrl().isBlank()) { -// images.add(result); -// } -// } -// -// response.setImages(images); -// response.setPage(page); -// response.setPageSize(pageSize); -// response.setTotalResults(totalResults); -// response.setTotalPages(totalPages); -// -// return response; -// } - - public List getAllImagesWithoutPagination(String searchApiJson, String findType) - throws IOException { - JsonNode root = objectMapper.readTree(searchApiJson); - JsonNode matches = root.path(findType); - - List allImages = new ArrayList<>(); - - if (matches.isArray()) { - for (JsonNode match : matches) { - YandexSearchResponse.ImageResult result = mapImageResult(match); - - if ("exact_matches".equals(findType)) { - JsonNode thumbnail = match.path("thumbnail"); - if (!thumbnail.isMissingNode() && thumbnail.asText().startsWith("data:image")) { - result.setUrl(thumbnail.asText()); - } - - JsonNode imageNode = match.path("image"); - if (!imageNode.isMissingNode()) { - String directUrl = imageNode.path("link").asText(null); - if (directUrl != null && !directUrl.isBlank()) { - result.setUrl(directUrl); - } - } - } - - if (result.getUrl() != null && !result.getUrl().isBlank()) { - allImages.add(result); - } - } - } - - return allImages; - } - - public List removeDuplicateUrls( - List yandexImages, - List googleImages) { - - Map uniqueByUrl = new LinkedHashMap<>(); - - for (YandexSearchResponse.ImageResult image : googleImages) { - String normalizedUrl = normalizeUrl(image.getUrl()); - uniqueByUrl.putIfAbsent(normalizedUrl, image); - } - - for (YandexSearchResponse.ImageResult image : yandexImages) { - String normalizedUrl = normalizeUrl(image.getUrl()); - uniqueByUrl.putIfAbsent(normalizedUrl, image); - } - - return new ArrayList<>(uniqueByUrl.values()); - } - - public List paginateResults( - List allResults, - int page, - int pageSize) { - - if (allResults.isEmpty()) { - return new ArrayList<>(); - } - - int totalPages = (int) Math.ceil((double) allResults.size() / pageSize); - - if (page > totalPages) { - return new ArrayList<>(); - } - - int startIndex = (page - 1) * pageSize; - int endIndex = Math.min(startIndex + pageSize, allResults.size()); - - return new ArrayList<>(allResults.subList(startIndex, endIndex)); - } - - private String normalizeUrl(String url) { - if (url == null) return null; - - try { - if (url.startsWith("data:image")) { - return url.length() > 100 ? url.substring(0, 100) : url; - } - - java.net.URI uri = new java.net.URI(url); - String host = uri.getHost(); - if (host != null) { - host = host.toLowerCase().replaceFirst("^www\\.", ""); - } - - String path = uri.getPath(); - if (path != null && path.length() > 1 && path.endsWith("/")) { - path = path.substring(0, path.length() - 1); - } - - return (host != null ? host : "") + (path != null ? path : ""); - } catch (Exception e) { - return url.toLowerCase() - .replaceFirst("^(https?://)?(www\\.)?", "") - .replaceFirst("/$", ""); - } - } - - private YandexSearchResponse.ImageResult mapImageResult(JsonNode match) { - - YandexSearchResponse.ImageResult result = - new YandexSearchResponse.ImageResult(); - - JsonNode imageNode = match.path("image"); - if (imageNode.isObject()) { - result.setUrl(imageNode.path("link").asText(null)); - result.setWidth(imageNode.path("width").asInt(0)); - result.setHeight(imageNode.path("height").asInt(0)); - } - - result.setPageUrl(match.path("link").asText(null)); - result.setPageTitle(match.path("title").asText(null)); - - String source = match.path("source").asText(null); - result.setHost(extractHostFromSource(source)); - - return result; - } - - private String extractHostFromSource(String source) { - if (source == null || source.isEmpty()) { - return ""; - } - source = source.replaceFirst("^(https?://)?(www\\.)?", ""); - - int slashIndex = source.indexOf('/'); - if (slashIndex > 0) { - return source.substring(0, slashIndex); - } - - return source; - } -} diff --git a/src/main/java/ru/soune/nocopy/service/search/SearchImageService.java b/src/main/java/ru/soune/nocopy/service/search/SearchImageService.java index fe88b9f..1b219a8 100644 --- a/src/main/java/ru/soune/nocopy/service/search/SearchImageService.java +++ b/src/main/java/ru/soune/nocopy/service/search/SearchImageService.java @@ -1,5 +1,6 @@ package ru.soune.nocopy.service.search; +import com.fasterxml.jackson.databind.JsonNode; import com.fasterxml.jackson.databind.ObjectMapper; import com.fasterxml.jackson.databind.DeserializationFeature; import jakarta.annotation.PostConstruct; @@ -9,11 +10,15 @@ import org.springframework.beans.factory.annotation.Value; import org.springframework.stereotype.Service; import ru.soune.nocopy.dto.BaseResponse; import ru.soune.nocopy.dto.MessageCode; +import ru.soune.nocopy.dto.file.YandexSearchResponse; import ru.soune.nocopy.entity.file.FileEntity; import ru.soune.nocopy.exception.NotValidFieldException; import java.io.*; import java.net.SocketTimeoutException; +import java.util.ArrayList; +import java.util.LinkedHashMap; +import java.util.List; import java.util.Map; import java.util.concurrent.*; @@ -101,7 +106,6 @@ public class SearchImageService { .addQueryParameter("engine", engine) .addQueryParameter("api_key", searchApiKey) .addQueryParameter("url", imageUrl) -// .addQueryParameter("search_type", "visual_matches") .addQueryParameter("search_type", searchType) .addQueryParameter("wait", "true") .build(); @@ -132,120 +136,140 @@ public class SearchImageService { } } -// public YandexSearchResponse searchByFileEntity(FileEntity fileEntity) throws IOException { -// byte[] fileBytes; -// -// if (!isImageFile(fileEntity)) { -// log.error("File not image: {}", fileEntity.getMimeType()); -// throw new NotValidFieldException("File not image", new BaseResponse(20007, -// MessageCode.INVALID_FIELD.getCode(), MessageCode.INVALID_FIELD.getDescription(), -// Map.of("file_type", fileEntity.getMimeType()))); -// } -// -// try { -// fileBytes = readFileFromDisk(fileEntity); -// } catch (IOException e) { -// throw new NotValidFieldException("File not found or cannot read file", new BaseResponse(20007, -// MessageCode.FILE_NOT_FOUND.getCode(), MessageCode.FILE_NOT_FOUND.getDescription(), -// Map.of("fileId", fileEntity.getId(), -// "filePath", fileEntity.getFilePath()))); -// } -// -// return callYandexApi(fileBytes); -// } + public List getAllImagesWithoutPagination(String searchApiJson, String findType) + throws IOException { + JsonNode root = objectMapper.readTree(searchApiJson); + JsonNode matches = root.path(findType); + List allImages = new ArrayList<>(); -// public String searchReverseByPublicUrl(FileEntity fileEntity) throws IOException { -// if (!isImageFile(fileEntity)) { -// log.error("File not image: {}", fileEntity.getMimeType()); -// throw new NotValidFieldException("File not image", new BaseResponse(20007, -// MessageCode.INVALID_FIELD.getCode(), MessageCode.INVALID_FIELD.getDescription(), -// Map.of("file_type", fileEntity.getMimeType()))); -// } -// -// String publicUrl = String.format("%s/api/files/public/%s", appBaseUrl, fileEntity.getId()); -// -// log.info("Searching reverse for image: {}", publicUrl); -// -// return callReverseImageApiByUrl(publicUrl); -// } + if (matches.isArray()) { + for (JsonNode match : matches) { + YandexSearchResponse.ImageResult result = mapImageResult(match); -// private byte[] readFileFromDisk(FileEntity fileEntity) throws IOException { -// Path filePath = Path.of(fileEntity.getFilePath()); -// -// if (!Files.exists(filePath)) { -// throw new IOException("File not found: " + fileEntity.getFilePath()); -// } -// -// if (!Files.isReadable(filePath)) { -// throw new IOException("Cannot read file: " + fileEntity.getFilePath()); -// } -// -// return Files.readAllBytes(filePath); -// } + if ("exact_matches".equals(findType)) { + JsonNode thumbnail = match.path("thumbnail"); + if (!thumbnail.isMissingNode() && thumbnail.asText().startsWith("data:image")) { + result.setUrl(thumbnail.asText()); + } -// private YandexSearchResponse callYandexApi(byte[] imageBytes) throws IOException { -// String imageBase64 = Base64.getEncoder().encodeToString(imageBytes); -// String jsonRequest = String.format("{\"folderId\":\"%s\",\"data\":\"%s\",\"page\":0,\"showSimilarImages\":true}", -// folderId, imageBase64); -// -// URL url = new URL(searchUrl); -// HttpURLConnection connection = (HttpURLConnection) url.openConnection(); -// -// try { -// connection.setRequestMethod("POST"); -// connection.setRequestProperty("Authorization", "Api-Key " + apiKey); -// connection.setRequestProperty("Content-Type", "application/json"); -// connection.setRequestProperty("Accept", "application/json"); -// connection.setConnectTimeout(30000); -// connection.setReadTimeout(30000); -// connection.setDoOutput(true); -// -// try (OutputStream os = connection.getOutputStream()) { -// byte[] input = jsonRequest.getBytes("utf-8"); -// os.write(input, 0, input.length); -// os.flush(); -// } -// -// int responseCode = connection.getResponseCode(); -// -// String responseBody; -// if (responseCode == HttpURLConnection.HTTP_OK) { -// try (BufferedReader br = new BufferedReader( -// new InputStreamReader(connection.getInputStream(), "utf-8"))) { -// responseBody = readAll(br); -// } -// } else { -// try (BufferedReader br = new BufferedReader( -// new InputStreamReader(connection.getErrorStream(), "utf-8"))) { -// responseBody = readAll(br); -// } -// throw new IOException("Error Yandex API: " + responseCode + " - " + responseBody); -// } -// -// return parseJsonResponse(responseBody); -// -// } finally { -// connection.disconnect(); -// } -// } -// private String readAll(BufferedReader reader) throws IOException { -// StringBuilder sb = new StringBuilder(); -// String line; -// while ((line = reader.readLine()) != null) { -// sb.append(line); -// } -// return sb.toString(); -// } -// -// private YandexSearchResponse parseJsonResponse(String json) { -// YandexSearchResponse response = null; -// try { -// response = objectMapper.readValue(json, YandexSearchResponse.class); -// } catch (JsonProcessingException e) { -// throw new RuntimeException(e); -// } -// -// return response; -// } + JsonNode imageNode = match.path("image"); + if (!imageNode.isMissingNode()) { + String directUrl = imageNode.path("link").asText(null); + if (directUrl != null && !directUrl.isBlank()) { + result.setUrl(directUrl); + } + } + } + + if (result.getUrl() != null && !result.getUrl().isBlank()) { + allImages.add(result); + } + } + } + + return allImages; + } + + public List removeDuplicateUrls( + List yandexImages, + List googleImages) { + + Map uniqueByUrl = new LinkedHashMap<>(); + + for (YandexSearchResponse.ImageResult image : googleImages) { + String normalizedUrl = normalizeUrl(image.getUrl()); + uniqueByUrl.putIfAbsent(normalizedUrl, image); + } + + for (YandexSearchResponse.ImageResult image : yandexImages) { + String normalizedUrl = normalizeUrl(image.getUrl()); + uniqueByUrl.putIfAbsent(normalizedUrl, image); + } + + return new ArrayList<>(uniqueByUrl.values()); + } + + public List paginateResults( + List allResults, + int page, + int pageSize) { + + if (allResults.isEmpty()) { + return new ArrayList<>(); + } + + int totalPages = (int) Math.ceil((double) allResults.size() / pageSize); + + if (page > totalPages) { + return new ArrayList<>(); + } + + int startIndex = (page - 1) * pageSize; + int endIndex = Math.min(startIndex + pageSize, allResults.size()); + + return new ArrayList<>(allResults.subList(startIndex, endIndex)); + } + + private String normalizeUrl(String url) { + if (url == null) return null; + + try { + if (url.startsWith("data:image")) { + return url.length() > 100 ? url.substring(0, 100) : url; + } + + java.net.URI uri = new java.net.URI(url); + String host = uri.getHost(); + if (host != null) { + host = host.toLowerCase().replaceFirst("^www\\.", ""); + } + + String path = uri.getPath(); + if (path != null && path.length() > 1 && path.endsWith("/")) { + path = path.substring(0, path.length() - 1); + } + + return (host != null ? host : "") + (path != null ? path : ""); + } catch (Exception e) { + return url.toLowerCase() + .replaceFirst("^(https?://)?(www\\.)?", "") + .replaceFirst("/$", ""); + } + } + + private YandexSearchResponse.ImageResult mapImageResult(JsonNode match) { + + YandexSearchResponse.ImageResult result = + new YandexSearchResponse.ImageResult(); + + JsonNode imageNode = match.path("image"); + if (imageNode.isObject()) { + result.setUrl(imageNode.path("link").asText(null)); + result.setWidth(imageNode.path("width").asInt(0)); + result.setHeight(imageNode.path("height").asInt(0)); + } + + result.setPageUrl(match.path("link").asText(null)); + result.setPageTitle(match.path("title").asText(null)); + + String source = match.path("source").asText(null); + result.setHost(extractHostFromSource(source)); + + return result; + } + + private String extractHostFromSource(String source) { + if (source == null || source.isEmpty()) { + return ""; + } + source = source.replaceFirst("^(https?://)?(www\\.)?", ""); + + int slashIndex = source.indexOf('/'); + if (slashIndex > 0) { + return source.substring(0, slashIndex); + } + + return source; + } }