From 2bf5e99fbbdb0b07f676c2c322ec8f5349b16886 Mon Sep 17 00:00:00 2001 From: vladp Date: Fri, 20 Mar 2026 17:33:09 +0700 Subject: [PATCH] dev add violation notion --- .../service/search/SearchImageService.java | 442 ++---------------- 1 file changed, 40 insertions(+), 402 deletions(-) diff --git a/src/main/java/ru/soune/nocopy/service/search/SearchImageService.java b/src/main/java/ru/soune/nocopy/service/search/SearchImageService.java index ed334a4..83d93c2 100644 --- a/src/main/java/ru/soune/nocopy/service/search/SearchImageService.java +++ b/src/main/java/ru/soune/nocopy/service/search/SearchImageService.java @@ -37,10 +37,10 @@ public class SearchImageService { this.yandexHttpClient = new OkHttpClient.Builder() // .proxy(proxy) - .connectTimeout(120, TimeUnit.SECONDS) - .writeTimeout(120, TimeUnit.SECONDS) - .readTimeout(120, TimeUnit.SECONDS) - .callTimeout(120, TimeUnit.SECONDS) + .connectTimeout(60, TimeUnit.SECONDS) + .writeTimeout(60, TimeUnit.SECONDS) + .readTimeout(60, TimeUnit.SECONDS) + .callTimeout(60, TimeUnit.SECONDS) .followRedirects(true) .followSslRedirects(true) .retryOnConnectionFailure(true) @@ -48,10 +48,10 @@ public class SearchImageService { this.googleHttpClient = new OkHttpClient.Builder() // .proxy(proxy) - .connectTimeout(120, TimeUnit.SECONDS) - .writeTimeout(120, TimeUnit.SECONDS) - .readTimeout(120, TimeUnit.SECONDS) - .callTimeout(120, TimeUnit.SECONDS) + .connectTimeout(35, TimeUnit.SECONDS) + .writeTimeout(35, TimeUnit.SECONDS) + .readTimeout(35, TimeUnit.SECONDS) + .callTimeout(35, TimeUnit.SECONDS) .followRedirects(true) .followSslRedirects(true) .retryOnConnectionFailure(true) @@ -103,7 +103,7 @@ public class SearchImageService { try { return callReverseImageApiByUrl(publicUrl, engine, searchType); } catch (SocketTimeoutException e) { - log.error("Search timeout for file: {}", fileEntity.getId()); + log.error("Yandex search timeout after {}", fileEntity.getId()); throw new TimeoutException("Search timeout"); } } @@ -131,7 +131,6 @@ public class SearchImageService { .addQueryParameter("api_key", searchApiKey) .addQueryParameter("url", imageUrl) .addQueryParameter("search_type", searchType) - .addQueryParameter("num", "10") .addQueryParameter("t_", String.valueOf(System.currentTimeMillis())) .build(); @@ -156,129 +155,62 @@ public class SearchImageService { throw new IOException("Empty response body"); } - String responseBody = readWithTimeout(body, 60000); + String responseBody = body.string(); - log.info("=== RESPONSE BODY LENGTH: {} bytes received ===", responseBody.length()); - - if (responseBody.length() < 1000) { - log.info("=== RESPONSE BODY: {} ===", responseBody); - } + log.info("=== RESPONSE BODY LENGTH: {} ===", responseBody.length()); + log.info("=== RESPONSE BODY START: {} ===", + responseBody.length() > 200 ? responseBody.substring(0, 200) : responseBody); if (!response.isSuccessful()) { log.error("Response not successful: {}", response.code()); - throw new IOException("API error " + response.code() + ": " + - (responseBody.length() > 200 ? responseBody.substring(0, 200) : responseBody)); - } - - if (responseBody.length() > 0 && !responseBody.endsWith("}")) { - responseBody = fixTruncatedJson(responseBody); - log.info("Fixed truncated JSON, new length: {}", responseBody.length()); + throw new IOException("API error " + response.code() + ": " + responseBody); } return responseBody; - } catch (InterruptedIOException e) { - log.warn("Timeout/Interrupted during reading for engine: {}", engine); - throw new IOException("Partial response received", e); } catch (Exception e) { log.error("Exception in search API call: {}", e.getMessage(), e); throw e; } } - private String readWithTimeout(ResponseBody body, long timeoutMs) throws IOException { - StringBuilder result = new StringBuilder(); - long startTime = System.currentTimeMillis(); - - try (BufferedReader reader = new BufferedReader(body.charStream())) { - char[] buffer = new char[8192]; - int read; - - while ((read = reader.read(buffer)) != -1) { - result.append(buffer, 0, read); - - if (System.currentTimeMillis() - startTime > timeoutMs) { - log.warn("Partial read timeout after {} ms, received {} bytes", - timeoutMs, result.length()); - break; - } - } - } - - return result.toString(); - } - - private String fixTruncatedJson(String partialJson) { - if (partialJson == null || partialJson.isEmpty()) { - return "{}"; - } - - String[] matchTypes = {"\"visual_matches\":", "\"exact_matches\":"}; - - for (String matchType : matchTypes) { - int matchesStart = partialJson.indexOf(matchType); - if (matchesStart > 0) { - int lastCompleteMatch = partialJson.lastIndexOf("},"); - int arrayEnd = partialJson.lastIndexOf("]"); - - if (lastCompleteMatch > matchesStart && arrayEnd < lastCompleteMatch) { - return partialJson.substring(0, lastCompleteMatch + 1) + "]}"; - } - break; - } - } - - if (!partialJson.endsWith("}")) { - return partialJson + "}"; - } - - return partialJson; - } - public List getAllImagesWithoutPagination(String searchApiJson, String findType) throws IOException { - try { - JsonNode root = objectMapper.readTree(searchApiJson); - JsonNode matches = root.path(findType); + JsonNode root = objectMapper.readTree(searchApiJson); + JsonNode matches = root.path(findType); - List allImages = new ArrayList<>(); + List allImages = new ArrayList<>(); - if (matches.isArray()) { - for (JsonNode match : matches) { - YandexSearchResponse.ImageResult result = mapImageResult(match); + if (matches.isArray()) { + for (JsonNode match : matches) { + YandexSearchResponse.ImageResult result = mapImageResult(match); - if ("exact_matches".equals(findType)) { - JsonNode thumbnail = match.path("thumbnail"); - if (!thumbnail.isMissingNode()) { - String thumbnailUrl = thumbnail.asText(); - if (thumbnailUrl.startsWith("data:image")) { - result.setUrl(thumbnailUrl); - } else if (!thumbnailUrl.isBlank()) { - result.setUrl(thumbnailUrl); - } - } - - JsonNode imageNode = match.path("image"); - if (!imageNode.isMissingNode()) { - String directUrl = imageNode.path("link").asText(); - if (directUrl != null && !directUrl.isBlank()) { - result.setUrl(directUrl); - } + if ("exact_matches".equals(findType)) { + JsonNode thumbnail = match.path("thumbnail"); + if (!thumbnail.isMissingNode()) { + String thumbnailUrl = thumbnail.asText(); + if (thumbnailUrl.startsWith("data:image")) { + result.setUrl(thumbnailUrl); + } else if (!thumbnailUrl.isBlank()) { + result.setUrl(thumbnailUrl); } } - if (result.getUrl() != null && !result.getUrl().isBlank()) { - allImages.add(result); + JsonNode imageNode = match.path("image"); + if (!imageNode.isMissingNode()) { + String directUrl = imageNode.path("link").asText(); + if (directUrl != null && !directUrl.isBlank()) { + result.setUrl(directUrl); + } } } + + if (result.getUrl() != null && !result.getUrl().isBlank()) { + allImages.add(result); + } } - - log.info("Parsed {} images from response (findType: {})", allImages.size(), findType); - return allImages; - - } catch (Exception e) { - log.error("Failed to parse JSON response: {}", e.getMessage()); - return new ArrayList<>(); } + + return allImages; } public List removeDuplicateUrls( @@ -383,297 +315,3 @@ public class SearchImageService { return source; } } -//@Slf4j -//@Service -//public class SearchImageService { -// -// private final ObjectMapper objectMapper; -// -// private final OkHttpClient yandexHttpClient; -// -// private final OkHttpClient googleHttpClient; -// -// public SearchImageService() { -// Proxy proxy = new Proxy(Proxy.Type.HTTP, -// new InetSocketAddress("193.46.217.94", 3128)); -// -// this.yandexHttpClient = new OkHttpClient.Builder() -//// .proxy(proxy) -// .connectTimeout(60, TimeUnit.SECONDS) -// .writeTimeout(60, TimeUnit.SECONDS) -// .readTimeout(60, TimeUnit.SECONDS) -// .callTimeout(60, TimeUnit.SECONDS) -// .followRedirects(true) -// .followSslRedirects(true) -// .retryOnConnectionFailure(true) -// .build(); -// -// this.googleHttpClient = new OkHttpClient.Builder() -//// .proxy(proxy) -// .connectTimeout(35, TimeUnit.SECONDS) -// .writeTimeout(35, TimeUnit.SECONDS) -// .readTimeout(35, TimeUnit.SECONDS) -// .callTimeout(35, TimeUnit.SECONDS) -// .followRedirects(true) -// .followSslRedirects(true) -// .retryOnConnectionFailure(true) -// .build(); -// -// this.objectMapper = new ObjectMapper(); -// } -// -// @Value("${yandex.api-key}") -// private String apiKey; -// -// @Value("${yandex.folder-id}") -// private String folderId; -// -// @Value("${yandex.search-url}") -// private String searchUrl; -// -// @Value("${searchapi.api-key:}") -// private String searchApiKey; -// -// @Value("${server.baseurl}") -// private String appBaseUrl; -// -// @PostConstruct -// public void init() { -// objectMapper.configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, false); -// objectMapper.configure(DeserializationFeature.ACCEPT_SINGLE_VALUE_AS_ARRAY, true); -// } -// -// public String searchReverseByPublicUrl(FileEntity fileEntity, String engine, String searchType) -// throws IOException, TimeoutException { -// -// if (!isImageFile(fileEntity)) { -// throw new NotValidFieldException( -// "File not image", -// new BaseResponse( -// 20007, -// MessageCode.INVALID_FIELD.getCode(), -// MessageCode.INVALID_FIELD.getDescription(), -// Map.of("file_type", fileEntity.getMimeType()) -// ) -// ); -// } -// -// String publicUrl = String.format("%s/api/files/public/%s", appBaseUrl, fileEntity.getId()); -// -// log.info("Searching reverse for image: {}", publicUrl); -// -// try { -// return callReverseImageApiByUrl(publicUrl, engine, searchType); -// } catch (SocketTimeoutException e) { -// log.error("Yandex search timeout after {}", fileEntity.getId()); -// throw new TimeoutException("Search timeout"); -// } -// } -// -// private boolean isImageFile(FileEntity fileEntity) { -// String mimeType = fileEntity.getMimeType(); -// return mimeType != null && mimeType.startsWith("image"); -// } -// -// private String callReverseImageApiByUrl(String imageUrl, String engine, String searchType) throws IOException { -// log.info("=== START SEARCH API CALL ==="); -// log.info("Engine: {}, SearchType: {}", engine, searchType); -// log.info("ImageUrl: {}", imageUrl); -// -// if (searchApiKey == null || searchApiKey.isBlank()) { -// log.error("SearchAPI key not configured"); -// throw new IllegalStateException("SearchAPI key not configured"); -// } -// -// OkHttpClient client = engine.contains("yandex_reverse_image") ? yandexHttpClient : googleHttpClient; -// -// HttpUrl url = HttpUrl.parse("https://www.searchapi.io/api/v1/search") -// .newBuilder() -// .addQueryParameter("engine", engine) -// .addQueryParameter("api_key", searchApiKey) -// .addQueryParameter("url", imageUrl) -// .addQueryParameter("search_type", searchType) -// .addQueryParameter("t_", String.valueOf(System.currentTimeMillis())) -// .build(); -// -// log.info("Request URL: {}", url); -// -// Request request = new Request.Builder() -// .url(url) -// .header("Accept", "application/json") -// .header("User-Agent", "Mozilla/5.0") -// .build(); -// -// long start = System.currentTimeMillis(); -// -// try (Response response = client.newCall(request).execute()) { -// long duration = System.currentTimeMillis() - start; -// log.info("SearchAPI response code={}, duration={}ms, engine={}", -// response.code(), duration, engine); -// -// ResponseBody body = response.body(); -// if (body == null) { -// log.error("Response body is null"); -// throw new IOException("Empty response body"); -// } -// -// String responseBody = body.string(); -// -// log.info("=== RESPONSE BODY LENGTH: {} ===", responseBody.length()); -// log.info("=== RESPONSE BODY START: {} ===", -// responseBody.length() > 200 ? responseBody.substring(0, 200) : responseBody); -// -// if (!response.isSuccessful()) { -// log.error("Response not successful: {}", response.code()); -// throw new IOException("API error " + response.code() + ": " + responseBody); -// } -// -// return responseBody; -// } catch (Exception e) { -// log.error("Exception in search API call: {}", e.getMessage(), e); -// throw e; -// } -// } -// -// public List getAllImagesWithoutPagination(String searchApiJson, String findType) -// throws IOException { -// JsonNode root = objectMapper.readTree(searchApiJson); -// JsonNode matches = root.path(findType); -// -// List allImages = new ArrayList<>(); -// -// if (matches.isArray()) { -// for (JsonNode match : matches) { -// YandexSearchResponse.ImageResult result = mapImageResult(match); -// -// if ("exact_matches".equals(findType)) { -// JsonNode thumbnail = match.path("thumbnail"); -// if (!thumbnail.isMissingNode()) { -// String thumbnailUrl = thumbnail.asText(); -// if (thumbnailUrl.startsWith("data:image")) { -// result.setUrl(thumbnailUrl); -// } else if (!thumbnailUrl.isBlank()) { -// result.setUrl(thumbnailUrl); -// } -// } -// -// JsonNode imageNode = match.path("image"); -// if (!imageNode.isMissingNode()) { -// String directUrl = imageNode.path("link").asText(); -// if (directUrl != null && !directUrl.isBlank()) { -// result.setUrl(directUrl); -// } -// } -// } -// -// if (result.getUrl() != null && !result.getUrl().isBlank()) { -// allImages.add(result); -// } -// } -// } -// -// return allImages; -// } -// -// public List removeDuplicateUrls( -// List yandexImages, -// List googleImages) { -// -// Map uniqueByUrl = new LinkedHashMap<>(); -// -// for (YandexSearchResponse.ImageResult image : googleImages) { -// String normalizedUrl = normalizeUrl(image.getUrl()); -// uniqueByUrl.putIfAbsent(normalizedUrl, image); -// } -// -// for (YandexSearchResponse.ImageResult image : yandexImages) { -// String normalizedUrl = normalizeUrl(image.getUrl()); -// uniqueByUrl.putIfAbsent(normalizedUrl, image); -// } -// -// return new ArrayList<>(uniqueByUrl.values()); -// } -// -// public List paginateResults( -// List allResults, -// int page, -// int pageSize) { -// -// if (allResults.isEmpty()) { -// return new ArrayList<>(); -// } -// -// int totalPages = (int) Math.ceil((double) allResults.size() / pageSize); -// -// if (page > totalPages) { -// return new ArrayList<>(); -// } -// -// int startIndex = (page - 1) * pageSize; -// int endIndex = Math.min(startIndex + pageSize, allResults.size()); -// -// return new ArrayList<>(allResults.subList(startIndex, endIndex)); -// } -// -// private String normalizeUrl(String url) { -// if (url == null) return null; -// -// try { -// if (url.startsWith("data:image")) { -// return url.length() > 100 ? url.substring(0, 100) : url; -// } -// -// java.net.URI uri = new java.net.URI(url); -// String host = uri.getHost(); -// if (host != null) { -// host = host.toLowerCase().replaceFirst("^www\\.", ""); -// } -// -// String path = uri.getPath(); -// if (path != null && path.length() > 1 && path.endsWith("/")) { -// path = path.substring(0, path.length() - 1); -// } -// -// return (host != null ? host : "") + (path != null ? path : ""); -// } catch (Exception e) { -// return url.toLowerCase() -// .replaceFirst("^(https?://)?(www\\.)?", "") -// .replaceFirst("/$", ""); -// } -// } -// -// private YandexSearchResponse.ImageResult mapImageResult(JsonNode match) { -// -// YandexSearchResponse.ImageResult result = -// new YandexSearchResponse.ImageResult(); -// -// JsonNode imageNode = match.path("image"); -// if (imageNode.isObject()) { -// result.setUrl(imageNode.path("link").asText()); -// result.setWidth(imageNode.path("width").asInt(0)); -// result.setHeight(imageNode.path("height").asInt(0)); -// } -// -// result.setPageUrl(match.path("link").asText()); -// result.setPageTitle(match.path("title").asText()); -// -// String source = match.path("source").asText(); -// result.setHost(extractHostFromSource(source)); -// -// return result; -// } -// -// private String extractHostFromSource(String source) { -// if (source == null || source.isEmpty()) { -// return ""; -// } -// source = source.replaceFirst("^(https?://)?(www\\.)?", ""); -// -// int slashIndex = source.indexOf('/'); -// if (slashIndex > 0) { -// return source.substring(0, slashIndex); -// } -// -// return source; -// } -//}