diff --git a/src/main/java/ru/soune/nocopy/service/search/SearchImageService.java b/src/main/java/ru/soune/nocopy/service/search/SearchImageService.java index 83d93c2..ed334a4 100644 --- a/src/main/java/ru/soune/nocopy/service/search/SearchImageService.java +++ b/src/main/java/ru/soune/nocopy/service/search/SearchImageService.java @@ -37,10 +37,10 @@ public class SearchImageService { this.yandexHttpClient = new OkHttpClient.Builder() // .proxy(proxy) - .connectTimeout(60, TimeUnit.SECONDS) - .writeTimeout(60, TimeUnit.SECONDS) - .readTimeout(60, TimeUnit.SECONDS) - .callTimeout(60, TimeUnit.SECONDS) + .connectTimeout(120, TimeUnit.SECONDS) + .writeTimeout(120, TimeUnit.SECONDS) + .readTimeout(120, TimeUnit.SECONDS) + .callTimeout(120, TimeUnit.SECONDS) .followRedirects(true) .followSslRedirects(true) .retryOnConnectionFailure(true) @@ -48,10 +48,10 @@ public class SearchImageService { this.googleHttpClient = new OkHttpClient.Builder() // .proxy(proxy) - .connectTimeout(35, TimeUnit.SECONDS) - .writeTimeout(35, TimeUnit.SECONDS) - .readTimeout(35, TimeUnit.SECONDS) - .callTimeout(35, TimeUnit.SECONDS) + .connectTimeout(120, TimeUnit.SECONDS) + .writeTimeout(120, TimeUnit.SECONDS) + .readTimeout(120, TimeUnit.SECONDS) + .callTimeout(120, TimeUnit.SECONDS) .followRedirects(true) .followSslRedirects(true) .retryOnConnectionFailure(true) @@ -103,7 +103,7 @@ public class SearchImageService { try { return callReverseImageApiByUrl(publicUrl, engine, searchType); } catch (SocketTimeoutException e) { - log.error("Yandex search timeout after {}", fileEntity.getId()); + log.error("Search timeout for file: {}", fileEntity.getId()); throw new TimeoutException("Search timeout"); } } @@ -131,6 +131,7 @@ public class SearchImageService { .addQueryParameter("api_key", searchApiKey) .addQueryParameter("url", imageUrl) .addQueryParameter("search_type", searchType) + .addQueryParameter("num", "10") .addQueryParameter("t_", String.valueOf(System.currentTimeMillis())) .build(); @@ -155,62 +156,129 @@ public class SearchImageService { throw new IOException("Empty response body"); } - String responseBody = body.string(); + String responseBody = readWithTimeout(body, 60000); - log.info("=== RESPONSE BODY LENGTH: {} ===", responseBody.length()); - log.info("=== RESPONSE BODY START: {} ===", - responseBody.length() > 200 ? responseBody.substring(0, 200) : responseBody); + log.info("=== RESPONSE BODY LENGTH: {} bytes received ===", responseBody.length()); + + if (responseBody.length() < 1000) { + log.info("=== RESPONSE BODY: {} ===", responseBody); + } if (!response.isSuccessful()) { log.error("Response not successful: {}", response.code()); - throw new IOException("API error " + response.code() + ": " + responseBody); + throw new IOException("API error " + response.code() + ": " + + (responseBody.length() > 200 ? responseBody.substring(0, 200) : responseBody)); + } + + if (responseBody.length() > 0 && !responseBody.endsWith("}")) { + responseBody = fixTruncatedJson(responseBody); + log.info("Fixed truncated JSON, new length: {}", responseBody.length()); } return responseBody; + } catch (InterruptedIOException e) { + log.warn("Timeout/Interrupted during reading for engine: {}", engine); + throw new IOException("Partial response received", e); } catch (Exception e) { log.error("Exception in search API call: {}", e.getMessage(), e); throw e; } } - public List getAllImagesWithoutPagination(String searchApiJson, String findType) - throws IOException { - JsonNode root = objectMapper.readTree(searchApiJson); - JsonNode matches = root.path(findType); + private String readWithTimeout(ResponseBody body, long timeoutMs) throws IOException { + StringBuilder result = new StringBuilder(); + long startTime = System.currentTimeMillis(); - List allImages = new ArrayList<>(); + try (BufferedReader reader = new BufferedReader(body.charStream())) { + char[] buffer = new char[8192]; + int read; - if (matches.isArray()) { - for (JsonNode match : matches) { - YandexSearchResponse.ImageResult result = mapImageResult(match); + while ((read = reader.read(buffer)) != -1) { + result.append(buffer, 0, read); - if ("exact_matches".equals(findType)) { - JsonNode thumbnail = match.path("thumbnail"); - if (!thumbnail.isMissingNode()) { - String thumbnailUrl = thumbnail.asText(); - if (thumbnailUrl.startsWith("data:image")) { - result.setUrl(thumbnailUrl); - } else if (!thumbnailUrl.isBlank()) { - result.setUrl(thumbnailUrl); - } - } - - JsonNode imageNode = match.path("image"); - if (!imageNode.isMissingNode()) { - String directUrl = imageNode.path("link").asText(); - if (directUrl != null && !directUrl.isBlank()) { - result.setUrl(directUrl); - } - } - } - - if (result.getUrl() != null && !result.getUrl().isBlank()) { - allImages.add(result); + if (System.currentTimeMillis() - startTime > timeoutMs) { + log.warn("Partial read timeout after {} ms, received {} bytes", + timeoutMs, result.length()); + break; } } } - return allImages; + return result.toString(); + } + + private String fixTruncatedJson(String partialJson) { + if (partialJson == null || partialJson.isEmpty()) { + return "{}"; + } + + String[] matchTypes = {"\"visual_matches\":", "\"exact_matches\":"}; + + for (String matchType : matchTypes) { + int matchesStart = partialJson.indexOf(matchType); + if (matchesStart > 0) { + int lastCompleteMatch = partialJson.lastIndexOf("},"); + int arrayEnd = partialJson.lastIndexOf("]"); + + if (lastCompleteMatch > matchesStart && arrayEnd < lastCompleteMatch) { + return partialJson.substring(0, lastCompleteMatch + 1) + "]}"; + } + break; + } + } + + if (!partialJson.endsWith("}")) { + return partialJson + "}"; + } + + return partialJson; + } + + public List getAllImagesWithoutPagination(String searchApiJson, String findType) + throws IOException { + try { + JsonNode root = objectMapper.readTree(searchApiJson); + JsonNode matches = root.path(findType); + + List allImages = new ArrayList<>(); + + if (matches.isArray()) { + for (JsonNode match : matches) { + YandexSearchResponse.ImageResult result = mapImageResult(match); + + if ("exact_matches".equals(findType)) { + JsonNode thumbnail = match.path("thumbnail"); + if (!thumbnail.isMissingNode()) { + String thumbnailUrl = thumbnail.asText(); + if (thumbnailUrl.startsWith("data:image")) { + result.setUrl(thumbnailUrl); + } else if (!thumbnailUrl.isBlank()) { + result.setUrl(thumbnailUrl); + } + } + + JsonNode imageNode = match.path("image"); + if (!imageNode.isMissingNode()) { + String directUrl = imageNode.path("link").asText(); + if (directUrl != null && !directUrl.isBlank()) { + result.setUrl(directUrl); + } + } + } + + if (result.getUrl() != null && !result.getUrl().isBlank()) { + allImages.add(result); + } + } + } + + log.info("Parsed {} images from response (findType: {})", allImages.size(), findType); + return allImages; + + } catch (Exception e) { + log.error("Failed to parse JSON response: {}", e.getMessage()); + return new ArrayList<>(); + } } public List removeDuplicateUrls( @@ -315,3 +383,297 @@ public class SearchImageService { return source; } } +//@Slf4j +//@Service +//public class SearchImageService { +// +// private final ObjectMapper objectMapper; +// +// private final OkHttpClient yandexHttpClient; +// +// private final OkHttpClient googleHttpClient; +// +// public SearchImageService() { +// Proxy proxy = new Proxy(Proxy.Type.HTTP, +// new InetSocketAddress("193.46.217.94", 3128)); +// +// this.yandexHttpClient = new OkHttpClient.Builder() +//// .proxy(proxy) +// .connectTimeout(60, TimeUnit.SECONDS) +// .writeTimeout(60, TimeUnit.SECONDS) +// .readTimeout(60, TimeUnit.SECONDS) +// .callTimeout(60, TimeUnit.SECONDS) +// .followRedirects(true) +// .followSslRedirects(true) +// .retryOnConnectionFailure(true) +// .build(); +// +// this.googleHttpClient = new OkHttpClient.Builder() +//// .proxy(proxy) +// .connectTimeout(35, TimeUnit.SECONDS) +// .writeTimeout(35, TimeUnit.SECONDS) +// .readTimeout(35, TimeUnit.SECONDS) +// .callTimeout(35, TimeUnit.SECONDS) +// .followRedirects(true) +// .followSslRedirects(true) +// .retryOnConnectionFailure(true) +// .build(); +// +// this.objectMapper = new ObjectMapper(); +// } +// +// @Value("${yandex.api-key}") +// private String apiKey; +// +// @Value("${yandex.folder-id}") +// private String folderId; +// +// @Value("${yandex.search-url}") +// private String searchUrl; +// +// @Value("${searchapi.api-key:}") +// private String searchApiKey; +// +// @Value("${server.baseurl}") +// private String appBaseUrl; +// +// @PostConstruct +// public void init() { +// objectMapper.configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, false); +// objectMapper.configure(DeserializationFeature.ACCEPT_SINGLE_VALUE_AS_ARRAY, true); +// } +// +// public String searchReverseByPublicUrl(FileEntity fileEntity, String engine, String searchType) +// throws IOException, TimeoutException { +// +// if (!isImageFile(fileEntity)) { +// throw new NotValidFieldException( +// "File not image", +// new BaseResponse( +// 20007, +// MessageCode.INVALID_FIELD.getCode(), +// MessageCode.INVALID_FIELD.getDescription(), +// Map.of("file_type", fileEntity.getMimeType()) +// ) +// ); +// } +// +// String publicUrl = String.format("%s/api/files/public/%s", appBaseUrl, fileEntity.getId()); +// +// log.info("Searching reverse for image: {}", publicUrl); +// +// try { +// return callReverseImageApiByUrl(publicUrl, engine, searchType); +// } catch (SocketTimeoutException e) { +// log.error("Yandex search timeout after {}", fileEntity.getId()); +// throw new TimeoutException("Search timeout"); +// } +// } +// +// private boolean isImageFile(FileEntity fileEntity) { +// String mimeType = fileEntity.getMimeType(); +// return mimeType != null && mimeType.startsWith("image"); +// } +// +// private String callReverseImageApiByUrl(String imageUrl, String engine, String searchType) throws IOException { +// log.info("=== START SEARCH API CALL ==="); +// log.info("Engine: {}, SearchType: {}", engine, searchType); +// log.info("ImageUrl: {}", imageUrl); +// +// if (searchApiKey == null || searchApiKey.isBlank()) { +// log.error("SearchAPI key not configured"); +// throw new IllegalStateException("SearchAPI key not configured"); +// } +// +// OkHttpClient client = engine.contains("yandex_reverse_image") ? yandexHttpClient : googleHttpClient; +// +// HttpUrl url = HttpUrl.parse("https://www.searchapi.io/api/v1/search") +// .newBuilder() +// .addQueryParameter("engine", engine) +// .addQueryParameter("api_key", searchApiKey) +// .addQueryParameter("url", imageUrl) +// .addQueryParameter("search_type", searchType) +// .addQueryParameter("t_", String.valueOf(System.currentTimeMillis())) +// .build(); +// +// log.info("Request URL: {}", url); +// +// Request request = new Request.Builder() +// .url(url) +// .header("Accept", "application/json") +// .header("User-Agent", "Mozilla/5.0") +// .build(); +// +// long start = System.currentTimeMillis(); +// +// try (Response response = client.newCall(request).execute()) { +// long duration = System.currentTimeMillis() - start; +// log.info("SearchAPI response code={}, duration={}ms, engine={}", +// response.code(), duration, engine); +// +// ResponseBody body = response.body(); +// if (body == null) { +// log.error("Response body is null"); +// throw new IOException("Empty response body"); +// } +// +// String responseBody = body.string(); +// +// log.info("=== RESPONSE BODY LENGTH: {} ===", responseBody.length()); +// log.info("=== RESPONSE BODY START: {} ===", +// responseBody.length() > 200 ? responseBody.substring(0, 200) : responseBody); +// +// if (!response.isSuccessful()) { +// log.error("Response not successful: {}", response.code()); +// throw new IOException("API error " + response.code() + ": " + responseBody); +// } +// +// return responseBody; +// } catch (Exception e) { +// log.error("Exception in search API call: {}", e.getMessage(), e); +// throw e; +// } +// } +// +// public List getAllImagesWithoutPagination(String searchApiJson, String findType) +// throws IOException { +// JsonNode root = objectMapper.readTree(searchApiJson); +// JsonNode matches = root.path(findType); +// +// List allImages = new ArrayList<>(); +// +// if (matches.isArray()) { +// for (JsonNode match : matches) { +// YandexSearchResponse.ImageResult result = mapImageResult(match); +// +// if ("exact_matches".equals(findType)) { +// JsonNode thumbnail = match.path("thumbnail"); +// if (!thumbnail.isMissingNode()) { +// String thumbnailUrl = thumbnail.asText(); +// if (thumbnailUrl.startsWith("data:image")) { +// result.setUrl(thumbnailUrl); +// } else if (!thumbnailUrl.isBlank()) { +// result.setUrl(thumbnailUrl); +// } +// } +// +// JsonNode imageNode = match.path("image"); +// if (!imageNode.isMissingNode()) { +// String directUrl = imageNode.path("link").asText(); +// if (directUrl != null && !directUrl.isBlank()) { +// result.setUrl(directUrl); +// } +// } +// } +// +// if (result.getUrl() != null && !result.getUrl().isBlank()) { +// allImages.add(result); +// } +// } +// } +// +// return allImages; +// } +// +// public List removeDuplicateUrls( +// List yandexImages, +// List googleImages) { +// +// Map uniqueByUrl = new LinkedHashMap<>(); +// +// for (YandexSearchResponse.ImageResult image : googleImages) { +// String normalizedUrl = normalizeUrl(image.getUrl()); +// uniqueByUrl.putIfAbsent(normalizedUrl, image); +// } +// +// for (YandexSearchResponse.ImageResult image : yandexImages) { +// String normalizedUrl = normalizeUrl(image.getUrl()); +// uniqueByUrl.putIfAbsent(normalizedUrl, image); +// } +// +// return new ArrayList<>(uniqueByUrl.values()); +// } +// +// public List paginateResults( +// List allResults, +// int page, +// int pageSize) { +// +// if (allResults.isEmpty()) { +// return new ArrayList<>(); +// } +// +// int totalPages = (int) Math.ceil((double) allResults.size() / pageSize); +// +// if (page > totalPages) { +// return new ArrayList<>(); +// } +// +// int startIndex = (page - 1) * pageSize; +// int endIndex = Math.min(startIndex + pageSize, allResults.size()); +// +// return new ArrayList<>(allResults.subList(startIndex, endIndex)); +// } +// +// private String normalizeUrl(String url) { +// if (url == null) return null; +// +// try { +// if (url.startsWith("data:image")) { +// return url.length() > 100 ? url.substring(0, 100) : url; +// } +// +// java.net.URI uri = new java.net.URI(url); +// String host = uri.getHost(); +// if (host != null) { +// host = host.toLowerCase().replaceFirst("^www\\.", ""); +// } +// +// String path = uri.getPath(); +// if (path != null && path.length() > 1 && path.endsWith("/")) { +// path = path.substring(0, path.length() - 1); +// } +// +// return (host != null ? host : "") + (path != null ? path : ""); +// } catch (Exception e) { +// return url.toLowerCase() +// .replaceFirst("^(https?://)?(www\\.)?", "") +// .replaceFirst("/$", ""); +// } +// } +// +// private YandexSearchResponse.ImageResult mapImageResult(JsonNode match) { +// +// YandexSearchResponse.ImageResult result = +// new YandexSearchResponse.ImageResult(); +// +// JsonNode imageNode = match.path("image"); +// if (imageNode.isObject()) { +// result.setUrl(imageNode.path("link").asText()); +// result.setWidth(imageNode.path("width").asInt(0)); +// result.setHeight(imageNode.path("height").asInt(0)); +// } +// +// result.setPageUrl(match.path("link").asText()); +// result.setPageTitle(match.path("title").asText()); +// +// String source = match.path("source").asText(); +// result.setHost(extractHostFromSource(source)); +// +// return result; +// } +// +// private String extractHostFromSource(String source) { +// if (source == null || source.isEmpty()) { +// return ""; +// } +// source = source.replaceFirst("^(https?://)?(www\\.)?", ""); +// +// int slashIndex = source.indexOf('/'); +// if (slashIndex > 0) { +// return source.substring(0, slashIndex); +// } +// +// return source; +// } +//}