dev add violation notion

src/main/java/ru/soune/nocopy/service/search/SearchImageService.java

@@ -37,10 +37,10 @@ public class SearchImageService {
 
         this.yandexHttpClient = new OkHttpClient.Builder()
 //                .proxy(proxy)
-                .connectTimeout(60, TimeUnit.SECONDS)
-                .writeTimeout(60, TimeUnit.SECONDS)
-                .readTimeout(60, TimeUnit.SECONDS)
-                .callTimeout(60, TimeUnit.SECONDS)
+                .connectTimeout(120, TimeUnit.SECONDS)
+                .writeTimeout(120, TimeUnit.SECONDS)
+                .readTimeout(120, TimeUnit.SECONDS)
+                .callTimeout(120, TimeUnit.SECONDS)
                 .followRedirects(true)
                 .followSslRedirects(true)
                 .retryOnConnectionFailure(true)
@@ -48,10 +48,10 @@ public class SearchImageService {
 
         this.googleHttpClient = new OkHttpClient.Builder()
 //                .proxy(proxy)
-                .connectTimeout(35, TimeUnit.SECONDS)
-                .writeTimeout(35, TimeUnit.SECONDS)
-                .readTimeout(35, TimeUnit.SECONDS)
-                .callTimeout(35, TimeUnit.SECONDS)
+                .connectTimeout(120, TimeUnit.SECONDS)
+                .writeTimeout(120, TimeUnit.SECONDS)
+                .readTimeout(120, TimeUnit.SECONDS)
+                .callTimeout(120, TimeUnit.SECONDS)
                 .followRedirects(true)
                 .followSslRedirects(true)
                 .retryOnConnectionFailure(true)
@@ -103,7 +103,7 @@ public class SearchImageService {
         try {
             return callReverseImageApiByUrl(publicUrl, engine, searchType);
         } catch (SocketTimeoutException e) {
-            log.error("Yandex search timeout after {}", fileEntity.getId());
+            log.error("Search timeout for file: {}", fileEntity.getId());
             throw new TimeoutException("Search timeout");
         }
     }
@@ -131,6 +131,7 @@ public class SearchImageService {
                 .addQueryParameter("api_key", searchApiKey)
                 .addQueryParameter("url", imageUrl)
                 .addQueryParameter("search_type", searchType)
+                .addQueryParameter("num", "10")
                 .addQueryParameter("t_", String.valueOf(System.currentTimeMillis()))
                 .build();
 
@@ -155,62 +156,129 @@ public class SearchImageService {
                 throw new IOException("Empty response body");
             }
 
-            String responseBody = body.string();
+            String responseBody = readWithTimeout(body, 60000);
 
-            log.info("=== RESPONSE BODY LENGTH: {} ===", responseBody.length());
-            log.info("=== RESPONSE BODY START: {} ===",
-                    responseBody.length() > 200 ? responseBody.substring(0, 200) : responseBody);
+            log.info("=== RESPONSE BODY LENGTH: {} bytes received ===", responseBody.length());
+
+            if (responseBody.length() < 1000) {
+                log.info("=== RESPONSE BODY: {} ===", responseBody);
+            }
 
             if (!response.isSuccessful()) {
                 log.error("Response not successful: {}", response.code());
-                throw new IOException("API error " + response.code() + ": " + responseBody);
+                throw new IOException("API error " + response.code() + ": " +
+                        (responseBody.length() > 200 ? responseBody.substring(0, 200) : responseBody));
+            }
+
+            if (responseBody.length() > 0 && !responseBody.endsWith("}")) {
+                responseBody = fixTruncatedJson(responseBody);
+                log.info("Fixed truncated JSON, new length: {}", responseBody.length());
             }
 
             return responseBody;
+        } catch (InterruptedIOException e) {
+            log.warn("Timeout/Interrupted during reading for engine: {}", engine);
+            throw new IOException("Partial response received", e);
         } catch (Exception e) {
             log.error("Exception in search API call: {}", e.getMessage(), e);
             throw e;
         }
     }
 
-    public List<YandexSearchResponse.ImageResult> getAllImagesWithoutPagination(String searchApiJson, String findType)
-            throws IOException {
-        JsonNode root = objectMapper.readTree(searchApiJson);
-        JsonNode matches = root.path(findType);
+    private String readWithTimeout(ResponseBody body, long timeoutMs) throws IOException {
+        StringBuilder result = new StringBuilder();
+        long startTime = System.currentTimeMillis();
 
-        List<YandexSearchResponse.ImageResult> allImages = new ArrayList<>();
+        try (BufferedReader reader = new BufferedReader(body.charStream())) {
+            char[] buffer = new char[8192];
+            int read;
 
-        if (matches.isArray()) {
-            for (JsonNode match : matches) {
-                YandexSearchResponse.ImageResult result = mapImageResult(match);
+            while ((read = reader.read(buffer)) != -1) {
+                result.append(buffer, 0, read);
 
-                if ("exact_matches".equals(findType)) {
-                    JsonNode thumbnail = match.path("thumbnail");
-                    if (!thumbnail.isMissingNode()) {
-                        String thumbnailUrl = thumbnail.asText();
-                        if (thumbnailUrl.startsWith("data:image")) {
-                            result.setUrl(thumbnailUrl);
-                        } else if (!thumbnailUrl.isBlank()) {
-                            result.setUrl(thumbnailUrl);
-                        }
-                    }
-
-                    JsonNode imageNode = match.path("image");
-                    if (!imageNode.isMissingNode()) {
-                        String directUrl = imageNode.path("link").asText();
-                        if (directUrl != null && !directUrl.isBlank()) {
-                            result.setUrl(directUrl);
-                        }
-                    }
-                }
-
-                if (result.getUrl() != null && !result.getUrl().isBlank()) {
-                    allImages.add(result);
+                if (System.currentTimeMillis() - startTime > timeoutMs) {
+                    log.warn("Partial read timeout after {} ms, received {} bytes",
+                            timeoutMs, result.length());
+                    break;
                 }
             }
         }
 
-        return allImages;
+        return result.toString();
+    }
+
+    private String fixTruncatedJson(String partialJson) {
+        if (partialJson == null || partialJson.isEmpty()) {
+            return "{}";
+        }
+
+        String[] matchTypes = {"\"visual_matches\":", "\"exact_matches\":"};
+
+        for (String matchType : matchTypes) {
+            int matchesStart = partialJson.indexOf(matchType);
+            if (matchesStart > 0) {
+                int lastCompleteMatch = partialJson.lastIndexOf("},");
+                int arrayEnd = partialJson.lastIndexOf("]");
+
+                if (lastCompleteMatch > matchesStart && arrayEnd < lastCompleteMatch) {
+                    return partialJson.substring(0, lastCompleteMatch + 1) + "]}";
+                }
+                break;
+            }
+        }
+
+        if (!partialJson.endsWith("}")) {
+            return partialJson + "}";
+        }
+
+        return partialJson;
+    }
+
+    public List<YandexSearchResponse.ImageResult> getAllImagesWithoutPagination(String searchApiJson, String findType)
+            throws IOException {
+        try {
+            JsonNode root = objectMapper.readTree(searchApiJson);
+            JsonNode matches = root.path(findType);
+
+            List<YandexSearchResponse.ImageResult> allImages = new ArrayList<>();
+
+            if (matches.isArray()) {
+                for (JsonNode match : matches) {
+                    YandexSearchResponse.ImageResult result = mapImageResult(match);
+
+                    if ("exact_matches".equals(findType)) {
+                        JsonNode thumbnail = match.path("thumbnail");
+                        if (!thumbnail.isMissingNode()) {
+                            String thumbnailUrl = thumbnail.asText();
+                            if (thumbnailUrl.startsWith("data:image")) {
+                                result.setUrl(thumbnailUrl);
+                            } else if (!thumbnailUrl.isBlank()) {
+                                result.setUrl(thumbnailUrl);
+                            }
+                        }
+
+                        JsonNode imageNode = match.path("image");
+                        if (!imageNode.isMissingNode()) {
+                            String directUrl = imageNode.path("link").asText();
+                            if (directUrl != null && !directUrl.isBlank()) {
+                                result.setUrl(directUrl);
+                            }
+                        }
+                    }
+
+                    if (result.getUrl() != null && !result.getUrl().isBlank()) {
+                        allImages.add(result);
+                    }
+                }
+            }
+
+            log.info("Parsed {} images from response (findType: {})", allImages.size(), findType);
+            return allImages;
+
+        } catch (Exception e) {
+            log.error("Failed to parse JSON response: {}", e.getMessage());
+            return new ArrayList<>();
+        }
     }
 
     public List<YandexSearchResponse.ImageResult> removeDuplicateUrls(
@@ -315,3 +383,297 @@ public class SearchImageService {
         return source;
     }
 }
+//@Slf4j
+//@Service
+//public class SearchImageService {
+//
+//    private final ObjectMapper objectMapper;
+//
+//    private final OkHttpClient yandexHttpClient;
+//
+//    private final OkHttpClient googleHttpClient;
+//
+//    public SearchImageService() {
+//        Proxy proxy = new Proxy(Proxy.Type.HTTP,
+//                new InetSocketAddress("193.46.217.94", 3128));
+//
+//        this.yandexHttpClient = new OkHttpClient.Builder()
+////                .proxy(proxy)
+//                .connectTimeout(60, TimeUnit.SECONDS)
+//                .writeTimeout(60, TimeUnit.SECONDS)
+//                .readTimeout(60, TimeUnit.SECONDS)
+//                .callTimeout(60, TimeUnit.SECONDS)
+//                .followRedirects(true)
+//                .followSslRedirects(true)
+//                .retryOnConnectionFailure(true)
+//                .build();
+//
+//        this.googleHttpClient = new OkHttpClient.Builder()
+////                .proxy(proxy)
+//                .connectTimeout(35, TimeUnit.SECONDS)
+//                .writeTimeout(35, TimeUnit.SECONDS)
+//                .readTimeout(35, TimeUnit.SECONDS)
+//                .callTimeout(35, TimeUnit.SECONDS)
+//                .followRedirects(true)
+//                .followSslRedirects(true)
+//                .retryOnConnectionFailure(true)
+//                .build();
+//
+//        this.objectMapper = new ObjectMapper();
+//    }
+//
+//    @Value("${yandex.api-key}")
+//    private String apiKey;
+//
+//    @Value("${yandex.folder-id}")
+//    private String folderId;
+//
+//    @Value("${yandex.search-url}")
+//    private String searchUrl;
+//
+//    @Value("${searchapi.api-key:}")
+//    private String searchApiKey;
+//
+//    @Value("${server.baseurl}")
+//    private String appBaseUrl;
+//
+//    @PostConstruct
+//    public void init() {
+//        objectMapper.configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, false);
+//        objectMapper.configure(DeserializationFeature.ACCEPT_SINGLE_VALUE_AS_ARRAY, true);
+//    }
+//
+//    public String searchReverseByPublicUrl(FileEntity fileEntity, String engine, String searchType)
+//            throws IOException, TimeoutException {
+//
+//        if (!isImageFile(fileEntity)) {
+//            throw new NotValidFieldException(
+//                    "File not image",
+//                    new BaseResponse(
+//                            20007,
+//                            MessageCode.INVALID_FIELD.getCode(),
+//                            MessageCode.INVALID_FIELD.getDescription(),
+//                            Map.of("file_type", fileEntity.getMimeType())
+//                    )
+//            );
+//        }
+//
+//        String publicUrl = String.format("%s/api/files/public/%s", appBaseUrl, fileEntity.getId());
+//
+//        log.info("Searching reverse for image: {}", publicUrl);
+//
+//        try {
+//            return callReverseImageApiByUrl(publicUrl, engine, searchType);
+//        } catch (SocketTimeoutException e) {
+//            log.error("Yandex search timeout after {}", fileEntity.getId());
+//            throw new TimeoutException("Search timeout");
+//        }
+//    }
+//
+//    private boolean isImageFile(FileEntity fileEntity) {
+//        String mimeType = fileEntity.getMimeType();
+//        return mimeType != null && mimeType.startsWith("image");
+//    }
+//
+//    private String callReverseImageApiByUrl(String imageUrl, String engine, String searchType) throws IOException {
+//        log.info("=== START SEARCH API CALL ===");
+//        log.info("Engine: {}, SearchType: {}", engine, searchType);
+//        log.info("ImageUrl: {}", imageUrl);
+//
+//        if (searchApiKey == null || searchApiKey.isBlank()) {
+//            log.error("SearchAPI key not configured");
+//            throw new IllegalStateException("SearchAPI key not configured");
+//        }
+//
+//        OkHttpClient client = engine.contains("yandex_reverse_image") ? yandexHttpClient : googleHttpClient;
+//
+//        HttpUrl url = HttpUrl.parse("https://www.searchapi.io/api/v1/search")
+//                .newBuilder()
+//                .addQueryParameter("engine", engine)
+//                .addQueryParameter("api_key", searchApiKey)
+//                .addQueryParameter("url", imageUrl)
+//                .addQueryParameter("search_type", searchType)
+//                .addQueryParameter("t_", String.valueOf(System.currentTimeMillis()))
+//                .build();
+//
+//        log.info("Request URL: {}", url);
+//
+//        Request request = new Request.Builder()
+//                .url(url)
+//                .header("Accept", "application/json")
+//                .header("User-Agent", "Mozilla/5.0")
+//                .build();
+//
+//        long start = System.currentTimeMillis();
+//
+//        try (Response response = client.newCall(request).execute()) {
+//            long duration = System.currentTimeMillis() - start;
+//            log.info("SearchAPI response code={}, duration={}ms, engine={}",
+//                    response.code(), duration, engine);
+//
+//            ResponseBody body = response.body();
+//            if (body == null) {
+//                log.error("Response body is null");
+//                throw new IOException("Empty response body");
+//            }
+//
+//            String responseBody = body.string();
+//
+//            log.info("=== RESPONSE BODY LENGTH: {} ===", responseBody.length());
+//            log.info("=== RESPONSE BODY START: {} ===",
+//                    responseBody.length() > 200 ? responseBody.substring(0, 200) : responseBody);
+//
+//            if (!response.isSuccessful()) {
+//                log.error("Response not successful: {}", response.code());
+//                throw new IOException("API error " + response.code() + ": " + responseBody);
+//            }
+//
+//            return responseBody;
+//        } catch (Exception e) {
+//            log.error("Exception in search API call: {}", e.getMessage(), e);
+//            throw e;
+//        }
+//    }
+//
+//    public List<YandexSearchResponse.ImageResult> getAllImagesWithoutPagination(String searchApiJson, String findType)
+//            throws IOException {
+//        JsonNode root = objectMapper.readTree(searchApiJson);
+//        JsonNode matches = root.path(findType);
+//
+//        List<YandexSearchResponse.ImageResult> allImages = new ArrayList<>();
+//
+//        if (matches.isArray()) {
+//            for (JsonNode match : matches) {
+//                YandexSearchResponse.ImageResult result = mapImageResult(match);
+//
+//                if ("exact_matches".equals(findType)) {
+//                    JsonNode thumbnail = match.path("thumbnail");
+//                    if (!thumbnail.isMissingNode()) {
+//                        String thumbnailUrl = thumbnail.asText();
+//                        if (thumbnailUrl.startsWith("data:image")) {
+//                            result.setUrl(thumbnailUrl);
+//                        } else if (!thumbnailUrl.isBlank()) {
+//                            result.setUrl(thumbnailUrl);
+//                        }
+//                    }
+//
+//                    JsonNode imageNode = match.path("image");
+//                    if (!imageNode.isMissingNode()) {
+//                        String directUrl = imageNode.path("link").asText();
+//                        if (directUrl != null && !directUrl.isBlank()) {
+//                            result.setUrl(directUrl);
+//                        }
+//                    }
+//                }
+//
+//                if (result.getUrl() != null && !result.getUrl().isBlank()) {
+//                    allImages.add(result);
+//                }
+//            }
+//        }
+//
+//        return allImages;
+//    }
+//
+//    public List<YandexSearchResponse.ImageResult> removeDuplicateUrls(
+//            List<YandexSearchResponse.ImageResult> yandexImages,
+//            List<YandexSearchResponse.ImageResult> googleImages) {
+//
+//        Map<String, YandexSearchResponse.ImageResult> uniqueByUrl = new LinkedHashMap<>();
+//
+//        for (YandexSearchResponse.ImageResult image : googleImages) {
+//            String normalizedUrl = normalizeUrl(image.getUrl());
+//            uniqueByUrl.putIfAbsent(normalizedUrl, image);
+//        }
+//
+//        for (YandexSearchResponse.ImageResult image : yandexImages) {
+//            String normalizedUrl = normalizeUrl(image.getUrl());
+//            uniqueByUrl.putIfAbsent(normalizedUrl, image);
+//        }
+//
+//        return new ArrayList<>(uniqueByUrl.values());
+//    }
+//
+//    public List<YandexSearchResponse.ImageResult> paginateResults(
+//            List<YandexSearchResponse.ImageResult> allResults,
+//            int page,
+//            int pageSize) {
+//
+//        if (allResults.isEmpty()) {
+//            return new ArrayList<>();
+//        }
+//
+//        int totalPages = (int) Math.ceil((double) allResults.size() / pageSize);
+//
+//        if (page > totalPages) {
+//            return new ArrayList<>();
+//        }
+//
+//        int startIndex = (page - 1) * pageSize;
+//        int endIndex = Math.min(startIndex + pageSize, allResults.size());
+//
+//        return new ArrayList<>(allResults.subList(startIndex, endIndex));
+//    }
+//
+//    private String normalizeUrl(String url) {
+//        if (url == null) return null;
+//
+//        try {
+//            if (url.startsWith("data:image")) {
+//                return url.length() > 100 ? url.substring(0, 100) : url;
+//            }
+//
+//            java.net.URI uri = new java.net.URI(url);
+//            String host = uri.getHost();
+//            if (host != null) {
+//                host = host.toLowerCase().replaceFirst("^www\\.", "");
+//            }
+//
+//            String path = uri.getPath();
+//            if (path != null && path.length() > 1 && path.endsWith("/")) {
+//                path = path.substring(0, path.length() - 1);
+//            }
+//
+//            return (host != null ? host : "") + (path != null ? path : "");
+//        } catch (Exception e) {
+//            return url.toLowerCase()
+//                    .replaceFirst("^(https?://)?(www\\.)?", "")
+//                    .replaceFirst("/$", "");
+//        }
+//    }
+//
+//    private YandexSearchResponse.ImageResult mapImageResult(JsonNode match) {
+//
+//        YandexSearchResponse.ImageResult result =
+//                new YandexSearchResponse.ImageResult();
+//
+//        JsonNode imageNode = match.path("image");
+//        if (imageNode.isObject()) {
+//            result.setUrl(imageNode.path("link").asText());
+//            result.setWidth(imageNode.path("width").asInt(0));
+//            result.setHeight(imageNode.path("height").asInt(0));
+//        }
+//
+//        result.setPageUrl(match.path("link").asText());
+//        result.setPageTitle(match.path("title").asText());
+//
+//        String source = match.path("source").asText();
+//        result.setHost(extractHostFromSource(source));
+//
+//        return result;
+//    }
+//
+//    private String extractHostFromSource(String source) {
+//        if (source == null || source.isEmpty()) {
+//            return "";
+//        }
+//        source = source.replaceFirst("^(https?://)?(www\\.)?", "");
+//
+//        int slashIndex = source.indexOf('/');
+//        if (slashIndex > 0) {
+//            return source.substring(0, slashIndex);
+//        }
+//
+//        return source;
+//    }
+//}