diff --git a/.env b/.env new file mode 100644 index 0000000..91f491f --- /dev/null +++ b/.env @@ -0,0 +1,15 @@ +POSTGRES_DB: monitoring_db +POSTGRES_USER: adminMonitoring +POSTGRES_PASSWORD: monitoringDbApp +POSTGRES_PORT=5353 +POSTGRES_HOST=db + +REDIS_HOST=redis +REDIS_PORT=6379 + +SERVER_PORT=8080 + +MAIL_HOST_PROD=postfix-production +MAIL_PORT_PROD=25 +MAIL_USERNAME_PROD=noreply@nocopy.com +SMTP_PASSWORD_PROD=nocopy!nocopy! \ No newline at end of file diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 0000000..d9be650 --- /dev/null +++ b/Dockerfile @@ -0,0 +1,14 @@ +FROM gradle:8.14.2-jdk21 AS build + +WORKDIR /app +COPY . . +RUN gradle --no-daemon clean bootJar -x test + +FROM eclipse-temurin:21-jre + +WORKDIR /app +COPY --from=build /app/build/libs/*.jar app.jar + +EXPOSE 8083 + +CMD ["java", "-jar", "app.jar"] \ No newline at end of file diff --git a/build.gradle b/build.gradle index 00044d1..066d8fc 100644 --- a/build.gradle +++ b/build.gradle @@ -1,11 +1,12 @@ plugins { id 'java' - id 'org.springframework.boot' version '4.0.6' + id 'org.springframework.boot' version '3.5.6' id 'io.spring.dependency-management' version '1.1.7' } group = 'ru.no-copy' version = '0.0.1-SNAPSHOT' +description = 'monitoring' java { toolchain { @@ -13,16 +14,25 @@ java { } } +configurations { + compileOnly { + extendsFrom annotationProcessor + } +} + repositories { mavenCentral() } dependencies { implementation 'org.springframework.boot:spring-boot-starter-actuator' + implementation 'org.springframework.boot:spring-boot-starter' + implementation 'org.springframework.kafka:spring-kafka' implementation 'org.springframework.boot:spring-boot-starter-data-jpa' implementation 'org.springframework.boot:spring-boot-starter-security' implementation 'org.springframework.boot:spring-boot-starter-thymeleaf' implementation 'org.springframework.boot:spring-boot-starter-validation' + annotationProcessor 'org.projectlombok:lombok' implementation 'org.thymeleaf.extras:thymeleaf-extras-springsecurity6' developmentOnly 'org.springframework.boot:spring-boot-devtools' runtimeOnly 'org.postgresql:postgresql' @@ -32,6 +42,11 @@ dependencies { testImplementation 'org.springframework.boot:spring-boot-starter-thymeleaf-test' testImplementation 'org.springframework.boot:spring-boot-starter-validation-test' testRuntimeOnly 'org.junit.platform:junit-platform-launcher' + implementation 'com.fasterxml.jackson.dataformat:jackson-dataformat-xml:2.17.0' + implementation 'tools.jackson.core:jackson-core:3.0.3' + implementation group: 'com.fasterxml.jackson.core', name: 'jackson-databind', version: '2.17.2' + implementation group: 'com.squareup.okhttp3', name: 'okhttp', version: '4.12.0' + } tasks.named('test') { diff --git a/docker-compose.yaml b/docker-compose.yaml new file mode 100644 index 0000000..7ec1267 --- /dev/null +++ b/docker-compose.yaml @@ -0,0 +1,81 @@ +version: '3.9' + +services: + db: + image: postgres:17.7 + container_name: monitoring-db + environment: + POSTGRES_DB: monitoring_db + POSTGRES_USER: adminMonitoring + POSTGRES_PASSWORD: monitoringDbApp + volumes: + - postgres_data:/var/lib/postgresql/data + ports: + - "5353:5432" + restart: unless-stopped + + monitoring: + build: . + container_name: monitoring-backend + ports: + - "${SERVER_PORT:-8083}:${SERVER_PORT:-8083}" + environment: + SPRING_PROFILES_ACTIVE: prod + POSTGRES_HOST: db + POSTGRES_PORT: 5432 + POSTGRES_DB: monitoring_db + POSTGRES_USER: adminMonitoring + POSTGRES_PASSWORD: monitoringDbApp + SERVER_PORT: ${SERVER_PORT:-8083} +# SPRING_KAFKA_BOOTSTRAP_SERVERS: kafka:9092 + depends_on: + - db + restart: unless-stopped + networks: + - app-network + + kafka: + image: apache/kafka:latest + container_name: kafka + ports: + - "9092:9092" + environment: + KAFKA_NODE_ID: 1 + KAFKA_PROCESS_ROLES: 'broker,controller' + KAFKA_CONTROLLER_QUORUM_VOTERS: '1@kafka:9093' + KAFKA_LISTENERS: 'PLAINTEXT://0.0.0.0:9092,CONTROLLER://0.0.0.0:9093' + KAFKA_ADVERTISED_LISTENERS: 'PLAINTEXT://kafka:9092' + KAFKA_CONTROLLER_LISTENER_NAMES: 'CONTROLLER' + KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: 'CONTROLLER:PLAINTEXT,PLAINTEXT:PLAINTEXT' + KAFKA_AUTO_CREATE_TOPICS_ENABLE: 'true' + KAFKA_DELETE_TOPIC_ENABLE: 'true' + KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR: 1 + KAFKA_DEFAULT_REPLICATION_FACTOR: 1 + KAFKA_MIN_INSYNC_REPLICAS: 1 + volumes: + - kafka_data:/var/lib/kafka/data + networks: + - app-network + + kafka-ui: + image: provectuslabs/kafka-ui:latest + container_name: kafka-ui + ports: + - "8010:8080" + environment: + DYNAMIC_CONFIG_ENABLED: 'true' + KAFKA_CLUSTERS_0_NAME: 'local-cluster' + KAFKA_CLUSTERS_0_BOOTSTRAPSERVERS: 'kafka:9092' + depends_on: + - kafka + networks: + - app-network + + +volumes: + kafka_data: + postgres_data: + +networks: + app-network: + external: true \ No newline at end of file diff --git a/src/main/java/ru/no_copy/monitoring/controller/HealthCheckController.java b/src/main/java/ru/no_copy/monitoring/controller/HealthCheckController.java new file mode 100644 index 0000000..cad81f7 --- /dev/null +++ b/src/main/java/ru/no_copy/monitoring/controller/HealthCheckController.java @@ -0,0 +1,17 @@ +package ru.no_copy.monitoring.controller; + +import org.springframework.http.ResponseEntity; +import org.springframework.web.bind.annotation.GetMapping; +import org.springframework.web.bind.annotation.RestController; + +import java.util.Map; + +@RestController("/api/monitoring") +public class HealthCheckController { + + @GetMapping + public ResponseEntity> healthCheck() { + + return ResponseEntity.ok().body(Map.of("status","ok")); + } +} diff --git a/src/main/java/ru/no_copy/monitoring/dto/MonitoringDTO.java b/src/main/java/ru/no_copy/monitoring/dto/MonitoringDTO.java new file mode 100644 index 0000000..a1c1218 --- /dev/null +++ b/src/main/java/ru/no_copy/monitoring/dto/MonitoringDTO.java @@ -0,0 +1,17 @@ +package ru.no_copy.monitoring.dto; + +import lombok.AllArgsConstructor; +import lombok.Builder; +import lombok.Data; +import lombok.NoArgsConstructor; + +@Data +@Builder +@NoArgsConstructor +@AllArgsConstructor +public class MonitoringDTO { + String fileId; + String baseUrl; + String engine; + String searchType; +} diff --git a/src/main/java/ru/no_copy/monitoring/dto/SearchResponse.java b/src/main/java/ru/no_copy/monitoring/dto/SearchResponse.java new file mode 100644 index 0000000..34befff --- /dev/null +++ b/src/main/java/ru/no_copy/monitoring/dto/SearchResponse.java @@ -0,0 +1,47 @@ +package ru.no_copy.monitoring.dto; + +import com.fasterxml.jackson.annotation.JsonIgnoreProperties; +import com.fasterxml.jackson.annotation.JsonProperty; +import lombok.Data; + +import java.util.List; + +@Data +@JsonIgnoreProperties(ignoreUnknown = true) +public class SearchResponse { + @JsonProperty("images") + private List images; + + @Data + @JsonIgnoreProperties(ignoreUnknown = true) + public static class ImageResult { + @JsonProperty("url") + private String url; + + @JsonProperty("pageUrl") + private String pageUrl; + + @JsonProperty("pageTitle") + private String pageTitle; + + @JsonProperty("width") + private Integer width; + + @JsonProperty("height") + private Integer height; + + @JsonProperty("host") + private String host; + + @JsonProperty("file_id") + private String fileId; + } + + private int page; + + private int pageSize; + + private int totalResults; + + private int totalPages; +} diff --git a/src/main/java/ru/no_copy/monitoring/kafka/MonitoringCommandListener.java b/src/main/java/ru/no_copy/monitoring/kafka/MonitoringCommandListener.java new file mode 100644 index 0000000..332df85 --- /dev/null +++ b/src/main/java/ru/no_copy/monitoring/kafka/MonitoringCommandListener.java @@ -0,0 +1,32 @@ +package ru.no_copy.monitoring.kafka; + +import lombok.extern.slf4j.Slf4j; +import org.springframework.kafka.annotation.KafkaListener; +import org.springframework.stereotype.Component; +import ru.no_copy.monitoring.service.MonitoringService; + +import java.io.IOException; +import java.util.concurrent.CompletableFuture; +import java.util.concurrent.TimeoutException; + +@Component +@Slf4j +public class MonitoringCommandListener { + private final MonitoringService monitoringService; + + public MonitoringCommandListener(MonitoringService monitoringService) { + this.monitoringService = monitoringService; + } + + @KafkaListener(topics = "monitoring-commands", groupId = "monitoring-service") + public void handleCommand(String commandMessage) { + log.info("message: " + commandMessage); + CompletableFuture.runAsync(() -> { + try { + monitoringService.execute(commandMessage); + } catch (IOException | TimeoutException e) { + throw new RuntimeException(e); + } + }); + } +} diff --git a/src/main/java/ru/no_copy/monitoring/searcher/SearchImageService.java b/src/main/java/ru/no_copy/monitoring/searcher/SearchImageService.java new file mode 100644 index 0000000..35ffaac --- /dev/null +++ b/src/main/java/ru/no_copy/monitoring/searcher/SearchImageService.java @@ -0,0 +1,173 @@ +package ru.no_copy.monitoring.searcher; + +import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.ObjectMapper; +import lombok.RequiredArgsConstructor; +import lombok.extern.slf4j.Slf4j; +import okhttp3.*; +import org.springframework.beans.factory.annotation.Value; +import org.springframework.stereotype.Service; +import ru.no_copy.monitoring.dto.SearchResponse; + +import java.io.IOException; +import java.net.SocketTimeoutException; +import java.util.ArrayList; +import java.util.List; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.TimeoutException; + +@Slf4j +@Service +@RequiredArgsConstructor +public class SearchImageService { + + private final ObjectMapper objectMapper; + + @Value("${searchapi.api-key:}") + private String searchApiKey; + + public String searchReverseByPublicUrl(String fileId, String baseUrl, String engine, String searchType) + throws IOException, TimeoutException { + + String publicUrl = String.format("%s/api/files/public/%s", baseUrl, fileId); + + try { + return callReverseImageApiByUrl(publicUrl, engine, searchType); + } catch (SocketTimeoutException e) { + log.error("Yandex search timeout after {}", fileId); + throw new TimeoutException("Search timeout"); + } + } + + private String callReverseImageApiByUrl(String imageUrl, String engine, String searchType) throws IOException { + if (searchApiKey == null || searchApiKey.isBlank()) { + throw new IllegalStateException("SearchAPI key not configured"); + } + + OkHttpClient client = createHttpClient(); + + HttpUrl url = HttpUrl.parse("https://www.searchapi.io/api/v1/search") + .newBuilder() + .addQueryParameter("engine", engine) + .addQueryParameter("api_key", searchApiKey) + .addQueryParameter("url", imageUrl) + .addQueryParameter("search_type", searchType) + .addQueryParameter("t_", String.valueOf(System.currentTimeMillis())) + .build(); + + Request request = new Request.Builder() + .url(url) + .header("Accept", "application/json") + .header("User-Agent", "Mozilla/5.0") + .build(); + + long start = System.currentTimeMillis(); + + try (Response response = client.newCall(request).execute()) { + ResponseBody body = response.body(); + long duration = System.currentTimeMillis() - start; + log.info("SearchAPI response code={}, duration={}ms, engine={}", + response.code(), duration, engine); + + if (!response.isSuccessful()) { + String errorBody = response.body() != null ? response.body().string() : "null"; + throw new IOException("API error " + response.code() + ": " + errorBody); + } + + if (body == null) { + throw new IOException("Empty response body"); + } + + return body.string(); + } + } + + public List getAllImagesWithoutPagination(String searchApiJson, String findType, String fileId) + throws IOException { + JsonNode root = objectMapper.readTree(searchApiJson); + JsonNode matches = root.path(findType); + + List allImages = new ArrayList<>(); + + if (matches.isArray()) { + for (JsonNode match : matches) { + SearchResponse.ImageResult result = mapImageResult(match); + result.setFileId(fileId); + + if ("exact_matches".equals(findType)) { + JsonNode thumbnail = match.path("thumbnail"); + if (!thumbnail.isMissingNode()) { + String thumbnailUrl = thumbnail.asText(); + if (thumbnailUrl.startsWith("data:image")) { + result.setUrl(thumbnailUrl); + } else if (!thumbnailUrl.isBlank()) { + result.setUrl(thumbnailUrl); + } + } + + JsonNode imageNode = match.path("image"); + if (!imageNode.isMissingNode()) { + String directUrl = imageNode.path("link").asText(); + if (directUrl != null && !directUrl.isBlank()) { + result.setUrl(directUrl); + } + } + } + + if (result.getUrl() != null && !result.getUrl().isBlank()) { + allImages.add(result); + } + } + } + + return allImages; + } + + private SearchResponse.ImageResult mapImageResult(JsonNode match) { + + SearchResponse.ImageResult result = + new SearchResponse.ImageResult(); + + JsonNode imageNode = match.path("image"); + if (imageNode.isObject()) { + result.setUrl(imageNode.path("link").asText()); + result.setWidth(imageNode.path("width").asInt(0)); + result.setHeight(imageNode.path("height").asInt(0)); + } + + result.setPageUrl(match.path("link").asText()); + result.setPageTitle(match.path("title").asText()); + + String source = match.path("source").asText(); + result.setHost(extractHostFromSource(source)); + + return result; + } + + private OkHttpClient createHttpClient() { + OkHttpClient.Builder builder = new OkHttpClient.Builder() + .connectTimeout(35, TimeUnit.SECONDS) + .writeTimeout(35, TimeUnit.SECONDS) + .readTimeout(35, TimeUnit.SECONDS) + .callTimeout(35, TimeUnit.SECONDS) + .followRedirects(true) + .followSslRedirects(true) + .retryOnConnectionFailure(true); + + return builder.build(); + } + + private String extractHostFromSource(String source) { + if (source == null || source.isEmpty()) { + return ""; + } + source = source.replaceFirst("^(https?://)?(www\\.)?", ""); + + int slashIndex = source.indexOf('/'); + if (slashIndex > 0) { + return source.substring(0, slashIndex); + } + + return source; + } +} diff --git a/src/main/java/ru/no_copy/monitoring/service/MonitoringService.java b/src/main/java/ru/no_copy/monitoring/service/MonitoringService.java new file mode 100644 index 0000000..f9aa8fc --- /dev/null +++ b/src/main/java/ru/no_copy/monitoring/service/MonitoringService.java @@ -0,0 +1,66 @@ +package ru.no_copy.monitoring.service; + +import com.fasterxml.jackson.databind.ObjectMapper; +import lombok.RequiredArgsConstructor; +import lombok.extern.slf4j.Slf4j; +import org.springframework.kafka.core.KafkaTemplate; +import org.springframework.stereotype.Service; +import ru.no_copy.monitoring.dto.MonitoringDTO; +import ru.no_copy.monitoring.dto.SearchResponse; +import ru.no_copy.monitoring.searcher.SearchImageService; + +import java.io.IOException; +import java.util.List; +import java.util.Set; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.TimeoutException; +import java.util.function.Function; + +@Service +@RequiredArgsConstructor +@Slf4j +public class MonitoringService { + + private final SearchImageService imageService; + + private final ObjectMapper objectMapper; + + private final KafkaTemplate kafkaTemplate; + + public void execute(String message) throws IOException, TimeoutException { + MonitoringDTO monitoring; + try { + monitoring = objectMapper.readValue(message, MonitoringDTO.class); + log.info("Monitoring object создан: fileId={}, engine={}", monitoring.getFileId(), monitoring.getEngine()); + } catch (Exception e) { + log.error("Exception readValue: {}", e.getMessage(), e); + throw new RuntimeException(e); + } + + String searchResponse = imageService.searchReverseByPublicUrl(monitoring.getFileId(), monitoring.getBaseUrl(), + monitoring.getEngine(), monitoring.getSearchType()); + log.info("Search response: {}", searchResponse); + + List images = + imageService.getAllImagesWithoutPagination(searchResponse, + monitoring.getSearchType(), + monitoring.getFileId()); + + log.info("Images count before filter: {}", images.size()); + + images = images.stream() + .filter(distinctByKey(SearchResponse.ImageResult::getUrl)) + .toList(); + + log.info("Images count after filter: {}", images.size()); + + for (SearchResponse.ImageResult imageResult: images) { + kafkaTemplate.send("monitoring-results", imageResult); + } + } + + private static java.util.function.Predicate distinctByKey(Function keyExtractor) { + Set seen = ConcurrentHashMap.newKeySet(); + return t -> seen.add(keyExtractor.apply(t)); + } +} diff --git a/src/main/resources/application.properties b/src/main/resources/application.properties deleted file mode 100644 index ee1e996..0000000 --- a/src/main/resources/application.properties +++ /dev/null @@ -1 +0,0 @@ -spring.application.name=monitoring diff --git a/src/main/resources/application.yaml b/src/main/resources/application.yaml new file mode 100644 index 0000000..01da4c3 --- /dev/null +++ b/src/main/resources/application.yaml @@ -0,0 +1,44 @@ +spring: + kafka: + bootstrap-servers: ${KAFKA_BOOTSTRAP_SERVERS:kafka:9092} + consumer: + group-id: monitoring-service + key-deserializer: org.apache.kafka.common.serialization.StringDeserializer + value-deserializer: org.apache.kafka.common.serialization.StringDeserializer + producer: + key-serializer: org.apache.kafka.common.serialization.StringSerializer + value-serializer: org.springframework.kafka.support.serializer.JsonSerializer + + cloud: + compatibility-verifier: + enabled: false + + datasource: + url: jdbc:postgresql://${POSTGRES_HOST:db}:${POSTGRES_PORT:5432}/${POSTGRES_DB:monitoring_db} + username: ${POSTGRES_USER:adminMonitoring} + password: ${POSTGRES_PASSWORD:monitoringDbApp} + + jpa: + hibernate: + ddl-auto: update + properties: + hibernate: + dialect: org.hibernate.dialect.PostgreSQLDialect + format_sql: true + +server: + port: ${SERVER_PORT:8083} + +searchapi: + api-key: ${SEARCHAPI_API_KEY:5jyYZC8jSaxhZTwjMUhwtAXi} + reverse-image-url: "https://searchapi.io/api/v1/search" + +app: + internal-api-key: "tljzkXiEYF1klSHuG2hPZKjx6EsBX8RQP6UrzMdQanSKbRYuHuOBwrXejZkn7V4FICvIahoDmYD2hjNBPw61NFbGIt4scOzwZpyCiXEa1YKLAeJSPso4S43LIZlKjO4S" + +jwt: + secret: Z29nb3Bvd2VycmFuZ2VydHVtYmF5YW1iZjMyNDIyMjh3aW5lcndpbmVy + expiration: 3600000 + +nocopy-dashboard: + url: http://172.17.0.1:3001 \ No newline at end of file diff --git a/src/test/java/ru/no_copy/monitoring/MonitoringApplicationTests.java b/src/test/java/ru/no_copy/monitoring/MonitoringApplicationTests.java deleted file mode 100644 index 9246d03..0000000 --- a/src/test/java/ru/no_copy/monitoring/MonitoringApplicationTests.java +++ /dev/null @@ -1,13 +0,0 @@ -package ru.no_copy.monitoring; - -import org.junit.jupiter.api.Test; -import org.springframework.boot.test.context.SpringBootTest; - -@SpringBootTest -class MonitoringApplicationTests { - - @Test - void contextLoads() { - } - -}