NCBACK-3 Add found by yandex image

Dockerfile

@@ -12,8 +12,6 @@ WORKDIR /app
 
 COPY --from=build /app/build/libs/*.jar app.jar
 
-RUN mkdir -p /data/uploads && chmod 755 /data/uploads
-
 EXPOSE 8080
 
 CMD ["java", "-jar", "app.jar"]

build.gradle

@@ -31,6 +31,10 @@ dependencies {
 	implementation 'org.mapstruct:mapstruct:1.5.5.Final'
 	implementation 'commons-validator:commons-validator:1.7'
 
+	implementation 'com.fasterxml.jackson.dataformat:jackson-dataformat-xml:2.17.0'
+	implementation 'tools.jackson.core:jackson-core:3.0.3'
+	implementation 'jakarta.xml.bind:jakarta.xml.bind-api:4.0.0'
+
 	annotationProcessor 'org.mapstruct:mapstruct-processor:1.5.5.Final'
 	annotationProcessor 'org.projectlombok:lombok'
 

docker-compose.yaml

@@ -1,16 +1,6 @@
 version: '3.9'
 
 services:
-  storage:
-    image: alpine:latest
-    container_name: file-storage
-    networks:
-      - app-network
-    volumes:
-      - uploads_data:/storage:rw
-    command: tail -f /dev/null
-    restart: unless-stopped
-
   db:
     image: postgres:17
     restart: always
@@ -40,7 +30,8 @@ services:
       POSTGRES_PASSWORD: postgres
       POSTGRES_PORT: 5432
       POSTGRES_HOST: db
-      STORAGE_SERVICE_URL: http://storage:8081
+      YANDEX_SEARCH_API_KEY: AQVNyaVaUmgUb1GMCtf5zSEqFxy0woXrcMOOB43q
+      YANDEX_SEARCH_FOLDER_ID: b1gokpdbm6qfpsou8pcd
     depends_on:
       - db
     ports:
@@ -52,7 +43,7 @@ services:
           - backend
           - api
     volumes:
-      - uploads_data:/data/uploads:rw
+      - ./uploads:/data/uploads:rw
 
   grafana:
     image: grafana/grafana:10.3.1
@@ -157,7 +148,7 @@ volumes:
   loki_chunks:
   loki_index:
   loki_rules:
-  uploads_data:
+#  uploads_volume:
 
 networks:
   app-network:

infrastructure/jenkins/deploy.groovy

@@ -114,7 +114,6 @@ EOF
                                   --network app-network \\
                                   --network-alias app \\
                                   -p 80:8080 \\
-                                  -v /opt/uploads:/data/uploads:rw \\
                                   -e POSTGRES_DB=no_copy_ \\
                                   -e POSTGRES_USER=postgres \\
                                   -e POSTGRES_PASSWORD=postgres \\

src/main/java/ru/soune/nocopy/configuration/HandlerConfig.java

@@ -16,7 +16,8 @@ public class HandlerConfig {
             LoginRequestHandler login,
             FileUploadHandler upload,
             FileEntityHandler file,
-            LogoutRequestHandler logoutHandler
+            LogoutRequestHandler logoutHandler,
+            ImageFoundRequestHandler imageFoundRequestHandler
     ) {
         Map<Integer, RequestHandler> map = new HashMap<>();
         map.put(20001, login);
@@ -24,6 +25,7 @@ public class HandlerConfig {
         map.put(20004, upload);
         map.put(20005, file);
         map.put(20006, logoutHandler);
+        map.put(20007, imageFoundRequestHandler);
 
         return map;
     }

src/main/java/ru/soune/nocopy/configuration/JacksonConfig.java

@@ -35,6 +35,8 @@ public class JacksonConfig {
 
         mapper.registerModule(javaTimeModule);
         mapper.disable(SerializationFeature.WRITE_DATES_AS_TIMESTAMPS);
+        mapper.configure(com.fasterxml.jackson.databind.DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, false);
+        mapper.enable(SerializationFeature.INDENT_OUTPUT);
 
         return mapper;
     }

src/main/java/ru/soune/nocopy/configuration/file/YandexConfig.java

@@ -0,0 +1,14 @@
+package ru.soune.nocopy.configuration.file;
+
+import lombok.Data;
+import org.springframework.boot.context.properties.ConfigurationProperties;
+import org.springframework.context.annotation.Configuration;
+
+@Data
+@Configuration
+@ConfigurationProperties(prefix = "yandex")
+public class YandexConfig {
+    private String apiKey;
+    private String folderId;
+    private String searchUrl;
+}

src/main/java/ru/soune/nocopy/controller/ApiController.java

@@ -387,11 +387,11 @@ public class ApiController {
             return false;
         }
 
-        return mimeType.startsWith("image") ||
+        return mimeType.startsWith("image/") ||
-                mimeType.startsWith("text") ||
+                mimeType.startsWith("text/") ||
-                mimeType.equals("pdf") ||
+                mimeType.equals("application/pdf") ||
-                mimeType.startsWith("video") ||
+                mimeType.startsWith("video/") ||
-                mimeType.startsWith("audio");
+                mimeType.startsWith("audio/");
     }
 
     private Long getUserIdFromToken(String tokenHeader) {

src/main/java/ru/soune/nocopy/dto/MessageCode.java

@@ -8,6 +8,7 @@ public enum MessageCode {
     INVALID_TOKEN(2, "Invalid token"),
     INVALID_ACTION(2, "Invalid action"),
     FILE_UPLOAD_ERROR(2, "File upload error"),
+    IMAGE_FOUND_ERROR(2, "Image found error"),
     INVALID_JSON_BODY(2, "Invalid fields in JSON object"),
     INCOMPLETE_UPLOAD(2, "Not load all chunks"),
     MSG_ID_NOT_FOUND(4, "Message id not found"),

src/main/java/ru/soune/nocopy/dto/file/ImageSearchRequest.java

@@ -0,0 +1,10 @@
+package ru.soune.nocopy.dto.file;
+
+import com.fasterxml.jackson.annotation.JsonProperty;
+import lombok.Data;
+
+@Data
+public class ImageSearchRequest {
+    @JsonProperty("file_id")
+    private String fileId;
+}

src/main/java/ru/soune/nocopy/dto/file/YandexSearchResponse.java

@@ -0,0 +1,36 @@
+package ru.soune.nocopy.dto.file;
+
+import com.fasterxml.jackson.annotation.JsonIgnoreProperties;
+import com.fasterxml.jackson.annotation.JsonProperty;
+import lombok.Data;
+import java.util.List;
+
+@Data
+@JsonIgnoreProperties(ignoreUnknown = true)
+public class YandexSearchResponse {
+
+    @JsonProperty("images")
+    private List<ImageResult> images;
+
+    @Data
+    @JsonIgnoreProperties(ignoreUnknown = true)
+    public static class ImageResult {
+        @JsonProperty("url")
+        private String url;
+
+        @JsonProperty("pageUrl")
+        private String pageUrl;
+
+        @JsonProperty("pageTitle")
+        private String pageTitle;
+
+        @JsonProperty("width")
+        private Integer width;
+
+        @JsonProperty("height")
+        private Integer height;
+
+        @JsonProperty("host")
+        private String host;
+    }
+}

src/main/java/ru/soune/nocopy/entity/file/FileType.java

@@ -2,47 +2,21 @@ package ru.soune.nocopy.entity.file;
 
 import lombok.Getter;
 
-import java.util.Arrays;
-import java.util.List;
-
 @Getter
 public enum FileType {
-    PHOTO("photo", Arrays.asList("jpg", "jpeg", "png", "gif", "bmp")),
+    PHOTO("photo"),
-    IMAGE("image", Arrays.asList("jpg", "jpeg", "png", "gif", "bmp", "svg", "tiff", "tif", "ico",
+    IMAGE("image"),
-            "psd", "ai", "eps", "raw", "heic", "heif")),
+    VIDEO("video"),
-    VIDEO("video", Arrays.asList("mp4", "avi", "mov", "wmv", "flv", "mkv", "webm", "m4v", "mpg", "mpeg",
+    AUDIO("audio"),
-            "3gp", "3g2", "f4v", "m2ts", "mts", "vob", "ogv", "divx")),
+    DOCUMENT("document");
-    AUDIO("audio", Arrays.asList("mp3", "wav", "ogg", "aac", "flac", "m4a", "wma", "aiff", "aif", "amr",
-            "opus", "mka", "ac3", "alac")),
-    DOCUMENT("document", Arrays.asList(
-            "pdf", "txt", "rtf",
-            "doc", "docx", "xls", "xlsx", "ppt", "pptx", "pps", "ppsx", "dot", "dotx", "xlt", "xltx", "pot", "potx",
-            "odt", "ods", "odp", "odg", "odf", "odb", "odc", "odi", "odm", "ott", "ots", "otp", "otg", "oth",
-            "sxw", "sxc", "sxi", "sxd", "sxg", "stc", "sti", "stw", "sxm",
-            "pages", "numbers", "key",
-            "csv", "tsv", "xml", "html", "htm", "tex", "md", "markdown",
-            "epub", "mobi", "azw", "azw3", "fb2",
-            "wps", "wpt", "et", "dps", "vsd", "vsdx",
-            "java", "py", "cpp", "c", "h", "js", "css", "php", "sql", "json", "yaml", "yml", "sh", "bat",
-            "one", "note"));
 
-    private final String displayName;
+    private final String code;
-    private final List<String> allowedExtensions;
 
-    FileType(String displayName, List<String> allowedExtensions) {
+    FileType(String code) {
-        this.displayName = displayName;
+        this.code = code;
-        this.allowedExtensions = allowedExtensions;
     }
 
-    public String getDisplayName() {
+    public String getCode() {
-        return displayName;
+        return code;
-    }
-
-    public List<String> getAllowedExtensions() {
-        return allowedExtensions;
-    }
-
-    public boolean supportsExtension(String extension) {
-        return allowedExtensions.contains(extension.toLowerCase());
     }
 }

src/main/java/ru/soune/nocopy/handler/FileUploadHandler.java

@@ -6,7 +6,6 @@ import lombok.extern.slf4j.Slf4j;
 import org.springframework.stereotype.Component;
 import org.springframework.validation.BeanPropertyBindingResult;
 import org.springframework.validation.BindingResult;
-import org.springframework.validation.FieldError;
 import ru.soune.nocopy.dto.*;
 import ru.soune.nocopy.dto.file.*;
 import ru.soune.nocopy.entity.AuthToken;
@@ -18,8 +17,10 @@ import ru.soune.nocopy.repository.AuthTokenRepository;
 import ru.soune.nocopy.repository.FileUploadSessionRepository;
 import ru.soune.nocopy.service.file.FileUploadService;
 
-import java.util.*;
+import java.util.Arrays;
-import java.util.stream.Collectors;
+import java.util.HashMap;
+import java.util.Map;
+import java.util.Optional;
 
 @Slf4j
 @Component
@@ -69,31 +70,18 @@ public class FileUploadHandler implements RequestHandler {
     }
 
     private BaseResponse handleInitUpload(BaseRequest request, FileUploadRequest fileRequest) {
+        try {
             String token = fileRequest.getToken();
 
             Optional<AuthToken> tokenOptional = authTokenRepository.findByToken(token);
 
-            if (tokenOptional.isEmpty()) {
-                return new BaseResponse(request.getMsgId(), MessageCode.INVALID_TOKEN.getCode(),
-                        MessageCode.INVALID_TOKEN.getDescription(), Map.of("token", token));
-            }
-
             AuthToken authToken = tokenOptional.orElseThrow(() -> new NotFoundAuthToken("Token not found"));
 
             BindingResult bindingResult = new BeanPropertyBindingResult(fileRequest, "fileRequest");
             fileUploadRequestValidator.validate(fileRequest, bindingResult);
 
             if (bindingResult.hasErrors()) {
-                Map<String, String> fieldErrors = bindingResult.getFieldErrors()
+                throw new ValidationException(bindingResult, request.getMsgId());
-                        .stream()
-                        .collect(Collectors.toMap(
-                                FieldError::getField,
-                                fieldError -> fieldError.getDefaultMessage() != null
-                                        ? fieldError.getDefaultMessage()
-                                        : "Validation error"));
-
-                return new BaseResponse(request.getMsgId(), MessageCode.INVALID_FIELD.getCode(),
-                        MessageCode.INVALID_FIELD.getDescription(), fieldErrors);
             }
 
             FileUploadSession session = fileUploadService.initUpload(
@@ -113,6 +101,15 @@ public class FileUploadHandler implements RequestHandler {
 
             return new BaseResponse(request.getMsgId(), MessageCode.SUCCESS.getCode(),
                     MessageCode.SUCCESS.getDescription(), fileResponse);
+
+        } catch (NotFoundAuthToken e) {
+            InitFileResponse initFileResponse = InitFileResponse.builder()
+                    .build();
+
+            throw new NotValidFieldException("Invalid or expired token: " + fileRequest.getToken(),
+                    new BaseResponse(request.getMsgId(), MessageCode.INVALID_TOKEN.getCode(),
+                            MessageCode.INVALID_TOKEN.getDescription(), initFileResponse));
+        }
     }
 
     private BaseResponse handleGetProgress(BaseRequest request, FileUploadRequest fileRequest) {

src/main/java/ru/soune/nocopy/handler/ImageFoundRequestHandler.java

@@ -0,0 +1,35 @@
+package ru.soune.nocopy.handler;
+
+import com.fasterxml.jackson.databind.ObjectMapper;
+import lombok.RequiredArgsConstructor;
+import lombok.extern.slf4j.Slf4j;
+import org.springframework.stereotype.Component;
+import ru.soune.nocopy.dto.BaseRequest;
+import ru.soune.nocopy.dto.BaseResponse;
+import ru.soune.nocopy.dto.MessageCode;
+import ru.soune.nocopy.dto.file.ImageSearchRequest;
+import ru.soune.nocopy.dto.file.YandexSearchResponse;
+import ru.soune.nocopy.service.YandexSearchService;
+
+@Slf4j
+@Component
+@RequiredArgsConstructor
+public class ImageFoundRequestHandler implements RequestHandler {
+
+    private final ObjectMapper objectMapper;
+
+    private final YandexSearchService yandexSearchService;
+
+    @Override
+    public BaseResponse handle(BaseRequest request) throws Exception {
+        ImageSearchRequest imageSearchRequest = objectMapper.convertValue(request.getMessageBody(),
+                ImageSearchRequest.class);
+
+        String fileId = imageSearchRequest.getFileId();
+
+        YandexSearchResponse response = yandexSearchService.searchByFileEntity(fileId);
+
+        return new BaseResponse(request.getMsgId(), MessageCode.SUCCESS.getCode(),
+                MessageCode.SUCCESS.getDescription(), response);
+    }
+}

src/main/java/ru/soune/nocopy/handler/validator/FileUploadRequestValidator.java

@@ -7,12 +7,9 @@ import ru.soune.nocopy.dto.file.FileUploadRequest;
 import ru.soune.nocopy.entity.file.FileType;
 
 import java.util.Arrays;
-import java.util.List;
 
 @Component
 public class FileUploadRequestValidator implements Validator {
-    private final List<FileType> supportedFileTypes = Arrays.asList(FileType.values());
-
     @Override
     public boolean supports(Class<?> clazz) {
         return FileUploadRequest.class.isAssignableFrom(clazz);
@@ -24,7 +21,7 @@ public class FileUploadRequestValidator implements Validator {
 
         validateType(request.getFileType(), errors);
         validateFileName(request.getFileName(), errors);
-        validateExtension(request.getExtension(), request.getFileType(), errors);
+        validateExtension(request.getExtension(), errors);
     }
 
     private void validateType(String fileType, Errors errors) {
@@ -34,12 +31,7 @@ public class FileUploadRequestValidator implements Validator {
         }
 
         try {
-            FileType parsedType = FileType.valueOf(fileType.toUpperCase());
+            FileType.valueOf(fileType.toUpperCase());
-
-            if (!supportedFileTypes.contains(parsedType)) {
-                errors.rejectValue("fileType", "fileType.unsupported",
-                        "Unsupported file type. Valid types: " + Arrays.toString(FileType.values()));
-            }
         } catch (IllegalArgumentException e) {
             errors.rejectValue("fileType", "fileType.invalid",
                     "Invalid file type. Valid types: " + Arrays.toString(FileType.values()));
@@ -58,34 +50,10 @@ public class FileUploadRequestValidator implements Validator {
         }
     }
 
-    private void validateExtension(String extension, String fileType,  Errors errors) {
+    private void validateExtension(String extension, Errors errors) {
         if (extension == null || extension.isBlank()) {
-            errors.rejectValue("fileType", "fileType.required", "File type is required");
+            errors.rejectValue("extension", "extension.required", "Extension is required");
             return;
         }
-
-        if (extension.contains(".")) {
-            errors.rejectValue("extension", "extension.required", "Extension contains comma");
-            return;
-        }
-
-        try {
-            FileType parsedType = FileType.valueOf(fileType.toUpperCase());
-
-            if (!parsedType.supportsExtension(extension)) {
-                errors.rejectValue("fileName", "fileType.extension.mismatch",
-                        String.format("File extension '%s' does not match file type '%s'. Allowed extensions for %s: %s",
-                                extension, parsedType.getDisplayName(), parsedType.getDisplayName(),
-                                parsedType.getAllowedExtensions()));
-            }
-
-        } catch (IllegalArgumentException e) {
-            errors.rejectValue("fileType", "fileType.invalid",
-                    "Invalid file type. Valid types: " + Arrays.toString(FileType.values()));
-        }
-    }
-
-    private String getFileExtension(String fileName) {
-        return fileName.substring(fileName.lastIndexOf('.') + 1).toLowerCase();
     }
 }

src/main/java/ru/soune/nocopy/repository/FileEntityRepository.java

@@ -14,6 +14,7 @@ import java.util.Optional;
 public interface FileEntityRepository extends JpaRepository<FileEntity, String> {
 
     List<FileEntity> findByUserId(Long userId);
+    Optional<FileEntity> findByUserIdAndChecksum(Long userId,  String imageHash);
 
     List<FileEntity> findByUserIdAndStatus(Long userId, FileStatus status);
 

src/main/java/ru/soune/nocopy/service/YandexSearchService.java

@@ -0,0 +1,155 @@
+package ru.soune.nocopy.service;
+
+import com.fasterxml.jackson.core.JsonProcessingException;
+import com.fasterxml.jackson.databind.ObjectMapper;
+import com.fasterxml.jackson.databind.DeserializationFeature;
+import jakarta.annotation.PostConstruct;
+import lombok.RequiredArgsConstructor;
+import lombok.extern.slf4j.Slf4j;
+import org.springframework.beans.factory.annotation.Value;
+import org.springframework.stereotype.Service;
+import ru.soune.nocopy.dto.BaseResponse;
+import ru.soune.nocopy.dto.MessageCode;
+import ru.soune.nocopy.dto.file.YandexSearchResponse;
+import ru.soune.nocopy.entity.file.FileEntity;
+import ru.soune.nocopy.exception.FileEntityNotFoundException;
+import ru.soune.nocopy.exception.NotValidFieldException;
+import ru.soune.nocopy.repository.FileEntityRepository;
+
+import java.io.*;
+import java.net.HttpURLConnection;
+import java.net.URL;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.util.Base64;
+import java.util.Map;
+
+@Slf4j
+@Service
+@RequiredArgsConstructor
+public class YandexSearchService {
+
+    private final FileEntityRepository fileEntityRepository;
+    private final ObjectMapper objectMapper;
+
+    @Value("${yandex.api-key}")
+    private String apiKey;
+
+    @Value("${yandex.folder-id}")
+    private String folderId;
+
+    @Value("${yandex.search-url}")
+    private String searchUrl;
+
+    @PostConstruct
+    public void init() {
+        objectMapper.configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, false);
+        objectMapper.configure(DeserializationFeature.ACCEPT_SINGLE_VALUE_AS_ARRAY, true);
+    }
+
+    public YandexSearchResponse searchByFileEntity(String fileId) throws IOException {
+        FileEntity fileEntity = fileEntityRepository.findById(fileId)
+                .orElseThrow(() -> {
+                    throw new NotValidFieldException("File not found", new BaseResponse(20007,
+                            MessageCode.FILE_NOT_FOUND.getCode(), MessageCode.FILE_NOT_FOUND.getDescription(),
+                            Map.of("fileId",fileId)));
+                });
+
+        if (!isImageFile(fileEntity)) {
+            log.error("File not image: {}", fileEntity.getMimeType());
+            throw new NotValidFieldException("File not image", new BaseResponse(20007,
+                    MessageCode.INVALID_FIELD.getCode(), MessageCode.INVALID_FIELD.getDescription(),
+                    Map.of("file_type", fileEntity.getMimeType())));
+        }
+
+        byte[] fileBytes = readFileFromDisk(fileEntity);
+
+        return callYandexApi(fileBytes);
+    }
+
+    private boolean isImageFile(FileEntity fileEntity) {
+        String mimeType = fileEntity.getMimeType();
+        return mimeType != null && mimeType.startsWith("image");
+    }
+
+    private byte[] readFileFromDisk(FileEntity fileEntity) throws IOException {
+        Path filePath = Path.of(fileEntity.getFilePath());
+
+        if (!Files.exists(filePath)) {
+            throw new IOException("File not found: " + fileEntity.getFilePath());
+        }
+
+        if (!Files.isReadable(filePath)) {
+            throw new IOException("Cannot read file: " + fileEntity.getFilePath());
+        }
+
+        return Files.readAllBytes(filePath);
+    }
+
+    private YandexSearchResponse callYandexApi(byte[] imageBytes) throws IOException {
+        String imageBase64 = Base64.getEncoder().encodeToString(imageBytes);
+        String jsonRequest = String.format("{\"folderId\":\"%s\",\"data\":\"%s\",\"page\":0,\"showSimilarImages\":true}",
+                folderId, imageBase64);
+
+        URL url = new URL(searchUrl);
+        HttpURLConnection connection = (HttpURLConnection) url.openConnection();
+
+        try {
+            connection.setRequestMethod("POST");
+            connection.setRequestProperty("Authorization", "Api-Key " + apiKey);
+            connection.setRequestProperty("Content-Type", "application/json");
+            connection.setRequestProperty("Accept", "application/json");
+//            connection.setRequestProperty("User-Agent", "Mozilla/5.0");
+            connection.setConnectTimeout(30000);
+            connection.setReadTimeout(30000);
+            connection.setDoOutput(true);
+
+            try (OutputStream os = connection.getOutputStream()) {
+                byte[] input = jsonRequest.getBytes("utf-8");
+                os.write(input, 0, input.length);
+                os.flush();
+            }
+
+            int responseCode = connection.getResponseCode();
+
+            String responseBody;
+            if (responseCode == HttpURLConnection.HTTP_OK) {
+                try (BufferedReader br = new BufferedReader(
+                        new InputStreamReader(connection.getInputStream(), "utf-8"))) {
+                    responseBody = readAll(br);
+                }
+            } else {
+                try (BufferedReader br = new BufferedReader(
+                        new InputStreamReader(connection.getErrorStream(), "utf-8"))) {
+                    responseBody = readAll(br);
+                }
+                throw new IOException("Error Yandex API: " + responseCode + " - " + responseBody);
+            }
+
+            return parseJsonResponse(responseBody);
+
+        } finally {
+            connection.disconnect();
+        }
+    }
+
+    private String readAll(BufferedReader reader) throws IOException {
+        StringBuilder sb = new StringBuilder();
+        String line;
+        while ((line = reader.readLine()) != null) {
+            sb.append(line);
+        }
+        return sb.toString();
+    }
+
+    private YandexSearchResponse parseJsonResponse(String json) {
+        YandexSearchResponse response = null;
+        try {
+            response = objectMapper.readValue(json, YandexSearchResponse.class);
+        } catch (JsonProcessingException e) {
+                throw new RuntimeException(e);
+        }
+
+        return response;
+    }
+}

src/main/resources/application.yaml

@@ -38,6 +38,11 @@ security:
     allowed-headers: "*"
     allow-credentials: true
 
+yandex:
+    api-key: ${YANDEX_SEARCH_API_KEY}
+    folder-id: ${YANDEX_SEARCH_FOLDER_ID}
+    search-url: "https://searchapi.api.cloud.yandex.net/v2/image/search_by_image"
+
 server:
   port: ${SERVER_PORT:8080}