NCBACK-26 add docker container with storage

Dockerfile

@@ -12,6 +12,8 @@ WORKDIR /app
 
 COPY --from=build /app/build/libs/*.jar app.jar
 
+RUN mkdir -p /data/uploads && chmod 755 /data/uploads
+
 EXPOSE 8080
 
 CMD ["java", "-jar", "app.jar"]

build.gradle

@@ -31,10 +31,6 @@ dependencies {
 	implementation 'org.mapstruct:mapstruct:1.5.5.Final'
 	implementation 'commons-validator:commons-validator:1.7'
 
-	implementation 'com.fasterxml.jackson.dataformat:jackson-dataformat-xml:2.17.0'
-	implementation 'tools.jackson.core:jackson-core:3.0.3'
-	implementation 'jakarta.xml.bind:jakarta.xml.bind-api:4.0.0'
-
 	annotationProcessor 'org.mapstruct:mapstruct-processor:1.5.5.Final'
 	annotationProcessor 'org.projectlombok:lombok'
 

docker-compose.yaml

@@ -1,6 +1,16 @@
 version: '3.9'
 
 services:
+  storage:
+    image: alpine:latest
+    container_name: file-storage
+    networks:
+      - app-network
+    volumes:
+      - uploads_data:/storage:rw
+    command: tail -f /dev/null
+    restart: unless-stopped
+
   db:
     image: postgres:17
     restart: always
@@ -30,8 +40,7 @@ services:
       POSTGRES_PASSWORD: postgres
       POSTGRES_PORT: 5432
       POSTGRES_HOST: db
-      YANDEX_SEARCH_API_KEY: AQVNyaVaUmgUb1GMCtf5zSEqFxy0woXrcMOOB43q
+      STORAGE_SERVICE_URL: http://storage:8081
-      YANDEX_SEARCH_FOLDER_ID: b1gokpdbm6qfpsou8pcd
     depends_on:
       - db
     ports:
@@ -43,7 +52,7 @@ services:
           - backend
           - api
     volumes:
-      - ./uploads:/data/uploads:rw
+      - uploads_data:/data/uploads:rw
 
   grafana:
     image: grafana/grafana:10.3.1
@@ -148,7 +157,7 @@ volumes:
   loki_chunks:
   loki_index:
   loki_rules:
-#  uploads_volume:
+  uploads_data:
 
 networks:
   app-network:

infrastructure/jenkins/deploy.groovy

@@ -114,6 +114,7 @@ EOF
                                   --network app-network \\
                                   --network-alias app \\
                                   -p 80:8080 \\
+                                  -v /opt/uploads:/data/uploads:rw \\
                                   -e POSTGRES_DB=no_copy_ \\
                                   -e POSTGRES_USER=postgres \\
                                   -e POSTGRES_PASSWORD=postgres \\

src/main/java/ru/soune/nocopy/configuration/HandlerConfig.java

@@ -16,8 +16,7 @@ public class HandlerConfig {
             LoginRequestHandler login,
             FileUploadHandler upload,
             FileEntityHandler file,
-            LogoutRequestHandler logoutHandler,
+            LogoutRequestHandler logoutHandler
-            ImageFoundRequestHandler imageFoundRequestHandler
     ) {
         Map<Integer, RequestHandler> map = new HashMap<>();
         map.put(20001, login);
@@ -25,7 +24,6 @@ public class HandlerConfig {
         map.put(20004, upload);
         map.put(20005, file);
         map.put(20006, logoutHandler);
-        map.put(20007, imageFoundRequestHandler);
 
         return map;
     }

src/main/java/ru/soune/nocopy/configuration/JacksonConfig.java

@@ -35,8 +35,6 @@ public class JacksonConfig {
 
         mapper.registerModule(javaTimeModule);
         mapper.disable(SerializationFeature.WRITE_DATES_AS_TIMESTAMPS);
-        mapper.configure(com.fasterxml.jackson.databind.DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, false);
-        mapper.enable(SerializationFeature.INDENT_OUTPUT);
 
         return mapper;
     }

src/main/java/ru/soune/nocopy/configuration/file/YandexConfig.java

@@ -1,14 +0,0 @@
-package ru.soune.nocopy.configuration.file;
-
-import lombok.Data;
-import org.springframework.boot.context.properties.ConfigurationProperties;
-import org.springframework.context.annotation.Configuration;
-
-@Data
-@Configuration
-@ConfigurationProperties(prefix = "yandex")
-public class YandexConfig {
-    private String apiKey;
-    private String folderId;
-    private String searchUrl;
-}

src/main/java/ru/soune/nocopy/controller/ApiController.java

@@ -387,11 +387,11 @@ public class ApiController {
             return false;
         }
 
-        return mimeType.startsWith("image/") ||
+        return mimeType.startsWith("image") ||
-                mimeType.startsWith("text/") ||
+                mimeType.startsWith("text") ||
-                mimeType.equals("application/pdf") ||
+                mimeType.equals("pdf") ||
-                mimeType.startsWith("video/") ||
+                mimeType.startsWith("video") ||
-                mimeType.startsWith("audio/");
+                mimeType.startsWith("audio");
     }
 
     private Long getUserIdFromToken(String tokenHeader) {

src/main/java/ru/soune/nocopy/dto/MessageCode.java

@@ -8,7 +8,6 @@ public enum MessageCode {
     INVALID_TOKEN(2, "Invalid token"),
     INVALID_ACTION(2, "Invalid action"),
     FILE_UPLOAD_ERROR(2, "File upload error"),
-    IMAGE_FOUND_ERROR(2, "Image found error"),
     INVALID_JSON_BODY(2, "Invalid fields in JSON object"),
     INCOMPLETE_UPLOAD(2, "Not load all chunks"),
     MSG_ID_NOT_FOUND(4, "Message id not found"),

src/main/java/ru/soune/nocopy/dto/file/ImageSearchRequest.java

@@ -1,10 +0,0 @@
-package ru.soune.nocopy.dto.file;
-
-import com.fasterxml.jackson.annotation.JsonProperty;
-import lombok.Data;
-
-@Data
-public class ImageSearchRequest {
-    @JsonProperty("file_id")
-    private String fileId;
-}

src/main/java/ru/soune/nocopy/dto/file/YandexSearchResponse.java

@@ -1,36 +0,0 @@
-package ru.soune.nocopy.dto.file;
-
-import com.fasterxml.jackson.annotation.JsonIgnoreProperties;
-import com.fasterxml.jackson.annotation.JsonProperty;
-import lombok.Data;
-import java.util.List;
-
-@Data
-@JsonIgnoreProperties(ignoreUnknown = true)
-public class YandexSearchResponse {
-
-    @JsonProperty("images")
-    private List<ImageResult> images;
-
-    @Data
-    @JsonIgnoreProperties(ignoreUnknown = true)
-    public static class ImageResult {
-        @JsonProperty("url")
-        private String url;
-
-        @JsonProperty("pageUrl")
-        private String pageUrl;
-
-        @JsonProperty("pageTitle")
-        private String pageTitle;
-
-        @JsonProperty("width")
-        private Integer width;
-
-        @JsonProperty("height")
-        private Integer height;
-
-        @JsonProperty("host")
-        private String host;
-    }
-}

src/main/java/ru/soune/nocopy/entity/file/FileType.java

@@ -2,21 +2,47 @@ package ru.soune.nocopy.entity.file;
 
 import lombok.Getter;
 
+import java.util.Arrays;
+import java.util.List;
+
 @Getter
 public enum FileType {
-    PHOTO("photo"),
+    PHOTO("photo", Arrays.asList("jpg", "jpeg", "png", "gif", "bmp")),
-    IMAGE("image"),
+    IMAGE("image", Arrays.asList("jpg", "jpeg", "png", "gif", "bmp", "svg", "tiff", "tif", "ico",
-    VIDEO("video"),
+            "psd", "ai", "eps", "raw", "heic", "heif")),
-    AUDIO("audio"),
+    VIDEO("video", Arrays.asList("mp4", "avi", "mov", "wmv", "flv", "mkv", "webm", "m4v", "mpg", "mpeg",
-    DOCUMENT("document");
+            "3gp", "3g2", "f4v", "m2ts", "mts", "vob", "ogv", "divx")),
+    AUDIO("audio", Arrays.asList("mp3", "wav", "ogg", "aac", "flac", "m4a", "wma", "aiff", "aif", "amr",
+            "opus", "mka", "ac3", "alac")),
+    DOCUMENT("document", Arrays.asList(
+            "pdf", "txt", "rtf",
+            "doc", "docx", "xls", "xlsx", "ppt", "pptx", "pps", "ppsx", "dot", "dotx", "xlt", "xltx", "pot", "potx",
+            "odt", "ods", "odp", "odg", "odf", "odb", "odc", "odi", "odm", "ott", "ots", "otp", "otg", "oth",
+            "sxw", "sxc", "sxi", "sxd", "sxg", "stc", "sti", "stw", "sxm",
+            "pages", "numbers", "key",
+            "csv", "tsv", "xml", "html", "htm", "tex", "md", "markdown",
+            "epub", "mobi", "azw", "azw3", "fb2",
+            "wps", "wpt", "et", "dps", "vsd", "vsdx",
+            "java", "py", "cpp", "c", "h", "js", "css", "php", "sql", "json", "yaml", "yml", "sh", "bat",
+            "one", "note"));
 
-    private final String code;
+    private final String displayName;
+    private final List<String> allowedExtensions;
 
-    FileType(String code) {
+    FileType(String displayName, List<String> allowedExtensions) {
-        this.code = code;
+        this.displayName = displayName;
+        this.allowedExtensions = allowedExtensions;
     }
 
-    public String getCode() {
+    public String getDisplayName() {
-        return code;
+        return displayName;
+    }
+
+    public List<String> getAllowedExtensions() {
+        return allowedExtensions;
+    }
+
+    public boolean supportsExtension(String extension) {
+        return allowedExtensions.contains(extension.toLowerCase());
     }
 }

src/main/java/ru/soune/nocopy/handler/FileUploadHandler.java

@@ -6,6 +6,7 @@ import lombok.extern.slf4j.Slf4j;
 import org.springframework.stereotype.Component;
 import org.springframework.validation.BeanPropertyBindingResult;
 import org.springframework.validation.BindingResult;
+import org.springframework.validation.FieldError;
 import ru.soune.nocopy.dto.*;
 import ru.soune.nocopy.dto.file.*;
 import ru.soune.nocopy.entity.AuthToken;
@@ -17,10 +18,8 @@ import ru.soune.nocopy.repository.AuthTokenRepository;
 import ru.soune.nocopy.repository.FileUploadSessionRepository;
 import ru.soune.nocopy.service.file.FileUploadService;
 
-import java.util.Arrays;
+import java.util.*;
-import java.util.HashMap;
+import java.util.stream.Collectors;
-import java.util.Map;
-import java.util.Optional;
 
 @Slf4j
 @Component
@@ -70,18 +69,31 @@ public class FileUploadHandler implements RequestHandler {
     }
 
     private BaseResponse handleInitUpload(BaseRequest request, FileUploadRequest fileRequest) {
-        try {
             String token = fileRequest.getToken();
 
             Optional<AuthToken> tokenOptional = authTokenRepository.findByToken(token);
 
+            if (tokenOptional.isEmpty()) {
+                return new BaseResponse(request.getMsgId(), MessageCode.INVALID_TOKEN.getCode(),
+                        MessageCode.INVALID_TOKEN.getDescription(), Map.of("token", token));
+            }
+
             AuthToken authToken = tokenOptional.orElseThrow(() -> new NotFoundAuthToken("Token not found"));
 
             BindingResult bindingResult = new BeanPropertyBindingResult(fileRequest, "fileRequest");
             fileUploadRequestValidator.validate(fileRequest, bindingResult);
 
             if (bindingResult.hasErrors()) {
-                throw new ValidationException(bindingResult, request.getMsgId());
+                Map<String, String> fieldErrors = bindingResult.getFieldErrors()
+                        .stream()
+                        .collect(Collectors.toMap(
+                                FieldError::getField,
+                                fieldError -> fieldError.getDefaultMessage() != null
+                                        ? fieldError.getDefaultMessage()
+                                        : "Validation error"));
+
+                return new BaseResponse(request.getMsgId(), MessageCode.INVALID_FIELD.getCode(),
+                        MessageCode.INVALID_FIELD.getDescription(), fieldErrors);
             }
 
             FileUploadSession session = fileUploadService.initUpload(
@@ -101,15 +113,6 @@ public class FileUploadHandler implements RequestHandler {
 
             return new BaseResponse(request.getMsgId(), MessageCode.SUCCESS.getCode(),
                     MessageCode.SUCCESS.getDescription(), fileResponse);
-
-        } catch (NotFoundAuthToken e) {
-            InitFileResponse initFileResponse = InitFileResponse.builder()
-                    .build();
-
-            throw new NotValidFieldException("Invalid or expired token: " + fileRequest.getToken(),
-                    new BaseResponse(request.getMsgId(), MessageCode.INVALID_TOKEN.getCode(),
-                            MessageCode.INVALID_TOKEN.getDescription(), initFileResponse));
-        }
     }
 
     private BaseResponse handleGetProgress(BaseRequest request, FileUploadRequest fileRequest) {

src/main/java/ru/soune/nocopy/handler/ImageFoundRequestHandler.java

@@ -1,35 +0,0 @@
-package ru.soune.nocopy.handler;
-
-import com.fasterxml.jackson.databind.ObjectMapper;
-import lombok.RequiredArgsConstructor;
-import lombok.extern.slf4j.Slf4j;
-import org.springframework.stereotype.Component;
-import ru.soune.nocopy.dto.BaseRequest;
-import ru.soune.nocopy.dto.BaseResponse;
-import ru.soune.nocopy.dto.MessageCode;
-import ru.soune.nocopy.dto.file.ImageSearchRequest;
-import ru.soune.nocopy.dto.file.YandexSearchResponse;
-import ru.soune.nocopy.service.YandexSearchService;
-
-@Slf4j
-@Component
-@RequiredArgsConstructor
-public class ImageFoundRequestHandler implements RequestHandler {
-
-    private final ObjectMapper objectMapper;
-
-    private final YandexSearchService yandexSearchService;
-
-    @Override
-    public BaseResponse handle(BaseRequest request) throws Exception {
-        ImageSearchRequest imageSearchRequest = objectMapper.convertValue(request.getMessageBody(),
-                ImageSearchRequest.class);
-
-        String fileId = imageSearchRequest.getFileId();
-
-        YandexSearchResponse response = yandexSearchService.searchByFileEntity(fileId);
-
-        return new BaseResponse(request.getMsgId(), MessageCode.SUCCESS.getCode(),
-                MessageCode.SUCCESS.getDescription(), response);
-    }
-}

src/main/java/ru/soune/nocopy/handler/validator/FileUploadRequestValidator.java

@@ -7,9 +7,12 @@ import ru.soune.nocopy.dto.file.FileUploadRequest;
 import ru.soune.nocopy.entity.file.FileType;
 
 import java.util.Arrays;
+import java.util.List;
 
 @Component
 public class FileUploadRequestValidator implements Validator {
+    private final List<FileType> supportedFileTypes = Arrays.asList(FileType.values());
+
     @Override
     public boolean supports(Class<?> clazz) {
         return FileUploadRequest.class.isAssignableFrom(clazz);
@@ -21,7 +24,7 @@ public class FileUploadRequestValidator implements Validator {
 
         validateType(request.getFileType(), errors);
         validateFileName(request.getFileName(), errors);
-        validateExtension(request.getExtension(), errors);
+        validateExtension(request.getExtension(), request.getFileType(), errors);
     }
 
     private void validateType(String fileType, Errors errors) {
@@ -31,7 +34,12 @@ public class FileUploadRequestValidator implements Validator {
         }
 
         try {
-            FileType.valueOf(fileType.toUpperCase());
+            FileType parsedType = FileType.valueOf(fileType.toUpperCase());
+
+            if (!supportedFileTypes.contains(parsedType)) {
+                errors.rejectValue("fileType", "fileType.unsupported",
+                        "Unsupported file type. Valid types: " + Arrays.toString(FileType.values()));
+            }
         } catch (IllegalArgumentException e) {
             errors.rejectValue("fileType", "fileType.invalid",
                     "Invalid file type. Valid types: " + Arrays.toString(FileType.values()));
@@ -50,10 +58,34 @@ public class FileUploadRequestValidator implements Validator {
         }
     }
 
-    private void validateExtension(String extension, Errors errors) {
+    private void validateExtension(String extension, String fileType,  Errors errors) {
         if (extension == null || extension.isBlank()) {
-            errors.rejectValue("extension", "extension.required", "Extension is required");
+            errors.rejectValue("fileType", "fileType.required", "File type is required");
             return;
         }
+
+        if (extension.contains(".")) {
+            errors.rejectValue("extension", "extension.required", "Extension contains comma");
+            return;
+        }
+
+        try {
+            FileType parsedType = FileType.valueOf(fileType.toUpperCase());
+
+            if (!parsedType.supportsExtension(extension)) {
+                errors.rejectValue("fileName", "fileType.extension.mismatch",
+                        String.format("File extension '%s' does not match file type '%s'. Allowed extensions for %s: %s",
+                                extension, parsedType.getDisplayName(), parsedType.getDisplayName(),
+                                parsedType.getAllowedExtensions()));
+            }
+
+        } catch (IllegalArgumentException e) {
+            errors.rejectValue("fileType", "fileType.invalid",
+                    "Invalid file type. Valid types: " + Arrays.toString(FileType.values()));
+        }
+    }
+
+    private String getFileExtension(String fileName) {
+        return fileName.substring(fileName.lastIndexOf('.') + 1).toLowerCase();
     }
 }

src/main/java/ru/soune/nocopy/repository/FileEntityRepository.java

@@ -14,7 +14,6 @@ import java.util.Optional;
 public interface FileEntityRepository extends JpaRepository<FileEntity, String> {
 
     List<FileEntity> findByUserId(Long userId);
-    Optional<FileEntity> findByUserIdAndChecksum(Long userId,  String imageHash);
 
     List<FileEntity> findByUserIdAndStatus(Long userId, FileStatus status);
 

src/main/java/ru/soune/nocopy/service/YandexSearchService.java

@@ -1,155 +0,0 @@
-package ru.soune.nocopy.service;
-
-import com.fasterxml.jackson.core.JsonProcessingException;
-import com.fasterxml.jackson.databind.ObjectMapper;
-import com.fasterxml.jackson.databind.DeserializationFeature;
-import jakarta.annotation.PostConstruct;
-import lombok.RequiredArgsConstructor;
-import lombok.extern.slf4j.Slf4j;
-import org.springframework.beans.factory.annotation.Value;
-import org.springframework.stereotype.Service;
-import ru.soune.nocopy.dto.BaseResponse;
-import ru.soune.nocopy.dto.MessageCode;
-import ru.soune.nocopy.dto.file.YandexSearchResponse;
-import ru.soune.nocopy.entity.file.FileEntity;
-import ru.soune.nocopy.exception.FileEntityNotFoundException;
-import ru.soune.nocopy.exception.NotValidFieldException;
-import ru.soune.nocopy.repository.FileEntityRepository;
-
-import java.io.*;
-import java.net.HttpURLConnection;
-import java.net.URL;
-import java.nio.file.Files;
-import java.nio.file.Path;
-import java.util.Base64;
-import java.util.Map;
-
-@Slf4j
-@Service
-@RequiredArgsConstructor
-public class YandexSearchService {
-
-    private final FileEntityRepository fileEntityRepository;
-    private final ObjectMapper objectMapper;
-
-    @Value("${yandex.api-key}")
-    private String apiKey;
-
-    @Value("${yandex.folder-id}")
-    private String folderId;
-
-    @Value("${yandex.search-url}")
-    private String searchUrl;
-
-    @PostConstruct
-    public void init() {
-        objectMapper.configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, false);
-        objectMapper.configure(DeserializationFeature.ACCEPT_SINGLE_VALUE_AS_ARRAY, true);
-    }
-
-    public YandexSearchResponse searchByFileEntity(String fileId) throws IOException {
-        FileEntity fileEntity = fileEntityRepository.findById(fileId)
-                .orElseThrow(() -> {
-                    throw new NotValidFieldException("File not found", new BaseResponse(20007,
-                            MessageCode.FILE_NOT_FOUND.getCode(), MessageCode.FILE_NOT_FOUND.getDescription(),
-                            Map.of("fileId",fileId)));
-                });
-
-        if (!isImageFile(fileEntity)) {
-            log.error("File not image: {}", fileEntity.getMimeType());
-            throw new NotValidFieldException("File not image", new BaseResponse(20007,
-                    MessageCode.INVALID_FIELD.getCode(), MessageCode.INVALID_FIELD.getDescription(),
-                    Map.of("file_type", fileEntity.getMimeType())));
-        }
-
-        byte[] fileBytes = readFileFromDisk(fileEntity);
-
-        return callYandexApi(fileBytes);
-    }
-
-    private boolean isImageFile(FileEntity fileEntity) {
-        String mimeType = fileEntity.getMimeType();
-        return mimeType != null && mimeType.startsWith("image");
-    }
-
-    private byte[] readFileFromDisk(FileEntity fileEntity) throws IOException {
-        Path filePath = Path.of(fileEntity.getFilePath());
-
-        if (!Files.exists(filePath)) {
-            throw new IOException("File not found: " + fileEntity.getFilePath());
-        }
-
-        if (!Files.isReadable(filePath)) {
-            throw new IOException("Cannot read file: " + fileEntity.getFilePath());
-        }
-
-        return Files.readAllBytes(filePath);
-    }
-
-    private YandexSearchResponse callYandexApi(byte[] imageBytes) throws IOException {
-        String imageBase64 = Base64.getEncoder().encodeToString(imageBytes);
-        String jsonRequest = String.format("{\"folderId\":\"%s\",\"data\":\"%s\",\"page\":0,\"showSimilarImages\":true}",
-                folderId, imageBase64);
-
-        URL url = new URL(searchUrl);
-        HttpURLConnection connection = (HttpURLConnection) url.openConnection();
-
-        try {
-            connection.setRequestMethod("POST");
-            connection.setRequestProperty("Authorization", "Api-Key " + apiKey);
-            connection.setRequestProperty("Content-Type", "application/json");
-            connection.setRequestProperty("Accept", "application/json");
-//            connection.setRequestProperty("User-Agent", "Mozilla/5.0");
-            connection.setConnectTimeout(30000);
-            connection.setReadTimeout(30000);
-            connection.setDoOutput(true);
-
-            try (OutputStream os = connection.getOutputStream()) {
-                byte[] input = jsonRequest.getBytes("utf-8");
-                os.write(input, 0, input.length);
-                os.flush();
-            }
-
-            int responseCode = connection.getResponseCode();
-
-            String responseBody;
-            if (responseCode == HttpURLConnection.HTTP_OK) {
-                try (BufferedReader br = new BufferedReader(
-                        new InputStreamReader(connection.getInputStream(), "utf-8"))) {
-                    responseBody = readAll(br);
-                }
-            } else {
-                try (BufferedReader br = new BufferedReader(
-                        new InputStreamReader(connection.getErrorStream(), "utf-8"))) {
-                    responseBody = readAll(br);
-                }
-                throw new IOException("Error Yandex API: " + responseCode + " - " + responseBody);
-            }
-
-            return parseJsonResponse(responseBody);
-
-        } finally {
-            connection.disconnect();
-        }
-    }
-
-    private String readAll(BufferedReader reader) throws IOException {
-        StringBuilder sb = new StringBuilder();
-        String line;
-        while ((line = reader.readLine()) != null) {
-            sb.append(line);
-        }
-        return sb.toString();
-    }
-
-    private YandexSearchResponse parseJsonResponse(String json) {
-        YandexSearchResponse response = null;
-        try {
-            response = objectMapper.readValue(json, YandexSearchResponse.class);
-        } catch (JsonProcessingException e) {
-                throw new RuntimeException(e);
-        }
-
-        return response;
-    }
-}

src/main/resources/application.yaml

@@ -38,11 +38,6 @@ security:
     allowed-headers: "*"
     allow-credentials: true
 
-yandex:
-    api-key: ${YANDEX_SEARCH_API_KEY}
-    folder-id: ${YANDEX_SEARCH_FOLDER_ID}
-    search-url: "https://searchapi.api.cloud.yandex.net/v2/image/search_by_image"
-
 server:
   port: ${SERVER_PORT:8080}