NCBACK-3 Add found by yandex image #10

Merged
backdev merged 4 commits from NCBACK-3 into dev 2026-01-10 01:03:33 +08:00
21 changed files with 745 additions and 432 deletions
Showing only changes of commit edd508e5d8 - Show all commits
+2
View File
@@ -12,6 +12,8 @@ WORKDIR /app
COPY --from=build /app/build/libs/*.jar app.jar COPY --from=build /app/build/libs/*.jar app.jar
RUN mkdir -p /data/uploads && chmod 755 /data/uploads
EXPOSE 8080 EXPOSE 8080
CMD ["java", "-jar", "app.jar"] CMD ["java", "-jar", "app.jar"]
+87 -4
View File
@@ -1,13 +1,39 @@
version: '3.9' version: '3.9'
services: services:
storage:
image: alpine:latest
container_name: file-storage
deploy:
resources:
limits:
memory: 128M
reservations:
memory: 64M
networks:
- app-network
volumes:
- uploads_data:/storage:rw
command: tail -f /dev/null
restart: unless-stopped
db: db:
image: postgres:17 image: postgres:17
restart: always restart: always
deploy:
resources:
limits:
cpus: '2.0'
memory: 2G
reservations:
cpus: '1.0'
memory: 1G
environment: environment:
POSTGRES_DB: no_copy_ POSTGRES_DB: no_copy_
POSTGRES_USER: postgres POSTGRES_USER: postgres
POSTGRES_PASSWORD: postgres POSTGRES_PASSWORD: postgres
POSTGRES_SHARED_BUFFERS: 512MB
POSTGRES_EFFECTIVE_CACHE_SIZE: 1536MB
ports: ports:
- "54320:5432" - "54320:5432"
volumes: volumes:
@@ -21,10 +47,19 @@ services:
app: app:
build: . build: .
container_name: app-backend container_name: app-backend
deploy:
resources:
limits:
cpus: '1.5'
memory: 1G
reservations:
cpus: '0.5'
memory: 512M
environment: environment:
FILE_STORAGE_PATH: /data/uploads FILE_STORAGE_PATH: /data/uploads
MAX_FILE_SIZE: 1073741824 MAX_FILE_SIZE: 10737418240
FILE_CHUNK_SIZE: 5242880 # FILE_CHUNK_SIZE: 1048576
FILE_CHUNK_SIZE: 1000000
POSTGRES_DB: no_copy_ POSTGRES_DB: no_copy_
POSTGRES_USER: postgres POSTGRES_USER: postgres
POSTGRES_PASSWORD: postgres POSTGRES_PASSWORD: postgres
@@ -32,6 +67,7 @@ services:
POSTGRES_HOST: db POSTGRES_HOST: db
YANDEX_SEARCH_API_KEY: AQVNyaVaUmgUb1GMCtf5zSEqFxy0woXrcMOOB43q YANDEX_SEARCH_API_KEY: AQVNyaVaUmgUb1GMCtf5zSEqFxy0woXrcMOOB43q
YANDEX_SEARCH_FOLDER_ID: b1gokpdbm6qfpsou8pcd YANDEX_SEARCH_FOLDER_ID: b1gokpdbm6qfpsou8pcd
STORAGE_SERVICE_URL: http://storage:8081
depends_on: depends_on:
- db - db
ports: ports:
@@ -43,11 +79,18 @@ services:
- backend - backend
- api - api
volumes: volumes:
- ./uploads:/data/uploads:rw - uploads_data:/data/uploads:rw
grafana: grafana:
image: grafana/grafana:10.3.1 image: grafana/grafana:10.3.1
container_name: grafana container_name: grafana
deploy:
resources:
limits:
cpus: '0.5'
memory: 512M
reservations:
memory: 256M
ports: ports:
- "3000:3000" - "3000:3000"
depends_on: depends_on:
@@ -64,6 +107,8 @@ services:
GF_SECURITY_ADMIN_USER: admin GF_SECURITY_ADMIN_USER: admin
GF_SECURITY_ADMIN_PASSWORD: admin GF_SECURITY_ADMIN_PASSWORD: admin
GF_METRICS_ENABLED: "true" GF_METRICS_ENABLED: "true"
GF_DATABASE_MAX_IDLE_CONN: "2"
GF_DATABASE_MAX_OPEN_CONN: "10"
volumes: volumes:
- ./infrastructure/grafana/provisioning:/etc/grafana/provisioning - ./infrastructure/grafana/provisioning:/etc/grafana/provisioning
- ./infrastructure/grafana/dashboards:/var/lib/grafana/dashboards - ./infrastructure/grafana/dashboards:/var/lib/grafana/dashboards
@@ -78,10 +123,23 @@ services:
prometheus: prometheus:
image: prom/prometheus:latest image: prom/prometheus:latest
container_name: prometheus container_name: prometheus
deploy:
resources:
limits:
cpus: '0.5'
memory: 1G
reservations:
memory: 512M
ports: ports:
- "9090:9090" - "9090:9090"
volumes: volumes:
- ./infrastructure/prometheus/prometheus.yml:/etc/prometheus/prometheus.yml - ./infrastructure/prometheus/prometheus.yml:/etc/prometheus/prometheus.yml
- prometheus_data:/prometheus
command:
- '--config.file=/etc/prometheus/prometheus.yml'
- '--storage.tsdb.path=/prometheus'
- '--storage.tsdb.retention.time=15d' # Удерживать 15 дней
- '--web.enable-lifecycle'
healthcheck: healthcheck:
test: [ "CMD", "wget", "--spider", "http://localhost:9090/-/healthy" ] test: [ "CMD", "wget", "--spider", "http://localhost:9090/-/healthy" ]
interval: 10s interval: 10s
@@ -93,10 +151,20 @@ services:
loki: loki:
image: grafana/loki:2.9.2 image: grafana/loki:2.9.2
container_name: loki container_name: loki
deploy:
resources:
limits:
cpus: '1.0'
memory: 2G
reservations:
memory: 1G
ports: ports:
- "3100:3100" - "3100:3100"
volumes: volumes:
- ./infrastructure/loki/loki-config.yaml:/etc/loki/loki-config.yaml:ro - ./infrastructure/loki/loki-config.yaml:/etc/loki/loki-config.yaml:ro
- loki_data:/loki
command:
- -config.file=/etc/loki/loki-config.yaml
healthcheck: healthcheck:
test: [ "CMD", "wget", "--spider", "-q", "http://localhost:3100/ready" ] test: [ "CMD", "wget", "--spider", "-q", "http://localhost:3100/ready" ]
interval: 10s interval: 10s
@@ -108,6 +176,13 @@ services:
tempo: tempo:
image: grafana/tempo:2.4.1 image: grafana/tempo:2.4.1
container_name: tempo container_name: tempo
deploy:
resources:
limits:
cpus: '0.5'
memory: 512M
reservations:
memory: 256M
command: [ "-config.file=/etc/tempo/tempo.yaml" ] command: [ "-config.file=/etc/tempo/tempo.yaml" ]
volumes: volumes:
- ./infrastructure/tempo/tempo.yaml:/etc/tempo/tempo.yaml - ./infrastructure/tempo/tempo.yaml:/etc/tempo/tempo.yaml
@@ -120,6 +195,13 @@ services:
alloy: alloy:
image: grafana/alloy:latest image: grafana/alloy:latest
container_name: alloy container_name: alloy
deploy:
resources:
limits:
cpus: '0.5'
memory: 256M
reservations:
memory: 128M
user: root user: root
ports: ports:
- "9080:9080" - "9080:9080"
@@ -148,7 +230,8 @@ volumes:
loki_chunks: loki_chunks:
loki_index: loki_index:
loki_rules: loki_rules:
# uploads_volume: uploads_data:
prometheus_data:
networks: networks:
app-network: app-network:
+29 -85
View File
@@ -7,7 +7,6 @@ pipeline {
defaultValue: 'dev', defaultValue: 'dev',
description: 'Ветка для деплоя' description: 'Ветка для деплоя'
) )
string( string(
name: 'SERVER', name: 'SERVER',
defaultValue: '92.242.61.23', defaultValue: '92.242.61.23',
@@ -45,75 +44,38 @@ pipeline {
echo "Deploying branch: ${params.BRANCH}" echo "Deploying branch: ${params.BRANCH}"
echo "Copying files to server..." echo "Copying files to server..."
sshpass -p '$SSH_PASS' ssh $SSH_USER@$SERVER "mkdir -p /opt/deployments/${params.BRANCH}" sshpass -p '$SSH_PASS' ssh -o StrictHostKeyChecking=no $SSH_USER@$SERVER "mkdir -p /opt/deployments/${params.BRANCH}"
sshpass -p '$SSH_PASS' scp -r ./* $SSH_USER@$SERVER:/opt/deployments/${params.BRANCH}/ sshpass -p '$SSH_PASS' scp -r -o StrictHostKeyChecking=no ./* $SSH_USER@$SERVER:/opt/deployments/${params.BRANCH}/
sshpass -p '$SSH_PASS' ssh $SSH_USER@$SERVER " sshpass -p '$SSH_PASS' ssh -o StrictHostKeyChecking=no $SSH_USER@$SERVER "
cd /opt/deployments/${params.BRANCH} cd /opt/deployments/${params.BRANCH}
echo 'Step 1: Checking current state...' echo '1. Остановка старого приложения...'
docker stop app-backend 2>/dev/null || echo 'Контейнер не найден'
docker rm app-backend 2>/dev/null || echo 'Контейнер не найден'
# Смотрим что запущено echo '2. Удаление старых образов...'
docker ps --filter 'name=postgres|app-backend' --format 'table {{.Names}}\\t{{.Image}}\\t{{.Status}}\\t{{.Ports}}' docker images --filter 'reference=*app*' -q | xargs -r docker rmi -f 2>/dev/null || echo 'Нет образов для удаления'
echo 'Step 2: Force cleanup old application...' echo '3. Создание сети если нужно...'
docker network create app-network 2>/dev/null || echo 'Сеть уже существует'
# Принудительно удаляем старый контейнер и образ echo '4. Запуск инфраструктуры...'
docker stop app-backend 2>/dev/null || echo 'App not running' docker-compose up -d db storage
docker rm app-backend 2>/dev/null || echo 'App not found'
docker rmi app-backend:latest 2>/dev/null || echo 'Image not found'
# Удаляем все образы app-backend echo '5. Ожидание PostgreSQL...'
docker images --filter 'reference=app-backend*' -q | xargs -r docker rmi 2>/dev/null || echo 'No images to remove'
echo 'Step 3: Verify copied files...'
echo 'Files in directory:'
ls -la
echo ''
echo 'Checking Java sources:'
find . -name '*.java' | head -2
echo 'Step 4: Check if PostgreSQL is running...'
if ! docker ps | grep -q postgres; then
echo 'Starting PostgreSQL...'
docker-compose up -d db
sleep 10 sleep 10
else
echo 'PostgreSQL already running'
fi
echo 'Step 5: Build application with forced rebuild...' echo '6. Сборка нового образа приложения...'
echo 'Checking Dockerfile exists:' docker build --no-cache -t app-backend:latest .
ls -la Dockerfile 2>/dev/null || echo 'Dockerfile not found, creating simple one'
# Если нет Dockerfile, создаем простой echo '7. Запуск приложения...'
if [ ! -f Dockerfile ]; then
echo 'Creating simple Dockerfile...'
cat > Dockerfile << 'EOF'
FROM eclipse-temurin:21-jre
WORKDIR /app
COPY . /app/
CMD ["java", "-jar", "app.jar"]
EOF
fi
# Собираем с подробным выводом
echo 'Building Docker image...'
docker build --no-cache --progress=plain -t app-backend:latest . 2>&1 | tail -50
echo 'Step 6: Verify new image...'
echo 'Current app-backend images:'
docker images | grep app-backend
echo 'Image creation time:'
docker inspect app-backend:latest --format='{{.Created}}' 2>/dev/null || echo 'Cannot inspect image'
echo 'Step 7: Starting application with alias...'
docker run -d \\ docker run -d \\
--name app-backend \\ --name app-backend \\
--network app-network \\ --network app-network \\
--network-alias app \\ --network-alias app \\
-p 80:8080 \\ -p 80:8080 \\
-v uploads_data:/data/uploads:rw \\
-e POSTGRES_DB=no_copy_ \\ -e POSTGRES_DB=no_copy_ \\
-e POSTGRES_USER=postgres \\ -e POSTGRES_USER=postgres \\
-e POSTGRES_PASSWORD=postgres \\ -e POSTGRES_PASSWORD=postgres \\
@@ -122,40 +84,22 @@ EOF
--restart unless-stopped \\ --restart unless-stopped \\
app-backend:latest app-backend:latest
echo 'Step 8: Checking deployment...' echo '8. Запуск мониторинга...'
sleep 10 docker-compose up -d grafana prometheus loki tempo alloy
echo 'Containers status:' echo '9. Проверка...'
docker ps --filter 'name=postgres|app-backend' --format 'table {{.Names}}\\t{{.Image}}\\t{{.Status}}\\t{{.CreatedAt}}'
echo 'Checking image inside container:'
docker exec app-backend ls -la /app/ 2>/dev/null || echo 'Cannot check container files'
echo 'Application logs (last 10 lines):'
docker logs --tail=10 app-backend 2>/dev/null || echo 'Logs not available yet'
echo 'Step 9: Health check...'
if docker ps | grep -q app-backend; then
echo 'Container is running'
echo 'Testing application health...'
for i in {1..5}; do
if curl -s -f http://localhost:80/health > /dev/null 2>&1; then
echo 'Health check passed'
echo 'Deployment successful'
echo 'Application URL: http://${params.SERVER}:80'
exit 0
fi
echo 'Waiting for application to start... attempt ' \$i
sleep 5 sleep 5
done
echo 'Health check failed, but container is running' echo 'Статус контейнеров:'
echo 'Application might be starting slowly' docker ps --format 'table {{.Names}}\\t{{.Image}}\\t{{.Status}}'
echo 'Application URL: http://${params.SERVER}:80'
echo '10. Проверка health...'
if curl -s -f http://localhost:80/health > /dev/null 2>&1; then
echo 'Приложение работает'
echo 'URL: http://${params.SERVER}:80'
else else
echo 'Application failed to start' echo 'Проверка health не удалась'
docker logs app-backend docker logs app-backend --tail=20
exit 1
fi fi
" "
""" """
@@ -2,8 +2,10 @@ package ru.soune.nocopy;
import org.springframework.boot.SpringApplication; import org.springframework.boot.SpringApplication;
import org.springframework.boot.autoconfigure.SpringBootApplication; import org.springframework.boot.autoconfigure.SpringBootApplication;
import org.springframework.scheduling.annotation.EnableScheduling;
@SpringBootApplication @SpringBootApplication
@EnableScheduling
public class NoCopyApplication { public class NoCopyApplication {
public static void main(String[] args) { public static void main(String[] args) {
@@ -20,10 +20,9 @@ import ru.soune.nocopy.dto.file.CompleteUploadResponse;
import ru.soune.nocopy.dto.file.FileEntityResponse; import ru.soune.nocopy.dto.file.FileEntityResponse;
import ru.soune.nocopy.dto.file.UploadProgress; import ru.soune.nocopy.dto.file.UploadProgress;
import ru.soune.nocopy.entity.AuthToken; import ru.soune.nocopy.entity.AuthToken;
import ru.soune.nocopy.entity.file.FileStatus;
import ru.soune.nocopy.entity.file.UploadStatus; import ru.soune.nocopy.entity.file.UploadStatus;
import ru.soune.nocopy.exception.NotFoundAuthToken; import ru.soune.nocopy.exception.*;
import ru.soune.nocopy.exception.NotValidFieldException;
import ru.soune.nocopy.exception.ValidationException;
import ru.soune.nocopy.handler.*; import ru.soune.nocopy.handler.*;
import ru.soune.nocopy.repository.AuthTokenRepository; import ru.soune.nocopy.repository.AuthTokenRepository;
import ru.soune.nocopy.service.file.FileEntityService; import ru.soune.nocopy.service.file.FileEntityService;
@@ -89,29 +88,34 @@ public class ApiController {
@PostMapping("/v{version}/files/chunk") @PostMapping("/v{version}/files/chunk")
public ResponseEntity<BaseResponse> uploadChunk( public ResponseEntity<BaseResponse> uploadChunk(
@PathVariable("version") int version, @PathVariable("version") int version,
@RequestParam("upload_id") String uploadId, @RequestParam(value = "upload_id", required = false) String uploadId,
@RequestParam("chunk_number") Integer chunkNumber, @RequestParam(value = "chunk_number", required = false) Integer chunkNumber,
@RequestParam("chunk") MultipartFile chunk) { @RequestParam(value = "chunk", required = false) MultipartFile chunk) {
try {
log.info("Uploading chunk {} for session {}, file size: {} bytes, version: {}", if (chunk == null || chunk.isEmpty()) {
chunkNumber, uploadId, chunk.getSize(), version);
if (chunk.isEmpty()) {
return ResponseEntity.ok().body(new BaseResponse(20004, MessageCode.FILE_UPLOAD_ERROR.getCode(), return ResponseEntity.ok().body(new BaseResponse(20004, MessageCode.FILE_UPLOAD_ERROR.getCode(),
"Chunk file is empty", null)); "Chunk file null or empty", ChunkUploadResponse.builder()
.uploadId(uploadId)
.chunkNumber(chunkNumber)
.build()));
} }
if (uploadId == null || uploadId.isBlank()) { if (uploadId == null || uploadId.isBlank()) {
return ResponseEntity.ok().body(new BaseResponse(20004, MessageCode.FILE_UPLOAD_ERROR.getCode(), return ResponseEntity.ok().body(new BaseResponse(20004, MessageCode.FILE_UPLOAD_ERROR.getCode(),
"Upload ID is required", null)); "Upload ID is required", ChunkUploadResponse.builder()
.uploadId(uploadId)
.chunkNumber(chunkNumber)
.build()));
} }
if (chunkNumber == null || chunkNumber < 0) { if (chunkNumber == null || chunkNumber < 0) {
return ResponseEntity.ok().body(new BaseResponse(20004, MessageCode.FILE_UPLOAD_ERROR.getCode(), return ResponseEntity.ok().body(new BaseResponse(20004, MessageCode.FILE_UPLOAD_ERROR.getCode(),
"Valid chunk number is required", null)); "Valid chunk number is required", ChunkUploadResponse.builder()
.uploadId(uploadId)
.chunkNumber(chunkNumber)
.build()));
} }
try {
fileUploadService.uploadChunk(uploadId, chunkNumber, chunk); fileUploadService.uploadChunk(uploadId, chunkNumber, chunk);
ChunkUploadResponse responseBody = ChunkUploadResponse.builder() ChunkUploadResponse responseBody = ChunkUploadResponse.builder()
@@ -123,7 +127,6 @@ public class ApiController {
return ResponseEntity.ok().body(new BaseResponse(20004, MessageCode.SUCCESS.getCode(), return ResponseEntity.ok().body(new BaseResponse(20004, MessageCode.SUCCESS.getCode(),
"Chunk uploaded successfully", responseBody)); "Chunk uploaded successfully", responseBody));
} catch (Exception e) { } catch (Exception e) {
log.error("Error uploading chunk", e); log.error("Error uploading chunk", e);
@@ -237,29 +240,67 @@ public class ApiController {
@GetMapping("/v{version}/files/download/{fileId}") @GetMapping("/v{version}/files/download/{fileId}")
public ResponseEntity<Resource> downloadFile( public ResponseEntity<?> downloadFile(
@PathVariable String fileId, @PathVariable(required = false) String fileId,
@PathVariable Integer version, @PathVariable(required = false) Integer version,
@RequestHeader("Authorization") String tokenHeader) { @RequestHeader(value = "Authorization", required = false) String tokenHeader) {
try { try {
if (tokenHeader == null) {
Map<String, Object> errorData = new HashMap<>();
errorData.put("token", tokenHeader);
errorData.put("fileId", fileId);
errorData.put("version", version);
return ResponseEntity.ok().body(new BaseResponse(20004,
MessageCode.FILE_DOWNLOAD_ERROR_NOT_CORRECT_FIELD.getCode(),
MessageCode.FILE_DOWNLOAD_ERROR_NOT_CORRECT_FIELD.getDescription(),
errorData));
}
Long userId = getUserIdFromToken(tokenHeader); Long userId = getUserIdFromToken(tokenHeader);
FileEntityResponse fileInfo = fileEntityService.getById(fileId, version); FileEntityResponse entityResponse = fileEntityService.getById(fileId, version);
if (!fileInfo.getUserId().equals(userId)) { if (!entityResponse.getUserId().equals(userId)) {
return ResponseEntity.status(403).build(); Map<String, Object> errorData = new HashMap<>();
errorData.put("token", tokenHeader);
return ResponseEntity.ok().body(new BaseResponse(20004,
MessageCode.FILE_DOWNLOAD_ERROR.getCode(),
MessageCode.FILE_DOWNLOAD_ERROR.getDescription(),
errorData));
} }
if (!fileInfo.isExistsOnDisk()) { if (entityResponse.getStatus().equals(FileStatus.DELETED)) {
return ResponseEntity.status(404) Map<String, Object> errorData = new HashMap<>();
.body(null); errorData.put("file_status", entityResponse.getStatus());
return ResponseEntity.ok().body(new BaseResponse(20004,
MessageCode.FILE_DOWNLOAD_ERROR.getCode(),
MessageCode.FILE_DOWNLOAD_ERROR.getDescription(),
errorData));
} }
Path filePath = Paths.get(fileInfo.getFilePath()); if (!entityResponse.isExistsOnDisk()) {
Map<String, Object> errorData = new HashMap<>();
errorData.put("onDisk", entityResponse.isExistsOnDisk());
return ResponseEntity.ok().body(new BaseResponse(20004,
MessageCode.FILE_DOWNLOAD_ERROR.getCode(),
MessageCode.FILE_DOWNLOAD_ERROR.getDescription(),
errorData));
}
Path filePath = Paths.get(entityResponse.getFilePath());
Resource resource = new UrlResource(filePath.toUri()); Resource resource = new UrlResource(filePath.toUri());
if (!resource.exists()) { if (!resource.exists()) {
return ResponseEntity.status(404).build(); Map<String, Object> errorData = new HashMap<>();
errorData.put("resource", resource.exists());
return ResponseEntity.ok().body(new BaseResponse(20004,
MessageCode.FILE_DOWNLOAD_ERROR.getCode(),
MessageCode.FILE_DOWNLOAD_ERROR.getDescription(),
errorData));
} }
String contentType = determineContentType(filePath); String contentType = determineContentType(filePath);
@@ -267,85 +308,30 @@ public class ApiController {
return ResponseEntity.ok() return ResponseEntity.ok()
.contentType(MediaType.parseMediaType(contentType)) .contentType(MediaType.parseMediaType(contentType))
.header(HttpHeaders.CONTENT_DISPOSITION, .header(HttpHeaders.CONTENT_DISPOSITION,
"attachment; filename=\"" + fileInfo.getOriginalFileName() + "\"") "attachment; filename=\"" + entityResponse.getOriginalFileName() + "\"")
.header(HttpHeaders.CONTENT_LENGTH, String.valueOf(fileInfo.getFileSize())) .header(HttpHeaders.CONTENT_LENGTH, String.valueOf(entityResponse.getFileSize()))
.body(resource); .body(resource);
} catch (FileEntityNotFoundException e) {
Map<String, Object> errorData = new HashMap<>();
errorData.put("fileId", fileId);
} catch (NotFoundAuthToken e) { return ResponseEntity.ok().body(new BaseResponse(20004,
return ResponseEntity.status(401).build(); MessageCode.FILE_NOT_FOUND.getCode(),
} catch (Exception e) { MessageCode.FILE_NOT_FOUND.getDescription(),
log.error("Error downloading file", e); errorData));
return ResponseEntity.status(500).build(); } catch (NotFoundAuthToken | IOException e) {
Map<String, Object> errorData = new HashMap<>();
errorData.put("token", tokenHeader);
errorData.put("fileId", fileId);
errorData.put("version", version);
return ResponseEntity.ok().body(new BaseResponse(20004,
MessageCode.FILE_DOWNLOAD_ERROR_NOT_CORRECT_FIELD.getCode(),
MessageCode.FILE_DOWNLOAD_ERROR_NOT_CORRECT_FIELD.getDescription(),
errorData));
} }
} }
@GetMapping("/v{version}/files/info/{fileId}")
public ResponseEntity<FileEntityResponse> getFileInfo(
@PathVariable String fileId,
@PathVariable Integer version,
@RequestHeader("Authorization") String tokenHeader) {
try {
Long userId = getUserIdFromToken(tokenHeader);
FileEntityResponse fileInfo = fileEntityService.getById(fileId, version);
if (!fileInfo.getUserId().equals(userId)) {
return ResponseEntity.status(403).build();
}
return ResponseEntity.ok(fileInfo);
} catch (NotFoundAuthToken e) {
return ResponseEntity.status(401).build();
} catch (Exception e) {
log.error("Error getting file info", e);
return ResponseEntity.status(500).build();
}
}
@GetMapping("/v{version}/files/preview/{fileId}")
public ResponseEntity<Resource> previewFile(
@PathVariable String fileId,
@PathVariable Integer version,
@RequestHeader("Authorization") String tokenHeader) {
try {
Long userId = getUserIdFromToken(tokenHeader);
FileEntityResponse fileInfo = fileEntityService.getById(fileId, version);
if (!fileInfo.getUserId().equals(userId)) {
return ResponseEntity.status(403).build();
}
if (!isPreviewSupported(fileInfo.getMimeType())) {
return ResponseEntity.status(415)
.body(null);
}
Path filePath = Paths.get(fileInfo.getFilePath());
Resource resource = new UrlResource(filePath.toUri());
if (!resource.exists()) {
return ResponseEntity.status(404).build();
}
String contentType = determineContentType(filePath);
return ResponseEntity.ok()
.contentType(MediaType.parseMediaType(contentType))
.header(HttpHeaders.CONTENT_DISPOSITION,
"inline; filename=\"" + fileInfo.getOriginalFileName() + "\"")
.body(resource);
} catch (NotFoundAuthToken e) {
return ResponseEntity.status(401).build();
} catch (Exception e) {
log.error("Error previewing file", e);
return ResponseEntity.status(500).build();
}
}
private ResponseEntity<BaseResponse> createValidationErrorResponse(BindingResult bindingResult, Integer msgId) { private ResponseEntity<BaseResponse> createValidationErrorResponse(BindingResult bindingResult, Integer msgId) {
List<Map<String, String>> fieldErrors = bindingResult.getFieldErrors() List<Map<String, String>> fieldErrors = bindingResult.getFieldErrors()
.stream() .stream()
@@ -382,18 +368,6 @@ public class ApiController {
return contentType; return contentType;
} }
private boolean isPreviewSupported(String mimeType) {
if (mimeType == null) {
return false;
}
return mimeType.startsWith("image/") ||
mimeType.startsWith("text/") ||
mimeType.equals("application/pdf") ||
mimeType.startsWith("video/") ||
mimeType.startsWith("audio/");
}
private Long getUserIdFromToken(String tokenHeader) { private Long getUserIdFromToken(String tokenHeader) {
String token = tokenHeader.replace("Bearer ", ""); String token = tokenHeader.replace("Bearer ", "");
AuthToken authToken = authTokenRepository.findByToken(token) AuthToken authToken = authTokenRepository.findByToken(token)
@@ -8,6 +8,8 @@ public enum MessageCode {
INVALID_TOKEN(2, "Invalid token"), INVALID_TOKEN(2, "Invalid token"),
INVALID_ACTION(2, "Invalid action"), INVALID_ACTION(2, "Invalid action"),
FILE_UPLOAD_ERROR(2, "File upload error"), FILE_UPLOAD_ERROR(2, "File upload error"),
FILE_DOWNLOAD_ERROR(2, "File download error"),
FILE_DOWNLOAD_ERROR_NOT_CORRECT_FIELD(2, "Not correct field"),
IMAGE_FOUND_ERROR(2, "Image found error"), IMAGE_FOUND_ERROR(2, "Image found error"),
INVALID_JSON_BODY(2, "Invalid fields in JSON object"), INVALID_JSON_BODY(2, "Invalid fields in JSON object"),
INCOMPLETE_UPLOAD(2, "Not load all chunks"), INCOMPLETE_UPLOAD(2, "Not load all chunks"),
@@ -0,0 +1,20 @@
package ru.soune.nocopy.dto.file;
import com.fasterxml.jackson.annotation.JsonProperty;
import lombok.Builder;
import lombok.Data;
import java.util.List;
@Data
@Builder
public class FileExtensionResponse {
@JsonProperty("file_extension")
private List<String> extension;
@JsonProperty("count")
private Integer count;
@JsonProperty("max_file_size")
private Long maxFileSize;
}
@@ -0,0 +1,61 @@
package ru.soune.nocopy.dto.file;
import com.fasterxml.jackson.annotation.JsonProperty;
import lombok.AllArgsConstructor;
import lombok.Builder;
import lombok.Data;
import lombok.NoArgsConstructor;
@Data
@Builder
@NoArgsConstructor
@AllArgsConstructor
public class FileInfoUserResponse {
@JsonProperty("all_files_size")
private Long allFileSize;
@JsonProperty("all_files_quantity")
private Integer fileCount;
@JsonProperty("all_files_check")
private Integer filesCheck;
@JsonProperty("all_files_violation")
private Integer filesViolation;
@JsonProperty("images_size")
private Long imagesSize;
@JsonProperty("images_quantity")
private Integer imagesCount;
@JsonProperty("images_check")
private Integer imagesCheck;
@JsonProperty("images_violations")
private Integer imagesViolations;
@JsonProperty("videos_size")
private Long videosSize;
@JsonProperty("videos_quantity")
private Integer videosCount;
@JsonProperty("videos_check")
private Integer videosCheck;
@JsonProperty("videos_violations")
private Integer videosViolations;
@JsonProperty("audios_size")
private Long audiosSize;
@JsonProperty("audios_quantity")
private Integer audiosCount;
@JsonProperty("audios_check")
private Integer audiosCheck;
@JsonProperty("audios_violations")
private Integer audiosViolations;
}
@@ -4,5 +4,7 @@ public enum FileStatus {
ACTIVE, ACTIVE,
DELETED, DELETED,
PROCESSING, PROCESSING,
VIOLATION,
CHECKED,
ERROR ERROR
} }
@@ -2,21 +2,33 @@ package ru.soune.nocopy.entity.file;
import lombok.Getter; import lombok.Getter;
import java.util.Arrays;
import java.util.List;
@Getter @Getter
public enum FileType { public enum FileType {
PHOTO("photo"), IMAGE("image", Arrays.asList("jpg", "jpeg", "png", "gif", "bmp", "webp", "jfif")),
IMAGE("image"), VIDEO("video", Arrays.asList("mp4", "avi", "mov", "wmv", "flv", "mkv", "webm", "m4v", "mpg", "mpeg",
VIDEO("video"), "3gp", "3g2", "f4v", "m2ts", "mts", "vob", "ogv", "divx")),
AUDIO("audio"), AUDIO("audio", Arrays.asList("mp3", "wav", "flac"));
DOCUMENT("document");
private final String code; private final String displayName;
private final List<String> allowedExtensions;
FileType(String code) { FileType(String displayName, List<String> allowedExtensions) {
this.code = code; this.displayName = displayName;
this.allowedExtensions = allowedExtensions;
} }
public String getCode() { public String getDisplayName() {
return code; return displayName;
}
public List<String> getAllowedExtensions() {
return allowedExtensions;
}
public boolean supportsExtension(String extension) {
return allowedExtensions.contains(extension.toLowerCase());
} }
} }
@@ -86,7 +86,7 @@ public class FileUploadSession {
@PrePersist @PrePersist
public void prePersist() { public void prePersist() {
this.createdAt = LocalDateTime.now(); this.createdAt = LocalDateTime.now();
this.expiresAt = LocalDateTime.now().plusHours(24); this.expiresAt = LocalDateTime.now().plusMinutes(1);
if (this.chunksUploaded == null) { if (this.chunksUploaded == null) {
this.chunksUploaded = 0; this.chunksUploaded = 0;
} }
@@ -2,6 +2,6 @@ package ru.soune.nocopy.exception;
public class ChunkSizeExceededException extends RuntimeException { public class ChunkSizeExceededException extends RuntimeException {
public ChunkSizeExceededException(long actualSize, long maxSize) { public ChunkSizeExceededException(long actualSize, long maxSize) {
super(String.format("Chunk size exceeded: %d > %d", actualSize, maxSize)); super(String.format("Chunk size not valid: " + actualSize));
} }
} }
@@ -7,11 +7,17 @@ import org.springframework.stereotype.Component;
import ru.soune.nocopy.dto.*; import ru.soune.nocopy.dto.*;
import ru.soune.nocopy.dto.file.*; import ru.soune.nocopy.dto.file.*;
import ru.soune.nocopy.entity.AuthToken; import ru.soune.nocopy.entity.AuthToken;
import ru.soune.nocopy.entity.file.FileEntity;
import ru.soune.nocopy.entity.file.FileStatus;
import ru.soune.nocopy.exception.FileEntityNotFoundException;
import ru.soune.nocopy.exception.NotFoundAuthToken; import ru.soune.nocopy.exception.NotFoundAuthToken;
import ru.soune.nocopy.repository.AuthTokenRepository; import ru.soune.nocopy.repository.AuthTokenRepository;
import ru.soune.nocopy.repository.FileEntityRepository;
import ru.soune.nocopy.service.file.FileEntityService; import ru.soune.nocopy.service.file.FileEntityService;
import ru.soune.nocopy.service.file.FileStatsService;
import java.util.Arrays; import java.util.Arrays;
import java.util.List;
@Slf4j @Slf4j
@Component @Component
@@ -19,9 +25,15 @@ import java.util.Arrays;
public class FileEntityHandler implements RequestHandler { public class FileEntityHandler implements RequestHandler {
private final FileEntityService fileEntityService; private final FileEntityService fileEntityService;
private final FileStatsService fileStatsService;
private final AuthTokenRepository authTokenRepository; private final AuthTokenRepository authTokenRepository;
private final ObjectMapper objectMapper; private final ObjectMapper objectMapper;
private final FileEntityRepository fileEntityRepository;
@Override @Override
public BaseResponse handle(BaseRequest request) { public BaseResponse handle(BaseRequest request) {
try { try {
@@ -31,15 +43,17 @@ public class FileEntityHandler implements RequestHandler {
String action = fileRequest.getAction(); String action = fileRequest.getAction();
switch (action) { switch (action) {
case "get_file_info": case "file_info":
return handleGetFileInfo(request, fileRequest); return handleGetFileInfo(request, fileRequest);
case "get_file_by_session": case "user_files_info":
return handleGetFilesUserInfo(request, fileRequest);
case "file_by_session":
return handleGetFileBySession(request, fileRequest); return handleGetFileBySession(request, fileRequest);
case "get_user_files": case "user_files":
return handleGetUserFiles(request, fileRequest); return handleGetUserFiles(request, fileRequest);
case "search_files": case "search_files":
return handleSearchFiles(request, fileRequest); return handleSearchFiles(request, fileRequest);
case "get_storage_usage": case "storage_usage":
return handleGetStorageUsage(request, fileRequest); return handleGetStorageUsage(request, fileRequest);
case "delete_file": case "delete_file":
return handleDeleteFile(request, fileRequest); return handleDeleteFile(request, fileRequest);
@@ -47,8 +61,8 @@ public class FileEntityHandler implements RequestHandler {
ActionResponse response = ActionResponse.builder() ActionResponse response = ActionResponse.builder()
.action(action) .action(action)
.availableActions(Arrays.asList( .availableActions(Arrays.asList(
"get_file_info", "get_file_by_session", "get_user_files", "file_info", "file_by_session", "user_files",
"search_files", "get_storage_usage", "delete_file")) "search_files", "storage_usage", "delete_file"))
.build(); .build();
return new BaseResponse(request.getMsgId(), return new BaseResponse(request.getMsgId(),
@@ -65,6 +79,16 @@ public class FileEntityHandler implements RequestHandler {
} }
} }
private BaseResponse handleGetFilesUserInfo(BaseRequest request, FileEntityRequest fileRequest) {
Long userId = getUserIdFromToken(fileRequest.getToken());
FileInfoUserResponse userFileStats = fileStatsService.getUserFileStats(userId);
return new BaseResponse(request.getMsgId(),
MessageCode.SUCCESS.getCode(),
MessageCode.SUCCESS.getDescription(),
userFileStats);
}
private BaseResponse handleGetFileInfo(BaseRequest request, FileEntityRequest fileRequest) { private BaseResponse handleGetFileInfo(BaseRequest request, FileEntityRequest fileRequest) {
try { try {
Long userId = getUserIdFromToken(fileRequest.getToken()); Long userId = getUserIdFromToken(fileRequest.getToken());
@@ -168,17 +192,22 @@ public class FileEntityHandler implements RequestHandler {
FileResponse allFiles = fileEntityService.getUserFiles(userId, 1, 1000, request.getVersion()); FileResponse allFiles = fileEntityService.getUserFiles(userId, 1, 1000, request.getVersion());
var filteredFiles = allFiles.getFiles().stream() var filteredFiles = allFiles.getFiles().stream()
.filter(f -> f.getOriginalFileName().toLowerCase().contains(fileRequest.getQuery().toLowerCase())) .filter(f -> (f.getOriginalFileName() + f.getFileExtension()).toLowerCase()
.contains(fileRequest.getQuery().toLowerCase()))
.toList(); .toList();
int start = (page - 1) * pageSize; int start = (page - 1) * pageSize;
int end = Math.min(start + pageSize, filteredFiles.size()); int end = Math.min(start + pageSize, filteredFiles.size());
List<FileEntityResponse> fileEntityResponses = filteredFiles.subList(start, Math.min(end, filteredFiles.size()));
long sumSize = fileEntityResponses.stream()
.mapToLong(FileEntityResponse::getFileSize)
.sum();
FileListResponse response = FileListResponse.builder() FileListResponse response = FileListResponse.builder()
.files(filteredFiles.subList(start, Math.min(end, filteredFiles.size()))) .files(fileEntityResponses)
.totalCount(filteredFiles.size()) .totalCount(filteredFiles.size())
.totalSize(allFiles.getTotalSize()) .totalSize(sumSize)
.formattedTotalSize(allFiles.getFormattedTotalSize()) .formattedTotalSize(fileEntityService.formatFileSize(sumSize))
.page(page) .page(page)
.pageSize(pageSize) .pageSize(pageSize)
.build(); .build();
@@ -187,18 +216,13 @@ public class FileEntityHandler implements RequestHandler {
MessageCode.SUCCESS.getCode(), MessageCode.SUCCESS.getCode(),
MessageCode.SUCCESS.getDescription(), MessageCode.SUCCESS.getDescription(),
response); response);
} catch (NotFoundAuthToken e) { } catch (NotFoundAuthToken e) {
return new BaseResponse(request.getMsgId(), return new BaseResponse(request.getMsgId(), MessageCode.INVALID_TOKEN.getCode(),
MessageCode.INVALID_TOKEN.getCode(), "Authentication required", null);
"Authentication required",
null);
} catch (Exception e) { } catch (Exception e) {
log.error("Error searching files", e); log.error("Error searching files", e);
return new BaseResponse(request.getMsgId(), return new BaseResponse(request.getMsgId(), MessageCode.FILE_UPLOAD_ERROR.getCode(),
MessageCode.FILE_UPLOAD_ERROR.getCode(), "Failed to search files: " + e.getMessage(), null);
"Failed to search files: " + e.getMessage(),
null);
} }
} }
@@ -235,7 +259,9 @@ public class FileEntityHandler implements RequestHandler {
private BaseResponse handleDeleteFile(BaseRequest request, FileEntityRequest fileRequest) { private BaseResponse handleDeleteFile(BaseRequest request, FileEntityRequest fileRequest) {
try { try {
Long userId = getUserIdFromToken(fileRequest.getToken()); Long userId = getUserIdFromToken(fileRequest.getToken());
FileEntityResponse fileInfo = fileEntityService.getById(fileRequest.getFileId(), request.getVersion()); String fileId = fileRequest.getFileId();
FileEntityResponse fileInfo = fileEntityService.getById(fileId, request.getVersion());
DeleteFileResponse response;
if (!fileInfo.getUserId().equals(userId)) { if (!fileInfo.getUserId().equals(userId)) {
return new BaseResponse(request.getMsgId(), return new BaseResponse(request.getMsgId(),
@@ -244,18 +270,27 @@ public class FileEntityHandler implements RequestHandler {
null); null);
} }
fileEntityService.markAsDeleted(fileRequest.getFileId()); FileEntity fileEntity = fileEntityRepository.findById(fileId)
.orElseThrow(() -> new FileEntityNotFoundException(fileId));
DeleteFileResponse response = DeleteFileResponse.builder() if (fileEntity.getStatus().equals(FileStatus.DELETED)) {
fileEntityService.deleteFromDisk(fileEntity);
response = DeleteFileResponse.builder()
.fileId(fileRequest.getFileId())
.message("File deleted from disk")
.build();
} else {
fileEntityService.markAsDeleted(fileEntity);
response = DeleteFileResponse.builder()
.fileId(fileRequest.getFileId()) .fileId(fileRequest.getFileId())
.message("File marked as deleted") .message("File marked as deleted")
.build(); .build();
}
return new BaseResponse(request.getMsgId(), return new BaseResponse(request.getMsgId(),
MessageCode.SUCCESS.getCode(), MessageCode.SUCCESS.getCode(),
"File marked as deleted", "File marked as deleted",
response); response);
} catch (NotFoundAuthToken e) { } catch (NotFoundAuthToken e) {
return new BaseResponse(request.getMsgId(), return new BaseResponse(request.getMsgId(),
MessageCode.INVALID_TOKEN.getCode(), MessageCode.INVALID_TOKEN.getCode(),
@@ -3,9 +3,11 @@ package ru.soune.nocopy.handler;
import com.fasterxml.jackson.databind.ObjectMapper; import com.fasterxml.jackson.databind.ObjectMapper;
import lombok.RequiredArgsConstructor; import lombok.RequiredArgsConstructor;
import lombok.extern.slf4j.Slf4j; import lombok.extern.slf4j.Slf4j;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.stereotype.Component; import org.springframework.stereotype.Component;
import org.springframework.validation.BeanPropertyBindingResult; import org.springframework.validation.BeanPropertyBindingResult;
import org.springframework.validation.BindingResult; import org.springframework.validation.BindingResult;
import org.springframework.validation.FieldError;
import ru.soune.nocopy.dto.*; import ru.soune.nocopy.dto.*;
import ru.soune.nocopy.dto.file.*; import ru.soune.nocopy.dto.file.*;
import ru.soune.nocopy.entity.AuthToken; import ru.soune.nocopy.entity.AuthToken;
@@ -17,21 +19,26 @@ import ru.soune.nocopy.repository.AuthTokenRepository;
import ru.soune.nocopy.repository.FileUploadSessionRepository; import ru.soune.nocopy.repository.FileUploadSessionRepository;
import ru.soune.nocopy.service.file.FileUploadService; import ru.soune.nocopy.service.file.FileUploadService;
import java.util.Arrays; import java.util.*;
import java.util.HashMap; import java.util.stream.Collectors;
import java.util.Map;
import java.util.Optional;
@Slf4j @Slf4j
@Component @Component
@RequiredArgsConstructor @RequiredArgsConstructor
public class FileUploadHandler implements RequestHandler { public class FileUploadHandler implements RequestHandler {
private final FileUploadService fileUploadService; private final FileUploadService fileUploadService;
private final AuthTokenRepository authTokenRepository; private final AuthTokenRepository authTokenRepository;
private final FileUploadSessionRepository fileUploadSessionRepository; private final FileUploadSessionRepository fileUploadSessionRepository;
private final ObjectMapper objectMapper; private final ObjectMapper objectMapper;
private final FileUploadRequestValidator fileUploadRequestValidator; private final FileUploadRequestValidator fileUploadRequestValidator;
@Value("${file.storage.max-file-size}")
private long maxFileSize;
@Override @Override
public BaseResponse handle(BaseRequest request) { public BaseResponse handle(BaseRequest request) {
try { try {
@@ -45,18 +52,18 @@ public class FileUploadHandler implements RequestHandler {
return handleInitUpload(request, fileRequest); return handleInitUpload(request, fileRequest);
case "file_types": case "file_types":
return handleGetFileTypes(request); return handleGetFileTypes(request);
case "file_extension":
return handleGetFileExtensions(fileRequest, request);
case "chunks": case "chunks":
return handleGetChunkStatus(request, fileRequest); return handleGetChunkStatus(request, fileRequest);
case "cancel": case "cancel":
return handleCancelUpload(request, fileRequest); return handleCancelUpload(request, fileRequest);
case "progress": case "progress":
return handleGetProgress(request, fileRequest); return handleGetProgress(request, fileRequest);
case "retry":
return handleRetryUpload(request, fileRequest);
default: default:
ActionResponse response = ActionResponse.builder() ActionResponse response = ActionResponse.builder()
.action(action) .action(action)
.availableActions(Arrays.asList("init", "file_types", "chunks", "cancel", "progress", "retry")) .availableActions(Arrays.asList("init", "file_types", "chunks", "cancel", "progress"))
.build(); .build();
return new BaseResponse(request.getMsgId(), MessageCode.INVALID_ACTION.getCode(), return new BaseResponse(request.getMsgId(), MessageCode.INVALID_ACTION.getCode(),
@@ -70,18 +77,31 @@ public class FileUploadHandler implements RequestHandler {
} }
private BaseResponse handleInitUpload(BaseRequest request, FileUploadRequest fileRequest) { private BaseResponse handleInitUpload(BaseRequest request, FileUploadRequest fileRequest) {
try {
String token = fileRequest.getToken(); String token = fileRequest.getToken();
Optional<AuthToken> tokenOptional = authTokenRepository.findByToken(token); Optional<AuthToken> tokenOptional = authTokenRepository.findByToken(token);
if (tokenOptional.isEmpty()) {
return new BaseResponse(request.getMsgId(), MessageCode.INVALID_TOKEN.getCode(),
MessageCode.INVALID_TOKEN.getDescription(), Map.of("token", token));
}
AuthToken authToken = tokenOptional.orElseThrow(() -> new NotFoundAuthToken("Token not found")); AuthToken authToken = tokenOptional.orElseThrow(() -> new NotFoundAuthToken("Token not found"));
BindingResult bindingResult = new BeanPropertyBindingResult(fileRequest, "fileRequest"); BindingResult bindingResult = new BeanPropertyBindingResult(fileRequest, "fileRequest");
fileUploadRequestValidator.validate(fileRequest, bindingResult); fileUploadRequestValidator.validate(fileRequest, bindingResult);
if (bindingResult.hasErrors()) { if (bindingResult.hasErrors()) {
throw new ValidationException(bindingResult, request.getMsgId()); Map<String, String> fieldErrors = bindingResult.getFieldErrors()
.stream()
.collect(Collectors.toMap(
FieldError::getField,
fieldError -> fieldError.getDefaultMessage() != null
? fieldError.getDefaultMessage()
: "Validation error"));
return new BaseResponse(request.getMsgId(), MessageCode.INVALID_FIELD.getCode(),
MessageCode.INVALID_FIELD.getDescription(), fieldErrors);
} }
FileUploadSession session = fileUploadService.initUpload( FileUploadSession session = fileUploadService.initUpload(
@@ -101,15 +121,6 @@ public class FileUploadHandler implements RequestHandler {
return new BaseResponse(request.getMsgId(), MessageCode.SUCCESS.getCode(), return new BaseResponse(request.getMsgId(), MessageCode.SUCCESS.getCode(),
MessageCode.SUCCESS.getDescription(), fileResponse); MessageCode.SUCCESS.getDescription(), fileResponse);
} catch (NotFoundAuthToken e) {
InitFileResponse initFileResponse = InitFileResponse.builder()
.build();
throw new NotValidFieldException("Invalid or expired token: " + fileRequest.getToken(),
new BaseResponse(request.getMsgId(), MessageCode.INVALID_TOKEN.getCode(),
MessageCode.INVALID_TOKEN.getDescription(), initFileResponse));
}
} }
private BaseResponse handleGetProgress(BaseRequest request, FileUploadRequest fileRequest) { private BaseResponse handleGetProgress(BaseRequest request, FileUploadRequest fileRequest) {
@@ -240,4 +251,24 @@ public class FileUploadHandler implements RequestHandler {
"Failed to get file types", null); "Failed to get file types", null);
} }
} }
private BaseResponse handleGetFileExtensions(FileUploadRequest fileRequest, BaseRequest request) {
try {
FileType fileType = FileType.valueOf(fileRequest.getFileType().toUpperCase());
List<String> allowedExtensions = fileType.getAllowedExtensions();
FileExtensionResponse response = FileExtensionResponse.builder()
.extension(allowedExtensions)
.count(allowedExtensions.size())
.maxFileSize(maxFileSize)
.build();
return new BaseResponse(request.getMsgId(), MessageCode.SUCCESS.getCode(),
MessageCode.SUCCESS.getDescription(), response);
} catch (Exception e) {
log.error("Error getting file extensions", e);
return new BaseResponse(request.getMsgId(), MessageCode.FILE_UPLOAD_ERROR.getCode(),
"Failed to get file extensions", null);
}
}
} }
@@ -7,9 +7,12 @@ import ru.soune.nocopy.dto.file.FileUploadRequest;
import ru.soune.nocopy.entity.file.FileType; import ru.soune.nocopy.entity.file.FileType;
import java.util.Arrays; import java.util.Arrays;
import java.util.List;
@Component @Component
public class FileUploadRequestValidator implements Validator { public class FileUploadRequestValidator implements Validator {
private final List<FileType> supportedFileTypes = Arrays.asList(FileType.values());
@Override @Override
public boolean supports(Class<?> clazz) { public boolean supports(Class<?> clazz) {
return FileUploadRequest.class.isAssignableFrom(clazz); return FileUploadRequest.class.isAssignableFrom(clazz);
@@ -21,7 +24,7 @@ public class FileUploadRequestValidator implements Validator {
validateType(request.getFileType(), errors); validateType(request.getFileType(), errors);
validateFileName(request.getFileName(), errors); validateFileName(request.getFileName(), errors);
validateExtension(request.getExtension(), errors); validateExtension(request.getExtension(), request.getFileType(), errors);
} }
private void validateType(String fileType, Errors errors) { private void validateType(String fileType, Errors errors) {
@@ -31,7 +34,12 @@ public class FileUploadRequestValidator implements Validator {
} }
try { try {
FileType.valueOf(fileType.toUpperCase()); FileType parsedType = FileType.valueOf(fileType.toUpperCase());
if (!supportedFileTypes.contains(parsedType)) {
errors.rejectValue("fileType", "fileType.unsupported",
"Unsupported file type. Valid types: " + Arrays.toString(FileType.values()));
}
} catch (IllegalArgumentException e) { } catch (IllegalArgumentException e) {
errors.rejectValue("fileType", "fileType.invalid", errors.rejectValue("fileType", "fileType.invalid",
"Invalid file type. Valid types: " + Arrays.toString(FileType.values())); "Invalid file type. Valid types: " + Arrays.toString(FileType.values()));
@@ -50,10 +58,34 @@ public class FileUploadRequestValidator implements Validator {
} }
} }
private void validateExtension(String extension, Errors errors) { private void validateExtension(String extension, String fileType, Errors errors) {
if (extension == null || extension.isBlank()) { if (extension == null || extension.isBlank()) {
errors.rejectValue("extension", "extension.required", "Extension is required"); errors.rejectValue("fileType", "fileType.required", "File type is required");
return; return;
} }
if (extension.contains(".")) {
errors.rejectValue("extension", "extension.required", "Extension contains comma");
return;
}
try {
FileType parsedType = FileType.valueOf(fileType.toUpperCase());
if (!parsedType.supportsExtension(extension)) {
errors.rejectValue("fileName", "fileType.extension.mismatch",
String.format("File extension '%s' does not match file type '%s'. Allowed extensions for %s: %s",
extension, parsedType.getDisplayName(), parsedType.getDisplayName(),
parsedType.getAllowedExtensions()));
}
} catch (IllegalArgumentException e) {
errors.rejectValue("fileType", "fileType.invalid",
"Invalid file type. Valid types: " + Arrays.toString(FileType.values()));
}
}
private String getFileExtension(String fileName) {
return fileName.substring(fileName.lastIndexOf('.') + 1).toLowerCase();
} }
} }
@@ -17,6 +17,7 @@ import java.nio.file.Files;
import java.nio.file.Path; import java.nio.file.Path;
import java.nio.file.Paths; import java.nio.file.Paths;
import java.time.LocalDateTime; import java.time.LocalDateTime;
import java.time.ZoneOffset;
import java.util.List; import java.util.List;
import java.util.stream.Collectors; import java.util.stream.Collectors;
@@ -40,7 +41,6 @@ public class FileEntityService {
long fileSize = Files.size(filePath); long fileSize = Files.size(filePath);
String originalName = session.getFileName(); String originalName = session.getFileName();
String extension = extractFileExtension(originalName);
String storedName = filePath.getFileName().toString(); String storedName = filePath.getFileName().toString();
FileEntity fileEntity = FileEntity.builder() FileEntity fileEntity = FileEntity.builder()
@@ -50,7 +50,7 @@ public class FileEntityService {
.filePath(session.getFilePath()) .filePath(session.getFilePath())
.fileSize(fileSize) .fileSize(fileSize)
.mimeType(session.getFileType()) .mimeType(session.getFileType())
.fileExtension(extension) .fileExtension(session.getExtension())
.checksum(checksum) .checksum(checksum)
.uploadSessionId(session.getUploadId()) .uploadSessionId(session.getUploadId())
.status(FileStatus.ACTIVE) .status(FileStatus.ACTIVE)
@@ -168,15 +168,24 @@ public class FileEntityService {
} }
@Transactional @Transactional
public void markAsDeleted(String fileId) { public void markAsDeleted(FileEntity fileEntity) {
FileEntity fileEntity = fileEntityRepository.findById(fileId)
.orElseThrow(() -> new FileEntityNotFoundException(fileId));
fileEntity.setStatus(FileStatus.DELETED); fileEntity.setStatus(FileStatus.DELETED);
fileEntity.setUpdatedAt(LocalDateTime.now()); fileEntity.setUpdatedAt(LocalDateTime.now());
fileEntityRepository.save(fileEntity);
log.info("FileEntity marked as deleted: {}", fileId); fileEntityRepository.save(fileEntity);
}
@Transactional
public boolean deleteFromDisk(FileEntity fileEntity) throws IOException {
Path path = Paths.get(fileEntity.getFilePath());
if (!Files.exists(path)) {
return true;
}
Files.delete(path);
return true;
} }
@Transactional(readOnly = true) @Transactional(readOnly = true)
@@ -217,7 +226,7 @@ public class FileEntityService {
.build(); .build();
} }
private String formatFileSize(long size) { public String formatFileSize(long size) {
if (size < 1024) { if (size < 1024) {
return size + " B"; return size + " B";
} else if (size < 1024 * 1024) { } else if (size < 1024 * 1024) {
@@ -0,0 +1,93 @@
package ru.soune.nocopy.service.file;
import lombok.RequiredArgsConstructor;
import org.springframework.stereotype.Service;
import ru.soune.nocopy.dto.file.FileInfoUserResponse;
import ru.soune.nocopy.entity.file.FileEntity;
import ru.soune.nocopy.entity.file.FileStatus;
import ru.soune.nocopy.entity.file.FileType;
import ru.soune.nocopy.repository.FileEntityRepository;
import java.util.List;
@Service
@RequiredArgsConstructor
public class FileStatsService {
private final FileEntityRepository fileEntityRepository;
public FileInfoUserResponse getUserFileStats(Long userId) {
List<FileEntity> userFiles = fileEntityRepository.findByUserId(userId);
return calculateStats(userFiles);
}
public FileInfoUserResponse calculateStats(List<FileEntity> files) {
return FileInfoUserResponse.builder()
.allFileSize(calculateTotalSize(files))
.fileCount(calculateTotalCount(files))
.filesCheck(calculateByStatus(files, FileStatus.CHECKED))
.filesViolation(calculateByStatus(files, FileStatus.VIOLATION))
.imagesSize(calculateMediaSize(files, FileType.IMAGE))
.imagesCount(calculateMediaCount(files, FileType.IMAGE))
.imagesCheck(calculateMediaByStatus(files, FileType.IMAGE, FileStatus.CHECKED))
.imagesViolations(calculateMediaByStatus(files, FileType.IMAGE, FileStatus.VIOLATION))
.videosSize(calculateMediaSize(files, FileType.VIDEO))
.videosCount(calculateMediaCount(files, FileType.VIDEO))
.videosCheck(calculateMediaByStatus(files, FileType.VIDEO, FileStatus.CHECKED))
.videosViolations(calculateMediaByStatus(files, FileType.VIDEO, FileStatus.VIOLATION))
.audiosSize(calculateMediaSize(files, FileType.AUDIO))
.audiosCount(calculateMediaCount(files, FileType.AUDIO))
.audiosCheck(calculateMediaByStatus(files, FileType.AUDIO, FileStatus.CHECKED))
.audiosViolations(calculateMediaByStatus(files, FileType.AUDIO, FileStatus.VIOLATION))
.build();
}
private Long calculateTotalSize(List<FileEntity> files) {
return files.stream()
.filter(file -> file.getStatus() != FileStatus.DELETED)
.mapToLong(FileEntity::getFileSize)
.sum();
}
private Integer calculateTotalCount(List<FileEntity> files) {
return (int) files.stream()
.filter(file -> file.getStatus() != FileStatus.DELETED)
.count();
}
private Integer calculateByStatus(List<FileEntity> files, FileStatus status) {
return (int) files.stream()
.filter(file -> file.getStatus() == status)
.count();
}
private Long calculateMediaSize(List<FileEntity> files, FileType fileType) {
return files.stream()
.filter(file -> file.getStatus() != FileStatus.DELETED)
.filter(file -> isFileType(file, fileType))
.mapToLong(FileEntity::getFileSize)
.sum();
}
private Integer calculateMediaCount(List<FileEntity> files, FileType fileType) {
return (int) files.stream()
.filter(file -> file.getStatus() != FileStatus.DELETED)
.filter(file -> isFileType(file, fileType))
.count();
}
private Integer calculateMediaByStatus(List<FileEntity> files, FileType fileType, FileStatus status) {
return (int) files.stream()
.filter(file -> isFileType(file, fileType))
.filter(file -> file.getStatus() == status)
.count();
}
private boolean isFileType(FileEntity file, FileType fileType) {
if (file.getFileExtension() == null) return false;
String extension = file.getFileExtension().toLowerCase().replace(".", "");
return fileType.supportsExtension(extension);
}
}
@@ -15,7 +15,7 @@ public interface FileUploadService {
UploadProgressResponse getUploadProgress(String uploadId); UploadProgressResponse getUploadProgress(String uploadId);
void cleanupExpiredSessions(); void handleExpiredSession(FileUploadSession session);
public void retryFailedUpload(String uploadId); public void retryFailedUpload(String uploadId);
@@ -102,12 +102,14 @@ public class FileUploadServiceImpl implements FileUploadService {
int totalChunks = (int) Math.ceil((double) fileSize / chunkSize); int totalChunks = (int) Math.ceil((double) fileSize / chunkSize);
log.debug("File will be split into {} chunks (chunk size: {} bytes)", log.debug("File will be split into {} chunks (chunk size: {} bytes)",
totalChunks, chunkSize); totalChunks, chunkSize);
Long chunkSize = totalChunks == 1 ? fileSize : 1000000L;
FileUploadSession session = FileUploadSession.builder() FileUploadSession session = FileUploadSession.builder()
.userId(userId) .userId(userId)
.extension(extension) .extension(extension)
.fileName(fileName) .fileName(fileName)
.fileType(fileType) .fileType(fileType)
.chunkSize(chunkSize)
.fileSize(fileSize) .fileSize(fileSize)
.totalChunks(totalChunks) .totalChunks(totalChunks)
.chunksUploaded(0) .chunksUploaded(0)
@@ -143,17 +145,103 @@ public class FileUploadServiceImpl implements FileUploadService {
if (chunkNumber < 0 || chunkNumber >= session.getTotalChunks()) { if (chunkNumber < 0 || chunkNumber >= session.getTotalChunks()) {
throw new FileUploadException( throw new FileUploadException(
String.format("Invalid chunk number %d. Expected 0-%d", String.format("Invalid chunk number %d. Expected number: %d",
chunkNumber, session.getTotalChunks() - 1)); chunkNumber, session.getTotalChunks() - 1));
} }
if (chunkFile.getSize() > chunkSize) { if (chunkNumber == 0 && chunkFile.getSize() > chunkSize || chunkNumber + 1 == (session.getTotalChunks())
&& chunkFile.getSize() > chunkSize) {
throw new ChunkSizeExceededException(chunkFile.getSize(), chunkSize);
}
if (chunkNumber > 0 && chunkNumber + 1 < session.getTotalChunks() && chunkFile.getSize() != chunkSize) {
throw new ChunkSizeExceededException(chunkFile.getSize(), chunkSize); throw new ChunkSizeExceededException(chunkFile.getSize(), chunkSize);
} }
return processChunk(session, chunkNumber, chunkFile); return processChunk(session, chunkNumber, chunkFile);
} }
@Override
@Transactional
public void handleExpiredSession(FileUploadSession session) {
session.setStatus(UploadStatus.FAILED);
session.setLastError("Upload session expired");
sessionRepository.save(session);
CompletableFuture.runAsync(() -> cleanupSessionFiles(session));
}
@Async("fileUploadTaskExecutor")
@Transactional
public void assembleFileAsync(FileUploadSession session) {
try {
assembleFile(session);
log.info("File assembly completed successfully for session: {}",
session.getUploadId());
} catch (Exception e) {
log.error("Failed to assemble file for session {}: {}",
session.getUploadId(), e.getMessage(), e);
handleAssemblyFailure(session, e);
}
}
@Override
@Transactional
public void cancelUpload(String uploadId) {
FileUploadSession session = sessionRepository.findById(uploadId)
.orElseThrow(() -> new UploadSessionNotFoundException(uploadId));
if (session.getStatus() == UploadStatus.COMPLETED || session.getStatus() == UploadStatus.CANCELLED) {
throw new FileUploadException("Cannot cancel completed or cancelled upload");
}
session.setStatus(UploadStatus.CANCELLED);
sessionRepository.save(session);
CompletableFuture.runAsync(() -> cleanupSessionFiles(session));
log.info("Upload cancelled: {}", uploadId);
}
@Override
@Transactional
public void retryFailedUpload(String uploadId) {
FileUploadSession session = sessionRepository.findById(uploadId)
.orElseThrow(() -> new UploadSessionNotFoundException(uploadId));
if (session.getStatus() != UploadStatus.FAILED) {
throw new FileUploadException("Only failed uploads can be retried");
}
if (session.getRetryCount() >= maxRetryAttempts) {
throw new FileUploadException("Max retry attempts exceeded");
}
session.setStatus(UploadStatus.UPLOADING);
session.setRetryCount(session.getRetryCount() + 1);
session.setLastError(null);
sessionRepository.save(session);
if (session.getChunksUploaded().equals(session.getTotalChunks())) {
log.info("Retrying file assembly for session: {}", uploadId);
assembleFileAsync(session);
}
log.info("Upload retry initiated for session: {} (attempt {})",
uploadId, session.getRetryCount());
}
@Override
public UploadProgressResponse getUploadProgress(String uploadId) {
FileUploadSession session = sessionRepository.findById(uploadId)
.orElseThrow(() -> new UploadSessionNotFoundException(uploadId));
return UploadProgressResponse.fromSession(session);
}
private void validateSession(FileUploadSession session) { private void validateSession(FileUploadSession session) {
UploadStatus status = session.getStatus(); UploadStatus status = session.getStatus();
@@ -183,14 +271,6 @@ public class FileUploadServiceImpl implements FileUploadService {
} }
} }
private void handleExpiredSession(FileUploadSession session) {
session.setStatus(UploadStatus.FAILED);
session.setLastError("Upload session expired");
sessionRepository.save(session);
CompletableFuture.runAsync(() -> cleanupSessionFiles(session));
}
private UploadProgressResponse processChunk(FileUploadSession session, private UploadProgressResponse processChunk(FileUploadSession session,
Integer chunkNumber, Integer chunkNumber,
MultipartFile chunkFile) { MultipartFile chunkFile) {
@@ -206,6 +286,7 @@ public class FileUploadServiceImpl implements FileUploadService {
session.getChunkPaths().put(chunkNumber, chunkPath); session.getChunkPaths().put(chunkNumber, chunkPath);
session.setChunksUploaded(session.getChunksUploaded() + 1); session.setChunksUploaded(session.getChunksUploaded() + 1);
session.setStatus(UploadStatus.UPLOADING); session.setStatus(UploadStatus.UPLOADING);
session.setExpiresAt(LocalDateTime.now().plusMinutes(1));
log.debug("Chunk {} saved successfully. Uploaded: {}/{}", log.debug("Chunk {} saved successfully. Uploaded: {}/{}",
chunkNumber, session.getChunksUploaded(), session.getTotalChunks()); chunkNumber, session.getChunksUploaded(), session.getTotalChunks());
@@ -316,52 +397,6 @@ public class FileUploadServiceImpl implements FileUploadService {
} }
} }
@Async("fileUploadTaskExecutor")
@Transactional
public void assembleFileAsync(FileUploadSession session) {
try {
assembleFile(session);
log.info("File assembly completed successfully for session: {}",
session.getUploadId());
} catch (Exception e) {
log.error("Failed to assemble file for session {}: {}",
session.getUploadId(), e.getMessage(), e);
handleAssemblyFailure(session, e);
}
}
// private void assembleFile(FileUploadSession session) throws IOException {
// log.info("Starting file assembly for session: {} ({})",
// session.getUploadId(), session.getFileName());
//
// Path finalFilePath = null;
//
// try {
// finalFilePath = prepareFinalFile(session);
//
// validateAllChunksExist(session);
//
// mergeChunksToFile(session, finalFilePath);
//
// validateFinalFile(session, finalFilePath);
//
// updateSessionOnSuccess(session, finalFilePath);
//
// cleanupSessionFiles(session);
//
// log.info("File assembly completed: {} -> {} ({} bytes)",
// session.getFileName(), finalFilePath, session.getFileSize());
//
// } catch (Exception e) {
// if (finalFilePath != null) {
// Files.deleteIfExists(finalFilePath);
// }
// throw e;
// }
// }
private void assembleFile(FileUploadSession session) throws IOException { private void assembleFile(FileUploadSession session) throws IOException {
log.info("Starting file assembly for session: {} ({})", log.info("Starting file assembly for session: {} ({})",
session.getUploadId(), session.getFileName()); session.getUploadId(), session.getFileName());
@@ -526,14 +561,6 @@ public class FileUploadServiceImpl implements FileUploadService {
log.debug("Final file checksum: {}", checksum); log.debug("Final file checksum: {}", checksum);
} }
private void updateSessionOnSuccess(FileUploadSession session, Path finalPath) {
session.setFilePath(finalPath.toString());
session.setChecksum(calculateChecksum(finalPath));
session.setStatus(UploadStatus.COMPLETED);
session.setCompletedAt(LocalDateTime.now());
sessionRepository.save(session);
}
private void cleanupSessionFiles(FileUploadSession session) { private void cleanupSessionFiles(FileUploadSession session) {
try { try {
Path chunkDir = getChunkDirectory(session.getUserId(), session.getUploadId()); Path chunkDir = getChunkDirectory(session.getUserId(), session.getUploadId());
@@ -570,90 +597,37 @@ public class FileUploadServiceImpl implements FileUploadService {
session.getUploadId(), e.getMessage()); session.getUploadId(), e.getMessage());
} }
@Override
@Transactional
public void cancelUpload(String uploadId) {
FileUploadSession session = sessionRepository.findById(uploadId)
.orElseThrow(() -> new UploadSessionNotFoundException(uploadId));
if (session.getStatus() == UploadStatus.COMPLETED || session.getStatus() == UploadStatus.CANCELLED) {
throw new FileUploadException("Cannot cancel completed or cancelled upload");
}
session.setStatus(UploadStatus.CANCELLED); // @Override
sessionRepository.save(session); // @Transactional
// public void cleanupExpiredSessions() {
CompletableFuture.runAsync(() -> cleanupSessionFiles(session)); // log.info("Starting cleanup of expired upload sessions");
//
log.info("Upload cancelled: {}", uploadId); // LocalDateTime expiryThreshold = LocalDateTime.now().minusHours(sessionExpiryHours);
} //
// try {
@Override // var expiredSessions = sessionRepository.findByStatusInAndCreatedAtBefore(
@Transactional // Set.of(UploadStatus.INITIATED, UploadStatus.UPLOADING, UploadStatus.FAILED),
public void retryFailedUpload(String uploadId) { // expiryThreshold
FileUploadSession session = sessionRepository.findById(uploadId) // );
.orElseThrow(() -> new UploadSessionNotFoundException(uploadId)); //
// int cleanedCount = 0;
if (session.getStatus() != UploadStatus.FAILED) { // for (FileUploadSession session : expiredSessions) {
throw new FileUploadException("Only failed uploads can be retried"); // session.setStatus(UploadStatus.FAILED);
} // session.setLastError("Session expired during cleanup");
// sessionRepository.save(session);
if (session.getRetryCount() >= maxRetryAttempts) { //
throw new FileUploadException("Max retry attempts exceeded"); // CompletableFuture.runAsync(() -> cleanupSessionFiles(session));
} // cleanedCount++;
// }
session.setStatus(UploadStatus.UPLOADING); //
session.setRetryCount(session.getRetryCount() + 1); // log.info("Cleaned up {} expired upload sessions", cleanedCount);
session.setLastError(null); //
// } catch (Exception e) {
sessionRepository.save(session); // log.error("Error during session cleanup", e);
// }
if (session.getChunksUploaded().equals(session.getTotalChunks())) { // }
log.info("Retrying file assembly for session: {}", uploadId);
assembleFileAsync(session);
}
log.info("Upload retry initiated for session: {} (attempt {})",
uploadId, session.getRetryCount());
}
@Override
public UploadProgressResponse getUploadProgress(String uploadId) {
FileUploadSession session = sessionRepository.findById(uploadId)
.orElseThrow(() -> new UploadSessionNotFoundException(uploadId));
return UploadProgressResponse.fromSession(session);
}
@Override
@Transactional
public void cleanupExpiredSessions() {
log.info("Starting cleanup of expired upload sessions");
LocalDateTime expiryThreshold = LocalDateTime.now().minusHours(sessionExpiryHours);
try {
var expiredSessions = sessionRepository.findByStatusInAndCreatedAtBefore(
Set.of(UploadStatus.INITIATED, UploadStatus.UPLOADING, UploadStatus.FAILED),
expiryThreshold
);
int cleanedCount = 0;
for (FileUploadSession session : expiredSessions) {
session.setStatus(UploadStatus.FAILED);
session.setLastError("Session expired during cleanup");
sessionRepository.save(session);
CompletableFuture.runAsync(() -> cleanupSessionFiles(session));
cleanedCount++;
}
log.info("Cleaned up {} expired upload sessions", cleanedCount);
} catch (Exception e) {
log.error("Error during session cleanup", e);
}
}
private Path getChunkDirectory(Long userId, String uploadId) { private Path getChunkDirectory(Long userId, String uploadId) {
return storageRoot.resolve("temp") return storageRoot.resolve("temp")
@@ -0,0 +1,35 @@
package ru.soune.nocopy.service.file;
import lombok.RequiredArgsConstructor;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.scheduling.annotation.Scheduled;
import org.springframework.stereotype.Service;
import org.springframework.transaction.annotation.Transactional;
import ru.soune.nocopy.entity.file.FileUploadSession;
import ru.soune.nocopy.entity.file.UploadStatus;
import ru.soune.nocopy.repository.FileUploadSessionRepository;
import java.time.LocalDateTime;
import java.util.List;
import java.util.Set;
@Service
@RequiredArgsConstructor
public class UploadSessionCleanupService {
@Autowired
private FileUploadSessionRepository fileUploadSessionRepository;
@Autowired
private FileUploadService fileUploadService;
@Transactional
@Scheduled(fixedDelay = 30000)
public void cleanupSessions() {
List<FileUploadSession> expiredSessions = fileUploadSessionRepository.findExpiredSessions(
LocalDateTime.now(), Set.of(UploadStatus.INITIATED, UploadStatus.UPLOADING));
for (FileUploadSession session : expiredSessions) {
fileUploadService.handleExpiredSession(session);
}
}
}
+3 -1
View File
@@ -24,7 +24,9 @@ spring:
file: file:
storage: storage:
base-path: ${FILE_STORAGE_PATH:/data/uploads} base-path: ${FILE_STORAGE_PATH:/data/uploads}
chunk-size: ${FILE_CHUNK_SIZE:5242880} # 5MB # chunk-size: ${FILE_CHUNK_SIZE:5242880} # 5MB
# chunk-size: ${FILE_CHUNK_SIZE:1048576} # 1MB
chunk-size: ${FILE_CHUNK_SIZE:1000000} # 1MB
max-file-size: ${MAX_FILE_SIZE:10737418240} # 10GB max-file-size: ${MAX_FILE_SIZE:10737418240} # 10GB
max-retry-attempts: ${MAX_RETRY_ATTEMPTS:3} max-retry-attempts: ${MAX_RETRY_ATTEMPTS:3}
chunk-timeout-ms: ${CHUNK_TIMEOUT_MS:300000} # 5 минут chunk-timeout-ms: ${CHUNK_TIMEOUT_MS:300000} # 5 минут