62 Commits
Author SHA1 Message Date
vladp 3d679440f6 NCBACK-34 add logic for clear does not exist files,add dto for similar file search controller
Test Workflow / test (push) Successful in 4s
2026-01-16 14:30:45 +07:00
vladp 4a45f40e9f NCBACK-34 need fix delete filentity and hash
Test Workflow / test (push) Successful in 4s
2026-01-15 20:41:35 +07:00
vladp bbb0d7b17a Merge branch 'dev' into NCBACK-34 2026-01-15 16:55:47 +07:00
vladp 0ed3f958ff DEV_DB change db creds
Test Workflow / test (push) Successful in 7s
2026-01-15 16:52:33 +07:00
vladp de810e32b4 DEV_DB change db creds
Test Workflow / test (push) Successful in 4s
2026-01-15 16:42:37 +07:00
vladp 1a6823f7a9 DEV_DB change db creds
Test Workflow / test (push) Successful in 4s
2026-01-15 16:17:03 +07:00
vladp 82a57a2983 NCBACK-32 add all case for json
Test Workflow / test (push) Successful in 4s
2026-01-15 11:09:05 +07:00
vladp d3e50aa8d1 NCBACK-19 add all case for json
Test Workflow / test (push) Successful in 3s
2026-01-14 19:10:22 +07:00
vladp 5c6309e228 NCBACK-33 fix check company name
Test Workflow / test (push) Successful in 3s
2026-01-14 18:47:34 +07:00
vladp 8ac73ff045 NCBACK-34 add logic to find similar image in db
Test Workflow / test (push) Successful in 3s
2026-01-13 22:40:52 +07:00
vladp 344ec075d4 Dev
Test Workflow / test (push) Successful in 2s
2026-01-10 01:19:30 +07:00
vladp 7b8546fd4d Dev
Test Workflow / test (push) Successful in 2s
2026-01-10 01:06:32 +07:00
vladp eb12e219ec Dev
Test Workflow / test (push) Successful in 2s
2026-01-10 00:54:49 +07:00
vladp 920cde3cdf Dev 2026-01-10 00:51:03 +07:00
backdev 5b4dca23fb Merge pull request 'NCBACK-3' (#14) from NCBACK-3 into dev
Test Workflow / test (push) Successful in 3s
Reviewed-on: #14
2026-01-10 01:43:36 +08:00
vladp 7b1ed5fe67 NCBACK-3
Test Workflow / test (push) Successful in 3s
2026-01-10 00:35:29 +07:00
backdev 8102d72982 Merge pull request 'NCBACK-3 Add found by yandex image' (#10) from NCBACK-3 into dev
Test Workflow / test (push) Successful in 10s
Reviewed-on: #10
2026-01-10 01:03:33 +08:00
vladp b3a3d57ee3 NCBACK-3
Test Workflow / test (push) Successful in 3s
2026-01-09 23:41:19 +07:00
vladp d896f23872 add user_files_info for avaibale actions
Test Workflow / test (push) Successful in 2s
2026-01-09 23:05:37 +07:00
vladp edd508e5d8 Merge branch 'dev' into NCBACK-3
Test Workflow / test (push) Successful in 5s
# Conflicts:
#	docker-compose.yaml
#	src/main/java/ru/soune/nocopy/dto/MessageCode.java
2026-01-09 12:34:24 +07:00
vladp 63baaf7c80 update types
Test Workflow / test (push) Successful in 4s
2026-01-08 12:59:30 +07:00
vladp 3a9a5b48a2 update types
Test Workflow / test (push) Successful in 4s
2026-01-07 14:10:35 +07:00
vladp 6fff5884d2 set chunk
Test Workflow / test (push) Successful in 3s
2026-01-07 13:16:53 +07:00
vladp f8673dd581 set chunk
Test Workflow / test (push) Successful in 3s
2026-01-07 12:50:40 +07:00
vladp b9a401f113 correct file_extensions
Test Workflow / test (push) Successful in 8s
2026-01-07 11:56:03 +07:00
vladp 918b4894fc add endpoint with files user info
Test Workflow / test (push) Successful in 6s
2026-01-06 17:27:38 +07:00
vladp bf4d1ca450 add endpoint with files user info
Test Workflow / test (push) Successful in 4s
2026-01-06 12:50:39 +07:00
vladp bd9ecd4f84 change chunk size
Test Workflow / test (push) Successful in 3s
2025-12-30 12:25:22 +07:00
vladp 0f73c8319e add ram size
Test Workflow / test (push) Successful in 4s
2025-12-30 12:08:23 +07:00
vladp 28b0356b03 fix check status deleted
Test Workflow / test (push) Successful in 3s
2025-12-30 02:19:07 +07:00
vladp 72a4516b7d set correct file types
Test Workflow / test (push) Successful in 4s
2025-12-30 01:05:59 +07:00
vladp 851676b895 set chunk-size to 5
Test Workflow / test (push) Successful in 2s
2025-12-26 20:32:15 +07:00
vladp 254fdc0fbd set chunk-size to 5
Test Workflow / test (push) Successful in 8s
2025-12-26 20:30:38 +07:00
vladp a33c53fa6b set chunk-size to 5
Test Workflow / test (push) Successful in 3s
2025-12-26 20:18:18 +07:00
vladp 86e7371ffe set chunk-size to 1000000
Test Workflow / test (push) Successful in 2s
2025-12-26 19:50:32 +07:00
vladp 9541e89651 add post size
Test Workflow / test (push) Successful in 2s
2025-12-26 19:40:08 +07:00
vladp 291586dbf4 add post size
Test Workflow / test (push) Successful in 2s
2025-12-26 19:29:47 +07:00
vladp 91e9b0c785 add post size
Test Workflow / test (push) Successful in 2s
2025-12-26 19:26:39 +07:00
vladp 4fa6b83a10 add post size
Test Workflow / test (push) Successful in 3s
2025-12-26 19:15:45 +07:00
vladp 2c1a2aeb16 add post size
Test Workflow / test (push) Successful in 3s
2025-12-26 19:10:43 +07:00
vladp 2a9f33068a add logic with hard delete
Test Workflow / test (push) Successful in 2s
2025-12-26 16:23:17 +07:00
vladp 1ebe8b852f add file extension to search
Test Workflow / test (push) Successful in 2s
2025-12-26 14:51:02 +07:00
vladp 84993dec00 fix page_size and filtered values
Test Workflow / test (push) Successful in 3s
2025-12-26 13:52:53 +07:00
vladp a2c20ae645 add exceptions
Test Workflow / test (push) Successful in 2s
2025-12-26 12:22:40 +07:00
vladp 18ec9b51f3 add clean logic for uploadsessions
Test Workflow / test (push) Successful in 3s
2025-12-25 15:27:45 +07:00
vladp 45b7acc774 fix after test
Test Workflow / test (push) Successful in 2s
2025-12-25 13:04:49 +07:00
vladp b2cf06a512 update name controllers ends
Test Workflow / test (push) Successful in 3s
2025-12-24 17:26:13 +07:00
vladp d842af2c62 add file size
Test Workflow / test (push) Waiting to run
2025-12-24 16:55:07 +07:00
vladp c5ef58ddd4 fix after test
Test Workflow / test (push) Successful in 3s
2025-12-24 14:20:49 +07:00
vladp f6d4ebfe0e fix after test
Test Workflow / test (push) Successful in 3s
2025-12-24 13:58:46 +07:00
vladp a77a51f519 delete photo
Test Workflow / test (push) Successful in 1s
2025-12-23 18:33:53 +07:00
vladp 1aef1dcd96 add chunk size
Test Workflow / test (push) Successful in 1s
2025-12-23 16:31:09 +07:00
backdev 7dc32d055b Merge pull request 'NCBACK-26 add docker container with storage' (#11) from NCBACK-26 into dev
Test Workflow / test (push) Successful in 0s
Reviewed-on: #11
2025-12-23 17:10:36 +08:00
vladp 19d8f61e1d NCBACK-26 add docker container with storage
Test Workflow / test (push) Successful in 1s
2025-12-23 16:05:51 +07:00
vladp 8fe4d854a6 add create uploads filedir
Test Workflow / test (push) Successful in 1s
2025-12-23 13:10:36 +07:00
vladp 4c607f5bf9 fix check format
Test Workflow / test (push) Successful in 6s
2025-12-23 12:46:59 +07:00
vladp ce879fea18 add format for all office,os
Test Workflow / test (push) Successful in 1s
2025-12-23 12:03:49 +07:00
vladp 33479ea9af add check extensions
Test Workflow / test (push) Successful in 3s
2025-12-22 17:17:48 +07:00
vladp ffb147d716 NCBACK-3 Add found by yandex image
Test Workflow / test (push) Successful in 4s
2025-12-22 14:42:27 +07:00
vladp 647fd6e1a7 NCBACK-3 Add found by yandex image
Test Workflow / test (push) Successful in 3s
2025-12-22 14:07:44 +07:00
vladp 76e8f4ab1d add image type
Test Workflow / test (push) Successful in 3s
2025-12-22 10:59:36 +07:00
backdev c03ca2566d Merge pull request 'add bone for file system' (#9) from NCBACK-15 into dev
Test Workflow / test (push) Successful in 9s
Reviewed-on: #9
2025-12-21 17:19:34 +08:00
52 changed files with 1717 additions and 665 deletions
+2 -2
View File
@@ -1,6 +1,6 @@
POSTGRES_DB=no_copy_
POSTGRES_USER=postgres
POSTGRES_PASSWORD=postgres
POSTGRES_USER=ncp_db
POSTGRES_PASSWORD=ncpDbApp
POSTGRES_PORT=5432
POSTGRES_HOST=postgres
+2
View File
@@ -12,6 +12,8 @@ WORKDIR /app
COPY --from=build /app/build/libs/*.jar app.jar
RUN mkdir -p /data/uploads && chmod 755 /data/uploads
EXPOSE 8080
CMD ["java", "-jar", "app.jar"]
+48
View File
@@ -0,0 +1,48 @@
суперпользователь БД:
postgres/postgres
пользователь бд для бэка
ncp_db / ncpDbApp
Подключение к БД для приложения,если volume не существует :
Создаться автоматически.
Если уже существует volume :
docker exec -it postgres psql -U postgres -d postgres
CREATE USER ncp_db WITH PASSWORD 'ncpDbApp';
GRANT ALL PRIVILEGES ON DATABASE no_copy_ TO ncp_db;
\du
\q
-------
psql -U postgres -d no_copy_
-- база
GRANT CONNECT ON DATABASE no_copy_ TO ncp_db;
-- схема
GRANT USAGE ON SCHEMA public TO ncp_db;
-- существующие таблицы
GRANT ALL PRIVILEGES ON ALL TABLES IN SCHEMA public TO ncp_db;
-- существующие sequence (очень важно для id)
GRANT ALL PRIVILEGES ON ALL SEQUENCES IN SCHEMA public TO ncp_db;
-- будущие таблицы
ALTER DEFAULT PRIVILEGES IN SCHEMA public
GRANT ALL ON TABLES TO ncp_db;
ALTER DEFAULT PRIVILEGES IN SCHEMA public
GRANT ALL ON SEQUENCES TO ncp_db;
+6
View File
@@ -31,6 +31,12 @@ dependencies {
implementation 'org.mapstruct:mapstruct:1.5.5.Final'
implementation 'commons-validator:commons-validator:1.7'
implementation 'org.flywaydb:flyway-core:9.22.0'
implementation 'com.fasterxml.jackson.dataformat:jackson-dataformat-xml:2.17.0'
implementation 'tools.jackson.core:jackson-core:3.0.3'
implementation 'jakarta.xml.bind:jakarta.xml.bind-api:4.0.0'
annotationProcessor 'org.mapstruct:mapstruct-processor:1.5.5.Final'
annotationProcessor 'org.projectlombok:lombok'
+105 -9
View File
@@ -1,17 +1,46 @@
version: '3.9'
services:
storage:
image: alpine:latest
container_name: file-storage
deploy:
resources:
limits:
memory: 128M
reservations:
memory: 64M
networks:
- app-network
volumes:
- uploads_data:/storage:rw
command: tail -f /dev/null
restart: unless-stopped
db:
image: postgres:17
image: postgres:17.7
restart: always
deploy:
resources:
limits:
cpus: '2.0'
memory: 2G
reservations:
cpus: '1.0'
memory: 1G
environment:
POSTGRES_DB: no_copy_
POSTGRES_USER: postgres
POSTGRES_PASSWORD: postgres
# POSTGRES_USER: postgres
# POSTGRES_PASSWORD: postgres
POSTGRES_USER: ${POSTGRES_USER}
POSTGRES_PASSWORD: ${POSTGRES_PASSWORD}
POSTGRES_SHARED_BUFFERS: 512MB
POSTGRES_EFFECTIVE_CACHE_SIZE: 1536MB
ports:
- "54320:5432"
volumes:
- pgdata:/var/lib/postgresql/data
- ./init-scripts:/docker-entrypoint-initdb.d:ro
container_name: postgres
networks:
app-network:
@@ -21,15 +50,31 @@ services:
app:
build: .
container_name: app-backend
deploy:
resources:
limits:
cpus: '1.5'
memory: 1G
reservations:
cpus: '0.5'
memory: 512M
environment:
FILE_STORAGE_PATH: /data/uploads
MAX_FILE_SIZE: 1073741824
FILE_CHUNK_SIZE: 5242880
MAX_FILE_SIZE: 10737418240
# FILE_CHUNK_SIZE: 1048576
FILE_CHUNK_SIZE: 1000000
POSTGRES_DB: no_copy_
POSTGRES_USER: postgres
POSTGRES_PASSWORD: postgres
# POSTGRES_USER: postgres
# POSTGRES_PASSWORD: postgres
POSTGRES_USER: ${POSTGRES_USER}
POSTGRES_PASSWORD: ${POSTGRES_PASSWORD}
POSTGRES_PORT: 5432
POSTGRES_HOST: db
STORAGE_SERVICE_URL: http://storage:8081
SPRING_PROFILES_ACTIVE: docker
YANDEX_API_KEY: AQVNyaVaUmgUb1GMCtf5zSEqFxy0woXrcMOOB43q
YANDEX_FOLDER_ID: b1gokpdbm6qfpsou8pcd
YANDEX_SEARCH_URL: "https://searchapi.api.cloud.yandex.net/v2/image/search_by_image"
depends_on:
- db
ports:
@@ -41,11 +86,22 @@ services:
- backend
- api
volumes:
- ./uploads:/data/uploads:rw
- uploads_data:/data/uploads:rw
healthcheck:
test: [ "CMD-SHELL", "pg_isready -U ${POSTGRES_USER} -d no_copy_" ]
interval: 10s
retries: 5
grafana:
image: grafana/grafana:10.3.1
container_name: grafana
deploy:
resources:
limits:
cpus: '0.5'
memory: 512M
reservations:
memory: 256M
ports:
- "3000:3000"
depends_on:
@@ -62,6 +118,8 @@ services:
GF_SECURITY_ADMIN_USER: admin
GF_SECURITY_ADMIN_PASSWORD: admin
GF_METRICS_ENABLED: "true"
GF_DATABASE_MAX_IDLE_CONN: "2"
GF_DATABASE_MAX_OPEN_CONN: "10"
volumes:
- ./infrastructure/grafana/provisioning:/etc/grafana/provisioning
- ./infrastructure/grafana/dashboards:/var/lib/grafana/dashboards
@@ -76,10 +134,23 @@ services:
prometheus:
image: prom/prometheus:latest
container_name: prometheus
deploy:
resources:
limits:
cpus: '0.5'
memory: 1G
reservations:
memory: 512M
ports:
- "9090:9090"
volumes:
- ./infrastructure/prometheus/prometheus.yml:/etc/prometheus/prometheus.yml
- prometheus_data:/prometheus
command:
- '--config.file=/etc/prometheus/prometheus.yml'
- '--storage.tsdb.path=/prometheus'
- '--storage.tsdb.retention.time=15d' # Удерживать 15 дней
- '--web.enable-lifecycle'
healthcheck:
test: [ "CMD", "wget", "--spider", "http://localhost:9090/-/healthy" ]
interval: 10s
@@ -91,10 +162,20 @@ services:
loki:
image: grafana/loki:2.9.2
container_name: loki
deploy:
resources:
limits:
cpus: '1.0'
memory: 2G
reservations:
memory: 1G
ports:
- "3100:3100"
volumes:
- ./infrastructure/loki/loki-config.yaml:/etc/loki/loki-config.yaml:ro
- loki_data:/loki
command:
- -config.file=/etc/loki/loki-config.yaml
healthcheck:
test: [ "CMD", "wget", "--spider", "-q", "http://localhost:3100/ready" ]
interval: 10s
@@ -106,6 +187,13 @@ services:
tempo:
image: grafana/tempo:2.4.1
container_name: tempo
deploy:
resources:
limits:
cpus: '0.5'
memory: 512M
reservations:
memory: 256M
command: [ "-config.file=/etc/tempo/tempo.yaml" ]
volumes:
- ./infrastructure/tempo/tempo.yaml:/etc/tempo/tempo.yaml
@@ -118,6 +206,13 @@ services:
alloy:
image: grafana/alloy:latest
container_name: alloy
deploy:
resources:
limits:
cpus: '0.5'
memory: 256M
reservations:
memory: 128M
user: root
ports:
- "9080:9080"
@@ -146,7 +241,8 @@ volumes:
loki_chunks:
loki_index:
loki_rules:
# uploads_volume:
uploads_data:
prometheus_data:
networks:
app-network:
+35 -88
View File
@@ -7,7 +7,6 @@ pipeline {
defaultValue: 'dev',
description: 'Ветка для деплоя'
)
string(
name: 'SERVER',
defaultValue: '92.242.61.23',
@@ -31,6 +30,7 @@ pipeline {
}
}
stage('Deploy with docker-compose') {
steps {
script {
@@ -39,123 +39,70 @@ pipeline {
credentialsId: 'server-root-password',
usernameVariable: 'SSH_USER',
passwordVariable: 'SSH_PASS'
)
),
string(credentialsId: 'DB_USER', variable: 'DB_USER'),
string(credentialsId: 'DB_PASSWORD', variable: 'DB_PASSWORD')
]) {
sh """
echo "Deploying branch: ${params.BRANCH}"
echo "Copying files to server..."
sshpass -p '$SSH_PASS' ssh $SSH_USER@$SERVER "mkdir -p /opt/deployments/${params.BRANCH}"
sshpass -p '$SSH_PASS' scp -r ./* $SSH_USER@$SERVER:/opt/deployments/${params.BRANCH}/
sshpass -p '$SSH_PASS' ssh -o StrictHostKeyChecking=no $SSH_USER@$SERVER "mkdir -p /opt/deployments/${params.BRANCH}"
sshpass -p '$SSH_PASS' scp -r -o StrictHostKeyChecking=no ./* $SSH_USER@$SERVER:/opt/deployments/${params.BRANCH}/
sshpass -p '$SSH_PASS' ssh $SSH_USER@$SERVER "
sshpass -p '$SSH_PASS' ssh -o StrictHostKeyChecking=no $SSH_USER@$SERVER "
cd /opt/deployments/${params.BRANCH}
echo 'Step 1: Checking current state...'
echo '1. Остановка старого приложения...'
docker stop app-backend 2>/dev/null || echo 'Контейнер не найден'
docker rm app-backend 2>/dev/null || echo 'Контейнер не найден'
# Смотрим что запущено
docker ps --filter 'name=postgres|app-backend' --format 'table {{.Names}}\\t{{.Image}}\\t{{.Status}}\\t{{.Ports}}'
echo '2. Удаление старых образов...'
docker images --filter 'reference=*app*' -q | xargs -r docker rmi -f 2>/dev/null || echo 'Нет образов для удаления'
echo 'Step 2: Force cleanup old application...'
echo '3. Создание сети если нужно...'
docker network create app-network 2>/dev/null || echo 'Сеть уже существует'
# Принудительно удаляем старый контейнер и образ
docker stop app-backend 2>/dev/null || echo 'App not running'
docker rm app-backend 2>/dev/null || echo 'App not found'
docker rmi app-backend:latest 2>/dev/null || echo 'Image not found'
echo '4. Запуск инфраструктуры...'
docker-compose up -d db storage
# Удаляем все образы app-backend
docker images --filter 'reference=app-backend*' -q | xargs -r docker rmi 2>/dev/null || echo 'No images to remove'
echo 'Step 3: Verify copied files...'
echo 'Files in directory:'
ls -la
echo ''
echo 'Checking Java sources:'
find . -name '*.java' | head -2
echo 'Step 4: Check if PostgreSQL is running...'
if ! docker ps | grep -q postgres; then
echo 'Starting PostgreSQL...'
docker-compose up -d db
echo '5. Ожидание PostgreSQL...'
sleep 10
else
echo 'PostgreSQL already running'
fi
echo 'Step 5: Build application with forced rebuild...'
echo 'Checking Dockerfile exists:'
ls -la Dockerfile 2>/dev/null || echo 'Dockerfile not found, creating simple one'
echo '6. Сборка нового образа приложения...'
docker build --no-cache -t app-backend:latest .
# Если нет Dockerfile, создаем простой
if [ ! -f Dockerfile ]; then
echo 'Creating simple Dockerfile...'
cat > Dockerfile << 'EOF'
FROM eclipse-temurin:21-jre
WORKDIR /app
COPY . /app/
CMD ["java", "-jar", "app.jar"]
EOF
fi
# Собираем с подробным выводом
echo 'Building Docker image...'
docker build --no-cache --progress=plain -t app-backend:latest . 2>&1 | tail -50
echo 'Step 6: Verify new image...'
echo 'Current app-backend images:'
docker images | grep app-backend
echo 'Image creation time:'
docker inspect app-backend:latest --format='{{.Created}}' 2>/dev/null || echo 'Cannot inspect image'
echo 'Step 7: Starting application with alias...'
echo '7. Запуск приложения...'
docker run -d \\
--name app-backend \\
--network app-network \\
--network-alias app \\
-p 80:8080 \\
-v uploads_data:/data/uploads:rw \\
-e POSTGRES_DB=no_copy_ \\
-e POSTGRES_USER=postgres \\
-e POSTGRES_PASSWORD=postgres \\
-e POSTGRES_USER=$DB_USER \\
-e POSTGRES_PASSWORD=$DB_PASSWORD \\
-e POSTGRES_PORT=5432 \\
-e POSTGRES_HOST=db \\
--restart unless-stopped \\
app-backend:latest
echo 'Step 8: Checking deployment...'
sleep 10
echo '8. Запуск мониторинга...'
docker-compose up -d grafana prometheus loki tempo alloy
echo 'Containers status:'
docker ps --filter 'name=postgres|app-backend' --format 'table {{.Names}}\\t{{.Image}}\\t{{.Status}}\\t{{.CreatedAt}}'
echo 'Checking image inside container:'
docker exec app-backend ls -la /app/ 2>/dev/null || echo 'Cannot check container files'
echo 'Application logs (last 10 lines):'
docker logs --tail=10 app-backend 2>/dev/null || echo 'Logs not available yet'
echo 'Step 9: Health check...'
if docker ps | grep -q app-backend; then
echo 'Container is running'
echo 'Testing application health...'
for i in {1..5}; do
if curl -s -f http://localhost:80/health > /dev/null 2>&1; then
echo 'Health check passed'
echo 'Deployment successful'
echo 'Application URL: http://${params.SERVER}:80'
exit 0
fi
echo 'Waiting for application to start... attempt ' \$i
echo '9. Проверка...'
sleep 5
done
echo 'Health check failed, but container is running'
echo 'Application might be starting slowly'
echo 'Application URL: http://${params.SERVER}:80'
echo 'Статус контейнеров:'
docker ps --format 'table {{.Names}}\\t{{.Image}}\\t{{.Status}}'
echo '10. Проверка health...'
if curl -s -f http://localhost:80/health > /dev/null 2>&1; then
echo 'Приложение работает'
echo 'URL: http://${params.SERVER}:80'
else
echo 'Application failed to start'
docker logs app-backend
exit 1
echo 'Проверка health не удалась'
docker logs app-backend --tail=20
fi
"
"""
+19
View File
@@ -0,0 +1,19 @@
DO $$
BEGIN
IF NOT EXISTS (SELECT FROM pg_roles WHERE rolname = 'ncp_db') THEN
CREATE USER ncp_db WITH PASSWORD 'ncpDbApp';
END IF;
END
$$;
GRANT CONNECT ON DATABASE no_copy_ TO ncp_db;
GRANT USAGE ON SCHEMA public TO ncp_db;
GRANT ALL PRIVILEGES ON ALL TABLES IN SCHEMA public TO ncp_db;
GRANT ALL PRIVILEGES ON ALL SEQUENCES IN SCHEMA public TO ncp_db;
ALTER DEFAULT PRIVILEGES IN SCHEMA public
GRANT ALL ON TABLES TO ncp_db;
ALTER DEFAULT PRIVILEGES IN SCHEMA public
GRANT ALL ON SEQUENCES TO ncp_db;
@@ -2,8 +2,10 @@ package ru.soune.nocopy;
import org.springframework.boot.SpringApplication;
import org.springframework.boot.autoconfigure.SpringBootApplication;
import org.springframework.scheduling.annotation.EnableScheduling;
@SpringBootApplication
@EnableScheduling
public class NoCopyApplication {
public static void main(String[] args) {
@@ -16,7 +16,8 @@ public class HandlerConfig {
LoginRequestHandler login,
FileUploadHandler upload,
FileEntityHandler file,
LogoutRequestHandler logoutHandler
LogoutRequestHandler logoutHandler,
ImageFoundRequestHandler imageFoundRequestHandler
) {
Map<Integer, RequestHandler> map = new HashMap<>();
map.put(20001, login);
@@ -24,6 +25,7 @@ public class HandlerConfig {
map.put(20004, upload);
map.put(20005, file);
map.put(20006, logoutHandler);
map.put(20007, imageFoundRequestHandler);
return map;
}
@@ -35,6 +35,8 @@ public class JacksonConfig {
mapper.registerModule(javaTimeModule);
mapper.disable(SerializationFeature.WRITE_DATES_AS_TIMESTAMPS);
mapper.configure(com.fasterxml.jackson.databind.DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, false);
mapper.enable(SerializationFeature.INDENT_OUTPUT);
return mapper;
}
@@ -4,6 +4,9 @@ import lombok.RequiredArgsConstructor;
import lombok.extern.slf4j.Slf4j;
import org.springframework.core.io.Resource;
import org.springframework.core.io.UrlResource;
import org.springframework.data.domain.Page;
import org.springframework.data.domain.Pageable;
import org.springframework.data.web.PageableDefault;
import org.springframework.http.HttpHeaders;
import org.springframework.http.MediaType;
import org.springframework.http.ResponseEntity;
@@ -15,17 +18,16 @@ import ru.soune.nocopy.dto.BaseRequest;
import ru.soune.nocopy.dto.BaseResponse;
import ru.soune.nocopy.dto.MessageCode;
import ru.soune.nocopy.dto.RegAnswer;
import ru.soune.nocopy.dto.file.ChunkUploadResponse;
import ru.soune.nocopy.dto.file.CompleteUploadResponse;
import ru.soune.nocopy.dto.file.FileEntityResponse;
import ru.soune.nocopy.dto.file.UploadProgress;
import ru.soune.nocopy.dto.file.*;
import ru.soune.nocopy.entity.AuthToken;
import ru.soune.nocopy.entity.file.FileEntity;
import ru.soune.nocopy.entity.file.FileStatus;
import ru.soune.nocopy.entity.file.UploadStatus;
import ru.soune.nocopy.exception.NotFoundAuthToken;
import ru.soune.nocopy.exception.NotValidFieldException;
import ru.soune.nocopy.exception.ValidationException;
import ru.soune.nocopy.exception.*;
import ru.soune.nocopy.handler.*;
import ru.soune.nocopy.repository.AuthTokenRepository;
import ru.soune.nocopy.repository.FileEntityRepository;
import ru.soune.nocopy.service.FileSimilarityService;
import ru.soune.nocopy.service.file.FileEntityService;
import ru.soune.nocopy.service.file.FileUploadService;
@@ -36,6 +38,7 @@ import java.nio.file.Paths;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.Optional;
import java.util.stream.Collectors;
@Slf4j
@@ -51,6 +54,10 @@ public class ApiController {
private final AuthTokenRepository authTokenRepository;
private final FileSimilarityService fileSimilarityService;
private final FileEntityRepository fileEntityRepository;
@PostMapping("/v{version}/data")
public ResponseEntity<?> handlePostRequest(@RequestBody BaseRequest request,
@PathVariable("version") int version) {
@@ -89,55 +96,72 @@ public class ApiController {
@PostMapping("/v{version}/files/chunk")
public ResponseEntity<BaseResponse> uploadChunk(
@PathVariable("version") int version,
@RequestParam("upload_id") String uploadId,
@RequestParam("chunk_number") Integer chunkNumber,
@RequestParam("chunk") MultipartFile chunk) {
log.info("Uploading chunk {} for session {}, file size: {} bytes, version: {}",
chunkNumber, uploadId, chunk.getSize(), version);
if (chunk.isEmpty()) {
return ResponseEntity.ok().body(new BaseResponse(20004, MessageCode.FILE_UPLOAD_ERROR.getCode(),
"Chunk file is empty", null));
}
if (uploadId == null || uploadId.isBlank()) {
return ResponseEntity.ok().body(new BaseResponse(20004, MessageCode.FILE_UPLOAD_ERROR.getCode(),
"Upload ID is required", null));
}
if (chunkNumber == null || chunkNumber < 0) {
return ResponseEntity.ok().body(new BaseResponse(20004, MessageCode.FILE_UPLOAD_ERROR.getCode(),
"Valid chunk number is required", null));
}
@RequestParam(value = "upload_id", required = false) String uploadId,
@RequestParam(value = "chunk_number", required = false) Integer chunkNumber,
@RequestParam(value = "chunk", required = false) MultipartFile chunk) {
try {
fileUploadService.uploadChunk(uploadId, chunkNumber, chunk);
if (chunk == null || chunk.isEmpty()) {
return buildErrorResponse(uploadId, chunkNumber, "Chunk file null or empty");
}
if (uploadId == null || uploadId.isBlank()) {
return buildErrorResponse(uploadId, chunkNumber, "Upload ID is required");
}
if (chunkNumber == null || chunkNumber < 0) {
return buildErrorResponse(uploadId, chunkNumber, "Valid chunk number is required");
}
ChunkUploadResponse responseBody = ChunkUploadResponse.builder()
.uploadId(uploadId)
.chunkNumber(chunkNumber)
.chunkSize(chunk.getSize())
.message("Chunk uploaded successfully")
.build();
UploadProgressResponse uploadProgressResponse = fileUploadService.uploadChunk(uploadId, chunkNumber, chunk);
return ResponseEntity.ok().body(new BaseResponse(20004, MessageCode.SUCCESS.getCode(),
"Chunk uploaded successfully", responseBody));
if (uploadProgressResponse.getStatus().equals(UploadStatus.COMPLETED)) {
ResponseEntity<BaseResponse> duplicateCheckResult = checkForDuplicates(uploadProgressResponse.getUploadId());
if (duplicateCheckResult != null) {
return duplicateCheckResult;
}
}
return buildSuccessResponse(uploadId, chunkNumber, chunk);
} catch (Exception e) {
log.error("Error uploading chunk", e);
ChunkUploadResponse responseBody = ChunkUploadResponse.builder()
.uploadId(uploadId)
.chunkNumber(chunkNumber)
.build();
return ResponseEntity.ok().body(new BaseResponse(
20004, MessageCode.FILE_UPLOAD_ERROR.getCode(),
"Failed to upload chunk: " + e.getMessage(), responseBody));
return buildErrorResponse(uploadId, chunkNumber, "Failed to upload chunk: " + e.getMessage());
}
}
@GetMapping("/v{version}/files/{fileId}/similar")
public ResponseEntity<BaseResponse> findSimilarFiles(
@PathVariable("version") int version,
@PathVariable String fileId,
@RequestParam(required = false) List<String> similarityLevels,
@PageableDefault(size = 20, sort = "hammingDistance") Pageable pageable) {
SimilarityFilter filter = SimilarityFilter.builder()
.similarityLevels(similarityLevels)
.build();
Page<SimilarFileResponse> similarFiles = fileSimilarityService.findSimilarFiles(fileId, filter, pageable);
String messageDesc;
MessageCode success;
if (similarFiles.isEmpty()) {
messageDesc = MessageCode.FILE_NOT_FOUND.getDescription();
success = MessageCode.FILE_NOT_FOUND;
} else {
messageDesc = MessageCode.SIMILAR_FILES_FOUND.getDescription();
success = MessageCode.SIMILAR_FILES_FOUND;
}
Map<String, Object> responseData = new HashMap<>();
responseData.put("content", similarFiles.getContent());
responseData.put("page", similarFiles.getNumber());
responseData.put("size", similarFiles.getSize());
responseData.put("totalElements", similarFiles.getTotalElements());
responseData.put("totalPages", similarFiles.getTotalPages());
responseData.put("hasNext", similarFiles.hasNext());
responseData.put("hasPrevious", similarFiles.hasPrevious());
return ResponseEntity.ok()
.body(new BaseResponse(20004, success.getCode(), messageDesc, responseData));
}
@GetMapping("/v{version}/files/progress/{uploadId}")
public ResponseEntity<BaseResponse> getUploadProgress(
@PathVariable("version") int version,
@@ -161,7 +185,6 @@ public class ApiController {
return ResponseEntity.ok().body(new BaseResponse(
20004, MessageCode.SUCCESS.getCode(), MessageCode.SUCCESS.getDescription(), responseBody));
} catch (Exception e) {
log.error("Error getting progress for upload: {}", uploadId, e);
@@ -169,8 +192,7 @@ public class ApiController {
.uploadId(uploadId)
.build();
return ResponseEntity.ok().body(new BaseResponse(
20004, MessageCode.FILE_UPLOAD_ERROR.getCode(),
return ResponseEntity.ok().body(new BaseResponse(20004, MessageCode.FILE_UPLOAD_ERROR.getCode(),
"Failed to get upload progress: " + e.getMessage(), responseBody));
}
}
@@ -237,29 +259,67 @@ public class ApiController {
@GetMapping("/v{version}/files/download/{fileId}")
public ResponseEntity<Resource> downloadFile(
@PathVariable String fileId,
@PathVariable Integer version,
@RequestHeader("Authorization") String tokenHeader) {
public ResponseEntity<?> downloadFile(
@PathVariable(required = false) String fileId,
@PathVariable(required = false) Integer version,
@RequestHeader(value = "Authorization", required = false) String tokenHeader) {
try {
if (tokenHeader == null) {
Map<String, Object> errorData = new HashMap<>();
errorData.put("token", tokenHeader);
errorData.put("fileId", fileId);
errorData.put("version", version);
return ResponseEntity.ok().body(new BaseResponse(20004,
MessageCode.FILE_DOWNLOAD_ERROR_NOT_CORRECT_FIELD.getCode(),
MessageCode.FILE_DOWNLOAD_ERROR_NOT_CORRECT_FIELD.getDescription(),
errorData));
}
Long userId = getUserIdFromToken(tokenHeader);
FileEntityResponse fileInfo = fileEntityService.getById(fileId, version);
FileEntityResponse entityResponse = fileEntityService.getById(fileId, version);
if (!fileInfo.getUserId().equals(userId)) {
return ResponseEntity.status(403).build();
if (!entityResponse.getUserId().equals(userId)) {
Map<String, Object> errorData = new HashMap<>();
errorData.put("token", tokenHeader);
return ResponseEntity.ok().body(new BaseResponse(20004,
MessageCode.FILE_DOWNLOAD_ERROR.getCode(),
MessageCode.FILE_DOWNLOAD_ERROR.getDescription(),
errorData));
}
if (!fileInfo.isExistsOnDisk()) {
return ResponseEntity.status(404)
.body(null);
if (entityResponse.getStatus().equals(FileStatus.DELETED)) {
Map<String, Object> errorData = new HashMap<>();
errorData.put("file_status", entityResponse.getStatus());
return ResponseEntity.ok().body(new BaseResponse(20004,
MessageCode.FILE_DOWNLOAD_ERROR.getCode(),
MessageCode.FILE_DOWNLOAD_ERROR.getDescription(),
errorData));
}
Path filePath = Paths.get(fileInfo.getFilePath());
if (!entityResponse.isExistsOnDisk()) {
Map<String, Object> errorData = new HashMap<>();
errorData.put("onDisk", entityResponse.isExistsOnDisk());
return ResponseEntity.ok().body(new BaseResponse(20004,
MessageCode.FILE_DOWNLOAD_ERROR.getCode(),
MessageCode.FILE_DOWNLOAD_ERROR.getDescription(),
errorData));
}
Path filePath = Paths.get(entityResponse.getFilePath());
Resource resource = new UrlResource(filePath.toUri());
if (!resource.exists()) {
return ResponseEntity.status(404).build();
Map<String, Object> errorData = new HashMap<>();
errorData.put("resource", resource.exists());
return ResponseEntity.ok().body(new BaseResponse(20004,
MessageCode.FILE_DOWNLOAD_ERROR.getCode(),
MessageCode.FILE_DOWNLOAD_ERROR.getDescription(),
errorData));
}
String contentType = determineContentType(filePath);
@@ -267,84 +327,101 @@ public class ApiController {
return ResponseEntity.ok()
.contentType(MediaType.parseMediaType(contentType))
.header(HttpHeaders.CONTENT_DISPOSITION,
"attachment; filename=\"" + fileInfo.getOriginalFileName() + "\"")
.header(HttpHeaders.CONTENT_LENGTH, String.valueOf(fileInfo.getFileSize()))
"attachment; filename=\"" + entityResponse.getOriginalFileName() + "\"")
.header(HttpHeaders.CONTENT_LENGTH, String.valueOf(entityResponse.getFileSize()))
.body(resource);
} catch (FileEntityNotFoundException e) {
Map<String, Object> errorData = new HashMap<>();
errorData.put("fileId", fileId);
} catch (NotFoundAuthToken e) {
return ResponseEntity.status(401).build();
} catch (Exception e) {
log.error("Error downloading file", e);
return ResponseEntity.status(500).build();
return ResponseEntity.ok().body(new BaseResponse(20004,
MessageCode.FILE_NOT_FOUND.getCode(),
MessageCode.FILE_NOT_FOUND.getDescription(),
errorData));
} catch (NotFoundAuthToken | IOException e) {
Map<String, Object> errorData = new HashMap<>();
errorData.put("token", tokenHeader);
errorData.put("fileId", fileId);
errorData.put("version", version);
return ResponseEntity.ok().body(new BaseResponse(20004,
MessageCode.FILE_DOWNLOAD_ERROR_NOT_CORRECT_FIELD.getCode(),
MessageCode.FILE_DOWNLOAD_ERROR_NOT_CORRECT_FIELD.getDescription(),
errorData));
}
}
@GetMapping("/v{version}/files/info/{fileId}")
public ResponseEntity<FileEntityResponse> getFileInfo(
@PathVariable String fileId,
@PathVariable Integer version,
@RequestHeader("Authorization") String tokenHeader) {
private ResponseEntity<BaseResponse> checkForDuplicates(String uploadId) throws IOException {
Optional<FileEntity> uploadedFile = fileEntityRepository.findByUploadSessionId(uploadId);
try {
Long userId = getUserIdFromToken(tokenHeader);
FileEntityResponse fileInfo = fileEntityService.getById(fileId, version);
if (!fileInfo.getUserId().equals(userId)) {
return ResponseEntity.status(403).build();
if (uploadedFile.isEmpty() || !uploadedFile.get().getMimeType().equals("image")) {
return null;
}
return ResponseEntity.ok(fileInfo);
FileEntity fileEntity = uploadedFile.get();
List<SimilarFileResponse> similarFiles = fileSimilarityService.findSimilarFiles(fileEntity.getId());
} catch (NotFoundAuthToken e) {
return ResponseEntity.status(401).build();
} catch (Exception e) {
log.error("Error getting file info", e);
return ResponseEntity.status(500).build();
}
if (hasDuplicate(similarFiles)) {
return handleDuplicate(fileEntity, similarFiles);
}
@GetMapping("/v{version}/files/preview/{fileId}")
public ResponseEntity<Resource> previewFile(
@PathVariable String fileId,
@PathVariable Integer version,
@RequestHeader("Authorization") String tokenHeader) {
try {
Long userId = getUserIdFromToken(tokenHeader);
FileEntityResponse fileInfo = fileEntityService.getById(fileId, version);
if (!fileInfo.getUserId().equals(userId)) {
return ResponseEntity.status(403).build();
return null;
}
if (!isPreviewSupported(fileInfo.getMimeType())) {
return ResponseEntity.status(415)
.body(null);
private boolean hasDuplicate(List<SimilarFileResponse> similarFiles) {
return similarFiles.stream().anyMatch(f -> f.getHammingDistance() <= 5);
}
Path filePath = Paths.get(fileInfo.getFilePath());
Resource resource = new UrlResource(filePath.toUri());
private ResponseEntity<BaseResponse> handleDuplicate(FileEntity fileEntity, List<SimilarFileResponse> similarFiles)
throws IOException {
fileEntityService.deleteFromDisk(fileEntity);
if (!resource.exists()) {
return ResponseEntity.status(404).build();
fileEntityService.markAsDeleted(fileEntity);
Optional<FileEntity> originalFile = fileEntityRepository.findById(similarFiles.get(0).getFileId());
if (originalFile.isPresent()) {
Map<String, String> duplicateInfo = Map.of(
"duplicate_file_id", originalFile.get().getId(),
"owner_user_id", String.valueOf(originalFile.get().getUserId()));
return ResponseEntity.ok().body(new BaseResponse(
20004,
MessageCode.DUPLICATE_FILE_UPLOAD.getCode(),
"Failed to upload chunk, duplicate",
duplicateInfo));
}
String contentType = determineContentType(filePath);
return ResponseEntity.ok()
.contentType(MediaType.parseMediaType(contentType))
.header(HttpHeaders.CONTENT_DISPOSITION,
"inline; filename=\"" + fileInfo.getOriginalFileName() + "\"")
.body(resource);
} catch (NotFoundAuthToken e) {
return ResponseEntity.status(401).build();
} catch (Exception e) {
log.error("Error previewing file", e);
return ResponseEntity.status(500).build();
}
return null;
}
private ResponseEntity<BaseResponse> buildSuccessResponse(String uploadId, Integer chunkNumber, MultipartFile chunk) {
ChunkUploadResponse responseBody = ChunkUploadResponse.builder()
.uploadId(uploadId)
.chunkNumber(chunkNumber)
.chunkSize(chunk.getSize())
.message("Chunk uploaded successfully")
.build();
return ResponseEntity.ok().body(new BaseResponse(
20000,
MessageCode.SUCCESS.getCode(),
"Chunk uploaded successfully",
responseBody));
}
private ResponseEntity<BaseResponse> buildErrorResponse(String uploadId, Integer chunkNumber, String errorMessage) {
ChunkUploadResponse responseBody = ChunkUploadResponse.builder()
.uploadId(uploadId)
.chunkNumber(chunkNumber)
.build();
return ResponseEntity.ok().body(new BaseResponse(
20004,
MessageCode.FILE_UPLOAD_ERROR.getCode(),
errorMessage,
responseBody));
}
private ResponseEntity<BaseResponse> createValidationErrorResponse(BindingResult bindingResult, Integer msgId) {
List<Map<String, String>> fieldErrors = bindingResult.getFieldErrors()
@@ -382,18 +459,6 @@ public class ApiController {
return contentType;
}
private boolean isPreviewSupported(String mimeType) {
if (mimeType == null) {
return false;
}
return mimeType.startsWith("image/") ||
mimeType.startsWith("text/") ||
mimeType.equals("application/pdf") ||
mimeType.startsWith("video/") ||
mimeType.startsWith("audio/");
}
private Long getUserIdFromToken(String tokenHeader) {
String token = tokenHeader.replace("Bearer ", "");
AuthToken authToken = authTokenRepository.findByToken(token)
@@ -1,9 +1,11 @@
package ru.soune.nocopy.dto;
import com.fasterxml.jackson.annotation.JsonFormat;
import com.fasterxml.jackson.annotation.JsonProperty;
import lombok.Data;
@Data
@JsonFormat(with = JsonFormat.Feature.ACCEPT_CASE_INSENSITIVE_PROPERTIES)
public class BaseRequest {
@JsonProperty("version")
Integer version;
@@ -1,11 +1,13 @@
package ru.soune.nocopy.dto;
import com.fasterxml.jackson.annotation.JsonFormat;
import com.fasterxml.jackson.annotation.JsonProperty;
import lombok.AllArgsConstructor;
import lombok.Data;
@Data
@AllArgsConstructor
@JsonFormat(with = JsonFormat.Feature.ACCEPT_CASE_INSENSITIVE_PROPERTIES)
public class BaseResponse {
@JsonProperty("msg_id")
private Integer msgId;
@@ -1,5 +1,6 @@
package ru.soune.nocopy.dto;
import com.fasterxml.jackson.annotation.JsonFormat;
import jakarta.validation.constraints.Email;
import jakarta.validation.constraints.NotBlank;
import jakarta.validation.constraints.Size;
@@ -10,6 +11,7 @@ import lombok.NoArgsConstructor;
@Data
@AllArgsConstructor
@NoArgsConstructor
@JsonFormat(with = JsonFormat.Feature.ACCEPT_CASE_INSENSITIVE_PROPERTIES)
public class LoginRequest {
@NotBlank(message = "error.not.blank") @Email(message = "error.not.email") @Size(max = 128)
@@ -8,6 +8,10 @@ public enum MessageCode {
INVALID_TOKEN(2, "Invalid token"),
INVALID_ACTION(2, "Invalid action"),
FILE_UPLOAD_ERROR(2, "File upload error"),
DUPLICATE_FILE_UPLOAD(2, "Duplicate file upload"),
FILE_DOWNLOAD_ERROR(2, "File download error"),
FILE_DOWNLOAD_ERROR_NOT_CORRECT_FIELD(2, "Not correct field"),
IMAGE_FOUND_ERROR(2, "Image found error"),
INVALID_JSON_BODY(2, "Invalid fields in JSON object"),
INCOMPLETE_UPLOAD(2, "Not load all chunks"),
MSG_ID_NOT_FOUND(4, "Message id not found"),
@@ -17,7 +21,8 @@ public enum MessageCode {
AUTH_EMAIL_OR_TOKEN_NOT_FOUND(4, "Email or Token not found "),
AUTH_TOKEN_MISMATCH(4, "Token mismatch"),
FILE_NOT_FOUND(4, "File not found"),
AUTH_PASSWORD_NOT_MATCHES(2, "Password does not match");
AUTH_PASSWORD_NOT_MATCHES(2, "Password does not match"),
SIMILAR_FILES_FOUND(0, "Similar files found");
private final Integer code;
@@ -1,10 +1,12 @@
package ru.soune.nocopy.dto;
import com.fasterxml.jackson.annotation.JsonFormat;
import jakarta.validation.constraints.*;
import lombok.Data;
@Data
@JsonFormat(with = JsonFormat.Feature.ACCEPT_CASE_INSENSITIVE_PROPERTIES)
public class RegRequest {
@NotEmpty(message = "Full name is required")
private String fullName;
@@ -10,6 +10,7 @@ import ru.soune.nocopy.entity.GenderType;
import java.time.LocalDate;
@Data
@JsonFormat(with = JsonFormat.Feature.ACCEPT_CASE_INSENSITIVE_PROPERTIES)
public class UserRequest {
@Size(min = 2)
private String fullName;
@@ -0,0 +1,20 @@
package ru.soune.nocopy.dto.file;
import com.fasterxml.jackson.annotation.JsonProperty;
import lombok.Builder;
import lombok.Data;
import java.util.List;
@Data
@Builder
public class FileExtensionResponse {
@JsonProperty("file_extension")
private List<String> extension;
@JsonProperty("count")
private Integer count;
@JsonProperty("max_file_size")
private Long maxFileSize;
}
@@ -0,0 +1,61 @@
package ru.soune.nocopy.dto.file;
import com.fasterxml.jackson.annotation.JsonProperty;
import lombok.AllArgsConstructor;
import lombok.Builder;
import lombok.Data;
import lombok.NoArgsConstructor;
@Data
@Builder
@NoArgsConstructor
@AllArgsConstructor
public class FileInfoUserResponse {
@JsonProperty("all_files_size")
private Long allFileSize;
@JsonProperty("all_files_quantity")
private Integer fileCount;
@JsonProperty("all_files_check")
private Integer filesCheck;
@JsonProperty("all_files_violation")
private Integer filesViolation;
@JsonProperty("images_size")
private Long imagesSize;
@JsonProperty("images_quantity")
private Integer imagesCount;
@JsonProperty("images_check")
private Integer imagesCheck;
@JsonProperty("images_violations")
private Integer imagesViolations;
@JsonProperty("videos_size")
private Long videosSize;
@JsonProperty("videos_quantity")
private Integer videosCount;
@JsonProperty("videos_check")
private Integer videosCheck;
@JsonProperty("videos_violations")
private Integer videosViolations;
@JsonProperty("audios_size")
private Long audiosSize;
@JsonProperty("audios_quantity")
private Integer audiosCount;
@JsonProperty("audios_check")
private Integer audiosCheck;
@JsonProperty("audios_violations")
private Integer audiosViolations;
}
@@ -0,0 +1,10 @@
package ru.soune.nocopy.dto.file;
import com.fasterxml.jackson.annotation.JsonProperty;
import lombok.Data;
@Data
public class ImageSearchRequest {
@JsonProperty("file_id")
private String fileId;
}
@@ -0,0 +1,14 @@
package ru.soune.nocopy.dto.file;
import lombok.Builder;
import lombok.Value;
@Value
@Builder
public class SimilarFileResponse {
String fileId;
String originalFileName;
Long fileSize;
Integer hammingDistance;
String similarityLevel;
}
@@ -0,0 +1,18 @@
package ru.soune.nocopy.dto.file;
import lombok.AllArgsConstructor;
import lombok.Builder;
import lombok.Data;
import lombok.NoArgsConstructor;
import java.util.List;
@Data
@Builder
@NoArgsConstructor
@AllArgsConstructor
public class SimilarityFilter {
private List<String> similarityLevels;
private Integer page;
private Integer size;
}
@@ -0,0 +1,36 @@
package ru.soune.nocopy.dto.file;
import com.fasterxml.jackson.annotation.JsonIgnoreProperties;
import com.fasterxml.jackson.annotation.JsonProperty;
import lombok.Data;
import java.util.List;
@Data
@JsonIgnoreProperties(ignoreUnknown = true)
public class YandexSearchResponse {
@JsonProperty("images")
private List<ImageResult> images;
@Data
@JsonIgnoreProperties(ignoreUnknown = true)
public static class ImageResult {
@JsonProperty("url")
private String url;
@JsonProperty("pageUrl")
private String pageUrl;
@JsonProperty("pageTitle")
private String pageTitle;
@JsonProperty("width")
private Integer width;
@JsonProperty("height")
private Integer height;
@JsonProperty("host")
private String host;
}
}
@@ -1,5 +1,6 @@
package ru.soune.nocopy.entity.file;
import com.fasterxml.jackson.annotation.JsonIgnore;
import jakarta.persistence.*;
import lombok.*;
import org.springframework.data.annotation.CreatedDate;
@@ -60,6 +61,10 @@ public class FileEntity {
@Column(name = "updated_at")
private LocalDateTime updatedAt;
@OneToOne(mappedBy = "file", cascade = CascadeType.ALL, orphanRemoval = true)
@JsonIgnore
private ImageHashEntity imageHash;
@PrePersist
public void prePersist() {
if (this.status == null) {
@@ -4,5 +4,7 @@ public enum FileStatus {
ACTIVE,
DELETED,
PROCESSING,
VIOLATION,
CHECKED,
ERROR
}
@@ -2,20 +2,33 @@ package ru.soune.nocopy.entity.file;
import lombok.Getter;
import java.util.Arrays;
import java.util.List;
@Getter
public enum FileType {
PHOTO("photo"),
VIDEO("video"),
AUDIO("audio"),
DOCUMENT("document");
IMAGE("image", Arrays.asList("jpg", "jpeg", "png", "gif", "bmp", "webp", "jfif")),
VIDEO("video", Arrays.asList("mp4", "avi", "mov", "wmv", "flv", "mkv", "webm", "m4v", "mpg", "mpeg",
"3gp", "3g2", "f4v", "m2ts", "mts", "vob", "ogv", "divx")),
AUDIO("audio", Arrays.asList("mp3", "wav", "flac"));
private final String code;
private final String displayName;
private final List<String> allowedExtensions;
FileType(String code) {
this.code = code;
FileType(String displayName, List<String> allowedExtensions) {
this.displayName = displayName;
this.allowedExtensions = allowedExtensions;
}
public String getCode() {
return code;
public String getDisplayName() {
return displayName;
}
public List<String> getAllowedExtensions() {
return allowedExtensions;
}
public boolean supportsExtension(String extension) {
return allowedExtensions.contains(extension.toLowerCase());
}
}
@@ -69,6 +69,7 @@ public class FileUploadSession {
private String extension;
@Column(name = "retry_count")
@Builder.Default
private Integer retryCount = 0;
@Column(name = "completed_at")
@@ -81,12 +82,13 @@ public class FileUploadSession {
)
@MapKeyColumn(name = "chunk_number")
@Column(name = "chunk_path")
@Builder.Default
private Map<Integer, String> chunkPaths = new HashMap<>();
@PrePersist
public void prePersist() {
this.createdAt = LocalDateTime.now();
this.expiresAt = LocalDateTime.now().plusHours(24);
this.expiresAt = LocalDateTime.now().plusMinutes(1);
if (this.chunksUploaded == null) {
this.chunksUploaded = 0;
}
@@ -0,0 +1,39 @@
package ru.soune.nocopy.entity.file;
import jakarta.persistence.*;
import lombok.AllArgsConstructor;
import lombok.Builder;
import lombok.Data;
import lombok.NoArgsConstructor;
import java.time.LocalDateTime;
@Entity
@Table(name = "image_hashes")
@Data
@NoArgsConstructor
@AllArgsConstructor
@Builder
public class ImageHashEntity {
@Id
@Column(name = "file_id")
private String fileId;
@OneToOne(fetch = FetchType.LAZY)
@MapsId
@JoinColumn(name = "file_id")
private FileEntity file;
@Column(name = "hash64_hi")
private Integer hash64Hi;
@Column(name = "hash64_lo")
private Integer hash64Lo;
@Column(name = "hash_algorithm", nullable = false)
private String hashAlgorithm;
@Column(name = "created_at")
private LocalDateTime createdAt;
}
@@ -2,6 +2,6 @@ package ru.soune.nocopy.exception;
public class ChunkSizeExceededException extends RuntimeException {
public ChunkSizeExceededException(long actualSize, long maxSize) {
super(String.format("Chunk size exceeded: %d > %d", actualSize, maxSize));
super(String.format("Chunk size not valid: " + actualSize));
}
}
@@ -0,0 +1,7 @@
package ru.soune.nocopy.exception;
public class DuplicateImageException extends RuntimeException {
public DuplicateImageException(String message) {
super(message);
}
}
@@ -7,11 +7,17 @@ import org.springframework.stereotype.Component;
import ru.soune.nocopy.dto.*;
import ru.soune.nocopy.dto.file.*;
import ru.soune.nocopy.entity.AuthToken;
import ru.soune.nocopy.entity.file.FileEntity;
import ru.soune.nocopy.entity.file.FileStatus;
import ru.soune.nocopy.exception.FileEntityNotFoundException;
import ru.soune.nocopy.exception.NotFoundAuthToken;
import ru.soune.nocopy.repository.AuthTokenRepository;
import ru.soune.nocopy.repository.FileEntityRepository;
import ru.soune.nocopy.service.file.FileEntityService;
import ru.soune.nocopy.service.file.FileStatsService;
import java.util.Arrays;
import java.util.List;
@Slf4j
@Component
@@ -19,9 +25,15 @@ import java.util.Arrays;
public class FileEntityHandler implements RequestHandler {
private final FileEntityService fileEntityService;
private final FileStatsService fileStatsService;
private final AuthTokenRepository authTokenRepository;
private final ObjectMapper objectMapper;
private final FileEntityRepository fileEntityRepository;
@Override
public BaseResponse handle(BaseRequest request) {
try {
@@ -31,15 +43,17 @@ public class FileEntityHandler implements RequestHandler {
String action = fileRequest.getAction();
switch (action) {
case "get_file_info":
case "file_info":
return handleGetFileInfo(request, fileRequest);
case "get_file_by_session":
case "user_files_info":
return handleGetFilesUserInfo(request, fileRequest);
case "file_by_session":
return handleGetFileBySession(request, fileRequest);
case "get_user_files":
case "user_files":
return handleGetUserFiles(request, fileRequest);
case "search_files":
return handleSearchFiles(request, fileRequest);
case "get_storage_usage":
case "storage_usage":
return handleGetStorageUsage(request, fileRequest);
case "delete_file":
return handleDeleteFile(request, fileRequest);
@@ -47,8 +61,9 @@ public class FileEntityHandler implements RequestHandler {
ActionResponse response = ActionResponse.builder()
.action(action)
.availableActions(Arrays.asList(
"get_file_info", "get_file_by_session", "get_user_files",
"search_files", "get_storage_usage", "delete_file"))
"file_info", "file_by_session", "user_files",
"search_files", "storage_usage", "delete_file",
"user_files_info"))
.build();
return new BaseResponse(request.getMsgId(),
@@ -65,6 +80,16 @@ public class FileEntityHandler implements RequestHandler {
}
}
private BaseResponse handleGetFilesUserInfo(BaseRequest request, FileEntityRequest fileRequest) {
Long userId = getUserIdFromToken(fileRequest.getToken());
FileInfoUserResponse userFileStats = fileStatsService.getUserFileStats(userId);
return new BaseResponse(request.getMsgId(),
MessageCode.SUCCESS.getCode(),
MessageCode.SUCCESS.getDescription(),
userFileStats);
}
private BaseResponse handleGetFileInfo(BaseRequest request, FileEntityRequest fileRequest) {
try {
Long userId = getUserIdFromToken(fileRequest.getToken());
@@ -168,17 +193,22 @@ public class FileEntityHandler implements RequestHandler {
FileResponse allFiles = fileEntityService.getUserFiles(userId, 1, 1000, request.getVersion());
var filteredFiles = allFiles.getFiles().stream()
.filter(f -> f.getOriginalFileName().toLowerCase().contains(fileRequest.getQuery().toLowerCase()))
.filter(f -> (f.getOriginalFileName() + f.getFileExtension()).toLowerCase()
.contains(fileRequest.getQuery().toLowerCase()))
.toList();
int start = (page - 1) * pageSize;
int end = Math.min(start + pageSize, filteredFiles.size());
List<FileEntityResponse> fileEntityResponses = filteredFiles.subList(start, Math.min(end, filteredFiles.size()));
long sumSize = fileEntityResponses.stream()
.mapToLong(FileEntityResponse::getFileSize)
.sum();
FileListResponse response = FileListResponse.builder()
.files(filteredFiles.subList(start, Math.min(end, filteredFiles.size())))
.files(fileEntityResponses)
.totalCount(filteredFiles.size())
.totalSize(allFiles.getTotalSize())
.formattedTotalSize(allFiles.getFormattedTotalSize())
.totalSize(sumSize)
.formattedTotalSize(fileEntityService.formatFileSize(sumSize))
.page(page)
.pageSize(pageSize)
.build();
@@ -187,18 +217,13 @@ public class FileEntityHandler implements RequestHandler {
MessageCode.SUCCESS.getCode(),
MessageCode.SUCCESS.getDescription(),
response);
} catch (NotFoundAuthToken e) {
return new BaseResponse(request.getMsgId(),
MessageCode.INVALID_TOKEN.getCode(),
"Authentication required",
null);
return new BaseResponse(request.getMsgId(), MessageCode.INVALID_TOKEN.getCode(),
"Authentication required", null);
} catch (Exception e) {
log.error("Error searching files", e);
return new BaseResponse(request.getMsgId(),
MessageCode.FILE_UPLOAD_ERROR.getCode(),
"Failed to search files: " + e.getMessage(),
null);
return new BaseResponse(request.getMsgId(), MessageCode.FILE_UPLOAD_ERROR.getCode(),
"Failed to search files: " + e.getMessage(), null);
}
}
@@ -235,7 +260,9 @@ public class FileEntityHandler implements RequestHandler {
private BaseResponse handleDeleteFile(BaseRequest request, FileEntityRequest fileRequest) {
try {
Long userId = getUserIdFromToken(fileRequest.getToken());
FileEntityResponse fileInfo = fileEntityService.getById(fileRequest.getFileId(), request.getVersion());
String fileId = fileRequest.getFileId();
FileEntityResponse fileInfo = fileEntityService.getById(fileId, request.getVersion());
DeleteFileResponse response;
if (!fileInfo.getUserId().equals(userId)) {
return new BaseResponse(request.getMsgId(),
@@ -244,18 +271,27 @@ public class FileEntityHandler implements RequestHandler {
null);
}
fileEntityService.markAsDeleted(fileRequest.getFileId());
FileEntity fileEntity = fileEntityRepository.findById(fileId)
.orElseThrow(() -> new FileEntityNotFoundException(fileId));
DeleteFileResponse response = DeleteFileResponse.builder()
if (fileEntity.getStatus().equals(FileStatus.DELETED)) {
fileEntityService.deleteFromDisk(fileEntity);
response = DeleteFileResponse.builder()
.fileId(fileRequest.getFileId())
.message("File deleted from disk")
.build();
} else {
fileEntityService.markAsDeleted(fileEntity);
response = DeleteFileResponse.builder()
.fileId(fileRequest.getFileId())
.message("File marked as deleted")
.build();
}
return new BaseResponse(request.getMsgId(),
MessageCode.SUCCESS.getCode(),
"File marked as deleted",
response);
} catch (NotFoundAuthToken e) {
return new BaseResponse(request.getMsgId(),
MessageCode.INVALID_TOKEN.getCode(),
@@ -3,9 +3,11 @@ package ru.soune.nocopy.handler;
import com.fasterxml.jackson.databind.ObjectMapper;
import lombok.RequiredArgsConstructor;
import lombok.extern.slf4j.Slf4j;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.stereotype.Component;
import org.springframework.validation.BeanPropertyBindingResult;
import org.springframework.validation.BindingResult;
import org.springframework.validation.FieldError;
import ru.soune.nocopy.dto.*;
import ru.soune.nocopy.dto.file.*;
import ru.soune.nocopy.entity.AuthToken;
@@ -17,21 +19,26 @@ import ru.soune.nocopy.repository.AuthTokenRepository;
import ru.soune.nocopy.repository.FileUploadSessionRepository;
import ru.soune.nocopy.service.file.FileUploadService;
import java.util.Arrays;
import java.util.HashMap;
import java.util.Map;
import java.util.Optional;
import java.util.*;
import java.util.stream.Collectors;
@Slf4j
@Component
@RequiredArgsConstructor
public class FileUploadHandler implements RequestHandler {
private final FileUploadService fileUploadService;
private final AuthTokenRepository authTokenRepository;
private final FileUploadSessionRepository fileUploadSessionRepository;
private final ObjectMapper objectMapper;
private final FileUploadRequestValidator fileUploadRequestValidator;
@Value("${file.storage.max-file-size}")
private long maxFileSize;
@Override
public BaseResponse handle(BaseRequest request) {
try {
@@ -45,18 +52,18 @@ public class FileUploadHandler implements RequestHandler {
return handleInitUpload(request, fileRequest);
case "file_types":
return handleGetFileTypes(request);
case "file_extension":
return handleGetFileExtensions(fileRequest, request);
case "chunks":
return handleGetChunkStatus(request, fileRequest);
case "cancel":
return handleCancelUpload(request, fileRequest);
case "progress":
return handleGetProgress(request, fileRequest);
case "retry":
return handleRetryUpload(request, fileRequest);
default:
ActionResponse response = ActionResponse.builder()
.action(action)
.availableActions(Arrays.asList("init", "file_types", "chunks", "cancel", "progress", "retry"))
.availableActions(Arrays.asList("init", "file_types", "chunks", "cancel", "progress"))
.build();
return new BaseResponse(request.getMsgId(), MessageCode.INVALID_ACTION.getCode(),
@@ -70,18 +77,31 @@ public class FileUploadHandler implements RequestHandler {
}
private BaseResponse handleInitUpload(BaseRequest request, FileUploadRequest fileRequest) {
try {
String token = fileRequest.getToken();
Optional<AuthToken> tokenOptional = authTokenRepository.findByToken(token);
if (tokenOptional.isEmpty()) {
return new BaseResponse(request.getMsgId(), MessageCode.INVALID_TOKEN.getCode(),
MessageCode.INVALID_TOKEN.getDescription(), Map.of("token", token));
}
AuthToken authToken = tokenOptional.orElseThrow(() -> new NotFoundAuthToken("Token not found"));
BindingResult bindingResult = new BeanPropertyBindingResult(fileRequest, "fileRequest");
fileUploadRequestValidator.validate(fileRequest, bindingResult);
if (bindingResult.hasErrors()) {
throw new ValidationException(bindingResult, request.getMsgId());
Map<String, String> fieldErrors = bindingResult.getFieldErrors()
.stream()
.collect(Collectors.toMap(
FieldError::getField,
fieldError -> fieldError.getDefaultMessage() != null
? fieldError.getDefaultMessage()
: "Validation error"));
return new BaseResponse(request.getMsgId(), MessageCode.INVALID_FIELD.getCode(),
MessageCode.INVALID_FIELD.getDescription(), fieldErrors);
}
FileUploadSession session = fileUploadService.initUpload(
@@ -101,15 +121,6 @@ public class FileUploadHandler implements RequestHandler {
return new BaseResponse(request.getMsgId(), MessageCode.SUCCESS.getCode(),
MessageCode.SUCCESS.getDescription(), fileResponse);
} catch (NotFoundAuthToken e) {
InitFileResponse initFileResponse = InitFileResponse.builder()
.build();
throw new NotValidFieldException("Invalid or expired token: " + fileRequest.getToken(),
new BaseResponse(request.getMsgId(), MessageCode.INVALID_TOKEN.getCode(),
MessageCode.INVALID_TOKEN.getDescription(), initFileResponse));
}
}
private BaseResponse handleGetProgress(BaseRequest request, FileUploadRequest fileRequest) {
@@ -198,6 +209,7 @@ public class FileUploadHandler implements RequestHandler {
.orElseThrow(() -> new UploadSessionNotFoundException(fileRequest.getUploadId()));
Map<String, Boolean> chunkStatus = new HashMap<>();
for (int i = 0; i < session.getTotalChunks(); i++) {
chunkStatus.put("chunk_" + i, session.getChunkPaths().containsKey(i));
}
@@ -212,7 +224,6 @@ public class FileUploadHandler implements RequestHandler {
return new BaseResponse(request.getMsgId(), MessageCode.SUCCESS.getCode(),
MessageCode.SUCCESS.getDescription(), response);
} catch (UploadSessionNotFoundException e) {
return new BaseResponse(request.getMsgId(), MessageCode.FILE_NOT_FOUND.getCode(),
"Upload session not found", null);
@@ -240,4 +251,24 @@ public class FileUploadHandler implements RequestHandler {
"Failed to get file types", null);
}
}
private BaseResponse handleGetFileExtensions(FileUploadRequest fileRequest, BaseRequest request) {
try {
FileType fileType = FileType.valueOf(fileRequest.getFileType().toUpperCase());
List<String> allowedExtensions = fileType.getAllowedExtensions();
FileExtensionResponse response = FileExtensionResponse.builder()
.extension(allowedExtensions)
.count(allowedExtensions.size())
.maxFileSize(maxFileSize)
.build();
return new BaseResponse(request.getMsgId(), MessageCode.SUCCESS.getCode(),
MessageCode.SUCCESS.getDescription(), response);
} catch (Exception e) {
log.error("Error getting file extensions", e);
return new BaseResponse(request.getMsgId(), MessageCode.FILE_UPLOAD_ERROR.getCode(),
"Failed to get file extensions", null);
}
}
}
@@ -5,15 +5,16 @@ import org.springframework.context.support.DefaultMessageSourceResolvable;
import org.springframework.http.HttpStatus;
import org.springframework.http.ResponseEntity;
import org.springframework.web.bind.MethodArgumentNotValidException;
import org.springframework.web.bind.annotation.ExceptionHandler;
import org.springframework.web.bind.annotation.ResponseStatus;
import org.springframework.web.bind.annotation.RestControllerAdvice;
import org.springframework.web.bind.annotation.*;
import ru.soune.nocopy.dto.BaseResponse;
import ru.soune.nocopy.dto.MessageCode;
import ru.soune.nocopy.exception.*;
import java.util.Map;
@RestControllerAdvice
@AllArgsConstructor
@ControllerAdvice
public class GlobalExceptionHandler {
@ExceptionHandler(MethodArgumentNotValidException.class)
@@ -81,4 +82,12 @@ public class GlobalExceptionHandler {
"message" ,ex.getMessage()
));
}
@ExceptionHandler(DuplicateImageException.class)
@ResponseBody
public ResponseEntity<BaseResponse> handleDuplicateImage(DuplicateImageException e) {
return ResponseEntity.ok().body(new BaseResponse(
20004, MessageCode.DUPLICATE_FILE_UPLOAD.getCode(),
"Duplicate image detected", e.getMessage()));
}
}
@@ -0,0 +1,36 @@
package ru.soune.nocopy.handler;
import com.fasterxml.jackson.databind.ObjectMapper;
import lombok.RequiredArgsConstructor;
import lombok.extern.slf4j.Slf4j;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.stereotype.Component;
import ru.soune.nocopy.dto.BaseRequest;
import ru.soune.nocopy.dto.BaseResponse;
import ru.soune.nocopy.dto.MessageCode;
import ru.soune.nocopy.dto.file.ImageSearchRequest;
import ru.soune.nocopy.dto.file.YandexSearchResponse;
import ru.soune.nocopy.service.YandexSearchService;
@Slf4j
@Component
@RequiredArgsConstructor
public class ImageFoundRequestHandler implements RequestHandler {
private final ObjectMapper objectMapper;
private final YandexSearchService yandexSearchService;
@Override
public BaseResponse handle(BaseRequest request) throws Exception {
ImageSearchRequest imageSearchRequest = objectMapper.convertValue(request.getMessageBody(),
ImageSearchRequest.class);
String fileId = imageSearchRequest.getFileId();
YandexSearchResponse response = yandexSearchService.searchByFileEntity(fileId);
return new BaseResponse(request.getMsgId(), MessageCode.SUCCESS.getCode(),
MessageCode.SUCCESS.getDescription(), response);
}
}
@@ -7,9 +7,12 @@ import ru.soune.nocopy.dto.file.FileUploadRequest;
import ru.soune.nocopy.entity.file.FileType;
import java.util.Arrays;
import java.util.List;
@Component
public class FileUploadRequestValidator implements Validator {
private final List<FileType> supportedFileTypes = Arrays.asList(FileType.values());
@Override
public boolean supports(Class<?> clazz) {
return FileUploadRequest.class.isAssignableFrom(clazz);
@@ -21,7 +24,7 @@ public class FileUploadRequestValidator implements Validator {
validateType(request.getFileType(), errors);
validateFileName(request.getFileName(), errors);
validateExtension(request.getExtension(), errors);
validateExtension(request.getExtension(), request.getFileType(), errors);
}
private void validateType(String fileType, Errors errors) {
@@ -31,7 +34,12 @@ public class FileUploadRequestValidator implements Validator {
}
try {
FileType.valueOf(fileType.toUpperCase());
FileType parsedType = FileType.valueOf(fileType.toUpperCase());
if (!supportedFileTypes.contains(parsedType)) {
errors.rejectValue("fileType", "fileType.unsupported",
"Unsupported file type. Valid types: " + Arrays.toString(FileType.values()));
}
} catch (IllegalArgumentException e) {
errors.rejectValue("fileType", "fileType.invalid",
"Invalid file type. Valid types: " + Arrays.toString(FileType.values()));
@@ -50,10 +58,34 @@ public class FileUploadRequestValidator implements Validator {
}
}
private void validateExtension(String extension, Errors errors) {
private void validateExtension(String extension, String fileType, Errors errors) {
if (extension == null || extension.isBlank()) {
errors.rejectValue("extension", "extension.required", "Extension is required");
errors.rejectValue("fileType", "fileType.required", "File type is required");
return;
}
if (extension.contains(".")) {
errors.rejectValue("extension", "extension.required", "Extension contains comma");
return;
}
try {
FileType parsedType = FileType.valueOf(fileType.toUpperCase());
if (!parsedType.supportsExtension(extension)) {
errors.rejectValue("fileName", "fileType.extension.mismatch",
String.format("File extension '%s' does not match file type '%s'. Allowed extensions for %s: %s",
extension, parsedType.getDisplayName(), parsedType.getDisplayName(),
parsedType.getAllowedExtensions()));
}
} catch (IllegalArgumentException e) {
errors.rejectValue("fileType", "fileType.invalid",
"Invalid file type. Valid types: " + Arrays.toString(FileType.values()));
}
}
private String getFileExtension(String fileName) {
return fileName.substring(fileName.lastIndexOf('.') + 1).toLowerCase();
}
}
@@ -13,7 +13,7 @@ import java.util.*;
@Component
public class RegRequestValidator implements Validator {
private static final String COMPANY_REGEX = "^[a-zA-Zа-яА-ЯёЁ0-9\\s\\-&.,'()]{0,200}$";
private static final String COMPANY_REGEX = "^[a-zA-Zа-яА-ЯёЁ0-9\\s\\-&.,'()\"\"«»„‟”“”‘'’‹›❛❜❝❞〝〞〟]{1,200}$";
private static final String NAME_REGEX = "^[a-zA-Zа-яА-ЯёЁ\\s\\-'.]{2,100}$";
@@ -74,21 +74,6 @@ public class RegRequestValidator implements Validator {
errors.rejectValue("companyName", "companyName.invalid.chars",
"Company name contains invalid characters");
}
validateForbiddenWords(trimmedCompany, "companyName", errors,
Arrays.asList("admin", "root", "system", "test"));
}
private void validateForbiddenWords(String text, String fieldName,
Errors errors, List<String> forbiddenWords) {
String lowerText = text.toLowerCase();
for (String word : forbiddenWords) {
if (lowerText.contains(word)) {
errors.rejectValue(fieldName, fieldName + ".forbidden.word",
"Contains forbidden word: " + word);
break;
}
}
}
private void validateEmail(String email, Errors errors) {
@@ -14,6 +14,7 @@ import java.util.Optional;
public interface FileEntityRepository extends JpaRepository<FileEntity, String> {
List<FileEntity> findByUserId(Long userId);
Optional<FileEntity> findByUserIdAndChecksum(Long userId, String imageHash);
List<FileEntity> findByUserIdAndStatus(Long userId, FileStatus status);
@@ -29,5 +30,8 @@ public interface FileEntityRepository extends JpaRepository<FileEntity, String>
@Query("SELECT f FROM FileEntity f WHERE f.userId = :userId AND f.originalFileName LIKE %:keyword%")
List<FileEntity> searchByFileName(@Param("userId") Long userId, @Param("keyword") String keyword);
@Query("SELECT f FROM FileEntity f WHERE f.status = :status")
List<FileEntity> searchFileEntityByStatus(@Param("status") FileStatus status);
long countByUserId(Long userId);
}
@@ -0,0 +1,10 @@
package ru.soune.nocopy.repository;
import org.springframework.data.jpa.repository.JpaRepository;
import org.springframework.stereotype.Repository;
import ru.soune.nocopy.entity.file.ImageHashEntity;
@Repository
public interface ImageHashRepository extends JpaRepository<ImageHashEntity, String> {
void deleteByFileId(String fileId);
}
@@ -0,0 +1,35 @@
package ru.soune.nocopy.repository;
import org.springframework.data.jpa.repository.JpaRepository;
import org.springframework.data.jpa.repository.Query;
import org.springframework.data.repository.query.Param;
import org.springframework.stereotype.Repository;
import ru.soune.nocopy.entity.file.FileEntity;
import java.util.List;
@Repository
public interface ImageSimilarityRepository
extends JpaRepository<FileEntity, String> {
@Query(value = """
SELECT
f.id AS similarFileId,
f.original_file_name AS originalFileName,
f.file_size AS fileSize,
h.hash64_hi AS hash64Hi,
h.hash64_lo AS hash64Lo
FROM image_hashes ref
JOIN image_hashes h
ON ref.file_id <> h.file_id
JOIN file_entities f
ON f.id = h.file_id
WHERE ref.file_id = :fileId
""",
nativeQuery = true)
List<SimilarImageProjection> findCandidates(
@Param("fileId") String fileId
);
}
@@ -0,0 +1,14 @@
package ru.soune.nocopy.repository;
public interface SimilarImageProjection {
String getSimilarFileId();
String getOriginalFileName();
Long getFileSize();
Integer getHash64Hi();
Integer getHash64Lo();
}
@@ -0,0 +1,124 @@
package ru.soune.nocopy.service;
import lombok.RequiredArgsConstructor;
import org.springframework.data.domain.Page;
import org.springframework.data.domain.PageImpl;
import org.springframework.data.domain.Pageable;
import org.springframework.stereotype.Service;
import ru.soune.nocopy.dto.file.SimilarFileResponse;
import ru.soune.nocopy.dto.file.SimilarityFilter;
import ru.soune.nocopy.repository.ImageHashRepository;
import ru.soune.nocopy.repository.ImageSimilarityRepository;
import ru.soune.nocopy.repository.SimilarImageProjection;
import java.util.Comparator;
import java.util.List;
import java.util.stream.Collectors;
@Service
@RequiredArgsConstructor
public class FileSimilarityService {
private final ImageSimilarityRepository repository;
private final ImageHashRepository hashRepository;
public List<SimilarFileResponse> findSimilarFiles(String fileId) {
var imageHashEntity = hashRepository.findById(fileId)
.orElseThrow(() -> new RuntimeException("Hash not found"));
Integer hash64Hi = imageHashEntity.getHash64Hi();
Integer hash64Lo = imageHashEntity.getHash64Lo();
List<SimilarImageProjection> candidates =
repository.findCandidates(fileId);
return candidates.stream()
.map(c -> {
Integer cHi = c.getHash64Hi();
Integer cLo = c.getHash64Lo();
int hamming = hamming64(hash64Hi, hash64Lo, cHi, cLo);
String level;
if (hamming <= 5) {
level = "DUPLICATE";
} else if (hamming <= 12) {
level = "SIMILAR";
} else {
level = "DIFFERENT";
}
return SimilarFileResponse.builder()
.fileId(c.getSimilarFileId())
.originalFileName(c.getOriginalFileName())
.fileSize(c.getFileSize())
.hammingDistance(hamming)
.similarityLevel(level)
.build();
})
.sorted((a, b) ->
Integer.compare(a.getHammingDistance(), b.getHammingDistance()))
.toList();
}
public Page<SimilarFileResponse> findSimilarFiles(String fileId, SimilarityFilter filter, Pageable pageable) {
var imageHashEntity = hashRepository.findById(fileId)
.orElseThrow(() -> new RuntimeException("Hash not found"));
Integer hash64Hi = imageHashEntity.getHash64Hi();
Integer hash64Lo = imageHashEntity.getHash64Lo();
List<SimilarImageProjection> candidates = repository.findCandidates(fileId);
List<String> similarityLevels = (filter != null && filter.getSimilarityLevels() != null)
? filter.getSimilarityLevels()
: List.of("DUPLICATE", "SIMILAR", "DIFFERENT");
List<SimilarFileResponse> allResults = candidates.stream()
.map(c -> createSimilarFileResponse(c, hash64Hi, hash64Lo))
.filter(response -> similarityLevels.contains(response.getSimilarityLevel()))
.sorted(Comparator.comparingInt(SimilarFileResponse::getHammingDistance))
.collect(Collectors.toList());
int total = allResults.size();
int page = (pageable != null) ? pageable.getPageNumber() : 0;
int size = (pageable != null) ? pageable.getPageSize() : 20;
int fromIndex = Math.min(page * size, total);
int toIndex = Math.min(fromIndex + size, total);
List<SimilarFileResponse> pageContent = allResults.subList(fromIndex, toIndex);
return new PageImpl<>(pageContent, pageable, total);
}
private SimilarFileResponse createSimilarFileResponse(SimilarImageProjection similarImageProjection,
Integer hash64Hi, Integer hash64Lo) {
Integer imageProjectionHash64Hi = similarImageProjection.getHash64Hi();
Integer similarImageProjectionHash64Lo = similarImageProjection.getHash64Lo();
int hamming = hamming64(hash64Hi, hash64Lo, imageProjectionHash64Hi, similarImageProjectionHash64Lo);
String level;
if (hamming <= 5) {
level = "DUPLICATE";
} else if (hamming <= 12) {
level = "SIMILAR";
} else {
level = "DIFFERENT";
}
return SimilarFileResponse.builder()
.fileId(similarImageProjection.getSimilarFileId())
.originalFileName(similarImageProjection.getOriginalFileName())
.fileSize(similarImageProjection.getFileSize())
.hammingDistance(hamming)
.similarityLevel(level)
.build();
}
private int hamming64(int aHi, int aLo, int bHi, int bLo) {
return Integer.bitCount(aHi ^ bHi)
+ Integer.bitCount(aLo ^ bLo);
}
}
@@ -0,0 +1,154 @@
package ru.soune.nocopy.service;
import lombok.RequiredArgsConstructor;
import lombok.extern.slf4j.Slf4j;
import org.springframework.stereotype.Service;
import ru.soune.nocopy.entity.file.FileEntity;
import ru.soune.nocopy.entity.file.ImageHashEntity;
import ru.soune.nocopy.repository.ImageHashRepository;
import javax.imageio.ImageIO;
import java.awt.*;
import java.awt.image.BufferedImage;
import java.io.IOException;
import java.io.InputStream;
import java.nio.file.Files;
import java.nio.file.Path;
import java.time.LocalDateTime;
import java.util.Arrays;
@Service
@RequiredArgsConstructor
@Slf4j
public class ImageHashService {
private final ImageHashRepository repository;
public void create(FileEntity file, Path imagePath) {
try {
long hash64 = computePhash64(imagePath);
int hi = high32(hash64);
int lo = low32(hash64);
ImageHashEntity entity = ImageHashEntity.builder()
.file(file)
.hash64Hi(hi)
.hash64Lo(lo)
.hashAlgorithm("PHASH64")
.createdAt(LocalDateTime.now())
.build();
repository.save(entity);
} catch (IOException e) {
throw new RuntimeException("Failed to compute image hash", e);
}
}
private long computePhash64(Path path) throws IOException {
try (InputStream is = Files.newInputStream(path)) {
BufferedImage image = ImageIO.read(is);
if (image == null) throw new IOException("Cannot read image file: " + path);
BufferedImage gray = toGrayScale(image);
BufferedImage resized = resizeImage(gray, 32, 32);
double[][] dct = applyDCT(resized);
double[][] topLeft = extractTopLeft8x8(dct);
double median = calculateMedian(topLeft, true);
return create64BitHashLong(topLeft, median);
}
}
private BufferedImage toGrayScale(BufferedImage image) {
BufferedImage gray = new BufferedImage(image.getWidth(), image.getHeight(), BufferedImage.TYPE_BYTE_GRAY);
Graphics2D g = gray.createGraphics();
g.drawImage(image, 0, 0, null);
g.dispose();
return gray;
}
private BufferedImage resizeImage(BufferedImage image, int width, int height) {
BufferedImage resized = new BufferedImage(width, height, BufferedImage.TYPE_BYTE_GRAY);
Graphics2D g = resized.createGraphics();
g.setRenderingHint(RenderingHints.KEY_INTERPOLATION, RenderingHints.VALUE_INTERPOLATION_BILINEAR);
g.drawImage(image, 0, 0, width, height, null);
g.dispose();
return resized;
}
private double[][] applyDCT(BufferedImage image) {
int w = image.getWidth();
int h = image.getHeight();
double[][] pixels = new double[h][w];
for (int y = 0; y < h; y++) {
for (int x = 0; x < w; x++) {
int gray = image.getRGB(x, y) & 0xFF;
pixels[y][x] = gray;
}
}
double[][] dct = new double[h][w];
for (int u = 0; u < h; u++) {
double au = u == 0 ? 1.0 / Math.sqrt(h) : Math.sqrt(2.0 / h);
for (int v = 0; v < w; v++) {
double av = v == 0 ? 1.0 / Math.sqrt(w) : Math.sqrt(2.0 / w);
double sum = 0.0;
for (int x = 0; x < h; x++) {
for (int y = 0; y < w; y++) {
sum += pixels[x][y] *
Math.cos((2 * x + 1) * u * Math.PI / (2.0 * h)) *
Math.cos((2 * y + 1) * v * Math.PI / (2.0 * w));
}
}
dct[u][v] = au * av * sum;
}
}
return dct;
}
private double[][] extractTopLeft8x8(double[][] dct) {
double[][] block = new double[8][8];
for (int i = 0; i < 8; i++) System.arraycopy(dct[i], 0, block[i], 0, 8);
return block;
}
private double calculateMedian(double[][] block, boolean excludeDC) {
double[] vals = new double[63];
int idx = 0;
for (int i = 0; i < 8; i++) {
for (int j = 0; j < 8; j++) {
if (i == 0 && j == 0 && excludeDC) continue;
vals[idx++] = block[i][j];
}
}
Arrays.sort(vals);
return (vals.length % 2 == 0) ?
(vals[vals.length / 2 - 1] + vals[vals.length / 2]) / 2.0 :
vals[vals.length / 2];
}
private long create64BitHashLong(double[][] block, double median) {
long hash = 0L;
for (int i = 0; i < 8; i++) {
for (int j = 0; j < 8; j++) {
if (i == 0 && j == 0) continue;
hash <<= 1;
if (block[i][j] > median) hash |= 1;
}
}
return hash;
}
private int high32(long hash64) {
return (int) (hash64 >>> 32);
}
private int low32(long hash64) {
return (int) hash64;
}
}
@@ -0,0 +1,162 @@
package ru.soune.nocopy.service;
import com.fasterxml.jackson.core.JsonProcessingException;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.fasterxml.jackson.databind.DeserializationFeature;
import jakarta.annotation.PostConstruct;
import lombok.RequiredArgsConstructor;
import lombok.extern.slf4j.Slf4j;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.stereotype.Service;
import ru.soune.nocopy.dto.BaseResponse;
import ru.soune.nocopy.dto.MessageCode;
import ru.soune.nocopy.dto.file.YandexSearchResponse;
import ru.soune.nocopy.entity.file.FileEntity;
import ru.soune.nocopy.exception.FileEntityNotFoundException;
import ru.soune.nocopy.exception.NotValidFieldException;
import ru.soune.nocopy.repository.FileEntityRepository;
import java.io.*;
import java.net.HttpURLConnection;
import java.net.URL;
import java.nio.file.Files;
import java.nio.file.Path;
import java.util.Base64;
import java.util.Map;
@Slf4j
@Service
@RequiredArgsConstructor
public class YandexSearchService {
private final FileEntityRepository fileEntityRepository;
private final ObjectMapper objectMapper;
@Value("${yandex.api-key}")
private String apiKey;
@Value("${yandex.folder-id}")
private String folderId;
@Value("${yandex.search-url}")
private String searchUrl;
@PostConstruct
public void init() {
objectMapper.configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, false);
objectMapper.configure(DeserializationFeature.ACCEPT_SINGLE_VALUE_AS_ARRAY, true);
}
public YandexSearchResponse searchByFileEntity(String fileId) throws IOException {
FileEntity fileEntity = fileEntityRepository.findById(fileId)
.orElseThrow(() -> {
throw new NotValidFieldException("File not found", new BaseResponse(20007,
MessageCode.FILE_NOT_FOUND.getCode(), MessageCode.FILE_NOT_FOUND.getDescription(),
Map.of("fileId",fileId)));
});
byte[] fileBytes;
if (!isImageFile(fileEntity)) {
log.error("File not image: {}", fileEntity.getMimeType());
throw new NotValidFieldException("File not image", new BaseResponse(20007,
MessageCode.INVALID_FIELD.getCode(), MessageCode.INVALID_FIELD.getDescription(),
Map.of("file_type", fileEntity.getMimeType())));
}
try {
fileBytes = readFileFromDisk(fileEntity);
} catch (IOException e) {
throw new NotValidFieldException("File not found or cannot read file", new BaseResponse(20007,
MessageCode.FILE_NOT_FOUND.getCode(), MessageCode.FILE_NOT_FOUND.getDescription(),
Map.of("fileId", fileId,
"filePath", fileEntity.getFilePath())));
}
return callYandexApi(fileBytes);
}
private boolean isImageFile(FileEntity fileEntity) {
String mimeType = fileEntity.getMimeType();
return mimeType != null && mimeType.startsWith("image");
}
private byte[] readFileFromDisk(FileEntity fileEntity) throws IOException {
Path filePath = Path.of(fileEntity.getFilePath());
if (!Files.exists(filePath)) {
throw new IOException("File not found: " + fileEntity.getFilePath());
}
if (!Files.isReadable(filePath)) {
throw new IOException("Cannot read file: " + fileEntity.getFilePath());
}
return Files.readAllBytes(filePath);
}
private YandexSearchResponse callYandexApi(byte[] imageBytes) throws IOException {
String imageBase64 = Base64.getEncoder().encodeToString(imageBytes);
String jsonRequest = String.format("{\"folderId\":\"%s\",\"data\":\"%s\",\"page\":0,\"showSimilarImages\":true}",
folderId, imageBase64);
URL url = new URL(searchUrl);
HttpURLConnection connection = (HttpURLConnection) url.openConnection();
try {
connection.setRequestMethod("POST");
connection.setRequestProperty("Authorization", "Api-Key " + apiKey);
connection.setRequestProperty("Content-Type", "application/json");
connection.setRequestProperty("Accept", "application/json");
connection.setConnectTimeout(30000);
connection.setReadTimeout(30000);
connection.setDoOutput(true);
try (OutputStream os = connection.getOutputStream()) {
byte[] input = jsonRequest.getBytes("utf-8");
os.write(input, 0, input.length);
os.flush();
}
int responseCode = connection.getResponseCode();
String responseBody;
if (responseCode == HttpURLConnection.HTTP_OK) {
try (BufferedReader br = new BufferedReader(
new InputStreamReader(connection.getInputStream(), "utf-8"))) {
responseBody = readAll(br);
}
} else {
try (BufferedReader br = new BufferedReader(
new InputStreamReader(connection.getErrorStream(), "utf-8"))) {
responseBody = readAll(br);
}
throw new IOException("Error Yandex API: " + responseCode + " - " + responseBody);
}
return parseJsonResponse(responseBody);
} finally {
connection.disconnect();
}
}
private String readAll(BufferedReader reader) throws IOException {
StringBuilder sb = new StringBuilder();
String line;
while ((line = reader.readLine()) != null) {
sb.append(line);
}
return sb.toString();
}
private YandexSearchResponse parseJsonResponse(String json) {
YandexSearchResponse response = null;
try {
response = objectMapper.readValue(json, YandexSearchResponse.class);
} catch (JsonProcessingException e) {
throw new RuntimeException(e);
}
return response;
}
}
@@ -11,6 +11,8 @@ import ru.soune.nocopy.entity.file.FileStatus;
import ru.soune.nocopy.entity.file.FileUploadSession;
import ru.soune.nocopy.exception.FileEntityNotFoundException;
import ru.soune.nocopy.repository.FileEntityRepository;
import ru.soune.nocopy.service.FileSimilarityService;
import ru.soune.nocopy.service.ImageHashService;
import java.io.IOException;
import java.nio.file.Files;
@@ -27,6 +29,10 @@ public class FileEntityService {
private final FileEntityRepository fileEntityRepository;
private final ImageHashService imageHashService;
private final FileSimilarityService fileSimilarityService;
@Transactional
public FileEntity createFromUploadSession(FileUploadSession session, String checksum) {
log.info("Creating FileEntity for upload session: {}", session.getUploadId());
@@ -40,7 +46,6 @@ public class FileEntityService {
long fileSize = Files.size(filePath);
String originalName = session.getFileName();
String extension = extractFileExtension(originalName);
String storedName = filePath.getFileName().toString();
FileEntity fileEntity = FileEntity.builder()
@@ -50,18 +55,19 @@ public class FileEntityService {
.filePath(session.getFilePath())
.fileSize(fileSize)
.mimeType(session.getFileType())
.fileExtension(extension)
.fileExtension(session.getExtension())
.checksum(checksum)
.uploadSessionId(session.getUploadId())
.status(FileStatus.ACTIVE)
.build();
FileEntity saved = fileEntityRepository.save(fileEntity);
log.info("FileEntity created successfully: {} (size: {} bytes)",
saved.getId(), fileSize);
if (saved.getMimeType().startsWith("image")) {
imageHashService.create(saved, Path.of(saved.getFilePath()));
}
return saved;
} catch (IOException e) {
log.error("Failed to create FileEntity for session {}: {}",
session.getUploadId(), e.getMessage(), e);
@@ -69,15 +75,6 @@ public class FileEntityService {
}
}
private String extractFileExtension(String fileName) {
int dotIndex = fileName.lastIndexOf('.');
if (dotIndex > 0 && dotIndex < fileName.length() - 1) {
return fileName.substring(dotIndex + 1).toLowerCase();
}
return "";
}
@Transactional(readOnly = true)
public FileEntityResponse getById(String fileId, int version) {
FileEntity fileEntity = fileEntityRepository.findById(fileId)
@@ -95,39 +92,6 @@ public class FileEntityService {
return convertToResponse(fileEntity, version);
}
@Transactional(readOnly = true)
public FileEntityResponse getByFilePath(String filePath, int version) {
FileEntity fileEntity = fileEntityRepository.findByFilePath(filePath)
.orElseThrow(() -> new FileEntityNotFoundException("Path: " + filePath));
return convertToResponse(fileEntity, version);
}
@Transactional(readOnly = true)
public FileResponse getAllUserFiles(Long userId, int version) {
List<FileEntity> fileEntities = fileEntityRepository.findByUserIdAndStatus(
userId, FileStatus.ACTIVE);
List<FileEntityResponse> files = fileEntities.stream()
.map(file -> convertToResponse(file, version))
.collect(Collectors.toList());
long totalSize = fileEntities.stream()
.mapToLong(FileEntity::getFileSize)
.sum();
return FileResponse.builder()
.files(files)
.totalCount(files.size())
.totalSize(totalSize)
.formattedTotalSize(formatFileSize(totalSize))
.page(1)
.pageSize(files.size())
.build();
}
@Transactional(readOnly = true)
public FileResponse getUserFiles(Long userId, int page, int pageSize, int version) {
List<FileEntity> allFiles = fileEntityRepository.findByUserIdAndStatus(
@@ -168,15 +132,26 @@ public class FileEntityService {
}
@Transactional
public void markAsDeleted(String fileId) {
FileEntity fileEntity = fileEntityRepository.findById(fileId)
.orElseThrow(() -> new FileEntityNotFoundException(fileId));
public void markAsDeleted(FileEntity fileEntity) {
fileEntity.setStatus(FileStatus.DELETED);
fileEntity.setUpdatedAt(LocalDateTime.now());
fileEntityRepository.save(fileEntity);
log.info("FileEntity marked as deleted: {}", fileId);
fileEntityRepository.save(fileEntity);
}
@Transactional
public boolean deleteFromDisk(FileEntity fileEntity) throws IOException {
Path path = Paths.get(fileEntity.getFilePath());
if (!Files.exists(path)) {
return true;
}
Files.delete(path);
markAsDeleted(fileEntity);
return true;
}
@Transactional(readOnly = true)
@@ -217,7 +192,7 @@ public class FileEntityService {
.build();
}
private String formatFileSize(long size) {
public String formatFileSize(long size) {
if (size < 1024) {
return size + " B";
} else if (size < 1024 * 1024) {
@@ -0,0 +1,93 @@
package ru.soune.nocopy.service.file;
import lombok.RequiredArgsConstructor;
import org.springframework.stereotype.Service;
import ru.soune.nocopy.dto.file.FileInfoUserResponse;
import ru.soune.nocopy.entity.file.FileEntity;
import ru.soune.nocopy.entity.file.FileStatus;
import ru.soune.nocopy.entity.file.FileType;
import ru.soune.nocopy.repository.FileEntityRepository;
import java.util.List;
@Service
@RequiredArgsConstructor
public class FileStatsService {
private final FileEntityRepository fileEntityRepository;
public FileInfoUserResponse getUserFileStats(Long userId) {
List<FileEntity> userFiles = fileEntityRepository.findByUserId(userId);
return calculateStats(userFiles);
}
public FileInfoUserResponse calculateStats(List<FileEntity> files) {
return FileInfoUserResponse.builder()
.allFileSize(calculateTotalSize(files))
.fileCount(calculateTotalCount(files))
.filesCheck(calculateByStatus(files, FileStatus.CHECKED))
.filesViolation(calculateByStatus(files, FileStatus.VIOLATION))
.imagesSize(calculateMediaSize(files, FileType.IMAGE))
.imagesCount(calculateMediaCount(files, FileType.IMAGE))
.imagesCheck(calculateMediaByStatus(files, FileType.IMAGE, FileStatus.CHECKED))
.imagesViolations(calculateMediaByStatus(files, FileType.IMAGE, FileStatus.VIOLATION))
.videosSize(calculateMediaSize(files, FileType.VIDEO))
.videosCount(calculateMediaCount(files, FileType.VIDEO))
.videosCheck(calculateMediaByStatus(files, FileType.VIDEO, FileStatus.CHECKED))
.videosViolations(calculateMediaByStatus(files, FileType.VIDEO, FileStatus.VIOLATION))
.audiosSize(calculateMediaSize(files, FileType.AUDIO))
.audiosCount(calculateMediaCount(files, FileType.AUDIO))
.audiosCheck(calculateMediaByStatus(files, FileType.AUDIO, FileStatus.CHECKED))
.audiosViolations(calculateMediaByStatus(files, FileType.AUDIO, FileStatus.VIOLATION))
.build();
}
private Long calculateTotalSize(List<FileEntity> files) {
return files.stream()
.filter(file -> file.getStatus() != FileStatus.DELETED)
.mapToLong(FileEntity::getFileSize)
.sum();
}
private Integer calculateTotalCount(List<FileEntity> files) {
return (int) files.stream()
.filter(file -> file.getStatus() != FileStatus.DELETED)
.count();
}
private Integer calculateByStatus(List<FileEntity> files, FileStatus status) {
return (int) files.stream()
.filter(file -> file.getStatus() == status)
.count();
}
private Long calculateMediaSize(List<FileEntity> files, FileType fileType) {
return files.stream()
.filter(file -> file.getStatus() != FileStatus.DELETED)
.filter(file -> isFileType(file, fileType))
.mapToLong(FileEntity::getFileSize)
.sum();
}
private Integer calculateMediaCount(List<FileEntity> files, FileType fileType) {
return (int) files.stream()
.filter(file -> file.getStatus() != FileStatus.DELETED)
.filter(file -> isFileType(file, fileType))
.count();
}
private Integer calculateMediaByStatus(List<FileEntity> files, FileType fileType, FileStatus status) {
return (int) files.stream()
.filter(file -> isFileType(file, fileType))
.filter(file -> file.getStatus() == status)
.count();
}
private boolean isFileType(FileEntity file, FileType fileType) {
if (file.getFileExtension() == null) return false;
String extension = file.getFileExtension().toLowerCase().replace(".", "");
return fileType.supportsExtension(extension);
}
}
@@ -15,7 +15,7 @@ public interface FileUploadService {
UploadProgressResponse getUploadProgress(String uploadId);
void cleanupExpiredSessions();
void handleExpiredSession(FileUploadSession session);
public void retryFailedUpload(String uploadId);
@@ -102,12 +102,14 @@ public class FileUploadServiceImpl implements FileUploadService {
int totalChunks = (int) Math.ceil((double) fileSize / chunkSize);
log.debug("File will be split into {} chunks (chunk size: {} bytes)",
totalChunks, chunkSize);
Long chunkSize = totalChunks == 1 ? fileSize : 1000000L;
FileUploadSession session = FileUploadSession.builder()
.userId(userId)
.extension(extension)
.fileName(fileName)
.fileType(fileType)
.chunkSize(chunkSize)
.fileSize(fileSize)
.totalChunks(totalChunks)
.chunksUploaded(0)
@@ -143,17 +145,103 @@ public class FileUploadServiceImpl implements FileUploadService {
if (chunkNumber < 0 || chunkNumber >= session.getTotalChunks()) {
throw new FileUploadException(
String.format("Invalid chunk number %d. Expected 0-%d",
String.format("Invalid chunk number %d. Expected number: %d",
chunkNumber, session.getTotalChunks() - 1));
}
if (chunkFile.getSize() > chunkSize) {
if (chunkNumber == 0 && chunkFile.getSize() > chunkSize || chunkNumber + 1 == (session.getTotalChunks())
&& chunkFile.getSize() > chunkSize) {
throw new ChunkSizeExceededException(chunkFile.getSize(), chunkSize);
}
if (chunkNumber > 0 && chunkNumber + 1 < session.getTotalChunks() && chunkFile.getSize() != chunkSize) {
throw new ChunkSizeExceededException(chunkFile.getSize(), chunkSize);
}
return processChunk(session, chunkNumber, chunkFile);
}
@Override
@Transactional
public void handleExpiredSession(FileUploadSession session) {
session.setStatus(UploadStatus.FAILED);
session.setLastError("Upload session expired");
sessionRepository.save(session);
CompletableFuture.runAsync(() -> cleanupSessionFiles(session));
}
@Async("fileUploadTaskExecutor")
@Transactional
public void assembleFileAsync(FileUploadSession session) {
try {
assembleFile(session);
log.info("File assembly completed successfully for session: {}",
session.getUploadId());
} catch (Exception e) {
log.error("Failed to assemble file for session {}: {}",
session.getUploadId(), e.getMessage(), e);
handleAssemblyFailure(session, e);
}
}
@Override
@Transactional
public void cancelUpload(String uploadId) {
FileUploadSession session = sessionRepository.findById(uploadId)
.orElseThrow(() -> new UploadSessionNotFoundException(uploadId));
if (session.getStatus() == UploadStatus.COMPLETED || session.getStatus() == UploadStatus.CANCELLED) {
throw new FileUploadException("Cannot cancel completed or cancelled upload");
}
session.setStatus(UploadStatus.CANCELLED);
sessionRepository.save(session);
CompletableFuture.runAsync(() -> cleanupSessionFiles(session));
log.info("Upload cancelled: {}", uploadId);
}
@Override
@Transactional
public void retryFailedUpload(String uploadId) {
FileUploadSession session = sessionRepository.findById(uploadId)
.orElseThrow(() -> new UploadSessionNotFoundException(uploadId));
if (session.getStatus() != UploadStatus.FAILED) {
throw new FileUploadException("Only failed uploads can be retried");
}
if (session.getRetryCount() >= maxRetryAttempts) {
throw new FileUploadException("Max retry attempts exceeded");
}
session.setStatus(UploadStatus.UPLOADING);
session.setRetryCount(session.getRetryCount() + 1);
session.setLastError(null);
sessionRepository.save(session);
if (session.getChunksUploaded().equals(session.getTotalChunks())) {
log.info("Retrying file assembly for session: {}", uploadId);
assembleFileAsync(session);
}
log.info("Upload retry initiated for session: {} (attempt {})",
uploadId, session.getRetryCount());
}
@Override
public UploadProgressResponse getUploadProgress(String uploadId) {
FileUploadSession session = sessionRepository.findById(uploadId)
.orElseThrow(() -> new UploadSessionNotFoundException(uploadId));
return UploadProgressResponse.fromSession(session);
}
private void validateSession(FileUploadSession session) {
UploadStatus status = session.getStatus();
@@ -183,14 +271,6 @@ public class FileUploadServiceImpl implements FileUploadService {
}
}
private void handleExpiredSession(FileUploadSession session) {
session.setStatus(UploadStatus.FAILED);
session.setLastError("Upload session expired");
sessionRepository.save(session);
CompletableFuture.runAsync(() -> cleanupSessionFiles(session));
}
private UploadProgressResponse processChunk(FileUploadSession session,
Integer chunkNumber,
MultipartFile chunkFile) {
@@ -206,6 +286,7 @@ public class FileUploadServiceImpl implements FileUploadService {
session.getChunkPaths().put(chunkNumber, chunkPath);
session.setChunksUploaded(session.getChunksUploaded() + 1);
session.setStatus(UploadStatus.UPLOADING);
session.setExpiresAt(LocalDateTime.now().plusMinutes(1));
log.debug("Chunk {} saved successfully. Uploaded: {}/{}",
chunkNumber, session.getChunksUploaded(), session.getTotalChunks());
@@ -316,58 +397,12 @@ public class FileUploadServiceImpl implements FileUploadService {
}
}
@Async("fileUploadTaskExecutor")
@Transactional
public void assembleFileAsync(FileUploadSession session) {
try {
assembleFile(session);
log.info("File assembly completed successfully for session: {}",
session.getUploadId());
} catch (Exception e) {
log.error("Failed to assemble file for session {}: {}",
session.getUploadId(), e.getMessage(), e);
handleAssemblyFailure(session, e);
}
}
// private void assembleFile(FileUploadSession session) throws IOException {
// log.info("Starting file assembly for session: {} ({})",
// session.getUploadId(), session.getFileName());
//
// Path finalFilePath = null;
//
// try {
// finalFilePath = prepareFinalFile(session);
//
// validateAllChunksExist(session);
//
// mergeChunksToFile(session, finalFilePath);
//
// validateFinalFile(session, finalFilePath);
//
// updateSessionOnSuccess(session, finalFilePath);
//
// cleanupSessionFiles(session);
//
// log.info("File assembly completed: {} -> {} ({} bytes)",
// session.getFileName(), finalFilePath, session.getFileSize());
//
// } catch (Exception e) {
// if (finalFilePath != null) {
// Files.deleteIfExists(finalFilePath);
// }
// throw e;
// }
// }
private void assembleFile(FileUploadSession session) throws IOException {
log.info("Starting file assembly for session: {} ({})",
session.getUploadId(), session.getFileName());
Path finalFilePath = null;
String checksum = null;
String checksum;
try {
finalFilePath = prepareFinalFile(session);
@@ -526,14 +561,6 @@ public class FileUploadServiceImpl implements FileUploadService {
log.debug("Final file checksum: {}", checksum);
}
private void updateSessionOnSuccess(FileUploadSession session, Path finalPath) {
session.setFilePath(finalPath.toString());
session.setChecksum(calculateChecksum(finalPath));
session.setStatus(UploadStatus.COMPLETED);
session.setCompletedAt(LocalDateTime.now());
sessionRepository.save(session);
}
private void cleanupSessionFiles(FileUploadSession session) {
try {
Path chunkDir = getChunkDirectory(session.getUserId(), session.getUploadId());
@@ -564,95 +591,8 @@ public class FileUploadServiceImpl implements FileUploadService {
private void handleAssemblyFailure(FileUploadSession session, Exception e) {
session.setStatus(UploadStatus.FAILED);
session.setLastError(e.getMessage());
sessionRepository.save(session);
log.error("File assembly failed for session {}: {}",
session.getUploadId(), e.getMessage());
}
@Override
@Transactional
public void cancelUpload(String uploadId) {
FileUploadSession session = sessionRepository.findById(uploadId)
.orElseThrow(() -> new UploadSessionNotFoundException(uploadId));
if (session.getStatus() == UploadStatus.COMPLETED || session.getStatus() == UploadStatus.CANCELLED) {
throw new FileUploadException("Cannot cancel completed or cancelled upload");
}
session.setStatus(UploadStatus.CANCELLED);
sessionRepository.save(session);
CompletableFuture.runAsync(() -> cleanupSessionFiles(session));
log.info("Upload cancelled: {}", uploadId);
}
@Override
@Transactional
public void retryFailedUpload(String uploadId) {
FileUploadSession session = sessionRepository.findById(uploadId)
.orElseThrow(() -> new UploadSessionNotFoundException(uploadId));
if (session.getStatus() != UploadStatus.FAILED) {
throw new FileUploadException("Only failed uploads can be retried");
}
if (session.getRetryCount() >= maxRetryAttempts) {
throw new FileUploadException("Max retry attempts exceeded");
}
session.setStatus(UploadStatus.UPLOADING);
session.setRetryCount(session.getRetryCount() + 1);
session.setLastError(null);
sessionRepository.save(session);
if (session.getChunksUploaded().equals(session.getTotalChunks())) {
log.info("Retrying file assembly for session: {}", uploadId);
assembleFileAsync(session);
}
log.info("Upload retry initiated for session: {} (attempt {})",
uploadId, session.getRetryCount());
}
@Override
public UploadProgressResponse getUploadProgress(String uploadId) {
FileUploadSession session = sessionRepository.findById(uploadId)
.orElseThrow(() -> new UploadSessionNotFoundException(uploadId));
return UploadProgressResponse.fromSession(session);
}
@Override
@Transactional
public void cleanupExpiredSessions() {
log.info("Starting cleanup of expired upload sessions");
LocalDateTime expiryThreshold = LocalDateTime.now().minusHours(sessionExpiryHours);
try {
var expiredSessions = sessionRepository.findByStatusInAndCreatedAtBefore(
Set.of(UploadStatus.INITIATED, UploadStatus.UPLOADING, UploadStatus.FAILED),
expiryThreshold
);
int cleanedCount = 0;
for (FileUploadSession session : expiredSessions) {
session.setStatus(UploadStatus.FAILED);
session.setLastError("Session expired during cleanup");
sessionRepository.save(session);
CompletableFuture.runAsync(() -> cleanupSessionFiles(session));
cleanedCount++;
}
log.info("Cleaned up {} expired upload sessions", cleanedCount);
} catch (Exception e) {
log.error("Error during session cleanup", e);
}
}
private Path getChunkDirectory(Long userId, String uploadId) {
@@ -0,0 +1,35 @@
package ru.soune.nocopy.service.file;
import lombok.RequiredArgsConstructor;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.scheduling.annotation.Scheduled;
import org.springframework.stereotype.Service;
import org.springframework.transaction.annotation.Transactional;
import ru.soune.nocopy.entity.file.FileEntity;
import ru.soune.nocopy.entity.file.FileStatus;
import ru.soune.nocopy.repository.FileEntityRepository;
import java.nio.file.Files;
import java.nio.file.Paths;
import java.util.List;
@Service
@RequiredArgsConstructor
public class NotExistFilesCleanupService {
@Autowired
private FileEntityRepository fileEntityRepository;
@Transactional
@Scheduled(fixedDelay = 30000)
public void cleanUpNotExistFiles() {
List<FileEntity> fileEntities = fileEntityRepository.searchFileEntityByStatus(FileStatus.DELETED);
for (FileEntity fileEntity : fileEntities) {
boolean exists = Files.exists(Paths.get(fileEntity.getFilePath()));
if (!exists) {
fileEntityRepository.delete(fileEntity);
}
}
}
}
@@ -0,0 +1,35 @@
package ru.soune.nocopy.service.file;
import lombok.RequiredArgsConstructor;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.scheduling.annotation.Scheduled;
import org.springframework.stereotype.Service;
import org.springframework.transaction.annotation.Transactional;
import ru.soune.nocopy.entity.file.FileUploadSession;
import ru.soune.nocopy.entity.file.UploadStatus;
import ru.soune.nocopy.repository.FileUploadSessionRepository;
import java.time.LocalDateTime;
import java.util.List;
import java.util.Set;
@Service
@RequiredArgsConstructor
public class UploadSessionCleanupService {
@Autowired
private FileUploadSessionRepository fileUploadSessionRepository;
@Autowired
private FileUploadService fileUploadService;
@Transactional
@Scheduled(fixedDelay = 30000)
public void cleanupSessions() {
List<FileUploadSession> expiredSessions = fileUploadSessionRepository.findExpiredSessions(
LocalDateTime.now(), Set.of(UploadStatus.INITIATED, UploadStatus.UPLOADING));
for (FileUploadSession session : expiredSessions) {
fileUploadService.handleExpiredSession(session);
}
}
}
+12 -1
View File
@@ -1,4 +1,8 @@
spring:
flyway:
enabled: true
baseline-on-migrate: true
datasource:
url: jdbc:postgresql://${POSTGRES_HOST}:${POSTGRES_PORT}/${POSTGRES_DB}
username: ${POSTGRES_USER}
@@ -24,7 +28,9 @@ spring:
file:
storage:
base-path: ${FILE_STORAGE_PATH:/data/uploads}
chunk-size: ${FILE_CHUNK_SIZE:5242880} # 5MB
# chunk-size: ${FILE_CHUNK_SIZE:5242880} # 5MB
# chunk-size: ${FILE_CHUNK_SIZE:1048576} # 1MB
chunk-size: ${FILE_CHUNK_SIZE:1000000} # 1MB
max-file-size: ${MAX_FILE_SIZE:10737418240} # 10GB
max-retry-attempts: ${MAX_RETRY_ATTEMPTS:3}
chunk-timeout-ms: ${CHUNK_TIMEOUT_MS:300000} # 5 минут
@@ -50,3 +56,8 @@ logging:
name: logs/application.log
pattern:
console: "%d{yyyy-MM-dd HH:mm:ss} [%thread] %-5level %logger{36} - %msg%n"
yandex:
api-key: ${YANDEX_API_KEY:AQVNyaVaUmgUb1GMCtf5zSEqFxy0woXrcMOOB43q}
folder-id: ${YANDEX_FOLDER_ID:b1gokpdbm6qfpsou8pcd}
search-url: ${YANDEX_SEARCH_URL:https://searchapi.api.cloud.yandex.net/v2/image/search_by_image}
@@ -0,0 +1,2 @@
CREATE INDEX idx_image_hash_prefix
ON image_hashes ((hash_value >> 48));
@@ -1,101 +0,0 @@
package ru.soune.nocopy.service;
import org.junit.jupiter.api.BeforeEach;
import org.junit.jupiter.api.Test;
import org.mockito.ArgumentCaptor;
import org.mockito.InjectMocks;
import org.mockito.Mock;
import org.mockito.MockitoAnnotations;
import org.springframework.context.MessageSource;
import org.springframework.security.crypto.password.PasswordEncoder;
import ru.soune.nocopy.dto.LoginRequest;
import ru.soune.nocopy.entity.AuthToken;
import ru.soune.nocopy.entity.User;
import ru.soune.nocopy.exception.NotValidationPasswordException;
import ru.soune.nocopy.exception.UserNotFoundException;
import ru.soune.nocopy.repository.AuthTokenRepository;
import ru.soune.nocopy.repository.UserRepository;
import java.util.Optional;
import static org.junit.jupiter.api.Assertions.*;
import static org.mockito.ArgumentMatchers.any;
import static org.mockito.Mockito.verify;
import static org.mockito.Mockito.when;
public class AuthServiceTest {
@Mock
private UserRepository userRepository;
@Mock
private AuthTokenRepository authTokenRepository;
@Mock
private PasswordEncoder passwordEncoder;
@Mock
private MessageSource messageSource;
@InjectMocks
private AuthService authService;
@BeforeEach
void setUp() {
MockitoAnnotations.openMocks(this);
}
@Test
void login_success() {
LoginRequest request = new LoginRequest("test@mail.com", "password");
User user = new User();
user.setId(1L);
user.setEmail("test@mail.com");
user.setPassword("encoded_pass");
when(userRepository.findByEmail("test@mail.com"))
.thenReturn(Optional.of(user));
when(passwordEncoder.matches("password", "encoded_pass"))
.thenReturn(true);
when(authTokenRepository.save(any(AuthToken.class)))
.thenAnswer(invocation -> invocation.getArgument(0));
AuthToken token = authService.login(request);
assertNotNull(token);
assertNotNull(token.getToken());
// assertEquals(user.getTokens().get(0), token.getUser());
ArgumentCaptor<AuthToken> captor = ArgumentCaptor.forClass(AuthToken.class);
verify(authTokenRepository).save(captor.capture());
// assertEquals(user, captor.getValue().getUser());
assertNotNull(captor.getValue().getToken());
}
@Test
void login_userNotFound() {
LoginRequest request = new LoginRequest("missing@mail.com", "pwd");
when(userRepository.findByEmail("missing@mail.com")).thenReturn(Optional.empty());
assertThrows(UserNotFoundException.class, () -> authService.login(request));
}
@Test
void login_invalidPassword() {
LoginRequest request = new LoginRequest("test@mail.com", "wrong");
User user = new User();
user.setEmail("test@mail.com");
user.setPassword("encoded_pass");
when(userRepository.findByEmail("test@mail.com")).thenReturn(Optional.of(user));
when(passwordEncoder.matches("wrong", "encoded_pass")).thenReturn(false);
assertThrows(NotValidationPasswordException.class, () -> authService.login(request));
}
}