78 Commits
Author SHA1 Message Date
vladp ab3eef1668 NCBACK-25 change path to file
Test Workflow / test (push) Successful in 3s
2026-01-25 12:10:56 +07:00
vladp 63c3794aa4 fix
Test Workflow / test (push) Successful in 4s
2026-01-24 11:34:30 +07:00
vladp d638819ee9 NCBACK-25
Test Workflow / test (push) Successful in 2s
2026-01-23 13:25:56 +07:00
vladp 51654fd060 NCBACK-25 add protection for audio and use hash method from library
Test Workflow / test (push) Successful in 2s
2026-01-23 13:22:02 +07:00
vladp e37cc06cd0 NCBACK-25 add protection for audio and use hash method from library
Test Workflow / test (push) Successful in 3s
2026-01-22 20:53:25 +07:00
vladp 5a3e26b6a3 NCBACK-25 add protection for audio and use hash method from library
Test Workflow / test (push) Successful in 4s
2026-01-22 20:52:45 +07:00
vladp 409c677ba6 NCBACK-25 add protection for audio and use hash method from library
Test Workflow / test (push) Successful in 3s
2026-01-22 19:55:09 +07:00
vladp 95deda12f3 Merge branch 'refs/heads/dev' into NCBACK-25
# Conflicts:
#	src/main/java/ru/soune/nocopy/controller/ApiController.java
#	src/main/java/ru/soune/nocopy/entity/file/FileEntity.java
2026-01-22 14:15:09 +07:00
vladp ad9cbd96f6 NCBACK-34 update dev
Test Workflow / test (push) Successful in 4s
2026-01-22 11:24:37 +07:00
vladp 211b99b986 Merge branch 'dev' into NCBACK-34
# Conflicts:
#	src/main/java/ru/soune/nocopy/controller/ApiController.java
2026-01-22 11:23:58 +07:00
vladp 60678ef889 Dev add support id
Test Workflow / test (push) Successful in 4s
2026-01-20 21:55:42 +07:00
backdev f0a87648cf Merge pull request 'NCBACK-27' (#12) from NCBACK-27 into dev
Test Workflow / test (push) Successful in 3s
Reviewed-on: #12
2026-01-20 16:20:38 +08:00
vladp c2ffcdcd6e Merge branch 'dev' into NCBACK-27
Test Workflow / test (push) Successful in 3s
# Conflicts:
#	src/main/java/ru/soune/nocopy/configuration/HandlerConfig.java
2026-01-20 15:20:10 +07:00
vladp 0c0fadf5ec NCBACK-34 fix send exception
Test Workflow / test (push) Successful in 3s
2026-01-19 15:49:27 +07:00
vladp e979712b7c Merge branch 'dev' into NCBACK-34 2026-01-17 01:31:21 +07:00
vladp 5027920199 dev info for docker user
Test Workflow / test (push) Has been cancelled
2026-01-16 23:59:07 +07:00
vladp 3d679440f6 NCBACK-34 add logic for clear does not exist files,add dto for similar file search controller
Test Workflow / test (push) Successful in 4s
2026-01-16 14:30:45 +07:00
vladp 4a45f40e9f NCBACK-34 need fix delete filentity and hash
Test Workflow / test (push) Successful in 4s
2026-01-15 20:41:35 +07:00
vladp bbb0d7b17a Merge branch 'dev' into NCBACK-34 2026-01-15 16:55:47 +07:00
vladp 0ed3f958ff DEV_DB change db creds
Test Workflow / test (push) Successful in 7s
2026-01-15 16:52:33 +07:00
vladp de810e32b4 DEV_DB change db creds
Test Workflow / test (push) Successful in 4s
2026-01-15 16:42:37 +07:00
vladp 1a6823f7a9 DEV_DB change db creds
Test Workflow / test (push) Successful in 4s
2026-01-15 16:17:03 +07:00
vladp 82a57a2983 NCBACK-32 add all case for json
Test Workflow / test (push) Successful in 4s
2026-01-15 11:09:05 +07:00
vladp d3e50aa8d1 NCBACK-19 add all case for json
Test Workflow / test (push) Successful in 3s
2026-01-14 19:10:22 +07:00
vladp 5c6309e228 NCBACK-33 fix check company name
Test Workflow / test (push) Successful in 3s
2026-01-14 18:47:34 +07:00
vladp 8ac73ff045 NCBACK-34 add logic to find similar image in db
Test Workflow / test (push) Successful in 3s
2026-01-13 22:40:52 +07:00
vladp 20e1ab321d NCBACK-25 work version for protect file
Test Workflow / test (push) Successful in 3s
2026-01-13 13:28:09 +07:00
vladp 2b7a8b97e1 NCBACK-27 fix token life
Test Workflow / test (push) Successful in 3s
2026-01-10 02:09:39 +07:00
vladp 344ec075d4 Dev
Test Workflow / test (push) Successful in 2s
2026-01-10 01:19:30 +07:00
vladp 7b8546fd4d Dev
Test Workflow / test (push) Successful in 2s
2026-01-10 01:06:32 +07:00
vladp eb12e219ec Dev
Test Workflow / test (push) Successful in 2s
2026-01-10 00:54:49 +07:00
vladp 920cde3cdf Dev 2026-01-10 00:51:03 +07:00
backdev 5b4dca23fb Merge pull request 'NCBACK-3' (#14) from NCBACK-3 into dev
Test Workflow / test (push) Successful in 3s
Reviewed-on: #14
2026-01-10 01:43:36 +08:00
vladp 7b1ed5fe67 NCBACK-3
Test Workflow / test (push) Successful in 3s
2026-01-10 00:35:29 +07:00
backdev 8102d72982 Merge pull request 'NCBACK-3 Add found by yandex image' (#10) from NCBACK-3 into dev
Test Workflow / test (push) Successful in 10s
Reviewed-on: #10
2026-01-10 01:03:33 +08:00
vladp b3a3d57ee3 NCBACK-3
Test Workflow / test (push) Successful in 3s
2026-01-09 23:41:19 +07:00
vladp d896f23872 add user_files_info for avaibale actions
Test Workflow / test (push) Successful in 2s
2026-01-09 23:05:37 +07:00
vladp 3e06040123 NCBACK-27 fix build
Test Workflow / test (push) Successful in 3s
2026-01-09 22:55:06 +07:00
vladp 18d35cb0fb Merge branch 'dev' into NCBACK-27
Test Workflow / test (push) Successful in 2s
# Conflicts:
#	src/main/java/ru/soune/nocopy/controller/ApiController.java
#	src/main/java/ru/soune/nocopy/handler/FileEntityHandler.java
2026-01-09 12:41:32 +07:00
vladp edd508e5d8 Merge branch 'dev' into NCBACK-3
Test Workflow / test (push) Successful in 5s
# Conflicts:
#	docker-compose.yaml
#	src/main/java/ru/soune/nocopy/dto/MessageCode.java
2026-01-09 12:34:24 +07:00
vladp 63baaf7c80 update types
Test Workflow / test (push) Successful in 4s
2026-01-08 12:59:30 +07:00
vladp f136ac1f0a add methods and lib for protection files
Test Workflow / test (push) Successful in 4s
2026-01-08 03:22:32 +07:00
vladp 3a9a5b48a2 update types
Test Workflow / test (push) Successful in 4s
2026-01-07 14:10:35 +07:00
vladp 6fff5884d2 set chunk
Test Workflow / test (push) Successful in 3s
2026-01-07 13:16:53 +07:00
vladp f8673dd581 set chunk
Test Workflow / test (push) Successful in 3s
2026-01-07 12:50:40 +07:00
vladp b9a401f113 correct file_extensions
Test Workflow / test (push) Successful in 8s
2026-01-07 11:56:03 +07:00
vladp 918b4894fc add endpoint with files user info
Test Workflow / test (push) Successful in 6s
2026-01-06 17:27:38 +07:00
vladp bf4d1ca450 add endpoint with files user info
Test Workflow / test (push) Successful in 4s
2026-01-06 12:50:39 +07:00
vladp f1222ede57 add token life by hour
Test Workflow / test (push) Successful in 3s
2026-01-05 16:31:40 +07:00
vladp 0da0404488 add token life by hour
Test Workflow / test (push) Successful in 4s
2026-01-05 16:26:46 +07:00
vladp bd9ecd4f84 change chunk size
Test Workflow / test (push) Successful in 3s
2025-12-30 12:25:22 +07:00
vladp 0f73c8319e add ram size
Test Workflow / test (push) Successful in 4s
2025-12-30 12:08:23 +07:00
vladp 28b0356b03 fix check status deleted
Test Workflow / test (push) Successful in 3s
2025-12-30 02:19:07 +07:00
vladp 72a4516b7d set correct file types
Test Workflow / test (push) Successful in 4s
2025-12-30 01:05:59 +07:00
vladp 851676b895 set chunk-size to 5
Test Workflow / test (push) Successful in 2s
2025-12-26 20:32:15 +07:00
vladp 254fdc0fbd set chunk-size to 5
Test Workflow / test (push) Successful in 8s
2025-12-26 20:30:38 +07:00
vladp a33c53fa6b set chunk-size to 5
Test Workflow / test (push) Successful in 3s
2025-12-26 20:18:18 +07:00
vladp 86e7371ffe set chunk-size to 1000000
Test Workflow / test (push) Successful in 2s
2025-12-26 19:50:32 +07:00
vladp 9541e89651 add post size
Test Workflow / test (push) Successful in 2s
2025-12-26 19:40:08 +07:00
vladp 291586dbf4 add post size
Test Workflow / test (push) Successful in 2s
2025-12-26 19:29:47 +07:00
vladp 91e9b0c785 add post size
Test Workflow / test (push) Successful in 2s
2025-12-26 19:26:39 +07:00
vladp 4fa6b83a10 add post size
Test Workflow / test (push) Successful in 3s
2025-12-26 19:15:45 +07:00
vladp 2c1a2aeb16 add post size
Test Workflow / test (push) Successful in 3s
2025-12-26 19:10:43 +07:00
vladp 2a9f33068a add logic with hard delete
Test Workflow / test (push) Successful in 2s
2025-12-26 16:23:17 +07:00
vladp 1ebe8b852f add file extension to search
Test Workflow / test (push) Successful in 2s
2025-12-26 14:51:02 +07:00
vladp 84993dec00 fix page_size and filtered values
Test Workflow / test (push) Successful in 3s
2025-12-26 13:52:53 +07:00
vladp a2c20ae645 add exceptions
Test Workflow / test (push) Successful in 2s
2025-12-26 12:22:40 +07:00
vladp 18ec9b51f3 add clean logic for uploadsessions
Test Workflow / test (push) Successful in 3s
2025-12-25 15:27:45 +07:00
vladp 45b7acc774 fix after test
Test Workflow / test (push) Successful in 2s
2025-12-25 13:04:49 +07:00
vladp b2cf06a512 update name controllers ends
Test Workflow / test (push) Successful in 3s
2025-12-24 17:26:13 +07:00
vladp d842af2c62 add file size
Test Workflow / test (push) Waiting to run
2025-12-24 16:55:07 +07:00
vladp c5ef58ddd4 fix after test
Test Workflow / test (push) Successful in 3s
2025-12-24 14:20:49 +07:00
vladp f6d4ebfe0e fix after test
Test Workflow / test (push) Successful in 3s
2025-12-24 13:58:46 +07:00
vladp a77a51f519 delete photo
Test Workflow / test (push) Successful in 1s
2025-12-23 18:33:53 +07:00
vladp 1aef1dcd96 add chunk size
Test Workflow / test (push) Successful in 1s
2025-12-23 16:31:09 +07:00
backdev 7dc32d055b Merge pull request 'NCBACK-26 add docker container with storage' (#11) from NCBACK-26 into dev
Test Workflow / test (push) Successful in 0s
Reviewed-on: #11
2025-12-23 17:10:36 +08:00
vladp ffb147d716 NCBACK-3 Add found by yandex image
Test Workflow / test (push) Successful in 4s
2025-12-22 14:42:27 +07:00
vladp 647fd6e1a7 NCBACK-3 Add found by yandex image
Test Workflow / test (push) Successful in 3s
2025-12-22 14:07:44 +07:00
76 changed files with 2569 additions and 826 deletions
+2 -2
View File
@@ -1,6 +1,6 @@
POSTGRES_DB=no_copy_
POSTGRES_USER=postgres
POSTGRES_PASSWORD=postgres
POSTGRES_USER=ncp_db
POSTGRES_PASSWORD=ncpDbApp
POSTGRES_PORT=5432
POSTGRES_HOST=postgres
+78
View File
@@ -0,0 +1,78 @@
Создаем сеть:
docker network create app-network
Войти в БД:
docker exec -it postgres psql -U ncp_db -d no_copy_
Выполнить команду:
\du
Должен быть один пользователь:
List of roles
Role name | Attributes
-----------+------------------------------------------------------------
ncp_db | Superuser, Create role, Create DB, Replication, Bypass RLS
суперпользователь БД:
postgres/postgres
пользователь бд для бэка
ncp_db / ncpDbApp
Подключение к БД для приложения,если volume не существует :
Создаться автоматически.
Если уже существует volume :
docker exec -it postgres psql -U postgres -d postgres
CREATE USER ncp_db WITH PASSWORD 'ncpDbApp';
GRANT ALL PRIVILEGES ON DATABASE no_copy_ TO ncp_db;
\du
\q
-------
psql -U postgres -d no_copy_
-- база
GRANT CONNECT ON DATABASE no_copy_ TO ncp_db;
-- схема
GRANT USAGE ON SCHEMA public TO ncp_db;
-- существующие таблицы
GRANT ALL PRIVILEGES ON ALL TABLES IN SCHEMA public TO ncp_db;
-- существующие sequence (очень важно для id)
GRANT ALL PRIVILEGES ON ALL SEQUENCES IN SCHEMA public TO ncp_db;
-- будущие таблицы
ALTER DEFAULT PRIVILEGES IN SCHEMA public
GRANT ALL ON TABLES TO ncp_db;
ALTER DEFAULT PRIVILEGES IN SCHEMA public
GRANT ALL ON SEQUENCES TO ncp_db;
Зайти в контейнер:
docker exec -it {name} bash
------
Создать для сущности FileEntity sequence для икремента индетификатора
CREATE SEQUENCE IF NOT EXISTS file_support_id_seq START 1;
ALTER TABLE file_entities
ALTER COLUMN support_id
SET DEFAULT nextval('file_support_id_seq');
+12
View File
@@ -21,6 +21,9 @@ configurations {
repositories {
mavenCentral()
flatDir {
dirs 'libs'
}
}
dependencies {
@@ -31,6 +34,12 @@ dependencies {
implementation 'org.mapstruct:mapstruct:1.5.5.Final'
implementation 'commons-validator:commons-validator:1.7'
implementation 'org.flywaydb:flyway-core:9.22.0'
implementation 'com.fasterxml.jackson.dataformat:jackson-dataformat-xml:2.17.0'
implementation 'tools.jackson.core:jackson-core:3.0.3'
implementation 'jakarta.xml.bind:jakarta.xml.bind-api:4.0.0'
annotationProcessor 'org.mapstruct:mapstruct-processor:1.5.5.Final'
annotationProcessor 'org.projectlombok:lombok'
@@ -45,6 +54,9 @@ dependencies {
testImplementation 'org.springframework.boot:spring-boot-starter-test'
testImplementation 'org.mockito:mockito-core:5.3.1'
testRuntimeOnly 'org.junit.platform:junit-platform-launcher'
implementation name: 'testlib-fat-0.2.1-all'
}
tasks.named('test') {
+87 -7
View File
@@ -4,6 +4,12 @@ services:
storage:
image: alpine:latest
container_name: file-storage
deploy:
resources:
limits:
memory: 128M
reservations:
memory: 64M
networks:
- app-network
volumes:
@@ -12,16 +18,27 @@ services:
restart: unless-stopped
db:
image: postgres:17
image: postgres:17.7
restart: always
deploy:
resources:
limits:
cpus: '2.0'
memory: 2G
reservations:
cpus: '1.0'
memory: 1G
environment:
POSTGRES_DB: no_copy_
POSTGRES_USER: postgres
POSTGRES_PASSWORD: postgres
POSTGRES_USER: ${POSTGRES_USER}
POSTGRES_PASSWORD: ${POSTGRES_PASSWORD}
POSTGRES_SHARED_BUFFERS: 512MB
POSTGRES_EFFECTIVE_CACHE_SIZE: 1536MB
ports:
- "54320:5432"
volumes:
- pgdata:/var/lib/postgresql/data
- ./init-scripts:/docker-entrypoint-initdb.d:ro
container_name: postgres
networks:
app-network:
@@ -31,16 +48,28 @@ services:
app:
build: .
container_name: app-backend
deploy:
resources:
limits:
cpus: '1.5'
memory: 1G
reservations:
cpus: '0.5'
memory: 512M
environment:
FILE_STORAGE_PATH: /data/uploads
MAX_FILE_SIZE: 1073741824
FILE_CHUNK_SIZE: 5242880
MAX_FILE_SIZE: 10737418240
FILE_CHUNK_SIZE: 1000000
POSTGRES_DB: no_copy_
POSTGRES_USER: postgres
POSTGRES_PASSWORD: postgres
POSTGRES_USER: ${POSTGRES_USER}
POSTGRES_PASSWORD: ${POSTGRES_PASSWORD}
POSTGRES_PORT: 5432
POSTGRES_HOST: db
STORAGE_SERVICE_URL: http://storage:8081
SPRING_PROFILES_ACTIVE: docker
YANDEX_API_KEY: AQVNyaVaUmgUb1GMCtf5zSEqFxy0woXrcMOOB43q
YANDEX_FOLDER_ID: b1gokpdbm6qfpsou8pcd
YANDEX_SEARCH_URL: "https://searchapi.api.cloud.yandex.net/v2/image/search_by_image"
depends_on:
- db
ports:
@@ -53,10 +82,21 @@ services:
- api
volumes:
- uploads_data:/data/uploads:rw
healthcheck:
test: [ "CMD-SHELL", "pg_isready -U ${POSTGRES_USER} -d no_copy_" ]
interval: 10s
retries: 5
grafana:
image: grafana/grafana:10.3.1
container_name: grafana
deploy:
resources:
limits:
cpus: '0.5'
memory: 512M
reservations:
memory: 256M
ports:
- "3000:3000"
depends_on:
@@ -73,6 +113,8 @@ services:
GF_SECURITY_ADMIN_USER: admin
GF_SECURITY_ADMIN_PASSWORD: admin
GF_METRICS_ENABLED: "true"
GF_DATABASE_MAX_IDLE_CONN: "2"
GF_DATABASE_MAX_OPEN_CONN: "10"
volumes:
- ./infrastructure/grafana/provisioning:/etc/grafana/provisioning
- ./infrastructure/grafana/dashboards:/var/lib/grafana/dashboards
@@ -87,10 +129,23 @@ services:
prometheus:
image: prom/prometheus:latest
container_name: prometheus
deploy:
resources:
limits:
cpus: '0.5'
memory: 1G
reservations:
memory: 512M
ports:
- "9090:9090"
volumes:
- ./infrastructure/prometheus/prometheus.yml:/etc/prometheus/prometheus.yml
- prometheus_data:/prometheus
command:
- '--config.file=/etc/prometheus/prometheus.yml'
- '--storage.tsdb.path=/prometheus'
- '--storage.tsdb.retention.time=15d' # Удерживать 15 дней
- '--web.enable-lifecycle'
healthcheck:
test: [ "CMD", "wget", "--spider", "http://localhost:9090/-/healthy" ]
interval: 10s
@@ -102,10 +157,20 @@ services:
loki:
image: grafana/loki:2.9.2
container_name: loki
deploy:
resources:
limits:
cpus: '1.0'
memory: 2G
reservations:
memory: 1G
ports:
- "3100:3100"
volumes:
- ./infrastructure/loki/loki-config.yaml:/etc/loki/loki-config.yaml:ro
- loki_data:/loki
command:
- -config.file=/etc/loki/loki-config.yaml
healthcheck:
test: [ "CMD", "wget", "--spider", "-q", "http://localhost:3100/ready" ]
interval: 10s
@@ -117,6 +182,13 @@ services:
tempo:
image: grafana/tempo:2.4.1
container_name: tempo
deploy:
resources:
limits:
cpus: '0.5'
memory: 512M
reservations:
memory: 256M
command: [ "-config.file=/etc/tempo/tempo.yaml" ]
volumes:
- ./infrastructure/tempo/tempo.yaml:/etc/tempo/tempo.yaml
@@ -129,6 +201,13 @@ services:
alloy:
image: grafana/alloy:latest
container_name: alloy
deploy:
resources:
limits:
cpus: '0.5'
memory: 256M
reservations:
memory: 128M
user: root
ports:
- "9080:9080"
@@ -158,6 +237,7 @@ volumes:
loki_index:
loki_rules:
uploads_data:
prometheus_data:
networks:
app-network:
+35 -89
View File
@@ -7,7 +7,6 @@ pipeline {
defaultValue: 'dev',
description: 'Ветка для деплоя'
)
string(
name: 'SERVER',
defaultValue: '92.242.61.23',
@@ -31,6 +30,7 @@ pipeline {
}
}
stage('Deploy with docker-compose') {
steps {
script {
@@ -39,124 +39,70 @@ pipeline {
credentialsId: 'server-root-password',
usernameVariable: 'SSH_USER',
passwordVariable: 'SSH_PASS'
)
),
string(credentialsId: 'DB_USER', variable: 'DB_USER'),
string(credentialsId: 'DB_PASSWORD', variable: 'DB_PASSWORD')
]) {
sh """
echo "Deploying branch: ${params.BRANCH}"
echo "Copying files to server..."
sshpass -p '$SSH_PASS' ssh $SSH_USER@$SERVER "mkdir -p /opt/deployments/${params.BRANCH}"
sshpass -p '$SSH_PASS' scp -r ./* $SSH_USER@$SERVER:/opt/deployments/${params.BRANCH}/
sshpass -p '$SSH_PASS' ssh -o StrictHostKeyChecking=no $SSH_USER@$SERVER "mkdir -p /opt/deployments/${params.BRANCH}"
sshpass -p '$SSH_PASS' scp -r -o StrictHostKeyChecking=no ./* $SSH_USER@$SERVER:/opt/deployments/${params.BRANCH}/
sshpass -p '$SSH_PASS' ssh $SSH_USER@$SERVER "
sshpass -p '$SSH_PASS' ssh -o StrictHostKeyChecking=no $SSH_USER@$SERVER "
cd /opt/deployments/${params.BRANCH}
echo 'Step 1: Checking current state...'
echo '1. Остановка старого приложения...'
docker stop app-backend 2>/dev/null || echo 'Контейнер не найден'
docker rm app-backend 2>/dev/null || echo 'Контейнер не найден'
# Смотрим что запущено
docker ps --filter 'name=postgres|app-backend' --format 'table {{.Names}}\\t{{.Image}}\\t{{.Status}}\\t{{.Ports}}'
echo '2. Удаление старых образов...'
docker images --filter 'reference=*app*' -q | xargs -r docker rmi -f 2>/dev/null || echo 'Нет образов для удаления'
echo 'Step 2: Force cleanup old application...'
echo '3. Создание сети если нужно...'
docker network create app-network 2>/dev/null || echo 'Сеть уже существует'
# Принудительно удаляем старый контейнер и образ
docker stop app-backend 2>/dev/null || echo 'App not running'
docker rm app-backend 2>/dev/null || echo 'App not found'
docker rmi app-backend:latest 2>/dev/null || echo 'Image not found'
echo '4. Запуск инфраструктуры...'
docker-compose up -d db storage
# Удаляем все образы app-backend
docker images --filter 'reference=app-backend*' -q | xargs -r docker rmi 2>/dev/null || echo 'No images to remove'
echo 'Step 3: Verify copied files...'
echo 'Files in directory:'
ls -la
echo ''
echo 'Checking Java sources:'
find . -name '*.java' | head -2
echo 'Step 4: Check if PostgreSQL is running...'
if ! docker ps | grep -q postgres; then
echo 'Starting PostgreSQL...'
docker-compose up -d db
echo '5. Ожидание PostgreSQL...'
sleep 10
else
echo 'PostgreSQL already running'
fi
echo 'Step 5: Build application with forced rebuild...'
echo 'Checking Dockerfile exists:'
ls -la Dockerfile 2>/dev/null || echo 'Dockerfile not found, creating simple one'
echo '6. Сборка нового образа приложения...'
docker build --no-cache -t app-backend:latest .
# Если нет Dockerfile, создаем простой
if [ ! -f Dockerfile ]; then
echo 'Creating simple Dockerfile...'
cat > Dockerfile << 'EOF'
FROM eclipse-temurin:21-jre
WORKDIR /app
COPY . /app/
CMD ["java", "-jar", "app.jar"]
EOF
fi
# Собираем с подробным выводом
echo 'Building Docker image...'
docker build --no-cache --progress=plain -t app-backend:latest . 2>&1 | tail -50
echo 'Step 6: Verify new image...'
echo 'Current app-backend images:'
docker images | grep app-backend
echo 'Image creation time:'
docker inspect app-backend:latest --format='{{.Created}}' 2>/dev/null || echo 'Cannot inspect image'
echo 'Step 7: Starting application with alias...'
echo '7. Запуск приложения...'
docker run -d \\
--name app-backend \\
--network app-network \\
--network-alias app \\
-p 80:8080 \\
-v /opt/uploads:/data/uploads:rw \\
-v uploads_data:/data/uploads:rw \\
-e POSTGRES_DB=no_copy_ \\
-e POSTGRES_USER=postgres \\
-e POSTGRES_PASSWORD=postgres \\
-e POSTGRES_USER=$DB_USER \\
-e POSTGRES_PASSWORD=$DB_PASSWORD \\
-e POSTGRES_PORT=5432 \\
-e POSTGRES_HOST=db \\
--restart unless-stopped \\
app-backend:latest
echo 'Step 8: Checking deployment...'
sleep 10
echo '8. Запуск мониторинга...'
docker-compose up -d grafana prometheus loki tempo alloy
echo 'Containers status:'
docker ps --filter 'name=postgres|app-backend' --format 'table {{.Names}}\\t{{.Image}}\\t{{.Status}}\\t{{.CreatedAt}}'
echo 'Checking image inside container:'
docker exec app-backend ls -la /app/ 2>/dev/null || echo 'Cannot check container files'
echo 'Application logs (last 10 lines):'
docker logs --tail=10 app-backend 2>/dev/null || echo 'Logs not available yet'
echo 'Step 9: Health check...'
if docker ps | grep -q app-backend; then
echo 'Container is running'
echo 'Testing application health...'
for i in {1..5}; do
if curl -s -f http://localhost:80/health > /dev/null 2>&1; then
echo 'Health check passed'
echo 'Deployment successful'
echo 'Application URL: http://${params.SERVER}:80'
exit 0
fi
echo 'Waiting for application to start... attempt ' \$i
echo '9. Проверка...'
sleep 5
done
echo 'Health check failed, but container is running'
echo 'Application might be starting slowly'
echo 'Application URL: http://${params.SERVER}:80'
echo 'Статус контейнеров:'
docker ps --format 'table {{.Names}}\\t{{.Image}}\\t{{.Status}}'
echo '10. Проверка health...'
if curl -s -f http://localhost:80/health > /dev/null 2>&1; then
echo 'Приложение работает'
echo 'URL: http://${params.SERVER}:80'
else
echo 'Application failed to start'
docker logs app-backend
exit 1
echo 'Проверка health не удалась'
docker logs app-backend --tail=20
fi
"
"""
+19
View File
@@ -0,0 +1,19 @@
DO $$
BEGIN
IF NOT EXISTS (SELECT FROM pg_roles WHERE rolname = 'ncp_db') THEN
CREATE USER ncp_db WITH PASSWORD 'ncpDbApp';
END IF;
END
$$;
GRANT CONNECT ON DATABASE no_copy_ TO ncp_db;
GRANT USAGE ON SCHEMA public TO ncp_db;
GRANT ALL PRIVILEGES ON ALL TABLES IN SCHEMA public TO ncp_db;
GRANT ALL PRIVILEGES ON ALL SEQUENCES IN SCHEMA public TO ncp_db;
ALTER DEFAULT PRIVILEGES IN SCHEMA public
GRANT ALL ON TABLES TO ncp_db;
ALTER DEFAULT PRIVILEGES IN SCHEMA public
GRANT ALL ON SEQUENCES TO ncp_db;
Binary file not shown.
@@ -2,8 +2,10 @@ package ru.soune.nocopy;
import org.springframework.boot.SpringApplication;
import org.springframework.boot.autoconfigure.SpringBootApplication;
import org.springframework.scheduling.annotation.EnableScheduling;
@SpringBootApplication
@EnableScheduling
public class NoCopyApplication {
public static void main(String[] args) {
@@ -1,17 +1,49 @@
package ru.soune.nocopy.configuration;
import com.vrt.AudioFilePathProvider;
import com.vrt.fileprotection.FileProtector;
import com.vrt.fileprotection.audio.AudioLocalSearch;
import com.vrt.fileprotection.image.ImageLocalSearch;
import com.vrt.fileprotection.image.ImageUniqueCheck;
import com.vrt.fileprotection.image.phash.PerceptualHashHelper;
import lombok.AllArgsConstructor;
import org.springframework.boot.autoconfigure.EnableAutoConfiguration;
import org.springframework.context.annotation.Bean;
import org.springframework.context.annotation.Configuration;
import org.springframework.security.crypto.bcrypt.BCryptPasswordEncoder;
import org.springframework.security.crypto.password.PasswordEncoder;
import java.util.Collections;
@Configuration
@EnableAutoConfiguration
@AllArgsConstructor
public class ApplicationConfig {
@Bean
PasswordEncoder passwordEncoder() {
return new BCryptPasswordEncoder();
}
@Bean
public com.vrt.
NoCopyFileService noCopyFileService(
FileProtector.FileProvider fileProvider,
FileProtector.ProcessingListener processingListener,
ImageUniqueCheck imageUniqueCheck,
ImageLocalSearch imageLocalSearch,
AudioLocalSearch audioLocalSearch,
AudioFilePathProvider audioFilePathProvider) {
return new com.vrt.NoCopyFileService(
Collections.emptyList(),
fileProvider,
processingListener,
imageUniqueCheck,
imageLocalSearch,
audioLocalSearch,
audioFilePathProvider
);
}
}
@@ -16,7 +16,9 @@ public class HandlerConfig {
LoginRequestHandler login,
FileUploadHandler upload,
FileEntityHandler file,
LogoutRequestHandler logoutHandler
LogoutRequestHandler logoutHandler,
ImageFoundRequestHandler imageFoundRequestHandler,
AuthRequestHandler authRequestHandler
) {
Map<Integer, RequestHandler> map = new HashMap<>();
map.put(20001, login);
@@ -24,6 +26,8 @@ public class HandlerConfig {
map.put(20004, upload);
map.put(20005, file);
map.put(20006, logoutHandler);
map.put(20007, imageFoundRequestHandler);
map.put(20008, authRequestHandler);
return map;
}
@@ -35,6 +35,8 @@ public class JacksonConfig {
mapper.registerModule(javaTimeModule);
mapper.disable(SerializationFeature.WRITE_DATES_AS_TIMESTAMPS);
mapper.configure(com.fasterxml.jackson.databind.DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, false);
mapper.enable(SerializationFeature.INDENT_OUTPUT);
return mapper;
}
@@ -0,0 +1,19 @@
package ru.soune.nocopy.configuration.file;
import lombok.AllArgsConstructor;
import org.springframework.boot.context.event.ApplicationReadyEvent;
import org.springframework.context.event.EventListener;
import org.springframework.stereotype.Component;
import ru.soune.nocopy.service.file.FileProcessingOrchestrator;
@Component
@AllArgsConstructor
public class NoCopyInitializer {
private final FileProcessingOrchestrator orchestrator;
@EventListener(ApplicationReadyEvent.class)
public void initializeOnStartup() {
orchestrator.initializeProcessingQueue();
}
}
@@ -1,9 +1,15 @@
package ru.soune.nocopy.controller;
import com.vrt.NoCopyFileService;
import com.vrt.fileprotection.FileProtector;
import com.vrt.fileprotection.NoCopyCheckResult;
import lombok.RequiredArgsConstructor;
import lombok.extern.slf4j.Slf4j;
import org.springframework.core.io.Resource;
import org.springframework.core.io.UrlResource;
import org.springframework.data.domain.Page;
import org.springframework.data.domain.Pageable;
import org.springframework.data.web.PageableDefault;
import org.springframework.http.HttpHeaders;
import org.springframework.http.MediaType;
import org.springframework.http.ResponseEntity;
@@ -15,20 +21,25 @@ import ru.soune.nocopy.dto.BaseRequest;
import ru.soune.nocopy.dto.BaseResponse;
import ru.soune.nocopy.dto.MessageCode;
import ru.soune.nocopy.dto.RegAnswer;
import ru.soune.nocopy.dto.file.*;
import ru.soune.nocopy.entity.file.FileEntity;
import ru.soune.nocopy.dto.file.ChunkUploadResponse;
import ru.soune.nocopy.dto.file.CompleteUploadResponse;
import ru.soune.nocopy.dto.file.FileEntityResponse;
import ru.soune.nocopy.dto.file.UploadProgress;
import ru.soune.nocopy.entity.AuthToken;
import ru.soune.nocopy.entity.file.FileStatus;
import ru.soune.nocopy.entity.file.ProtectionStatus;
import ru.soune.nocopy.entity.file.UploadStatus;
import ru.soune.nocopy.exception.NotFoundAuthToken;
import ru.soune.nocopy.exception.NotValidFieldException;
import ru.soune.nocopy.exception.ValidationException;
import ru.soune.nocopy.exception.*;
import ru.soune.nocopy.handler.*;
import ru.soune.nocopy.repository.AuthTokenRepository;
import ru.soune.nocopy.repository.FileEntityRepository;
import ru.soune.nocopy.service.FileSimilarityService;
import ru.soune.nocopy.service.auth.AuthService;
import ru.soune.nocopy.service.file.FileEntityService;
import ru.soune.nocopy.service.file.FileUploadService;
import ru.soune.nocopy.util.FileUtil;
import java.io.File;
import java.io.IOException;
import java.nio.file.Files;
import java.nio.file.Path;
@@ -36,6 +47,7 @@ import java.nio.file.Paths;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.Optional;
import java.util.stream.Collectors;
@Slf4j
@@ -49,7 +61,15 @@ public class ApiController {
private final FileEntityService fileEntityService;
private final AuthTokenRepository authTokenRepository;
private final AuthService authService;
private final FileSimilarityService fileSimilarityService;
private final FileEntityRepository fileEntityRepository;
private final NoCopyFileService noCopyFileService;
private final FileUtil fileUtil;
@PostMapping("/v{version}/data")
public ResponseEntity<?> handlePostRequest(@RequestBody BaseRequest request,
@@ -72,6 +92,9 @@ public class ApiController {
return ResponseEntity.ok().body(response);
} catch (ValidationException e) {
return createValidationErrorResponse(e.getBindingResult(), e.getMsgId());
} catch (NotFoundAuthToken e) {
return ResponseEntity.ok().body(new BaseResponse(msgId, MessageCode.AUTH_TOKEN_NOT_FOUND.getCode(),
MessageCode.AUTH_TOKEN_NOT_FOUND.getDescription(), new HashMap<>()));
} catch (NotValidFieldException e) {
throw e;
} catch (Exception e) {
@@ -89,55 +112,68 @@ public class ApiController {
@PostMapping("/v{version}/files/chunk")
public ResponseEntity<BaseResponse> uploadChunk(
@PathVariable("version") int version,
@RequestParam("upload_id") String uploadId,
@RequestParam("chunk_number") Integer chunkNumber,
@RequestParam("chunk") MultipartFile chunk) {
log.info("Uploading chunk {} for session {}, file size: {} bytes, version: {}",
chunkNumber, uploadId, chunk.getSize(), version);
if (chunk.isEmpty()) {
return ResponseEntity.ok().body(new BaseResponse(20004, MessageCode.FILE_UPLOAD_ERROR.getCode(),
"Chunk file is empty", null));
}
if (uploadId == null || uploadId.isBlank()) {
return ResponseEntity.ok().body(new BaseResponse(20004, MessageCode.FILE_UPLOAD_ERROR.getCode(),
"Upload ID is required", null));
}
if (chunkNumber == null || chunkNumber < 0) {
return ResponseEntity.ok().body(new BaseResponse(20004, MessageCode.FILE_UPLOAD_ERROR.getCode(),
"Valid chunk number is required", null));
}
@RequestParam(value = "upload_id", required = false) String uploadId,
@RequestParam(value = "chunk_number", required = false) Integer chunkNumber,
@RequestParam(value = "chunk", required = false) MultipartFile chunk) {
try {
if (chunk == null || chunk.isEmpty()) {
return buildErrorResponse(uploadId, chunkNumber, "Chunk file null or empty");
}
if (uploadId == null || uploadId.isBlank()) {
return buildErrorResponse(uploadId, chunkNumber, "Upload ID is required");
}
if (chunkNumber == null || chunkNumber < 0) {
return buildErrorResponse(uploadId, chunkNumber, "Valid chunk number is required");
}
fileUploadService.uploadChunk(uploadId, chunkNumber, chunk);
ChunkUploadResponse responseBody = ChunkUploadResponse.builder()
.uploadId(uploadId)
.chunkNumber(chunkNumber)
.chunkSize(chunk.getSize())
.message("Chunk uploaded successfully")
.build();
return ResponseEntity.ok().body(new BaseResponse(20004, MessageCode.SUCCESS.getCode(),
"Chunk uploaded successfully", responseBody));
return buildSuccessResponse(uploadId, chunkNumber, chunk);
} catch (DuplicateImageException e) {
return ResponseEntity.ok().body(new BaseResponse(20004, MessageCode.DUPLICATE_FILE_UPLOAD.getCode(),
MessageCode.DUPLICATE_FILE_UPLOAD.getDescription(),
Map.of("duplicateOwnerId", e.userId(), "duplicateFileId", e.duplicateFileId())));
} catch (Exception e) {
log.error("Error uploading chunk", e);
ChunkUploadResponse responseBody = ChunkUploadResponse.builder()
.uploadId(uploadId)
.chunkNumber(chunkNumber)
.build();
return ResponseEntity.ok().body(new BaseResponse(
20004, MessageCode.FILE_UPLOAD_ERROR.getCode(),
"Failed to upload chunk: " + e.getMessage(), responseBody));
return buildErrorResponse(uploadId, chunkNumber, "Failed to upload chunk: " + e.getMessage());
}
}
@GetMapping("/v{version}/files/{fileId}/similar")
public ResponseEntity<BaseResponse> findSimilarFiles(
@PathVariable("version") int version,
@PathVariable String fileId,
@RequestParam(required = false) List<String> similarityLevels,
@PageableDefault(size = 20, sort = "hammingDistance") Pageable pageable) {
SimilarityFilter filter = SimilarityFilter.builder()
.similarityLevels(similarityLevels)
.build();
Page<SimilarFileDTO> similarFiles = fileSimilarityService.findSimilarFiles(fileId, filter, pageable);
String messageDesc;
MessageCode success;
if (similarFiles.isEmpty()) {
messageDesc = MessageCode.FILE_NOT_FOUND.getDescription();
success = MessageCode.FILE_NOT_FOUND;
} else {
messageDesc = MessageCode.SIMILAR_FILES_FOUND.getDescription();
success = MessageCode.SIMILAR_FILES_FOUND;
}
Map<String, Object> responseData = new HashMap<>();
responseData.put("content", similarFiles.getContent());
responseData.put("page", similarFiles.getNumber());
responseData.put("size", similarFiles.getSize());
responseData.put("totalElements", similarFiles.getTotalElements());
responseData.put("totalPages", similarFiles.getTotalPages());
responseData.put("hasNext", similarFiles.hasNext());
responseData.put("hasPrevious", similarFiles.hasPrevious());
return ResponseEntity.ok()
.body(new BaseResponse(20004, success.getCode(), messageDesc, responseData));
}
@GetMapping("/v{version}/files/progress/{uploadId}")
public ResponseEntity<BaseResponse> getUploadProgress(
@PathVariable("version") int version,
@@ -161,7 +197,6 @@ public class ApiController {
return ResponseEntity.ok().body(new BaseResponse(
20004, MessageCode.SUCCESS.getCode(), MessageCode.SUCCESS.getDescription(), responseBody));
} catch (Exception e) {
log.error("Error getting progress for upload: {}", uploadId, e);
@@ -237,29 +272,67 @@ public class ApiController {
@GetMapping("/v{version}/files/download/{fileId}")
public ResponseEntity<Resource> downloadFile(
@PathVariable String fileId,
@PathVariable Integer version,
@RequestHeader("Authorization") String tokenHeader) {
public ResponseEntity<?> downloadFile(
@PathVariable(required = false) String fileId,
@PathVariable(required = false) Integer version,
@RequestHeader(value = "Authorization", required = false) String tokenHeader) {
try {
Long userId = getUserIdFromToken(tokenHeader);
FileEntityResponse fileInfo = fileEntityService.getById(fileId, version);
if (tokenHeader == null) {
Map<String, Object> errorData = new HashMap<>();
errorData.put("token", tokenHeader);
errorData.put("fileId", fileId);
errorData.put("version", version);
if (!fileInfo.getUserId().equals(userId)) {
return ResponseEntity.status(403).build();
return ResponseEntity.ok().body(new BaseResponse(20004,
MessageCode.FILE_DOWNLOAD_ERROR_NOT_CORRECT_FIELD.getCode(),
MessageCode.FILE_DOWNLOAD_ERROR_NOT_CORRECT_FIELD.getDescription(),
errorData));
}
if (!fileInfo.isExistsOnDisk()) {
return ResponseEntity.status(404)
.body(null);
Long userId = authService.useUserAuthToken(tokenHeader);
FileEntityResponse entityResponse = fileEntityService.getById(fileId, version);
if (!entityResponse.getUserId().equals(userId)) {
Map<String, Object> errorData = new HashMap<>();
errorData.put("token", tokenHeader);
return ResponseEntity.ok().body(new BaseResponse(20004,
MessageCode.FILE_DOWNLOAD_ERROR.getCode(),
MessageCode.FILE_DOWNLOAD_ERROR.getDescription(),
errorData));
}
Path filePath = Paths.get(fileInfo.getFilePath());
if (entityResponse.getStatus().equals(FileStatus.DELETED)) {
Map<String, Object> errorData = new HashMap<>();
errorData.put("file_status", entityResponse.getStatus());
return ResponseEntity.ok().body(new BaseResponse(20004,
MessageCode.FILE_DOWNLOAD_ERROR.getCode(),
MessageCode.FILE_DOWNLOAD_ERROR.getDescription(),
errorData));
}
if (!entityResponse.isExistsOnDisk()) {
Map<String, Object> errorData = new HashMap<>();
errorData.put("onDisk", entityResponse.isExistsOnDisk());
return ResponseEntity.ok().body(new BaseResponse(20004,
MessageCode.FILE_DOWNLOAD_ERROR.getCode(),
MessageCode.FILE_DOWNLOAD_ERROR.getDescription(),
errorData));
}
Path filePath = Paths.get(entityResponse.getFilePath());
Resource resource = new UrlResource(filePath.toUri());
if (!resource.exists()) {
return ResponseEntity.status(404).build();
Map<String, Object> errorData = new HashMap<>();
errorData.put("resource", resource.exists());
return ResponseEntity.ok().body(new BaseResponse(20004,
MessageCode.FILE_DOWNLOAD_ERROR.getCode(),
MessageCode.FILE_DOWNLOAD_ERROR.getDescription(),
errorData));
}
String contentType = determineContentType(filePath);
@@ -267,84 +340,138 @@ public class ApiController {
return ResponseEntity.ok()
.contentType(MediaType.parseMediaType(contentType))
.header(HttpHeaders.CONTENT_DISPOSITION,
"attachment; filename=\"" + fileInfo.getOriginalFileName() + "\"")
.header(HttpHeaders.CONTENT_LENGTH, String.valueOf(fileInfo.getFileSize()))
"attachment; filename=\"" + entityResponse.getOriginalFileName() + "\"")
.header(HttpHeaders.CONTENT_LENGTH, String.valueOf(entityResponse.getFileSize()))
.body(resource);
} catch (FileEntityNotFoundException e) {
Map<String, Object> errorData = new HashMap<>();
errorData.put("fileId", fileId);
} catch (NotFoundAuthToken e) {
return ResponseEntity.status(401).build();
} catch (Exception e) {
log.error("Error downloading file", e);
return ResponseEntity.status(500).build();
return ResponseEntity.ok().body(new BaseResponse(20004,
MessageCode.FILE_NOT_FOUND.getCode(),
MessageCode.FILE_NOT_FOUND.getDescription(),
errorData));
} catch (NotFoundAuthToken | IOException e) {
Map<String, Object> errorData = new HashMap<>();
errorData.put("token", tokenHeader);
errorData.put("fileId", fileId);
errorData.put("version", version);
return ResponseEntity.ok().body(new BaseResponse(20004,
MessageCode.FILE_DOWNLOAD_ERROR_NOT_CORRECT_FIELD.getCode(),
MessageCode.FILE_DOWNLOAD_ERROR_NOT_CORRECT_FIELD.getDescription(),
errorData));
}
}
@GetMapping("/v{version}/files/info/{fileId}")
public ResponseEntity<FileEntityResponse> getFileInfo(
@PathVariable String fileId,
@PathVariable Integer version,
@RequestHeader("Authorization") String tokenHeader) {
@GetMapping("/protect/{fileId}")
public ResponseEntity<?> protect( @PathVariable(required = false) String fileId) {
Optional<FileEntity> optionalFileEntity = fileEntityRepository.findById(fileId);
try {
Long userId = getUserIdFromToken(tokenHeader);
FileEntityResponse fileInfo = fileEntityService.getById(fileId, version);
if (!optionalFileEntity.isPresent()) {
return ResponseEntity.notFound().build();
}
FileEntity fileEntity = optionalFileEntity.get();
FileProtector.FileInfo fileInfo = fileUtil.createFileInfo(fileEntity);
if (!fileInfo.getUserId().equals(userId)) {
return ResponseEntity.status(403).build();
noCopyFileService.addFile(fileInfo);
fileEntity.setProtectionStatus(ProtectionStatus.PROCESSING);
fileEntityRepository.save(fileEntity);
return ResponseEntity.ok().build();
}
return ResponseEntity.ok(fileInfo);
} catch (NotFoundAuthToken e) {
return ResponseEntity.status(401).build();
} catch (Exception e) {
log.error("Error getting file info", e);
return ResponseEntity.status(500).build();
}
@GetMapping("/check/{fileId}/{type}")
public ResponseEntity<?> check(@PathVariable(required = false) String fileId,
@PathVariable(required = false) String type) {
Optional<FileEntity> optionalFileEntity = fileEntityRepository.findById(fileId);
if (!optionalFileEntity.isPresent()) {
return ResponseEntity.notFound().build();
}
@GetMapping("/v{version}/files/preview/{fileId}")
public ResponseEntity<Resource> previewFile(
@PathVariable String fileId,
@PathVariable Integer version,
@RequestHeader("Authorization") String tokenHeader) {
FileEntity fileEntity = optionalFileEntity.get();
try {
Long userId = getUserIdFromToken(tokenHeader);
FileEntityResponse fileInfo = fileEntityService.getById(fileId, version);
Path path = Paths.get(fileEntity.getProtectedFilePath());
File file = path.toFile();
if (!fileInfo.getUserId().equals(userId)) {
return ResponseEntity.status(403).build();
NoCopyCheckResult noCopyCheckResult = noCopyFileService.checkFile(file,
FileProtector.Type.valueOf(type.toUpperCase()));
return ResponseEntity.ok().body(noCopyCheckResult);
}
if (!isPreviewSupported(fileInfo.getMimeType())) {
return ResponseEntity.status(415)
.body(null);
private ResponseEntity<BaseResponse> checkForDuplicates(String uploadId) throws IOException {
Optional<FileEntity> uploadedFile = fileEntityRepository.findByUploadSessionId(uploadId);
if (uploadedFile.isEmpty() || !uploadedFile.get().getMimeType().equals("image")) {
return null;
}
Path filePath = Paths.get(fileInfo.getFilePath());
Resource resource = new UrlResource(filePath.toUri());
FileEntity fileEntity = uploadedFile.get();
List<SimilarFileDTO> similarFiles = fileSimilarityService.findSimilarFiles(fileEntity.getId());
if (!resource.exists()) {
return ResponseEntity.status(404).build();
if (hasDuplicate(similarFiles)) {
return handleDuplicate(fileEntity, similarFiles);
}
String contentType = determineContentType(filePath);
return ResponseEntity.ok()
.contentType(MediaType.parseMediaType(contentType))
.header(HttpHeaders.CONTENT_DISPOSITION,
"inline; filename=\"" + fileInfo.getOriginalFileName() + "\"")
.body(resource);
} catch (NotFoundAuthToken e) {
return ResponseEntity.status(401).build();
} catch (Exception e) {
log.error("Error previewing file", e);
return ResponseEntity.status(500).build();
}
return null;
}
private boolean hasDuplicate(List<SimilarFileDTO> similarFiles) {
return similarFiles.stream().anyMatch(f -> f.getHammingDistance() <= 5);
}
private ResponseEntity<BaseResponse> handleDuplicate(FileEntity fileEntity, List<SimilarFileDTO> similarFiles)
throws IOException {
fileEntityService.deleteFromDisk(fileEntity);
fileEntityService.markAsDeleted(fileEntity);
Optional<FileEntity> originalFile = fileEntityRepository.findById(similarFiles.get(0).getFileId());
if (originalFile.isPresent()) {
Map<String, String> duplicateInfo = Map.of(
"duplicate_file_id", originalFile.get().getId(),
"owner_user_id", String.valueOf(originalFile.get().getUserId()));
return ResponseEntity.ok().body(new BaseResponse(
20004,
MessageCode.DUPLICATE_FILE_UPLOAD.getCode(),
"Failed to upload chunk, duplicate",
duplicateInfo));
}
return null;
}
private ResponseEntity<BaseResponse> buildSuccessResponse(String uploadId, Integer chunkNumber, MultipartFile chunk) {
ChunkUploadResponse responseBody = ChunkUploadResponse.builder()
.uploadId(uploadId)
.chunkNumber(chunkNumber)
.chunkSize(chunk.getSize())
.message("Chunk uploaded successfully")
.build();
return ResponseEntity.ok().body(new BaseResponse(
20000,
MessageCode.SUCCESS.getCode(),
"Chunk uploaded successfully",
responseBody));
}
private ResponseEntity<BaseResponse> buildErrorResponse(String uploadId, Integer chunkNumber, String errorMessage) {
ChunkUploadResponse responseBody = ChunkUploadResponse.builder()
.uploadId(uploadId)
.chunkNumber(chunkNumber)
.build();
return ResponseEntity.ok().body(new BaseResponse(
20004,
MessageCode.FILE_UPLOAD_ERROR.getCode(),
errorMessage,
responseBody));
}
private ResponseEntity<BaseResponse> createValidationErrorResponse(BindingResult bindingResult, Integer msgId) {
List<Map<String, String>> fieldErrors = bindingResult.getFieldErrors()
@@ -381,23 +508,4 @@ public class ApiController {
}
return contentType;
}
private boolean isPreviewSupported(String mimeType) {
if (mimeType == null) {
return false;
}
return mimeType.startsWith("image") ||
mimeType.startsWith("text") ||
mimeType.equals("pdf") ||
mimeType.startsWith("video") ||
mimeType.startsWith("audio");
}
private Long getUserIdFromToken(String tokenHeader) {
String token = tokenHeader.replace("Bearer ", "");
AuthToken authToken = authTokenRepository.findByToken(token)
.orElseThrow(() -> new NotFoundAuthToken("Token not found"));
return authToken.getUser().getId();
}
}
@@ -1,9 +1,11 @@
package ru.soune.nocopy.dto;
import com.fasterxml.jackson.annotation.JsonFormat;
import com.fasterxml.jackson.annotation.JsonProperty;
import lombok.Data;
@Data
@JsonFormat(with = JsonFormat.Feature.ACCEPT_CASE_INSENSITIVE_PROPERTIES)
public class BaseRequest {
@JsonProperty("version")
Integer version;
@@ -1,11 +1,13 @@
package ru.soune.nocopy.dto;
import com.fasterxml.jackson.annotation.JsonFormat;
import com.fasterxml.jackson.annotation.JsonProperty;
import lombok.AllArgsConstructor;
import lombok.Data;
@Data
@AllArgsConstructor
@JsonFormat(with = JsonFormat.Feature.ACCEPT_CASE_INSENSITIVE_PROPERTIES)
public class BaseResponse {
@JsonProperty("msg_id")
private Integer msgId;
@@ -1,5 +1,6 @@
package ru.soune.nocopy.dto;
import com.fasterxml.jackson.annotation.JsonFormat;
import jakarta.validation.constraints.Email;
import jakarta.validation.constraints.NotBlank;
import jakarta.validation.constraints.Size;
@@ -10,6 +11,7 @@ import lombok.NoArgsConstructor;
@Data
@AllArgsConstructor
@NoArgsConstructor
@JsonFormat(with = JsonFormat.Feature.ACCEPT_CASE_INSENSITIVE_PROPERTIES)
public class LoginRequest {
@NotBlank(message = "error.not.blank") @Email(message = "error.not.email") @Size(max = 128)
@@ -8,6 +8,10 @@ public enum MessageCode {
INVALID_TOKEN(2, "Invalid token"),
INVALID_ACTION(2, "Invalid action"),
FILE_UPLOAD_ERROR(2, "File upload error"),
DUPLICATE_FILE_UPLOAD(2, "Duplicate file upload"),
FILE_DOWNLOAD_ERROR(2, "File download error"),
FILE_DOWNLOAD_ERROR_NOT_CORRECT_FIELD(2, "Not correct field"),
IMAGE_FOUND_ERROR(2, "Image found error"),
INVALID_JSON_BODY(2, "Invalid fields in JSON object"),
INCOMPLETE_UPLOAD(2, "Not load all chunks"),
MSG_ID_NOT_FOUND(4, "Message id not found"),
@@ -16,8 +20,10 @@ public enum MessageCode {
AUTH_EMAIL_NOT_FOUND(4, "Email not found"),
AUTH_EMAIL_OR_TOKEN_NOT_FOUND(4, "Email or Token not found "),
AUTH_TOKEN_MISMATCH(4, "Token mismatch"),
AUTH_TOKEN_NOT_FOUND(4, "Token not found"),
FILE_NOT_FOUND(4, "File not found"),
AUTH_PASSWORD_NOT_MATCHES(2, "Password does not match");
AUTH_PASSWORD_NOT_MATCHES(2, "Password does not match"),
SIMILAR_FILES_FOUND(0, "Similar files found");
private final Integer code;
@@ -1,10 +1,12 @@
package ru.soune.nocopy.dto;
import com.fasterxml.jackson.annotation.JsonFormat;
import jakarta.validation.constraints.*;
import lombok.Data;
@Data
@JsonFormat(with = JsonFormat.Feature.ACCEPT_CASE_INSENSITIVE_PROPERTIES)
public class RegRequest {
@NotEmpty(message = "Full name is required")
private String fullName;
@@ -0,0 +1,11 @@
package ru.soune.nocopy.dto;
import com.fasterxml.jackson.annotation.JsonProperty;
import lombok.Data;
@Data
public class TokenProcessRequest {
@JsonProperty("token")
private String token;
}
@@ -10,6 +10,7 @@ import ru.soune.nocopy.entity.GenderType;
import java.time.LocalDate;
@Data
@JsonFormat(with = JsonFormat.Feature.ACCEPT_CASE_INSENSITIVE_PROPERTIES)
public class UserRequest {
@Size(min = 2)
private String fullName;
@@ -1,41 +0,0 @@
package ru.soune.nocopy.dto.file;
import lombok.AllArgsConstructor;
import lombok.Builder;
import lombok.Data;
import lombok.NoArgsConstructor;
@Data
@Builder
@NoArgsConstructor
@AllArgsConstructor
public class FileApiResponse<T> {
private boolean success;
private String message;
private T data;
private String error;
public static <T> FileApiResponse<T> success(T data) {
return FileApiResponse.<T>builder()
.success(true)
.message("Operation completed successfully")
.data(data)
.build();
}
public static <T> FileApiResponse<T> success(String message, T data) {
return FileApiResponse.<T>builder()
.success(true)
.message(message)
.data(data)
.build();
}
public static <T> FileApiResponse<T> error(String error) {
return FileApiResponse.<T>builder()
.success(false)
.error(error)
.build();
}
}
@@ -30,4 +30,5 @@ public class FileEntityResponse {
private String formattedSize;
private String downloadUrl;
private boolean existsOnDisk;
private Integer supportId;
}
@@ -0,0 +1,20 @@
package ru.soune.nocopy.dto.file;
import com.fasterxml.jackson.annotation.JsonProperty;
import lombok.Builder;
import lombok.Data;
import java.util.List;
@Data
@Builder
public class FileExtensionResponse {
@JsonProperty("file_extension")
private List<String> extension;
@JsonProperty("count")
private Integer count;
@JsonProperty("max_file_size")
private Long maxFileSize;
}
@@ -0,0 +1,61 @@
package ru.soune.nocopy.dto.file;
import com.fasterxml.jackson.annotation.JsonProperty;
import lombok.AllArgsConstructor;
import lombok.Builder;
import lombok.Data;
import lombok.NoArgsConstructor;
@Data
@Builder
@NoArgsConstructor
@AllArgsConstructor
public class FileInfoUserResponse {
@JsonProperty("all_files_size")
private Long allFileSize;
@JsonProperty("all_files_quantity")
private Integer fileCount;
@JsonProperty("all_files_check")
private Integer filesCheck;
@JsonProperty("all_files_violation")
private Integer filesViolation;
@JsonProperty("images_size")
private Long imagesSize;
@JsonProperty("images_quantity")
private Integer imagesCount;
@JsonProperty("images_check")
private Integer imagesCheck;
@JsonProperty("images_violations")
private Integer imagesViolations;
@JsonProperty("videos_size")
private Long videosSize;
@JsonProperty("videos_quantity")
private Integer videosCount;
@JsonProperty("videos_check")
private Integer videosCheck;
@JsonProperty("videos_violations")
private Integer videosViolations;
@JsonProperty("audios_size")
private Long audiosSize;
@JsonProperty("audios_quantity")
private Integer audiosCount;
@JsonProperty("audios_check")
private Integer audiosCheck;
@JsonProperty("audios_violations")
private Integer audiosViolations;
}
@@ -0,0 +1,10 @@
package ru.soune.nocopy.dto.file;
import com.fasterxml.jackson.annotation.JsonProperty;
import lombok.Data;
@Data
public class ImageSearchRequest {
@JsonProperty("file_id")
private String fileId;
}
@@ -0,0 +1,15 @@
package ru.soune.nocopy.dto.file;
import lombok.Builder;
import lombok.Value;
@Value
@Builder
public class SimilarFileDTO {
String fileId;
String originalFileName;
Long fileSize;
Integer hammingDistance;
String similarityLevel;
Long ownerId;
}
@@ -0,0 +1,18 @@
package ru.soune.nocopy.dto.file;
import lombok.AllArgsConstructor;
import lombok.Builder;
import lombok.Data;
import lombok.NoArgsConstructor;
import java.util.List;
@Data
@Builder
@NoArgsConstructor
@AllArgsConstructor
public class SimilarityFilter {
private List<String> similarityLevels;
private Integer page;
private Integer size;
}
@@ -0,0 +1,36 @@
package ru.soune.nocopy.dto.file;
import com.fasterxml.jackson.annotation.JsonIgnoreProperties;
import com.fasterxml.jackson.annotation.JsonProperty;
import lombok.Data;
import java.util.List;
@Data
@JsonIgnoreProperties(ignoreUnknown = true)
public class YandexSearchResponse {
@JsonProperty("images")
private List<ImageResult> images;
@Data
@JsonIgnoreProperties(ignoreUnknown = true)
public static class ImageResult {
@JsonProperty("url")
private String url;
@JsonProperty("pageUrl")
private String pageUrl;
@JsonProperty("pageTitle")
private String pageTitle;
@JsonProperty("width")
private Integer width;
@JsonProperty("height")
private Integer height;
@JsonProperty("host")
private String host;
}
}
@@ -30,7 +30,7 @@ public class AuthToken {
private String token;
@Column(name = "expires_at", nullable = false)
private LocalDateTime expiresAt = LocalDateTime.now().plusDays(30);
private LocalDateTime expiresAt = LocalDateTime.now().plusHours(1);
@CreatedDate
@Column(name = "created_at", updatable = false, nullable = false)
@@ -41,8 +41,4 @@ public class AuthToken {
@Column(name = "is_active")
private Boolean isActive = true;
public boolean isValid() {
return Boolean.TRUE.equals(isActive) && expiresAt.isAfter(LocalDateTime.now());
}
}
@@ -1,49 +0,0 @@
package ru.soune.nocopy.entity;
import jakarta.persistence.*;
import lombok.*;
import org.springframework.data.annotation.CreatedDate;
import org.springframework.data.jpa.domain.support.AuditingEntityListener;
import java.time.LocalDateTime;
@Entity
@Table(name = "image_protection")
@Getter
@Setter
@NoArgsConstructor
@AllArgsConstructor
@ToString
@EntityListeners(AuditingEntityListener.class)
public class ImageProtection {
@Id
@GeneratedValue(strategy = GenerationType.IDENTITY)
private Long protectionId;
@ManyToOne(fetch = FetchType.LAZY)
@JoinColumn(name = "user_id", nullable = false)
@ToString.Exclude
private User user;
@ManyToOne(fetch = FetchType.LAZY)
@JoinColumn(name = "content_id", nullable = false)
@ToString.Exclude
private UserContent content;
@Column(name = "protection_method", nullable = false, length = 50)
private String protectionMethod;
@Column(name = "protection_level", nullable = false)
private Integer protectionLevel;
@Column(name = "is_active", nullable = false)
private Boolean isActive = true;
@CreatedDate
@Column(name = "applied_at", nullable = false, updatable = false)
private LocalDateTime appliedAt;
@Column(name = "metadata", columnDefinition = "JSON")
private String metadata;
}
@@ -84,10 +84,4 @@ public class User {
@OneToMany(mappedBy = "user", cascade = CascadeType.ALL, orphanRemoval = true)
@JsonIgnore
@ToString.Exclude
private List<Violation> violations = new ArrayList<>();
@OneToMany(mappedBy = "user", cascade = CascadeType.ALL, orphanRemoval = true)
@JsonIgnore
@ToString.Exclude
private List<ImageProtection> imageProtections = new ArrayList<>();
}
private List<Violation> violations = new ArrayList<>();}
@@ -1,11 +1,15 @@
package ru.soune.nocopy.entity.file;
import com.fasterxml.jackson.annotation.JsonIgnore;
import jakarta.persistence.*;
import lombok.*;
import org.hibernate.annotations.GenerationTime;
import org.springframework.data.annotation.CreatedDate;
import org.springframework.data.annotation.LastModifiedDate;
import org.springframework.data.jpa.domain.support.AuditingEntityListener;
import java.sql.DriverManager;
import java.sql.SQLException;
import java.time.LocalDateTime;
@Data
@@ -16,11 +20,13 @@ import java.time.LocalDateTime;
@Table(name = "file_entities")
@EntityListeners(AuditingEntityListener.class)
public class FileEntity {
@Id
@GeneratedValue(strategy = GenerationType.UUID)
private String id;
@Column(name = "support_id", unique = true, updatable = false)
private Integer supportId;
@Column(name = "user_id", nullable = false)
private Long userId;
@@ -60,11 +66,34 @@ public class FileEntity {
@Column(name = "updated_at")
private LocalDateTime updatedAt;
@Column(name = "protected_file_path")
private String protectedFilePath;
@Column(name = "protection_status")
@Enumerated(EnumType.STRING)
private ProtectionStatus protectionStatus;
@Column(name = "protected_at")
private LocalDateTime protectedAt;
@Column(name = "signature")
@JsonIgnore
private String signature;
@OneToOne(mappedBy = "file", cascade = CascadeType.ALL, orphanRemoval = true)
@JsonIgnore
private ImageHashEntity imageHash;
@PrePersist
public void prePersist() {
if (this.status == null) {
this.status = FileStatus.ACTIVE;
}
if (this.protectionStatus == null) {
this.protectionStatus = ProtectionStatus.NOT_PROTECTED;
}
if (this.createdAt == null) {
this.createdAt = LocalDateTime.now();
}
@@ -4,5 +4,7 @@ public enum FileStatus {
ACTIVE,
DELETED,
PROCESSING,
VIOLATION,
CHECKED,
ERROR
}
@@ -7,24 +7,10 @@ import java.util.List;
@Getter
public enum FileType {
PHOTO("photo", Arrays.asList("jpg", "jpeg", "png", "gif", "bmp")),
IMAGE("image", Arrays.asList("jpg", "jpeg", "png", "gif", "bmp", "svg", "tiff", "tif", "ico",
"psd", "ai", "eps", "raw", "heic", "heif")),
IMAGE("image", Arrays.asList("jpg", "jpeg", "png", "gif", "bmp", "webp", "jfif")),
VIDEO("video", Arrays.asList("mp4", "avi", "mov", "wmv", "flv", "mkv", "webm", "m4v", "mpg", "mpeg",
"3gp", "3g2", "f4v", "m2ts", "mts", "vob", "ogv", "divx")),
AUDIO("audio", Arrays.asList("mp3", "wav", "ogg", "aac", "flac", "m4a", "wma", "aiff", "aif", "amr",
"opus", "mka", "ac3", "alac")),
DOCUMENT("document", Arrays.asList(
"pdf", "txt", "rtf",
"doc", "docx", "xls", "xlsx", "ppt", "pptx", "pps", "ppsx", "dot", "dotx", "xlt", "xltx", "pot", "potx",
"odt", "ods", "odp", "odg", "odf", "odb", "odc", "odi", "odm", "ott", "ots", "otp", "otg", "oth",
"sxw", "sxc", "sxi", "sxd", "sxg", "stc", "sti", "stw", "sxm",
"pages", "numbers", "key",
"csv", "tsv", "xml", "html", "htm", "tex", "md", "markdown",
"epub", "mobi", "azw", "azw3", "fb2",
"wps", "wpt", "et", "dps", "vsd", "vsdx",
"java", "py", "cpp", "c", "h", "js", "css", "php", "sql", "json", "yaml", "yml", "sh", "bat",
"one", "note"));
AUDIO("audio", List.of("wav"));
private final String displayName;
private final List<String> allowedExtensions;
@@ -69,6 +69,7 @@ public class FileUploadSession {
private String extension;
@Column(name = "retry_count")
@Builder.Default
private Integer retryCount = 0;
@Column(name = "completed_at")
@@ -81,12 +82,13 @@ public class FileUploadSession {
)
@MapKeyColumn(name = "chunk_number")
@Column(name = "chunk_path")
@Builder.Default
private Map<Integer, String> chunkPaths = new HashMap<>();
@PrePersist
public void prePersist() {
this.createdAt = LocalDateTime.now();
this.expiresAt = LocalDateTime.now().plusHours(24);
this.expiresAt = LocalDateTime.now().plusMinutes(1);
if (this.chunksUploaded == null) {
this.chunksUploaded = 0;
}
@@ -0,0 +1,39 @@
package ru.soune.nocopy.entity.file;
import jakarta.persistence.*;
import lombok.AllArgsConstructor;
import lombok.Builder;
import lombok.Data;
import lombok.NoArgsConstructor;
import java.time.LocalDateTime;
@Entity
@Table(name = "image_hashes")
@Data
@NoArgsConstructor
@AllArgsConstructor
@Builder
public class ImageHashEntity {
@Id
@Column(name = "file_id")
private String fileId;
@OneToOne(fetch = FetchType.LAZY)
@MapsId
@JoinColumn(name = "file_id")
private FileEntity file;
@Column(name = "hash64_hi")
private Long hash64Hi;
@Column(name = "hash64_lo")
private Long hash64Lo;
@Column(name = "hash_algorithm", nullable = false)
private String hashAlgorithm;
@Column(name = "created_at")
private LocalDateTime createdAt;
}
@@ -0,0 +1,9 @@
package ru.soune.nocopy.entity.file;
public enum ProtectionStatus {
NOT_PROTECTED,
PROCESSING,
PROTECTED,
FAILED,
FAILED_SAVE
}
@@ -2,6 +2,6 @@ package ru.soune.nocopy.exception;
public class ChunkSizeExceededException extends RuntimeException {
public ChunkSizeExceededException(long actualSize, long maxSize) {
super(String.format("Chunk size exceeded: %d > %d", actualSize, maxSize));
super(String.format("Chunk size not valid: " + actualSize));
}
}
@@ -0,0 +1,20 @@
package ru.soune.nocopy.exception;
public class DuplicateImageException extends RuntimeException {
private final String duplicateFileId;
private final Long userId;
public DuplicateImageException(String message, String duplicateFileId, Long userId) {
super(message);
this.duplicateFileId = duplicateFileId;
this.userId = userId;
}
public String duplicateFileId() {
return duplicateFileId;
}
public Long userId() {
return userId;
}
}
@@ -0,0 +1,37 @@
package ru.soune.nocopy.handler;
import com.fasterxml.jackson.databind.ObjectMapper;
import lombok.RequiredArgsConstructor;
import lombok.extern.slf4j.Slf4j;
import org.springframework.stereotype.Component;
import ru.soune.nocopy.dto.BaseRequest;
import ru.soune.nocopy.dto.BaseResponse;
import ru.soune.nocopy.dto.MessageCode;
import ru.soune.nocopy.dto.TokenProcessRequest;
import ru.soune.nocopy.exception.NotFoundAuthToken;
import ru.soune.nocopy.service.auth.AuthService;
import java.util.Map;
@Slf4j
@Component
@RequiredArgsConstructor
public class AuthRequestHandler implements RequestHandler {
private final ObjectMapper objectMapper;
private final AuthService authService;
@Override
public BaseResponse handle(BaseRequest request) throws NotFoundAuthToken {
TokenProcessRequest tokenProcessRequest = objectMapper.convertValue(request.getMessageBody(),
TokenProcessRequest.class);
authService.useUserAuthToken(tokenProcessRequest.getToken());
return new BaseResponse(request.getMsgId(),
MessageCode.SUCCESS.getCode(),
"Token processed successfully",
Map.of("token", tokenProcessRequest.getToken()));
}
}
@@ -6,12 +6,17 @@ import lombok.extern.slf4j.Slf4j;
import org.springframework.stereotype.Component;
import ru.soune.nocopy.dto.*;
import ru.soune.nocopy.dto.file.*;
import ru.soune.nocopy.entity.AuthToken;
import ru.soune.nocopy.entity.file.FileEntity;
import ru.soune.nocopy.entity.file.FileStatus;
import ru.soune.nocopy.exception.FileEntityNotFoundException;
import ru.soune.nocopy.exception.NotFoundAuthToken;
import ru.soune.nocopy.repository.AuthTokenRepository;
import ru.soune.nocopy.repository.FileEntityRepository;
import ru.soune.nocopy.service.auth.AuthService;
import ru.soune.nocopy.service.file.FileEntityService;
import ru.soune.nocopy.service.file.FileStatsService;
import java.util.Arrays;
import java.util.List;
@Slf4j
@Component
@@ -19,9 +24,15 @@ import java.util.Arrays;
public class FileEntityHandler implements RequestHandler {
private final FileEntityService fileEntityService;
private final AuthTokenRepository authTokenRepository;
private final AuthService authService;
private final FileStatsService fileStatsService;
private final ObjectMapper objectMapper;
private final FileEntityRepository fileEntityRepository;
@Override
public BaseResponse handle(BaseRequest request) {
try {
@@ -31,15 +42,17 @@ public class FileEntityHandler implements RequestHandler {
String action = fileRequest.getAction();
switch (action) {
case "get_file_info":
case "file_info":
return handleGetFileInfo(request, fileRequest);
case "get_file_by_session":
case "user_files_info":
return handleGetFilesUserInfo(request, fileRequest);
case "file_by_session":
return handleGetFileBySession(request, fileRequest);
case "get_user_files":
case "user_files":
return handleGetUserFiles(request, fileRequest);
case "search_files":
return handleSearchFiles(request, fileRequest);
case "get_storage_usage":
case "storage_usage":
return handleGetStorageUsage(request, fileRequest);
case "delete_file":
return handleDeleteFile(request, fileRequest);
@@ -47,8 +60,9 @@ public class FileEntityHandler implements RequestHandler {
ActionResponse response = ActionResponse.builder()
.action(action)
.availableActions(Arrays.asList(
"get_file_info", "get_file_by_session", "get_user_files",
"search_files", "get_storage_usage", "delete_file"))
"file_info", "file_by_session", "user_files",
"search_files", "storage_usage", "delete_file",
"user_files_info"))
.build();
return new BaseResponse(request.getMsgId(),
@@ -65,9 +79,19 @@ public class FileEntityHandler implements RequestHandler {
}
}
private BaseResponse handleGetFilesUserInfo(BaseRequest request, FileEntityRequest fileRequest) {
Long userId = authService.useUserAuthToken(fileRequest.getToken());
FileInfoUserResponse userFileStats = fileStatsService.getUserFileStats(userId);
return new BaseResponse(request.getMsgId(),
MessageCode.SUCCESS.getCode(),
MessageCode.SUCCESS.getDescription(),
userFileStats);
}
private BaseResponse handleGetFileInfo(BaseRequest request, FileEntityRequest fileRequest) {
try {
Long userId = getUserIdFromToken(fileRequest.getToken());
Long userId = authService.useUserAuthToken(fileRequest.getToken());
FileEntityResponse fileInfo = fileEntityService.getById(fileRequest.getFileId(), request.getVersion());
if (!fileInfo.getUserId().equals(userId)) {
@@ -98,7 +122,7 @@ public class FileEntityHandler implements RequestHandler {
private BaseResponse handleGetFileBySession(BaseRequest request, FileEntityRequest fileRequest) {
try {
Long userId = getUserIdFromToken(fileRequest.getToken());
Long userId = authService.useUserAuthToken(fileRequest.getToken());
FileEntityResponse fileInfo = fileEntityService.getByUploadSessionId(fileRequest.getUploadSessionId(),
request.getVersion());
@@ -130,7 +154,7 @@ public class FileEntityHandler implements RequestHandler {
private BaseResponse handleGetUserFiles(BaseRequest request, FileEntityRequest fileRequest) {
try {
Long userId = getUserIdFromToken(fileRequest.getToken());
Long userId = authService.useUserAuthToken(fileRequest.getToken());
int page = fileRequest.getPage() != null ? fileRequest.getPage() : 1;
int pageSize = fileRequest.getPageSize() != null ? fileRequest.getPageSize() : 20;
@@ -157,7 +181,7 @@ public class FileEntityHandler implements RequestHandler {
private BaseResponse handleSearchFiles(BaseRequest request, FileEntityRequest fileRequest) {
try {
Long userId = getUserIdFromToken(fileRequest.getToken());
Long userId = authService.useUserAuthToken(fileRequest.getToken());
int page = fileRequest.getPage() != null ? fileRequest.getPage() : 1;
int pageSize = fileRequest.getPageSize() != null ? fileRequest.getPageSize() : 20;
@@ -168,17 +192,22 @@ public class FileEntityHandler implements RequestHandler {
FileResponse allFiles = fileEntityService.getUserFiles(userId, 1, 1000, request.getVersion());
var filteredFiles = allFiles.getFiles().stream()
.filter(f -> f.getOriginalFileName().toLowerCase().contains(fileRequest.getQuery().toLowerCase()))
.filter(f -> (f.getOriginalFileName() + f.getFileExtension()).toLowerCase()
.contains(fileRequest.getQuery().toLowerCase()))
.toList();
int start = (page - 1) * pageSize;
int end = Math.min(start + pageSize, filteredFiles.size());
List<FileEntityResponse> fileEntityResponses = filteredFiles.subList(start, Math.min(end, filteredFiles.size()));
long sumSize = fileEntityResponses.stream()
.mapToLong(FileEntityResponse::getFileSize)
.sum();
FileListResponse response = FileListResponse.builder()
.files(filteredFiles.subList(start, Math.min(end, filteredFiles.size())))
.files(fileEntityResponses)
.totalCount(filteredFiles.size())
.totalSize(allFiles.getTotalSize())
.formattedTotalSize(allFiles.getFormattedTotalSize())
.totalSize(sumSize)
.formattedTotalSize(fileEntityService.formatFileSize(sumSize))
.page(page)
.pageSize(pageSize)
.build();
@@ -187,24 +216,19 @@ public class FileEntityHandler implements RequestHandler {
MessageCode.SUCCESS.getCode(),
MessageCode.SUCCESS.getDescription(),
response);
} catch (NotFoundAuthToken e) {
return new BaseResponse(request.getMsgId(),
MessageCode.INVALID_TOKEN.getCode(),
"Authentication required",
null);
return new BaseResponse(request.getMsgId(), MessageCode.INVALID_TOKEN.getCode(),
"Authentication required", null);
} catch (Exception e) {
log.error("Error searching files", e);
return new BaseResponse(request.getMsgId(),
MessageCode.FILE_UPLOAD_ERROR.getCode(),
"Failed to search files: " + e.getMessage(),
null);
return new BaseResponse(request.getMsgId(), MessageCode.FILE_UPLOAD_ERROR.getCode(),
"Failed to search files: " + e.getMessage(), null);
}
}
private BaseResponse handleGetStorageUsage(BaseRequest request, FileEntityRequest fileRequest) {
try {
Long userId = getUserIdFromToken(fileRequest.getToken());
Long userId = authService.useUserAuthToken(fileRequest.getToken());
long usage = fileEntityService.getUserStorageUsed(userId);
StorageUsageResponse response = StorageUsageResponse.builder()
@@ -234,8 +258,10 @@ public class FileEntityHandler implements RequestHandler {
private BaseResponse handleDeleteFile(BaseRequest request, FileEntityRequest fileRequest) {
try {
Long userId = getUserIdFromToken(fileRequest.getToken());
FileEntityResponse fileInfo = fileEntityService.getById(fileRequest.getFileId(), request.getVersion());
Long userId = authService.useUserAuthToken(fileRequest.getToken());
String fileId = fileRequest.getFileId();
FileEntityResponse fileInfo = fileEntityService.getById(fileId, request.getVersion());
DeleteFileResponse response;
if (!fileInfo.getUserId().equals(userId)) {
return new BaseResponse(request.getMsgId(),
@@ -244,18 +270,27 @@ public class FileEntityHandler implements RequestHandler {
null);
}
fileEntityService.markAsDeleted(fileRequest.getFileId());
FileEntity fileEntity = fileEntityRepository.findById(fileId)
.orElseThrow(() -> new FileEntityNotFoundException(fileId));
DeleteFileResponse response = DeleteFileResponse.builder()
if (fileEntity.getStatus().equals(FileStatus.DELETED)) {
fileEntityService.deleteFromDisk(fileEntity);
response = DeleteFileResponse.builder()
.fileId(fileRequest.getFileId())
.message("File deleted from disk")
.build();
} else {
fileEntityService.markAsDeleted(fileEntity);
response = DeleteFileResponse.builder()
.fileId(fileRequest.getFileId())
.message("File marked as deleted")
.build();
}
return new BaseResponse(request.getMsgId(),
MessageCode.SUCCESS.getCode(),
"File marked as deleted",
response);
} catch (NotFoundAuthToken e) {
return new BaseResponse(request.getMsgId(),
MessageCode.INVALID_TOKEN.getCode(),
@@ -270,20 +305,6 @@ public class FileEntityHandler implements RequestHandler {
}
}
private Long getUserIdFromToken(String token) {
if (token == null || token.isBlank()) {
throw new NotFoundAuthToken("Token is required");
}
if (token.startsWith("Bearer ")) {
token = token.replace("Bearer ", "");
}
AuthToken authToken = authTokenRepository.findByToken(token)
.orElseThrow(() -> new NotFoundAuthToken("Token not found"));
return authToken.getUser().getId();
}
private String formatFileSize(long size) {
if (size < 1024) return size + " B";
int exp = (int) (Math.log(size) / Math.log(1024));
@@ -3,6 +3,7 @@ package ru.soune.nocopy.handler;
import com.fasterxml.jackson.databind.ObjectMapper;
import lombok.RequiredArgsConstructor;
import lombok.extern.slf4j.Slf4j;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.stereotype.Component;
import org.springframework.validation.BeanPropertyBindingResult;
import org.springframework.validation.BindingResult;
@@ -16,6 +17,7 @@ import ru.soune.nocopy.exception.*;
import ru.soune.nocopy.handler.validator.FileUploadRequestValidator;
import ru.soune.nocopy.repository.AuthTokenRepository;
import ru.soune.nocopy.repository.FileUploadSessionRepository;
import ru.soune.nocopy.service.auth.AuthService;
import ru.soune.nocopy.service.file.FileUploadService;
import java.util.*;
@@ -26,11 +28,20 @@ import java.util.stream.Collectors;
@RequiredArgsConstructor
public class FileUploadHandler implements RequestHandler {
private final FileUploadService fileUploadService;
private final AuthService authService;
private final AuthTokenRepository authTokenRepository;
private final FileUploadSessionRepository fileUploadSessionRepository;
private final ObjectMapper objectMapper;
private final FileUploadRequestValidator fileUploadRequestValidator;
@Value("${file.storage.max-file-size}")
private long maxFileSize;
@Override
public BaseResponse handle(BaseRequest request) {
try {
@@ -44,18 +55,18 @@ public class FileUploadHandler implements RequestHandler {
return handleInitUpload(request, fileRequest);
case "file_types":
return handleGetFileTypes(request);
case "file_extension":
return handleGetFileExtensions(fileRequest, request);
case "chunks":
return handleGetChunkStatus(request, fileRequest);
case "cancel":
return handleCancelUpload(request, fileRequest);
case "progress":
return handleGetProgress(request, fileRequest);
case "retry":
return handleRetryUpload(request, fileRequest);
default:
ActionResponse response = ActionResponse.builder()
.action(action)
.availableActions(Arrays.asList("init", "file_types", "chunks", "cancel", "progress", "retry"))
.availableActions(Arrays.asList("init", "file_types", "chunks", "cancel", "progress"))
.build();
return new BaseResponse(request.getMsgId(), MessageCode.INVALID_ACTION.getCode(),
@@ -78,8 +89,7 @@ public class FileUploadHandler implements RequestHandler {
MessageCode.INVALID_TOKEN.getDescription(), Map.of("token", token));
}
AuthToken authToken = tokenOptional.orElseThrow(() -> new NotFoundAuthToken("Token not found"));
Long userId = authService.useUserAuthToken(token);
BindingResult bindingResult = new BeanPropertyBindingResult(fileRequest, "fileRequest");
fileUploadRequestValidator.validate(fileRequest, bindingResult);
@@ -97,7 +107,7 @@ public class FileUploadHandler implements RequestHandler {
}
FileUploadSession session = fileUploadService.initUpload(
authToken.getUser().getId(),
userId,
fileRequest.getFileName(),
fileRequest.getFileType(),
fileRequest.getExtension(),
@@ -167,40 +177,13 @@ public class FileUploadHandler implements RequestHandler {
}
}
private BaseResponse handleRetryUpload(BaseRequest request, FileUploadRequest fileRequest) {
try {
fileUploadService.retryFailedUpload(fileRequest.getUploadId());
ru.soune.nocopy.dto.file.UploadProgressResponse progress =
fileUploadService.getUploadProgress(fileRequest.getUploadId());
RetryUploadResponse response = RetryUploadResponse.builder()
.uploadId(progress.getUploadId())
.message("Upload retry initiated")
.status(progress.getStatus().toString())
.build();
return new BaseResponse(request.getMsgId(), MessageCode.SUCCESS.getCode(), "Upload retry initiated",
response);
} catch (UploadSessionNotFoundException e) {
return new BaseResponse(request.getMsgId(), MessageCode.FILE_NOT_FOUND.getCode(),
"Upload session not found", null);
} catch (FileUploadException e) {
return new BaseResponse(request.getMsgId(), MessageCode.FILE_UPLOAD_ERROR.getCode(), e.getMessage(),
null);
} catch (Exception e) {
log.error("Error retrying upload", e);
return new BaseResponse(request.getMsgId(), MessageCode.FILE_UPLOAD_ERROR.getCode(),
"Failed to retry upload", null);
}
}
private BaseResponse handleGetChunkStatus(BaseRequest request, FileUploadRequest fileRequest) {
try {
FileUploadSession session = fileUploadSessionRepository.findById(fileRequest.getUploadId())
.orElseThrow(() -> new UploadSessionNotFoundException(fileRequest.getUploadId()));
Map<String, Boolean> chunkStatus = new HashMap<>();
for (int i = 0; i < session.getTotalChunks(); i++) {
chunkStatus.put("chunk_" + i, session.getChunkPaths().containsKey(i));
}
@@ -215,7 +198,6 @@ public class FileUploadHandler implements RequestHandler {
return new BaseResponse(request.getMsgId(), MessageCode.SUCCESS.getCode(),
MessageCode.SUCCESS.getDescription(), response);
} catch (UploadSessionNotFoundException e) {
return new BaseResponse(request.getMsgId(), MessageCode.FILE_NOT_FOUND.getCode(),
"Upload session not found", null);
@@ -243,4 +225,24 @@ public class FileUploadHandler implements RequestHandler {
"Failed to get file types", null);
}
}
private BaseResponse handleGetFileExtensions(FileUploadRequest fileRequest, BaseRequest request) {
try {
FileType fileType = FileType.valueOf(fileRequest.getFileType().toUpperCase());
List<String> allowedExtensions = fileType.getAllowedExtensions();
FileExtensionResponse response = FileExtensionResponse.builder()
.extension(allowedExtensions)
.count(allowedExtensions.size())
.maxFileSize(maxFileSize)
.build();
return new BaseResponse(request.getMsgId(), MessageCode.SUCCESS.getCode(),
MessageCode.SUCCESS.getDescription(), response);
} catch (Exception e) {
log.error("Error getting file extensions", e);
return new BaseResponse(request.getMsgId(), MessageCode.FILE_UPLOAD_ERROR.getCode(),
"Failed to get file extensions", null);
}
}
}
@@ -5,15 +5,16 @@ import org.springframework.context.support.DefaultMessageSourceResolvable;
import org.springframework.http.HttpStatus;
import org.springframework.http.ResponseEntity;
import org.springframework.web.bind.MethodArgumentNotValidException;
import org.springframework.web.bind.annotation.ExceptionHandler;
import org.springframework.web.bind.annotation.ResponseStatus;
import org.springframework.web.bind.annotation.RestControllerAdvice;
import org.springframework.web.bind.annotation.*;
import ru.soune.nocopy.dto.BaseResponse;
import ru.soune.nocopy.dto.MessageCode;
import ru.soune.nocopy.exception.*;
import java.util.Map;
@RestControllerAdvice
@AllArgsConstructor
@ControllerAdvice
public class GlobalExceptionHandler {
@ExceptionHandler(MethodArgumentNotValidException.class)
@@ -81,4 +82,12 @@ public class GlobalExceptionHandler {
"message" ,ex.getMessage()
));
}
@ExceptionHandler(DuplicateImageException.class)
@ResponseBody
public ResponseEntity<BaseResponse> handleDuplicateImage(DuplicateImageException e) {
return ResponseEntity.ok().body(new BaseResponse(
20004, MessageCode.DUPLICATE_FILE_UPLOAD.getCode(),
"Duplicate image detected", e.getMessage()));
}
}
@@ -0,0 +1,36 @@
package ru.soune.nocopy.handler;
import com.fasterxml.jackson.databind.ObjectMapper;
import lombok.RequiredArgsConstructor;
import lombok.extern.slf4j.Slf4j;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.stereotype.Component;
import ru.soune.nocopy.dto.BaseRequest;
import ru.soune.nocopy.dto.BaseResponse;
import ru.soune.nocopy.dto.MessageCode;
import ru.soune.nocopy.dto.file.ImageSearchRequest;
import ru.soune.nocopy.dto.file.YandexSearchResponse;
import ru.soune.nocopy.service.YandexSearchService;
@Slf4j
@Component
@RequiredArgsConstructor
public class ImageFoundRequestHandler implements RequestHandler {
private final ObjectMapper objectMapper;
private final YandexSearchService yandexSearchService;
@Override
public BaseResponse handle(BaseRequest request) throws Exception {
ImageSearchRequest imageSearchRequest = objectMapper.convertValue(request.getMessageBody(),
ImageSearchRequest.class);
String fileId = imageSearchRequest.getFileId();
YandexSearchResponse response = yandexSearchService.searchByFileEntity(fileId);
return new BaseResponse(request.getMsgId(), MessageCode.SUCCESS.getCode(),
MessageCode.SUCCESS.getDescription(), response);
}
}
@@ -7,7 +7,7 @@ import ru.soune.nocopy.dto.*;
import ru.soune.nocopy.entity.AuthToken;
import ru.soune.nocopy.exception.NotValidFieldException;
import ru.soune.nocopy.repository.UserRepository;
import ru.soune.nocopy.service.AuthService;
import ru.soune.nocopy.service.auth.AuthService;
import java.util.Arrays;
import java.util.Map;
@@ -17,7 +17,9 @@ import java.util.Map;
public class LoginRequestHandler implements RequestHandler {
private final UserRepository userRepository;
private final AuthService authService;
private final ObjectMapper objectMapper;
@Override
@@ -34,9 +36,12 @@ public class LoginRequestHandler implements RequestHandler {
}
AuthToken authToken = authService.login(loginRequest);
String token = authToken.getToken();
LoginAnswer loginAnswer = new LoginAnswer();
loginAnswer.setToken(authToken.getToken());
loginAnswer.setToken(token);
authService.useUserAuthToken(token);
return new BaseResponse(request.getMsgId(), MessageCode.SUCCESS.getCode(),
MessageCode.SUCCESS.getDescription(), loginAnswer);
@@ -9,7 +9,7 @@ import ru.soune.nocopy.entity.User;
import ru.soune.nocopy.exception.NotValidFieldException;
import ru.soune.nocopy.repository.AuthTokenRepository;
import ru.soune.nocopy.repository.UserRepository;
import ru.soune.nocopy.service.AuthService;
import ru.soune.nocopy.service.auth.AuthService;
import java.util.Objects;
import java.util.Optional;
@@ -12,7 +12,7 @@ import ru.soune.nocopy.exception.NotValidFieldException;
import ru.soune.nocopy.exception.ValidationException;
import ru.soune.nocopy.handler.validator.RegRequestValidator;
import ru.soune.nocopy.repository.UserRepository;
import ru.soune.nocopy.service.AuthService;
import ru.soune.nocopy.service.auth.AuthService;
import java.util.Arrays;
import java.util.Map;
@@ -52,9 +52,11 @@ public class RegRequestHandler implements RequestHandler {
}
AuthToken authToken = authService.register(regRequest);
String token = authToken.getToken();
authService.useUserAuthToken(token);
RegAnswer regAnswer = new RegAnswer();
regAnswer.setToken(authToken.getToken());
regAnswer.setToken(token);
return new BaseResponse(request.getMsgId(), MessageCode.SUCCESS.getCode(),
MessageCode.SUCCESS.getDescription(), regAnswer);
@@ -1,19 +1,16 @@
package ru.soune.nocopy.handler.validator;
import org.apache.commons.validator.routines.DomainValidator;
import org.apache.commons.validator.routines.EmailValidator;
import org.springframework.stereotype.Component;
import org.springframework.validation.Errors;
import org.springframework.validation.Validator;
import ru.soune.nocopy.dto.RegRequest;
import java.net.InetAddress;
import java.net.UnknownHostException;
import java.util.*;
@Component
public class RegRequestValidator implements Validator {
private static final String COMPANY_REGEX = "^[a-zA-Zа-яА-ЯёЁ0-9\\s\\-&.,'()]{0,200}$";
private static final String COMPANY_REGEX = "^[a-zA-Zа-яА-ЯёЁ0-9\\s\\-&.,'()\"\"«»„‟”“”‘'’‹›❛❜❝❞〝〞〟]{1,200}$";
private static final String NAME_REGEX = "^[a-zA-Zа-яА-ЯёЁ\\s\\-'.]{2,100}$";
@@ -74,21 +71,6 @@ public class RegRequestValidator implements Validator {
errors.rejectValue("companyName", "companyName.invalid.chars",
"Company name contains invalid characters");
}
validateForbiddenWords(trimmedCompany, "companyName", errors,
Arrays.asList("admin", "root", "system", "test"));
}
private void validateForbiddenWords(String text, String fieldName,
Errors errors, List<String> forbiddenWords) {
String lowerText = text.toLowerCase();
for (String word : forbiddenWords) {
if (lowerText.contains(word)) {
errors.rejectValue(fieldName, fieldName + ".forbidden.word",
"Contains forbidden word: " + word);
break;
}
}
}
private void validateEmail(String email, Errors errors) {
@@ -156,15 +138,6 @@ public class RegRequestValidator implements Validator {
errors.rejectValue("password", "password.contains.spaces",
"Password cannot contain spaces");
}
// recomment if need complexity
// checkPasswordComplexity(password, errors);
// reccoment if need check simply standart password
// checkCommonPasswords(password, errors);
// reccoment if need check simply standart password
// checkForSequences(password, errors);
}
private void checkPasswordComplexity(String password, Errors errors) {
@@ -4,11 +4,13 @@ import org.springframework.data.jpa.repository.JpaRepository;
import org.springframework.stereotype.Repository;
import ru.soune.nocopy.entity.AuthToken;
import java.time.LocalDate;
import java.time.LocalDateTime;
import java.util.List;
import java.util.Optional;
@Repository
public interface AuthTokenRepository extends JpaRepository<AuthToken, Long> {
Optional<AuthToken> findByTokenAndExpiresAtAfter(String token, LocalDate expiresAtAfter);
List<AuthToken> findByExpiresAtBefore(LocalDateTime expiresAtBefore);
Optional<AuthToken> findByLastUsedAtBefore(LocalDateTime lastUsedAt);
Optional<AuthToken> findByToken(String token);
}
@@ -6,28 +6,45 @@ import org.springframework.data.repository.query.Param;
import org.springframework.stereotype.Repository;
import ru.soune.nocopy.entity.file.FileEntity;
import ru.soune.nocopy.entity.file.FileStatus;
import ru.soune.nocopy.entity.file.ProtectionStatus;
import java.util.List;
import java.util.Optional;
@Repository
public interface FileEntityRepository extends JpaRepository<FileEntity, String> {
List<FileEntity> findByUserId(Long userId);
Optional<FileEntity> findByUserIdAndChecksum(Long userId, String imageHash);
List<FileEntity> findByUserIdAndStatus(Long userId, FileStatus status);
Optional<FileEntity> findByFilePath(String filePath);
Optional<FileEntity> findByUploadSessionId(String uploadSessionId);
List<FileEntity> findByProtectionStatus(ProtectionStatus protectionStatus);
boolean existsByFilePath(String filePath);
@Query("SELECT SUM(f.fileSize) FROM FileEntity f WHERE f.userId = :userId AND f.status = 'ACTIVE'")
Long getTotalSizeByUserId(@Param("userId") Long userId);
@Query("SELECT f FROM FileEntity f WHERE f.protectionStatus = null AND f.status = 'ACTIVE' OR " +
"f.protectionStatus = 'NOT_PROTECTED' AND f.status = 'ACTIVE'")
List<FileEntity> findAllActiveFilesAndNotProtected();
@Query("SELECT f FROM FileEntity f WHERE f.userId = :userId AND f.originalFileName LIKE %:keyword%")
List<FileEntity> searchByFileName(@Param("userId") Long userId, @Param("keyword") String keyword);
@Query("SELECT f FROM FileEntity f WHERE f.status = :status")
List<FileEntity> searchFileEntityByStatus(@Param("status") FileStatus status);
@Query("SELECT f FROM FileEntity f WHERE f.id = :fileId")
FileEntity findByFileId(@Param("fileId") String fileId);
@Query("SELECT f FROM FileEntity f WHERE f.signature = :signature")
FileEntity findBySignature(@Param("signature") String signature);
long countByUserId(Long userId);
}
@@ -0,0 +1,14 @@
package ru.soune.nocopy.repository;
import org.springframework.data.jpa.repository.JpaRepository;
import org.springframework.stereotype.Repository;
import ru.soune.nocopy.entity.file.ImageHashEntity;
import java.util.List;
@Repository
public interface ImageHashRepository extends JpaRepository<ImageHashEntity, String> {
void deleteByFileId(String fileId);
List<ImageHashEntity> findByFileId(String fileId);
ImageHashEntity findByHash64HiAndHash64Lo(Long hash64Hi, Long hash64Lo);
}
@@ -0,0 +1,54 @@
package ru.soune.nocopy.repository;
import org.springframework.data.jpa.repository.JpaRepository;
import org.springframework.data.jpa.repository.Query;
import org.springframework.data.repository.query.Param;
import org.springframework.stereotype.Repository;
import ru.soune.nocopy.entity.file.FileEntity;
import java.util.List;
@Repository
public interface ImageSimilarityRepository
extends JpaRepository<FileEntity, String> {
@Query(value = """
SELECT
f.id AS similarFileId,
f.original_file_name AS originalFileName,
f.file_size AS fileSize,
h.hash64_hi AS hash64Hi,
h.hash64_lo AS hash64Lo
FROM image_hashes ref
JOIN image_hashes h
ON ref.file_id <> h.file_id
JOIN file_entities f
ON f.id = h.file_id
WHERE ref.file_id = :fileId
""",
nativeQuery = true)
List<SimilarImageProjection> findCandidates(
@Param("fileId") String fileId
);
@Query(value = """
SELECT
h.hash64_hi AS hash64Hi,
h.hash64_lo AS hash64Lo,
h.file_id AS fileId,
f.user_id AS userId,
f.original_file_name AS originalFileName,
f.file_size AS fileSize,
f.stored_file_name AS similarFileId
FROM image_hashes h
JOIN file_entities f ON f.id = h.file_id
WHERE h.hash64_hi = :hash64Hi
AND h.hash64_lo = :hash64Lo
""",
nativeQuery = true)
List<SimilarImageProjection> findExactDuplicates(
@Param("hash64Hi") Long hash64_hi,
@Param("hash64Lo") Long hash64_lo);
}
@@ -0,0 +1,10 @@
package ru.soune.nocopy.repository;
public interface SimilarImageProjection {
Long getHash64Hi();
Long getHash64Lo();
String getFileId();
Long getUserId();
String getOriginalFileName();
Long getFileSize();
}
@@ -0,0 +1,180 @@
package ru.soune.nocopy.service;
import lombok.RequiredArgsConstructor;
import org.springframework.data.domain.Page;
import org.springframework.data.domain.PageImpl;
import org.springframework.data.domain.Pageable;
import org.springframework.stereotype.Service;
import ru.soune.nocopy.dto.file.SimilarFileDTO;
import ru.soune.nocopy.dto.file.SimilarityFilter;
import ru.soune.nocopy.repository.ImageHashRepository;
import ru.soune.nocopy.repository.ImageSimilarityRepository;
import ru.soune.nocopy.repository.SimilarImageProjection;
import ru.soune.nocopy.util.FileUtil;
import java.util.AbstractMap;
import java.util.ArrayList;
import java.util.Comparator;
import java.util.List;
import java.util.stream.Collectors;
@Service
@RequiredArgsConstructor
public class FileSimilarityService {
private final ImageSimilarityRepository repository;
private final ImageHashRepository hashRepository;
private final FileUtil fileUtil;
public List<SimilarFileDTO> findSimilarFiles(String fileId) {
var imageHashEntity = hashRepository.findById(fileId)
.orElseThrow(() -> new RuntimeException("Hash not found"));
Long hash64Hi = imageHashEntity.getHash64Hi();
Long hash64Lo = imageHashEntity.getHash64Lo();
List<SimilarImageProjection> candidates = repository.findCandidates(fileId);
return candidates.stream()
.map(c -> {
Long cHi = c.getHash64Hi();
Long cLo = c.getHash64Lo();
int hamming = fileUtil.hamming64(hash64Hi, hash64Lo, cHi, cLo);
String level;
if (hamming <= 5) {
level = "DUPLICATE";
} else if (hamming <= 12) {
level = "SIMILAR";
} else {
level = "DIFFERENT";
}
return SimilarFileDTO.builder()
.fileId(c.getFileId())
.originalFileName(c.getOriginalFileName())
.fileSize(c.getFileSize())
.hammingDistance(hamming)
.similarityLevel(level)
.build();
})
.sorted((a, b) ->
Integer.compare(a.getHammingDistance(), b.getHammingDistance()))
.toList();
}
public List<SimilarFileDTO> findDuplicateByHammingDistance(String fileId, int hammingDistance,
int duplicate, int similar) {
List<SimilarImageProjection> candidates = repository.findCandidates(fileId);
var imageHashEntity = hashRepository.findById(fileId)
.orElseThrow(() -> new RuntimeException("Hash not found"));
Long hash64Hi = imageHashEntity.getHash64Hi();
Long hash64Lo = imageHashEntity.getHash64Lo();
return candidates.stream()
.map(c -> {
Long cHi = c.getHash64Hi();
Long cLo = c.getHash64Lo();
int hamming = fileUtil.hamming64(hash64Hi, hash64Lo, cHi, cLo);
return new AbstractMap.SimpleEntry<>(c, hamming);
})
.filter(entry -> entry.getValue() <= hammingDistance)
.sorted((a, b) -> Integer.compare(a.getValue(), b.getValue()))
.map(entry -> {
SimilarImageProjection similarImageProjection = entry.getKey();
int hamming = entry.getValue();
String level;
if (hamming <= duplicate) {
level = "DUPLICATE";
} else if (hamming <= similar) {
level = "SIMILAR";
} else {
level = "DIFFERENT";
}
return SimilarFileDTO.builder()
.fileId(similarImageProjection.getFileId())
.originalFileName(similarImageProjection.getOriginalFileName())
.fileSize(similarImageProjection.getFileSize())
.hammingDistance(hamming)
.similarityLevel(level)
.ownerId(similarImageProjection.getUserId())
.build();
})
.toList();
}
public List<SimilarImageProjection> findDuplicatedByHash(Long hash64Hi, Long hash64Lo) {
List<SimilarImageProjection> duplicates = repository.findExactDuplicates(hash64Hi,hash64Lo);
if (duplicates.isEmpty()) {
return new ArrayList<>();
}
return duplicates;
}
public Page<SimilarFileDTO> findSimilarFiles(String fileId, SimilarityFilter filter, Pageable pageable) {
var imageHashEntity = hashRepository.findById(fileId)
.orElseThrow(() -> new RuntimeException("Hash not found"));
Long hash64Hi = imageHashEntity.getHash64Hi();
Long hash64Lo = imageHashEntity.getHash64Lo();
List<SimilarImageProjection> candidates = repository.findCandidates(fileId);
List<String> similarityLevels = (filter != null && filter.getSimilarityLevels() != null)
? filter.getSimilarityLevels()
: List.of("DUPLICATE", "SIMILAR", "DIFFERENT");
List<SimilarFileDTO> allResults = candidates.stream()
.map(c -> createSimilarFileResponse(c, hash64Hi, hash64Lo))
.filter(response -> similarityLevels.contains(response.getSimilarityLevel()))
.sorted(Comparator.comparingInt(SimilarFileDTO::getHammingDistance))
.collect(Collectors.toList());
int total = allResults.size();
int page = (pageable != null) ? pageable.getPageNumber() : 0;
int size = (pageable != null) ? pageable.getPageSize() : 20;
int fromIndex = Math.min(page * size, total);
int toIndex = Math.min(fromIndex + size, total);
List<SimilarFileDTO> pageContent = allResults.subList(fromIndex, toIndex);
return new PageImpl<>(pageContent, pageable, total);
}
private SimilarFileDTO createSimilarFileResponse(SimilarImageProjection similarImageProjection,
Long hash64Hi, Long hash64Lo) {
Long imageProjectionHash64Hi = similarImageProjection.getHash64Hi();
Long similarImageProjectionHash64Lo = similarImageProjection.getHash64Lo();
int hamming = fileUtil.hamming64(hash64Hi, hash64Lo, imageProjectionHash64Hi, similarImageProjectionHash64Lo);
String level;
if (hamming <= 5) {
level = "DUPLICATE";
} else if (hamming <= 12) {
level = "SIMILAR";
} else {
level = "DIFFERENT";
}
return SimilarFileDTO.builder()
.fileId(similarImageProjection.getFileId())
.originalFileName(similarImageProjection.getOriginalFileName())
.fileSize(similarImageProjection.getFileSize())
.hammingDistance(hamming)
.similarityLevel(level)
.build();
}
}
@@ -0,0 +1,48 @@
package ru.soune.nocopy.service;
import com.vrt.fileprotection.image.phash.PHash;
import com.vrt.fileprotection.image.phash.PerceptualHashHelper;
import lombok.RequiredArgsConstructor;
import lombok.extern.slf4j.Slf4j;
import org.springframework.stereotype.Service;
import ru.soune.nocopy.entity.file.FileEntity;
import ru.soune.nocopy.entity.file.ImageHashEntity;
import ru.soune.nocopy.repository.ImageHashRepository;
import java.io.File;
import java.io.IOException;
import java.nio.file.Path;
import java.time.LocalDateTime;
import java.util.Map;
@Service
@RequiredArgsConstructor
@Slf4j
public class ImageHashService {
private final ImageHashRepository repository;
public Map<String, Long> calculateHash(Path imagePath) throws IOException {
File file = imagePath.toFile();
PHash pHash = PerceptualHashHelper.INSTANCE.generateDCTPerceptualHash(file);
Long firstPart = pHash.getFirstPart();
Long secondPart = pHash.getSecondPart();
Map<String, Long> hash = Map.of(
"hi", firstPart,
"low", secondPart);
return hash;
}
public void create(FileEntity file, Map<String, Long> stringIntegerMap) {
ImageHashEntity entity = ImageHashEntity.builder()
.file(file)
.hash64Hi(stringIntegerMap.get("hi"))
.hash64Lo(stringIntegerMap.get("low"))
.hashAlgorithm("PHASH64")
.createdAt(LocalDateTime.now())
.build();
repository.save(entity);
}
}
@@ -0,0 +1,161 @@
package ru.soune.nocopy.service;
import com.fasterxml.jackson.core.JsonProcessingException;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.fasterxml.jackson.databind.DeserializationFeature;
import jakarta.annotation.PostConstruct;
import lombok.RequiredArgsConstructor;
import lombok.extern.slf4j.Slf4j;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.stereotype.Service;
import ru.soune.nocopy.dto.BaseResponse;
import ru.soune.nocopy.dto.MessageCode;
import ru.soune.nocopy.dto.file.YandexSearchResponse;
import ru.soune.nocopy.entity.file.FileEntity;
import ru.soune.nocopy.exception.NotValidFieldException;
import ru.soune.nocopy.repository.FileEntityRepository;
import java.io.*;
import java.net.HttpURLConnection;
import java.net.URL;
import java.nio.file.Files;
import java.nio.file.Path;
import java.util.Base64;
import java.util.Map;
@Slf4j
@Service
@RequiredArgsConstructor
public class YandexSearchService {
private final FileEntityRepository fileEntityRepository;
private final ObjectMapper objectMapper;
@Value("${yandex.api-key}")
private String apiKey;
@Value("${yandex.folder-id}")
private String folderId;
@Value("${yandex.search-url}")
private String searchUrl;
@PostConstruct
public void init() {
objectMapper.configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, false);
objectMapper.configure(DeserializationFeature.ACCEPT_SINGLE_VALUE_AS_ARRAY, true);
}
public YandexSearchResponse searchByFileEntity(String fileId) throws IOException {
FileEntity fileEntity = fileEntityRepository.findById(fileId)
.orElseThrow(() -> {
throw new NotValidFieldException("File not found", new BaseResponse(20007,
MessageCode.FILE_NOT_FOUND.getCode(), MessageCode.FILE_NOT_FOUND.getDescription(),
Map.of("fileId",fileId)));
});
byte[] fileBytes;
if (!isImageFile(fileEntity)) {
log.error("File not image: {}", fileEntity.getMimeType());
throw new NotValidFieldException("File not image", new BaseResponse(20007,
MessageCode.INVALID_FIELD.getCode(), MessageCode.INVALID_FIELD.getDescription(),
Map.of("file_type", fileEntity.getMimeType())));
}
try {
fileBytes = readFileFromDisk(fileEntity);
} catch (IOException e) {
throw new NotValidFieldException("File not found or cannot read file", new BaseResponse(20007,
MessageCode.FILE_NOT_FOUND.getCode(), MessageCode.FILE_NOT_FOUND.getDescription(),
Map.of("fileId", fileId,
"filePath", fileEntity.getFilePath())));
}
return callYandexApi(fileBytes);
}
private boolean isImageFile(FileEntity fileEntity) {
String mimeType = fileEntity.getMimeType();
return mimeType != null && mimeType.startsWith("image");
}
private byte[] readFileFromDisk(FileEntity fileEntity) throws IOException {
Path filePath = Path.of(fileEntity.getFilePath());
if (!Files.exists(filePath)) {
throw new IOException("File not found: " + fileEntity.getFilePath());
}
if (!Files.isReadable(filePath)) {
throw new IOException("Cannot read file: " + fileEntity.getFilePath());
}
return Files.readAllBytes(filePath);
}
private YandexSearchResponse callYandexApi(byte[] imageBytes) throws IOException {
String imageBase64 = Base64.getEncoder().encodeToString(imageBytes);
String jsonRequest = String.format("{\"folderId\":\"%s\",\"data\":\"%s\",\"page\":0,\"showSimilarImages\":true}",
folderId, imageBase64);
URL url = new URL(searchUrl);
HttpURLConnection connection = (HttpURLConnection) url.openConnection();
try {
connection.setRequestMethod("POST");
connection.setRequestProperty("Authorization", "Api-Key " + apiKey);
connection.setRequestProperty("Content-Type", "application/json");
connection.setRequestProperty("Accept", "application/json");
connection.setConnectTimeout(30000);
connection.setReadTimeout(30000);
connection.setDoOutput(true);
try (OutputStream os = connection.getOutputStream()) {
byte[] input = jsonRequest.getBytes("utf-8");
os.write(input, 0, input.length);
os.flush();
}
int responseCode = connection.getResponseCode();
String responseBody;
if (responseCode == HttpURLConnection.HTTP_OK) {
try (BufferedReader br = new BufferedReader(
new InputStreamReader(connection.getInputStream(), "utf-8"))) {
responseBody = readAll(br);
}
} else {
try (BufferedReader br = new BufferedReader(
new InputStreamReader(connection.getErrorStream(), "utf-8"))) {
responseBody = readAll(br);
}
throw new IOException("Error Yandex API: " + responseCode + " - " + responseBody);
}
return parseJsonResponse(responseBody);
} finally {
connection.disconnect();
}
}
private String readAll(BufferedReader reader) throws IOException {
StringBuilder sb = new StringBuilder();
String line;
while ((line = reader.readLine()) != null) {
sb.append(line);
}
return sb.toString();
}
private YandexSearchResponse parseJsonResponse(String json) {
YandexSearchResponse response = null;
try {
response = objectMapper.readValue(json, YandexSearchResponse.class);
} catch (JsonProcessingException e) {
throw new RuntimeException(e);
}
return response;
}
}
@@ -1,13 +1,13 @@
package ru.soune.nocopy.service;
package ru.soune.nocopy.service.auth;
import lombok.RequiredArgsConstructor;
import org.springframework.context.MessageSource;
import org.springframework.security.crypto.password.PasswordEncoder;
import org.springframework.stereotype.Service;
import org.springframework.transaction.annotation.Transactional;
import ru.soune.nocopy.dto.*;
import ru.soune.nocopy.entity.AuthToken;
import ru.soune.nocopy.entity.User;
import ru.soune.nocopy.exception.NotFoundAuthToken;
import ru.soune.nocopy.exception.NotValidFieldException;
import ru.soune.nocopy.repository.AuthTokenRepository;
import ru.soune.nocopy.repository.UserRepository;
@@ -29,8 +29,6 @@ public class AuthService {
private final PasswordEncoder passwordEncoder;
private final MessageSource messageSource;
private final SecureRandom secureRandom = new SecureRandom();
@Transactional
@@ -81,6 +79,26 @@ public class AuthService {
return authTokenRepository.save(authToken);
}
@Transactional
public Long useUserAuthToken(String token) {
if (token == null || token.isBlank()) {
throw new NotFoundAuthToken("Token is required");
}
if (token.startsWith("Bearer ")) {
token = token.replace("Bearer ", "");
}
AuthToken authToken = authTokenRepository.findByToken(token)
.orElseThrow(() -> new NotFoundAuthToken("Token not found"));
authToken.setLastUsedAt(LocalDateTime.now());
authToken.setExpiresAt(LocalDateTime.now().plusHours(1));
authTokenRepository.save(authToken);
return authToken.getUser().getId();
}
@Transactional
public void logout(String token) {
authTokenRepository.findByToken(token)
@@ -0,0 +1,37 @@
package ru.soune.nocopy.service.auth;
import lombok.RequiredArgsConstructor;
import lombok.extern.slf4j.Slf4j;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.scheduling.annotation.Scheduled;
import org.springframework.stereotype.Component;
import org.springframework.transaction.annotation.Transactional;
import ru.soune.nocopy.entity.AuthToken;
import ru.soune.nocopy.repository.AuthTokenRepository;
import java.time.LocalDateTime;
import java.util.List;
@Slf4j
@Component
@RequiredArgsConstructor
public class CleanupTokenSessionsService {
@Autowired
private AuthTokenRepository authTokenRepository;
@Value("${file.storage.auth-token-life-hours}")
private int authTokenLife;
@Transactional
@Scheduled(fixedDelay = 30000)
public void cleanupExpiredTokens() {
LocalDateTime now = LocalDateTime.now();
List<AuthToken> expiredTokens = authTokenRepository.findByExpiresAtBefore(now);
if (!expiredTokens.isEmpty()) {
authTokenRepository.deleteAll(expiredTokens);
}
}
}
@@ -9,15 +9,22 @@ import ru.soune.nocopy.dto.file.FileResponse;
import ru.soune.nocopy.entity.file.FileEntity;
import ru.soune.nocopy.entity.file.FileStatus;
import ru.soune.nocopy.entity.file.FileUploadSession;
import ru.soune.nocopy.entity.file.ProtectionStatus;
import ru.soune.nocopy.exception.DuplicateImageException;
import ru.soune.nocopy.exception.FileEntityNotFoundException;
import ru.soune.nocopy.repository.FileEntityRepository;
import ru.soune.nocopy.repository.SimilarImageProjection;
import ru.soune.nocopy.service.FileSimilarityService;
import ru.soune.nocopy.service.ImageHashService;
import java.io.File;
import java.io.IOException;
import java.nio.file.Files;
import java.nio.file.Path;
import java.nio.file.Paths;
import java.time.LocalDateTime;
import java.util.List;
import java.util.Map;
import java.util.stream.Collectors;
@Slf4j
@@ -27,7 +34,11 @@ public class FileEntityService {
private final FileEntityRepository fileEntityRepository;
@Transactional
private final ImageHashService imageHashService;
private final FileSimilarityService fileSimilarityService;
@Transactional(noRollbackFor = DuplicateImageException.class)
public FileEntity createFromUploadSession(FileUploadSession session, String checksum) {
log.info("Creating FileEntity for upload session: {}", session.getUploadId());
@@ -38,9 +49,23 @@ public class FileEntityService {
throw new IOException("File not found on disk: " + filePath);
}
Map<String, Long> imageHash = Map.of();
if (session.getFileType().startsWith("image")) {
imageHash = imageHashService.calculateHash(filePath);
List<SimilarImageProjection> duplicatedByHash = fileSimilarityService.findDuplicatedByHash(
imageHash.get("hi"), imageHash.get("low"));
if (!duplicatedByHash.isEmpty()) {
SimilarImageProjection similarImageProjection = duplicatedByHash.get(0);
throw new DuplicateImageException("Duplicate", similarImageProjection.getFileId(),
similarImageProjection.getUserId());
}
}
long fileSize = Files.size(filePath);
String originalName = session.getFileName();
String extension = extractFileExtension(originalName);
String storedName = filePath.getFileName().toString();
FileEntity fileEntity = FileEntity.builder()
@@ -50,18 +75,19 @@ public class FileEntityService {
.filePath(session.getFilePath())
.fileSize(fileSize)
.mimeType(session.getFileType())
.fileExtension(extension)
.fileExtension(session.getExtension())
.checksum(checksum)
.uploadSessionId(session.getUploadId())
.status(FileStatus.ACTIVE)
.build();
FileEntity saved = fileEntityRepository.save(fileEntity);
log.info("FileEntity created successfully: {} (size: {} bytes)",
saved.getId(), fileSize);
if (!imageHash.isEmpty()) {
imageHashService.create(saved, imageHash);
}
return saved;
} catch (IOException e) {
log.error("Failed to create FileEntity for session {}: {}",
session.getUploadId(), e.getMessage(), e);
@@ -69,15 +95,6 @@ public class FileEntityService {
}
}
private String extractFileExtension(String fileName) {
int dotIndex = fileName.lastIndexOf('.');
if (dotIndex > 0 && dotIndex < fileName.length() - 1) {
return fileName.substring(dotIndex + 1).toLowerCase();
}
return "";
}
@Transactional(readOnly = true)
public FileEntityResponse getById(String fileId, int version) {
FileEntity fileEntity = fileEntityRepository.findById(fileId)
@@ -168,15 +185,26 @@ public class FileEntityService {
}
@Transactional
public void markAsDeleted(String fileId) {
FileEntity fileEntity = fileEntityRepository.findById(fileId)
.orElseThrow(() -> new FileEntityNotFoundException(fileId));
public void markAsDeleted(FileEntity fileEntity) {
fileEntity.setStatus(FileStatus.DELETED);
fileEntity.setUpdatedAt(LocalDateTime.now());
fileEntityRepository.save(fileEntity);
log.info("FileEntity marked as deleted: {}", fileId);
fileEntityRepository.save(fileEntity);
}
@Transactional
public boolean deleteFromDisk(FileEntity fileEntity) throws IOException {
Path path = Paths.get(fileEntity.getFilePath());
if (!Files.exists(path)) {
return true;
}
Files.delete(path);
markAsDeleted(fileEntity);
return true;
}
@Transactional(readOnly = true)
@@ -185,6 +213,95 @@ public class FileEntityService {
return totalSize != null ? totalSize : 0L;
}
public void changeStatus(ProtectionStatus newStatus, String fileId) {
FileEntity fileEntity = fileEntityRepository.findById(fileId)
.orElseThrow(() -> new FileEntityNotFoundException(fileId));
fileEntity.setProtectionStatus(newStatus);
fileEntityRepository.save(fileEntity);
}
@Transactional
public void writeProtectedFile(String id, byte[] data, String fileExt) throws IOException {
FileEntity fileEntity = fileEntityRepository.findById(id)
.orElseThrow(() -> new RuntimeException("File not found: " + id));
String extension = determineFileExtension(fileExt, fileEntity);
Path protectedFilePath = prepareProtectedPath(fileEntity, extension);
if (Files.exists(protectedFilePath)) {
Files.delete(protectedFilePath);
}
Files.write(protectedFilePath, data);
fileEntity.setProtectedFilePath(protectedFilePath.toString());
fileEntity.setProtectedAt(LocalDateTime.now());
fileEntity.setUpdatedAt(LocalDateTime.now());
fileEntity.setFileExtension(extension);
fileEntity.setProtectionStatus(ProtectionStatus.PROTECTED);
fileEntityRepository.save(fileEntity);
}
public FileEntity findBySignature(String signature) {
return fileEntityRepository.findBySignature(signature);
}
public File getFileById(String id) {
try {
FileEntity fileEntity = fileEntityRepository.findById(id).orElseThrow(() ->
new RuntimeException("File not found: " + id));
File file = new File(fileEntity.getFilePath());
if (!file.exists()) {
throw new RuntimeException("File not found on disk: " + fileEntity.getFilePath());
}
return file;
} catch (Exception e) {
log.error("Error getting file: {}", id, e);
return null;
}
}
public Path prepareProtectedPath(FileEntity fileEntity, String extension) throws IOException {
Path originalPath = Paths.get(fileEntity.getFilePath());
String pathStr = originalPath.toString();
pathStr = pathStr.replaceFirst("/uploads/uploads/", "/uploads/protected/");
Path protectedPath = Paths.get(pathStr);
String fileName = protectedPath.getFileName().toString();
if (extension != null) {
String nameWithoutExt = fileName;
int lastDotIndex = fileName.lastIndexOf('.');
if (lastDotIndex > 0) {
nameWithoutExt = fileName.substring(0, lastDotIndex);
}
fileName = nameWithoutExt + "." + extension;
protectedPath = protectedPath.getParent().resolve(fileName);
}
Files.createDirectories(protectedPath.getParent());
return protectedPath;
}
public void updateSignature(String signature, String fileId) throws IOException {
FileEntity fileEntity = fileEntityRepository.findByFileId(fileId);
fileEntity.setSignature(signature);
fileEntity.setProtectionStatus(ProtectionStatus.PROTECTED);
fileEntity.setUpdatedAt(LocalDateTime.now());
fileEntity.setProtectedAt(LocalDateTime.now());
fileEntity.setProtectedFilePath(prepareProtectedPath(fileEntity, fileEntity.getFileExtension()).toString());
fileEntityRepository.save(fileEntity);
}
private boolean checkFileExistsOnDisk(String filePath) {
try {
return Files.exists(Paths.get(filePath));
@@ -194,6 +311,14 @@ public class FileEntityService {
}
}
private String determineFileExtension(String fileExt, FileEntity fileEntity) {
if (fileExt != null && !fileExt.trim().isEmpty()) {
return fileExt;
} else {
return fileEntity.getFileExtension();
}
}
private FileEntityResponse convertToResponse(FileEntity fileEntity, int version) {
boolean existsOnDisk = checkFileExistsOnDisk(fileEntity.getFilePath());
@@ -214,10 +339,11 @@ public class FileEntityService {
.formattedSize(formatFileSize(fileEntity.getFileSize()))
.downloadUrl("/api/v" + version + "/files/download/" + fileEntity.getId())
.existsOnDisk(existsOnDisk)
.supportId(fileEntity.getSupportId())
.build();
}
private String formatFileSize(long size) {
public String formatFileSize(long size) {
if (size < 1024) {
return size + " B";
} else if (size < 1024 * 1024) {
@@ -0,0 +1,53 @@
package ru.soune.nocopy.service.file;
import com.vrt.NoCopyFileService;
import com.vrt.fileprotection.FileProtector;
import lombok.RequiredArgsConstructor;
import lombok.extern.slf4j.Slf4j;
import org.springframework.scheduling.annotation.Scheduled;
import org.springframework.stereotype.Service;
import ru.soune.nocopy.entity.file.FileEntity;
import ru.soune.nocopy.entity.file.ProtectionStatus;
import ru.soune.nocopy.repository.FileEntityRepository;
import ru.soune.nocopy.util.FileUtil;
import java.util.List;
@Slf4j
@Service
@RequiredArgsConstructor
public class FileProcessingOrchestrator {
private final NoCopyFileService noCopyFileService;
private final FileEntityRepository fileRepository;
private final FileUtil fileUtil;
public void initializeProcessingQueue() {
List<FileEntity> filesToProtect = fileRepository.findByProtectionStatus(ProtectionStatus.NOT_PROTECTED);
for (FileEntity fileEntity : filesToProtect) {
try {
FileProtector.FileInfo fileInfo = fileUtil.createFileInfo(fileEntity);
noCopyFileService.addFile(fileInfo);
log.info("Add to query: {}", fileEntity.getOriginalFileName());
} catch (Exception e) {
log.error("Fail add to query: {}", fileEntity.getId(), e);
}
}
}
@Scheduled(fixedDelay = 120000)
public void checkNewFilesForProtection() {
List<FileEntity> newFiles = fileRepository.findAllActiveFilesAndNotProtected();
for (FileEntity fileEntity : newFiles) {
FileProtector.FileInfo fileInfo = fileUtil.createFileInfo(fileEntity);
noCopyFileService.addFile(fileInfo);
}
}
}
@@ -0,0 +1,93 @@
package ru.soune.nocopy.service.file;
import lombok.RequiredArgsConstructor;
import org.springframework.stereotype.Service;
import ru.soune.nocopy.dto.file.FileInfoUserResponse;
import ru.soune.nocopy.entity.file.FileEntity;
import ru.soune.nocopy.entity.file.FileStatus;
import ru.soune.nocopy.entity.file.FileType;
import ru.soune.nocopy.repository.FileEntityRepository;
import java.util.List;
@Service
@RequiredArgsConstructor
public class FileStatsService {
private final FileEntityRepository fileEntityRepository;
public FileInfoUserResponse getUserFileStats(Long userId) {
List<FileEntity> userFiles = fileEntityRepository.findByUserId(userId);
return calculateStats(userFiles);
}
public FileInfoUserResponse calculateStats(List<FileEntity> files) {
return FileInfoUserResponse.builder()
.allFileSize(calculateTotalSize(files))
.fileCount(calculateTotalCount(files))
.filesCheck(calculateByStatus(files, FileStatus.CHECKED))
.filesViolation(calculateByStatus(files, FileStatus.VIOLATION))
.imagesSize(calculateMediaSize(files, FileType.IMAGE))
.imagesCount(calculateMediaCount(files, FileType.IMAGE))
.imagesCheck(calculateMediaByStatus(files, FileType.IMAGE, FileStatus.CHECKED))
.imagesViolations(calculateMediaByStatus(files, FileType.IMAGE, FileStatus.VIOLATION))
.videosSize(calculateMediaSize(files, FileType.VIDEO))
.videosCount(calculateMediaCount(files, FileType.VIDEO))
.videosCheck(calculateMediaByStatus(files, FileType.VIDEO, FileStatus.CHECKED))
.videosViolations(calculateMediaByStatus(files, FileType.VIDEO, FileStatus.VIOLATION))
.audiosSize(calculateMediaSize(files, FileType.AUDIO))
.audiosCount(calculateMediaCount(files, FileType.AUDIO))
.audiosCheck(calculateMediaByStatus(files, FileType.AUDIO, FileStatus.CHECKED))
.audiosViolations(calculateMediaByStatus(files, FileType.AUDIO, FileStatus.VIOLATION))
.build();
}
private Long calculateTotalSize(List<FileEntity> files) {
return files.stream()
.filter(file -> file.getStatus() != FileStatus.DELETED)
.mapToLong(FileEntity::getFileSize)
.sum();
}
private Integer calculateTotalCount(List<FileEntity> files) {
return (int) files.stream()
.filter(file -> file.getStatus() != FileStatus.DELETED)
.count();
}
private Integer calculateByStatus(List<FileEntity> files, FileStatus status) {
return (int) files.stream()
.filter(file -> file.getStatus() == status)
.count();
}
private Long calculateMediaSize(List<FileEntity> files, FileType fileType) {
return files.stream()
.filter(file -> file.getStatus() != FileStatus.DELETED)
.filter(file -> isFileType(file, fileType))
.mapToLong(FileEntity::getFileSize)
.sum();
}
private Integer calculateMediaCount(List<FileEntity> files, FileType fileType) {
return (int) files.stream()
.filter(file -> file.getStatus() != FileStatus.DELETED)
.filter(file -> isFileType(file, fileType))
.count();
}
private Integer calculateMediaByStatus(List<FileEntity> files, FileType fileType, FileStatus status) {
return (int) files.stream()
.filter(file -> isFileType(file, fileType))
.filter(file -> file.getStatus() == status)
.count();
}
private boolean isFileType(FileEntity file, FileType fileType) {
if (file.getFileExtension() == null) return false;
String extension = file.getFileExtension().toLowerCase().replace(".", "");
return fileType.supportsExtension(extension);
}
}
@@ -15,9 +15,7 @@ public interface FileUploadService {
UploadProgressResponse getUploadProgress(String uploadId);
void cleanupExpiredSessions();
public void retryFailedUpload(String uploadId);
void handleExpiredSession(FileUploadSession session);
public void cancelUpload(String uploadId);
}
@@ -0,0 +1,35 @@
package ru.soune.nocopy.service.file;
import lombok.RequiredArgsConstructor;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.scheduling.annotation.Scheduled;
import org.springframework.stereotype.Service;
import org.springframework.transaction.annotation.Transactional;
import ru.soune.nocopy.entity.file.FileEntity;
import ru.soune.nocopy.entity.file.FileStatus;
import ru.soune.nocopy.repository.FileEntityRepository;
import java.nio.file.Files;
import java.nio.file.Paths;
import java.util.List;
@Service
@RequiredArgsConstructor
public class NotExistFilesCleanupService {
@Autowired
private FileEntityRepository fileEntityRepository;
@Transactional
@Scheduled(fixedDelay = 30000)
public void cleanUpNotExistFiles() {
List<FileEntity> fileEntities = fileEntityRepository.searchFileEntityByStatus(FileStatus.DELETED);
for (FileEntity fileEntity : fileEntities) {
boolean exists = Files.exists(Paths.get(fileEntity.getFilePath()));
if (!exists) {
fileEntityRepository.delete(fileEntity);
}
}
}
}
@@ -0,0 +1,35 @@
package ru.soune.nocopy.service.file;
import lombok.RequiredArgsConstructor;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.scheduling.annotation.Scheduled;
import org.springframework.stereotype.Service;
import org.springframework.transaction.annotation.Transactional;
import ru.soune.nocopy.entity.file.FileUploadSession;
import ru.soune.nocopy.entity.file.UploadStatus;
import ru.soune.nocopy.repository.FileUploadSessionRepository;
import java.time.LocalDateTime;
import java.util.List;
import java.util.Set;
@Service
@RequiredArgsConstructor
public class UploadSessionCleanupService {
@Autowired
private FileUploadSessionRepository fileUploadSessionRepository;
@Autowired
private FileUploadService fileUploadService;
@Transactional
@Scheduled(fixedDelay = 30000)
public void cleanupSessions() {
List<FileUploadSession> expiredSessions = fileUploadSessionRepository.findExpiredSessions(
LocalDateTime.now(), Set.of(UploadStatus.INITIATED, UploadStatus.UPLOADING));
for (FileUploadSession session : expiredSessions) {
fileUploadService.handleExpiredSession(session);
}
}
}
@@ -0,0 +1,44 @@
package ru.soune.nocopy.service.file.impl;
import com.vrt.AudioFilePathProvider;
import com.vrt.fileprotection.FileProtector;
import lombok.RequiredArgsConstructor;
import lombok.extern.slf4j.Slf4j;
import org.jetbrains.annotations.NotNull;
import org.springframework.stereotype.Service;
import ru.soune.nocopy.entity.file.FileEntity;
import ru.soune.nocopy.repository.FileEntityRepository;
import ru.soune.nocopy.service.file.FileEntityService;
import java.io.IOException;
import java.util.Optional;
@Slf4j
@Service
@RequiredArgsConstructor
public class AudioFilePathProviderImpl implements AudioFilePathProvider {
private final FileEntityService fileEntityService;
private final FileEntityRepository fileEntityRepository;
@Override
public @NotNull String providePath(@NotNull FileProtector.FileInfo fileInfo) {
String filePath;
Optional<FileEntity> optionalFileEntity = fileEntityRepository.findById(fileInfo.getId());
if (optionalFileEntity.isEmpty()) {
throw new NullPointerException();
}
FileEntity fileEntity = optionalFileEntity.get();
try {
filePath = fileEntityService.prepareProtectedPath(fileEntity, fileEntity.getFileExtension()).toString();
} catch (IOException e) {
throw new RuntimeException(e);
}
return filePath;
}
}
@@ -0,0 +1,42 @@
package ru.soune.nocopy.service.file.impl;
import com.vrt.fileprotection.FileProtector;
import com.vrt.fileprotection.audio.AudioLocalSearch;
import lombok.RequiredArgsConstructor;
import lombok.extern.slf4j.Slf4j;
import org.jetbrains.annotations.NotNull;
import org.jetbrains.annotations.Nullable;
import org.springframework.stereotype.Service;
import ru.soune.nocopy.entity.file.FileEntity;
import ru.soune.nocopy.service.file.FileEntityService;
import ru.soune.nocopy.util.FileUtil;
import java.io.IOException;
@Slf4j
@Service
@RequiredArgsConstructor
public class AudioLocalSearchImpl implements AudioLocalSearch {
private final FileEntityService fileEntityService;
private final FileUtil fileUtil;
@Override
public @Nullable FileProtector.FileInfo findBySignature(@NotNull String signature) {
FileEntity fileEntity = fileEntityService.findBySignature(signature);
return fileUtil.createFileInfo(fileEntity);
}
@Override
public void updateSignature(@NotNull String signature, @NotNull FileProtector.FileInfo fileInfo) {
try {
fileEntityService.updateSignature(signature, fileInfo.getId());
} catch (IOException e) {
throw new RuntimeException(e);
}
}
}
@@ -1,5 +1,7 @@
package ru.soune.nocopy.service.file;
package ru.soune.nocopy.service.file.impl;
import com.vrt.NoCopyFileService;
import com.vrt.fileprotection.FileProtector;
import jakarta.annotation.PostConstruct;
import lombok.RequiredArgsConstructor;
import lombok.extern.slf4j.Slf4j;
@@ -10,12 +12,22 @@ import org.springframework.stereotype.Service;
import org.springframework.transaction.annotation.Transactional;
import org.springframework.web.multipart.MultipartFile;
import ru.soune.nocopy.dto.file.UploadProgressResponse;
import ru.soune.nocopy.entity.file.FileEntity;
import ru.soune.nocopy.entity.file.FileStatus;
import ru.soune.nocopy.entity.file.FileUploadSession;
import ru.soune.nocopy.entity.file.UploadStatus;
import ru.soune.nocopy.exception.ChunkSizeExceededException;
import ru.soune.nocopy.exception.DuplicateImageException;
import ru.soune.nocopy.exception.FileUploadException;
import ru.soune.nocopy.exception.UploadSessionNotFoundException;
import ru.soune.nocopy.repository.FileEntityRepository;
import ru.soune.nocopy.repository.FileUploadSessionRepository;
import ru.soune.nocopy.repository.SimilarImageProjection;
import ru.soune.nocopy.service.FileSimilarityService;
import ru.soune.nocopy.service.ImageHashService;
import ru.soune.nocopy.service.file.FileEntityService;
import ru.soune.nocopy.service.file.FileUploadService;
import ru.soune.nocopy.util.FileUtil;
import java.io.*;
import java.nio.file.*;
@@ -23,10 +35,7 @@ import java.nio.file.attribute.PosixFilePermission;
import java.security.MessageDigest;
import java.security.NoSuchAlgorithmException;
import java.time.LocalDateTime;
import java.util.Comparator;
import java.util.HashSet;
import java.util.Set;
import java.util.UUID;
import java.util.*;
import java.util.concurrent.CompletableFuture;
import java.util.stream.Stream;
@@ -57,9 +66,22 @@ public class FileUploadServiceImpl implements FileUploadService {
private Path storageRoot;
@Autowired
private ImageHashService imageHashService;
@Autowired
private FileSimilarityService fileSimilarityService;
@Autowired
private FileEntityRepository fileEntityRepository;
@Autowired
private FileEntityService fileEntityService;
private final NoCopyFileService noCopyFileService;
private final FileUtil fileUtil;
@PostConstruct
public void init() {
try {
@@ -102,12 +124,14 @@ public class FileUploadServiceImpl implements FileUploadService {
int totalChunks = (int) Math.ceil((double) fileSize / chunkSize);
log.debug("File will be split into {} chunks (chunk size: {} bytes)",
totalChunks, chunkSize);
Long chunkSize = totalChunks == 1 ? fileSize : 1000000L;
FileUploadSession session = FileUploadSession.builder()
.userId(userId)
.extension(extension)
.fileName(fileName)
.fileType(fileType)
.chunkSize(chunkSize)
.fileSize(fileSize)
.totalChunks(totalChunks)
.chunksUploaded(0)
@@ -127,10 +151,6 @@ public class FileUploadServiceImpl implements FileUploadService {
@Transactional
public UploadProgressResponse uploadChunk(String uploadId, Integer chunkNumber,
MultipartFile chunkFile) {
log.info("Processing chunk {} for session {}, size: {} bytes",
chunkNumber, uploadId, chunkFile.getSize());
FileUploadSession session = sessionRepository.findById(uploadId)
.orElseThrow(() -> new UploadSessionNotFoundException(uploadId));
@@ -143,17 +163,75 @@ public class FileUploadServiceImpl implements FileUploadService {
if (chunkNumber < 0 || chunkNumber >= session.getTotalChunks()) {
throw new FileUploadException(
String.format("Invalid chunk number %d. Expected 0-%d",
String.format("Invalid chunk number %d. Expected number: %d",
chunkNumber, session.getTotalChunks() - 1));
}
if (chunkFile.getSize() > chunkSize) {
if (chunkNumber == 0 && chunkFile.getSize() > chunkSize || chunkNumber + 1 == (session.getTotalChunks())
&& chunkFile.getSize() > chunkSize) {
throw new ChunkSizeExceededException(chunkFile.getSize(), chunkSize);
}
if (chunkNumber > 0 && chunkNumber + 1 < session.getTotalChunks() && chunkFile.getSize() != chunkSize) {
throw new ChunkSizeExceededException(chunkFile.getSize(), chunkSize);
}
return processChunk(session, chunkNumber, chunkFile);
}
@Override
@Transactional
public void handleExpiredSession(FileUploadSession session) {
session.setStatus(UploadStatus.FAILED);
session.setLastError("Upload session expired");
sessionRepository.save(session);
CompletableFuture.runAsync(() -> cleanupSessionFiles(session));
}
@Async("fileUploadTaskExecutor")
@Transactional
public void assembleFileAsync(FileUploadSession session) throws DuplicateImageException {
try {
assembleFile(session);
log.info("File assembly completed successfully for session: {}",
session.getUploadId());
} catch (DuplicateImageException e) {
throw new DuplicateImageException("DUBL", e.duplicateFileId(), e.userId());
} catch (Exception e) {
log.error("Failed to assemble file for session {}: {}",
session.getUploadId(), e.getMessage(), e);
handleAssemblyFailure(session, e);
}
}
@Override
@Transactional
public void cancelUpload(String uploadId) {
FileUploadSession session = sessionRepository.findById(uploadId)
.orElseThrow(() -> new UploadSessionNotFoundException(uploadId));
if (session.getStatus() == UploadStatus.COMPLETED || session.getStatus() == UploadStatus.CANCELLED) {
throw new FileUploadException("Cannot cancel completed or cancelled upload");
}
session.setStatus(UploadStatus.CANCELLED);
sessionRepository.save(session);
CompletableFuture.runAsync(() -> cleanupSessionFiles(session));
log.info("Upload cancelled: {}", uploadId);
}
@Override
public UploadProgressResponse getUploadProgress(String uploadId) {
FileUploadSession session = sessionRepository.findById(uploadId)
.orElseThrow(() -> new UploadSessionNotFoundException(uploadId));
return UploadProgressResponse.fromSession(session);
}
private void validateSession(FileUploadSession session) {
UploadStatus status = session.getStatus();
@@ -183,17 +261,47 @@ public class FileUploadServiceImpl implements FileUploadService {
}
}
private void handleExpiredSession(FileUploadSession session) {
session.setStatus(UploadStatus.FAILED);
session.setLastError("Upload session expired");
sessionRepository.save(session);
@Async("fileUploadTaskExecutor")
@Transactional
public void completeFileProcessingAsync(FileUploadSession session) {
try {
Path filePath = Paths.get(session.getFilePath());
String checksum = calculateChecksum(filePath);
CompletableFuture.runAsync(() -> cleanupSessionFiles(session));
FileEntity fileEntity = FileEntity.builder()
.userId(session.getUserId())
.originalFileName(session.getFileName())
.storedFileName(filePath.getFileName().toString())
.filePath(session.getFilePath())
.fileSize(session.getFileSize())
.mimeType(session.getFileType())
.fileExtension(session.getExtension())
.checksum(checksum)
.uploadSessionId(session.getUploadId())
.status(FileStatus.ACTIVE)
.build();
FileEntity saved = fileEntityRepository.save(fileEntity);
if (session.getFileType().equals("image")) {
Map<String, Long> hash = imageHashService.calculateHash(filePath);
imageHashService.create(saved, hash);
}
private UploadProgressResponse processChunk(FileUploadSession session,
Integer chunkNumber,
MultipartFile chunkFile) {
cleanupSessionFiles(session);
noCopyFileService.addFile(fileUtil.createFileInfo(fileEntity));
log.info("File processing completed for session: {}", session.getUploadId());
} catch (Exception e) {
log.error("Failed to complete file processing for session {}: {}",
session.getUploadId(), e.getMessage(), e);
}
}
private UploadProgressResponse processChunk(FileUploadSession session, Integer chunkNumber, MultipartFile chunkFile) {
String chunkPath = null;
try {
@@ -206,21 +314,47 @@ public class FileUploadServiceImpl implements FileUploadService {
session.getChunkPaths().put(chunkNumber, chunkPath);
session.setChunksUploaded(session.getChunksUploaded() + 1);
session.setStatus(UploadStatus.UPLOADING);
session.setExpiresAt(LocalDateTime.now().plusMinutes(1));
log.debug("Chunk {} saved successfully. Uploaded: {}/{}",
chunkNumber, session.getChunksUploaded(), session.getTotalChunks());
if (session.getChunksUploaded().equals(session.getTotalChunks())) {
boolean isLastChunk = session.getChunksUploaded().equals(session.getTotalChunks());
if (isLastChunk) {
log.info("All chunks uploaded for session {}. Starting assembly...",
session.getUploadId());
sessionRepository.save(session);
assembleFileAsync(session);
String finalFilePath = assembleFileSynchronously(session);
if (session.getFileType().startsWith("image")) {
checkForDuplicatesSynchronously(finalFilePath);
}
session.setStatus(UploadStatus.COMPLETED);
session.setFilePath(finalFilePath);
completeFileProcessingAsync(session);
} else {
sessionRepository.save(session);
}
return UploadProgressResponse.fromSession(session);
} catch (DuplicateImageException e) {
if (chunkPath != null) {
cleanupFailedChunk(chunkPath);
}
log.warn("Duplicate image found for session {}: {}",
session.getUploadId(), e.getMessage());
session.setStatus(UploadStatus.FAILED);
session.setLastError(e.getMessage());
sessionRepository.save(session);
throw e;
} catch (Exception e) {
if (chunkPath != null) {
cleanupFailedChunk(chunkPath);
@@ -237,6 +371,30 @@ public class FileUploadServiceImpl implements FileUploadService {
}
}
private String assembleFileSynchronously(FileUploadSession session) throws IOException {
Path finalFilePath = prepareFinalFile(session);
validateAllChunksExist(session);
mergeChunksToFile(session, finalFilePath);
validateFinalFile(session, finalFilePath);
return finalFilePath.toString();
}
private void checkForDuplicatesSynchronously(String filePath)
throws IOException {
Path path = Paths.get(filePath);
Map<String, Long> hash = imageHashService.calculateHash(path);
List<SimilarImageProjection> duplicates = fileSimilarityService.findDuplicatedByHash(
hash.get("hi"), hash.get("low"));
if (!duplicates.isEmpty()) {
throw new DuplicateImageException("Duplicate", duplicates.get(0).getFileId(),
duplicates.get(0).getUserId());
}
}
private UploadProgressResponse handleExistingChunk(FileUploadSession session,
Integer chunkNumber,
MultipartFile chunkFile) throws IOException {
@@ -244,25 +402,24 @@ public class FileUploadServiceImpl implements FileUploadService {
Path chunkPath = Paths.get(existingPath);
if (!Files.exists(chunkPath)) {
log.warn("Chunk file missing, re-uploading: {}", chunkPath);
session.getChunkPaths().remove(chunkNumber);
session.setChunksUploaded(session.getChunksUploaded() - 1);
sessionRepository.save(session);
return processChunk(session, chunkNumber, chunkFile);
}
long existingSize = Files.size(chunkPath);
if (existingSize != chunkFile.getSize()) {
log.warn("Chunk size mismatch, re-uploading: {} != {}",
existingSize, chunkFile.getSize());
Files.deleteIfExists(chunkPath);
session.getChunkPaths().remove(chunkNumber);
session.setChunksUploaded(session.getChunksUploaded() - 1);
sessionRepository.save(session);
return processChunk(session, chunkNumber, chunkFile);
}
log.debug("Chunk {} already uploaded and valid", chunkNumber);
return UploadProgressResponse.fromSession(session);
}
@@ -316,58 +473,12 @@ public class FileUploadServiceImpl implements FileUploadService {
}
}
@Async("fileUploadTaskExecutor")
@Transactional
public void assembleFileAsync(FileUploadSession session) {
try {
assembleFile(session);
log.info("File assembly completed successfully for session: {}",
session.getUploadId());
} catch (Exception e) {
log.error("Failed to assemble file for session {}: {}",
session.getUploadId(), e.getMessage(), e);
handleAssemblyFailure(session, e);
}
}
// private void assembleFile(FileUploadSession session) throws IOException {
// log.info("Starting file assembly for session: {} ({})",
// session.getUploadId(), session.getFileName());
//
// Path finalFilePath = null;
//
// try {
// finalFilePath = prepareFinalFile(session);
//
// validateAllChunksExist(session);
//
// mergeChunksToFile(session, finalFilePath);
//
// validateFinalFile(session, finalFilePath);
//
// updateSessionOnSuccess(session, finalFilePath);
//
// cleanupSessionFiles(session);
//
// log.info("File assembly completed: {} -> {} ({} bytes)",
// session.getFileName(), finalFilePath, session.getFileSize());
//
// } catch (Exception e) {
// if (finalFilePath != null) {
// Files.deleteIfExists(finalFilePath);
// }
// throw e;
// }
// }
private void assembleFile(FileUploadSession session) throws IOException {
log.info("Starting file assembly for session: {} ({})",
session.getUploadId(), session.getFileName());
Path finalFilePath = null;
String checksum = null;
String checksum;
try {
finalFilePath = prepareFinalFile(session);
@@ -386,13 +497,21 @@ public class FileUploadServiceImpl implements FileUploadService {
session.setChecksum(checksum);
session.setStatus(UploadStatus.COMPLETED);
session.setCompletedAt(LocalDateTime.now());
sessionRepository.save(session);
log.info("Upload session updated to COMPLETED: {}", session.getUploadId());
try {
fileEntityService.createFromUploadSession(session, checksum);
FileEntity fileEntity = fileEntityService.createFromUploadSession(session, checksum);
FileProtector.FileInfo fileInfo = fileUtil.createFileInfo(fileEntity);
noCopyFileService.addFile(fileInfo);
log.info("FileEntity successfully created for session: {}",
session.getUploadId());
} catch (DuplicateImageException e) {
throw new DuplicateImageException("Duplicate", e.duplicateFileId(), e.userId());
} catch (Exception e) {
log.error("Failed to create FileEntity for session {} (file uploaded, but metadata not saved): {}",
session.getUploadId(), e.getMessage());
@@ -526,14 +645,6 @@ public class FileUploadServiceImpl implements FileUploadService {
log.debug("Final file checksum: {}", checksum);
}
private void updateSessionOnSuccess(FileUploadSession session, Path finalPath) {
session.setFilePath(finalPath.toString());
session.setChecksum(calculateChecksum(finalPath));
session.setStatus(UploadStatus.COMPLETED);
session.setCompletedAt(LocalDateTime.now());
sessionRepository.save(session);
}
private void cleanupSessionFiles(FileUploadSession session) {
try {
Path chunkDir = getChunkDirectory(session.getUserId(), session.getUploadId());
@@ -564,95 +675,8 @@ public class FileUploadServiceImpl implements FileUploadService {
private void handleAssemblyFailure(FileUploadSession session, Exception e) {
session.setStatus(UploadStatus.FAILED);
session.setLastError(e.getMessage());
sessionRepository.save(session);
log.error("File assembly failed for session {}: {}",
session.getUploadId(), e.getMessage());
}
@Override
@Transactional
public void cancelUpload(String uploadId) {
FileUploadSession session = sessionRepository.findById(uploadId)
.orElseThrow(() -> new UploadSessionNotFoundException(uploadId));
if (session.getStatus() == UploadStatus.COMPLETED || session.getStatus() == UploadStatus.CANCELLED) {
throw new FileUploadException("Cannot cancel completed or cancelled upload");
}
session.setStatus(UploadStatus.CANCELLED);
sessionRepository.save(session);
CompletableFuture.runAsync(() -> cleanupSessionFiles(session));
log.info("Upload cancelled: {}", uploadId);
}
@Override
@Transactional
public void retryFailedUpload(String uploadId) {
FileUploadSession session = sessionRepository.findById(uploadId)
.orElseThrow(() -> new UploadSessionNotFoundException(uploadId));
if (session.getStatus() != UploadStatus.FAILED) {
throw new FileUploadException("Only failed uploads can be retried");
}
if (session.getRetryCount() >= maxRetryAttempts) {
throw new FileUploadException("Max retry attempts exceeded");
}
session.setStatus(UploadStatus.UPLOADING);
session.setRetryCount(session.getRetryCount() + 1);
session.setLastError(null);
sessionRepository.save(session);
if (session.getChunksUploaded().equals(session.getTotalChunks())) {
log.info("Retrying file assembly for session: {}", uploadId);
assembleFileAsync(session);
}
log.info("Upload retry initiated for session: {} (attempt {})",
uploadId, session.getRetryCount());
}
@Override
public UploadProgressResponse getUploadProgress(String uploadId) {
FileUploadSession session = sessionRepository.findById(uploadId)
.orElseThrow(() -> new UploadSessionNotFoundException(uploadId));
return UploadProgressResponse.fromSession(session);
}
@Override
@Transactional
public void cleanupExpiredSessions() {
log.info("Starting cleanup of expired upload sessions");
LocalDateTime expiryThreshold = LocalDateTime.now().minusHours(sessionExpiryHours);
try {
var expiredSessions = sessionRepository.findByStatusInAndCreatedAtBefore(
Set.of(UploadStatus.INITIATED, UploadStatus.UPLOADING, UploadStatus.FAILED),
expiryThreshold
);
int cleanedCount = 0;
for (FileUploadSession session : expiredSessions) {
session.setStatus(UploadStatus.FAILED);
session.setLastError("Session expired during cleanup");
sessionRepository.save(session);
CompletableFuture.runAsync(() -> cleanupSessionFiles(session));
cleanedCount++;
}
log.info("Cleaned up {} expired upload sessions", cleanedCount);
} catch (Exception e) {
log.error("Error during session cleanup", e);
}
}
private Path getChunkDirectory(Long userId, String uploadId) {
@@ -0,0 +1,54 @@
package ru.soune.nocopy.service.file.impl;
import com.vrt.fileprotection.FileProtector;
import com.vrt.fileprotection.image.ImageLocalSearch;
import com.vrt.fileprotection.image.phash.PHash;
import lombok.RequiredArgsConstructor;
import lombok.extern.slf4j.Slf4j;
import org.jetbrains.annotations.NotNull;
import org.jetbrains.annotations.Nullable;
import org.springframework.stereotype.Service;
import ru.soune.nocopy.dto.file.SimilarFileDTO;
import ru.soune.nocopy.entity.file.FileEntity;
import ru.soune.nocopy.entity.file.ImageHashEntity;
import ru.soune.nocopy.repository.FileEntityRepository;
import ru.soune.nocopy.repository.ImageHashRepository;
import ru.soune.nocopy.service.FileSimilarityService;
import ru.soune.nocopy.util.FileUtil;
import java.util.List;
@Slf4j
@Service
@RequiredArgsConstructor
public class ImageLocalSearchImpl implements ImageLocalSearch {
private final FileSimilarityService fileSimilarityService;
private final ImageHashRepository imageHashRepository;
private final FileUtil fileUtil;
private final FileEntityRepository fileEntityRepository;
@Override
public @NotNull List<Result> find(@NotNull FileProtector.FileInfo fileInfo) {
List<SimilarFileDTO> duplicateByHammingDistance =
fileSimilarityService.findDuplicateByHammingDistance(fileInfo.getId(), 10,
5, 10);
return duplicateByHammingDistance.stream()
.map(fileUtil::convertToResult)
.toList();
}
@Override
public @Nullable FileProtector.FileInfo findByPHash(@NotNull PHash pHash) {
ImageHashEntity imageHashEntity =
imageHashRepository.findByHash64HiAndHash64Lo(pHash.getFirstPart(),
pHash.getSecondPart());
FileEntity file = fileEntityRepository.findByFileId(imageHashEntity.getFileId());
return new FileProtector.FileInfo(FileProtector.Type.IMAGE, imageHashEntity.getFileId(),
String.valueOf(file.getUserId()));
}
}
@@ -0,0 +1,34 @@
package ru.soune.nocopy.service.file.impl;
import com.vrt.fileprotection.FileProtector;
import com.vrt.fileprotection.OperationResult;
import com.vrt.fileprotection.image.ImageUniqueCheck;
import lombok.RequiredArgsConstructor;
import lombok.extern.slf4j.Slf4j;
import org.jetbrains.annotations.NotNull;
import org.springframework.stereotype.Service;
import ru.soune.nocopy.dto.file.SimilarFileDTO;
import ru.soune.nocopy.service.FileSimilarityService;
import java.util.List;
@Slf4j
@Service
@RequiredArgsConstructor
public class ImageUniqueCheckImpl implements ImageUniqueCheck {
private final FileSimilarityService fileSimilarityService;
@Override
public @NotNull OperationResult check(@NotNull FileProtector.FileInfo fileInfo) {
String fileId = fileInfo.getId();
List<SimilarFileDTO> duplicateByHammingDistance =
fileSimilarityService.findDuplicateByHammingDistance(fileId, 3, 3, 4);
if (duplicateByHammingDistance == null || duplicateByHammingDistance.isEmpty()) {
return OperationResult.Companion.success();
} else {
return OperationResult.Companion.failure("Duplicate file");
}
}
}
@@ -0,0 +1,36 @@
package ru.soune.nocopy.service.file.impl;
import com.vrt.fileprotection.FileProtector;
import lombok.AllArgsConstructor;
import lombok.extern.slf4j.Slf4j;
import org.springframework.stereotype.Component;
import ru.soune.nocopy.entity.file.ProtectionStatus;
import ru.soune.nocopy.service.file.FileEntityService;
@Slf4j
@Component
@AllArgsConstructor
public class NoCopyProcessingListenerImpl implements FileProtector.ProcessingListener {
private final FileEntityService fileEntityService;
@Override
public void onStartProcessing(FileProtector.FileInfo fileInfo) {
fileEntityService.changeStatus(ProtectionStatus.PROCESSING, fileInfo.getId());
}
@Override
public void onProcessingFailed(FileProtector.FileInfo fileInfo) {
fileEntityService.changeStatus(ProtectionStatus.FAILED, fileInfo.getId());
}
@Override
public void onSavingFailed(FileProtector.FileInfo fileInfo) {
fileEntityService.changeStatus(ProtectionStatus.FAILED_SAVE, fileInfo.getId());
}
@Override
public void onFinish(FileProtector.FileInfo fileInfo) {
fileEntityService.changeStatus(ProtectionStatus.PROTECTED, fileInfo.getId());;
}
}
@@ -0,0 +1,108 @@
package ru.soune.nocopy.service.file.impl;
import com.vrt.fileprotection.FileProtector;
import com.vrt.fileprotection.OperationResult;
import lombok.RequiredArgsConstructor;
import lombok.extern.slf4j.Slf4j;
import org.jetbrains.annotations.NotNull;
import org.jetbrains.annotations.Nullable;
import org.springframework.stereotype.Component;
import ru.soune.nocopy.service.file.FileEntityService;
import java.io.File;
import java.io.IOException;
import java.nio.file.Files;
import java.nio.file.Path;
import java.nio.file.Paths;
@Slf4j
@Component
@RequiredArgsConstructor
public class ProtectionFileProviderImpl implements FileProtector.FileProvider {
private static final Path SIGNATURE_FILE_PATH = Paths.get("/data/uploads/signature");
private final FileEntityService fileEntityService;
@Nullable
@Override
public File getImageFile(@NotNull String id) {
return fileEntityService.getFileById(id);
}
@Nullable
@Override
public File getVideoFile(@NotNull String id) {
return fileEntityService.getFileById(id);
}
@Nullable
@Override
public File getAudioFile(@NotNull String id) {
return fileEntityService.getFileById(id);
}
@Override
public @Nullable File getSignature() {
File signatureFile = SIGNATURE_FILE_PATH.toFile();
if (signatureFile.exists() && signatureFile.isFile() && signatureFile.length() > 0) {
return signatureFile;
}
return null;
}
@Override
public @NotNull OperationResult writeSignature(@NotNull byte[] bytes) {
try {
Path directory = SIGNATURE_FILE_PATH.getParent();
if (directory != null && !Files.exists(directory)) {
Files.createDirectories(directory);
}
Files.write(SIGNATURE_FILE_PATH, bytes);
return OperationResult.Companion.success();
} catch (IOException e) {
return OperationResult.Companion.failure("Not saved signature file");
}
}
@NotNull
@Override
public com.vrt.fileprotection.OperationResult writeAudioFile(@NotNull String id, @NotNull byte[] data, @Nullable String fileExt) {
try {
fileEntityService.writeProtectedFile(id, data, fileExt);
return OperationResult.Companion.success();
} catch (Exception e) {
log.error("Failed to create protected file: {}", id, e);
return OperationResult.Companion.failure("Failed to create protected file");
}
}
@NotNull
@Override
public com.vrt.fileprotection.OperationResult writeVideoFile(@NotNull String id, @NotNull byte[] data, @Nullable String fileExt) {
try {
fileEntityService.writeProtectedFile(id, data, fileExt);
return OperationResult.Companion.success();
} catch (Exception e) {
log.error("Failed to create protected file: {}", id, e);
return OperationResult.Companion.failure("Failed to create protected file");
}
}
@NotNull
@Override
public com.vrt.fileprotection.OperationResult writeImageFile(@NotNull String id, @NotNull byte[] data, @Nullable String fileExt) {
try {
fileEntityService.writeProtectedFile(id, data, fileExt);
return OperationResult.Companion.success();
} catch (Exception e) {
log.error("Failed to create protected file: {}", id, e);
return OperationResult.Companion.failure("Failed to create protected file");
}
}
}
@@ -0,0 +1,48 @@
package ru.soune.nocopy.util;
import com.vrt.fileprotection.FileProtector;
import com.vrt.fileprotection.image.ImageLocalSearch;
import com.vrt.fileprotection.image.ImageScore;
import org.springframework.stereotype.Service;
import ru.soune.nocopy.dto.file.SimilarFileDTO;
import ru.soune.nocopy.entity.file.FileEntity;
@Service
public class FileUtil {
public ImageLocalSearch.Result convertToResult(SimilarFileDTO response) {
FileProtector.FileInfo fileInfo =
new FileProtector.FileInfo(FileProtector.Type.IMAGE, response.getFileId(),
String.valueOf(response.getOwnerId()));
ImageScore imageScore = new ImageScore(ImageScore.Rate.valueOf(response.getSimilarityLevel()),
response.getHammingDistance());
return new ImageLocalSearch.Result(fileInfo, imageScore);
}
public FileProtector.FileInfo createFileInfo(FileEntity fileEntity) {
FileProtector.Type type = determineFileType(fileEntity.getMimeType());
return new FileProtector.FileInfo(type, fileEntity.getId(), String.valueOf(fileEntity.getUserId()));
}
public int hamming64(long aHi, long aLo, long bHi, long bLo) {
return Long.bitCount(aHi ^ bHi)
+ Long.bitCount(aLo ^ bLo);
}
private FileProtector.Type determineFileType(String mimeType) {
if (mimeType == null) {
return FileProtector.Type.IMAGE;
}
if (mimeType.startsWith("image")) {
return FileProtector.Type.IMAGE;
} else if (mimeType.startsWith("video")) {
return FileProtector.Type.VIDEO;
} else if (mimeType.startsWith("audio")) {
return FileProtector.Type.AUDIO;
} else {
return FileProtector.Type.IMAGE;
}
}
}
+11 -1
View File
@@ -1,4 +1,8 @@
spring:
flyway:
enabled: true
baseline-on-migrate: true
datasource:
url: jdbc:postgresql://${POSTGRES_HOST}:${POSTGRES_PORT}/${POSTGRES_DB}
username: ${POSTGRES_USER}
@@ -24,11 +28,12 @@ spring:
file:
storage:
base-path: ${FILE_STORAGE_PATH:/data/uploads}
chunk-size: ${FILE_CHUNK_SIZE:5242880} # 5MB
chunk-size: ${FILE_CHUNK_SIZE:1000000} # 1MB
max-file-size: ${MAX_FILE_SIZE:10737418240} # 10GB
max-retry-attempts: ${MAX_RETRY_ATTEMPTS:3}
chunk-timeout-ms: ${CHUNK_TIMEOUT_MS:300000} # 5 минут
temp-ttl-hours: ${TEMP_TTL_HOURS:72} # 3 дня
auth-token-life-hours: ${AUTH_TOKEN_LIFE:1} # 1 час
session-expiry-hours: ${SESSION_EXPIRY_HOURS:24}
security:
@@ -50,3 +55,8 @@ logging:
name: logs/application.log
pattern:
console: "%d{yyyy-MM-dd HH:mm:ss} [%thread] %-5level %logger{36} - %msg%n"
yandex:
api-key: ${YANDEX_API_KEY:AQVNyaVaUmgUb1GMCtf5zSEqFxy0woXrcMOOB43q}
folder-id: ${YANDEX_FOLDER_ID:b1gokpdbm6qfpsou8pcd}
search-url: ${YANDEX_SEARCH_URL:https://searchapi.api.cloud.yandex.net/v2/image/search_by_image}
@@ -0,0 +1,2 @@
CREATE INDEX idx_image_hash_prefix
ON image_hashes ((hash_value >> 48));
@@ -1,101 +0,0 @@
package ru.soune.nocopy.service;
import org.junit.jupiter.api.BeforeEach;
import org.junit.jupiter.api.Test;
import org.mockito.ArgumentCaptor;
import org.mockito.InjectMocks;
import org.mockito.Mock;
import org.mockito.MockitoAnnotations;
import org.springframework.context.MessageSource;
import org.springframework.security.crypto.password.PasswordEncoder;
import ru.soune.nocopy.dto.LoginRequest;
import ru.soune.nocopy.entity.AuthToken;
import ru.soune.nocopy.entity.User;
import ru.soune.nocopy.exception.NotValidationPasswordException;
import ru.soune.nocopy.exception.UserNotFoundException;
import ru.soune.nocopy.repository.AuthTokenRepository;
import ru.soune.nocopy.repository.UserRepository;
import java.util.Optional;
import static org.junit.jupiter.api.Assertions.*;
import static org.mockito.ArgumentMatchers.any;
import static org.mockito.Mockito.verify;
import static org.mockito.Mockito.when;
public class AuthServiceTest {
@Mock
private UserRepository userRepository;
@Mock
private AuthTokenRepository authTokenRepository;
@Mock
private PasswordEncoder passwordEncoder;
@Mock
private MessageSource messageSource;
@InjectMocks
private AuthService authService;
@BeforeEach
void setUp() {
MockitoAnnotations.openMocks(this);
}
@Test
void login_success() {
LoginRequest request = new LoginRequest("test@mail.com", "password");
User user = new User();
user.setId(1L);
user.setEmail("test@mail.com");
user.setPassword("encoded_pass");
when(userRepository.findByEmail("test@mail.com"))
.thenReturn(Optional.of(user));
when(passwordEncoder.matches("password", "encoded_pass"))
.thenReturn(true);
when(authTokenRepository.save(any(AuthToken.class)))
.thenAnswer(invocation -> invocation.getArgument(0));
AuthToken token = authService.login(request);
assertNotNull(token);
assertNotNull(token.getToken());
// assertEquals(user.getTokens().get(0), token.getUser());
ArgumentCaptor<AuthToken> captor = ArgumentCaptor.forClass(AuthToken.class);
verify(authTokenRepository).save(captor.capture());
// assertEquals(user, captor.getValue().getUser());
assertNotNull(captor.getValue().getToken());
}
@Test
void login_userNotFound() {
LoginRequest request = new LoginRequest("missing@mail.com", "pwd");
when(userRepository.findByEmail("missing@mail.com")).thenReturn(Optional.empty());
assertThrows(UserNotFoundException.class, () -> authService.login(request));
}
@Test
void login_invalidPassword() {
LoginRequest request = new LoginRequest("test@mail.com", "wrong");
User user = new User();
user.setEmail("test@mail.com");
user.setPassword("encoded_pass");
when(userRepository.findByEmail("test@mail.com")).thenReturn(Optional.of(user));
when(passwordEncoder.matches("wrong", "encoded_pass")).thenReturn(false);
assertThrows(NotValidationPasswordException.class, () -> authService.login(request));
}
}