From bd9ecd4f84ad076b371cb5e93e9230257371ccac Mon Sep 17 00:00:00 2001 From: vladp Date: Tue, 30 Dec 2025 12:25:22 +0700 Subject: [PATCH] change chunk size --- docker-compose.yaml | 2 +- infrastructure/jenkins/deploy.groovy | 241 +++++++++++------- .../service/file/FileUploadServiceImpl.java | 2 +- src/main/resources/application.yaml | 2 +- 4 files changed, 146 insertions(+), 101 deletions(-) diff --git a/docker-compose.yaml b/docker-compose.yaml index f1e3e94..f64115c 100644 --- a/docker-compose.yaml +++ b/docker-compose.yaml @@ -58,7 +58,7 @@ services: environment: FILE_STORAGE_PATH: /data/uploads MAX_FILE_SIZE: 10737418240 - FILE_CHUNK_SIZE: 1000000 + FILE_CHUNK_SIZE: 1048576 POSTGRES_DB: no_copy_ POSTGRES_USER: postgres POSTGRES_PASSWORD: postgres diff --git a/infrastructure/jenkins/deploy.groovy b/infrastructure/jenkins/deploy.groovy index 248b0cf..a05e8ca 100644 --- a/infrastructure/jenkins/deploy.groovy +++ b/infrastructure/jenkins/deploy.groovy @@ -45,119 +45,162 @@ pipeline { echo "Deploying branch: ${params.BRANCH}" echo "Copying files to server..." - sshpass -p '$SSH_PASS' ssh $SSH_USER@$SERVER "mkdir -p /opt/deployments/${params.BRANCH}" - sshpass -p '$SSH_PASS' scp -r ./* $SSH_USER@$SERVER:/opt/deployments/${params.BRANCH}/ + sshpass -p '$SSH_PASS' ssh -o StrictHostKeyChecking=no $SSH_USER@$SERVER "mkdir -p /opt/deployments/${params.BRANCH}" + sshpass -p '$SSH_PASS' scp -r -o StrictHostKeyChecking=no ./* $SSH_USER@$SERVER:/opt/deployments/${params.BRANCH}/ - sshpass -p '$SSH_PASS' ssh $SSH_USER@$SERVER " + sshpass -p '$SSH_PASS' ssh -o StrictHostKeyChecking=no $SSH_USER@$SERVER " cd /opt/deployments/${params.BRANCH} + echo '========================================' echo 'Step 1: Checking current state...' + echo '========================================' - # Смотрим что запущено - docker ps --filter 'name=postgres|app-backend' --format 'table {{.Names}}\\t{{.Image}}\\t{{.Status}}\\t{{.Ports}}' + docker ps --filter 'name=postgres|app-backend|storage|grafana|prometheus|loki|tempo|alloy' --format 'table {{.Names}}\\t{{.Image}}\\t{{.Status}}' - echo 'Step 2: Force cleanup old application...' + echo '========================================' + echo 'Step 2: Setup environment for compatibility...' + echo '========================================' - # Принудительно удаляем старый контейнер и образ - docker stop app-backend 2>/dev/null || echo 'App not running' - docker rm app-backend 2>/dev/null || echo 'App not found' - docker rmi app-backend:latest 2>/dev/null || echo 'Image not found' + docker network create app-network 2>/dev/null || echo 'Network app-network already exists' - # Удаляем все образы app-backend - docker images --filter 'reference=app-backend*' -q | xargs -r docker rmi 2>/dev/null || echo 'No images to remove' + echo 'COMPOSE_COMPATIBILITY=true' > .env - echo 'Step 3: Verify copied files...' - echo 'Files in directory:' - ls -la - echo '' - echo 'Checking Java sources:' - find . -name '*.java' | head -2 + echo '========================================' + echo 'Step 3: Force cleanup old application...' + echo '========================================' - echo 'Step 4: Check if PostgreSQL is running...' - if ! docker ps | grep -q postgres; then - echo 'Starting PostgreSQL...' - docker-compose up -d db - sleep 10 - else - echo 'PostgreSQL already running' - fi + docker-compose --compatibility down 2>/dev/null || echo 'No previous compose stack' - echo 'Step 5: Build application with forced rebuild...' - echo 'Checking Dockerfile exists:' - ls -la Dockerfile 2>/dev/null || echo 'Dockerfile not found, creating simple one' + docker images --filter 'reference=*app-backend*' -q | xargs -r docker rmi -f 2>/dev/null || echo 'No images to remove' - # Если нет Dockerfile, создаем простой - if [ ! -f Dockerfile ]; then - echo 'Creating simple Dockerfile...' - cat > Dockerfile << 'EOF' -FROM eclipse-temurin:21-jre -WORKDIR /app -COPY . /app/ -CMD ["java", "-jar", "app.jar"] -EOF - fi + echo '========================================' + echo 'Step 4: Verify files and fix compose...' + echo '========================================' - # Собираем с подробным выводом - echo 'Building Docker image...' - docker build --no-cache --progress=plain -t app-backend:latest . 2>&1 | tail -50 - - echo 'Step 6: Verify new image...' - echo 'Current app-backend images:' - docker images | grep app-backend - - echo 'Image creation time:' - docker inspect app-backend:latest --format='{{.Created}}' 2>/dev/null || echo 'Cannot inspect image' - - echo 'Step 7: Starting application with alias...' - docker run -d \\ - --name app-backend \\ - --network app-network \\ - --network-alias app \\ - -p 80:8080 \\ - -v /opt/uploads:/data/uploads:rw \\ - -e POSTGRES_DB=no_copy_ \\ - -e POSTGRES_USER=postgres \\ - -e POSTGRES_PASSWORD=postgres \\ - -e POSTGRES_PORT=5432 \\ - -e POSTGRES_HOST=db \\ - --restart unless-stopped \\ - app-backend:latest - - echo 'Step 8: Checking deployment...' - sleep 10 - - echo 'Containers status:' - docker ps --filter 'name=postgres|app-backend' --format 'table {{.Names}}\\t{{.Image}}\\t{{.Status}}\\t{{.CreatedAt}}' - - echo 'Checking image inside container:' - docker exec app-backend ls -la /app/ 2>/dev/null || echo 'Cannot check container files' - - echo 'Application logs (last 10 lines):' - docker logs --tail=10 app-backend 2>/dev/null || echo 'Logs not available yet' - - echo 'Step 9: Health check...' - if docker ps | grep -q app-backend; then - echo 'Container is running' - echo 'Testing application health...' - for i in {1..5}; do - if curl -s -f http://localhost:80/health > /dev/null 2>&1; then - echo 'Health check passed' - echo 'Deployment successful' - echo 'Application URL: http://${params.SERVER}:80' - exit 0 - fi - echo 'Waiting for application to start... attempt ' \$i - sleep 5 - done - - echo 'Health check failed, but container is running' - echo 'Application might be starting slowly' - echo 'Application URL: http://${params.SERVER}:80' - else - echo 'Application failed to start' - docker logs app-backend + if [ ! -f docker-compose.yaml ]; then + echo 'ERROR: docker-compose.yaml not found!' + ls -la exit 1 fi + + echo 'Checking docker-compose.yaml syntax...' + docker-compose --compatibility config 2>&1 | head -20 || echo 'Config check output' + + echo '========================================' + echo 'Step 5: Start infrastructure services...' + echo '========================================' + + echo 'Starting PostgreSQL with resource limits...' + docker-compose --compatibility up -d db + + echo 'Starting storage service...' + docker-compose --compatibility up -d storage + + echo 'Waiting for PostgreSQL to be ready...' + for i in {1..30}; do + if docker-compose --compatibility exec db pg_isready -U postgres 2>/dev/null; then + echo 'PostgreSQL is ready' + break + fi + echo 'Waiting... attempt ' \$i + sleep 2 + done + + echo '========================================' + echo 'Step 6: Build application image...' + echo '========================================' + + if [ ! -f Dockerfile ]; then + echo 'ERROR: Dockerfile not found!' + exit 1 + fi + + echo 'Building Docker image...' + docker build --no-cache -t app-backend:latest . 2>&1 | tail -30 + + echo 'Image created successfully' + docker images app-backend:latest --format 'table {{.Repository}}\\t{{.Tag}}\\t{{.Size}}\\t{{.CreatedAt}}' + + echo '========================================' + echo 'Step 7: Start application with resource limits...' + echo '========================================' + + docker volume create uploads_data 2>/dev/null || echo 'Volume uploads_data already exists' + + echo 'Starting main application...' + docker-compose --compatibility up -d app + + echo 'Waiting for application to start...' + sleep 15 + + echo '========================================' + echo 'Step 8: Start monitoring stack...' + echo '========================================' + + docker-compose --compatibility up -d grafana prometheus loki tempo alloy + + echo '========================================' + echo 'Step 9: Verify deployment...' + echo '========================================' + + echo 'All containers status:' + docker-compose --compatibility ps + + echo '' + echo 'Resource limits check:' + echo 'App backend limits:' + docker inspect app-backend --format='{{.HostConfig.Memory}} {{.HostConfig.CpuQuota}} {{.HostConfig.CpuPeriod}}' 2>/dev/null || echo 'Cannot inspect' + + echo '' + echo 'PostgreSQL limits:' + docker inspect postgres --format='{{.HostConfig.Memory}} {{.HostConfig.CpuQuota}} {{.HostConfig.CpuPeriod}}' 2>/dev/null || echo 'Cannot inspect' + + echo '' + echo 'Container resource usage:' + docker stats --no-stream --format 'table {{.Name}}\\t{{.CPUPerc}}\\t{{.MemUsage}}\\t{{.MemPerc}}' postgres app-backend storage 2>/dev/null || echo 'Stats not available' + + echo '========================================' + echo 'Step 10: Health checks...' + echo '========================================' + + echo 'Application health check...' + for i in {1..10}; do + if curl -s -f http://localhost:80/health 2>/dev/null; then + echo '✓ Application health check PASSED' + break + elif [ \$i -eq 10 ]; then + echo '✗ Application health check FAILED' + echo 'Last logs:' + docker-compose --compatibility logs --tail=20 app + else + echo 'Waiting for application... attempt ' \$i + sleep 5 + fi + done + + echo 'Checking monitoring services...' + if curl -s -f http://localhost:3000/api/health 2>/dev/null; then + echo '✓ Grafana is healthy' + else + echo '⚠ Grafana may be starting' + fi + + if curl -s -f http://localhost:9090/-/healthy 2>/dev/null; then + echo '✓ Prometheus is healthy' + fi + + echo '========================================' + echo 'Deployment Summary:' + echo '========================================' + echo 'Application URL: http://${params.SERVER}:80' + echo 'Grafana: http://${params.SERVER}:3000 (admin/admin)' + echo 'Prometheus: http://${params.SERVER}:9090' + echo 'Loki: http://${params.SERVER}:3100' + echo '' + echo 'To view logs: docker-compose logs -f app' + echo 'To view all containers: docker-compose ps' + echo 'To stop: docker-compose down' + echo '========================================' " """ } @@ -170,6 +213,8 @@ EOF success { echo "Deployment successful" echo "Application URL: http://${params.SERVER}:80" + echo "Grafana: http://${params.SERVER}:3000" + echo "Prometheus: http://${params.SERVER}:9090" } failure { echo "Deployment failed for branch ${params.BRANCH}" diff --git a/src/main/java/ru/soune/nocopy/service/file/FileUploadServiceImpl.java b/src/main/java/ru/soune/nocopy/service/file/FileUploadServiceImpl.java index 3e09822..f54cec9 100644 --- a/src/main/java/ru/soune/nocopy/service/file/FileUploadServiceImpl.java +++ b/src/main/java/ru/soune/nocopy/service/file/FileUploadServiceImpl.java @@ -102,7 +102,7 @@ public class FileUploadServiceImpl implements FileUploadService { int totalChunks = (int) Math.ceil((double) fileSize / chunkSize); log.debug("File will be split into {} chunks (chunk size: {} bytes)", totalChunks, chunkSize); - Long chunkSize = totalChunks == 1 ? fileSize : 1000000L; + Long chunkSize = totalChunks == 1 ? fileSize : 1048576L; FileUploadSession session = FileUploadSession.builder() .userId(userId) diff --git a/src/main/resources/application.yaml b/src/main/resources/application.yaml index 1dd8dbe..1fd4fab 100644 --- a/src/main/resources/application.yaml +++ b/src/main/resources/application.yaml @@ -25,7 +25,7 @@ file: storage: base-path: ${FILE_STORAGE_PATH:/data/uploads} # chunk-size: ${FILE_CHUNK_SIZE:5242880} # 5MB - chunk-size: ${FILE_CHUNK_SIZE:1000000} # 15MB + chunk-size: ${FILE_CHUNK_SIZE:1048576} # 15MB max-file-size: ${MAX_FILE_SIZE:10737418240} # 10GB max-retry-attempts: ${MAX_RETRY_ATTEMPTS:3} chunk-timeout-ms: ${CHUNK_TIMEOUT_MS:300000} # 5 минут