NCBACK-27 #12

Merged
backdev merged 6 commits from NCBACK-27 into dev 2026-01-20 16:20:38 +08:00
9 changed files with 221 additions and 160 deletions
Showing only changes of commit 18d35cb0fb - Show all commits
+5 -4
View File
@@ -58,7 +58,8 @@ services:
environment: environment:
FILE_STORAGE_PATH: /data/uploads FILE_STORAGE_PATH: /data/uploads
MAX_FILE_SIZE: 10737418240 MAX_FILE_SIZE: 10737418240
FILE_CHUNK_SIZE: 1048576 # FILE_CHUNK_SIZE: 1048576
FILE_CHUNK_SIZE: 1000000
POSTGRES_DB: no_copy_ POSTGRES_DB: no_copy_
POSTGRES_USER: postgres POSTGRES_USER: postgres
POSTGRES_PASSWORD: postgres POSTGRES_PASSWORD: postgres
@@ -120,7 +121,7 @@ services:
prometheus: prometheus:
image: prom/prometheus:latest image: prom/prometheus:latest
container_name: prometheus container_name: prometheus
deploy: # ← ДОБАВЛЕНО deploy:
resources: resources:
limits: limits:
cpus: '0.5' cpus: '0.5'
@@ -148,7 +149,7 @@ services:
loki: loki:
image: grafana/loki:2.9.2 image: grafana/loki:2.9.2
container_name: loki container_name: loki
deploy: # ← ДОБАВЛЕНО deploy:
resources: resources:
limits: limits:
cpus: '1.0' cpus: '1.0'
@@ -173,7 +174,7 @@ services:
tempo: tempo:
image: grafana/tempo:2.4.1 image: grafana/tempo:2.4.1
container_name: tempo container_name: tempo
deploy: # ← ДОБАВЛЕНО deploy:
resources: resources:
limits: limits:
cpus: '0.5' cpus: '0.5'
+39 -141
View File
@@ -7,7 +7,6 @@ pipeline {
defaultValue: 'dev', defaultValue: 'dev',
description: 'Ветка для деплоя' description: 'Ветка для деплоя'
) )
string( string(
name: 'SERVER', name: 'SERVER',
defaultValue: '92.242.61.23', defaultValue: '92.242.61.23',
@@ -51,156 +50,57 @@ pipeline {
sshpass -p '$SSH_PASS' ssh -o StrictHostKeyChecking=no $SSH_USER@$SERVER " sshpass -p '$SSH_PASS' ssh -o StrictHostKeyChecking=no $SSH_USER@$SERVER "
cd /opt/deployments/${params.BRANCH} cd /opt/deployments/${params.BRANCH}
echo '========================================' echo '1. Остановка старого приложения...'
echo 'Step 1: Checking current state...' docker stop app-backend 2>/dev/null || echo 'Контейнер не найден'
echo '========================================' docker rm app-backend 2>/dev/null || echo 'Контейнер не найден'
docker ps --filter 'name=postgres|app-backend|storage|grafana|prometheus|loki|tempo|alloy' --format 'table {{.Names}}\\t{{.Image}}\\t{{.Status}}' echo '2. Удаление старых образов...'
docker images --filter 'reference=*app*' -q | xargs -r docker rmi -f 2>/dev/null || echo 'Нет образов для удаления'
echo '========================================' echo '3. Создание сети если нужно...'
echo 'Step 2: Setup environment for compatibility...' docker network create app-network 2>/dev/null || echo 'Сеть уже существует'
echo '========================================'
docker network create app-network 2>/dev/null || echo 'Network app-network already exists' echo '4. Запуск инфраструктуры...'
docker-compose up -d db storage
echo 'COMPOSE_COMPATIBILITY=true' > .env echo '5. Ожидание PostgreSQL...'
sleep 10
echo '========================================' echo '6. Сборка нового образа приложения...'
echo 'Step 3: Force cleanup old application...' docker build --no-cache -t app-backend:latest .
echo '========================================'
docker-compose --compatibility down 2>/dev/null || echo 'No previous compose stack' echo '7. Запуск приложения...'
docker run -d \\
--name app-backend \\
--network app-network \\
--network-alias app \\
-p 80:8080 \\
-v uploads_data:/data/uploads:rw \\
-e POSTGRES_DB=no_copy_ \\
-e POSTGRES_USER=postgres \\
-e POSTGRES_PASSWORD=postgres \\
-e POSTGRES_PORT=5432 \\
-e POSTGRES_HOST=db \\
--restart unless-stopped \\
app-backend:latest
docker images --filter 'reference=*app-backend*' -q | xargs -r docker rmi -f 2>/dev/null || echo 'No images to remove' echo '8. Запуск мониторинга...'
docker-compose up -d grafana prometheus loki tempo alloy
echo '========================================' echo '9. Проверка...'
echo 'Step 4: Verify files and fix compose...'
echo '========================================'
if [ ! -f docker-compose.yaml ]; then
echo 'ERROR: docker-compose.yaml not found!'
ls -la
exit 1
fi
echo 'Checking docker-compose.yaml syntax...'
docker-compose --compatibility config 2>&1 | head -20 || echo 'Config check output'
echo '========================================'
echo 'Step 5: Start infrastructure services...'
echo '========================================'
echo 'Starting PostgreSQL with resource limits...'
docker-compose --compatibility up -d db
echo 'Starting storage service...'
docker-compose --compatibility up -d storage
echo 'Waiting for PostgreSQL to be ready...'
for i in {1..30}; do
if docker-compose --compatibility exec db pg_isready -U postgres 2>/dev/null; then
echo 'PostgreSQL is ready'
break
fi
echo 'Waiting... attempt ' \$i
sleep 2
done
echo '========================================'
echo 'Step 6: Build application image...'
echo '========================================'
if [ ! -f Dockerfile ]; then
echo 'ERROR: Dockerfile not found!'
exit 1
fi
echo 'Building Docker image...'
docker build --no-cache -t app-backend:latest . 2>&1 | tail -30
echo 'Image created successfully'
docker images app-backend:latest --format 'table {{.Repository}}\\t{{.Tag}}\\t{{.Size}}\\t{{.CreatedAt}}'
echo '========================================'
echo 'Step 7: Start application with resource limits...'
echo '========================================'
docker volume create uploads_data 2>/dev/null || echo 'Volume uploads_data already exists'
echo 'Starting main application...'
docker-compose --compatibility up -d app
echo 'Waiting for application to start...'
sleep 15
echo '========================================'
echo 'Step 8: Start monitoring stack...'
echo '========================================'
docker-compose --compatibility up -d grafana prometheus loki tempo alloy
echo '========================================'
echo 'Step 9: Verify deployment...'
echo '========================================'
echo 'All containers status:'
docker-compose --compatibility ps
echo ''
echo 'Resource limits check:'
echo 'App backend limits:'
docker inspect app-backend --format='{{.HostConfig.Memory}} {{.HostConfig.CpuQuota}} {{.HostConfig.CpuPeriod}}' 2>/dev/null || echo 'Cannot inspect'
echo ''
echo 'PostgreSQL limits:'
docker inspect postgres --format='{{.HostConfig.Memory}} {{.HostConfig.CpuQuota}} {{.HostConfig.CpuPeriod}}' 2>/dev/null || echo 'Cannot inspect'
echo ''
echo 'Container resource usage:'
docker stats --no-stream --format 'table {{.Name}}\\t{{.CPUPerc}}\\t{{.MemUsage}}\\t{{.MemPerc}}' postgres app-backend storage 2>/dev/null || echo 'Stats not available'
echo '========================================'
echo 'Step 10: Health checks...'
echo '========================================'
echo 'Application health check...'
for i in {1..10}; do
if curl -s -f http://localhost:80/health 2>/dev/null; then
echo '✓ Application health check PASSED'
break
elif [ \$i -eq 10 ]; then
echo '✗ Application health check FAILED'
echo 'Last logs:'
docker-compose --compatibility logs --tail=20 app
else
echo 'Waiting for application... attempt ' \$i
sleep 5 sleep 5
fi
done
echo 'Checking monitoring services...' echo 'Статус контейнеров:'
if curl -s -f http://localhost:3000/api/health 2>/dev/null; then docker ps --format 'table {{.Names}}\\t{{.Image}}\\t{{.Status}}'
echo '✓ Grafana is healthy'
echo '10. Проверка health...'
if curl -s -f http://localhost:80/health > /dev/null 2>&1; then
echo 'Приложение работает'
echo 'URL: http://${params.SERVER}:80'
else else
echo '⚠ Grafana may be starting' echo 'Проверка health не удалась'
docker logs app-backend --tail=20
fi fi
if curl -s -f http://localhost:9090/-/healthy 2>/dev/null; then
echo '✓ Prometheus is healthy'
fi
echo '========================================'
echo 'Deployment Summary:'
echo '========================================'
echo 'Application URL: http://${params.SERVER}:80'
echo 'Grafana: http://${params.SERVER}:3000 (admin/admin)'
echo 'Prometheus: http://${params.SERVER}:9090'
echo 'Loki: http://${params.SERVER}:3100'
echo ''
echo 'To view logs: docker-compose logs -f app'
echo 'To view all containers: docker-compose ps'
echo 'To stop: docker-compose down'
echo '========================================'
" "
""" """
} }
@@ -213,8 +113,6 @@ pipeline {
success { success {
echo "Deployment successful" echo "Deployment successful"
echo "Application URL: http://${params.SERVER}:80" echo "Application URL: http://${params.SERVER}:80"
echo "Grafana: http://${params.SERVER}:3000"
echo "Prometheus: http://${params.SERVER}:9090"
} }
failure { failure {
echo "Deployment failed for branch ${params.BRANCH}" echo "Deployment failed for branch ${params.BRANCH}"
@@ -0,0 +1,61 @@
package ru.soune.nocopy.dto.file;
import com.fasterxml.jackson.annotation.JsonProperty;
import lombok.AllArgsConstructor;
import lombok.Builder;
import lombok.Data;
import lombok.NoArgsConstructor;
@Data
@Builder
@NoArgsConstructor
@AllArgsConstructor
public class FileInfoUserResponse {
@JsonProperty("all_files_size")
private Long allFileSize;
@JsonProperty("all_files_quantity")
private Integer fileCount;
@JsonProperty("all_files_check")
private Integer filesCheck;
@JsonProperty("all_files_violation")
private Integer filesViolation;
@JsonProperty("images_size")
private Long imagesSize;
@JsonProperty("images_quantity")
private Integer imagesCount;
@JsonProperty("images_check")
private Integer imagesCheck;
@JsonProperty("images_violations")
private Integer imagesViolations;
@JsonProperty("videos_size")
private Long videosSize;
@JsonProperty("videos_quantity")
private Integer videosCount;
@JsonProperty("videos_check")
private Integer videosCheck;
@JsonProperty("videos_violations")
private Integer videosViolations;
@JsonProperty("audios_size")
private Long audiosSize;
@JsonProperty("audios_quantity")
private Integer audiosCount;
@JsonProperty("audios_check")
private Integer audiosCheck;
@JsonProperty("audios_violations")
private Integer audiosViolations;
}
@@ -4,5 +4,7 @@ public enum FileStatus {
ACTIVE, ACTIVE,
DELETED, DELETED,
PROCESSING, PROCESSING,
VIOLATION,
CHECKED,
ERROR ERROR
} }
@@ -7,21 +7,10 @@ import java.util.List;
@Getter @Getter
public enum FileType { public enum FileType {
IMAGE("image", Arrays.asList("jpg", "jpeg", "png", "gif", "bmp", "webp")), IMAGE("image", Arrays.asList("jpg", "jpeg", "png", "gif", "bmp", "webp", "jfif")),
VIDEO("video", Arrays.asList("mp4", "avi", "mov", "wmv", "flv", "mkv", "webm", "m4v", "mpg", "mpeg", VIDEO("video", Arrays.asList("mp4", "avi", "mov", "wmv", "flv", "mkv", "webm", "m4v", "mpg", "mpeg",
"3gp", "3g2", "f4v", "m2ts", "mts", "vob", "ogv", "divx")), "3gp", "3g2", "f4v", "m2ts", "mts", "vob", "ogv", "divx")),
AUDIO("audio", Arrays.asList("mp3", "wav", "flac")), AUDIO("audio", Arrays.asList("mp3", "wav", "flac"));
DOCUMENT("document", Arrays.asList(
"pdf", "txt", "rtf",
"doc", "docx", "xls", "xlsx", "ppt", "pptx", "pps", "ppsx", "dot", "dotx", "xlt", "xltx", "pot", "potx",
"odt", "ods", "odp", "odg", "odf", "odb", "odc", "odi", "odm", "ott", "ots", "otp", "otg", "oth",
"sxw", "sxc", "sxi", "sxd", "sxg", "stc", "sti", "stw", "sxm",
"pages", "numbers", "key",
"csv", "tsv", "xml", "html", "htm", "tex", "md", "markdown",
"epub", "mobi", "azw", "azw3", "fb2",
"wps", "wpt", "et", "dps", "vsd", "vsdx",
"java", "py", "cpp", "c", "h", "js", "css", "php", "sql", "json", "yaml", "yml", "sh", "bat",
"one", "note"));
private final String displayName; private final String displayName;
private final List<String> allowedExtensions; private final List<String> allowedExtensions;
@@ -14,6 +14,7 @@ import ru.soune.nocopy.repository.AuthTokenRepository;
import ru.soune.nocopy.repository.FileEntityRepository; import ru.soune.nocopy.repository.FileEntityRepository;
import ru.soune.nocopy.service.auth.AuthService; import ru.soune.nocopy.service.auth.AuthService;
import ru.soune.nocopy.service.file.FileEntityService; import ru.soune.nocopy.service.file.FileEntityService;
import ru.soune.nocopy.service.file.FileStatsService;
import java.util.Arrays; import java.util.Arrays;
import java.util.List; import java.util.List;
@@ -22,10 +23,13 @@ import java.util.List;
@Component @Component
@RequiredArgsConstructor @RequiredArgsConstructor
public class FileEntityHandler implements RequestHandler { public class FileEntityHandler implements RequestHandler {
private final FileEntityService fileEntityService; private final FileEntityService fileEntityService;
private final AuthService authService; private final AuthService authService;
private final FileStatsService fileStatsService;
private final AuthTokenRepository authTokenRepository; private final AuthTokenRepository authTokenRepository;
private final ObjectMapper objectMapper; private final ObjectMapper objectMapper;
@@ -43,6 +47,8 @@ public class FileEntityHandler implements RequestHandler {
switch (action) { switch (action) {
case "file_info": case "file_info":
return handleGetFileInfo(request, fileRequest); return handleGetFileInfo(request, fileRequest);
case "user_files_info":
return handleGetFilesUserInfo(request, fileRequest);
case "file_by_session": case "file_by_session":
return handleGetFileBySession(request, fileRequest); return handleGetFileBySession(request, fileRequest);
case "user_files": case "user_files":
@@ -75,6 +81,16 @@ public class FileEntityHandler implements RequestHandler {
} }
} }
private BaseResponse handleGetFilesUserInfo(BaseRequest request, FileEntityRequest fileRequest) {
Long userId = getUserIdFromToken(fileRequest.getToken());
FileInfoUserResponse userFileStats = fileStatsService.getUserFileStats(userId);
return new BaseResponse(request.getMsgId(),
MessageCode.SUCCESS.getCode(),
MessageCode.SUCCESS.getDescription(),
userFileStats);
}
private BaseResponse handleGetFileInfo(BaseRequest request, FileEntityRequest fileRequest) { private BaseResponse handleGetFileInfo(BaseRequest request, FileEntityRequest fileRequest) {
try { try {
Long userId = authService.useUserAuthToken(fileRequest.getToken()); Long userId = authService.useUserAuthToken(fileRequest.getToken());
@@ -0,0 +1,93 @@
package ru.soune.nocopy.service.file;
import lombok.RequiredArgsConstructor;
import org.springframework.stereotype.Service;
import ru.soune.nocopy.dto.file.FileInfoUserResponse;
import ru.soune.nocopy.entity.file.FileEntity;
import ru.soune.nocopy.entity.file.FileStatus;
import ru.soune.nocopy.entity.file.FileType;
import ru.soune.nocopy.repository.FileEntityRepository;
import java.util.List;
@Service
@RequiredArgsConstructor
public class FileStatsService {
private final FileEntityRepository fileEntityRepository;
public FileInfoUserResponse getUserFileStats(Long userId) {
List<FileEntity> userFiles = fileEntityRepository.findByUserId(userId);
return calculateStats(userFiles);
}
public FileInfoUserResponse calculateStats(List<FileEntity> files) {
return FileInfoUserResponse.builder()
.allFileSize(calculateTotalSize(files))
.fileCount(calculateTotalCount(files))
.filesCheck(calculateByStatus(files, FileStatus.CHECKED))
.filesViolation(calculateByStatus(files, FileStatus.VIOLATION))
.imagesSize(calculateMediaSize(files, FileType.IMAGE))
.imagesCount(calculateMediaCount(files, FileType.IMAGE))
.imagesCheck(calculateMediaByStatus(files, FileType.IMAGE, FileStatus.CHECKED))
.imagesViolations(calculateMediaByStatus(files, FileType.IMAGE, FileStatus.VIOLATION))
.videosSize(calculateMediaSize(files, FileType.VIDEO))
.videosCount(calculateMediaCount(files, FileType.VIDEO))
.videosCheck(calculateMediaByStatus(files, FileType.VIDEO, FileStatus.CHECKED))
.videosViolations(calculateMediaByStatus(files, FileType.VIDEO, FileStatus.VIOLATION))
.audiosSize(calculateMediaSize(files, FileType.AUDIO))
.audiosCount(calculateMediaCount(files, FileType.AUDIO))
.audiosCheck(calculateMediaByStatus(files, FileType.AUDIO, FileStatus.CHECKED))
.audiosViolations(calculateMediaByStatus(files, FileType.AUDIO, FileStatus.VIOLATION))
.build();
}
private Long calculateTotalSize(List<FileEntity> files) {
return files.stream()
.filter(file -> file.getStatus() != FileStatus.DELETED)
.mapToLong(FileEntity::getFileSize)
.sum();
}
private Integer calculateTotalCount(List<FileEntity> files) {
return (int) files.stream()
.filter(file -> file.getStatus() != FileStatus.DELETED)
.count();
}
private Integer calculateByStatus(List<FileEntity> files, FileStatus status) {
return (int) files.stream()
.filter(file -> file.getStatus() == status)
.count();
}
private Long calculateMediaSize(List<FileEntity> files, FileType fileType) {
return files.stream()
.filter(file -> file.getStatus() != FileStatus.DELETED)
.filter(file -> isFileType(file, fileType))
.mapToLong(FileEntity::getFileSize)
.sum();
}
private Integer calculateMediaCount(List<FileEntity> files, FileType fileType) {
return (int) files.stream()
.filter(file -> file.getStatus() != FileStatus.DELETED)
.filter(file -> isFileType(file, fileType))
.count();
}
private Integer calculateMediaByStatus(List<FileEntity> files, FileType fileType, FileStatus status) {
return (int) files.stream()
.filter(file -> isFileType(file, fileType))
.filter(file -> file.getStatus() == status)
.count();
}
private boolean isFileType(FileEntity file, FileType fileType) {
if (file.getFileExtension() == null) return false;
String extension = file.getFileExtension().toLowerCase().replace(".", "");
return fileType.supportsExtension(extension);
}
}
@@ -102,7 +102,7 @@ public class FileUploadServiceImpl implements FileUploadService {
int totalChunks = (int) Math.ceil((double) fileSize / chunkSize); int totalChunks = (int) Math.ceil((double) fileSize / chunkSize);
log.debug("File will be split into {} chunks (chunk size: {} bytes)", log.debug("File will be split into {} chunks (chunk size: {} bytes)",
totalChunks, chunkSize); totalChunks, chunkSize);
Long chunkSize = totalChunks == 1 ? fileSize : 1048576L; Long chunkSize = totalChunks == 1 ? fileSize : 1000000L;
FileUploadSession session = FileUploadSession.builder() FileUploadSession session = FileUploadSession.builder()
.userId(userId) .userId(userId)
+2 -1
View File
@@ -25,7 +25,8 @@ file:
storage: storage:
base-path: ${FILE_STORAGE_PATH:/data/uploads} base-path: ${FILE_STORAGE_PATH:/data/uploads}
# chunk-size: ${FILE_CHUNK_SIZE:5242880} # 5MB # chunk-size: ${FILE_CHUNK_SIZE:5242880} # 5MB
chunk-size: ${FILE_CHUNK_SIZE:1048576} # 15MB # chunk-size: ${FILE_CHUNK_SIZE:1048576} # 1MB
chunk-size: ${FILE_CHUNK_SIZE:1000000} # 1MB
max-file-size: ${MAX_FILE_SIZE:10737418240} # 10GB max-file-size: ${MAX_FILE_SIZE:10737418240} # 10GB
max-retry-attempts: ${MAX_RETRY_ATTEMPTS:3} max-retry-attempts: ${MAX_RETRY_ATTEMPTS:3}
chunk-timeout-ms: ${CHUNK_TIMEOUT_MS:300000} # 5 минут chunk-timeout-ms: ${CHUNK_TIMEOUT_MS:300000} # 5 минут