dev add stats for search check
Test Workflow / test (push) Successful in 3s

This commit is contained in:
vladp
2026-02-05 18:55:54 +07:00
parent adc978bea8
commit f90b8a0bce
15 changed files with 261 additions and 53 deletions
@@ -5,10 +5,7 @@ import com.vrt.fileprotection.FileProtector;
import com.vrt.fileprotection.NoCopyCheckResult;
import lombok.RequiredArgsConstructor;
import lombok.extern.slf4j.Slf4j;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.core.io.FileSystemResource;
import org.springframework.core.io.Resource;
import org.springframework.core.io.UrlResource;
import org.springframework.data.domain.Page;
import org.springframework.data.domain.Pageable;
import org.springframework.data.web.PageableDefault;
@@ -32,6 +29,7 @@ import ru.soune.nocopy.exception.*;
import ru.soune.nocopy.handler.*;
import ru.soune.nocopy.repository.FileEntityRepository;
import ru.soune.nocopy.service.FileSimilarityService;
import ru.soune.nocopy.service.file.ProtectionsLimitService;
import ru.soune.nocopy.service.register.AuthService;
import ru.soune.nocopy.service.file.FileEntityService;
import ru.soune.nocopy.service.file.FileUploadService;
@@ -69,6 +67,8 @@ public class ApiController {
private final FileUtil fileUtil;
private final ProtectionsLimitService protectionsLimitService;
@PostMapping("/v{version}/data")
public ResponseEntity<?> handlePostRequest(@RequestBody BaseRequest request,
@PathVariable("version") int version) {
@@ -422,6 +422,13 @@ public class ApiController {
return ResponseEntity.ok().body(noCopyCheckResult);
}
@GetMapping("/check/file_stats/{userId}")
public ResponseEntity<?> checkFileProtectStats(@PathVariable(required = false) Long userId) {
Map<String, Object> fileTypeStats = protectionsLimitService.getFileTypeStats(userId);
return ResponseEntity.ok().body(fileTypeStats);
}
private boolean hasDuplicate(List<SimilarFileDTO> similarFiles) {
return similarFiles.stream().anyMatch(f -> f.getHammingDistance() <= 5);
}
@@ -15,21 +15,11 @@ public class CheckIncrementResult {
private Integer remainingLimit;
private String message;
public static CheckIncrementResult success(Integer countChecked, Integer remainingLimit) {
public static CheckIncrementResult success(Integer countChecked) {
return CheckIncrementResult.builder()
.success(true)
.countChecked(countChecked)
.remainingLimit(remainingLimit)
.message("Check count incremented successfully")
.build();
}
public static CheckIncrementResult limitExceeded(Integer remainingLimit) {
return CheckIncrementResult.builder()
.success(false)
.countChecked(null)
.remainingLimit(remainingLimit)
.message("Check limit exceeded")
.build();
}
}
@@ -0,0 +1,16 @@
package ru.soune.nocopy.dto.file;
import lombok.AllArgsConstructor;
import lombok.Builder;
import lombok.Data;
import lombok.NoArgsConstructor;
@Data
@Builder
@NoArgsConstructor
@AllArgsConstructor
public class FileTypeStatsDto {
private String fileType;
private Integer count;
}
@@ -0,0 +1,20 @@
package ru.soune.nocopy.dto.file;
import lombok.AllArgsConstructor;
import lombok.Builder;
import lombok.Data;
import lombok.NoArgsConstructor;
import java.time.LocalDate;
import java.util.Map;
@Data
@Builder
@NoArgsConstructor
@AllArgsConstructor
class PeriodHistoryDto {
private LocalDate periodStart;
private LocalDate periodEnd;
private Integer totalChecks;
private Map<String, Integer> checksByType;
}
@@ -0,0 +1,29 @@
package ru.soune.nocopy.dto.file;
import com.fasterxml.jackson.annotation.JsonFormat;
import lombok.AllArgsConstructor;
import lombok.Builder;
import lombok.Data;
import lombok.NoArgsConstructor;
import java.time.LocalDate;
import java.time.LocalDateTime;
import java.util.Map;
@Data
@Builder
@NoArgsConstructor
@AllArgsConstructor
public class ProtectionLimitsDto {
private Long userId;
private String userName;
private String userEmail;
private LocalDate lastResetDate;
private Integer totalChecksAllTime;
private Integer periodChecksCount;
private Map<String, Integer> checksByFileType;
private LocalDate periodStartDate;
private LocalDate periodEndDate;
@JsonFormat(pattern = "yyyy-MM-dd HH:mm:ss")
private LocalDateTime lastCheckAt;
}
@@ -2,10 +2,13 @@ package ru.soune.nocopy.entity.user;
import jakarta.persistence.*;
import lombok.*;
import org.hibernate.annotations.ColumnDefault;
import org.springframework.data.jpa.domain.support.AuditingEntityListener;
import java.time.LocalDate;
import java.time.LocalDateTime;
import java.time.temporal.ChronoUnit;
import java.util.HashMap;
import java.util.Map;
@Entity
@Table(name = "protect_check")
@@ -25,11 +28,6 @@ public class ProtectedFileCheck {
@JoinColumn(name = "user_id", unique = true, nullable = false)
private User user;
@Column(nullable = false)
@ColumnDefault(value = "10")
@Builder.Default
private Integer limitCheck = 10;
@Column(nullable = false)
@Builder.Default
private Integer countChecked = 0;
@@ -40,14 +38,66 @@ public class ProtectedFileCheck {
@Version
private Long version;
public boolean incrementCheck() {
if (this.limitCheck <= 0) {
return false;
}
@ElementCollection
@CollectionTable(
name = "protection_usage_by_type",
joinColumns = @JoinColumn(name = "protection_usage_id"))
@MapKeyColumn(name = "file_type")
@Column(name = "count")
@Builder.Default
private Map<String, Integer> usageByType = new HashMap<>();
@Column(name = "period_start_date")
@Builder.Default
private LocalDate periodStartDate = LocalDate.now();
@Column(name = "period_count")
@Builder.Default
private Integer periodCount = 0;
public synchronized boolean writeCheck(String fileType) {
resetPeriodIfNeeded();
countChecked++;
periodCount++;
usageByType.merge(fileType, 1, Integer::sum);
lastCheckAt = LocalDateTime.now();
this.countChecked++;
this.limitCheck--;
this.lastCheckAt = LocalDateTime.now();
return true;
}
private void resetPeriodIfNeeded() {
LocalDate today = LocalDate.now();
if (periodStartDate == null ||
ChronoUnit.DAYS.between(periodStartDate, today) >= 30) {
cleanupOldData();
periodStartDate = today;
periodCount = 0;
}
}
public void cleanupOldData() {
if (usageByType != null && !usageByType.isEmpty()) {
usageByType.clear();
periodCount = 0;
}
lastCheckAt = LocalDateTime.now();
}
public int getPeriodCount() {
resetPeriodIfNeeded();
return periodCount;
}
public LocalDate getPeriodStartDate() {
resetPeriodIfNeeded();
return periodStartDate;
}
}
@@ -58,13 +58,13 @@ public class ImageFoundRequestHandler implements RequestHandler {
//TODO uncommited when add billing
// GoogleVisionSearchResponse googleVisionSearchResponse = googleVisionSearchService.searchByFileEntity(fileId);
// checkCounterService.incrementCheckCount(fileEntity.getUserId());
String yandexReverseSearchResponse = yandexSearchService.searchReverseByPublicUrl(fileEntity);
// String yandexReverseSearchResponse = yandexSearchService.searchReverseByPublicUrl(fileEntity);
tariffInfoService.writeOffTokens(fileEntity.getUserId(), TariffConstants.TOKEN_VALUE_FOR_SEARCH);
checkCounterService.incrementCheckCount(fileEntity.getUserId(), fileEntity.getMimeType());
return new BaseResponse(request.getMsgId(), MessageCode.SUCCESS.getCode(),
MessageCode.SUCCESS.getDescription(), mapper.mapToYandexResponse(yandexReverseSearchResponse));
MessageCode.SUCCESS.getDescription()," mapper.mapToYandexResponse(yandexReverseSearchResponse)");
}
}
@@ -4,6 +4,8 @@ import org.springframework.data.jpa.repository.JpaRepository;
import org.springframework.stereotype.Repository;
import ru.soune.nocopy.entity.user.ProtectedFileCheck;
import java.time.LocalDate;
import java.util.List;
import java.util.Optional;
@Repository
@@ -11,5 +13,9 @@ public interface ProtectedFileCheckRepository extends JpaRepository<ProtectedFil
Optional<ProtectedFileCheck> findByUser_Id(Long userId);
ProtectedFileCheck findByUserId(Long userId);
boolean existsByUser_Id(Long userId);
List<ProtectedFileCheck> findByPeriodStartDateBefore(LocalDate thirtyDaysAgo);
}
@@ -11,6 +11,7 @@ import ru.soune.nocopy.entity.user.ProtectedFileCheck;
import ru.soune.nocopy.entity.user.User;
import ru.soune.nocopy.exception.RetryableException;
import ru.soune.nocopy.repository.ProtectedFileCheckRepository;
import ru.soune.nocopy.repository.UserRepository;
@Service
@RequiredArgsConstructor
@@ -19,22 +20,25 @@ public class CheckCounterService {
private final ProtectedFileCheckRepository protectedFileCheckRepository;
@Transactional
public void incrementCheckCount(Long userId) {
try {
ProtectedFileCheck check = protectedFileCheckRepository.findByUser_Id(userId)
.orElseThrow(NullPointerException::new);
private final UserRepository userRepository;
boolean success = check.incrementCheck();
@Transactional
public void incrementCheckCount(Long userId, String mimeType) {
try {
ProtectedFileCheck check = protectedFileCheckRepository.findByUser_Id(userId).orElse(null);
User user = userRepository.findById(userId).orElse(null);
if (check == null) {
check = initProtectCheckLimit(user,0);
}
boolean success = check.writeCheck(mimeType);
if (success) {
protectedFileCheckRepository.save(check);
log.info("Check count incremented for user {}: count={}, limit={}",
userId, check.getCountChecked(), check.getLimitCheck());
CheckIncrementResult.success(check.getCountChecked(), check.getLimitCheck());
} else {
log.warn("Check limit exceeded for user {}", userId);
CheckIncrementResult.limitExceeded(check.getLimitCheck());
log.info("Check count incremented for user {}: count={}",
userId, check.getCountChecked());
CheckIncrementResult.success(check.getCountChecked());
}
} catch (ObjectOptimisticLockingFailureException e) {
@@ -51,27 +55,23 @@ public class CheckCounterService {
return CheckStatus.builder()
.userId(userId)
.countChecked(check.getCountChecked())
.remainingLimit(check.getLimitCheck())
.lastCheckAt(check.getLastCheckAt())
.build();
}
public void initProtectCheckLimit(User user, Integer checked, Integer limit) {
public ProtectedFileCheck initProtectCheckLimit(User user, Integer checked) {
ProtectedFileCheck check = ProtectedFileCheck.builder()
.user(user)
.limitCheck(limit)
.countChecked(checked)
.build();
protectedFileCheckRepository.save(check);
return protectedFileCheckRepository.save(check);
}
@Transactional
public ProtectedFileCheck updateLimit(User user, Integer limit) {
ProtectedFileCheck check = ProtectedFileCheck.builder()
.user(user)
.limitCheck(limit)
.build();
return protectedFileCheckRepository.save(check);
@@ -5,13 +5,17 @@ import org.springframework.scheduling.annotation.Scheduled;
import org.springframework.stereotype.Component;
import lombok.RequiredArgsConstructor;
import lombok.extern.slf4j.Slf4j;
import ru.soune.nocopy.entity.user.ProtectedFileCheck;
import ru.soune.nocopy.repository.ProtectedFileCheckRepository;
import java.io.IOException;
import java.nio.file.*;
import java.nio.file.attribute.BasicFileAttributes;
import java.time.Instant;
import java.time.LocalDate;
import java.time.LocalDateTime;
import java.time.ZoneId;
import java.util.List;
@Slf4j
@Component
@@ -24,10 +28,14 @@ public class FileCleanupService {
@Value("${file.storage.temp-ttl-hours}")
private int tempTtlHours;
private final ProtectedFileCheckRepository protectedFileCheckRepository;
@Scheduled(cron = "0 0 3 * * *")
public void cleanupExpiredFiles() {
log.info("Starting cleanup of expired temporary files");
cleanupOldPeriods();
Path tempDir = Paths.get(basePath, "temp");
if (!Files.exists(tempDir)) {
return;
@@ -77,4 +85,17 @@ public class FileCleanupService {
log.error("Error during cleanup", e);
}
}
private void cleanupOldPeriods() {
LocalDate thirtyDaysAgo = LocalDate.now().minusDays(30);
List<ProtectedFileCheck> checks = protectedFileCheckRepository.findByPeriodStartDateBefore(thirtyDaysAgo);
for (ProtectedFileCheck check : checks) {
check.cleanupOldData();
}
protectedFileCheckRepository.saveAll(checks);
log.info("Cleaned up {} old protection usage records", checks.size());
}
}
@@ -0,0 +1,46 @@
package ru.soune.nocopy.service.file;
import lombok.RequiredArgsConstructor;
import lombok.extern.slf4j.Slf4j;
import org.springframework.stereotype.Service;
import org.springframework.transaction.annotation.Transactional;
import ru.soune.nocopy.dto.file.FileTypeStatsDto;
import ru.soune.nocopy.entity.user.ProtectedFileCheck;
import ru.soune.nocopy.repository.ProtectedFileCheckRepository;
import java.util.Map;
import java.util.stream.Collectors;
@Service
@RequiredArgsConstructor
@Slf4j
public class ProtectionsLimitService {
private final ProtectedFileCheckRepository protectedFileCheckRepository;
@Transactional(readOnly = true)
public Map<String, Object> getFileTypeStats(Long userId) {
ProtectedFileCheck protection = protectedFileCheckRepository
.findByUserId(userId);
Map<String, Integer> checksByType = protection.getUsageByType();
int totalChecks = protection.getPeriodCount();
if (totalChecks == 0) {
return Map.of();
}
Map<String, Object> collect = checksByType.entrySet().stream()
.collect(Collectors.toMap(
Map.Entry::getKey,
entry -> FileTypeStatsDto.builder()
.fileType(entry.getKey())
.count(entry.getValue())
.build()
));
collect.put("allCountForPeriod", totalChecks);
return collect;
}
}
@@ -331,8 +331,11 @@ public class FileUploadServiceImpl implements FileUploadService {
completeFileProcessingAsync(session, status);
if (status != FileStatus.TEMP) {
tariffInfoService.writeOffTokens(session.getUserId(), TariffConstants.TOKEN_VALUE);
String fileType = session.getFileType();
tariffInfoService.writeOffTokens(session.getUserId(), protectCost(fileType));
}
} else {
sessionRepository.save(session);
@@ -369,6 +372,23 @@ public class FileUploadServiceImpl implements FileUploadService {
}
}
private int protectCost(String fileType) {
int cost = 0;
switch (fileType) {
case "video":
cost = TariffConstants.VIDEO_TOKEN_VALUE;
case "image":
cost = TariffConstants.IMAGE_TOKEN_VALUE;
case "audio":
cost = TariffConstants.AUDIO_TOKEN_VALUE;
case "pdf":
cost = TariffConstants.PDF_TOKEN_VALUE;
}
return cost;
}
private String assembleFileSynchronously(FileUploadSession session) throws IOException {
Path finalFilePath = prepareFinalFile(session);
log.info("file path: " + finalFilePath);
@@ -76,7 +76,7 @@ public class AuthService {
AuthToken authToken = genereateAuthToken(savedUser);
checkCounterService.initProtectCheckLimit(user, 0, 10);
checkCounterService.initProtectCheckLimit(user, 0);
return authTokenRepository.save(authToken);
}
@@ -13,6 +13,9 @@ public class TariffConstants {
public static final Integer PRO_TOKENS = 300;
public static final Integer ENTERPRISE_TOKENS = 400;
public static final Integer TOKEN_VALUE = 3;
public static final Integer IMAGE_TOKEN_VALUE = 3;
public static final Integer VIDEO_TOKEN_VALUE = 16;
public static final Integer AUDIO_TOKEN_VALUE = 8;
public static final Integer PDF_TOKEN_VALUE = 6;
public static final Integer TOKEN_VALUE_FOR_SEARCH = 5;
}
@@ -54,7 +54,7 @@ public class TariffInfoService {
Integer tokens = activeTariffInfo.getTokens();
if (tokens - value > 0) {
activeTariffInfo.setTokens(activeTariffInfo.getTokens() - TariffConstants.TOKEN_VALUE);
activeTariffInfo.setTokens(activeTariffInfo.getTokens() - value);
tariffInfoRepository.save(activeTariffInfo);
} else {