SolarNetwork/Swarm
2
3
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

🐛 Fix file reanalysis service didn't got configured

Browse Source
This commit is contained in:
LittleSheep
2026-01-14 22:19:43 +08:00
parent 2daf8f5d77
commit 68d0881e34
4 changed files with 43 additions and 25 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
DysonNetwork.Drive/Program.cs
View File

@@ -21,7 +21,7 @@ builder.Services.AddRingService();
builder.Services.AddAccountService(); builder.Services.AddAccountService();
builder.Services.AddAppFlushHandlers(); builder.Services.AddAppFlushHandlers();
builder.Services.AddAppBusinessServices(); builder.Services.AddAppBusinessServices(builder.Configuration);
builder.Services.AddAppScheduledJobs(); builder.Services.AddAppScheduledJobs();
builder.AddSwaggerManifest( builder.AddSwaggerManifest(

4
DysonNetwork.Drive/Startup/ServiceCollectionExtensions.cs
View File

@@ -52,8 +52,10 @@ public static class ServiceCollectionExtensions
return services; return services;
} }
public IServiceCollection AddAppBusinessServices() public IServiceCollection AddAppBusinessServices(IConfiguration configuration)
{ {
services.Configure<Storage.Options.FileReanalysisOptions>(configuration.GetSection("FileReanalysis"));
services.AddScoped<Storage.FileService>(); services.AddScoped<Storage.FileService>();
services.AddScoped<Storage.FileReanalysisService>(); services.AddScoped<Storage.FileReanalysisService>();
services.AddScoped<Storage.PersistentTaskService>(); services.AddScoped<Storage.PersistentTaskService>();

54
DysonNetwork.Drive/Storage/FileReanalysisService.cs
View File

@@ -1,5 +1,6 @@
using System.Globalization; using System.Globalization;
using System.Security.Cryptography; using System.Security.Cryptography;
using DysonNetwork.Drive.Storage.Options;
using FFMpegCore; using FFMpegCore;
using Microsoft.EntityFrameworkCore; using Microsoft.EntityFrameworkCore;
using Microsoft.Extensions.Options; using Microsoft.Extensions.Options;
@@ -33,7 +34,8 @@ public class FileReanalysisService(
.Include(f => f.Object) .Include(f => f.Object)
.ThenInclude(f => f.FileReplicas) .ThenInclude(f => f.FileReplicas)
.Where(f => ((f.Object!.MimeType == null || !f.Object.MimeType.StartsWith("application/")) && .Where(f => ((f.Object!.MimeType == null || !f.Object.MimeType.StartsWith("application/")) &&
(f.Object!.Meta == null || f.Object.Meta.Count == 0)) || f.Object.Size == 0 || f.Object.Hash == null) (f.Object!.Meta == null || f.Object.Meta.Count == 0)) || f.Object.Size == 0 ||
f.Object.Hash == null)
.Where(f => f.Object!.FileReplicas.Count > 0) .Where(f => f.Object!.FileReplicas.Count > 0)
.Where(f => f.CreatedAt <= deadline) .Where(f => f.CreatedAt <= deadline)
.OrderBy(f => f.Object!.UpdatedAt) .OrderBy(f => f.Object!.UpdatedAt)
@@ -239,7 +241,8 @@ public class FileReanalysisService(
} }
} }
private async Task ValidateBatchCompressionAndThumbnailAsync(List<SnCloudFile> files, bool validateCompression, bool validateThumbnail) private async Task ValidateBatchCompressionAndThumbnailAsync(List<SnCloudFile> files, bool validateCompression,
bool validateThumbnail)
{ {
// Collect unique pool IDs and fetch all pools in one query // Collect unique pool IDs and fetch all pools in one query
var poolIds = files.Select(f => f.Object!.FileReplicas.First(r => r.IsPrimary).PoolId) var poolIds = files.Select(f => f.Object!.FileReplicas.First(r => r.IsPrimary).PoolId)
@@ -272,7 +275,8 @@ public class FileReanalysisService(
var client = CreateMinioClient(dest); var client = CreateMinioClient(dest);
if (client == null) if (client == null)
{ {
logger.LogWarning("Failed to create Minio client for pool {PoolId}, skipping batch validation", poolId); logger.LogWarning("Failed to create Minio client for pool {PoolId}, skipping batch validation",
poolId);
return; return;
} }
@@ -295,7 +299,9 @@ public class FileReanalysisService(
{ {
if (!objectNames.Contains(primaryReplica.StorageId + ".compressed")) if (!objectNames.Contains(primaryReplica.StorageId + ".compressed"))
{ {
logger.LogInformation("File {FileId} has compression flag but compressed version not found, setting HasCompression to false", file.Id); logger.LogInformation(
"File {FileId} has compression flag but compressed version not found, setting HasCompression to false",
file.Id);
file.Object.HasCompression = false; file.Object.HasCompression = false;
updated = true; updated = true;
} }
@@ -305,7 +311,9 @@ public class FileReanalysisService(
{ {
if (!objectNames.Contains(primaryReplica.StorageId + ".thumbnail")) if (!objectNames.Contains(primaryReplica.StorageId + ".thumbnail"))
{ {
logger.LogInformation("File {FileId} has thumbnail flag but thumbnail not found, setting HasThumbnail to false", file.Id); logger.LogInformation(
"File {FileId} has thumbnail flag but thumbnail not found, setting HasThumbnail to false",
file.Id);
file.Object.HasThumbnail = false; file.Object.HasThumbnail = false;
updated = true; updated = true;
} }
@@ -353,19 +361,16 @@ public class FileReanalysisService(
public async Task ProcessNextFileAsync() public async Task ProcessNextFileAsync()
{ {
var reanalysisFiles = await GetFilesNeedingReanalysisAsync(10); List<SnCloudFile> reanalysisFiles = [];
reanalysisFiles = reanalysisFiles.Where(f => !_failedFileIds.Contains(f.Id.ToString())).ToList(); if (_options.Enabled)
if (reanalysisFiles.Count > 0)
{ {
if (!_options.Enabled) reanalysisFiles = await GetFilesNeedingReanalysisAsync(10);
{ reanalysisFiles = reanalysisFiles.Where(f => !_failedFileIds.Contains(f.Id.ToString())).ToList();
logger.LogDebug("File reanalysis is disabled, skipping reanalysis but continuing with validation");
} if (reanalysisFiles.Count > 0)
else
{ {
var file = reanalysisFiles[0]; var file = reanalysisFiles[0];
bool success = await ReanalyzeFileAsync(file); var success = await ReanalyzeFileAsync(file);
if (!success) if (!success)
{ {
logger.LogWarning("Failed to reanalyze file {FileId}, skipping for now", file.Id); logger.LogWarning("Failed to reanalyze file {FileId}, skipping for now", file.Id);
@@ -376,13 +381,22 @@ public class FileReanalysisService(
{ {
_reanalysisSuccess++; _reanalysisSuccess++;
} }
_totalProcessed++; _totalProcessed++;
var successRate = (_reanalysisSuccess + _reanalysisFailure) > 0 ? (double)_reanalysisSuccess / (_reanalysisSuccess + _reanalysisFailure) * 100 : 0; var successRate = (_reanalysisSuccess + _reanalysisFailure) > 0
logger.LogInformation("Reanalysis progress: {ReanalysisSuccess} succeeded, {ReanalysisFailure} failed ({SuccessRate:F1}%)", _reanalysisSuccess, _reanalysisFailure, successRate); ? (double)_reanalysisSuccess / (_reanalysisSuccess + _reanalysisFailure) * 100
: 0;
logger.LogInformation(
"Reanalysis progress: {ReanalysisSuccess} succeeded, {ReanalysisFailure} failed ({SuccessRate:F1}%)",
_reanalysisSuccess, _reanalysisFailure, successRate);
return; return;
} }
} }
else
{
logger.LogDebug("File reanalysis is disabled, skipping reanalysis but continuing with validation");
}
if (_options.ValidateCompression) if (_options.ValidateCompression)
{ {
@@ -392,7 +406,8 @@ public class FileReanalysisService(
await ValidateBatchCompressionAndThumbnailAsync(compressionFiles, true, false); await ValidateBatchCompressionAndThumbnailAsync(compressionFiles, true, false);
_validationProcessed += compressionFiles.Count; _validationProcessed += compressionFiles.Count;
_totalProcessed += compressionFiles.Count; _totalProcessed += compressionFiles.Count;
logger.LogInformation("Batch compression validation progress: {ValidationProcessed} processed", _validationProcessed); logger.LogInformation("Batch compression validation progress: {ValidationProcessed} processed",
_validationProcessed);
return; return;
} }
} }
@@ -405,7 +420,8 @@ public class FileReanalysisService(
await ValidateBatchCompressionAndThumbnailAsync(thumbnailFiles, false, true); await ValidateBatchCompressionAndThumbnailAsync(thumbnailFiles, false, true);
_validationProcessed += thumbnailFiles.Count; _validationProcessed += thumbnailFiles.Count;
_totalProcessed += thumbnailFiles.Count; _totalProcessed += thumbnailFiles.Count;
logger.LogInformation("Batch thumbnail validation progress: {ValidationProcessed} processed", _validationProcessed); logger.LogInformation("Batch thumbnail validation progress: {ValidationProcessed} processed",
_validationProcessed);
return; return;
} }
} }

8
DysonNetwork.Drive/Storage/Options/FileReanalysisOptions.cs
View File

@@ -1,8 +1,8 @@
namespace DysonNetwork.Drive.Storage; namespace DysonNetwork.Drive.Storage.Options;
public class FileReanalysisOptions public class FileReanalysisOptions
{ {
public bool Enabled { get; set; } = true; public bool Enabled { get; init; } = true;
public bool ValidateCompression { get; set; } = true; public bool ValidateCompression { get; init; } = true;
public bool ValidateThumbnails { get; set; } = true; public bool ValidateThumbnails { get; init; } = true;
} }