diff --git a/DysonNetwork.Drive/Storage/FileReanalysisService.cs b/DysonNetwork.Drive/Storage/FileReanalysisService.cs index d352bad1..4416c169 100644 --- a/DysonNetwork.Drive/Storage/FileReanalysisService.cs +++ b/DysonNetwork.Drive/Storage/FileReanalysisService.cs @@ -2,6 +2,7 @@ using System.Globalization; using System.Security.Cryptography; using FFMpegCore; using Microsoft.EntityFrameworkCore; +using Microsoft.Extensions.Options; using Minio; using Minio.DataModel.Args; using Minio.Exceptions; @@ -13,9 +14,10 @@ namespace DysonNetwork.Drive.Storage; public class FileReanalysisService( AppDatabase db, - ILogger logger -) + ILogger logger, + IOptions options) { + private readonly FileReanalysisOptions _options = options.Value; private readonly HashSet _failedFileIds = new(); private async Task> GetFilesNeedingReanalysisAsync(int limit = 100) @@ -34,6 +36,32 @@ public class FileReanalysisService( .ToListAsync(); } + private async Task> GetFilesNeedingCompressionValidationAsync(int limit = 100) + { + return await db.Files + .Where(f => f.ObjectId != null) + .Include(f => f.Object) + .ThenInclude(f => f.FileReplicas) + .Where(f => f.Object != null && f.Object.HasCompression) + .Where(f => f.Object!.FileReplicas.Count > 0) + .OrderBy(f => f.Object!.UpdatedAt) + .Take(limit) + .ToListAsync(); + } + + private async Task> GetFilesNeedingThumbnailValidationAsync(int limit = 100) + { + return await db.Files + .Where(f => f.ObjectId != null) + .Include(f => f.Object) + .ThenInclude(f => f.FileReplicas) + .Where(f => f.Object != null && f.Object.HasThumbnail) + .Where(f => f.Object!.FileReplicas.Count > 0) + .OrderBy(f => f.Object!.UpdatedAt) + .Take(limit) + .ToListAsync(); + } + public async Task ReanalyzeFileAsync(SnCloudFile file) { logger.LogInformation("Starting reanalysis for file {FileId}: {FileName}", file.Id, file.Name); @@ -102,6 +130,11 @@ public class FileReanalysisService( logger.LogInformation("File {FileId} already up to date", file.Id); } + if (_options.ValidateCompression || _options.ValidateThumbnails) + { + await ValidateCompressionAndThumbnailAsync(file); + } + return true; } catch (ObjectNotFoundException) @@ -125,11 +158,112 @@ public class FileReanalysisService( } } + public async Task ValidateCompressionAndThumbnailAsync(SnCloudFile file) + { + if (file.Object == null) + { + logger.LogWarning("File {FileId} missing object, skipping validation", file.Id); + return; + } + + var primaryReplica = file.Object.FileReplicas.FirstOrDefault(r => r.IsPrimary); + if (primaryReplica == null) + { + logger.LogWarning("File {FileId} has no primary replica, skipping validation", file.Id); + return; + } + + try + { + var pool = await db.Pools.FindAsync(primaryReplica.PoolId.Value); + if (pool == null) + { + logger.LogWarning("No pool found for replica {ReplicaId}, skipping validation", primaryReplica.Id); + return; + } + + var dest = pool.StorageConfig; + var client = CreateMinioClient(dest); + if (client == null) + { + logger.LogWarning("Failed to create Minio client for pool {PoolId}, skipping validation", primaryReplica.PoolId); + return; + } + + bool updated = false; + + if (_options.ValidateCompression && file.Object.HasCompression) + { + var compressedExists = await ObjectExistsAsync(client, dest.Bucket, primaryReplica.StorageId + ".compressed"); + if (!compressedExists) + { + logger.LogInformation("File {FileId} has compression flag but compressed version not found, setting HasCompression to false", file.Id); + file.Object.HasCompression = false; + updated = true; + } + } + + if (_options.ValidateThumbnails && file.Object.HasThumbnail) + { + var thumbnailExists = await ObjectExistsAsync(client, dest.Bucket, primaryReplica.StorageId + ".thumbnail"); + if (!thumbnailExists) + { + logger.LogInformation("File {FileId} has thumbnail flag but thumbnail not found, setting HasThumbnail to false", file.Id); + file.Object.HasThumbnail = false; + updated = true; + } + } + + if (updated) + { + await db.SaveChangesAsync(); + logger.LogInformation("Updated compression/thumbnail status for file {FileId}", file.Id); + } + } + catch (Exception ex) + { + logger.LogError(ex, "Failed to validate compression/thumbnail for file {FileId}", file.Id); + } + } + + private async Task ObjectExistsAsync(IMinioClient client, string bucket, string objectName) + { + try + { + var statArgs = new StatObjectArgs() + .WithBucket(bucket) + .WithObject(objectName); + await client.StatObjectAsync(statArgs); + return true; + } + catch (ObjectNotFoundException) + { + return false; + } + } + public async Task ProcessNextFileAsync() { + if (!_options.Enabled) + { + logger.LogDebug("File reanalysis is disabled, skipping"); + return; + } + var files = await GetFilesNeedingReanalysisAsync(10); files = files.Where(f => !_failedFileIds.Contains(f.Id.ToString())).ToList(); if (files.Count == 0) + { + if (_options.ValidateCompression || _options.ValidateThumbnails) + { + files = await GetFilesNeedingCompressionValidationAsync(5); + if (files.Count == 0) + { + files = await GetFilesNeedingThumbnailValidationAsync(5); + } + } + } + if (files.Count == 0) { logger.LogInformation("No files found needing reanalysis"); return; diff --git a/DysonNetwork.Drive/Storage/Options/FileReanalysisOptions.cs b/DysonNetwork.Drive/Storage/Options/FileReanalysisOptions.cs new file mode 100644 index 00000000..fb8e7845 --- /dev/null +++ b/DysonNetwork.Drive/Storage/Options/FileReanalysisOptions.cs @@ -0,0 +1,8 @@ +namespace DysonNetwork.Drive.Storage; + +public class FileReanalysisOptions +{ + public bool Enabled { get; set; } = true; + public bool ValidateCompression { get; set; } = true; + public bool ValidateThumbnails { get; set; } = true; +} diff --git a/DysonNetwork.Drive/appsettings.json b/DysonNetwork.Drive/appsettings.json index 8fb888bc..e0506560 100644 --- a/DysonNetwork.Drive/appsettings.json +++ b/DysonNetwork.Drive/appsettings.json @@ -115,5 +115,10 @@ "KnownProxies": [ "127.0.0.1", "::1" - ] + ], + "FileReanalysis": { + "Enabled": true, + "ValidateCompression": true, + "ValidateThumbnails": true + } }