✨ File reanalysis now validate the thumbnail / compressed

DysonNetwork.Drive/Storage/FileReanalysisService.cs

@@ -2,6 +2,7 @@ using System.Globalization;
 using System.Security.Cryptography;
 using FFMpegCore;
 using Microsoft.EntityFrameworkCore;
+using Microsoft.Extensions.Options;
 using Minio;
 using Minio.DataModel.Args;
 using Minio.Exceptions;
@@ -13,9 +14,10 @@ namespace DysonNetwork.Drive.Storage;
 
 public class FileReanalysisService(
     AppDatabase db,
-    ILogger<FileReanalysisService> logger
-)
+    ILogger<FileReanalysisService> logger,
+    IOptions<FileReanalysisOptions> options)
 {
+    private readonly FileReanalysisOptions _options = options.Value;
     private readonly HashSet<string> _failedFileIds = new();
 
     private async Task<List<SnCloudFile>> GetFilesNeedingReanalysisAsync(int limit = 100)
@@ -34,6 +36,32 @@ public class FileReanalysisService(
             .ToListAsync();
     }
 
+    private async Task<List<SnCloudFile>> GetFilesNeedingCompressionValidationAsync(int limit = 100)
+    {
+        return await db.Files
+            .Where(f => f.ObjectId != null)
+            .Include(f => f.Object)
+            .ThenInclude(f => f.FileReplicas)
+            .Where(f => f.Object != null && f.Object.HasCompression)
+            .Where(f => f.Object!.FileReplicas.Count > 0)
+            .OrderBy(f => f.Object!.UpdatedAt)
+            .Take(limit)
+            .ToListAsync();
+    }
+
+    private async Task<List<SnCloudFile>> GetFilesNeedingThumbnailValidationAsync(int limit = 100)
+    {
+        return await db.Files
+            .Where(f => f.ObjectId != null)
+            .Include(f => f.Object)
+            .ThenInclude(f => f.FileReplicas)
+            .Where(f => f.Object != null && f.Object.HasThumbnail)
+            .Where(f => f.Object!.FileReplicas.Count > 0)
+            .OrderBy(f => f.Object!.UpdatedAt)
+            .Take(limit)
+            .ToListAsync();
+    }
+
     public async Task<bool> ReanalyzeFileAsync(SnCloudFile file)
     {
         logger.LogInformation("Starting reanalysis for file {FileId}: {FileName}", file.Id, file.Name);
@@ -102,6 +130,11 @@ public class FileReanalysisService(
                 logger.LogInformation("File {FileId} already up to date", file.Id);
             }
 
+            if (_options.ValidateCompression || _options.ValidateThumbnails)
+            {
+                await ValidateCompressionAndThumbnailAsync(file);
+            }
+
             return true;
         }
         catch (ObjectNotFoundException)
@@ -125,11 +158,112 @@ public class FileReanalysisService(
         }
     }
 
+    public async Task ValidateCompressionAndThumbnailAsync(SnCloudFile file)
+    {
+        if (file.Object == null)
+        {
+            logger.LogWarning("File {FileId} missing object, skipping validation", file.Id);
+            return;
+        }
+
+        var primaryReplica = file.Object.FileReplicas.FirstOrDefault(r => r.IsPrimary);
+        if (primaryReplica == null)
+        {
+            logger.LogWarning("File {FileId} has no primary replica, skipping validation", file.Id);
+            return;
+        }
+
+        try
+        {
+            var pool = await db.Pools.FindAsync(primaryReplica.PoolId.Value);
+            if (pool == null)
+            {
+                logger.LogWarning("No pool found for replica {ReplicaId}, skipping validation", primaryReplica.Id);
+                return;
+            }
+
+            var dest = pool.StorageConfig;
+            var client = CreateMinioClient(dest);
+            if (client == null)
+            {
+                logger.LogWarning("Failed to create Minio client for pool {PoolId}, skipping validation", primaryReplica.PoolId);
+                return;
+            }
+
+            bool updated = false;
+
+            if (_options.ValidateCompression && file.Object.HasCompression)
+            {
+                var compressedExists = await ObjectExistsAsync(client, dest.Bucket, primaryReplica.StorageId + ".compressed");
+                if (!compressedExists)
+                {
+                    logger.LogInformation("File {FileId} has compression flag but compressed version not found, setting HasCompression to false", file.Id);
+                    file.Object.HasCompression = false;
+                    updated = true;
+                }
+            }
+
+            if (_options.ValidateThumbnails && file.Object.HasThumbnail)
+            {
+                var thumbnailExists = await ObjectExistsAsync(client, dest.Bucket, primaryReplica.StorageId + ".thumbnail");
+                if (!thumbnailExists)
+                {
+                    logger.LogInformation("File {FileId} has thumbnail flag but thumbnail not found, setting HasThumbnail to false", file.Id);
+                    file.Object.HasThumbnail = false;
+                    updated = true;
+                }
+            }
+
+            if (updated)
+            {
+                await db.SaveChangesAsync();
+                logger.LogInformation("Updated compression/thumbnail status for file {FileId}", file.Id);
+            }
+        }
+        catch (Exception ex)
+        {
+            logger.LogError(ex, "Failed to validate compression/thumbnail for file {FileId}", file.Id);
+        }
+    }
+
+    private async Task<bool> ObjectExistsAsync(IMinioClient client, string bucket, string objectName)
+    {
+        try
+        {
+            var statArgs = new StatObjectArgs()
+                .WithBucket(bucket)
+                .WithObject(objectName);
+            await client.StatObjectAsync(statArgs);
+            return true;
+        }
+        catch (ObjectNotFoundException)
+        {
+            return false;
+        }
+    }
+
     public async Task ProcessNextFileAsync()
     {
+        if (!_options.Enabled)
+        {
+            logger.LogDebug("File reanalysis is disabled, skipping");
+            return;
+        }
+
         var files = await GetFilesNeedingReanalysisAsync(10);
         files = files.Where(f => !_failedFileIds.Contains(f.Id.ToString())).ToList();
         if (files.Count == 0)
+        {
+            if (_options.ValidateCompression || _options.ValidateThumbnails)
+            {
+                files = await GetFilesNeedingCompressionValidationAsync(5);
+                if (files.Count == 0)
+                {
+                    files = await GetFilesNeedingThumbnailValidationAsync(5);
+                }
+            }
+        }
+        if (files.Count == 0)
         {
             logger.LogInformation("No files found needing reanalysis");
             return;

DysonNetwork.Drive/Storage/Options/FileReanalysisOptions.cs

@@ -0,0 +1,8 @@
+namespace DysonNetwork.Drive.Storage;
+
+public class FileReanalysisOptions
+{
+    public bool Enabled { get; set; } = true;
+    public bool ValidateCompression { get; set; } = true;
+    public bool ValidateThumbnails { get; set; } = true;
+}

DysonNetwork.Drive/appsettings.json

@@ -115,5 +115,10 @@
     "KnownProxies": [
         "127.0.0.1",
         "::1"
-    ]
+    ],
+    "FileReanalysis": {
+        "Enabled": true,
+        "ValidateCompression": true,
+        "ValidateThumbnails": true
+    }
 }