⚡ Much larger batch in validation files

DysonNetwork.Drive/Storage/FileReanalysisService.cs

@@ -239,6 +239,102 @@ public class FileReanalysisService(
         }
     }
 
+    private async Task ValidateBatchCompressionAndThumbnailAsync(List<SnCloudFile> files, bool validateCompression, bool validateThumbnail)
+    {
+        // Collect unique pool IDs and fetch all pools in one query
+        var poolIds = files.Select(f => f.Object!.FileReplicas.First(r => r.IsPrimary).PoolId)
+            .Where(pid => pid.HasValue)
+            .Select(pid => pid.Value)
+            .Distinct()
+            .ToList();
+        var pools = await db.Pools.Where(p => poolIds.Contains(p.Id)).ToDictionaryAsync(p => p.Id);
+
+        var semaphore = new SemaphoreSlim(20);
+        var groupedByPool = files.GroupBy(f => f.Object!.FileReplicas.First(r => r.IsPrimary).PoolId);
+
+        var tasks = new List<Task>();
+        foreach (var group in groupedByPool)
+        {
+            if (!group.Key.HasValue) continue;
+            var poolId = group.Key.Value;
+            var poolFiles = group.ToList();
+            var task = semaphore.WaitAsync().ContinueWith(async _ =>
+            {
+                try
+                {
+                    if (!pools.TryGetValue(poolId, out var pool))
+                    {
+                        logger.LogWarning("No pool found for pool {PoolId}, skipping batch validation", poolId);
+                        return;
+                    }
+
+                    var dest = pool.StorageConfig;
+                    var client = CreateMinioClient(dest);
+                    if (client == null)
+                    {
+                        logger.LogWarning("Failed to create Minio client for pool {PoolId}, skipping batch validation", poolId);
+                        return;
+                    }
+
+                    // List all objects in the bucket to merge existence checks
+                    var listArgs = new ListObjectsArgs().WithBucket(dest.Bucket);
+                    var objectNames = new HashSet<string>();
+                    await foreach (var item in client.ListObjectsEnumAsync(listArgs))
+                        objectNames.Add(item.Key);
+
+                    // Check existence for each file in the group
+                    foreach (var file in poolFiles)
+                    {
+                        if (file.Object == null) continue;
+                        var primaryReplica = file.Object.FileReplicas.FirstOrDefault(r => r.IsPrimary);
+                        if (primaryReplica == null) continue;
+
+                        var updated = false;
+
+                        if (validateCompression && file.Object.HasCompression)
+                        {
+                            if (!objectNames.Contains(primaryReplica.StorageId + ".compressed"))
+                            {
+                                logger.LogInformation("File {FileId} has compression flag but compressed version not found, setting HasCompression to false", file.Id);
+                                file.Object.HasCompression = false;
+                                updated = true;
+                            }
+                        }
+
+                        if (validateThumbnail && file.Object.HasThumbnail)
+                        {
+                            if (!objectNames.Contains(primaryReplica.StorageId + ".thumbnail"))
+                            {
+                                logger.LogInformation("File {FileId} has thumbnail flag but thumbnail not found, setting HasThumbnail to false", file.Id);
+                                file.Object.HasThumbnail = false;
+                                updated = true;
+                            }
+                        }
+
+                        if (updated)
+                        {
+                            logger.LogInformation("Updated compression/thumbnail status for file {FileId}", file.Id);
+                        }
+                    }
+
+                    // Save changes for the group
+                    await db.SaveChangesAsync();
+                }
+                catch (Exception ex)
+                {
+                    logger.LogError(ex, "Failed to batch validate compression/thumbnail for pool {PoolId}", poolId);
+                }
+                finally
+                {
+                    semaphore.Release();
+                }
+            }).Unwrap();
+            tasks.Add(task);
+        }
+
+        await Task.WhenAll(tasks);
+    }
+
     private static async Task<bool> ObjectExistsAsync(IMinioClient client, string bucket, string objectName)
     {
         try
@@ -290,28 +386,26 @@ public class FileReanalysisService(
 
         if (_options.ValidateCompression)
         {
-            var compressionFiles = await GetFilesNeedingCompressionValidationAsync(5);
+            var compressionFiles = await GetFilesNeedingCompressionValidationAsync();
             if (compressionFiles.Count > 0)
             {
-                var file = compressionFiles[0];
+                await ValidateBatchCompressionAndThumbnailAsync(compressionFiles, true, false);
-                await ValidateCompressionAndThumbnailAsync(file);
+                _validationProcessed += compressionFiles.Count;
-                _validationProcessed++;
+                _totalProcessed += compressionFiles.Count;
-                _totalProcessed++;
+                logger.LogInformation("Batch compression validation progress: {ValidationProcessed} processed", _validationProcessed);
-                logger.LogInformation("Validation progress: {ValidationProcessed} processed", _validationProcessed);
                 return;
             }
         }
 
         if (_options.ValidateThumbnails)
         {
-            var thumbnailFiles = await GetFilesNeedingThumbnailValidationAsync(5);
+            var thumbnailFiles = await GetFilesNeedingThumbnailValidationAsync();
             if (thumbnailFiles.Count > 0)
             {
-                var file = thumbnailFiles[0];
+                await ValidateBatchCompressionAndThumbnailAsync(thumbnailFiles, false, true);
-                await ValidateCompressionAndThumbnailAsync(file);
+                _validationProcessed += thumbnailFiles.Count;
-                _validationProcessed++;
+                _totalProcessed += thumbnailFiles.Count;
-                _totalProcessed++;
+                logger.LogInformation("Batch thumbnail validation progress: {ValidationProcessed} processed", _validationProcessed);
-                logger.LogInformation("Validation progress: {ValidationProcessed} processed", _validationProcessed);
                 return;
             }
         }

DysonNetwork.sln.DotSettings.user

@@ -76,6 +76,7 @@
 	<s:String x:Key="/Default/CodeInspection/ExcludedFiles/FilesAndFoldersToSkip2/=7020124F_002D9FFC_002D4AC3_002D8F3D_002DAAB8E0240759_002Ff_003AHttpStatusCode_002Ecs_002Fl_003A_002E_002E_003F_002E_002E_003F_002E_002E_003FLibrary_003FApplication_0020Support_003FJetBrains_003FRider2025_002E1_003Fresharper_002Dhost_003FDecompilerCache_003Fdecompiler_003Fb3f2e07d4b3f4b42a41fbcf3137e534f3be00_003Fe2_003F215f9441_003FHttpStatusCode_002Ecs/@EntryIndexedValue">ForceIncluded</s:String>
 	<s:String x:Key="/Default/CodeInspection/ExcludedFiles/FilesAndFoldersToSkip2/=7020124F_002D9FFC_002D4AC3_002D8F3D_002DAAB8E0240759_002Ff_003AHttpTransformer_002Ecs_002Fl_003A_002E_002E_003F_002E_002E_003F_002E_002E_003FLibrary_003FApplication_0020Support_003FJetBrains_003FRider2025_002E1_003Fresharper_002Dhost_003FDecompilerCache_003Fdecompiler_003Fbf3f51607a3e4e76b5b91640cd7409195c430_003F8a_003Fd9fba048_003FHttpTransformer_002Ecs/@EntryIndexedValue">ForceIncluded</s:String>
 	<s:String x:Key="/Default/CodeInspection/ExcludedFiles/FilesAndFoldersToSkip2/=7020124F_002D9FFC_002D4AC3_002D8F3D_002DAAB8E0240759_002Ff_003AHttpUtility_002Ecs_002Fl_003A_002E_002E_003F_002E_002E_003F_002E_002E_003FLibrary_003FApplication_0020Support_003FJetBrains_003FRider2025_002E1_003Fresharper_002Dhost_003FDecompilerCache_003Fdecompiler_003F95cd5fa21c574d4087dec626d8227d77be00_003F08_003Fdd41228e_003FHttpUtility_002Ecs/@EntryIndexedValue">ForceIncluded</s:String>
+	<s:String x:Key="/Default/CodeInspection/ExcludedFiles/FilesAndFoldersToSkip2/=7020124F_002D9FFC_002D4AC3_002D8F3D_002DAAB8E0240759_002Ff_003AIBucketOperations_002Ecs_002Fl_003A_002E_002E_003F_002E_002E_003F_002E_002E_003FLibrary_003FApplication_0020Support_003FJetBrains_003FRider2025_002E3_003Fresharper_002Dhost_003FSourcesCache_003F41f33e119895b8c586d433c16b50ee21ce9dd4df26558f47a90de2a55138_003FIBucketOperations_002Ecs/@EntryIndexedValue">ForceIncluded</s:String>
 	<s:String x:Key="/Default/CodeInspection/ExcludedFiles/FilesAndFoldersToSkip2/=7020124F_002D9FFC_002D4AC3_002D8F3D_002DAAB8E0240759_002Ff_003AIConfiguration_002Ecs_002Fl_003A_002E_002E_003F_002E_002E_003F_002E_002E_003FLibrary_003FApplication_0020Support_003FJetBrains_003FRider2025_002E1_003Fresharper_002Dhost_003FDecompilerCache_003Fdecompiler_003Fbb55221b2bd14b31a20b0d8bdcc7ff457328_003F19_003F707d23be_003FIConfiguration_002Ecs/@EntryIndexedValue">ForceIncluded</s:String>
 	<s:String x:Key="/Default/CodeInspection/ExcludedFiles/FilesAndFoldersToSkip2/=7020124F_002D9FFC_002D4AC3_002D8F3D_002DAAB8E0240759_002Ff_003AIConnectionMultiplexer_002Ecs_002Fl_003A_002E_002E_003F_002E_002E_003F_002E_002E_003FLibrary_003FApplication_0020Support_003FJetBrains_003FRider2025_002E2_003Fresharper_002Dhost_003FSourcesCache_003Ffd5f2a75d480e8c786b15cfa0ac11aa9bf445a667ad13d25dc9db61f2cb1b_003FIConnectionMultiplexer_002Ecs/@EntryIndexedValue">ForceIncluded</s:String>
 	<s:String x:Key="/Default/CodeInspection/ExcludedFiles/FilesAndFoldersToSkip2/=7020124F_002D9FFC_002D4AC3_002D8F3D_002DAAB8E0240759_002Ff_003AIDatabaseAsync_002Ecs_002Fl_003A_002E_002E_003F_002E_002E_003F_002E_002E_003FLibrary_003FApplication_0020Support_003FJetBrains_003FRider2025_002E2_003Fresharper_002Dhost_003FSourcesCache_003F93b441a9e8201c5bdfa1b1fdc8061a77b86ccbb8566d7bae85036aba8c618f7_003FIDatabaseAsync_002Ecs/@EntryIndexedValue">ForceIncluded</s:String>