💥 ♻️ Refactor cloud files' references, and loading system

2025-06-01 19:18:23 +08:00
parent 02ae634690
commit 00229fd406
32 changed files with 5204 additions and 582 deletions
--- a/DysonNetwork.Sphere/Storage/CloudFileUnusedRecyclingJob.cs
+++ b/DysonNetwork.Sphere/Storage/CloudFileUnusedRecyclingJob.cs
@ -0,0 +1,127 @@
+using Microsoft.EntityFrameworkCore;
+using NodaTime;
+using Quartz;
+
+namespace DysonNetwork.Sphere.Storage;
+
+public class CloudFileUnusedRecyclingJob(
+    AppDatabase db,
+    FileService fs,
+    FileReferenceService fileRefService,
+    ILogger<CloudFileUnusedRecyclingJob> logger
+)
+    : IJob
+{
+    public async Task Execute(IJobExecutionContext context)
+    {
+        logger.LogInformation("Deleting unused cloud files...");
+
+        var cutoff = SystemClock.Instance.GetCurrentInstant() - Duration.FromHours(1);
+        var now = SystemClock.Instance.GetCurrentInstant();
+
+        // Get files that are either expired or created more than an hour ago
+        var fileIds = await db.Files
+            .Select(f => f.Id)
+            .ToListAsync();
+
+        // Filter to only include files that have no references or all references have expired
+        var deletionPlan = new List<string>();
+        foreach (var batch in fileIds.Chunk(100)) // Process in batches to avoid excessive query size
+        {
+            var references = await fileRefService.GetReferencesAsync(batch);
+            deletionPlan.AddRange(from refer in references
+                where refer.Value.Count == 0 || refer.Value.All(r => r.ExpiredAt != null && now >= r.ExpiredAt)
+                select refer.Key);
+        }
+
+        if (deletionPlan.Count == 0)
+        {
+            logger.LogInformation("No files to delete");
+            return;
+        }
+
+        // Get the actual file objects for the files to be deleted
+        var files = await db.Files
+            .Where(f => deletionPlan.Contains(f.Id))
+            .ToListAsync();
+
+        logger.LogInformation($"Found {files.Count} files to delete...");
+
+        // Group files by StorageId and find which ones are safe to delete
+        var storageIds = files.Where(f => f.StorageId != null)
+            .Select(f => f.StorageId!)
+            .Distinct()
+            .ToList();
+
+        // Check if any other files with the same storage IDs are referenced
+        var usedStorageIds = new List<string>();
+        var filesWithSameStorageId = await db.Files
+            .Where(f => f.StorageId != null &&
+                        storageIds.Contains(f.StorageId) &&
+                        !files.Select(ff => ff.Id).Contains(f.Id))
+            .ToListAsync();
+
+        foreach (var file in filesWithSameStorageId)
+        {
+            // Get all references for the file
+            var references = await fileRefService.GetReferencesAsync(file.Id);
+
+            // Check if file has active references (non-expired)
+            if (references.Any(r => r.ExpiredAt == null || r.ExpiredAt > now) && file.StorageId != null)
+            {
+                usedStorageIds.Add(file.StorageId);
+            }
+        }
+
+        // Group files for deletion
+        var filesToDelete = files.Where(f => f.StorageId == null || !usedStorageIds.Contains(f.StorageId))
+            .GroupBy(f => f.UploadedTo)
+            .ToDictionary(grouping => grouping.Key!, grouping => grouping.ToList());
+
+        // Delete files by remote storage
+        foreach (var group in filesToDelete.Where(group => !string.IsNullOrEmpty(group.Key)))
+        {
+            try
+            {
+                var dest = fs.GetRemoteStorageConfig(group.Key);
+                var client = fs.CreateMinioClient(dest);
+                if (client == null) continue;
+
+                // Create delete tasks for each file in the group
+                var deleteTasks = group.Value.Select(file =>
+                {
+                    var objectId = file.StorageId ?? file.Id;
+                    var tasks = new List<Task>
+                    {
+                        client.RemoveObjectAsync(new Minio.DataModel.Args.RemoveObjectArgs()
+                            .WithBucket(dest.Bucket)
+                            .WithObject(objectId))
+                    };
+
+                    if (file.HasCompression)
+                    {
+                        tasks.Add(client.RemoveObjectAsync(new Minio.DataModel.Args.RemoveObjectArgs()
+                            .WithBucket(dest.Bucket)
+                            .WithObject(objectId + ".compressed")));
+                    }
+
+                    return Task.WhenAll(tasks);
+                });
+
+                await Task.WhenAll(deleteTasks);
+            }
+            catch (Exception ex)
+            {
+                logger.LogError(ex, "Error deleting files from remote storage {remote}", group.Key);
+            }
+        }
+
+        // Delete all file records from the database
+        var fileIdsToDelete = files.Select(f => f.Id).ToList();
+        await db.Files
+            .Where(f => fileIdsToDelete.Contains(f.Id))
+            .ExecuteDeleteAsync();
+
+        logger.LogInformation($"Completed deleting {files.Count} files");
+    }
+}