♻️ File organize system v2

DysonNetwork.Drive/AppDatabase.cs

@@ -23,6 +23,9 @@ public class AppDatabase(
     public DbSet<QuotaRecord> QuotaRecords { get; set; } = null!;
     
     public DbSet<SnCloudFile> Files { get; set; } = null!;
+    public DbSet<SnFileObject> FileObjects { get; set; } = null!;
+    public DbSet<SnFileReplica> FileReplicas { get; set; } = null!;
+    public DbSet<SnFilePermission> FilePermissions { get; set; } = null!;
     public DbSet<SnCloudFileReference> FileReferences { get; set; } = null!;
     public DbSet<SnCloudFileIndex> FileIndexes { get; set; }
 

DysonNetwork.Drive/Program.cs

@@ -1,5 +1,6 @@
 using DysonNetwork.Drive;
 using DysonNetwork.Drive.Startup;
+using DysonNetwork.Drive.Storage;
 using DysonNetwork.Shared.Auth;
 using DysonNetwork.Shared.Http;
 using DysonNetwork.Shared.Registry;
@@ -38,6 +39,10 @@ using (var scope = app.Services.CreateScope())
 {
     var db = scope.ServiceProvider.GetRequiredService<AppDatabase>();
     await db.Database.MigrateAsync();
+
+    // Run one-time migration
+    var migrationService = scope.ServiceProvider.GetRequiredService<FileMigrationService>();
+    await migrationService.MigrateCloudFilesAsync();
 }
 
 app.ConfigureAppMiddleware();

DysonNetwork.Drive/Startup/ServiceCollectionExtensions.cs

@@ -54,6 +54,7 @@ public static class ServiceCollectionExtensions
 
         public IServiceCollection AddAppBusinessServices()
         {
+            services.AddScoped<Storage.FileMigrationService>();
             services.AddScoped<Storage.FileService>();
             services.AddScoped<Storage.FileReferenceService>();
             services.AddScoped<Storage.PersistentTaskService>();

DysonNetwork.Drive/Storage/FileMigrationService.cs

@@ -0,0 +1,59 @@
+using DysonNetwork.Shared.Models;
+using Microsoft.EntityFrameworkCore;
+
+namespace DysonNetwork.Drive.Storage;
+
+public class FileMigrationService(AppDatabase db, ILogger<FileMigrationService> logger)
+{
+    public async Task MigrateCloudFilesAsync()
+    {
+        logger.LogInformation("Starting cloud file migration.");
+        
+        var cloudFiles = await db.Files
+            .Where(f =>
+                f.ObjectId == null &&
+                f.StorageId != null &&
+                f.PoolId != null &&
+                !db.FileObjects.Any(fo => fo.Id == f.Id)
+            )
+            .ToListAsync();
+
+        logger.LogDebug("Found {Count} cloud files to migrate.", cloudFiles.Count);
+
+        foreach (var cf in cloudFiles)
+        {
+            var fileObject = new SnFileObject
+            {
+                Id = cf.Id,
+                Size = cf.Size,
+                Meta = cf.FileMeta,
+                MimeType = cf.MimeType,
+                Hash = cf.Hash,
+                HasCompression = cf.HasCompression,
+                HasThumbnail = cf.HasThumbnail
+            };
+
+            var fileReplica = new SnFileReplica
+            {
+                Id = Guid.NewGuid(),
+                ObjectId = fileObject.Id,
+                PoolId = cf.PoolId!.Value,
+                StorageId = cf.StorageId ?? cf.Id,
+                Status = SnFileReplicaStatus.Available,
+                IsPrimary = true
+            };
+
+            fileObject.FileReplicas.Add(fileReplica);
+
+            db.FileObjects.Add(fileObject);
+            db.FileReplicas.Add(fileReplica);
+
+            cf.ObjectId = fileObject.Id;
+            cf.Object = fileObject;
+        }
+
+        await db.SaveChangesAsync();
+        
+        logger.LogInformation("Cloud file migration completed.");
+    }
+}

DysonNetwork.Drive/Storage/FileService.cs

@@ -762,27 +762,6 @@ public class FileService(
         await db.SaveChangesAsync();
         return count;
     }
-
-    public async Task<string> CreateFastUploadLinkAsync(SnCloudFile file)
-    {
-        if (file.PoolId is null) throw new InvalidOperationException("Pool ID is null");
-
-        var dest = await GetRemoteStorageConfig(file.PoolId.Value);
-        if (dest is null) throw new InvalidOperationException($"No remote storage configured for pool {file.PoolId}");
-        var client = CreateMinioClient(dest);
-        if (client is null)
-            throw new InvalidOperationException(
-                $"Failed to configure client for remote destination '{file.PoolId}'"
-            );
-
-        var url = await client.PresignedPutObjectAsync(
-            new PresignedPutObjectArgs()
-                .WithBucket(dest.Bucket)
-                .WithObject(file.Id)
-                .WithExpiry(60 * 60 * 24)
-        );
-        return url;
-    }
 }
 
 file class UpdatableCloudFile(SnCloudFile file)

DysonNetwork.Shared/Models/CloudFile.cs

@@ -27,6 +27,9 @@ public class SnCloudFile : ModelBase, ICloudFile, IIdentifiedResource
     public bool HasThumbnail { get; set; } = false;
     public bool IsEncrypted { get; set; } = false;
     
+    [MaxLength(32)] public string? ObjectId { get; set; }
+    public SnFileObject? Object { get; set; }
+    
     public FilePool? Pool { get; set; }
     public Guid? PoolId { get; set; }
     [JsonIgnore] public SnFileBundle? Bundle { get; set; }

DysonNetwork.Shared/Models/CloudFileObject.cs

@@ -0,0 +1,67 @@
+using System.ComponentModel.DataAnnotations;
+using System.ComponentModel.DataAnnotations.Schema;
+
+namespace DysonNetwork.Shared.Models;
+
+public class SnFileObject : ModelBase
+{
+    [MaxLength(32)] public string Id { get; set; }
+
+    public long Size { get; set; }
+
+    [Column(TypeName = "jsonb")] public Dictionary<string, object?>? Meta { get; set; }
+    [MaxLength(256)] public string? MimeType { get; set; }
+    [MaxLength(256)] public string? Hash { get; set; }
+    public bool HasCompression { get; set; } = false;
+    public bool HasThumbnail { get; set; } = false;
+
+    public List<SnFileReplica> FileReplicas { get; set; } = [];
+}
+
+public enum SnFileReplicaStatus
+{
+    Pending,
+    Uploading,
+    Available,
+    Failed,
+    Deprecated
+}
+
+public class SnFileReplica : ModelBase
+{
+    public Guid Id { get; set; }
+
+    [MaxLength(32)] public string ObjectId { get; set; }
+    public SnFileObject Object { get; set; } = null!;
+
+    public Guid PoolId { get; set; }
+    public FilePool Pool { get; set; } = null!;
+
+    [MaxLength(128)]
+    public string StorageId { get; set; } = null!;
+
+    public SnFileReplicaStatus Status { get; set; } = SnFileReplicaStatus.Pending;
+
+    public bool IsPrimary { get; set; } = false;
+}
+
+public enum SnFilePermissionType
+{
+    Someone,
+    Anyone
+}
+
+public enum SnFilePermissionLevel
+{
+    Read,
+    Write
+}
+
+public class SnFilePermission : ModelBase
+{
+    public Guid Id { get; set; }
+    public string FileId { get; set; } = null!;
+    public SnFilePermissionType SubjectType { get; set; } = SnFilePermissionType.Someone;
+    public string SubjectId { get; set; } = null!;
+    public SnFilePermissionLevel Permission { get; set; } = SnFilePermissionLevel.Read;
+}