From 8177bda232b4737550ffbf00057a31ac9cac77d6 Mon Sep 17 00:00:00 2001 From: LittleSheep Date: Sat, 10 Jan 2026 14:34:53 +0800 Subject: [PATCH] :recycle: File organize system v2 --- DysonNetwork.Drive/AppDatabase.cs | 3 + DysonNetwork.Drive/Program.cs | 5 ++ .../Startup/ServiceCollectionExtensions.cs | 1 + .../Storage/FileMigrationService.cs | 59 ++++++++++++++++ DysonNetwork.Drive/Storage/FileService.cs | 21 ------ DysonNetwork.Shared/Models/CloudFile.cs | 5 +- DysonNetwork.Shared/Models/CloudFileObject.cs | 67 +++++++++++++++++++ 7 files changed, 139 insertions(+), 22 deletions(-) create mode 100644 DysonNetwork.Drive/Storage/FileMigrationService.cs create mode 100644 DysonNetwork.Shared/Models/CloudFileObject.cs diff --git a/DysonNetwork.Drive/AppDatabase.cs b/DysonNetwork.Drive/AppDatabase.cs index 386463b4..e9c12fc9 100644 --- a/DysonNetwork.Drive/AppDatabase.cs +++ b/DysonNetwork.Drive/AppDatabase.cs @@ -23,6 +23,9 @@ public class AppDatabase( public DbSet QuotaRecords { get; set; } = null!; public DbSet Files { get; set; } = null!; + public DbSet FileObjects { get; set; } = null!; + public DbSet FileReplicas { get; set; } = null!; + public DbSet FilePermissions { get; set; } = null!; public DbSet FileReferences { get; set; } = null!; public DbSet FileIndexes { get; set; } diff --git a/DysonNetwork.Drive/Program.cs b/DysonNetwork.Drive/Program.cs index c7245a28..65908c9e 100644 --- a/DysonNetwork.Drive/Program.cs +++ b/DysonNetwork.Drive/Program.cs @@ -1,5 +1,6 @@ using DysonNetwork.Drive; using DysonNetwork.Drive.Startup; +using DysonNetwork.Drive.Storage; using DysonNetwork.Shared.Auth; using DysonNetwork.Shared.Http; using DysonNetwork.Shared.Registry; @@ -38,6 +39,10 @@ using (var scope = app.Services.CreateScope()) { var db = scope.ServiceProvider.GetRequiredService(); await db.Database.MigrateAsync(); + + // Run one-time migration + var migrationService = scope.ServiceProvider.GetRequiredService(); + await migrationService.MigrateCloudFilesAsync(); } app.ConfigureAppMiddleware(); diff --git a/DysonNetwork.Drive/Startup/ServiceCollectionExtensions.cs b/DysonNetwork.Drive/Startup/ServiceCollectionExtensions.cs index ce5ead3a..3a1753c5 100644 --- a/DysonNetwork.Drive/Startup/ServiceCollectionExtensions.cs +++ b/DysonNetwork.Drive/Startup/ServiceCollectionExtensions.cs @@ -54,6 +54,7 @@ public static class ServiceCollectionExtensions public IServiceCollection AddAppBusinessServices() { + services.AddScoped(); services.AddScoped(); services.AddScoped(); services.AddScoped(); diff --git a/DysonNetwork.Drive/Storage/FileMigrationService.cs b/DysonNetwork.Drive/Storage/FileMigrationService.cs new file mode 100644 index 00000000..13c56aec --- /dev/null +++ b/DysonNetwork.Drive/Storage/FileMigrationService.cs @@ -0,0 +1,59 @@ +using DysonNetwork.Shared.Models; +using Microsoft.EntityFrameworkCore; + +namespace DysonNetwork.Drive.Storage; + +public class FileMigrationService(AppDatabase db, ILogger logger) +{ + public async Task MigrateCloudFilesAsync() + { + logger.LogInformation("Starting cloud file migration."); + + var cloudFiles = await db.Files + .Where(f => + f.ObjectId == null && + f.StorageId != null && + f.PoolId != null && + !db.FileObjects.Any(fo => fo.Id == f.Id) + ) + .ToListAsync(); + + logger.LogDebug("Found {Count} cloud files to migrate.", cloudFiles.Count); + + foreach (var cf in cloudFiles) + { + var fileObject = new SnFileObject + { + Id = cf.Id, + Size = cf.Size, + Meta = cf.FileMeta, + MimeType = cf.MimeType, + Hash = cf.Hash, + HasCompression = cf.HasCompression, + HasThumbnail = cf.HasThumbnail + }; + + var fileReplica = new SnFileReplica + { + Id = Guid.NewGuid(), + ObjectId = fileObject.Id, + PoolId = cf.PoolId!.Value, + StorageId = cf.StorageId ?? cf.Id, + Status = SnFileReplicaStatus.Available, + IsPrimary = true + }; + + fileObject.FileReplicas.Add(fileReplica); + + db.FileObjects.Add(fileObject); + db.FileReplicas.Add(fileReplica); + + cf.ObjectId = fileObject.Id; + cf.Object = fileObject; + } + + await db.SaveChangesAsync(); + + logger.LogInformation("Cloud file migration completed."); + } +} diff --git a/DysonNetwork.Drive/Storage/FileService.cs b/DysonNetwork.Drive/Storage/FileService.cs index 29ae955c..6c8cfcb9 100644 --- a/DysonNetwork.Drive/Storage/FileService.cs +++ b/DysonNetwork.Drive/Storage/FileService.cs @@ -762,27 +762,6 @@ public class FileService( await db.SaveChangesAsync(); return count; } - - public async Task CreateFastUploadLinkAsync(SnCloudFile file) - { - if (file.PoolId is null) throw new InvalidOperationException("Pool ID is null"); - - var dest = await GetRemoteStorageConfig(file.PoolId.Value); - if (dest is null) throw new InvalidOperationException($"No remote storage configured for pool {file.PoolId}"); - var client = CreateMinioClient(dest); - if (client is null) - throw new InvalidOperationException( - $"Failed to configure client for remote destination '{file.PoolId}'" - ); - - var url = await client.PresignedPutObjectAsync( - new PresignedPutObjectArgs() - .WithBucket(dest.Bucket) - .WithObject(file.Id) - .WithExpiry(60 * 60 * 24) - ); - return url; - } } file class UpdatableCloudFile(SnCloudFile file) diff --git a/DysonNetwork.Shared/Models/CloudFile.cs b/DysonNetwork.Shared/Models/CloudFile.cs index 190f5c2e..b7d4513f 100644 --- a/DysonNetwork.Shared/Models/CloudFile.cs +++ b/DysonNetwork.Shared/Models/CloudFile.cs @@ -26,7 +26,10 @@ public class SnCloudFile : ModelBase, ICloudFile, IIdentifiedResource public bool HasCompression { get; set; } = false; public bool HasThumbnail { get; set; } = false; public bool IsEncrypted { get; set; } = false; - + + [MaxLength(32)] public string? ObjectId { get; set; } + public SnFileObject? Object { get; set; } + public FilePool? Pool { get; set; } public Guid? PoolId { get; set; } [JsonIgnore] public SnFileBundle? Bundle { get; set; } diff --git a/DysonNetwork.Shared/Models/CloudFileObject.cs b/DysonNetwork.Shared/Models/CloudFileObject.cs new file mode 100644 index 00000000..2279dc0d --- /dev/null +++ b/DysonNetwork.Shared/Models/CloudFileObject.cs @@ -0,0 +1,67 @@ +using System.ComponentModel.DataAnnotations; +using System.ComponentModel.DataAnnotations.Schema; + +namespace DysonNetwork.Shared.Models; + +public class SnFileObject : ModelBase +{ + [MaxLength(32)] public string Id { get; set; } + + public long Size { get; set; } + + [Column(TypeName = "jsonb")] public Dictionary? Meta { get; set; } + [MaxLength(256)] public string? MimeType { get; set; } + [MaxLength(256)] public string? Hash { get; set; } + public bool HasCompression { get; set; } = false; + public bool HasThumbnail { get; set; } = false; + + public List FileReplicas { get; set; } = []; +} + +public enum SnFileReplicaStatus +{ + Pending, + Uploading, + Available, + Failed, + Deprecated +} + +public class SnFileReplica : ModelBase +{ + public Guid Id { get; set; } + + [MaxLength(32)] public string ObjectId { get; set; } + public SnFileObject Object { get; set; } = null!; + + public Guid PoolId { get; set; } + public FilePool Pool { get; set; } = null!; + + [MaxLength(128)] + public string StorageId { get; set; } = null!; + + public SnFileReplicaStatus Status { get; set; } = SnFileReplicaStatus.Pending; + + public bool IsPrimary { get; set; } = false; +} + +public enum SnFilePermissionType +{ + Someone, + Anyone +} + +public enum SnFilePermissionLevel +{ + Read, + Write +} + +public class SnFilePermission : ModelBase +{ + public Guid Id { get; set; } + public string FileId { get; set; } = null!; + public SnFilePermissionType SubjectType { get; set; } = SnFilePermissionType.Someone; + public string SubjectId { get; set; } = null!; + public SnFilePermissionLevel Permission { get; set; } = SnFilePermissionLevel.Read; +} \ No newline at end of file