From fc1edf0ea3829dfb77353a39c9f77a0825d44644 Mon Sep 17 00:00:00 2001 From: LittleSheep Date: Tue, 13 Jan 2026 23:26:09 +0800 Subject: [PATCH] :recycle: Update the usage counting since the pool id logic changed --- DysonNetwork.Drive/Billing/UsageService.cs | 99 +-- .../Index/FileIndexController.cs | 2 +- ...152536_RemovePoolFromCloudFile.Designer.cs | 640 ++++++++++++++++++ .../20260113152536_RemovePoolFromCloudFile.cs | 49 ++ .../Migrations/AppDatabaseModelSnapshot.cs | 14 - .../Startup/BroadcastEventHandler.cs | 13 +- .../Startup/ServiceCollectionExtensions.cs | 1 - .../Storage/CloudFileUnusedRecyclingJob.cs | 27 +- DysonNetwork.Drive/Storage/FileController.cs | 8 +- .../Storage/FileMigrationService.cs | 127 ---- .../Storage/FileReanalysisService.cs | 21 +- DysonNetwork.Drive/Storage/FileService.cs | 61 +- DysonNetwork.Shared/Models/CloudFile.cs | 2 - 13 files changed, 833 insertions(+), 231 deletions(-) create mode 100644 DysonNetwork.Drive/Migrations/20260113152536_RemovePoolFromCloudFile.Designer.cs create mode 100644 DysonNetwork.Drive/Migrations/20260113152536_RemovePoolFromCloudFile.cs delete mode 100644 DysonNetwork.Drive/Storage/FileMigrationService.cs diff --git a/DysonNetwork.Drive/Billing/UsageService.cs b/DysonNetwork.Drive/Billing/UsageService.cs index 1c9877e6..b295f44d 100644 --- a/DysonNetwork.Drive/Billing/UsageService.cs +++ b/DysonNetwork.Drive/Billing/UsageService.cs @@ -1,5 +1,6 @@ using Microsoft.EntityFrameworkCore; using NodaTime; +using DysonNetwork.Shared.Models; namespace DysonNetwork.Drive.Billing; @@ -29,28 +30,42 @@ public class UsageService(AppDatabase db) public async Task GetTotalUsage(Guid accountId) { var now = SystemClock.Instance.GetCurrentInstant(); - var fileQuery = db.Files - .Where(f => !f.IsMarkedRecycle) - .Where(f => !f.ExpiredAt.HasValue || f.ExpiredAt > now) - .Where(f => f.AccountId == accountId) - .AsQueryable(); var poolUsages = await db.Pools .Select(p => new UsageDetails { PoolId = p.Id, PoolName = p.Name, - UsageBytes = fileQuery - .Where(f => f.PoolId == p.Id) - .Include(f => f.Object) - .Sum(f => f.Size), - Cost = fileQuery - .Where(f => f.PoolId == p.Id) - .Include(f => f.Object) - .Sum(f => f.Size) / 1024.0 / 1024.0 * - (p.BillingConfig.CostMultiplier ?? 1.0), - FileCount = fileQuery - .Count(f => f.PoolId == p.Id) + UsageBytes = db.Files + .Where(f => f.AccountId == accountId) + .Where(f => !f.IsMarkedRecycle) + .Where(f => !f.ExpiredAt.HasValue || f.ExpiredAt > now) + .SelectMany(f => f.Object!.FileReplicas + .Where(r => r.PoolId == p.Id && r.Status == SnFileReplicaStatus.Available)) + .Join(db.FileObjects, + r => r.ObjectId, + o => o.Id, + (r, o) => o.Size) + .DefaultIfEmpty(0L) + .Sum(), + Cost = db.Files + .Where(f => f.AccountId == accountId) + .Where(f => !f.IsMarkedRecycle) + .Where(f => !f.ExpiredAt.HasValue || f.ExpiredAt > now) + .SelectMany(f => f.Object!.FileReplicas + .Where(r => r.PoolId == p.Id && r.Status == SnFileReplicaStatus.Available)) + .Join(db.FileObjects, + r => r.ObjectId, + o => o.Id, + (r, o) => new { Size = o.Size, Multiplier = p.BillingConfig.CostMultiplier ?? 1.0 }) + .Sum(x => x.Size * x.Multiplier) / 1024.0 / 1024.0, + FileCount = db.Files + .Where(f => f.AccountId == accountId) + .Where(f => !f.IsMarkedRecycle) + .Where(f => !f.ExpiredAt.HasValue || f.ExpiredAt > now) + .SelectMany(f => f.Object!.FileReplicas + .Where(r => r.PoolId == p.Id && r.Status == SnFileReplicaStatus.Available)) + .Count() }) .ToListAsync(); @@ -75,18 +90,22 @@ public class UsageService(AppDatabase db) } var now = SystemClock.Instance.GetCurrentInstant(); - var fileQuery = db.Files - .Where(f => !f.IsMarkedRecycle) - .Where(f => f.ExpiredAt.HasValue && f.ExpiredAt > now) + var replicaQuery = db.Files .Where(f => f.AccountId == accountId) - .AsQueryable(); + .Where(f => !f.IsMarkedRecycle) + .Where(f => !f.ExpiredAt.HasValue || f.ExpiredAt > now) + .SelectMany(f => f.Object!.FileReplicas + .Where(r => r.PoolId == poolId && r.Status == SnFileReplicaStatus.Available)); - var usageBytes = await fileQuery - .Include(f => f.Object) - .SumAsync(f => f.Size); + var usageBytes = await replicaQuery + .Join(db.FileObjects, + r => r.ObjectId, + o => o.Id, + (r, o) => o.Size) + .DefaultIfEmpty(0L) + .SumAsync(); - var fileCount = await fileQuery - .CountAsync(); + var fileCount = await replicaQuery.CountAsync(); var cost = usageBytes / 1024.0 / 1024.0 * (pool.BillingConfig.CostMultiplier ?? 1.0); @@ -104,22 +123,24 @@ public class UsageService(AppDatabase db) public async Task GetTotalBillableUsage(Guid accountId) { var now = SystemClock.Instance.GetCurrentInstant(); - var files = await db.Files - .Where(f => f.AccountId == accountId) - .Where(f => f.PoolId.HasValue) - .Where(f => !f.IsMarkedRecycle) - .Include(f => f.Pool) - .Include(f => f.Object) - .Where(f => !f.ExpiredAt.HasValue || f.ExpiredAt > now) - .Select(f => new + + var billingData = await (from f in db.Files + where f.AccountId == accountId + where !f.IsMarkedRecycle + where !f.ExpiredAt.HasValue || f.ExpiredAt > now + from r in f.Object!.FileReplicas + where r.Status == SnFileReplicaStatus.Available + where r.PoolId.HasValue + join p in db.Pools on r.PoolId equals p.Id + join o in db.FileObjects on r.ObjectId equals o.Id + select new { - f.Size, - Multiplier = f.Pool!.BillingConfig.CostMultiplier ?? 1.0 - }) - .ToListAsync(); + Size = o.Size, + Multiplier = p.BillingConfig.CostMultiplier ?? 1.0 + }).ToListAsync(); - var totalCost = files.Sum(f => f.Size * f.Multiplier) / 1024.0 / 1024.0; + var totalCost = billingData.Sum(x => x.Size * x.Multiplier) / 1024.0 / 1024.0; return (long)Math.Ceiling(totalCost); } -} \ No newline at end of file +} diff --git a/DysonNetwork.Drive/Index/FileIndexController.cs b/DysonNetwork.Drive/Index/FileIndexController.cs index 50fe0ecb..72db0470 100644 --- a/DysonNetwork.Drive/Index/FileIndexController.cs +++ b/DysonNetwork.Drive/Index/FileIndexController.cs @@ -230,7 +230,7 @@ public class FileIndexController( : filesQuery.OrderBy(f => f.CreatedAt) }; - if (pool.HasValue) filesQuery = filesQuery.Where(f => f.PoolId == pool); + if (pool.HasValue) filesQuery = filesQuery.Where(f => f.Object!.FileReplicas.Any(r => r.PoolId == pool.Value)); if (!string.IsNullOrWhiteSpace(query)) { diff --git a/DysonNetwork.Drive/Migrations/20260113152536_RemovePoolFromCloudFile.Designer.cs b/DysonNetwork.Drive/Migrations/20260113152536_RemovePoolFromCloudFile.Designer.cs new file mode 100644 index 00000000..2dab7abb --- /dev/null +++ b/DysonNetwork.Drive/Migrations/20260113152536_RemovePoolFromCloudFile.Designer.cs @@ -0,0 +1,640 @@ +// +using System; +using System.Collections.Generic; +using DysonNetwork.Drive; +using DysonNetwork.Shared.Models; +using Microsoft.EntityFrameworkCore; +using Microsoft.EntityFrameworkCore.Infrastructure; +using Microsoft.EntityFrameworkCore.Migrations; +using Microsoft.EntityFrameworkCore.Storage.ValueConversion; +using NodaTime; +using Npgsql.EntityFrameworkCore.PostgreSQL.Metadata; + +#nullable disable + +namespace DysonNetwork.Drive.Migrations +{ + [DbContext(typeof(AppDatabase))] + [Migration("20260113152536_RemovePoolFromCloudFile")] + partial class RemovePoolFromCloudFile + { + /// + protected override void BuildTargetModel(ModelBuilder modelBuilder) + { +#pragma warning disable 612, 618 + modelBuilder + .HasAnnotation("ProductVersion", "10.0.1") + .HasAnnotation("Relational:MaxIdentifierLength", 63); + + NpgsqlModelBuilderExtensions.UseIdentityByDefaultColumns(modelBuilder); + + modelBuilder.Entity("DysonNetwork.Drive.Billing.QuotaRecord", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("uuid") + .HasColumnName("id"); + + b.Property("AccountId") + .HasColumnType("uuid") + .HasColumnName("account_id"); + + b.Property("CreatedAt") + .HasColumnType("timestamp with time zone") + .HasColumnName("created_at"); + + b.Property("DeletedAt") + .HasColumnType("timestamp with time zone") + .HasColumnName("deleted_at"); + + b.Property("Description") + .IsRequired() + .HasColumnType("text") + .HasColumnName("description"); + + b.Property("ExpiredAt") + .HasColumnType("timestamp with time zone") + .HasColumnName("expired_at"); + + b.Property("Name") + .IsRequired() + .HasColumnType("text") + .HasColumnName("name"); + + b.Property("Quota") + .HasColumnType("bigint") + .HasColumnName("quota"); + + b.Property("UpdatedAt") + .HasColumnType("timestamp with time zone") + .HasColumnName("updated_at"); + + b.HasKey("Id") + .HasName("pk_quota_records"); + + b.ToTable("quota_records", (string)null); + }); + + modelBuilder.Entity("DysonNetwork.Drive.Storage.Model.PersistentTask", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("uuid") + .HasColumnName("id"); + + b.Property("AccountId") + .HasColumnType("uuid") + .HasColumnName("account_id"); + + b.Property("CompletedAt") + .HasColumnType("timestamp with time zone") + .HasColumnName("completed_at"); + + b.Property("CreatedAt") + .HasColumnType("timestamp with time zone") + .HasColumnName("created_at"); + + b.Property("DeletedAt") + .HasColumnType("timestamp with time zone") + .HasColumnName("deleted_at"); + + b.Property("Description") + .HasMaxLength(1024) + .HasColumnType("character varying(1024)") + .HasColumnName("description"); + + b.Property("ErrorMessage") + .HasMaxLength(1024) + .HasColumnType("character varying(1024)") + .HasColumnName("error_message"); + + b.Property("EstimatedDurationSeconds") + .HasColumnType("bigint") + .HasColumnName("estimated_duration_seconds"); + + b.Property("ExpiredAt") + .HasColumnType("timestamp with time zone") + .HasColumnName("expired_at"); + + b.Property("LastActivity") + .HasColumnType("timestamp with time zone") + .HasColumnName("last_activity"); + + b.Property("Name") + .IsRequired() + .HasMaxLength(256) + .HasColumnType("character varying(256)") + .HasColumnName("name"); + + b.Property>("Parameters") + .IsRequired() + .HasColumnType("jsonb") + .HasColumnName("parameters"); + + b.Property("Priority") + .HasColumnType("integer") + .HasColumnName("priority"); + + b.Property("Progress") + .HasColumnType("double precision") + .HasColumnName("progress"); + + b.Property>("Results") + .IsRequired() + .HasColumnType("jsonb") + .HasColumnName("results"); + + b.Property("StartedAt") + .HasColumnType("timestamp with time zone") + .HasColumnName("started_at"); + + b.Property("Status") + .HasColumnType("integer") + .HasColumnName("status"); + + b.Property("TaskId") + .IsRequired() + .HasMaxLength(64) + .HasColumnType("character varying(64)") + .HasColumnName("task_id"); + + b.Property("Type") + .HasColumnType("integer") + .HasColumnName("type"); + + b.Property("UpdatedAt") + .HasColumnType("timestamp with time zone") + .HasColumnName("updated_at"); + + b.HasKey("Id") + .HasName("pk_tasks"); + + b.ToTable("tasks", (string)null); + }); + + modelBuilder.Entity("DysonNetwork.Shared.Models.FilePool", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("uuid") + .HasColumnName("id"); + + b.Property("AccountId") + .HasColumnType("uuid") + .HasColumnName("account_id"); + + b.Property("BillingConfig") + .IsRequired() + .HasColumnType("jsonb") + .HasColumnName("billing_config"); + + b.Property("CreatedAt") + .HasColumnType("timestamp with time zone") + .HasColumnName("created_at"); + + b.Property("DeletedAt") + .HasColumnType("timestamp with time zone") + .HasColumnName("deleted_at"); + + b.Property("Description") + .IsRequired() + .HasMaxLength(8192) + .HasColumnType("character varying(8192)") + .HasColumnName("description"); + + b.Property("IsHidden") + .HasColumnType("boolean") + .HasColumnName("is_hidden"); + + b.Property("Name") + .IsRequired() + .HasMaxLength(1024) + .HasColumnType("character varying(1024)") + .HasColumnName("name"); + + b.Property("PolicyConfig") + .IsRequired() + .HasColumnType("jsonb") + .HasColumnName("policy_config"); + + b.Property("StorageConfig") + .IsRequired() + .HasColumnType("jsonb") + .HasColumnName("storage_config"); + + b.Property("UpdatedAt") + .HasColumnType("timestamp with time zone") + .HasColumnName("updated_at"); + + b.HasKey("Id") + .HasName("pk_pools"); + + b.ToTable("pools", (string)null); + }); + + modelBuilder.Entity("DysonNetwork.Shared.Models.SnCloudFile", b => + { + b.Property("Id") + .HasMaxLength(32) + .HasColumnType("character varying(32)") + .HasColumnName("id"); + + b.Property("AccountId") + .HasColumnType("uuid") + .HasColumnName("account_id"); + + b.Property("BundleId") + .HasColumnType("uuid") + .HasColumnName("bundle_id"); + + b.Property("CreatedAt") + .HasColumnType("timestamp with time zone") + .HasColumnName("created_at"); + + b.Property("DeletedAt") + .HasColumnType("timestamp with time zone") + .HasColumnName("deleted_at"); + + b.Property("Description") + .HasMaxLength(4096) + .HasColumnType("character varying(4096)") + .HasColumnName("description"); + + b.Property("ExpiredAt") + .HasColumnType("timestamp with time zone") + .HasColumnName("expired_at"); + + b.Property("IsMarkedRecycle") + .HasColumnType("boolean") + .HasColumnName("is_marked_recycle"); + + b.Property("Name") + .IsRequired() + .HasMaxLength(1024) + .HasColumnType("character varying(1024)") + .HasColumnName("name"); + + b.Property("ObjectId") + .HasMaxLength(32) + .HasColumnType("character varying(32)") + .HasColumnName("object_id"); + + b.PrimitiveCollection("SensitiveMarks") + .HasColumnType("jsonb") + .HasColumnName("sensitive_marks"); + + b.Property("StorageId") + .HasMaxLength(32) + .HasColumnType("character varying(32)") + .HasColumnName("storage_id"); + + b.Property("StorageUrl") + .HasMaxLength(4096) + .HasColumnType("character varying(4096)") + .HasColumnName("storage_url"); + + b.Property("UpdatedAt") + .HasColumnType("timestamp with time zone") + .HasColumnName("updated_at"); + + b.Property("UploadedAt") + .HasColumnType("timestamp with time zone") + .HasColumnName("uploaded_at"); + + b.Property>("UserMeta") + .HasColumnType("jsonb") + .HasColumnName("user_meta"); + + b.HasKey("Id") + .HasName("pk_files"); + + b.HasIndex("BundleId") + .HasDatabaseName("ix_files_bundle_id"); + + b.HasIndex("ObjectId") + .HasDatabaseName("ix_files_object_id"); + + b.ToTable("files", (string)null); + }); + + modelBuilder.Entity("DysonNetwork.Shared.Models.SnCloudFileIndex", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("uuid") + .HasColumnName("id"); + + b.Property("AccountId") + .HasColumnType("uuid") + .HasColumnName("account_id"); + + b.Property("CreatedAt") + .HasColumnType("timestamp with time zone") + .HasColumnName("created_at"); + + b.Property("DeletedAt") + .HasColumnType("timestamp with time zone") + .HasColumnName("deleted_at"); + + b.Property("FileId") + .IsRequired() + .HasMaxLength(32) + .HasColumnType("character varying(32)") + .HasColumnName("file_id"); + + b.Property("Path") + .IsRequired() + .HasMaxLength(8192) + .HasColumnType("character varying(8192)") + .HasColumnName("path"); + + b.Property("UpdatedAt") + .HasColumnType("timestamp with time zone") + .HasColumnName("updated_at"); + + b.HasKey("Id") + .HasName("pk_file_indexes"); + + b.HasIndex("FileId") + .HasDatabaseName("ix_file_indexes_file_id"); + + b.HasIndex("Path", "AccountId") + .HasDatabaseName("ix_file_indexes_path_account_id"); + + b.ToTable("file_indexes", (string)null); + }); + + modelBuilder.Entity("DysonNetwork.Shared.Models.SnFileBundle", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("uuid") + .HasColumnName("id"); + + b.Property("AccountId") + .HasColumnType("uuid") + .HasColumnName("account_id"); + + b.Property("CreatedAt") + .HasColumnType("timestamp with time zone") + .HasColumnName("created_at"); + + b.Property("DeletedAt") + .HasColumnType("timestamp with time zone") + .HasColumnName("deleted_at"); + + b.Property("Description") + .HasMaxLength(8192) + .HasColumnType("character varying(8192)") + .HasColumnName("description"); + + b.Property("ExpiredAt") + .HasColumnType("timestamp with time zone") + .HasColumnName("expired_at"); + + b.Property("Name") + .IsRequired() + .HasMaxLength(1024) + .HasColumnType("character varying(1024)") + .HasColumnName("name"); + + b.Property("Passcode") + .HasMaxLength(256) + .HasColumnType("character varying(256)") + .HasColumnName("passcode"); + + b.Property("Slug") + .IsRequired() + .HasMaxLength(1024) + .HasColumnType("character varying(1024)") + .HasColumnName("slug"); + + b.Property("UpdatedAt") + .HasColumnType("timestamp with time zone") + .HasColumnName("updated_at"); + + b.HasKey("Id") + .HasName("pk_bundles"); + + b.HasIndex("Slug") + .IsUnique() + .HasDatabaseName("ix_bundles_slug"); + + b.ToTable("bundles", (string)null); + }); + + modelBuilder.Entity("DysonNetwork.Shared.Models.SnFileObject", b => + { + b.Property("Id") + .HasMaxLength(32) + .HasColumnType("character varying(32)") + .HasColumnName("id"); + + b.Property("CreatedAt") + .HasColumnType("timestamp with time zone") + .HasColumnName("created_at"); + + b.Property("DeletedAt") + .HasColumnType("timestamp with time zone") + .HasColumnName("deleted_at"); + + b.Property("HasCompression") + .HasColumnType("boolean") + .HasColumnName("has_compression"); + + b.Property("HasThumbnail") + .HasColumnType("boolean") + .HasColumnName("has_thumbnail"); + + b.Property("Hash") + .HasMaxLength(256) + .HasColumnType("character varying(256)") + .HasColumnName("hash"); + + b.Property>("Meta") + .HasColumnType("jsonb") + .HasColumnName("meta"); + + b.Property("MimeType") + .HasMaxLength(256) + .HasColumnType("character varying(256)") + .HasColumnName("mime_type"); + + b.Property("Size") + .HasColumnType("bigint") + .HasColumnName("size"); + + b.Property("UpdatedAt") + .HasColumnType("timestamp with time zone") + .HasColumnName("updated_at"); + + b.HasKey("Id") + .HasName("pk_file_objects"); + + b.ToTable("file_objects", (string)null); + }); + + modelBuilder.Entity("DysonNetwork.Shared.Models.SnFilePermission", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("uuid") + .HasColumnName("id"); + + b.Property("CreatedAt") + .HasColumnType("timestamp with time zone") + .HasColumnName("created_at"); + + b.Property("DeletedAt") + .HasColumnType("timestamp with time zone") + .HasColumnName("deleted_at"); + + b.Property("FileId") + .IsRequired() + .HasColumnType("text") + .HasColumnName("file_id"); + + b.Property("Permission") + .HasColumnType("integer") + .HasColumnName("permission"); + + b.Property("SubjectId") + .IsRequired() + .HasColumnType("text") + .HasColumnName("subject_id"); + + b.Property("SubjectType") + .HasColumnType("integer") + .HasColumnName("subject_type"); + + b.Property("UpdatedAt") + .HasColumnType("timestamp with time zone") + .HasColumnName("updated_at"); + + b.HasKey("Id") + .HasName("pk_file_permissions"); + + b.ToTable("file_permissions", (string)null); + }); + + modelBuilder.Entity("DysonNetwork.Shared.Models.SnFileReplica", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("uuid") + .HasColumnName("id"); + + b.Property("CreatedAt") + .HasColumnType("timestamp with time zone") + .HasColumnName("created_at"); + + b.Property("DeletedAt") + .HasColumnType("timestamp with time zone") + .HasColumnName("deleted_at"); + + b.Property("IsPrimary") + .HasColumnType("boolean") + .HasColumnName("is_primary"); + + b.Property("ObjectId") + .IsRequired() + .HasMaxLength(32) + .HasColumnType("character varying(32)") + .HasColumnName("object_id"); + + b.Property("PoolId") + .HasColumnType("uuid") + .HasColumnName("pool_id"); + + b.Property("Status") + .HasColumnType("integer") + .HasColumnName("status"); + + b.Property("StorageId") + .IsRequired() + .HasMaxLength(128) + .HasColumnType("character varying(128)") + .HasColumnName("storage_id"); + + b.Property("UpdatedAt") + .HasColumnType("timestamp with time zone") + .HasColumnName("updated_at"); + + b.HasKey("Id") + .HasName("pk_file_replicas"); + + b.HasIndex("ObjectId") + .HasDatabaseName("ix_file_replicas_object_id"); + + b.HasIndex("PoolId") + .HasDatabaseName("ix_file_replicas_pool_id"); + + b.ToTable("file_replicas", (string)null); + }); + + modelBuilder.Entity("DysonNetwork.Shared.Models.SnCloudFile", b => + { + b.HasOne("DysonNetwork.Shared.Models.SnFileBundle", "Bundle") + .WithMany("Files") + .HasForeignKey("BundleId") + .HasConstraintName("fk_files_bundles_bundle_id"); + + b.HasOne("DysonNetwork.Shared.Models.SnFileObject", "Object") + .WithMany() + .HasForeignKey("ObjectId") + .HasConstraintName("fk_files_file_objects_object_id"); + + b.Navigation("Bundle"); + + b.Navigation("Object"); + }); + + modelBuilder.Entity("DysonNetwork.Shared.Models.SnCloudFileIndex", b => + { + b.HasOne("DysonNetwork.Shared.Models.SnCloudFile", "File") + .WithMany("FileIndexes") + .HasForeignKey("FileId") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired() + .HasConstraintName("fk_file_indexes_files_file_id"); + + b.Navigation("File"); + }); + + modelBuilder.Entity("DysonNetwork.Shared.Models.SnFileReplica", b => + { + b.HasOne("DysonNetwork.Shared.Models.SnFileObject", "Object") + .WithMany("FileReplicas") + .HasForeignKey("ObjectId") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired() + .HasConstraintName("fk_file_replicas_file_objects_object_id"); + + b.HasOne("DysonNetwork.Shared.Models.FilePool", "Pool") + .WithMany() + .HasForeignKey("PoolId") + .HasConstraintName("fk_file_replicas_pools_pool_id"); + + b.Navigation("Object"); + + b.Navigation("Pool"); + }); + + modelBuilder.Entity("DysonNetwork.Shared.Models.SnCloudFile", b => + { + b.Navigation("FileIndexes"); + }); + + modelBuilder.Entity("DysonNetwork.Shared.Models.SnFileBundle", b => + { + b.Navigation("Files"); + }); + + modelBuilder.Entity("DysonNetwork.Shared.Models.SnFileObject", b => + { + b.Navigation("FileReplicas"); + }); +#pragma warning restore 612, 618 + } + } +} diff --git a/DysonNetwork.Drive/Migrations/20260113152536_RemovePoolFromCloudFile.cs b/DysonNetwork.Drive/Migrations/20260113152536_RemovePoolFromCloudFile.cs new file mode 100644 index 00000000..eb2d9de5 --- /dev/null +++ b/DysonNetwork.Drive/Migrations/20260113152536_RemovePoolFromCloudFile.cs @@ -0,0 +1,49 @@ +using System; +using Microsoft.EntityFrameworkCore.Migrations; + +#nullable disable + +namespace DysonNetwork.Drive.Migrations +{ + /// + public partial class RemovePoolFromCloudFile : Migration + { + /// + protected override void Up(MigrationBuilder migrationBuilder) + { + migrationBuilder.DropForeignKey( + name: "fk_files_pools_pool_id", + table: "files"); + + migrationBuilder.DropIndex( + name: "ix_files_pool_id", + table: "files"); + + migrationBuilder.DropColumn( + name: "pool_id", + table: "files"); + } + + /// + protected override void Down(MigrationBuilder migrationBuilder) + { + migrationBuilder.AddColumn( + name: "pool_id", + table: "files", + type: "uuid", + nullable: true); + + migrationBuilder.CreateIndex( + name: "ix_files_pool_id", + table: "files", + column: "pool_id"); + + migrationBuilder.AddForeignKey( + name: "fk_files_pools_pool_id", + table: "files", + column: "pool_id", + principalTable: "pools", + principalColumn: "id"); + } + } +} diff --git a/DysonNetwork.Drive/Migrations/AppDatabaseModelSnapshot.cs b/DysonNetwork.Drive/Migrations/AppDatabaseModelSnapshot.cs index b8aa2936..749adcf4 100644 --- a/DysonNetwork.Drive/Migrations/AppDatabaseModelSnapshot.cs +++ b/DysonNetwork.Drive/Migrations/AppDatabaseModelSnapshot.cs @@ -276,10 +276,6 @@ namespace DysonNetwork.Drive.Migrations .HasColumnType("character varying(32)") .HasColumnName("object_id"); - b.Property("PoolId") - .HasColumnType("uuid") - .HasColumnName("pool_id"); - b.PrimitiveCollection("SensitiveMarks") .HasColumnType("jsonb") .HasColumnName("sensitive_marks"); @@ -315,9 +311,6 @@ namespace DysonNetwork.Drive.Migrations b.HasIndex("ObjectId") .HasDatabaseName("ix_files_object_id"); - b.HasIndex("PoolId") - .HasDatabaseName("ix_files_pool_id"); - b.ToTable("files", (string)null); }); @@ -588,16 +581,9 @@ namespace DysonNetwork.Drive.Migrations .HasForeignKey("ObjectId") .HasConstraintName("fk_files_file_objects_object_id"); - b.HasOne("DysonNetwork.Shared.Models.FilePool", "Pool") - .WithMany() - .HasForeignKey("PoolId") - .HasConstraintName("fk_files_pools_pool_id"); - b.Navigation("Bundle"); b.Navigation("Object"); - - b.Navigation("Pool"); }); modelBuilder.Entity("DysonNetwork.Shared.Models.SnCloudFileIndex", b => diff --git a/DysonNetwork.Drive/Startup/BroadcastEventHandler.cs b/DysonNetwork.Drive/Startup/BroadcastEventHandler.cs index 69993918..2d02fbea 100644 --- a/DysonNetwork.Drive/Startup/BroadcastEventHandler.cs +++ b/DysonNetwork.Drive/Startup/BroadcastEventHandler.cs @@ -314,9 +314,20 @@ public class BroadcastEventHandler( logger.LogInformation("Uploaded file {FileId} done!", fileId); var now = SystemClock.Instance.GetCurrentInstant(); + + var newReplica = new SnFileReplica + { + Id = Guid.NewGuid(), + ObjectId = fileId, + PoolId = destPool, + StorageId = storageId, + Status = SnFileReplicaStatus.Available, + IsPrimary = false + }; + scopedDb.FileReplicas.Add(newReplica); + await scopedDb.Files.Where(f => f.Id == fileId).ExecuteUpdateAsync(setter => setter .SetProperty(f => f.UploadedAt, now) - .SetProperty(f => f.PoolId, destPool) ); await scopedDb.FileObjects.Where(fo => fo.Id == fileId).ExecuteUpdateAsync(setter => setter diff --git a/DysonNetwork.Drive/Startup/ServiceCollectionExtensions.cs b/DysonNetwork.Drive/Startup/ServiceCollectionExtensions.cs index bc8cbd46..5c786add 100644 --- a/DysonNetwork.Drive/Startup/ServiceCollectionExtensions.cs +++ b/DysonNetwork.Drive/Startup/ServiceCollectionExtensions.cs @@ -54,7 +54,6 @@ public static class ServiceCollectionExtensions public IServiceCollection AddAppBusinessServices() { - services.AddScoped(); services.AddScoped(); services.AddScoped(); services.AddScoped(); diff --git a/DysonNetwork.Drive/Storage/CloudFileUnusedRecyclingJob.cs b/DysonNetwork.Drive/Storage/CloudFileUnusedRecyclingJob.cs index 2941fe18..6ad02656 100644 --- a/DysonNetwork.Drive/Storage/CloudFileUnusedRecyclingJob.cs +++ b/DysonNetwork.Drive/Storage/CloudFileUnusedRecyclingJob.cs @@ -1,5 +1,6 @@ using Microsoft.EntityFrameworkCore; using NodaTime; +using DysonNetwork.Shared.Models; using Quartz; namespace DysonNetwork.Drive.Storage; @@ -40,8 +41,10 @@ public class CloudFileUnusedRecyclingJob( var markedCount = 0; var totalFiles = await db.Files .Where(f => f.FileIndexes.Count == 0) - .Where(f => f.PoolId.HasValue && recyclablePools.Contains(f.PoolId.Value)) + .Where(f => f.Object!.FileReplicas.Any(r => r.PoolId.HasValue && recyclablePools.Contains(r.PoolId.Value))) .Where(f => !f.IsMarkedRecycle) + .Include(f => f.Object) + .ThenInclude(o => o.FileReplicas) .CountAsync(); logger.LogInformation("Found {TotalFiles} files to check for unused status", totalFiles); @@ -56,17 +59,18 @@ public class CloudFileUnusedRecyclingJob( while (hasMoreFiles) { - // Query for the next batch of files using keyset pagination - var filesQuery = db.Files - .Where(f => f.PoolId.HasValue && recyclablePools.Contains(f.PoolId.Value)) + IQueryable baseQuery = db.Files + .Where(f => f.Object!.FileReplicas.Any(r => r.PoolId.HasValue && recyclablePools.Contains(r.PoolId.Value))) .Where(f => !f.IsMarkedRecycle) - .Where(f => f.CreatedAt <= ageThreshold); // Only process older files first + .Where(f => f.CreatedAt <= ageThreshold) + .Include(f => f.Object) + .ThenInclude(o => o.FileReplicas); if (lastProcessedId != null) - filesQuery = filesQuery.Where(f => string.Compare(f.Id, lastProcessedId) > 0); + baseQuery = baseQuery.Where(f => string.Compare(f.Id, lastProcessedId) > 0); - var fileBatch = await filesQuery - .OrderBy(f => f.Id) // Ensure consistent ordering for pagination + var fileBatch = await baseQuery + .OrderBy(f => f.Id) .Take(batchSize) .Select(f => f.Id) .ToListAsync(); @@ -80,12 +84,11 @@ public class CloudFileUnusedRecyclingJob( processedCount += fileBatch.Count; lastProcessedId = fileBatch.Last(); - // Optimized query: Find files that have no other cloud files sharing the same object - // A file is considered "unused" if no other SnCloudFile shares its ObjectId + // Optimized query: Find files that have no file object or no replicas + // A file is considered "unused" if its file object has no replicas var filesToMark = await db.Files .Where(f => fileBatch.Contains(f.Id)) - .Where(f => f.ObjectId == null || // No file object at all - !db.Files.Any(cf => cf.ObjectId == f.ObjectId && cf.Id != f.Id)) // Or no other files share this object + .Where(f => f.Object == null || f.Object.FileReplicas.Count == 0) .Select(f => f.Id) .ToListAsync(); diff --git a/DysonNetwork.Drive/Storage/FileController.cs b/DysonNetwork.Drive/Storage/FileController.cs index 4c83040c..021c35bb 100644 --- a/DysonNetwork.Drive/Storage/FileController.cs +++ b/DysonNetwork.Drive/Storage/FileController.cs @@ -253,11 +253,12 @@ public class FileController( string? overrideMimeType ) { - if (!file.PoolId.HasValue) + var primaryReplica = file.Object?.FileReplicas.FirstOrDefault(r => r.IsPrimary); + if (primaryReplica == null || primaryReplica.PoolId == null) return StatusCode(StatusCodes.Status500InternalServerError, "File is in an inconsistent state: uploaded but no pool ID."); - var pool = await fs.GetPoolAsync(file.PoolId.Value); + var pool = await fs.GetPoolAsync(primaryReplica.PoolId.Value); if (pool is null) return StatusCode(StatusCodes.Status410Gone, "The pool of the file no longer exists or not accessible."); @@ -461,11 +462,10 @@ public class FileController( var filesQuery = db.Files .Where(e => e.IsMarkedRecycle == recycled) .Where(e => e.AccountId == accountId) - .Include(e => e.Pool) .Include(e => e.Object) .AsQueryable(); - if (pool.HasValue) filesQuery = filesQuery.Where(e => e.PoolId == pool); + if (pool.HasValue) filesQuery = filesQuery.Where(e => e.Object!.FileReplicas.Any(r => r.PoolId == pool.Value)); if (!string.IsNullOrWhiteSpace(query)) { diff --git a/DysonNetwork.Drive/Storage/FileMigrationService.cs b/DysonNetwork.Drive/Storage/FileMigrationService.cs deleted file mode 100644 index 8d42a737..00000000 --- a/DysonNetwork.Drive/Storage/FileMigrationService.cs +++ /dev/null @@ -1,127 +0,0 @@ -using DysonNetwork.Shared.Models; -using Microsoft.EntityFrameworkCore; - -namespace DysonNetwork.Drive.Storage; - -public class FileMigrationService(AppDatabase db, ILogger logger) -{ - public async Task MigrateCloudFilesAsync() - { - logger.LogInformation("Starting cloud file migration."); - - var cloudFiles = await db.Files - .Where(f => - f.ObjectId == null && - f.PoolId != null - ) - .ToListAsync(); - - logger.LogDebug("Found {Count} cloud files to migrate.", cloudFiles.Count); - - foreach (var cf in cloudFiles) - { - try - { - var ext = Path.GetExtension(cf.Name); - var mimeType = ext != "" && MimeTypes.TryGetMimeType(ext, out var mime) ? mime : "application/octet-stream"; - - var fileObject = await db.FileObjects.FindAsync(cf.Id); - - if (fileObject == null) - { - fileObject = new SnFileObject - { - Id = cf.Id, - MimeType = mimeType, - HasCompression = mimeType.StartsWith("image/"), - HasThumbnail = mimeType.StartsWith("video/") - }; - - db.FileObjects.Add(fileObject); - } - - var replicaExists = await db.FileReplicas.AnyAsync(r => - r.ObjectId == fileObject.Id && - r.PoolId == cf.PoolId!.Value); - - if (!replicaExists) - { - var fileReplica = new SnFileReplica - { - Id = Guid.NewGuid(), - ObjectId = fileObject.Id, - PoolId = cf.PoolId!.Value, - StorageId = cf.StorageId ?? cf.Id, - Status = SnFileReplicaStatus.Available, - IsPrimary = true - }; - - fileObject.FileReplicas.Add(fileReplica); - db.FileReplicas.Add(fileReplica); - } - - var permissionExists = await db.FilePermissions.AnyAsync(p => p.FileId == cf.Id); - - if (!permissionExists) - { - var permission = new SnFilePermission - { - Id = Guid.NewGuid(), - FileId = cf.Id, - SubjectType = SnFilePermissionType.Anyone, - SubjectId = string.Empty, - Permission = SnFilePermissionLevel.Read - }; - - db.FilePermissions.Add(permission); - } - - cf.ObjectId = fileObject.Id; - cf.Object = fileObject; - - await db.SaveChangesAsync(); - logger.LogInformation("Migrated file {FileId} successfully.", cf.Id); - } - catch (Exception ex) - { - logger.LogError(ex, - "Failed migrating file {FileId}. ObjectId={ObjectId}, PoolId={PoolId}, StorageId={StorageId}", - cf.Id, - cf.ObjectId, - cf.PoolId, - cf.StorageId); - } - } - - logger.LogInformation("Cloud file migration completed."); - } - - public async Task MigratePermissionsAsync() - { - logger.LogInformation("Starting file permission migration."); - - var filesWithoutPermission = await db.Files - .Where(f => !db.FilePermissions.Any(p => p.FileId == f.Id)) - .ToListAsync(); - - logger.LogDebug("Found {Count} files without permissions.", filesWithoutPermission.Count); - - foreach (var file in filesWithoutPermission) - { - var permission = new SnFilePermission - { - Id = Guid.NewGuid(), - FileId = file.Id, - SubjectType = SnFilePermissionType.Anyone, - SubjectId = string.Empty, - Permission = SnFilePermissionLevel.Read - }; - - db.FilePermissions.Add(permission); - } - - await db.SaveChangesAsync(); - - logger.LogInformation("Permission migration completed. Created {Count} permissions.", filesWithoutPermission.Count); - } -} diff --git a/DysonNetwork.Drive/Storage/FileReanalysisService.cs b/DysonNetwork.Drive/Storage/FileReanalysisService.cs index 021e05c4..d352bad1 100644 --- a/DysonNetwork.Drive/Storage/FileReanalysisService.cs +++ b/DysonNetwork.Drive/Storage/FileReanalysisService.cs @@ -23,10 +23,9 @@ public class FileReanalysisService( var now = SystemClock.Instance.GetCurrentInstant(); var deadline = now.Minus(Duration.FromMinutes(30)); return await db.Files - .Where(f => f.ObjectId != null && f.PoolId != null) + .Where(f => f.ObjectId != null) .Include(f => f.Object) .ThenInclude(f => f.FileReplicas) - .Include(f => f.Pool) .Where(f => f.Object != null && (f.Object.Meta == null || f.Object.Meta.Count == 0)) .Where(f => f.Object!.FileReplicas.Count > 0) .Where(f => f.CreatedAt <= deadline) @@ -39,9 +38,9 @@ public class FileReanalysisService( { logger.LogInformation("Starting reanalysis for file {FileId}: {FileName}", file.Id, file.Name); - if (file.Object == null || file.Pool == null) + if (file.Object == null) { - logger.LogWarning("File {FileId} missing object or pool, skipping reanalysis", file.Id); + logger.LogWarning("File {FileId} missing object, skipping reanalysis", file.Id); return true; // not a failure } @@ -147,16 +146,22 @@ public class FileReanalysisService( private async Task DownloadFileAsync(SnCloudFile file, SnFileReplica replica, string tempPath) { - var dest = file.Pool!.StorageConfig; - if (dest == null) + if (replica.PoolId == null) { - throw new InvalidOperationException($"No remote storage configured for pool {file.PoolId}"); + throw new InvalidOperationException($"Replica for file {file.Id} has no pool ID"); } + var pool = await db.Pools.FindAsync(replica.PoolId.Value); + if (pool == null) + { + throw new InvalidOperationException($"No remote storage configured for pool {replica.PoolId}"); + } + var dest = pool.StorageConfig; + var client = CreateMinioClient(dest); if (client == null) { - throw new InvalidOperationException($"Failed to create Minio client for pool {file.PoolId}"); + throw new InvalidOperationException($"Failed to create Minio client for pool {replica.PoolId}"); } await using var fileStream = File.Create(tempPath); diff --git a/DysonNetwork.Drive/Storage/FileService.cs b/DysonNetwork.Drive/Storage/FileService.cs index 0380518e..5f6d7c99 100644 --- a/DysonNetwork.Drive/Storage/FileService.cs +++ b/DysonNetwork.Drive/Storage/FileService.cs @@ -38,7 +38,6 @@ public class FileService( var file = await db.Files .Where(f => f.Id == fileId) - .Include(f => f.Pool) .Include(f => f.Bundle) .Include(f => f.Object) .ThenInclude(o => o.FileReplicas) @@ -70,7 +69,7 @@ public class FileService( { var dbFiles = await db.Files .Where(f => uncachedIds.Contains(f.Id)) - .Include(f => f.Pool) + .Include(f => f.Bundle) .Include(f => f.Object) .ThenInclude(o => o.FileReplicas) .ToListAsync(); @@ -124,7 +123,7 @@ public class FileService( fileObject.Hash = await HashFileAsync(processingPath); - await SaveFileToDatabaseAsync(file, fileObject); + await SaveFileToDatabaseAsync(file, fileObject, pool.Id); await PublishFileUploadedEventAsync(file, pool, processingPath, isTempFile); @@ -245,13 +244,13 @@ public class FileService( return Task.FromResult((encryptedPath, true)); } - private async Task SaveFileToDatabaseAsync(SnCloudFile file, SnFileObject fileObject) + private async Task SaveFileToDatabaseAsync(SnCloudFile file, SnFileObject fileObject, Guid poolId) { var replica = new SnFileReplica { Id = Guid.NewGuid(), ObjectId = file.Id, - PoolId = file.PoolId, + PoolId = poolId, StorageId = file.StorageId ?? file.Id, Status = SnFileReplicaStatus.Available, IsPrimary = true @@ -540,7 +539,30 @@ public class FileService( public async Task DeleteFileDataAsync(SnCloudFile file, bool force = false) { - if (!file.PoolId.HasValue || file.ObjectId == null) return; + if (file.ObjectId == null) return; + + var replicas = await db.FileReplicas + .Where(r => r.ObjectId == file.ObjectId) + .ToListAsync(); + + if (replicas.Count == 0) + { + logger.LogWarning("No replicas found for file object {ObjectId}", file.ObjectId); + return; + } + + var primaryReplica = replicas.FirstOrDefault(r => r.IsPrimary); + if (primaryReplica == null) + { + logger.LogWarning("No primary replica found for file object {ObjectId}", file.ObjectId); + return; + } + + if (primaryReplica.PoolId == null) + { + logger.LogWarning("Primary replica has no pool ID for file object {ObjectId}", file.ObjectId); + return; + } if (!force) { @@ -553,23 +575,12 @@ public class FileService( return; } - var replicas = await db.FileReplicas - .Where(r => r.ObjectId == file.ObjectId) - .ToListAsync(); - - if (replicas.Count == 0) - { - logger.LogWarning("No replicas found for file object {ObjectId}", file.ObjectId); - return; - } - - var primaryReplica = replicas.First(r => r.IsPrimary); - var dest = await GetRemoteStorageConfig(file.PoolId.Value); - if (dest is null) throw new InvalidOperationException($"No remote storage configured for pool {file.PoolId}"); + var dest = await GetRemoteStorageConfig(primaryReplica.PoolId.Value); + if (dest is null) throw new InvalidOperationException($"No remote storage configured for pool {primaryReplica.PoolId}"); var client = CreateMinioClient(dest); if (client is null) throw new InvalidOperationException( - $"Failed to configure client for remote destination '{file.PoolId}'" + $"Failed to configure client for remote destination '{primaryReplica.PoolId}'" ); var bucket = dest.Bucket; @@ -615,7 +626,7 @@ public class FileService( public async Task DeleteFileDataBatchAsync(List files) { - files = files.Where(f => f.PoolId.HasValue && f.ObjectId != null).ToList(); + files = files.Where(f => f.ObjectId != null).ToList(); var objectIds = files.Select(f => f.ObjectId).Distinct().ToList(); var replicas = await db.FileReplicas @@ -759,8 +770,14 @@ public class FileService( public async Task DeletePoolRecycledFilesAsync(Guid poolId) { + var fileIdsWithReplicas = await db.FileReplicas + .Where(r => r.PoolId == poolId) + .Select(r => r.ObjectId) + .Distinct() + .ToListAsync(); + var files = await db.Files - .Where(f => f.PoolId == poolId && f.IsMarkedRecycle) + .Where(f => fileIdsWithReplicas.Contains(f.Id) && f.IsMarkedRecycle) .ToListAsync(); var count = files.Count; var fileIds = files.Select(f => f.Id).ToList(); diff --git a/DysonNetwork.Shared/Models/CloudFile.cs b/DysonNetwork.Shared/Models/CloudFile.cs index 437e0436..b32c728f 100644 --- a/DysonNetwork.Shared/Models/CloudFile.cs +++ b/DysonNetwork.Shared/Models/CloudFile.cs @@ -32,8 +32,6 @@ public class SnCloudFile : ModelBase, ICloudFile, IIdentifiedResource [MaxLength(32)] public string? ObjectId { get; set; } public SnFileObject? Object { get; set; } - public FilePool? Pool { get; set; } - public Guid? PoolId { get; set; } [JsonIgnore] public SnFileBundle? Bundle { get; set; } public Guid? BundleId { get; set; } [JsonIgnore] public List FileIndexes { get; set; } = [];