From 98c100c864daf820e04bcbe47939862b938671b0 Mon Sep 17 00:00:00 2001 From: LittleSheep Date: Sat, 10 Jan 2026 16:54:22 +0800 Subject: [PATCH] :alembic: Testing out the File Storing System v2 --- DysonNetwork.Drive/AppDatabase.cs | 1 - ...eferencesAndAddFileObjectOwner.Designer.cs | 762 ++++++++++++++++++ ...moveFileReferencesAndAddFileObjectOwner.cs | 173 ++++ .../Migrations/AppDatabaseModelSnapshot.cs | 246 ++++-- .../Startup/ApplicationBuilderExtensions.cs | 1 - .../Startup/ScheduledJobsConfiguration.cs | 11 +- .../Startup/ServiceCollectionExtensions.cs | 1 - .../Storage/CloudFileUnusedRecyclingJob.cs | 15 +- DysonNetwork.Drive/Storage/FileController.cs | 13 +- .../Storage/FileExpirationJob.cs | 70 -- .../Storage/FileObjectCleanupJob.cs | 101 +++ .../Storage/FileReferenceService.cs | 532 ------------ .../Storage/FileReferenceServiceGrpc.cs | 174 ---- DysonNetwork.Drive/Storage/FileService.cs | 130 ++- DysonNetwork.Shared/Models/CloudFile.cs | 35 +- DysonNetwork.Shared/Models/CloudFileObject.cs | 1 + DysonNetwork.Sphere/Post/PostService.cs | 29 - .../Publisher/PublisherController.cs | 44 - .../Publisher/PublisherService.cs | 49 -- 19 files changed, 1353 insertions(+), 1035 deletions(-) create mode 100644 DysonNetwork.Drive/Migrations/20260110084758_RemoveFileReferencesAndAddFileObjectOwner.Designer.cs create mode 100644 DysonNetwork.Drive/Migrations/20260110084758_RemoveFileReferencesAndAddFileObjectOwner.cs delete mode 100644 DysonNetwork.Drive/Storage/FileExpirationJob.cs create mode 100644 DysonNetwork.Drive/Storage/FileObjectCleanupJob.cs delete mode 100644 DysonNetwork.Drive/Storage/FileReferenceService.cs delete mode 100644 DysonNetwork.Drive/Storage/FileReferenceServiceGrpc.cs diff --git a/DysonNetwork.Drive/AppDatabase.cs b/DysonNetwork.Drive/AppDatabase.cs index e9c12fc9..2843a2c6 100644 --- a/DysonNetwork.Drive/AppDatabase.cs +++ b/DysonNetwork.Drive/AppDatabase.cs @@ -26,7 +26,6 @@ public class AppDatabase( public DbSet FileObjects { get; set; } = null!; public DbSet FileReplicas { get; set; } = null!; public DbSet FilePermissions { get; set; } = null!; - public DbSet FileReferences { get; set; } = null!; public DbSet FileIndexes { get; set; } public DbSet Tasks { get; set; } = null!; diff --git a/DysonNetwork.Drive/Migrations/20260110084758_RemoveFileReferencesAndAddFileObjectOwner.Designer.cs b/DysonNetwork.Drive/Migrations/20260110084758_RemoveFileReferencesAndAddFileObjectOwner.Designer.cs new file mode 100644 index 00000000..71241c91 --- /dev/null +++ b/DysonNetwork.Drive/Migrations/20260110084758_RemoveFileReferencesAndAddFileObjectOwner.Designer.cs @@ -0,0 +1,762 @@ +// +using System; +using System.Collections.Generic; +using DysonNetwork.Drive; +using DysonNetwork.Shared.Models; +using Microsoft.EntityFrameworkCore; +using Microsoft.EntityFrameworkCore.Infrastructure; +using Microsoft.EntityFrameworkCore.Migrations; +using Microsoft.EntityFrameworkCore.Storage.ValueConversion; +using NodaTime; +using Npgsql.EntityFrameworkCore.PostgreSQL.Metadata; + +#nullable disable + +namespace DysonNetwork.Drive.Migrations +{ + [DbContext(typeof(AppDatabase))] + [Migration("20260110084758_RemoveFileReferencesAndAddFileObjectOwner")] + partial class RemoveFileReferencesAndAddFileObjectOwner + { + /// + protected override void BuildTargetModel(ModelBuilder modelBuilder) + { +#pragma warning disable 612, 618 + modelBuilder + .HasAnnotation("ProductVersion", "10.0.1") + .HasAnnotation("Relational:MaxIdentifierLength", 63); + + NpgsqlModelBuilderExtensions.UseIdentityByDefaultColumns(modelBuilder); + + modelBuilder.Entity("DysonNetwork.Drive.Billing.QuotaRecord", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("uuid") + .HasColumnName("id"); + + b.Property("AccountId") + .HasColumnType("uuid") + .HasColumnName("account_id"); + + b.Property("CreatedAt") + .HasColumnType("timestamp with time zone") + .HasColumnName("created_at"); + + b.Property("DeletedAt") + .HasColumnType("timestamp with time zone") + .HasColumnName("deleted_at"); + + b.Property("Description") + .IsRequired() + .HasColumnType("text") + .HasColumnName("description"); + + b.Property("ExpiredAt") + .HasColumnType("timestamp with time zone") + .HasColumnName("expired_at"); + + b.Property("Name") + .IsRequired() + .HasColumnType("text") + .HasColumnName("name"); + + b.Property("Quota") + .HasColumnType("bigint") + .HasColumnName("quota"); + + b.Property("UpdatedAt") + .HasColumnType("timestamp with time zone") + .HasColumnName("updated_at"); + + b.HasKey("Id") + .HasName("pk_quota_records"); + + b.ToTable("quota_records", (string)null); + }); + + modelBuilder.Entity("DysonNetwork.Drive.Storage.Model.PersistentTask", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("uuid") + .HasColumnName("id"); + + b.Property("AccountId") + .HasColumnType("uuid") + .HasColumnName("account_id"); + + b.Property("CompletedAt") + .HasColumnType("timestamp with time zone") + .HasColumnName("completed_at"); + + b.Property("CreatedAt") + .HasColumnType("timestamp with time zone") + .HasColumnName("created_at"); + + b.Property("DeletedAt") + .HasColumnType("timestamp with time zone") + .HasColumnName("deleted_at"); + + b.Property("Description") + .HasMaxLength(1024) + .HasColumnType("character varying(1024)") + .HasColumnName("description"); + + b.Property("Discriminator") + .IsRequired() + .HasMaxLength(21) + .HasColumnType("character varying(21)") + .HasColumnName("discriminator"); + + b.Property("ErrorMessage") + .HasMaxLength(1024) + .HasColumnType("character varying(1024)") + .HasColumnName("error_message"); + + b.Property("EstimatedDurationSeconds") + .HasColumnType("bigint") + .HasColumnName("estimated_duration_seconds"); + + b.Property("ExpiredAt") + .HasColumnType("timestamp with time zone") + .HasColumnName("expired_at"); + + b.Property("LastActivity") + .HasColumnType("timestamp with time zone") + .HasColumnName("last_activity"); + + b.Property("Name") + .IsRequired() + .HasMaxLength(256) + .HasColumnType("character varying(256)") + .HasColumnName("name"); + + b.Property>("Parameters") + .IsRequired() + .HasColumnType("jsonb") + .HasColumnName("parameters"); + + b.Property("Priority") + .HasColumnType("integer") + .HasColumnName("priority"); + + b.Property("Progress") + .HasColumnType("double precision") + .HasColumnName("progress"); + + b.Property>("Results") + .IsRequired() + .HasColumnType("jsonb") + .HasColumnName("results"); + + b.Property("StartedAt") + .HasColumnType("timestamp with time zone") + .HasColumnName("started_at"); + + b.Property("Status") + .HasColumnType("integer") + .HasColumnName("status"); + + b.Property("TaskId") + .IsRequired() + .HasMaxLength(64) + .HasColumnType("character varying(64)") + .HasColumnName("task_id"); + + b.Property("Type") + .HasColumnType("integer") + .HasColumnName("type"); + + b.Property("UpdatedAt") + .HasColumnType("timestamp with time zone") + .HasColumnName("updated_at"); + + b.HasKey("Id") + .HasName("pk_tasks"); + + b.ToTable("tasks", (string)null); + + b.HasDiscriminator().HasValue("PersistentTask"); + + b.UseTphMappingStrategy(); + }); + + modelBuilder.Entity("DysonNetwork.Shared.Models.FilePool", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("uuid") + .HasColumnName("id"); + + b.Property("AccountId") + .HasColumnType("uuid") + .HasColumnName("account_id"); + + b.Property("BillingConfig") + .IsRequired() + .HasColumnType("jsonb") + .HasColumnName("billing_config"); + + b.Property("CreatedAt") + .HasColumnType("timestamp with time zone") + .HasColumnName("created_at"); + + b.Property("DeletedAt") + .HasColumnType("timestamp with time zone") + .HasColumnName("deleted_at"); + + b.Property("Description") + .IsRequired() + .HasMaxLength(8192) + .HasColumnType("character varying(8192)") + .HasColumnName("description"); + + b.Property("IsHidden") + .HasColumnType("boolean") + .HasColumnName("is_hidden"); + + b.Property("Name") + .IsRequired() + .HasMaxLength(1024) + .HasColumnType("character varying(1024)") + .HasColumnName("name"); + + b.Property("PolicyConfig") + .IsRequired() + .HasColumnType("jsonb") + .HasColumnName("policy_config"); + + b.Property("StorageConfig") + .IsRequired() + .HasColumnType("jsonb") + .HasColumnName("storage_config"); + + b.Property("UpdatedAt") + .HasColumnType("timestamp with time zone") + .HasColumnName("updated_at"); + + b.HasKey("Id") + .HasName("pk_pools"); + + b.ToTable("pools", (string)null); + }); + + modelBuilder.Entity("DysonNetwork.Shared.Models.SnCloudFile", b => + { + b.Property("Id") + .HasMaxLength(32) + .HasColumnType("character varying(32)") + .HasColumnName("id"); + + b.Property("AccountId") + .HasColumnType("uuid") + .HasColumnName("account_id"); + + b.Property("BundleId") + .HasColumnType("uuid") + .HasColumnName("bundle_id"); + + b.Property("CreatedAt") + .HasColumnType("timestamp with time zone") + .HasColumnName("created_at"); + + b.Property("DeletedAt") + .HasColumnType("timestamp with time zone") + .HasColumnName("deleted_at"); + + b.Property("Description") + .HasMaxLength(4096) + .HasColumnType("character varying(4096)") + .HasColumnName("description"); + + b.Property("ExpiredAt") + .HasColumnType("timestamp with time zone") + .HasColumnName("expired_at"); + + b.Property>("FileMeta") + .HasColumnType("jsonb") + .HasColumnName("file_meta"); + + b.Property("HasCompression") + .HasColumnType("boolean") + .HasColumnName("has_compression"); + + b.Property("HasThumbnail") + .HasColumnType("boolean") + .HasColumnName("has_thumbnail"); + + b.Property("Hash") + .HasMaxLength(256) + .HasColumnType("character varying(256)") + .HasColumnName("hash"); + + b.Property("IsEncrypted") + .HasColumnType("boolean") + .HasColumnName("is_encrypted"); + + b.Property("IsMarkedRecycle") + .HasColumnType("boolean") + .HasColumnName("is_marked_recycle"); + + b.Property("MimeType") + .HasMaxLength(256) + .HasColumnType("character varying(256)") + .HasColumnName("mime_type"); + + b.Property("Name") + .IsRequired() + .HasMaxLength(1024) + .HasColumnType("character varying(1024)") + .HasColumnName("name"); + + b.Property("ObjectId") + .HasMaxLength(32) + .HasColumnType("character varying(32)") + .HasColumnName("object_id"); + + b.Property("PoolId") + .HasColumnType("uuid") + .HasColumnName("pool_id"); + + b.PrimitiveCollection("SensitiveMarks") + .HasColumnType("jsonb") + .HasColumnName("sensitive_marks"); + + b.Property("Size") + .HasColumnType("bigint") + .HasColumnName("size"); + + b.Property("StorageId") + .HasMaxLength(32) + .HasColumnType("character varying(32)") + .HasColumnName("storage_id"); + + b.Property("StorageUrl") + .HasMaxLength(4096) + .HasColumnType("character varying(4096)") + .HasColumnName("storage_url"); + + b.Property("UpdatedAt") + .HasColumnType("timestamp with time zone") + .HasColumnName("updated_at"); + + b.Property("UploadedAt") + .HasColumnType("timestamp with time zone") + .HasColumnName("uploaded_at"); + + b.Property>("UserMeta") + .HasColumnType("jsonb") + .HasColumnName("user_meta"); + + b.HasKey("Id") + .HasName("pk_files"); + + b.HasIndex("BundleId") + .HasDatabaseName("ix_files_bundle_id"); + + b.HasIndex("ObjectId") + .HasDatabaseName("ix_files_object_id"); + + b.HasIndex("PoolId") + .HasDatabaseName("ix_files_pool_id"); + + b.ToTable("files", (string)null); + }); + + modelBuilder.Entity("DysonNetwork.Shared.Models.SnCloudFileIndex", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("uuid") + .HasColumnName("id"); + + b.Property("AccountId") + .HasColumnType("uuid") + .HasColumnName("account_id"); + + b.Property("CreatedAt") + .HasColumnType("timestamp with time zone") + .HasColumnName("created_at"); + + b.Property("DeletedAt") + .HasColumnType("timestamp with time zone") + .HasColumnName("deleted_at"); + + b.Property("FileId") + .IsRequired() + .HasMaxLength(32) + .HasColumnType("character varying(32)") + .HasColumnName("file_id"); + + b.Property("Path") + .IsRequired() + .HasMaxLength(8192) + .HasColumnType("character varying(8192)") + .HasColumnName("path"); + + b.Property("UpdatedAt") + .HasColumnType("timestamp with time zone") + .HasColumnName("updated_at"); + + b.HasKey("Id") + .HasName("pk_file_indexes"); + + b.HasIndex("FileId") + .HasDatabaseName("ix_file_indexes_file_id"); + + b.HasIndex("Path", "AccountId") + .HasDatabaseName("ix_file_indexes_path_account_id"); + + b.ToTable("file_indexes", (string)null); + }); + + modelBuilder.Entity("DysonNetwork.Shared.Models.SnFileBundle", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("uuid") + .HasColumnName("id"); + + b.Property("AccountId") + .HasColumnType("uuid") + .HasColumnName("account_id"); + + b.Property("CreatedAt") + .HasColumnType("timestamp with time zone") + .HasColumnName("created_at"); + + b.Property("DeletedAt") + .HasColumnType("timestamp with time zone") + .HasColumnName("deleted_at"); + + b.Property("Description") + .HasMaxLength(8192) + .HasColumnType("character varying(8192)") + .HasColumnName("description"); + + b.Property("ExpiredAt") + .HasColumnType("timestamp with time zone") + .HasColumnName("expired_at"); + + b.Property("Name") + .IsRequired() + .HasMaxLength(1024) + .HasColumnType("character varying(1024)") + .HasColumnName("name"); + + b.Property("Passcode") + .HasMaxLength(256) + .HasColumnType("character varying(256)") + .HasColumnName("passcode"); + + b.Property("Slug") + .IsRequired() + .HasMaxLength(1024) + .HasColumnType("character varying(1024)") + .HasColumnName("slug"); + + b.Property("UpdatedAt") + .HasColumnType("timestamp with time zone") + .HasColumnName("updated_at"); + + b.HasKey("Id") + .HasName("pk_bundles"); + + b.HasIndex("Slug") + .IsUnique() + .HasDatabaseName("ix_bundles_slug"); + + b.ToTable("bundles", (string)null); + }); + + modelBuilder.Entity("DysonNetwork.Shared.Models.SnFileObject", b => + { + b.Property("Id") + .HasMaxLength(32) + .HasColumnType("character varying(32)") + .HasColumnName("id"); + + b.Property("AccountId") + .HasColumnType("uuid") + .HasColumnName("account_id"); + + b.Property("CreatedAt") + .HasColumnType("timestamp with time zone") + .HasColumnName("created_at"); + + b.Property("DeletedAt") + .HasColumnType("timestamp with time zone") + .HasColumnName("deleted_at"); + + b.Property("HasCompression") + .HasColumnType("boolean") + .HasColumnName("has_compression"); + + b.Property("HasThumbnail") + .HasColumnType("boolean") + .HasColumnName("has_thumbnail"); + + b.Property("Hash") + .HasMaxLength(256) + .HasColumnType("character varying(256)") + .HasColumnName("hash"); + + b.Property>("Meta") + .HasColumnType("jsonb") + .HasColumnName("meta"); + + b.Property("MimeType") + .HasMaxLength(256) + .HasColumnType("character varying(256)") + .HasColumnName("mime_type"); + + b.Property("Size") + .HasColumnType("bigint") + .HasColumnName("size"); + + b.Property("UpdatedAt") + .HasColumnType("timestamp with time zone") + .HasColumnName("updated_at"); + + b.HasKey("Id") + .HasName("pk_file_objects"); + + b.ToTable("file_objects", (string)null); + }); + + modelBuilder.Entity("DysonNetwork.Shared.Models.SnFilePermission", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("uuid") + .HasColumnName("id"); + + b.Property("CreatedAt") + .HasColumnType("timestamp with time zone") + .HasColumnName("created_at"); + + b.Property("DeletedAt") + .HasColumnType("timestamp with time zone") + .HasColumnName("deleted_at"); + + b.Property("FileId") + .IsRequired() + .HasColumnType("text") + .HasColumnName("file_id"); + + b.Property("Permission") + .HasColumnType("integer") + .HasColumnName("permission"); + + b.Property("SubjectId") + .IsRequired() + .HasColumnType("text") + .HasColumnName("subject_id"); + + b.Property("SubjectType") + .HasColumnType("integer") + .HasColumnName("subject_type"); + + b.Property("UpdatedAt") + .HasColumnType("timestamp with time zone") + .HasColumnName("updated_at"); + + b.HasKey("Id") + .HasName("pk_file_permissions"); + + b.ToTable("file_permissions", (string)null); + }); + + modelBuilder.Entity("DysonNetwork.Shared.Models.SnFileReplica", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("uuid") + .HasColumnName("id"); + + b.Property("CreatedAt") + .HasColumnType("timestamp with time zone") + .HasColumnName("created_at"); + + b.Property("DeletedAt") + .HasColumnType("timestamp with time zone") + .HasColumnName("deleted_at"); + + b.Property("IsPrimary") + .HasColumnType("boolean") + .HasColumnName("is_primary"); + + b.Property("ObjectId") + .IsRequired() + .HasMaxLength(32) + .HasColumnType("character varying(32)") + .HasColumnName("object_id"); + + b.Property("PoolId") + .HasColumnType("uuid") + .HasColumnName("pool_id"); + + b.Property("Status") + .HasColumnType("integer") + .HasColumnName("status"); + + b.Property("StorageId") + .IsRequired() + .HasMaxLength(128) + .HasColumnType("character varying(128)") + .HasColumnName("storage_id"); + + b.Property("UpdatedAt") + .HasColumnType("timestamp with time zone") + .HasColumnName("updated_at"); + + b.HasKey("Id") + .HasName("pk_file_replicas"); + + b.HasIndex("ObjectId") + .HasDatabaseName("ix_file_replicas_object_id"); + + b.HasIndex("PoolId") + .HasDatabaseName("ix_file_replicas_pool_id"); + + b.ToTable("file_replicas", (string)null); + }); + + modelBuilder.Entity("DysonNetwork.Drive.Storage.Model.PersistentUploadTask", b => + { + b.HasBaseType("DysonNetwork.Drive.Storage.Model.PersistentTask"); + + b.Property("BundleId") + .HasColumnType("uuid") + .HasColumnName("bundle_id"); + + b.Property("ChunkSize") + .HasColumnType("bigint") + .HasColumnName("chunk_size"); + + b.Property("ChunksCount") + .HasColumnType("integer") + .HasColumnName("chunks_count"); + + b.Property("ChunksUploaded") + .HasColumnType("integer") + .HasColumnName("chunks_uploaded"); + + b.Property("ContentType") + .IsRequired() + .HasMaxLength(128) + .HasColumnType("character varying(128)") + .HasColumnName("content_type"); + + b.Property("EncryptPassword") + .HasMaxLength(256) + .HasColumnType("character varying(256)") + .HasColumnName("encrypt_password"); + + b.Property("FileName") + .IsRequired() + .HasMaxLength(256) + .HasColumnType("character varying(256)") + .HasColumnName("file_name"); + + b.Property("FileSize") + .HasColumnType("bigint") + .HasColumnName("file_size"); + + b.Property("Hash") + .IsRequired() + .HasColumnType("text") + .HasColumnName("hash"); + + b.Property("Path") + .HasColumnType("text") + .HasColumnName("path"); + + b.Property("PoolId") + .HasColumnType("uuid") + .HasColumnName("pool_id"); + + b.PrimitiveCollection>("UploadedChunks") + .IsRequired() + .HasColumnType("integer[]") + .HasColumnName("uploaded_chunks"); + + b.HasDiscriminator().HasValue("PersistentUploadTask"); + }); + + modelBuilder.Entity("DysonNetwork.Shared.Models.SnCloudFile", b => + { + b.HasOne("DysonNetwork.Shared.Models.SnFileBundle", "Bundle") + .WithMany("Files") + .HasForeignKey("BundleId") + .HasConstraintName("fk_files_bundles_bundle_id"); + + b.HasOne("DysonNetwork.Shared.Models.SnFileObject", "Object") + .WithMany() + .HasForeignKey("ObjectId") + .HasConstraintName("fk_files_file_objects_object_id"); + + b.HasOne("DysonNetwork.Shared.Models.FilePool", "Pool") + .WithMany() + .HasForeignKey("PoolId") + .HasConstraintName("fk_files_pools_pool_id"); + + b.Navigation("Bundle"); + + b.Navigation("Object"); + + b.Navigation("Pool"); + }); + + modelBuilder.Entity("DysonNetwork.Shared.Models.SnCloudFileIndex", b => + { + b.HasOne("DysonNetwork.Shared.Models.SnCloudFile", "File") + .WithMany("FileIndexes") + .HasForeignKey("FileId") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired() + .HasConstraintName("fk_file_indexes_files_file_id"); + + b.Navigation("File"); + }); + + modelBuilder.Entity("DysonNetwork.Shared.Models.SnFileReplica", b => + { + b.HasOne("DysonNetwork.Shared.Models.SnFileObject", "Object") + .WithMany("FileReplicas") + .HasForeignKey("ObjectId") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired() + .HasConstraintName("fk_file_replicas_file_objects_object_id"); + + b.HasOne("DysonNetwork.Shared.Models.FilePool", "Pool") + .WithMany() + .HasForeignKey("PoolId") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired() + .HasConstraintName("fk_file_replicas_pools_pool_id"); + + b.Navigation("Object"); + + b.Navigation("Pool"); + }); + + modelBuilder.Entity("DysonNetwork.Shared.Models.SnCloudFile", b => + { + b.Navigation("FileIndexes"); + }); + + modelBuilder.Entity("DysonNetwork.Shared.Models.SnFileBundle", b => + { + b.Navigation("Files"); + }); + + modelBuilder.Entity("DysonNetwork.Shared.Models.SnFileObject", b => + { + b.Navigation("FileReplicas"); + }); +#pragma warning restore 612, 618 + } + } +} diff --git a/DysonNetwork.Drive/Migrations/20260110084758_RemoveFileReferencesAndAddFileObjectOwner.cs b/DysonNetwork.Drive/Migrations/20260110084758_RemoveFileReferencesAndAddFileObjectOwner.cs new file mode 100644 index 00000000..4258f7c9 --- /dev/null +++ b/DysonNetwork.Drive/Migrations/20260110084758_RemoveFileReferencesAndAddFileObjectOwner.cs @@ -0,0 +1,173 @@ +using System; +using System.Collections.Generic; +using Microsoft.EntityFrameworkCore.Migrations; +using NodaTime; + +#nullable disable + +namespace DysonNetwork.Drive.Migrations +{ + /// + public partial class RemoveFileReferencesAndAddFileObjectOwner : Migration + { + /// + protected override void Up(MigrationBuilder migrationBuilder) + { + migrationBuilder.DropTable( + name: "file_references"); + + migrationBuilder.AddColumn( + name: "object_id", + table: "files", + type: "character varying(32)", + maxLength: 32, + nullable: true); + + migrationBuilder.CreateTable( + name: "file_objects", + columns: table => new + { + id = table.Column(type: "character varying(32)", maxLength: 32, nullable: false), + account_id = table.Column(type: "uuid", nullable: false), + size = table.Column(type: "bigint", nullable: false), + meta = table.Column>(type: "jsonb", nullable: true), + mime_type = table.Column(type: "character varying(256)", maxLength: 256, nullable: true), + hash = table.Column(type: "character varying(256)", maxLength: 256, nullable: true), + has_compression = table.Column(type: "boolean", nullable: false), + has_thumbnail = table.Column(type: "boolean", nullable: false), + created_at = table.Column(type: "timestamp with time zone", nullable: false), + updated_at = table.Column(type: "timestamp with time zone", nullable: false), + deleted_at = table.Column(type: "timestamp with time zone", nullable: true) + }, + constraints: table => + { + table.PrimaryKey("pk_file_objects", x => x.id); + }); + + migrationBuilder.CreateTable( + name: "file_permissions", + columns: table => new + { + id = table.Column(type: "uuid", nullable: false), + file_id = table.Column(type: "text", nullable: false), + subject_type = table.Column(type: "integer", nullable: false), + subject_id = table.Column(type: "text", nullable: false), + permission = table.Column(type: "integer", nullable: false), + created_at = table.Column(type: "timestamp with time zone", nullable: false), + updated_at = table.Column(type: "timestamp with time zone", nullable: false), + deleted_at = table.Column(type: "timestamp with time zone", nullable: true) + }, + constraints: table => + { + table.PrimaryKey("pk_file_permissions", x => x.id); + }); + + migrationBuilder.CreateTable( + name: "file_replicas", + columns: table => new + { + id = table.Column(type: "uuid", nullable: false), + object_id = table.Column(type: "character varying(32)", maxLength: 32, nullable: false), + pool_id = table.Column(type: "uuid", nullable: false), + storage_id = table.Column(type: "character varying(128)", maxLength: 128, nullable: false), + status = table.Column(type: "integer", nullable: false), + is_primary = table.Column(type: "boolean", nullable: false), + created_at = table.Column(type: "timestamp with time zone", nullable: false), + updated_at = table.Column(type: "timestamp with time zone", nullable: false), + deleted_at = table.Column(type: "timestamp with time zone", nullable: true) + }, + constraints: table => + { + table.PrimaryKey("pk_file_replicas", x => x.id); + table.ForeignKey( + name: "fk_file_replicas_file_objects_object_id", + column: x => x.object_id, + principalTable: "file_objects", + principalColumn: "id", + onDelete: ReferentialAction.Cascade); + table.ForeignKey( + name: "fk_file_replicas_pools_pool_id", + column: x => x.pool_id, + principalTable: "pools", + principalColumn: "id", + onDelete: ReferentialAction.Cascade); + }); + + migrationBuilder.CreateIndex( + name: "ix_files_object_id", + table: "files", + column: "object_id"); + + migrationBuilder.CreateIndex( + name: "ix_file_replicas_object_id", + table: "file_replicas", + column: "object_id"); + + migrationBuilder.CreateIndex( + name: "ix_file_replicas_pool_id", + table: "file_replicas", + column: "pool_id"); + + migrationBuilder.AddForeignKey( + name: "fk_files_file_objects_object_id", + table: "files", + column: "object_id", + principalTable: "file_objects", + principalColumn: "id"); + } + + /// + protected override void Down(MigrationBuilder migrationBuilder) + { + migrationBuilder.DropForeignKey( + name: "fk_files_file_objects_object_id", + table: "files"); + + migrationBuilder.DropTable( + name: "file_permissions"); + + migrationBuilder.DropTable( + name: "file_replicas"); + + migrationBuilder.DropTable( + name: "file_objects"); + + migrationBuilder.DropIndex( + name: "ix_files_object_id", + table: "files"); + + migrationBuilder.DropColumn( + name: "object_id", + table: "files"); + + migrationBuilder.CreateTable( + name: "file_references", + columns: table => new + { + id = table.Column(type: "uuid", nullable: false), + file_id = table.Column(type: "character varying(32)", maxLength: 32, nullable: false), + created_at = table.Column(type: "timestamp with time zone", nullable: false), + deleted_at = table.Column(type: "timestamp with time zone", nullable: true), + expired_at = table.Column(type: "timestamp with time zone", nullable: true), + resource_id = table.Column(type: "character varying(1024)", maxLength: 1024, nullable: false), + updated_at = table.Column(type: "timestamp with time zone", nullable: false), + usage = table.Column(type: "character varying(1024)", maxLength: 1024, nullable: false) + }, + constraints: table => + { + table.PrimaryKey("pk_file_references", x => x.id); + table.ForeignKey( + name: "fk_file_references_files_file_id", + column: x => x.file_id, + principalTable: "files", + principalColumn: "id", + onDelete: ReferentialAction.Cascade); + }); + + migrationBuilder.CreateIndex( + name: "ix_file_references_file_id", + table: "file_references", + column: "file_id"); + } + } +} diff --git a/DysonNetwork.Drive/Migrations/AppDatabaseModelSnapshot.cs b/DysonNetwork.Drive/Migrations/AppDatabaseModelSnapshot.cs index f29dd8a7..a3effc72 100644 --- a/DysonNetwork.Drive/Migrations/AppDatabaseModelSnapshot.cs +++ b/DysonNetwork.Drive/Migrations/AppDatabaseModelSnapshot.cs @@ -307,6 +307,11 @@ namespace DysonNetwork.Drive.Migrations .HasColumnType("character varying(1024)") .HasColumnName("name"); + b.Property("ObjectId") + .HasMaxLength(32) + .HasColumnType("character varying(32)") + .HasColumnName("object_id"); + b.Property("PoolId") .HasColumnType("uuid") .HasColumnName("pool_id"); @@ -347,6 +352,9 @@ namespace DysonNetwork.Drive.Migrations b.HasIndex("BundleId") .HasDatabaseName("ix_files_bundle_id"); + b.HasIndex("ObjectId") + .HasDatabaseName("ix_files_object_id"); + b.HasIndex("PoolId") .HasDatabaseName("ix_files_pool_id"); @@ -400,56 +408,6 @@ namespace DysonNetwork.Drive.Migrations b.ToTable("file_indexes", (string)null); }); - modelBuilder.Entity("DysonNetwork.Shared.Models.SnCloudFileReference", b => - { - b.Property("Id") - .ValueGeneratedOnAdd() - .HasColumnType("uuid") - .HasColumnName("id"); - - b.Property("CreatedAt") - .HasColumnType("timestamp with time zone") - .HasColumnName("created_at"); - - b.Property("DeletedAt") - .HasColumnType("timestamp with time zone") - .HasColumnName("deleted_at"); - - b.Property("ExpiredAt") - .HasColumnType("timestamp with time zone") - .HasColumnName("expired_at"); - - b.Property("FileId") - .IsRequired() - .HasMaxLength(32) - .HasColumnType("character varying(32)") - .HasColumnName("file_id"); - - b.Property("ResourceId") - .IsRequired() - .HasMaxLength(1024) - .HasColumnType("character varying(1024)") - .HasColumnName("resource_id"); - - b.Property("UpdatedAt") - .HasColumnType("timestamp with time zone") - .HasColumnName("updated_at"); - - b.Property("Usage") - .IsRequired() - .HasMaxLength(1024) - .HasColumnType("character varying(1024)") - .HasColumnName("usage"); - - b.HasKey("Id") - .HasName("pk_file_references"); - - b.HasIndex("FileId") - .HasDatabaseName("ix_file_references_file_id"); - - b.ToTable("file_references", (string)null); - }); - modelBuilder.Entity("DysonNetwork.Shared.Models.SnFileBundle", b => { b.Property("Id") @@ -509,6 +467,159 @@ namespace DysonNetwork.Drive.Migrations b.ToTable("bundles", (string)null); }); + modelBuilder.Entity("DysonNetwork.Shared.Models.SnFileObject", b => + { + b.Property("Id") + .HasMaxLength(32) + .HasColumnType("character varying(32)") + .HasColumnName("id"); + + b.Property("AccountId") + .HasColumnType("uuid") + .HasColumnName("account_id"); + + b.Property("CreatedAt") + .HasColumnType("timestamp with time zone") + .HasColumnName("created_at"); + + b.Property("DeletedAt") + .HasColumnType("timestamp with time zone") + .HasColumnName("deleted_at"); + + b.Property("HasCompression") + .HasColumnType("boolean") + .HasColumnName("has_compression"); + + b.Property("HasThumbnail") + .HasColumnType("boolean") + .HasColumnName("has_thumbnail"); + + b.Property("Hash") + .HasMaxLength(256) + .HasColumnType("character varying(256)") + .HasColumnName("hash"); + + b.Property>("Meta") + .HasColumnType("jsonb") + .HasColumnName("meta"); + + b.Property("MimeType") + .HasMaxLength(256) + .HasColumnType("character varying(256)") + .HasColumnName("mime_type"); + + b.Property("Size") + .HasColumnType("bigint") + .HasColumnName("size"); + + b.Property("UpdatedAt") + .HasColumnType("timestamp with time zone") + .HasColumnName("updated_at"); + + b.HasKey("Id") + .HasName("pk_file_objects"); + + b.ToTable("file_objects", (string)null); + }); + + modelBuilder.Entity("DysonNetwork.Shared.Models.SnFilePermission", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("uuid") + .HasColumnName("id"); + + b.Property("CreatedAt") + .HasColumnType("timestamp with time zone") + .HasColumnName("created_at"); + + b.Property("DeletedAt") + .HasColumnType("timestamp with time zone") + .HasColumnName("deleted_at"); + + b.Property("FileId") + .IsRequired() + .HasColumnType("text") + .HasColumnName("file_id"); + + b.Property("Permission") + .HasColumnType("integer") + .HasColumnName("permission"); + + b.Property("SubjectId") + .IsRequired() + .HasColumnType("text") + .HasColumnName("subject_id"); + + b.Property("SubjectType") + .HasColumnType("integer") + .HasColumnName("subject_type"); + + b.Property("UpdatedAt") + .HasColumnType("timestamp with time zone") + .HasColumnName("updated_at"); + + b.HasKey("Id") + .HasName("pk_file_permissions"); + + b.ToTable("file_permissions", (string)null); + }); + + modelBuilder.Entity("DysonNetwork.Shared.Models.SnFileReplica", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("uuid") + .HasColumnName("id"); + + b.Property("CreatedAt") + .HasColumnType("timestamp with time zone") + .HasColumnName("created_at"); + + b.Property("DeletedAt") + .HasColumnType("timestamp with time zone") + .HasColumnName("deleted_at"); + + b.Property("IsPrimary") + .HasColumnType("boolean") + .HasColumnName("is_primary"); + + b.Property("ObjectId") + .IsRequired() + .HasMaxLength(32) + .HasColumnType("character varying(32)") + .HasColumnName("object_id"); + + b.Property("PoolId") + .HasColumnType("uuid") + .HasColumnName("pool_id"); + + b.Property("Status") + .HasColumnType("integer") + .HasColumnName("status"); + + b.Property("StorageId") + .IsRequired() + .HasMaxLength(128) + .HasColumnType("character varying(128)") + .HasColumnName("storage_id"); + + b.Property("UpdatedAt") + .HasColumnType("timestamp with time zone") + .HasColumnName("updated_at"); + + b.HasKey("Id") + .HasName("pk_file_replicas"); + + b.HasIndex("ObjectId") + .HasDatabaseName("ix_file_replicas_object_id"); + + b.HasIndex("PoolId") + .HasDatabaseName("ix_file_replicas_pool_id"); + + b.ToTable("file_replicas", (string)null); + }); + modelBuilder.Entity("DysonNetwork.Drive.Storage.Model.PersistentUploadTask", b => { b.HasBaseType("DysonNetwork.Drive.Storage.Model.PersistentTask"); @@ -578,6 +689,11 @@ namespace DysonNetwork.Drive.Migrations .HasForeignKey("BundleId") .HasConstraintName("fk_files_bundles_bundle_id"); + b.HasOne("DysonNetwork.Shared.Models.SnFileObject", "Object") + .WithMany() + .HasForeignKey("ObjectId") + .HasConstraintName("fk_files_file_objects_object_id"); + b.HasOne("DysonNetwork.Shared.Models.FilePool", "Pool") .WithMany() .HasForeignKey("PoolId") @@ -585,6 +701,8 @@ namespace DysonNetwork.Drive.Migrations b.Navigation("Bundle"); + b.Navigation("Object"); + b.Navigation("Pool"); }); @@ -600,29 +718,41 @@ namespace DysonNetwork.Drive.Migrations b.Navigation("File"); }); - modelBuilder.Entity("DysonNetwork.Shared.Models.SnCloudFileReference", b => + modelBuilder.Entity("DysonNetwork.Shared.Models.SnFileReplica", b => { - b.HasOne("DysonNetwork.Shared.Models.SnCloudFile", "File") - .WithMany("References") - .HasForeignKey("FileId") + b.HasOne("DysonNetwork.Shared.Models.SnFileObject", "Object") + .WithMany("FileReplicas") + .HasForeignKey("ObjectId") .OnDelete(DeleteBehavior.Cascade) .IsRequired() - .HasConstraintName("fk_file_references_files_file_id"); + .HasConstraintName("fk_file_replicas_file_objects_object_id"); - b.Navigation("File"); + b.HasOne("DysonNetwork.Shared.Models.FilePool", "Pool") + .WithMany() + .HasForeignKey("PoolId") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired() + .HasConstraintName("fk_file_replicas_pools_pool_id"); + + b.Navigation("Object"); + + b.Navigation("Pool"); }); modelBuilder.Entity("DysonNetwork.Shared.Models.SnCloudFile", b => { b.Navigation("FileIndexes"); - - b.Navigation("References"); }); modelBuilder.Entity("DysonNetwork.Shared.Models.SnFileBundle", b => { b.Navigation("Files"); }); + + modelBuilder.Entity("DysonNetwork.Shared.Models.SnFileObject", b => + { + b.Navigation("FileReplicas"); + }); #pragma warning restore 612, 618 } } diff --git a/DysonNetwork.Drive/Startup/ApplicationBuilderExtensions.cs b/DysonNetwork.Drive/Startup/ApplicationBuilderExtensions.cs index d91bed7f..ffaf8d0c 100644 --- a/DysonNetwork.Drive/Startup/ApplicationBuilderExtensions.cs +++ b/DysonNetwork.Drive/Startup/ApplicationBuilderExtensions.cs @@ -16,7 +16,6 @@ public static class ApplicationBuilderExtensions { // Map your gRPC services here app.MapGrpcService(); - app.MapGrpcService(); app.MapGrpcReflectionService(); return app; diff --git a/DysonNetwork.Drive/Startup/ScheduledJobsConfiguration.cs b/DysonNetwork.Drive/Startup/ScheduledJobsConfiguration.cs index 58745a5c..ec35cc8f 100644 --- a/DysonNetwork.Drive/Startup/ScheduledJobsConfiguration.cs +++ b/DysonNetwork.Drive/Startup/ScheduledJobsConfiguration.cs @@ -15,20 +15,27 @@ public static class ScheduledJobsConfiguration .ForJob(appDatabaseRecyclingJob) .WithIdentity("AppDatabaseRecyclingTrigger") .WithCronSchedule("0 0 0 * * ?")); - + var cloudFileUnusedRecyclingJob = new JobKey("CloudFileUnusedRecycling"); q.AddJob(opts => opts.WithIdentity(cloudFileUnusedRecyclingJob)); q.AddTrigger(opts => opts .ForJob(cloudFileUnusedRecyclingJob) .WithIdentity("CloudFileUnusedRecyclingTrigger") .WithCronSchedule("0 0 0 * * ?")); - + var persistentTaskCleanupJob = new JobKey("PersistentTaskCleanup"); q.AddJob(opts => opts.WithIdentity(persistentTaskCleanupJob)); q.AddTrigger(opts => opts .ForJob(persistentTaskCleanupJob) .WithIdentity("PersistentTaskCleanupTrigger") .WithCronSchedule("0 0 2 * * ?")); // Run daily at 2 AM + + var fileObjectCleanupJob = new JobKey("FileObjectCleanup"); + q.AddJob(opts => opts.WithIdentity(fileObjectCleanupJob)); + q.AddTrigger(opts => opts + .ForJob(fileObjectCleanupJob) + .WithIdentity("FileObjectCleanupTrigger") + .WithCronSchedule("0 0 1 * * ?")); // Run daily at 1 AM }); services.AddQuartzHostedService(q => q.WaitForJobsToComplete = true); diff --git a/DysonNetwork.Drive/Startup/ServiceCollectionExtensions.cs b/DysonNetwork.Drive/Startup/ServiceCollectionExtensions.cs index 3a1753c5..72c21636 100644 --- a/DysonNetwork.Drive/Startup/ServiceCollectionExtensions.cs +++ b/DysonNetwork.Drive/Startup/ServiceCollectionExtensions.cs @@ -56,7 +56,6 @@ public static class ServiceCollectionExtensions { services.AddScoped(); services.AddScoped(); - services.AddScoped(); services.AddScoped(); services.AddScoped(); services.AddScoped(); diff --git a/DysonNetwork.Drive/Storage/CloudFileUnusedRecyclingJob.cs b/DysonNetwork.Drive/Storage/CloudFileUnusedRecyclingJob.cs index e66dd503..2941fe18 100644 --- a/DysonNetwork.Drive/Storage/CloudFileUnusedRecyclingJob.cs +++ b/DysonNetwork.Drive/Storage/CloudFileUnusedRecyclingJob.cs @@ -26,7 +26,7 @@ public class CloudFileUnusedRecyclingJob( File.Delete(file); } } - + logger.LogInformation("Marking unused cloud files..."); var recyclablePools = await db.Pools @@ -47,7 +47,7 @@ public class CloudFileUnusedRecyclingJob( logger.LogInformation("Found {TotalFiles} files to check for unused status", totalFiles); // Define a timestamp to limit the age of files we're processing in this run - // This spreads the processing across multiple job runs for very large databases + // This spreads processing across multiple job runs for very large databases var ageThreshold = now - Duration.FromDays(30); // Process files up to 90 days old in this run // Instead of loading all files at once, use pagination @@ -80,13 +80,12 @@ public class CloudFileUnusedRecyclingJob( processedCount += fileBatch.Count; lastProcessedId = fileBatch.Last(); - // Optimized query: Find files that have no references OR all references are expired - // This replaces the memory-intensive approach of loading all references + // Optimized query: Find files that have no other cloud files sharing the same object + // A file is considered "unused" if no other SnCloudFile shares its ObjectId var filesToMark = await db.Files .Where(f => fileBatch.Contains(f.Id)) - .Where(f => !db.FileReferences.Any(r => r.FileId == f.Id) || // No references at all - !db.FileReferences.Any(r => r.FileId == f.Id && // OR has references but all are expired - (r.ExpiredAt == null || r.ExpiredAt > now))) + .Where(f => f.ObjectId == null || // No file object at all + !db.Files.Any(cf => cf.ObjectId == f.ObjectId && cf.Id != f.Id)) // Or no other files share this object .Select(f => f.Id) .ToListAsync(); @@ -112,7 +111,7 @@ public class CloudFileUnusedRecyclingJob( ); } } - + var expiredCount = await db.Files .Where(f => f.ExpiredAt.HasValue && f.ExpiredAt.Value <= now) .ExecuteUpdateAsync(s => s.SetProperty(f => f.IsMarkedRecycle, true)); diff --git a/DysonNetwork.Drive/Storage/FileController.cs b/DysonNetwork.Drive/Storage/FileController.cs index 747fd4b6..be665559 100644 --- a/DysonNetwork.Drive/Storage/FileController.cs +++ b/DysonNetwork.Drive/Storage/FileController.cs @@ -14,8 +14,7 @@ public class FileController( AppDatabase db, FileService fs, IConfiguration configuration, - IWebHostEnvironment env, - FileReferenceService fileReferenceService + IWebHostEnvironment env ) : ControllerBase { [HttpGet("{id}")] @@ -232,17 +231,19 @@ public class FileController( } [HttpGet("{id}/references")] - public async Task>> GetFileReferences(string id) + public async Task>> GetFileReferences(string id) { var file = await fs.GetFileAsync(id); if (file is null) return NotFound("File not found."); - // Check if user has access to the file + // Check if user has access to var accessResult = await ValidateFileAccess(file, null); if (accessResult is not null) return accessResult; - // Get references using the injected FileReferenceService - var references = await fileReferenceService.GetReferencesAsync(id); + // Get other cloud files sharing the same object + var references = await db.Files + .Where(f => f.ObjectId == file.ObjectId && f.Id != file.Id) + .ToListAsync(); return Ok(references); } diff --git a/DysonNetwork.Drive/Storage/FileExpirationJob.cs b/DysonNetwork.Drive/Storage/FileExpirationJob.cs deleted file mode 100644 index 53aa2e28..00000000 --- a/DysonNetwork.Drive/Storage/FileExpirationJob.cs +++ /dev/null @@ -1,70 +0,0 @@ -using Microsoft.EntityFrameworkCore; -using NodaTime; -using Quartz; - -namespace DysonNetwork.Drive.Storage; - -/// -/// Job responsible for cleaning up expired file references -/// -public class FileExpirationJob(AppDatabase db, FileService fileService, ILogger logger) : IJob -{ - public async Task Execute(IJobExecutionContext context) - { - var now = SystemClock.Instance.GetCurrentInstant(); - logger.LogInformation("Running file reference expiration job at {now}", now); - - // Delete expired references in bulk and get affected file IDs - var affectedFileIds = await db.FileReferences - .Where(r => r.ExpiredAt < now && r.ExpiredAt != null) - .Select(r => r.FileId) - .Distinct() - .ToListAsync(); - - if (!affectedFileIds.Any()) - { - logger.LogInformation("No expired file references found"); - return; - } - - logger.LogInformation("Found expired references for {count} files", affectedFileIds.Count); - - // Delete expired references in bulk - var deletedReferencesCount = await db.FileReferences - .Where(r => r.ExpiredAt < now && r.ExpiredAt != null) - .ExecuteDeleteAsync(); - - logger.LogInformation("Deleted {count} expired file references", deletedReferencesCount); - - // Find files that now have no remaining references (bulk operation) - var filesToDelete = await db.Files - .Where(f => affectedFileIds.Contains(f.Id)) - .Where(f => !db.FileReferences.Any(r => r.FileId == f.Id)) - .Select(f => f.Id) - .ToListAsync(); - - if (filesToDelete.Any()) - { - logger.LogInformation("Deleting {count} files that have no remaining references", filesToDelete.Count); - - // Get files for deletion - var files = await db.Files - .Where(f => filesToDelete.Contains(f.Id)) - .ToListAsync(); - - // Delete files and their data in parallel - var deleteTasks = files.Select(f => fileService.DeleteFileAsync(f)); - await Task.WhenAll(deleteTasks); - } - - // Purge cache for files that still have references - var filesWithRemainingRefs = affectedFileIds.Except(filesToDelete).ToList(); - if (filesWithRemainingRefs.Any()) - { - var cachePurgeTasks = filesWithRemainingRefs.Select(fileService._PurgeCacheAsync); - await Task.WhenAll(cachePurgeTasks); - } - - logger.LogInformation("Completed file reference expiration job"); - } -} diff --git a/DysonNetwork.Drive/Storage/FileObjectCleanupJob.cs b/DysonNetwork.Drive/Storage/FileObjectCleanupJob.cs new file mode 100644 index 00000000..91c778ff --- /dev/null +++ b/DysonNetwork.Drive/Storage/FileObjectCleanupJob.cs @@ -0,0 +1,101 @@ +using Microsoft.EntityFrameworkCore; +using Minio.DataModel.Args; +using NodaTime; +using Quartz; + +namespace DysonNetwork.Drive.Storage; + +/// +/// Job responsible for cleaning up orphaned file objects +/// When no SnCloudFile references a SnFileObject, the file object is considered orphaned +/// and should be deleted from disk and database +/// +public class FileObjectCleanupJob(AppDatabase db, FileService fileService, ILogger logger) : IJob +{ + public async Task Execute(IJobExecutionContext context) + { + var now = SystemClock.Instance.GetCurrentInstant(); + logger.LogInformation("Running file object cleanup job at {now}", now); + + // Find orphaned file objects (objects with no cloud files referencing them) + var referencedObjectIds = await db.Files + .Where(f => f.ObjectId != null) + .Select(f => f.ObjectId) + .Distinct() + .ToListAsync(); + + var orphanedObjects = await db.FileObjects + .Where(fo => !referencedObjectIds.Contains(fo.Id)) + .ToListAsync(); + + if (!orphanedObjects.Any()) + { + logger.LogInformation("No orphaned file objects found"); + return; + } + + logger.LogInformation("Found {count} orphaned file objects", orphanedObjects.Count); + + // Delete orphaned objects and their data + foreach (var fileObject in orphanedObjects) + { + try + { + var replicas = await db.FileReplicas + .Where(r => r.ObjectId == fileObject.Id) + .ToListAsync(); + + foreach (var replica in replicas) + { + var dest = await fileService.GetRemoteStorageConfig(replica.PoolId); + if (dest != null) + { + var client = fileService.CreateMinioClient(dest); + if (client != null) + { + try + { + await client.RemoveObjectAsync( + new RemoveObjectArgs() + .WithBucket(dest.Bucket) + .WithObject(replica.StorageId) + ); + if (fileObject.HasCompression) + { + await client.RemoveObjectAsync( + new RemoveObjectArgs() + .WithBucket(dest.Bucket) + .WithObject(replica.StorageId + ".compressed") + ); + } + if (fileObject.HasThumbnail) + { + await client.RemoveObjectAsync( + new RemoveObjectArgs() + .WithBucket(dest.Bucket) + .WithObject(replica.StorageId + ".thumbnail") + ); + } + } + catch (Exception ex) + { + logger.LogError(ex, "Failed to delete orphaned file object {ObjectId} from remote storage", fileObject.Id); + } + } + } + } + + db.FileReplicas.RemoveRange(replicas); + db.FileObjects.Remove(fileObject); + await db.SaveChangesAsync(); + logger.LogInformation("Deleted orphaned file object {ObjectId}", fileObject.Id); + } + catch (Exception ex) + { + logger.LogError(ex, "Failed to clean up orphaned file object {ObjectId}", fileObject.Id); + } + } + + logger.LogInformation("Completed file object cleanup job"); + } +} diff --git a/DysonNetwork.Drive/Storage/FileReferenceService.cs b/DysonNetwork.Drive/Storage/FileReferenceService.cs deleted file mode 100644 index 09599760..00000000 --- a/DysonNetwork.Drive/Storage/FileReferenceService.cs +++ /dev/null @@ -1,532 +0,0 @@ -using DysonNetwork.Shared.Cache; -using DysonNetwork.Shared.Data; -using DysonNetwork.Shared.Models; -using Microsoft.EntityFrameworkCore; -using NodaTime; - -namespace DysonNetwork.Drive.Storage; - -public class FileReferenceService(AppDatabase db, FileService fileService, ICacheService cache) -{ - private const string CacheKeyPrefix = "file:ref:"; - private static readonly TimeSpan CacheDuration = TimeSpan.FromMinutes(15); - - /// - /// Creates a new reference to a file for a specific resource - /// - /// The ID of the file to reference - /// The usage context (e.g., "avatar", "post-attachment") - /// The ID of the resource using the file - /// Optional expiration time for the file - /// Optional duration after which the file expires (alternative to expiredAt) - /// The created file reference - public async Task CreateReferenceAsync( - string fileId, - string usage, - string resourceId, - Instant? expiredAt = null, - Duration? duration = null - ) - { - // Calculate expiration time if needed - var finalExpiration = expiredAt; - if (duration.HasValue) - finalExpiration = SystemClock.Instance.GetCurrentInstant() + duration.Value; - - var reference = new SnCloudFileReference - { - FileId = fileId, - Usage = usage, - ResourceId = resourceId, - ExpiredAt = finalExpiration - }; - - db.FileReferences.Add(reference); - - await db.SaveChangesAsync(); - await fileService._PurgeCacheAsync(fileId); - - return reference; - } - - public async Task> CreateReferencesAsync( - List fileId, - string usage, - string resourceId, - Instant? expiredAt = null, - Duration? duration = null - ) - { - var now = SystemClock.Instance.GetCurrentInstant(); - var finalExpiredAt = expiredAt; - if (finalExpiredAt == null && duration.HasValue) - { - finalExpiredAt = now + duration.Value; - } - - var data = fileId.Select(id => new SnCloudFileReference - { - FileId = id, - Usage = usage, - ResourceId = resourceId, - ExpiredAt = finalExpiredAt, - CreatedAt = now, - UpdatedAt = now - }) - .ToList(); - - db.FileReferences.AddRange(data); - await db.SaveChangesAsync(); - return data; - } - - /// - /// Gets all references to a file - /// - /// The ID of the file - /// A list of all references to the file - public async Task> GetReferencesAsync(string fileId) - { - var cacheKey = $"{CacheKeyPrefix}list:{fileId}"; - - var cachedReferences = await cache.GetAsync>(cacheKey); - if (cachedReferences is not null) - return cachedReferences; - - var references = await db.FileReferences - .Where(r => r.FileId == fileId) - .ToListAsync(); - - await cache.SetAsync(cacheKey, references, CacheDuration); - - return references; - } - - public async Task>> GetReferencesAsync(IEnumerable fileIds) - { - var fileIdList = fileIds.ToList(); - var result = new Dictionary>(); - - // Check cache for each file ID - var uncachedFileIds = new List(); - foreach (var fileId in fileIdList) - { - var cacheKey = $"{CacheKeyPrefix}list:{fileId}"; - var cachedReferences = await cache.GetAsync>(cacheKey); - if (cachedReferences is not null) - { - result[fileId] = cachedReferences; - } - else - { - uncachedFileIds.Add(fileId); - } - } - - // Fetch uncached references from database - if (uncachedFileIds.Any()) - { - var dbReferences = await db.FileReferences - .Where(r => uncachedFileIds.Contains(r.FileId)) - .GroupBy(r => r.FileId) - .ToDictionaryAsync(r => r.Key, r => r.ToList()); - - // Cache the results - foreach (var kvp in dbReferences) - { - var cacheKey = $"{CacheKeyPrefix}list:{kvp.Key}"; - await cache.SetAsync(cacheKey, kvp.Value, CacheDuration); - result[kvp.Key] = kvp.Value; - } - } - - return result; - } - - /// - /// Gets the number of references to a file - /// - /// The ID of the file - /// The number of references to the file - public async Task GetReferenceCountAsync(string fileId) - { - var cacheKey = $"{CacheKeyPrefix}count:{fileId}"; - - var cachedCount = await cache.GetAsync(cacheKey); - if (cachedCount.HasValue) - return cachedCount.Value; - - var count = await db.FileReferences - .Where(r => r.FileId == fileId) - .CountAsync(); - - await cache.SetAsync(cacheKey, count, CacheDuration); - - return count; - } - - /// - /// Gets all references for a specific resource - /// - /// The ID of the resource - /// A list of file references associated with the resource - public async Task> GetResourceReferencesAsync(string resourceId) - { - var cacheKey = $"{CacheKeyPrefix}resource:{resourceId}"; - - var cachedReferences = await cache.GetAsync>(cacheKey); - if (cachedReferences is not null) - return cachedReferences; - - var references = await db.FileReferences - .Where(r => r.ResourceId == resourceId) - .ToListAsync(); - - await cache.SetAsync(cacheKey, references, CacheDuration); - - return references; - } - - /// - /// Gets all file references for a specific usage context - /// - /// The usage context - /// A list of file references with the specified usage - public async Task> GetUsageReferencesAsync(string usage) - { - var cacheKey = $"{CacheKeyPrefix}usage:{usage}"; - - var cachedReferences = await cache.GetAsync>(cacheKey); - if (cachedReferences is not null) - return cachedReferences; - - var references = await db.FileReferences - .Where(r => r.Usage == usage) - .ToListAsync(); - - await cache.SetAsync(cacheKey, references, CacheDuration); - - return references; - } - - /// - /// Deletes references for a specific resource - /// - /// The ID of the resource - /// The number of deleted references - public async Task DeleteResourceReferencesAsync(string resourceId) - { - var references = await db.FileReferences - .Where(r => r.ResourceId == resourceId) - .ToListAsync(); - - var fileIds = references.Select(r => r.FileId).Distinct().ToList(); - - db.FileReferences.RemoveRange(references); - var deletedCount = await db.SaveChangesAsync(); - - // Purge caches - var tasks = fileIds.Select(fileService._PurgeCacheAsync).ToList(); - tasks.Add(PurgeCacheForResourceAsync(resourceId)); - await Task.WhenAll(tasks); - - return deletedCount; - } - - /// - /// Deletes references for a specific resource and usage - /// - /// The ID of the resource - /// The usage context - /// The number of deleted references - public async Task DeleteResourceReferencesAsync(string resourceId, string usage) - { - var references = await db.FileReferences - .Where(r => r.ResourceId == resourceId && r.Usage == usage) - .ToListAsync(); - - if (references.Count == 0) - return 0; - - var fileIds = references.Select(r => r.FileId).Distinct().ToList(); - - db.FileReferences.RemoveRange(references); - var deletedCount = await db.SaveChangesAsync(); - - // Purge caches - var tasks = fileIds.Select(fileService._PurgeCacheAsync).ToList(); - tasks.Add(PurgeCacheForResourceAsync(resourceId)); - await Task.WhenAll(tasks); - - return deletedCount; - } - - public async Task DeleteResourceReferencesBatchAsync(IEnumerable resourceIds, string? usage = null) - { - var resourceIdList = resourceIds.ToList(); - var references = await db.FileReferences - .Where(r => resourceIdList.Contains(r.ResourceId)) - .If(usage != null, q => q.Where(q => q.Usage == usage)) - .ToListAsync(); - - if (references.Count == 0) - return 0; - - var fileIds = references.Select(r => r.FileId).Distinct().ToList(); - - db.FileReferences.RemoveRange(references); - var deletedCount = await db.SaveChangesAsync(); - - // Purge caches for files and resources - var tasks = fileIds.Select(fileService._PurgeCacheAsync).ToList(); - tasks.AddRange(resourceIdList.Select(PurgeCacheForResourceAsync)); - await Task.WhenAll(tasks); - - return deletedCount; - } - - /// - /// Deletes a specific file reference - /// - /// The ID of the reference to delete - /// True if the reference was deleted, false otherwise - public async Task DeleteReferenceAsync(Guid referenceId) - { - var reference = await db.FileReferences - .FirstOrDefaultAsync(r => r.Id == referenceId); - - if (reference == null) - return false; - - db.FileReferences.Remove(reference); - await db.SaveChangesAsync(); - - // Purge caches - await fileService._PurgeCacheAsync(reference.FileId); - await PurgeCacheForResourceAsync(reference.ResourceId); - await PurgeCacheForFileAsync(reference.FileId); - - return true; - } - - /// - /// Updates the files referenced by a resource - /// - /// The ID of the resource - /// The new list of file IDs - /// The usage context - /// Optional expiration time for newly added files - /// Optional duration after which newly added files expire - /// A list of the updated file references - public async Task> UpdateResourceFilesAsync( - string resourceId, - IEnumerable? newFileIds, - string usage, - Instant? expiredAt = null, - Duration? duration = null) - { - if (newFileIds == null) - return new List(); - - var existingReferences = await db.FileReferences - .Where(r => r.ResourceId == resourceId && r.Usage == usage) - .ToListAsync(); - - var existingFileIds = existingReferences.Select(r => r.FileId).ToHashSet(); - var newFileIdsList = newFileIds.ToList(); - var newFileIdsSet = newFileIdsList.ToHashSet(); - - // Files to remove - var toRemove = existingReferences - .Where(r => !newFileIdsSet.Contains(r.FileId)) - .ToList(); - - // Files to add - var toAdd = newFileIdsList - .Where(id => !existingFileIds.Contains(id)) - .Select(id => new SnCloudFileReference - { - FileId = id, - Usage = usage, - ResourceId = resourceId - }) - .ToList(); - - // Apply changes - if (toRemove.Any()) - db.FileReferences.RemoveRange(toRemove); - - if (toAdd.Any()) - db.FileReferences.AddRange(toAdd); - - await db.SaveChangesAsync(); - - // Update expiration for newly added references if specified - if ((expiredAt.HasValue || duration.HasValue) && toAdd.Any()) - { - var finalExpiration = expiredAt; - if (duration.HasValue) - { - finalExpiration = SystemClock.Instance.GetCurrentInstant() + duration.Value; - } - - // Update newly added references with the expiration time - var referenceIds = await db.FileReferences - .Where(r => toAdd.Select(a => a.FileId).Contains(r.FileId) && - r.ResourceId == resourceId && - r.Usage == usage) - .Select(r => r.Id) - .ToListAsync(); - - await db.FileReferences - .Where(r => referenceIds.Contains(r.Id)) - .ExecuteUpdateAsync(setter => setter.SetProperty( - r => r.ExpiredAt, - _ => finalExpiration - )); - } - - // Purge caches - var allFileIds = existingFileIds.Union(newFileIdsSet).ToList(); - var tasks = allFileIds.Select(fileService._PurgeCacheAsync).ToList(); - tasks.Add(PurgeCacheForResourceAsync(resourceId)); - await Task.WhenAll(tasks); - - // Return updated references - return await db.FileReferences - .Where(r => r.ResourceId == resourceId && r.Usage == usage) - .ToListAsync(); - } - - /// - /// Gets all files referenced by a resource - /// - /// The ID of the resource - /// Optional filter by usage context - /// A list of files referenced by the resource - public async Task> GetResourceFilesAsync(string resourceId, string? usage = null) - { - var query = db.FileReferences.Where(r => r.ResourceId == resourceId); - - if (usage != null) - query = query.Where(r => r.Usage == usage); - - var references = await query.ToListAsync(); - var fileIds = references.Select(r => r.FileId).ToList(); - - return await db.Files - .Where(f => fileIds.Contains(f.Id)) - .ToListAsync(); - } - - /// - /// Purges all caches related to a resource - /// - private async Task PurgeCacheForResourceAsync(string resourceId) - { - var cacheKey = $"{CacheKeyPrefix}resource:{resourceId}"; - await cache.RemoveAsync(cacheKey); - } - - /// - /// Purges all caches related to a file - /// - private async Task PurgeCacheForFileAsync(string fileId) - { - var cacheKeys = new[] - { - $"{CacheKeyPrefix}list:{fileId}", - $"{CacheKeyPrefix}count:{fileId}" - }; - - var tasks = cacheKeys.Select(cache.RemoveAsync); - await Task.WhenAll(tasks); - } - - /// - /// Updates the expiration time for a file reference - /// - /// The ID of the reference - /// The new expiration time, or null to remove expiration - /// True if the reference was found and updated, false otherwise - public async Task SetReferenceExpirationAsync(Guid referenceId, Instant? expiredAt) - { - var reference = await db.FileReferences - .FirstOrDefaultAsync(r => r.Id == referenceId); - - if (reference == null) - return false; - - reference.ExpiredAt = expiredAt; - await db.SaveChangesAsync(); - - await PurgeCacheForFileAsync(reference.FileId); - await PurgeCacheForResourceAsync(reference.ResourceId); - - return true; - } - - /// - /// Updates the expiration time for all references to a file - /// - /// The ID of the file - /// The new expiration time, or null to remove expiration - /// The number of references updated - public async Task SetFileReferencesExpirationAsync(string fileId, Instant? expiredAt) - { - var rowsAffected = await db.FileReferences - .Where(r => r.FileId == fileId) - .ExecuteUpdateAsync(setter => setter.SetProperty( - r => r.ExpiredAt, - _ => expiredAt - )); - - if (rowsAffected > 0) - { - await fileService._PurgeCacheAsync(fileId); - await PurgeCacheForFileAsync(fileId); - } - - return rowsAffected; - } - - /// - /// Get all file references for a specific resource and usage type - /// - /// The resource ID - /// The usage type - /// List of file references - public async Task> GetResourceReferencesAsync(string resourceId, string usageType) - { - return await db.FileReferences - .Where(r => r.ResourceId == resourceId && r.Usage == usageType) - .ToListAsync(); - } - - /// - /// Check if a file has any references - /// - /// The file ID to check - /// True if the file has references, false otherwise - public async Task HasFileReferencesAsync(string fileId) - { - return await db.FileReferences.AnyAsync(r => r.FileId == fileId); - } - - /// - /// Updates the expiration time for a file reference using a duration from now - /// - /// The ID of the reference - /// The duration after which the reference expires, or null to remove expiration - /// True if the reference was found and updated, false otherwise - public async Task SetReferenceExpirationDurationAsync(Guid referenceId, Duration? duration) - { - Instant? expiredAt = null; - if (duration.HasValue) - { - expiredAt = SystemClock.Instance.GetCurrentInstant() + duration.Value; - } - - return await SetReferenceExpirationAsync(referenceId, expiredAt); - } -} diff --git a/DysonNetwork.Drive/Storage/FileReferenceServiceGrpc.cs b/DysonNetwork.Drive/Storage/FileReferenceServiceGrpc.cs deleted file mode 100644 index b0e62d14..00000000 --- a/DysonNetwork.Drive/Storage/FileReferenceServiceGrpc.cs +++ /dev/null @@ -1,174 +0,0 @@ -using DysonNetwork.Shared.Proto; -using Grpc.Core; -using NodaTime; -using Duration = NodaTime.Duration; - -namespace DysonNetwork.Drive.Storage; - -public class FileReferenceServiceGrpc(FileReferenceService fileReferenceService) - : Shared.Proto.FileReferenceService.FileReferenceServiceBase -{ - public override async Task CreateReference(CreateReferenceRequest request, - ServerCallContext context) - { - Instant? expiredAt = null; - if (request.ExpiredAt != null) - expiredAt = Instant.FromUnixTimeSeconds(request.ExpiredAt.Seconds); - else if (request.Duration != null) - expiredAt = SystemClock.Instance.GetCurrentInstant() + - Duration.FromTimeSpan(request.Duration.ToTimeSpan()); - - var reference = await fileReferenceService.CreateReferenceAsync( - request.FileId, - request.Usage, - request.ResourceId, - expiredAt - ); - return reference.ToProtoValue(); - } - - public override async Task CreateReferenceBatch(CreateReferenceBatchRequest request, - ServerCallContext context) - { - Instant? expiredAt = null; - if (request.ExpiredAt != null) - expiredAt = Instant.FromUnixTimeSeconds(request.ExpiredAt.Seconds); - else if (request.Duration != null) - expiredAt = SystemClock.Instance.GetCurrentInstant() + - Duration.FromTimeSpan(request.Duration.ToTimeSpan()); - - var references = await fileReferenceService.CreateReferencesAsync( - request.FilesId.ToList(), - request.Usage, - request.ResourceId, - expiredAt - ); - var response = new CreateReferenceBatchResponse(); - response.References.AddRange(references.Select(r => r.ToProtoValue())); - return response; - } - - public override async Task GetReferences(GetReferencesRequest request, - ServerCallContext context) - { - var references = await fileReferenceService.GetReferencesAsync(request.FileId); - var response = new GetReferencesResponse(); - response.References.AddRange(references.Select(r => r.ToProtoValue())); - return response; - } - - public override async Task GetReferenceCount(GetReferenceCountRequest request, - ServerCallContext context) - { - var count = await fileReferenceService.GetReferenceCountAsync(request.FileId); - return new GetReferenceCountResponse { Count = count }; - } - - public override async Task GetResourceReferences(GetResourceReferencesRequest request, - ServerCallContext context) - { - var references = await fileReferenceService.GetResourceReferencesAsync(request.ResourceId, request.Usage); - var response = new GetReferencesResponse(); - response.References.AddRange(references.Select(r => r.ToProtoValue())); - return response; - } - - public override async Task GetResourceFiles(GetResourceFilesRequest request, - ServerCallContext context) - { - var files = await fileReferenceService.GetResourceFilesAsync(request.ResourceId, request.Usage); - var response = new GetResourceFilesResponse(); - response.Files.AddRange(files.Select(f => f.ToProtoValue())); - return response; - } - - public override async Task DeleteResourceReferences( - DeleteResourceReferencesRequest request, ServerCallContext context) - { - int deletedCount; - if (request.Usage is null) - deletedCount = await fileReferenceService.DeleteResourceReferencesAsync(request.ResourceId); - else - deletedCount = - await fileReferenceService.DeleteResourceReferencesAsync(request.ResourceId, request.Usage!); - return new DeleteResourceReferencesResponse { DeletedCount = deletedCount }; - } - - public override async Task DeleteResourceReferencesBatch(DeleteResourceReferencesBatchRequest request, ServerCallContext context) - { - var resourceIds = request.ResourceIds.ToList(); - int deletedCount; - if (request.Usage is null) - deletedCount = await fileReferenceService.DeleteResourceReferencesBatchAsync(resourceIds); - else - deletedCount = - await fileReferenceService.DeleteResourceReferencesBatchAsync(resourceIds, request.Usage!); - return new DeleteResourceReferencesResponse { DeletedCount = deletedCount }; - } - - public override async Task DeleteReference(DeleteReferenceRequest request, - ServerCallContext context) - { - var success = await fileReferenceService.DeleteReferenceAsync(Guid.Parse(request.ReferenceId)); - return new DeleteReferenceResponse { Success = success }; - } - - public override async Task UpdateResourceFiles(UpdateResourceFilesRequest request, - ServerCallContext context) - { - Instant? expiredAt = null; - if (request.ExpiredAt != null) - { - expiredAt = Instant.FromUnixTimeSeconds(request.ExpiredAt.Seconds); - } - else if (request.Duration != null) - { - expiredAt = SystemClock.Instance.GetCurrentInstant() + - Duration.FromTimeSpan(request.Duration.ToTimeSpan()); - } - - var references = await fileReferenceService.UpdateResourceFilesAsync( - request.ResourceId, - request.FileIds, - request.Usage, - expiredAt - ); - var response = new UpdateResourceFilesResponse(); - response.References.AddRange(references.Select(r => r.ToProtoValue())); - return response; - } - - public override async Task SetReferenceExpiration( - SetReferenceExpirationRequest request, ServerCallContext context) - { - Instant? expiredAt = null; - if (request.ExpiredAt != null) - { - expiredAt = Instant.FromUnixTimeSeconds(request.ExpiredAt.Seconds); - } - else if (request.Duration != null) - { - expiredAt = SystemClock.Instance.GetCurrentInstant() + - Duration.FromTimeSpan(request.Duration.ToTimeSpan()); - } - - var success = - await fileReferenceService.SetReferenceExpirationAsync(Guid.Parse(request.ReferenceId), expiredAt); - return new SetReferenceExpirationResponse { Success = success }; - } - - public override async Task SetFileReferencesExpiration( - SetFileReferencesExpirationRequest request, ServerCallContext context) - { - var expiredAt = Instant.FromUnixTimeSeconds(request.ExpiredAt.Seconds); - var updatedCount = await fileReferenceService.SetFileReferencesExpirationAsync(request.FileId, expiredAt); - return new SetFileReferencesExpirationResponse { UpdatedCount = updatedCount }; - } - - public override async Task HasFileReferences(HasFileReferencesRequest request, - ServerCallContext context) - { - var hasReferences = await fileReferenceService.HasFileReferencesAsync(request.FileId); - return new HasFileReferencesResponse { HasReferences = hasReferences }; - } -} \ No newline at end of file diff --git a/DysonNetwork.Drive/Storage/FileService.cs b/DysonNetwork.Drive/Storage/FileService.cs index 6c8cfcb9..3ad7d33c 100644 --- a/DysonNetwork.Drive/Storage/FileService.cs +++ b/DysonNetwork.Drive/Storage/FileService.cs @@ -40,6 +40,8 @@ public class FileService( .Where(f => f.Id == fileId) .Include(f => f.Pool) .Include(f => f.Bundle) + .Include(f => f.Object) + .ThenInclude(o => o.FileReplicas) .FirstOrDefaultAsync(); if (file != null) @@ -69,6 +71,8 @@ public class FileService( var dbFiles = await db.Files .Where(f => uncachedIds.Contains(f.Id)) .Include(f => f.Pool) + .Include(f => f.Object) + .ThenInclude(o => o.FileReplicas) .ToListAsync(); foreach (var file in dbFiles) @@ -228,8 +232,34 @@ public class FileService( private async Task SaveFileToDatabaseAsync(SnCloudFile file) { + var fileObject = new SnFileObject + { + Id = file.Id, + AccountId = file.AccountId, + Size = file.Size, + Meta = file.FileMeta, + MimeType = file.MimeType, + Hash = file.Hash, + HasCompression = file.HasCompression, + HasThumbnail = file.HasThumbnail + }; + + var replica = new SnFileReplica + { + Id = Guid.NewGuid(), + ObjectId = file.Id, + PoolId = file.PoolId!.Value, + StorageId = file.StorageId ?? file.Id, + Status = SnFileReplicaStatus.Available, + IsPrimary = true + }; + db.Files.Add(file); + db.FileObjects.Add(fileObject); + db.FileReplicas.Add(replica); + await db.SaveChangesAsync(); + file.ObjectId = file.Id; file.StorageId ??= file.Id; } @@ -470,8 +500,20 @@ public class FileService( await db.Files.Where(f => f.Id == file.Id).ExecuteUpdateAsync(updatable.ToSetPropertyCalls()); + if (updateMask.Paths.Contains("file_meta")) + { + await db.FileObjects + .Where(fo => fo.Id == file.ObjectId) + .ExecuteUpdateAsync(setter => setter + .SetProperty(fo => fo.Meta, file.FileMeta)); + } + await _PurgeCacheAsync(file.Id); - return await db.Files.AsNoTracking().FirstAsync(f => f.Id == file.Id); + return await db.Files + .AsNoTracking() + .Include(f => f.Object) + .ThenInclude(o => o.FileReplicas) + .FirstAsync(f => f.Id == file.Id); } public async Task DeleteFileAsync(SnCloudFile file, bool skipData = false) @@ -481,17 +523,23 @@ public class FileService( await _PurgeCacheAsync(file.Id); if (!skipData) - await DeleteFileDataAsync(file); + { + var hasOtherReferences = await db.Files + .AnyAsync(f => f.ObjectId == file.ObjectId && f.Id != file.Id); + + if (!hasOtherReferences) + await DeleteFileDataAsync(file); + } } public async Task DeleteFileDataAsync(SnCloudFile file, bool force = false) { - if (!file.PoolId.HasValue) return; + if (!file.PoolId.HasValue || file.ObjectId == null) return; if (!force) { var sameOriginFiles = await db.Files - .Where(f => f.StorageId == file.StorageId && f.Id != file.Id) + .Where(f => f.ObjectId == file.ObjectId && f.Id != file.Id) .Select(f => f.Id) .ToListAsync(); @@ -499,6 +547,17 @@ public class FileService( return; } + var replicas = await db.FileReplicas + .Where(r => r.ObjectId == file.ObjectId) + .ToListAsync(); + + if (replicas.Count == 0) + { + logger.LogWarning("No replicas found for file object {ObjectId}", file.ObjectId); + return; + } + + var primaryReplica = replicas.First(r => r.IsPrimary); var dest = await GetRemoteStorageConfig(file.PoolId.Value); if (dest is null) throw new InvalidOperationException($"No remote storage configured for pool {file.PoolId}"); var client = CreateMinioClient(dest); @@ -508,7 +567,7 @@ public class FileService( ); var bucket = dest.Bucket; - var objectId = file.StorageId ?? file.Id; + var objectId = primaryReplica.StorageId; await client.RemoveObjectAsync( new RemoveObjectArgs().WithBucket(bucket).WithObject(objectId) @@ -541,36 +600,55 @@ public class FileService( logger.LogWarning("Failed to delete thumbnail of file {fileId}", file.Id); } } + + db.FileReplicas.RemoveRange(replicas); + var fileObject = await db.FileObjects.FindAsync(file.ObjectId); + if (fileObject != null) db.FileObjects.Remove(fileObject); + await db.SaveChangesAsync(); } public async Task DeleteFileDataBatchAsync(List files) { - files = files.Where(f => f.PoolId.HasValue).ToList(); + files = files.Where(f => f.PoolId.HasValue && f.ObjectId != null).ToList(); - foreach (var fileGroup in files.GroupBy(f => f.PoolId!.Value)) + var objectIds = files.Select(f => f.ObjectId).Distinct().ToList(); + var replicas = await db.FileReplicas + .Where(r => objectIds.Contains(r.ObjectId)) + .ToListAsync(); + + foreach (var poolGroup in replicas.GroupBy(r => r.PoolId)) { - var dest = await GetRemoteStorageConfig(fileGroup.Key); + var dest = await GetRemoteStorageConfig(poolGroup.Key); if (dest is null) - throw new InvalidOperationException($"No remote storage configured for pool {fileGroup.Key}"); + throw new InvalidOperationException($"No remote storage configured for pool {poolGroup.Key}"); var client = CreateMinioClient(dest); if (client is null) throw new InvalidOperationException( - $"Failed to configure client for remote destination '{fileGroup.Key}'" + $"Failed to configure client for remote destination '{poolGroup.Key}'" ); List objectsToDelete = []; - foreach (var file in fileGroup) + foreach (var replica in poolGroup) { - objectsToDelete.Add(file.StorageId ?? file.Id); - if (file.HasCompression) objectsToDelete.Add(file.StorageId ?? file.Id + ".compressed"); - if (file.HasThumbnail) objectsToDelete.Add(file.StorageId ?? file.Id + ".thumbnail"); + var file = files.First(f => f.ObjectId == replica.ObjectId); + objectsToDelete.Add(replica.StorageId); + if (file.HasCompression) objectsToDelete.Add(replica.StorageId + ".compressed"); + if (file.HasThumbnail) objectsToDelete.Add(replica.StorageId + ".thumbnail"); } await client.RemoveObjectsAsync( new RemoveObjectsArgs().WithBucket(dest.Bucket).WithObjects(objectsToDelete) ); + + db.FileReplicas.RemoveRange(poolGroup); } + + var fileObjects = await db.FileObjects + .Where(fo => objectIds.Contains(fo.Id)) + .ToListAsync(); + db.FileObjects.RemoveRange(fileObjects); + await db.SaveChangesAsync(); } private async Task GetBundleAsync(Guid id, Guid accountId) @@ -654,6 +732,8 @@ public class FileService( { var dbFiles = await db.Files .Where(f => uncachedIds.Contains(f.Id)) + .Include(f => f.Object) + .ThenInclude(o => o.FileReplicas) .ToListAsync(); foreach (var file in dbFiles) @@ -674,15 +754,21 @@ public class FileService( public async Task GetReferenceCountAsync(string fileId) { - return await db.FileReferences - .Where(r => r.FileId == fileId) + var file = await db.Files.FirstOrDefaultAsync(f => f.Id == fileId); + if (file == null || file.ObjectId == null) return 0; + + return await db.Files + .Where(f => f.ObjectId == file.ObjectId && f.Id != fileId) .CountAsync(); } public async Task IsReferencedAsync(string fileId) { - return await db.FileReferences - .Where(r => r.FileId == fileId) + var file = await db.Files.FirstOrDefaultAsync(f => f.Id == fileId); + if (file == null || file.ObjectId == null) return false; + + return await db.Files + .Where(f => f.ObjectId == file.ObjectId && f.Id != fileId) .AnyAsync(); } @@ -709,8 +795,6 @@ public class FileService( .Where(f => f.AccountId == accountId && f.IsMarkedRecycle) .ToListAsync(); var count = files.Count; - var tasks = files.Select(f => DeleteFileDataAsync(f, true)); - await Task.WhenAll(tasks); var fileIds = files.Select(f => f.Id).ToList(); await _PurgeCacheRangeAsync(fileIds); db.RemoveRange(files); @@ -724,8 +808,6 @@ public class FileService( .Where(f => f.AccountId == accountId && fileIds.Contains(f.Id)) .ToListAsync(); var count = files.Count; - var tasks = files.Select(f => DeleteFileDataAsync(f, true)); - await Task.WhenAll(tasks); var fileIdsList = files.Select(f => f.Id).ToList(); await _PurgeCacheRangeAsync(fileIdsList); db.RemoveRange(files); @@ -739,8 +821,6 @@ public class FileService( .Where(f => f.PoolId == poolId && f.IsMarkedRecycle) .ToListAsync(); var count = files.Count; - var tasks = files.Select(f => DeleteFileDataAsync(f, true)); - await Task.WhenAll(tasks); var fileIds = files.Select(f => f.Id).ToList(); await _PurgeCacheRangeAsync(fileIds); db.RemoveRange(files); @@ -754,8 +834,6 @@ public class FileService( .Where(f => f.IsMarkedRecycle) .ToListAsync(); var count = files.Count; - var tasks = files.Select(f => DeleteFileDataAsync(f, true)); - await Task.WhenAll(tasks); var fileIds = files.Select(f => f.Id).ToList(); await _PurgeCacheRangeAsync(fileIds); db.RemoveRange(files); diff --git a/DysonNetwork.Shared/Models/CloudFile.cs b/DysonNetwork.Shared/Models/CloudFile.cs index b7d4513f..305d6e31 100644 --- a/DysonNetwork.Shared/Models/CloudFile.cs +++ b/DysonNetwork.Shared/Models/CloudFile.cs @@ -60,9 +60,7 @@ public class SnCloudFile : ModelBase, ICloudFile, IIdentifiedResource [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] public string? FastUploadLink { get; set; } - public List References { get; set; } = new List(); - - public Guid AccountId { get; set; } + public Guid AccountId { get; set; } public SnCloudFileReferenceObject ToReferenceObject() { @@ -112,34 +110,3 @@ public class SnCloudFile : ModelBase, ICloudFile, IIdentifiedResource return proto; } } - -public class SnCloudFileReference : ModelBase -{ - public Guid Id { get; set; } = Guid.NewGuid(); - [MaxLength(32)] public string FileId { get; set; } = null!; - [JsonIgnore] public SnCloudFile File { get; set; } = null!; - [MaxLength(1024)] public string Usage { get; set; } = null!; - [MaxLength(1024)] public string ResourceId { get; set; } = null!; - - /// - /// Optional expiration date for the file reference - /// - public Instant? ExpiredAt { get; set; } - - /// - /// Converts the SnCloudFileReference to a protobuf message - /// - /// The protobuf message representation of this object - public CloudFileReference ToProtoValue() - { - return new CloudFileReference - { - Id = Id.ToString(), - FileId = FileId, - File = File?.ToProtoValue(), - Usage = Usage, - ResourceId = ResourceId, - ExpiredAt = ExpiredAt?.ToTimestamp() - }; - } -} diff --git a/DysonNetwork.Shared/Models/CloudFileObject.cs b/DysonNetwork.Shared/Models/CloudFileObject.cs index 2279dc0d..bc1955af 100644 --- a/DysonNetwork.Shared/Models/CloudFileObject.cs +++ b/DysonNetwork.Shared/Models/CloudFileObject.cs @@ -6,6 +6,7 @@ namespace DysonNetwork.Shared.Models; public class SnFileObject : ModelBase { [MaxLength(32)] public string Id { get; set; } + public Guid AccountId { get; set; } public long Size { get; set; } diff --git a/DysonNetwork.Sphere/Post/PostService.cs b/DysonNetwork.Sphere/Post/PostService.cs index 90e37029..f05adc0b 100644 --- a/DysonNetwork.Sphere/Post/PostService.cs +++ b/DysonNetwork.Sphere/Post/PostService.cs @@ -25,7 +25,6 @@ public partial class PostService( ICacheService cache, ILogger logger, FileService.FileServiceClient files, - FileReferenceService.FileReferenceServiceClient fileRefs, Publisher.PublisherService ps, RemoteWebReaderService reader, AccountService.AccountServiceClient accounts, @@ -194,18 +193,6 @@ public partial class PostService( db.Posts.Add(post); await db.SaveChangesAsync(); - // Create file references for each attachment - if (post.Attachments.Count != 0) - { - var request = new CreateReferenceBatchRequest - { - Usage = PostFileUsageIdentifier, - ResourceId = post.ResourceIdentifier, - }; - request.FilesId.AddRange(post.Attachments.Select(a => a.Id)); - await fileRefs.CreateReferenceBatchAsync(request); - } - if (post.PublishedAt is not null && post.PublishedAt.Value.ToDateTimeUtc() <= DateTime.UtcNow) _ = Task.Run(async () => { @@ -306,17 +293,6 @@ public partial class PostService( if (attachments is not null) { - var postResourceId = $"post:{post.Id}"; - - // Update resource references using the new file list - var request = new UpdateResourceFilesRequest - { - ResourceId = postResourceId, - Usage = PostFileUsageIdentifier, - }; - request.FileIds.AddRange(attachments); - await fileRefs.UpdateResourceFilesAsync(request); - // Update post attachments by getting files from database var queryRequest = new GetFileBatchRequest(); queryRequest.Ids.AddRange(attachments); @@ -475,11 +451,6 @@ public partial class PostService( public async Task DeletePostAsync(SnPost post) { - // Delete all file references for this post - await fileRefs.DeleteResourceReferencesAsync( - new DeleteResourceReferencesRequest { ResourceId = post.ResourceIdentifier } - ); - var now = SystemClock.Instance.GetCurrentInstant(); await using var transaction = await db.Database.BeginTransactionAsync(); try diff --git a/DysonNetwork.Sphere/Publisher/PublisherController.cs b/DysonNetwork.Sphere/Publisher/PublisherController.cs index ee7ba4e1..3092d3fd 100644 --- a/DysonNetwork.Sphere/Publisher/PublisherController.cs +++ b/DysonNetwork.Sphere/Publisher/PublisherController.cs @@ -19,7 +19,6 @@ public class PublisherController( PublisherService ps, AccountService.AccountServiceClient accounts, FileService.FileServiceClient files, - FileReferenceService.FileReferenceServiceClient fileRefs, ActionLogService.ActionLogServiceClient als, RemoteRealmService remoteRealmService, IServiceScopeFactory factory @@ -569,25 +568,7 @@ public class PublisherController( ); var picture = SnCloudFileReferenceObject.FromProtoValue(queryResult); - // Remove old references for the publisher picture - if (publisher.Picture is not null) - await fileRefs.DeleteResourceReferencesAsync( - new DeleteResourceReferencesRequest - { - ResourceId = publisher.ResourceIdentifier, - } - ); - publisher.Picture = picture; - - await fileRefs.CreateReferenceAsync( - new CreateReferenceRequest - { - FileId = picture.Id, - Usage = "publisher.picture", - ResourceId = publisher.ResourceIdentifier, - } - ); } if (request.BackgroundId is not null) @@ -601,27 +582,7 @@ public class PublisherController( ); var background = SnCloudFileReferenceObject.FromProtoValue(queryResult); - // Remove old references for the publisher background - if (publisher.Background is not null) - { - await fileRefs.DeleteResourceReferencesAsync( - new DeleteResourceReferencesRequest - { - ResourceId = publisher.ResourceIdentifier, - } - ); - } - publisher.Background = background; - - await fileRefs.CreateReferenceAsync( - new CreateReferenceRequest - { - FileId = background.Id, - Usage = "publisher.background", - ResourceId = publisher.ResourceIdentifier, - } - ); } db.Update(publisher); @@ -717,11 +678,6 @@ public class PublisherController( var publisherResourceId = $"publisher:{publisher.Id}"; - // Delete all file references for this publisher - await fileRefs.DeleteResourceReferencesAsync( - new DeleteResourceReferencesRequest { ResourceId = publisherResourceId } - ); - db.Publishers.Remove(publisher); await db.SaveChangesAsync(); diff --git a/DysonNetwork.Sphere/Publisher/PublisherService.cs b/DysonNetwork.Sphere/Publisher/PublisherService.cs index 50d680fe..7f82d30a 100644 --- a/DysonNetwork.Sphere/Publisher/PublisherService.cs +++ b/DysonNetwork.Sphere/Publisher/PublisherService.cs @@ -22,7 +22,6 @@ public class FediverseStatus public class PublisherService( AppDatabase db, - FileReferenceService.FileReferenceServiceClient fileRefs, SocialCreditService.SocialCreditServiceClient socialCredits, ExperienceService.ExperienceServiceClient experiences, ICacheService cache, @@ -210,30 +209,6 @@ public class PublisherService( db.Publishers.Add(publisher); await db.SaveChangesAsync(); - if (publisher.Picture is not null) - { - await fileRefs.CreateReferenceAsync( - new CreateReferenceRequest - { - FileId = publisher.Picture.Id, - Usage = "publisher.picture", - ResourceId = publisher.ResourceIdentifier, - } - ); - } - - if (publisher.Background is not null) - { - await fileRefs.CreateReferenceAsync( - new CreateReferenceRequest - { - FileId = publisher.Background.Id, - Usage = "publisher.background", - ResourceId = publisher.ResourceIdentifier, - } - ); - } - return publisher; } @@ -270,30 +245,6 @@ public class PublisherService( db.Publishers.Add(publisher); await db.SaveChangesAsync(); - if (publisher.Picture is not null) - { - await fileRefs.CreateReferenceAsync( - new CreateReferenceRequest - { - FileId = publisher.Picture.Id, - Usage = "publisher.picture", - ResourceId = publisher.ResourceIdentifier, - } - ); - } - - if (publisher.Background is not null) - { - await fileRefs.CreateReferenceAsync( - new CreateReferenceRequest - { - FileId = publisher.Background.Id, - Usage = "publisher.background", - ResourceId = publisher.ResourceIdentifier, - } - ); - } - return publisher; }