SolarNetwork/Swarm
2
3
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

⚗️ Testing out the File Storing System v2

Browse Source
This commit is contained in:
LittleSheep
2026-01-10 16:54:22 +08:00
parent 8177bda232
commit 98c100c864
19 changed files with 1353 additions and 1035 deletions
Download Patch File Download Diff File Expand all files Collapse all files

1
DysonNetwork.Drive/AppDatabase.cs
View File

@@ -26,7 +26,6 @@ public class AppDatabase(
public DbSet<SnFileObject> FileObjects { get; set; } = null!; public DbSet<SnFileObject> FileObjects { get; set; } = null!;
public DbSet<SnFileReplica> FileReplicas { get; set; } = null!; public DbSet<SnFileReplica> FileReplicas { get; set; } = null!;
public DbSet<SnFilePermission> FilePermissions { get; set; } = null!; public DbSet<SnFilePermission> FilePermissions { get; set; } = null!;
public DbSet<SnCloudFileReference> FileReferences { get; set; } = null!;
public DbSet<SnCloudFileIndex> FileIndexes { get; set; } public DbSet<SnCloudFileIndex> FileIndexes { get; set; }
public DbSet<PersistentTask> Tasks { get; set; } = null!; public DbSet<PersistentTask> Tasks { get; set; } = null!;

762
DysonNetwork.Drive/Migrations/20260110084758_RemoveFileReferencesAndAddFileObjectOwner.Designer.cs generated Normal file
View File

@@ -0,0 +1,762 @@
// <auto-generated />
using System;
using System.Collections.Generic;
using DysonNetwork.Drive;
using DysonNetwork.Shared.Models;
using Microsoft.EntityFrameworkCore;
using Microsoft.EntityFrameworkCore.Infrastructure;
using Microsoft.EntityFrameworkCore.Migrations;
using Microsoft.EntityFrameworkCore.Storage.ValueConversion;
using NodaTime;
using Npgsql.EntityFrameworkCore.PostgreSQL.Metadata;
#nullable disable
namespace DysonNetwork.Drive.Migrations
{
[DbContext(typeof(AppDatabase))]
[Migration("20260110084758_RemoveFileReferencesAndAddFileObjectOwner")]
partial class RemoveFileReferencesAndAddFileObjectOwner
{
/// <inheritdoc />
protected override void BuildTargetModel(ModelBuilder modelBuilder)
{
#pragma warning disable 612, 618
modelBuilder
.HasAnnotation("ProductVersion", "10.0.1")
.HasAnnotation("Relational:MaxIdentifierLength", 63);
NpgsqlModelBuilderExtensions.UseIdentityByDefaultColumns(modelBuilder);
modelBuilder.Entity("DysonNetwork.Drive.Billing.QuotaRecord", b =>
{
b.Property<Guid>("Id")
.ValueGeneratedOnAdd()
.HasColumnType("uuid")
.HasColumnName("id");
b.Property<Guid>("AccountId")
.HasColumnType("uuid")
.HasColumnName("account_id");
b.Property<Instant>("CreatedAt")
.HasColumnType("timestamp with time zone")
.HasColumnName("created_at");
b.Property<Instant?>("DeletedAt")
.HasColumnType("timestamp with time zone")
.HasColumnName("deleted_at");
b.Property<string>("Description")
.IsRequired()
.HasColumnType("text")
.HasColumnName("description");
b.Property<Instant?>("ExpiredAt")
.HasColumnType("timestamp with time zone")
.HasColumnName("expired_at");
b.Property<string>("Name")
.IsRequired()
.HasColumnType("text")
.HasColumnName("name");
b.Property<long>("Quota")
.HasColumnType("bigint")
.HasColumnName("quota");
b.Property<Instant>("UpdatedAt")
.HasColumnType("timestamp with time zone")
.HasColumnName("updated_at");
b.HasKey("Id")
.HasName("pk_quota_records");
b.ToTable("quota_records", (string)null);
});
modelBuilder.Entity("DysonNetwork.Drive.Storage.Model.PersistentTask", b =>
{
b.Property<Guid>("Id")
.ValueGeneratedOnAdd()
.HasColumnType("uuid")
.HasColumnName("id");
b.Property<Guid>("AccountId")
.HasColumnType("uuid")
.HasColumnName("account_id");
b.Property<Instant?>("CompletedAt")
.HasColumnType("timestamp with time zone")
.HasColumnName("completed_at");
b.Property<Instant>("CreatedAt")
.HasColumnType("timestamp with time zone")
.HasColumnName("created_at");
b.Property<Instant?>("DeletedAt")
.HasColumnType("timestamp with time zone")
.HasColumnName("deleted_at");
b.Property<string>("Description")
.HasMaxLength(1024)
.HasColumnType("character varying(1024)")
.HasColumnName("description");
b.Property<string>("Discriminator")
.IsRequired()
.HasMaxLength(21)
.HasColumnType("character varying(21)")
.HasColumnName("discriminator");
b.Property<string>("ErrorMessage")
.HasMaxLength(1024)
.HasColumnType("character varying(1024)")
.HasColumnName("error_message");
b.Property<long?>("EstimatedDurationSeconds")
.HasColumnType("bigint")
.HasColumnName("estimated_duration_seconds");
b.Property<Instant?>("ExpiredAt")
.HasColumnType("timestamp with time zone")
.HasColumnName("expired_at");
b.Property<Instant>("LastActivity")
.HasColumnType("timestamp with time zone")
.HasColumnName("last_activity");
b.Property<string>("Name")
.IsRequired()
.HasMaxLength(256)
.HasColumnType("character varying(256)")
.HasColumnName("name");
b.Property<Dictionary<string, object>>("Parameters")
.IsRequired()
.HasColumnType("jsonb")
.HasColumnName("parameters");
b.Property<int>("Priority")
.HasColumnType("integer")
.HasColumnName("priority");
b.Property<double>("Progress")
.HasColumnType("double precision")
.HasColumnName("progress");
b.Property<Dictionary<string, object>>("Results")
.IsRequired()
.HasColumnType("jsonb")
.HasColumnName("results");
b.Property<Instant?>("StartedAt")
.HasColumnType("timestamp with time zone")
.HasColumnName("started_at");
b.Property<int>("Status")
.HasColumnType("integer")
.HasColumnName("status");
b.Property<string>("TaskId")
.IsRequired()
.HasMaxLength(64)
.HasColumnType("character varying(64)")
.HasColumnName("task_id");
b.Property<int>("Type")
.HasColumnType("integer")
.HasColumnName("type");
b.Property<Instant>("UpdatedAt")
.HasColumnType("timestamp with time zone")
.HasColumnName("updated_at");
b.HasKey("Id")
.HasName("pk_tasks");
b.ToTable("tasks", (string)null);
b.HasDiscriminator().HasValue("PersistentTask");
b.UseTphMappingStrategy();
});
modelBuilder.Entity("DysonNetwork.Shared.Models.FilePool", b =>
{
b.Property<Guid>("Id")
.ValueGeneratedOnAdd()
.HasColumnType("uuid")
.HasColumnName("id");
b.Property<Guid?>("AccountId")
.HasColumnType("uuid")
.HasColumnName("account_id");
b.Property<BillingConfig>("BillingConfig")
.IsRequired()
.HasColumnType("jsonb")
.HasColumnName("billing_config");
b.Property<Instant>("CreatedAt")
.HasColumnType("timestamp with time zone")
.HasColumnName("created_at");
b.Property<Instant?>("DeletedAt")
.HasColumnType("timestamp with time zone")
.HasColumnName("deleted_at");
b.Property<string>("Description")
.IsRequired()
.HasMaxLength(8192)
.HasColumnType("character varying(8192)")
.HasColumnName("description");
b.Property<bool>("IsHidden")
.HasColumnType("boolean")
.HasColumnName("is_hidden");
b.Property<string>("Name")
.IsRequired()
.HasMaxLength(1024)
.HasColumnType("character varying(1024)")
.HasColumnName("name");
b.Property<PolicyConfig>("PolicyConfig")
.IsRequired()
.HasColumnType("jsonb")
.HasColumnName("policy_config");
b.Property<RemoteStorageConfig>("StorageConfig")
.IsRequired()
.HasColumnType("jsonb")
.HasColumnName("storage_config");
b.Property<Instant>("UpdatedAt")
.HasColumnType("timestamp with time zone")
.HasColumnName("updated_at");
b.HasKey("Id")
.HasName("pk_pools");
b.ToTable("pools", (string)null);
});
modelBuilder.Entity("DysonNetwork.Shared.Models.SnCloudFile", b =>
{
b.Property<string>("Id")
.HasMaxLength(32)
.HasColumnType("character varying(32)")
.HasColumnName("id");
b.Property<Guid>("AccountId")
.HasColumnType("uuid")
.HasColumnName("account_id");
b.Property<Guid?>("BundleId")
.HasColumnType("uuid")
.HasColumnName("bundle_id");
b.Property<Instant>("CreatedAt")
.HasColumnType("timestamp with time zone")
.HasColumnName("created_at");
b.Property<Instant?>("DeletedAt")
.HasColumnType("timestamp with time zone")
.HasColumnName("deleted_at");
b.Property<string>("Description")
.HasMaxLength(4096)
.HasColumnType("character varying(4096)")
.HasColumnName("description");
b.Property<Instant?>("ExpiredAt")
.HasColumnType("timestamp with time zone")
.HasColumnName("expired_at");
b.Property<Dictionary<string, object>>("FileMeta")
.HasColumnType("jsonb")
.HasColumnName("file_meta");
b.Property<bool>("HasCompression")
.HasColumnType("boolean")
.HasColumnName("has_compression");
b.Property<bool>("HasThumbnail")
.HasColumnType("boolean")
.HasColumnName("has_thumbnail");
b.Property<string>("Hash")
.HasMaxLength(256)
.HasColumnType("character varying(256)")
.HasColumnName("hash");
b.Property<bool>("IsEncrypted")
.HasColumnType("boolean")
.HasColumnName("is_encrypted");
b.Property<bool>("IsMarkedRecycle")
.HasColumnType("boolean")
.HasColumnName("is_marked_recycle");
b.Property<string>("MimeType")
.HasMaxLength(256)
.HasColumnType("character varying(256)")
.HasColumnName("mime_type");
b.Property<string>("Name")
.IsRequired()
.HasMaxLength(1024)
.HasColumnType("character varying(1024)")
.HasColumnName("name");
b.Property<string>("ObjectId")
.HasMaxLength(32)
.HasColumnType("character varying(32)")
.HasColumnName("object_id");
b.Property<Guid?>("PoolId")
.HasColumnType("uuid")
.HasColumnName("pool_id");
b.PrimitiveCollection<string>("SensitiveMarks")
.HasColumnType("jsonb")
.HasColumnName("sensitive_marks");
b.Property<long>("Size")
.HasColumnType("bigint")
.HasColumnName("size");
b.Property<string>("StorageId")
.HasMaxLength(32)
.HasColumnType("character varying(32)")
.HasColumnName("storage_id");
b.Property<string>("StorageUrl")
.HasMaxLength(4096)
.HasColumnType("character varying(4096)")
.HasColumnName("storage_url");
b.Property<Instant>("UpdatedAt")
.HasColumnType("timestamp with time zone")
.HasColumnName("updated_at");
b.Property<Instant?>("UploadedAt")
.HasColumnType("timestamp with time zone")
.HasColumnName("uploaded_at");
b.Property<Dictionary<string, object>>("UserMeta")
.HasColumnType("jsonb")
.HasColumnName("user_meta");
b.HasKey("Id")
.HasName("pk_files");
b.HasIndex("BundleId")
.HasDatabaseName("ix_files_bundle_id");
b.HasIndex("ObjectId")
.HasDatabaseName("ix_files_object_id");
b.HasIndex("PoolId")
.HasDatabaseName("ix_files_pool_id");
b.ToTable("files", (string)null);
});
modelBuilder.Entity("DysonNetwork.Shared.Models.SnCloudFileIndex", b =>
{
b.Property<Guid>("Id")
.ValueGeneratedOnAdd()
.HasColumnType("uuid")
.HasColumnName("id");
b.Property<Guid>("AccountId")
.HasColumnType("uuid")
.HasColumnName("account_id");
b.Property<Instant>("CreatedAt")
.HasColumnType("timestamp with time zone")
.HasColumnName("created_at");
b.Property<Instant?>("DeletedAt")
.HasColumnType("timestamp with time zone")
.HasColumnName("deleted_at");
b.Property<string>("FileId")
.IsRequired()
.HasMaxLength(32)
.HasColumnType("character varying(32)")
.HasColumnName("file_id");
b.Property<string>("Path")
.IsRequired()
.HasMaxLength(8192)
.HasColumnType("character varying(8192)")
.HasColumnName("path");
b.Property<Instant>("UpdatedAt")
.HasColumnType("timestamp with time zone")
.HasColumnName("updated_at");
b.HasKey("Id")
.HasName("pk_file_indexes");
b.HasIndex("FileId")
.HasDatabaseName("ix_file_indexes_file_id");
b.HasIndex("Path", "AccountId")
.HasDatabaseName("ix_file_indexes_path_account_id");
b.ToTable("file_indexes", (string)null);
});
modelBuilder.Entity("DysonNetwork.Shared.Models.SnFileBundle", b =>
{
b.Property<Guid>("Id")
.ValueGeneratedOnAdd()
.HasColumnType("uuid")
.HasColumnName("id");
b.Property<Guid>("AccountId")
.HasColumnType("uuid")
.HasColumnName("account_id");
b.Property<Instant>("CreatedAt")
.HasColumnType("timestamp with time zone")
.HasColumnName("created_at");
b.Property<Instant?>("DeletedAt")
.HasColumnType("timestamp with time zone")
.HasColumnName("deleted_at");
b.Property<string>("Description")
.HasMaxLength(8192)
.HasColumnType("character varying(8192)")
.HasColumnName("description");
b.Property<Instant?>("ExpiredAt")
.HasColumnType("timestamp with time zone")
.HasColumnName("expired_at");
b.Property<string>("Name")
.IsRequired()
.HasMaxLength(1024)
.HasColumnType("character varying(1024)")
.HasColumnName("name");
b.Property<string>("Passcode")
.HasMaxLength(256)
.HasColumnType("character varying(256)")
.HasColumnName("passcode");
b.Property<string>("Slug")
.IsRequired()
.HasMaxLength(1024)
.HasColumnType("character varying(1024)")
.HasColumnName("slug");
b.Property<Instant>("UpdatedAt")
.HasColumnType("timestamp with time zone")
.HasColumnName("updated_at");
b.HasKey("Id")
.HasName("pk_bundles");
b.HasIndex("Slug")
.IsUnique()
.HasDatabaseName("ix_bundles_slug");
b.ToTable("bundles", (string)null);
});
modelBuilder.Entity("DysonNetwork.Shared.Models.SnFileObject", b =>
{
b.Property<string>("Id")
.HasMaxLength(32)
.HasColumnType("character varying(32)")
.HasColumnName("id");
b.Property<Guid>("AccountId")
.HasColumnType("uuid")
.HasColumnName("account_id");
b.Property<Instant>("CreatedAt")
.HasColumnType("timestamp with time zone")
.HasColumnName("created_at");
b.Property<Instant?>("DeletedAt")
.HasColumnType("timestamp with time zone")
.HasColumnName("deleted_at");
b.Property<bool>("HasCompression")
.HasColumnType("boolean")
.HasColumnName("has_compression");
b.Property<bool>("HasThumbnail")
.HasColumnType("boolean")
.HasColumnName("has_thumbnail");
b.Property<string>("Hash")
.HasMaxLength(256)
.HasColumnType("character varying(256)")
.HasColumnName("hash");
b.Property<Dictionary<string, object>>("Meta")
.HasColumnType("jsonb")
.HasColumnName("meta");
b.Property<string>("MimeType")
.HasMaxLength(256)
.HasColumnType("character varying(256)")
.HasColumnName("mime_type");
b.Property<long>("Size")
.HasColumnType("bigint")
.HasColumnName("size");
b.Property<Instant>("UpdatedAt")
.HasColumnType("timestamp with time zone")
.HasColumnName("updated_at");
b.HasKey("Id")
.HasName("pk_file_objects");
b.ToTable("file_objects", (string)null);
});
modelBuilder.Entity("DysonNetwork.Shared.Models.SnFilePermission", b =>
{
b.Property<Guid>("Id")
.ValueGeneratedOnAdd()
.HasColumnType("uuid")
.HasColumnName("id");
b.Property<Instant>("CreatedAt")
.HasColumnType("timestamp with time zone")
.HasColumnName("created_at");
b.Property<Instant?>("DeletedAt")
.HasColumnType("timestamp with time zone")
.HasColumnName("deleted_at");
b.Property<string>("FileId")
.IsRequired()
.HasColumnType("text")
.HasColumnName("file_id");
b.Property<int>("Permission")
.HasColumnType("integer")
.HasColumnName("permission");
b.Property<string>("SubjectId")
.IsRequired()
.HasColumnType("text")
.HasColumnName("subject_id");
b.Property<int>("SubjectType")
.HasColumnType("integer")
.HasColumnName("subject_type");
b.Property<Instant>("UpdatedAt")
.HasColumnType("timestamp with time zone")
.HasColumnName("updated_at");
b.HasKey("Id")
.HasName("pk_file_permissions");
b.ToTable("file_permissions", (string)null);
});
modelBuilder.Entity("DysonNetwork.Shared.Models.SnFileReplica", b =>
{
b.Property<Guid>("Id")
.ValueGeneratedOnAdd()
.HasColumnType("uuid")
.HasColumnName("id");
b.Property<Instant>("CreatedAt")
.HasColumnType("timestamp with time zone")
.HasColumnName("created_at");
b.Property<Instant?>("DeletedAt")
.HasColumnType("timestamp with time zone")
.HasColumnName("deleted_at");
b.Property<bool>("IsPrimary")
.HasColumnType("boolean")
.HasColumnName("is_primary");
b.Property<string>("ObjectId")
.IsRequired()
.HasMaxLength(32)
.HasColumnType("character varying(32)")
.HasColumnName("object_id");
b.Property<Guid>("PoolId")
.HasColumnType("uuid")
.HasColumnName("pool_id");
b.Property<int>("Status")
.HasColumnType("integer")
.HasColumnName("status");
b.Property<string>("StorageId")
.IsRequired()
.HasMaxLength(128)
.HasColumnType("character varying(128)")
.HasColumnName("storage_id");
b.Property<Instant>("UpdatedAt")
.HasColumnType("timestamp with time zone")
.HasColumnName("updated_at");
b.HasKey("Id")
.HasName("pk_file_replicas");
b.HasIndex("ObjectId")
.HasDatabaseName("ix_file_replicas_object_id");
b.HasIndex("PoolId")
.HasDatabaseName("ix_file_replicas_pool_id");
b.ToTable("file_replicas", (string)null);
});
modelBuilder.Entity("DysonNetwork.Drive.Storage.Model.PersistentUploadTask", b =>
{
b.HasBaseType("DysonNetwork.Drive.Storage.Model.PersistentTask");
b.Property<Guid?>("BundleId")
.HasColumnType("uuid")
.HasColumnName("bundle_id");
b.Property<long>("ChunkSize")
.HasColumnType("bigint")
.HasColumnName("chunk_size");
b.Property<int>("ChunksCount")
.HasColumnType("integer")
.HasColumnName("chunks_count");
b.Property<int>("ChunksUploaded")
.HasColumnType("integer")
.HasColumnName("chunks_uploaded");
b.Property<string>("ContentType")
.IsRequired()
.HasMaxLength(128)
.HasColumnType("character varying(128)")
.HasColumnName("content_type");
b.Property<string>("EncryptPassword")
.HasMaxLength(256)
.HasColumnType("character varying(256)")
.HasColumnName("encrypt_password");
b.Property<string>("FileName")
.IsRequired()
.HasMaxLength(256)
.HasColumnType("character varying(256)")
.HasColumnName("file_name");
b.Property<long>("FileSize")
.HasColumnType("bigint")
.HasColumnName("file_size");
b.Property<string>("Hash")
.IsRequired()
.HasColumnType("text")
.HasColumnName("hash");
b.Property<string>("Path")
.HasColumnType("text")
.HasColumnName("path");
b.Property<Guid>("PoolId")
.HasColumnType("uuid")
.HasColumnName("pool_id");
b.PrimitiveCollection<List<int>>("UploadedChunks")
.IsRequired()
.HasColumnType("integer[]")
.HasColumnName("uploaded_chunks");
b.HasDiscriminator().HasValue("PersistentUploadTask");
});
modelBuilder.Entity("DysonNetwork.Shared.Models.SnCloudFile", b =>
{
b.HasOne("DysonNetwork.Shared.Models.SnFileBundle", "Bundle")
.WithMany("Files")
.HasForeignKey("BundleId")
.HasConstraintName("fk_files_bundles_bundle_id");
b.HasOne("DysonNetwork.Shared.Models.SnFileObject", "Object")
.WithMany()
.HasForeignKey("ObjectId")
.HasConstraintName("fk_files_file_objects_object_id");
b.HasOne("DysonNetwork.Shared.Models.FilePool", "Pool")
.WithMany()
.HasForeignKey("PoolId")
.HasConstraintName("fk_files_pools_pool_id");
b.Navigation("Bundle");
b.Navigation("Object");
b.Navigation("Pool");
});
modelBuilder.Entity("DysonNetwork.Shared.Models.SnCloudFileIndex", b =>
{
b.HasOne("DysonNetwork.Shared.Models.SnCloudFile", "File")
.WithMany("FileIndexes")
.HasForeignKey("FileId")
.OnDelete(DeleteBehavior.Cascade)
.IsRequired()
.HasConstraintName("fk_file_indexes_files_file_id");
b.Navigation("File");
});
modelBuilder.Entity("DysonNetwork.Shared.Models.SnFileReplica", b =>
{
b.HasOne("DysonNetwork.Shared.Models.SnFileObject", "Object")
.WithMany("FileReplicas")
.HasForeignKey("ObjectId")
.OnDelete(DeleteBehavior.Cascade)
.IsRequired()
.HasConstraintName("fk_file_replicas_file_objects_object_id");
b.HasOne("DysonNetwork.Shared.Models.FilePool", "Pool")
.WithMany()
.HasForeignKey("PoolId")
.OnDelete(DeleteBehavior.Cascade)
.IsRequired()
.HasConstraintName("fk_file_replicas_pools_pool_id");
b.Navigation("Object");
b.Navigation("Pool");
});
modelBuilder.Entity("DysonNetwork.Shared.Models.SnCloudFile", b =>
{
b.Navigation("FileIndexes");
});
modelBuilder.Entity("DysonNetwork.Shared.Models.SnFileBundle", b =>
{
b.Navigation("Files");
});
modelBuilder.Entity("DysonNetwork.Shared.Models.SnFileObject", b =>
{
b.Navigation("FileReplicas");
});
#pragma warning restore 612, 618
}
}
}

173
DysonNetwork.Drive/Migrations/20260110084758_RemoveFileReferencesAndAddFileObjectOwner.cs Normal file
View File

@@ -0,0 +1,173 @@
using System;
using System.Collections.Generic;
using Microsoft.EntityFrameworkCore.Migrations;
using NodaTime;
#nullable disable
namespace DysonNetwork.Drive.Migrations
{
/// <inheritdoc />
public partial class RemoveFileReferencesAndAddFileObjectOwner : Migration
{
/// <inheritdoc />
protected override void Up(MigrationBuilder migrationBuilder)
{
migrationBuilder.DropTable(
name: "file_references");
migrationBuilder.AddColumn<string>(
name: "object_id",
table: "files",
type: "character varying(32)",
maxLength: 32,
nullable: true);
migrationBuilder.CreateTable(
name: "file_objects",
columns: table => new
{
id = table.Column<string>(type: "character varying(32)", maxLength: 32, nullable: false),
account_id = table.Column<Guid>(type: "uuid", nullable: false),
size = table.Column<long>(type: "bigint", nullable: false),
meta = table.Column<Dictionary<string, object>>(type: "jsonb", nullable: true),
mime_type = table.Column<string>(type: "character varying(256)", maxLength: 256, nullable: true),
hash = table.Column<string>(type: "character varying(256)", maxLength: 256, nullable: true),
has_compression = table.Column<bool>(type: "boolean", nullable: false),
has_thumbnail = table.Column<bool>(type: "boolean", nullable: false),
created_at = table.Column<Instant>(type: "timestamp with time zone", nullable: false),
updated_at = table.Column<Instant>(type: "timestamp with time zone", nullable: false),
deleted_at = table.Column<Instant>(type: "timestamp with time zone", nullable: true)
},
constraints: table =>
{
table.PrimaryKey("pk_file_objects", x => x.id);
});
migrationBuilder.CreateTable(
name: "file_permissions",
columns: table => new
{
id = table.Column<Guid>(type: "uuid", nullable: false),
file_id = table.Column<string>(type: "text", nullable: false),
subject_type = table.Column<int>(type: "integer", nullable: false),
subject_id = table.Column<string>(type: "text", nullable: false),
permission = table.Column<int>(type: "integer", nullable: false),
created_at = table.Column<Instant>(type: "timestamp with time zone", nullable: false),
updated_at = table.Column<Instant>(type: "timestamp with time zone", nullable: false),
deleted_at = table.Column<Instant>(type: "timestamp with time zone", nullable: true)
},
constraints: table =>
{
table.PrimaryKey("pk_file_permissions", x => x.id);
});
migrationBuilder.CreateTable(
name: "file_replicas",
columns: table => new
{
id = table.Column<Guid>(type: "uuid", nullable: false),
object_id = table.Column<string>(type: "character varying(32)", maxLength: 32, nullable: false),
pool_id = table.Column<Guid>(type: "uuid", nullable: false),
storage_id = table.Column<string>(type: "character varying(128)", maxLength: 128, nullable: false),
status = table.Column<int>(type: "integer", nullable: false),
is_primary = table.Column<bool>(type: "boolean", nullable: false),
created_at = table.Column<Instant>(type: "timestamp with time zone", nullable: false),
updated_at = table.Column<Instant>(type: "timestamp with time zone", nullable: false),
deleted_at = table.Column<Instant>(type: "timestamp with time zone", nullable: true)
},
constraints: table =>
{
table.PrimaryKey("pk_file_replicas", x => x.id);
table.ForeignKey(
name: "fk_file_replicas_file_objects_object_id",
column: x => x.object_id,
principalTable: "file_objects",
principalColumn: "id",
onDelete: ReferentialAction.Cascade);
table.ForeignKey(
name: "fk_file_replicas_pools_pool_id",
column: x => x.pool_id,
principalTable: "pools",
principalColumn: "id",
onDelete: ReferentialAction.Cascade);
});
migrationBuilder.CreateIndex(
name: "ix_files_object_id",
table: "files",
column: "object_id");
migrationBuilder.CreateIndex(
name: "ix_file_replicas_object_id",
table: "file_replicas",
column: "object_id");
migrationBuilder.CreateIndex(
name: "ix_file_replicas_pool_id",
table: "file_replicas",
column: "pool_id");
migrationBuilder.AddForeignKey(
name: "fk_files_file_objects_object_id",
table: "files",
column: "object_id",
principalTable: "file_objects",
principalColumn: "id");
}
/// <inheritdoc />
protected override void Down(MigrationBuilder migrationBuilder)
{
migrationBuilder.DropForeignKey(
name: "fk_files_file_objects_object_id",
table: "files");
migrationBuilder.DropTable(
name: "file_permissions");
migrationBuilder.DropTable(
name: "file_replicas");
migrationBuilder.DropTable(
name: "file_objects");
migrationBuilder.DropIndex(
name: "ix_files_object_id",
table: "files");
migrationBuilder.DropColumn(
name: "object_id",
table: "files");
migrationBuilder.CreateTable(
name: "file_references",
columns: table => new
{
id = table.Column<Guid>(type: "uuid", nullable: false),
file_id = table.Column<string>(type: "character varying(32)", maxLength: 32, nullable: false),
created_at = table.Column<Instant>(type: "timestamp with time zone", nullable: false),
deleted_at = table.Column<Instant>(type: "timestamp with time zone", nullable: true),
expired_at = table.Column<Instant>(type: "timestamp with time zone", nullable: true),
resource_id = table.Column<string>(type: "character varying(1024)", maxLength: 1024, nullable: false),
updated_at = table.Column<Instant>(type: "timestamp with time zone", nullable: false),
usage = table.Column<string>(type: "character varying(1024)", maxLength: 1024, nullable: false)
},
constraints: table =>
{
table.PrimaryKey("pk_file_references", x => x.id);
table.ForeignKey(
name: "fk_file_references_files_file_id",
column: x => x.file_id,
principalTable: "files",
principalColumn: "id",
onDelete: ReferentialAction.Cascade);
});
migrationBuilder.CreateIndex(
name: "ix_file_references_file_id",
table: "file_references",
column: "file_id");
}
}
}

246
DysonNetwork.Drive/Migrations/AppDatabaseModelSnapshot.cs
View File

@@ -307,6 +307,11 @@ namespace DysonNetwork.Drive.Migrations
.HasColumnType("character varying(1024)") .HasColumnType("character varying(1024)")
.HasColumnName("name"); .HasColumnName("name");
b.Property<string>("ObjectId")
.HasMaxLength(32)
.HasColumnType("character varying(32)")
.HasColumnName("object_id");
b.Property<Guid?>("PoolId") b.Property<Guid?>("PoolId")
.HasColumnType("uuid") .HasColumnType("uuid")
.HasColumnName("pool_id"); .HasColumnName("pool_id");
@@ -347,6 +352,9 @@ namespace DysonNetwork.Drive.Migrations
b.HasIndex("BundleId") b.HasIndex("BundleId")
.HasDatabaseName("ix_files_bundle_id"); .HasDatabaseName("ix_files_bundle_id");
b.HasIndex("ObjectId")
.HasDatabaseName("ix_files_object_id");
b.HasIndex("PoolId") b.HasIndex("PoolId")
.HasDatabaseName("ix_files_pool_id"); .HasDatabaseName("ix_files_pool_id");
@@ -400,56 +408,6 @@ namespace DysonNetwork.Drive.Migrations
b.ToTable("file_indexes", (string)null); b.ToTable("file_indexes", (string)null);
}); });
modelBuilder.Entity("DysonNetwork.Shared.Models.SnCloudFileReference", b =>
{
b.Property<Guid>("Id")
.ValueGeneratedOnAdd()
.HasColumnType("uuid")
.HasColumnName("id");
b.Property<Instant>("CreatedAt")
.HasColumnType("timestamp with time zone")
.HasColumnName("created_at");
b.Property<Instant?>("DeletedAt")
.HasColumnType("timestamp with time zone")
.HasColumnName("deleted_at");
b.Property<Instant?>("ExpiredAt")
.HasColumnType("timestamp with time zone")
.HasColumnName("expired_at");
b.Property<string>("FileId")
.IsRequired()
.HasMaxLength(32)
.HasColumnType("character varying(32)")
.HasColumnName("file_id");
b.Property<string>("ResourceId")
.IsRequired()
.HasMaxLength(1024)
.HasColumnType("character varying(1024)")
.HasColumnName("resource_id");
b.Property<Instant>("UpdatedAt")
.HasColumnType("timestamp with time zone")
.HasColumnName("updated_at");
b.Property<string>("Usage")
.IsRequired()
.HasMaxLength(1024)
.HasColumnType("character varying(1024)")
.HasColumnName("usage");
b.HasKey("Id")
.HasName("pk_file_references");
b.HasIndex("FileId")
.HasDatabaseName("ix_file_references_file_id");
b.ToTable("file_references", (string)null);
});
modelBuilder.Entity("DysonNetwork.Shared.Models.SnFileBundle", b => modelBuilder.Entity("DysonNetwork.Shared.Models.SnFileBundle", b =>
{ {
b.Property<Guid>("Id") b.Property<Guid>("Id")
@@ -509,6 +467,159 @@ namespace DysonNetwork.Drive.Migrations
b.ToTable("bundles", (string)null); b.ToTable("bundles", (string)null);
}); });
modelBuilder.Entity("DysonNetwork.Shared.Models.SnFileObject", b =>
{
b.Property<string>("Id")
.HasMaxLength(32)
.HasColumnType("character varying(32)")
.HasColumnName("id");
b.Property<Guid>("AccountId")
.HasColumnType("uuid")
.HasColumnName("account_id");
b.Property<Instant>("CreatedAt")
.HasColumnType("timestamp with time zone")
.HasColumnName("created_at");
b.Property<Instant?>("DeletedAt")
.HasColumnType("timestamp with time zone")
.HasColumnName("deleted_at");
b.Property<bool>("HasCompression")
.HasColumnType("boolean")
.HasColumnName("has_compression");
b.Property<bool>("HasThumbnail")
.HasColumnType("boolean")
.HasColumnName("has_thumbnail");
b.Property<string>("Hash")
.HasMaxLength(256)
.HasColumnType("character varying(256)")
.HasColumnName("hash");
b.Property<Dictionary<string, object>>("Meta")
.HasColumnType("jsonb")
.HasColumnName("meta");
b.Property<string>("MimeType")
.HasMaxLength(256)
.HasColumnType("character varying(256)")
.HasColumnName("mime_type");
b.Property<long>("Size")
.HasColumnType("bigint")
.HasColumnName("size");
b.Property<Instant>("UpdatedAt")
.HasColumnType("timestamp with time zone")
.HasColumnName("updated_at");
b.HasKey("Id")
.HasName("pk_file_objects");
b.ToTable("file_objects", (string)null);
});
modelBuilder.Entity("DysonNetwork.Shared.Models.SnFilePermission", b =>
{
b.Property<Guid>("Id")
.ValueGeneratedOnAdd()
.HasColumnType("uuid")
.HasColumnName("id");
b.Property<Instant>("CreatedAt")
.HasColumnType("timestamp with time zone")
.HasColumnName("created_at");
b.Property<Instant?>("DeletedAt")
.HasColumnType("timestamp with time zone")
.HasColumnName("deleted_at");
b.Property<string>("FileId")
.IsRequired()
.HasColumnType("text")
.HasColumnName("file_id");
b.Property<int>("Permission")
.HasColumnType("integer")
.HasColumnName("permission");
b.Property<string>("SubjectId")
.IsRequired()
.HasColumnType("text")
.HasColumnName("subject_id");
b.Property<int>("SubjectType")
.HasColumnType("integer")
.HasColumnName("subject_type");
b.Property<Instant>("UpdatedAt")
.HasColumnType("timestamp with time zone")
.HasColumnName("updated_at");
b.HasKey("Id")
.HasName("pk_file_permissions");
b.ToTable("file_permissions", (string)null);
});
modelBuilder.Entity("DysonNetwork.Shared.Models.SnFileReplica", b =>
{
b.Property<Guid>("Id")
.ValueGeneratedOnAdd()
.HasColumnType("uuid")
.HasColumnName("id");
b.Property<Instant>("CreatedAt")
.HasColumnType("timestamp with time zone")
.HasColumnName("created_at");
b.Property<Instant?>("DeletedAt")
.HasColumnType("timestamp with time zone")
.HasColumnName("deleted_at");
b.Property<bool>("IsPrimary")
.HasColumnType("boolean")
.HasColumnName("is_primary");
b.Property<string>("ObjectId")
.IsRequired()
.HasMaxLength(32)
.HasColumnType("character varying(32)")
.HasColumnName("object_id");
b.Property<Guid>("PoolId")
.HasColumnType("uuid")
.HasColumnName("pool_id");
b.Property<int>("Status")
.HasColumnType("integer")
.HasColumnName("status");
b.Property<string>("StorageId")
.IsRequired()
.HasMaxLength(128)
.HasColumnType("character varying(128)")
.HasColumnName("storage_id");
b.Property<Instant>("UpdatedAt")
.HasColumnType("timestamp with time zone")
.HasColumnName("updated_at");
b.HasKey("Id")
.HasName("pk_file_replicas");
b.HasIndex("ObjectId")
.HasDatabaseName("ix_file_replicas_object_id");
b.HasIndex("PoolId")
.HasDatabaseName("ix_file_replicas_pool_id");
b.ToTable("file_replicas", (string)null);
});
modelBuilder.Entity("DysonNetwork.Drive.Storage.Model.PersistentUploadTask", b => modelBuilder.Entity("DysonNetwork.Drive.Storage.Model.PersistentUploadTask", b =>
{ {
b.HasBaseType("DysonNetwork.Drive.Storage.Model.PersistentTask"); b.HasBaseType("DysonNetwork.Drive.Storage.Model.PersistentTask");
@@ -578,6 +689,11 @@ namespace DysonNetwork.Drive.Migrations
.HasForeignKey("BundleId") .HasForeignKey("BundleId")
.HasConstraintName("fk_files_bundles_bundle_id"); .HasConstraintName("fk_files_bundles_bundle_id");
b.HasOne("DysonNetwork.Shared.Models.SnFileObject", "Object")
.WithMany()
.HasForeignKey("ObjectId")
.HasConstraintName("fk_files_file_objects_object_id");
b.HasOne("DysonNetwork.Shared.Models.FilePool", "Pool") b.HasOne("DysonNetwork.Shared.Models.FilePool", "Pool")
.WithMany() .WithMany()
.HasForeignKey("PoolId") .HasForeignKey("PoolId")
@@ -585,6 +701,8 @@ namespace DysonNetwork.Drive.Migrations
b.Navigation("Bundle"); b.Navigation("Bundle");
b.Navigation("Object");
b.Navigation("Pool"); b.Navigation("Pool");
}); });
@@ -600,29 +718,41 @@ namespace DysonNetwork.Drive.Migrations
b.Navigation("File"); b.Navigation("File");
}); });
modelBuilder.Entity("DysonNetwork.Shared.Models.SnCloudFileReference", b => modelBuilder.Entity("DysonNetwork.Shared.Models.SnFileReplica", b =>
{ {
b.HasOne("DysonNetwork.Shared.Models.SnCloudFile", "File") b.HasOne("DysonNetwork.Shared.Models.SnFileObject", "Object")
.WithMany("References") .WithMany("FileReplicas")
.HasForeignKey("FileId") .HasForeignKey("ObjectId")
.OnDelete(DeleteBehavior.Cascade) .OnDelete(DeleteBehavior.Cascade)
.IsRequired() .IsRequired()
.HasConstraintName("fk_file_references_files_file_id"); .HasConstraintName("fk_file_replicas_file_objects_object_id");
b.Navigation("File"); b.HasOne("DysonNetwork.Shared.Models.FilePool", "Pool")
.WithMany()
.HasForeignKey("PoolId")
.OnDelete(DeleteBehavior.Cascade)
.IsRequired()
.HasConstraintName("fk_file_replicas_pools_pool_id");
b.Navigation("Object");
b.Navigation("Pool");
}); });
modelBuilder.Entity("DysonNetwork.Shared.Models.SnCloudFile", b => modelBuilder.Entity("DysonNetwork.Shared.Models.SnCloudFile", b =>
{ {
b.Navigation("FileIndexes"); b.Navigation("FileIndexes");
b.Navigation("References");
}); });
modelBuilder.Entity("DysonNetwork.Shared.Models.SnFileBundle", b => modelBuilder.Entity("DysonNetwork.Shared.Models.SnFileBundle", b =>
{ {
b.Navigation("Files"); b.Navigation("Files");
}); });
modelBuilder.Entity("DysonNetwork.Shared.Models.SnFileObject", b =>
{
b.Navigation("FileReplicas");
});
#pragma warning restore 612, 618 #pragma warning restore 612, 618
} }
} }

1
DysonNetwork.Drive/Startup/ApplicationBuilderExtensions.cs
View File

@@ -16,7 +16,6 @@ public static class ApplicationBuilderExtensions
{ {
// Map your gRPC services here // Map your gRPC services here
app.MapGrpcService<FileServiceGrpc>(); app.MapGrpcService<FileServiceGrpc>();
app.MapGrpcService<FileReferenceServiceGrpc>();
app.MapGrpcReflectionService(); app.MapGrpcReflectionService();
return app; return app;

7
DysonNetwork.Drive/Startup/ScheduledJobsConfiguration.cs
View File

@@ -29,6 +29,13 @@ public static class ScheduledJobsConfiguration
.ForJob(persistentTaskCleanupJob) .ForJob(persistentTaskCleanupJob)
.WithIdentity("PersistentTaskCleanupTrigger") .WithIdentity("PersistentTaskCleanupTrigger")
.WithCronSchedule("0 0 2 * * ?")); // Run daily at 2 AM .WithCronSchedule("0 0 2 * * ?")); // Run daily at 2 AM
var fileObjectCleanupJob = new JobKey("FileObjectCleanup");
q.AddJob<FileObjectCleanupJob>(opts => opts.WithIdentity(fileObjectCleanupJob));
q.AddTrigger(opts => opts
.ForJob(fileObjectCleanupJob)
.WithIdentity("FileObjectCleanupTrigger")
.WithCronSchedule("0 0 1 * * ?")); // Run daily at 1 AM
}); });
services.AddQuartzHostedService(q => q.WaitForJobsToComplete = true); services.AddQuartzHostedService(q => q.WaitForJobsToComplete = true);

1
DysonNetwork.Drive/Startup/ServiceCollectionExtensions.cs
View File

@@ -56,7 +56,6 @@ public static class ServiceCollectionExtensions
{ {
services.AddScoped<Storage.FileMigrationService>(); services.AddScoped<Storage.FileMigrationService>();
services.AddScoped<Storage.FileService>(); services.AddScoped<Storage.FileService>();
services.AddScoped<Storage.FileReferenceService>();
services.AddScoped<Storage.PersistentTaskService>(); services.AddScoped<Storage.PersistentTaskService>();
services.AddScoped<FileIndexService>(); services.AddScoped<FileIndexService>();
services.AddScoped<Billing.UsageService>(); services.AddScoped<Billing.UsageService>();

11
DysonNetwork.Drive/Storage/CloudFileUnusedRecyclingJob.cs
View File

@@ -47,7 +47,7 @@ public class CloudFileUnusedRecyclingJob(
logger.LogInformation("Found {TotalFiles} files to check for unused status", totalFiles); logger.LogInformation("Found {TotalFiles} files to check for unused status", totalFiles);
// Define a timestamp to limit the age of files we're processing in this run // Define a timestamp to limit the age of files we're processing in this run
// This spreads the processing across multiple job runs for very large databases // This spreads processing across multiple job runs for very large databases
var ageThreshold = now - Duration.FromDays(30); // Process files up to 90 days old in this run var ageThreshold = now - Duration.FromDays(30); // Process files up to 90 days old in this run
// Instead of loading all files at once, use pagination // Instead of loading all files at once, use pagination
@@ -80,13 +80,12 @@ public class CloudFileUnusedRecyclingJob(
processedCount += fileBatch.Count; processedCount += fileBatch.Count;
lastProcessedId = fileBatch.Last(); lastProcessedId = fileBatch.Last();
// Optimized query: Find files that have no references OR all references are expired // Optimized query: Find files that have no other cloud files sharing the same object
// This replaces the memory-intensive approach of loading all references // A file is considered "unused" if no other SnCloudFile shares its ObjectId
var filesToMark = await db.Files var filesToMark = await db.Files
.Where(f => fileBatch.Contains(f.Id)) .Where(f => fileBatch.Contains(f.Id))
.Where(f => !db.FileReferences.Any(r => r.FileId == f.Id) || // No references at all .Where(f => f.ObjectId == null || // No file object at all
!db.FileReferences.Any(r => r.FileId == f.Id && // OR has references but all are expired !db.Files.Any(cf => cf.ObjectId == f.ObjectId && cf.Id != f.Id)) // Or no other files share this object
(r.ExpiredAt == null || r.ExpiredAt > now)))
.Select(f => f.Id) .Select(f => f.Id)
.ToListAsync(); .ToListAsync();

13
DysonNetwork.Drive/Storage/FileController.cs
View File

@@ -14,8 +14,7 @@ public class FileController(
AppDatabase db, AppDatabase db,
FileService fs, FileService fs,
IConfiguration configuration, IConfiguration configuration,
IWebHostEnvironment env, IWebHostEnvironment env
FileReferenceService fileReferenceService
) : ControllerBase ) : ControllerBase
{ {
[HttpGet("{id}")] [HttpGet("{id}")]
@@ -232,17 +231,19 @@ public class FileController(
} }
[HttpGet("{id}/references")] [HttpGet("{id}/references")]
public async Task<ActionResult<List<Shared.Models.SnCloudFileReference>>> GetFileReferences(string id) public async Task<ActionResult<List<SnCloudFile>>> GetFileReferences(string id)
{ {
var file = await fs.GetFileAsync(id); var file = await fs.GetFileAsync(id);
if (file is null) return NotFound("File not found."); if (file is null) return NotFound("File not found.");
// Check if user has access to the file // Check if user has access to
var accessResult = await ValidateFileAccess(file, null); var accessResult = await ValidateFileAccess(file, null);
if (accessResult is not null) return accessResult; if (accessResult is not null) return accessResult;
// Get references using the injected FileReferenceService // Get other cloud files sharing the same object
var references = await fileReferenceService.GetReferencesAsync(id); var references = await db.Files
.Where(f => f.ObjectId == file.ObjectId && f.Id != file.Id)
.ToListAsync();
return Ok(references); return Ok(references);
} }

70
DysonNetwork.Drive/Storage/FileExpirationJob.cs
View File

@@ -1,70 +0,0 @@
using Microsoft.EntityFrameworkCore;
using NodaTime;
using Quartz;
namespace DysonNetwork.Drive.Storage;
/// <summary>
/// Job responsible for cleaning up expired file references
/// </summary>
public class FileExpirationJob(AppDatabase db, FileService fileService, ILogger<FileExpirationJob> logger) : IJob
{
public async Task Execute(IJobExecutionContext context)
{
var now = SystemClock.Instance.GetCurrentInstant();
logger.LogInformation("Running file reference expiration job at {now}", now);
// Delete expired references in bulk and get affected file IDs
var affectedFileIds = await db.FileReferences
.Where(r => r.ExpiredAt < now && r.ExpiredAt != null)
.Select(r => r.FileId)
.Distinct()
.ToListAsync();
if (!affectedFileIds.Any())
{
logger.LogInformation("No expired file references found");
return;
}
logger.LogInformation("Found expired references for {count} files", affectedFileIds.Count);
// Delete expired references in bulk
var deletedReferencesCount = await db.FileReferences
.Where(r => r.ExpiredAt < now && r.ExpiredAt != null)
.ExecuteDeleteAsync();
logger.LogInformation("Deleted {count} expired file references", deletedReferencesCount);
// Find files that now have no remaining references (bulk operation)
var filesToDelete = await db.Files
.Where(f => affectedFileIds.Contains(f.Id))
.Where(f => !db.FileReferences.Any(r => r.FileId == f.Id))
.Select(f => f.Id)
.ToListAsync();
if (filesToDelete.Any())
{
logger.LogInformation("Deleting {count} files that have no remaining references", filesToDelete.Count);
// Get files for deletion
var files = await db.Files
.Where(f => filesToDelete.Contains(f.Id))
.ToListAsync();
// Delete files and their data in parallel
var deleteTasks = files.Select(f => fileService.DeleteFileAsync(f));
await Task.WhenAll(deleteTasks);
}
// Purge cache for files that still have references
var filesWithRemainingRefs = affectedFileIds.Except(filesToDelete).ToList();
if (filesWithRemainingRefs.Any())
{
var cachePurgeTasks = filesWithRemainingRefs.Select(fileService._PurgeCacheAsync);
await Task.WhenAll(cachePurgeTasks);
}
logger.LogInformation("Completed file reference expiration job");
}
}

101
DysonNetwork.Drive/Storage/FileObjectCleanupJob.cs Normal file
View File

@@ -0,0 +1,101 @@
using Microsoft.EntityFrameworkCore;
using Minio.DataModel.Args;
using NodaTime;
using Quartz;
namespace DysonNetwork.Drive.Storage;
/// <summary>
/// Job responsible for cleaning up orphaned file objects
/// When no SnCloudFile references a SnFileObject, the file object is considered orphaned
/// and should be deleted from disk and database
/// </summary>
public class FileObjectCleanupJob(AppDatabase db, FileService fileService, ILogger<FileObjectCleanupJob> logger) : IJob
{
public async Task Execute(IJobExecutionContext context)
{
var now = SystemClock.Instance.GetCurrentInstant();
logger.LogInformation("Running file object cleanup job at {now}", now);
// Find orphaned file objects (objects with no cloud files referencing them)
var referencedObjectIds = await db.Files
.Where(f => f.ObjectId != null)
.Select(f => f.ObjectId)
.Distinct()
.ToListAsync();
var orphanedObjects = await db.FileObjects
.Where(fo => !referencedObjectIds.Contains(fo.Id))
.ToListAsync();
if (!orphanedObjects.Any())
{
logger.LogInformation("No orphaned file objects found");
return;
}
logger.LogInformation("Found {count} orphaned file objects", orphanedObjects.Count);
// Delete orphaned objects and their data
foreach (var fileObject in orphanedObjects)
{
try
{
var replicas = await db.FileReplicas
.Where(r => r.ObjectId == fileObject.Id)
.ToListAsync();
foreach (var replica in replicas)
{
var dest = await fileService.GetRemoteStorageConfig(replica.PoolId);
if (dest != null)
{
var client = fileService.CreateMinioClient(dest);
if (client != null)
{
try
{
await client.RemoveObjectAsync(
new RemoveObjectArgs()
.WithBucket(dest.Bucket)
.WithObject(replica.StorageId)
);
if (fileObject.HasCompression)
{
await client.RemoveObjectAsync(
new RemoveObjectArgs()
.WithBucket(dest.Bucket)
.WithObject(replica.StorageId + ".compressed")
);
}
if (fileObject.HasThumbnail)
{
await client.RemoveObjectAsync(
new RemoveObjectArgs()
.WithBucket(dest.Bucket)
.WithObject(replica.StorageId + ".thumbnail")
);
}
}
catch (Exception ex)
{
logger.LogError(ex, "Failed to delete orphaned file object {ObjectId} from remote storage", fileObject.Id);
}
}
}
}
db.FileReplicas.RemoveRange(replicas);
db.FileObjects.Remove(fileObject);
await db.SaveChangesAsync();
logger.LogInformation("Deleted orphaned file object {ObjectId}", fileObject.Id);
}
catch (Exception ex)
{
logger.LogError(ex, "Failed to clean up orphaned file object {ObjectId}", fileObject.Id);
}
}
logger.LogInformation("Completed file object cleanup job");
}
}

532
DysonNetwork.Drive/Storage/FileReferenceService.cs
View File

@@ -1,532 +0,0 @@
using DysonNetwork.Shared.Cache;
using DysonNetwork.Shared.Data;
using DysonNetwork.Shared.Models;
using Microsoft.EntityFrameworkCore;
using NodaTime;
namespace DysonNetwork.Drive.Storage;
public class FileReferenceService(AppDatabase db, FileService fileService, ICacheService cache)
{
private const string CacheKeyPrefix = "file:ref:";
private static readonly TimeSpan CacheDuration = TimeSpan.FromMinutes(15);
/// <summary>
/// Creates a new reference to a file for a specific resource
/// </summary>
/// <param name="fileId">The ID of the file to reference</param>
/// <param name="usage">The usage context (e.g., "avatar", "post-attachment")</param>
/// <param name="resourceId">The ID of the resource using the file</param>
/// <param name="expiredAt">Optional expiration time for the file</param>
/// <param name="duration">Optional duration after which the file expires (alternative to expiredAt)</param>
/// <returns>The created file reference</returns>
public async Task<SnCloudFileReference> CreateReferenceAsync(
string fileId,
string usage,
string resourceId,
Instant? expiredAt = null,
Duration? duration = null
)
{
// Calculate expiration time if needed
var finalExpiration = expiredAt;
if (duration.HasValue)
finalExpiration = SystemClock.Instance.GetCurrentInstant() + duration.Value;
var reference = new SnCloudFileReference
{
FileId = fileId,
Usage = usage,
ResourceId = resourceId,
ExpiredAt = finalExpiration
};
db.FileReferences.Add(reference);
await db.SaveChangesAsync();
await fileService._PurgeCacheAsync(fileId);
return reference;
}
public async Task<List<SnCloudFileReference>> CreateReferencesAsync(
List<string> fileId,
string usage,
string resourceId,
Instant? expiredAt = null,
Duration? duration = null
)
{
var now = SystemClock.Instance.GetCurrentInstant();
var finalExpiredAt = expiredAt;
if (finalExpiredAt == null && duration.HasValue)
{
finalExpiredAt = now + duration.Value;
}
var data = fileId.Select(id => new SnCloudFileReference
{
FileId = id,
Usage = usage,
ResourceId = resourceId,
ExpiredAt = finalExpiredAt,
CreatedAt = now,
UpdatedAt = now
})
.ToList();
db.FileReferences.AddRange(data);
await db.SaveChangesAsync();
return data;
}
/// <summary>
/// Gets all references to a file
/// </summary>
/// <param name="fileId">The ID of the file</param>
/// <returns>A list of all references to the file</returns>
public async Task<List<SnCloudFileReference>> GetReferencesAsync(string fileId)
{
var cacheKey = $"{CacheKeyPrefix}list:{fileId}";
var cachedReferences = await cache.GetAsync<List<SnCloudFileReference>>(cacheKey);
if (cachedReferences is not null)
return cachedReferences;
var references = await db.FileReferences
.Where(r => r.FileId == fileId)
.ToListAsync();
await cache.SetAsync(cacheKey, references, CacheDuration);
return references;
}
public async Task<Dictionary<string, List<SnCloudFileReference>>> GetReferencesAsync(IEnumerable<string> fileIds)
{
var fileIdList = fileIds.ToList();
var result = new Dictionary<string, List<SnCloudFileReference>>();
// Check cache for each file ID
var uncachedFileIds = new List<string>();
foreach (var fileId in fileIdList)
{
var cacheKey = $"{CacheKeyPrefix}list:{fileId}";
var cachedReferences = await cache.GetAsync<List<SnCloudFileReference>>(cacheKey);
if (cachedReferences is not null)
{
result[fileId] = cachedReferences;
}
else
{
uncachedFileIds.Add(fileId);
}
}
// Fetch uncached references from database
if (uncachedFileIds.Any())
{
var dbReferences = await db.FileReferences
.Where(r => uncachedFileIds.Contains(r.FileId))
.GroupBy(r => r.FileId)
.ToDictionaryAsync(r => r.Key, r => r.ToList());
// Cache the results
foreach (var kvp in dbReferences)
{
var cacheKey = $"{CacheKeyPrefix}list:{kvp.Key}";
await cache.SetAsync(cacheKey, kvp.Value, CacheDuration);
result[kvp.Key] = kvp.Value;
}
}
return result;
}
/// <summary>
/// Gets the number of references to a file
/// </summary>
/// <param name="fileId">The ID of the file</param>
/// <returns>The number of references to the file</returns>
public async Task<int> GetReferenceCountAsync(string fileId)
{
var cacheKey = $"{CacheKeyPrefix}count:{fileId}";
var cachedCount = await cache.GetAsync<int?>(cacheKey);
if (cachedCount.HasValue)
return cachedCount.Value;
var count = await db.FileReferences
.Where(r => r.FileId == fileId)
.CountAsync();
await cache.SetAsync(cacheKey, count, CacheDuration);
return count;
}
/// <summary>
/// Gets all references for a specific resource
/// </summary>
/// <param name="resourceId">The ID of the resource</param>
/// <returns>A list of file references associated with the resource</returns>
public async Task<List<SnCloudFileReference>> GetResourceReferencesAsync(string resourceId)
{
var cacheKey = $"{CacheKeyPrefix}resource:{resourceId}";
var cachedReferences = await cache.GetAsync<List<SnCloudFileReference>>(cacheKey);
if (cachedReferences is not null)
return cachedReferences;
var references = await db.FileReferences
.Where(r => r.ResourceId == resourceId)
.ToListAsync();
await cache.SetAsync(cacheKey, references, CacheDuration);
return references;
}
/// <summary>
/// Gets all file references for a specific usage context
/// </summary>
/// <param name="usage">The usage context</param>
/// <returns>A list of file references with the specified usage</returns>
public async Task<List<SnCloudFileReference>> GetUsageReferencesAsync(string usage)
{
var cacheKey = $"{CacheKeyPrefix}usage:{usage}";
var cachedReferences = await cache.GetAsync<List<SnCloudFileReference>>(cacheKey);
if (cachedReferences is not null)
return cachedReferences;
var references = await db.FileReferences
.Where(r => r.Usage == usage)
.ToListAsync();
await cache.SetAsync(cacheKey, references, CacheDuration);
return references;
}
/// <summary>
/// Deletes references for a specific resource
/// </summary>
/// <param name="resourceId">The ID of the resource</param>
/// <returns>The number of deleted references</returns>
public async Task<int> DeleteResourceReferencesAsync(string resourceId)
{
var references = await db.FileReferences
.Where(r => r.ResourceId == resourceId)
.ToListAsync();
var fileIds = references.Select(r => r.FileId).Distinct().ToList();
db.FileReferences.RemoveRange(references);
var deletedCount = await db.SaveChangesAsync();
// Purge caches
var tasks = fileIds.Select(fileService._PurgeCacheAsync).ToList();
tasks.Add(PurgeCacheForResourceAsync(resourceId));
await Task.WhenAll(tasks);
return deletedCount;
}
/// <summary>
/// Deletes references for a specific resource and usage
/// </summary>
/// <param name="resourceId">The ID of the resource</param>
/// <param name="usage">The usage context</param>
/// <returns>The number of deleted references</returns>
public async Task<int> DeleteResourceReferencesAsync(string resourceId, string usage)
{
var references = await db.FileReferences
.Where(r => r.ResourceId == resourceId && r.Usage == usage)
.ToListAsync();
if (references.Count == 0)
return 0;
var fileIds = references.Select(r => r.FileId).Distinct().ToList();
db.FileReferences.RemoveRange(references);
var deletedCount = await db.SaveChangesAsync();
// Purge caches
var tasks = fileIds.Select(fileService._PurgeCacheAsync).ToList();
tasks.Add(PurgeCacheForResourceAsync(resourceId));
await Task.WhenAll(tasks);
return deletedCount;
}
public async Task<int> DeleteResourceReferencesBatchAsync(IEnumerable<string> resourceIds, string? usage = null)
{
var resourceIdList = resourceIds.ToList();
var references = await db.FileReferences
.Where(r => resourceIdList.Contains(r.ResourceId))
.If(usage != null, q => q.Where(q => q.Usage == usage))
.ToListAsync();
if (references.Count == 0)
return 0;
var fileIds = references.Select(r => r.FileId).Distinct().ToList();
db.FileReferences.RemoveRange(references);
var deletedCount = await db.SaveChangesAsync();
// Purge caches for files and resources
var tasks = fileIds.Select(fileService._PurgeCacheAsync).ToList();
tasks.AddRange(resourceIdList.Select(PurgeCacheForResourceAsync));
await Task.WhenAll(tasks);
return deletedCount;
}
/// <summary>
/// Deletes a specific file reference
/// </summary>
/// <param name="referenceId">The ID of the reference to delete</param>
/// <returns>True if the reference was deleted, false otherwise</returns>
public async Task<bool> DeleteReferenceAsync(Guid referenceId)
{
var reference = await db.FileReferences
.FirstOrDefaultAsync(r => r.Id == referenceId);
if (reference == null)
return false;
db.FileReferences.Remove(reference);
await db.SaveChangesAsync();
// Purge caches
await fileService._PurgeCacheAsync(reference.FileId);
await PurgeCacheForResourceAsync(reference.ResourceId);
await PurgeCacheForFileAsync(reference.FileId);
return true;
}
/// <summary>
/// Updates the files referenced by a resource
/// </summary>
/// <param name="resourceId">The ID of the resource</param>
/// <param name="newFileIds">The new list of file IDs</param>
/// <param name="usage">The usage context</param>
/// <param name="expiredAt">Optional expiration time for newly added files</param>
/// <param name="duration">Optional duration after which newly added files expire</param>
/// <returns>A list of the updated file references</returns>
public async Task<List<SnCloudFileReference>> UpdateResourceFilesAsync(
string resourceId,
IEnumerable<string>? newFileIds,
string usage,
Instant? expiredAt = null,
Duration? duration = null)
{
if (newFileIds == null)
return new List<SnCloudFileReference>();
var existingReferences = await db.FileReferences
.Where(r => r.ResourceId == resourceId && r.Usage == usage)
.ToListAsync();
var existingFileIds = existingReferences.Select(r => r.FileId).ToHashSet();
var newFileIdsList = newFileIds.ToList();
var newFileIdsSet = newFileIdsList.ToHashSet();
// Files to remove
var toRemove = existingReferences
.Where(r => !newFileIdsSet.Contains(r.FileId))
.ToList();
// Files to add
var toAdd = newFileIdsList
.Where(id => !existingFileIds.Contains(id))
.Select(id => new SnCloudFileReference
{
FileId = id,
Usage = usage,
ResourceId = resourceId
})
.ToList();
// Apply changes
if (toRemove.Any())
db.FileReferences.RemoveRange(toRemove);
if (toAdd.Any())
db.FileReferences.AddRange(toAdd);
await db.SaveChangesAsync();
// Update expiration for newly added references if specified
if ((expiredAt.HasValue || duration.HasValue) && toAdd.Any())
{
var finalExpiration = expiredAt;
if (duration.HasValue)
{
finalExpiration = SystemClock.Instance.GetCurrentInstant() + duration.Value;
}
// Update newly added references with the expiration time
var referenceIds = await db.FileReferences
.Where(r => toAdd.Select(a => a.FileId).Contains(r.FileId) &&
r.ResourceId == resourceId &&
r.Usage == usage)
.Select(r => r.Id)
.ToListAsync();
await db.FileReferences
.Where(r => referenceIds.Contains(r.Id))
.ExecuteUpdateAsync(setter => setter.SetProperty(
r => r.ExpiredAt,
_ => finalExpiration
));
}
// Purge caches
var allFileIds = existingFileIds.Union(newFileIdsSet).ToList();
var tasks = allFileIds.Select(fileService._PurgeCacheAsync).ToList();
tasks.Add(PurgeCacheForResourceAsync(resourceId));
await Task.WhenAll(tasks);
// Return updated references
return await db.FileReferences
.Where(r => r.ResourceId == resourceId && r.Usage == usage)
.ToListAsync();
}
/// <summary>
/// Gets all files referenced by a resource
/// </summary>
/// <param name="resourceId">The ID of the resource</param>
/// <param name="usage">Optional filter by usage context</param>
/// <returns>A list of files referenced by the resource</returns>
public async Task<List<SnCloudFile>> GetResourceFilesAsync(string resourceId, string? usage = null)
{
var query = db.FileReferences.Where(r => r.ResourceId == resourceId);
if (usage != null)
query = query.Where(r => r.Usage == usage);
var references = await query.ToListAsync();
var fileIds = references.Select(r => r.FileId).ToList();
return await db.Files
.Where(f => fileIds.Contains(f.Id))
.ToListAsync();
}
/// <summary>
/// Purges all caches related to a resource
/// </summary>
private async Task PurgeCacheForResourceAsync(string resourceId)
{
var cacheKey = $"{CacheKeyPrefix}resource:{resourceId}";
await cache.RemoveAsync(cacheKey);
}
/// <summary>
/// Purges all caches related to a file
/// </summary>
private async Task PurgeCacheForFileAsync(string fileId)
{
var cacheKeys = new[]
{
$"{CacheKeyPrefix}list:{fileId}",
$"{CacheKeyPrefix}count:{fileId}"
};
var tasks = cacheKeys.Select(cache.RemoveAsync);
await Task.WhenAll(tasks);
}
/// <summary>
/// Updates the expiration time for a file reference
/// </summary>
/// <param name="referenceId">The ID of the reference</param>
/// <param name="expiredAt">The new expiration time, or null to remove expiration</param>
/// <returns>True if the reference was found and updated, false otherwise</returns>
public async Task<bool> SetReferenceExpirationAsync(Guid referenceId, Instant? expiredAt)
{
var reference = await db.FileReferences
.FirstOrDefaultAsync(r => r.Id == referenceId);
if (reference == null)
return false;
reference.ExpiredAt = expiredAt;
await db.SaveChangesAsync();
await PurgeCacheForFileAsync(reference.FileId);
await PurgeCacheForResourceAsync(reference.ResourceId);
return true;
}
/// <summary>
/// Updates the expiration time for all references to a file
/// </summary>
/// <param name="fileId">The ID of the file</param>
/// <param name="expiredAt">The new expiration time, or null to remove expiration</param>
/// <returns>The number of references updated</returns>
public async Task<int> SetFileReferencesExpirationAsync(string fileId, Instant? expiredAt)
{
var rowsAffected = await db.FileReferences
.Where(r => r.FileId == fileId)
.ExecuteUpdateAsync(setter => setter.SetProperty(
r => r.ExpiredAt,
_ => expiredAt
));
if (rowsAffected > 0)
{
await fileService._PurgeCacheAsync(fileId);
await PurgeCacheForFileAsync(fileId);
}
return rowsAffected;
}
/// <summary>
/// Get all file references for a specific resource and usage type
/// </summary>
/// <param name="resourceId">The resource ID</param>
/// <param name="usageType">The usage type</param>
/// <returns>List of file references</returns>
public async Task<List<SnCloudFileReference>> GetResourceReferencesAsync(string resourceId, string usageType)
{
return await db.FileReferences
.Where(r => r.ResourceId == resourceId && r.Usage == usageType)
.ToListAsync();
}
/// <summary>
/// Check if a file has any references
/// </summary>
/// <param name="fileId">The file ID to check</param>
/// <returns>True if the file has references, false otherwise</returns>
public async Task<bool> HasFileReferencesAsync(string fileId)
{
return await db.FileReferences.AnyAsync(r => r.FileId == fileId);
}
/// <summary>
/// Updates the expiration time for a file reference using a duration from now
/// </summary>
/// <param name="referenceId">The ID of the reference</param>
/// <param name="duration">The duration after which the reference expires, or null to remove expiration</param>
/// <returns>True if the reference was found and updated, false otherwise</returns>
public async Task<bool> SetReferenceExpirationDurationAsync(Guid referenceId, Duration? duration)
{
Instant? expiredAt = null;
if (duration.HasValue)
{
expiredAt = SystemClock.Instance.GetCurrentInstant() + duration.Value;
}
return await SetReferenceExpirationAsync(referenceId, expiredAt);
}
}

174
DysonNetwork.Drive/Storage/FileReferenceServiceGrpc.cs
View File

@@ -1,174 +0,0 @@
using DysonNetwork.Shared.Proto;
using Grpc.Core;
using NodaTime;
using Duration = NodaTime.Duration;
namespace DysonNetwork.Drive.Storage;
public class FileReferenceServiceGrpc(FileReferenceService fileReferenceService)
: Shared.Proto.FileReferenceService.FileReferenceServiceBase
{
public override async Task<Shared.Proto.CloudFileReference> CreateReference(CreateReferenceRequest request,
ServerCallContext context)
{
Instant? expiredAt = null;
if (request.ExpiredAt != null)
expiredAt = Instant.FromUnixTimeSeconds(request.ExpiredAt.Seconds);
else if (request.Duration != null)
expiredAt = SystemClock.Instance.GetCurrentInstant() +
Duration.FromTimeSpan(request.Duration.ToTimeSpan());
var reference = await fileReferenceService.CreateReferenceAsync(
request.FileId,
request.Usage,
request.ResourceId,
expiredAt
);
return reference.ToProtoValue();
}
public override async Task<CreateReferenceBatchResponse> CreateReferenceBatch(CreateReferenceBatchRequest request,
ServerCallContext context)
{
Instant? expiredAt = null;
if (request.ExpiredAt != null)
expiredAt = Instant.FromUnixTimeSeconds(request.ExpiredAt.Seconds);
else if (request.Duration != null)
expiredAt = SystemClock.Instance.GetCurrentInstant() +
Duration.FromTimeSpan(request.Duration.ToTimeSpan());
var references = await fileReferenceService.CreateReferencesAsync(
request.FilesId.ToList(),
request.Usage,
request.ResourceId,
expiredAt
);
var response = new CreateReferenceBatchResponse();
response.References.AddRange(references.Select(r => r.ToProtoValue()));
return response;
}
public override async Task<GetReferencesResponse> GetReferences(GetReferencesRequest request,
ServerCallContext context)
{
var references = await fileReferenceService.GetReferencesAsync(request.FileId);
var response = new GetReferencesResponse();
response.References.AddRange(references.Select(r => r.ToProtoValue()));
return response;
}
public override async Task<GetReferenceCountResponse> GetReferenceCount(GetReferenceCountRequest request,
ServerCallContext context)
{
var count = await fileReferenceService.GetReferenceCountAsync(request.FileId);
return new GetReferenceCountResponse { Count = count };
}
public override async Task<GetReferencesResponse> GetResourceReferences(GetResourceReferencesRequest request,
ServerCallContext context)
{
var references = await fileReferenceService.GetResourceReferencesAsync(request.ResourceId, request.Usage);
var response = new GetReferencesResponse();
response.References.AddRange(references.Select(r => r.ToProtoValue()));
return response;
}
public override async Task<GetResourceFilesResponse> GetResourceFiles(GetResourceFilesRequest request,
ServerCallContext context)
{
var files = await fileReferenceService.GetResourceFilesAsync(request.ResourceId, request.Usage);
var response = new GetResourceFilesResponse();
response.Files.AddRange(files.Select(f => f.ToProtoValue()));
return response;
}
public override async Task<DeleteResourceReferencesResponse> DeleteResourceReferences(
DeleteResourceReferencesRequest request, ServerCallContext context)
{
int deletedCount;
if (request.Usage is null)
deletedCount = await fileReferenceService.DeleteResourceReferencesAsync(request.ResourceId);
else
deletedCount =
await fileReferenceService.DeleteResourceReferencesAsync(request.ResourceId, request.Usage!);
return new DeleteResourceReferencesResponse { DeletedCount = deletedCount };
}
public override async Task<DeleteResourceReferencesResponse> DeleteResourceReferencesBatch(DeleteResourceReferencesBatchRequest request, ServerCallContext context)
{
var resourceIds = request.ResourceIds.ToList();
int deletedCount;
if (request.Usage is null)
deletedCount = await fileReferenceService.DeleteResourceReferencesBatchAsync(resourceIds);
else
deletedCount =
await fileReferenceService.DeleteResourceReferencesBatchAsync(resourceIds, request.Usage!);
return new DeleteResourceReferencesResponse { DeletedCount = deletedCount };
}
public override async Task<DeleteReferenceResponse> DeleteReference(DeleteReferenceRequest request,
ServerCallContext context)
{
var success = await fileReferenceService.DeleteReferenceAsync(Guid.Parse(request.ReferenceId));
return new DeleteReferenceResponse { Success = success };
}
public override async Task<UpdateResourceFilesResponse> UpdateResourceFiles(UpdateResourceFilesRequest request,
ServerCallContext context)
{
Instant? expiredAt = null;
if (request.ExpiredAt != null)
{
expiredAt = Instant.FromUnixTimeSeconds(request.ExpiredAt.Seconds);
}
else if (request.Duration != null)
{
expiredAt = SystemClock.Instance.GetCurrentInstant() +
Duration.FromTimeSpan(request.Duration.ToTimeSpan());
}
var references = await fileReferenceService.UpdateResourceFilesAsync(
request.ResourceId,
request.FileIds,
request.Usage,
expiredAt
);
var response = new UpdateResourceFilesResponse();
response.References.AddRange(references.Select(r => r.ToProtoValue()));
return response;
}
public override async Task<SetReferenceExpirationResponse> SetReferenceExpiration(
SetReferenceExpirationRequest request, ServerCallContext context)
{
Instant? expiredAt = null;
if (request.ExpiredAt != null)
{
expiredAt = Instant.FromUnixTimeSeconds(request.ExpiredAt.Seconds);
}
else if (request.Duration != null)
{
expiredAt = SystemClock.Instance.GetCurrentInstant() +
Duration.FromTimeSpan(request.Duration.ToTimeSpan());
}
var success =
await fileReferenceService.SetReferenceExpirationAsync(Guid.Parse(request.ReferenceId), expiredAt);
return new SetReferenceExpirationResponse { Success = success };
}
public override async Task<SetFileReferencesExpirationResponse> SetFileReferencesExpiration(
SetFileReferencesExpirationRequest request, ServerCallContext context)
{
var expiredAt = Instant.FromUnixTimeSeconds(request.ExpiredAt.Seconds);
var updatedCount = await fileReferenceService.SetFileReferencesExpirationAsync(request.FileId, expiredAt);
return new SetFileReferencesExpirationResponse { UpdatedCount = updatedCount };
}
public override async Task<HasFileReferencesResponse> HasFileReferences(HasFileReferencesRequest request,
ServerCallContext context)
{
var hasReferences = await fileReferenceService.HasFileReferencesAsync(request.FileId);
return new HasFileReferencesResponse { HasReferences = hasReferences };
}
}

130
DysonNetwork.Drive/Storage/FileService.cs
View File

@@ -40,6 +40,8 @@ public class FileService(
.Where(f => f.Id == fileId) .Where(f => f.Id == fileId)
.Include(f => f.Pool) .Include(f => f.Pool)
.Include(f => f.Bundle) .Include(f => f.Bundle)
.Include(f => f.Object)
.ThenInclude(o => o.FileReplicas)
.FirstOrDefaultAsync(); .FirstOrDefaultAsync();
if (file != null) if (file != null)
@@ -69,6 +71,8 @@ public class FileService(
var dbFiles = await db.Files var dbFiles = await db.Files
.Where(f => uncachedIds.Contains(f.Id)) .Where(f => uncachedIds.Contains(f.Id))
.Include(f => f.Pool) .Include(f => f.Pool)
.Include(f => f.Object)
.ThenInclude(o => o.FileReplicas)
.ToListAsync(); .ToListAsync();
foreach (var file in dbFiles) foreach (var file in dbFiles)
@@ -228,8 +232,34 @@ public class FileService(
private async Task SaveFileToDatabaseAsync(SnCloudFile file) private async Task SaveFileToDatabaseAsync(SnCloudFile file)
{ {
var fileObject = new SnFileObject
{
Id = file.Id,
AccountId = file.AccountId,
Size = file.Size,
Meta = file.FileMeta,
MimeType = file.MimeType,
Hash = file.Hash,
HasCompression = file.HasCompression,
HasThumbnail = file.HasThumbnail
};
var replica = new SnFileReplica
{
Id = Guid.NewGuid(),
ObjectId = file.Id,
PoolId = file.PoolId!.Value,
StorageId = file.StorageId ?? file.Id,
Status = SnFileReplicaStatus.Available,
IsPrimary = true
};
db.Files.Add(file); db.Files.Add(file);
db.FileObjects.Add(fileObject);
db.FileReplicas.Add(replica);
await db.SaveChangesAsync(); await db.SaveChangesAsync();
file.ObjectId = file.Id;
file.StorageId ??= file.Id; file.StorageId ??= file.Id;
} }
@@ -470,8 +500,20 @@ public class FileService(
await db.Files.Where(f => f.Id == file.Id).ExecuteUpdateAsync(updatable.ToSetPropertyCalls()); await db.Files.Where(f => f.Id == file.Id).ExecuteUpdateAsync(updatable.ToSetPropertyCalls());
if (updateMask.Paths.Contains("file_meta"))
{
await db.FileObjects
.Where(fo => fo.Id == file.ObjectId)
.ExecuteUpdateAsync(setter => setter
.SetProperty(fo => fo.Meta, file.FileMeta));
}
await _PurgeCacheAsync(file.Id); await _PurgeCacheAsync(file.Id);
return await db.Files.AsNoTracking().FirstAsync(f => f.Id == file.Id); return await db.Files
.AsNoTracking()
.Include(f => f.Object)
.ThenInclude(o => o.FileReplicas)
.FirstAsync(f => f.Id == file.Id);
} }
public async Task DeleteFileAsync(SnCloudFile file, bool skipData = false) public async Task DeleteFileAsync(SnCloudFile file, bool skipData = false)
@@ -481,17 +523,23 @@ public class FileService(
await _PurgeCacheAsync(file.Id); await _PurgeCacheAsync(file.Id);
if (!skipData) if (!skipData)
await DeleteFileDataAsync(file); {
var hasOtherReferences = await db.Files
.AnyAsync(f => f.ObjectId == file.ObjectId && f.Id != file.Id);
if (!hasOtherReferences)
await DeleteFileDataAsync(file);
}
} }
public async Task DeleteFileDataAsync(SnCloudFile file, bool force = false) public async Task DeleteFileDataAsync(SnCloudFile file, bool force = false)
{ {
if (!file.PoolId.HasValue) return; if (!file.PoolId.HasValue || file.ObjectId == null) return;
if (!force) if (!force)
{ {
var sameOriginFiles = await db.Files var sameOriginFiles = await db.Files
.Where(f => f.StorageId == file.StorageId && f.Id != file.Id) .Where(f => f.ObjectId == file.ObjectId && f.Id != file.Id)
.Select(f => f.Id) .Select(f => f.Id)
.ToListAsync(); .ToListAsync();
@@ -499,6 +547,17 @@ public class FileService(
return; return;
} }
var replicas = await db.FileReplicas
.Where(r => r.ObjectId == file.ObjectId)
.ToListAsync();
if (replicas.Count == 0)
{
logger.LogWarning("No replicas found for file object {ObjectId}", file.ObjectId);
return;
}
var primaryReplica = replicas.First(r => r.IsPrimary);
var dest = await GetRemoteStorageConfig(file.PoolId.Value); var dest = await GetRemoteStorageConfig(file.PoolId.Value);
if (dest is null) throw new InvalidOperationException($"No remote storage configured for pool {file.PoolId}"); if (dest is null) throw new InvalidOperationException($"No remote storage configured for pool {file.PoolId}");
var client = CreateMinioClient(dest); var client = CreateMinioClient(dest);
@@ -508,7 +567,7 @@ public class FileService(
); );
var bucket = dest.Bucket; var bucket = dest.Bucket;
var objectId = file.StorageId ?? file.Id; var objectId = primaryReplica.StorageId;
await client.RemoveObjectAsync( await client.RemoveObjectAsync(
new RemoveObjectArgs().WithBucket(bucket).WithObject(objectId) new RemoveObjectArgs().WithBucket(bucket).WithObject(objectId)
@@ -541,36 +600,55 @@ public class FileService(
logger.LogWarning("Failed to delete thumbnail of file {fileId}", file.Id); logger.LogWarning("Failed to delete thumbnail of file {fileId}", file.Id);
} }
} }
db.FileReplicas.RemoveRange(replicas);
var fileObject = await db.FileObjects.FindAsync(file.ObjectId);
if (fileObject != null) db.FileObjects.Remove(fileObject);
await db.SaveChangesAsync();
} }
public async Task DeleteFileDataBatchAsync(List<SnCloudFile> files) public async Task DeleteFileDataBatchAsync(List<SnCloudFile> files)
{ {
files = files.Where(f => f.PoolId.HasValue).ToList(); files = files.Where(f => f.PoolId.HasValue && f.ObjectId != null).ToList();
foreach (var fileGroup in files.GroupBy(f => f.PoolId!.Value)) var objectIds = files.Select(f => f.ObjectId).Distinct().ToList();
var replicas = await db.FileReplicas
.Where(r => objectIds.Contains(r.ObjectId))
.ToListAsync();
foreach (var poolGroup in replicas.GroupBy(r => r.PoolId))
{ {
var dest = await GetRemoteStorageConfig(fileGroup.Key); var dest = await GetRemoteStorageConfig(poolGroup.Key);
if (dest is null) if (dest is null)
throw new InvalidOperationException($"No remote storage configured for pool {fileGroup.Key}"); throw new InvalidOperationException($"No remote storage configured for pool {poolGroup.Key}");
var client = CreateMinioClient(dest); var client = CreateMinioClient(dest);
if (client is null) if (client is null)
throw new InvalidOperationException( throw new InvalidOperationException(
$"Failed to configure client for remote destination '{fileGroup.Key}'" $"Failed to configure client for remote destination '{poolGroup.Key}'"
); );
List<string> objectsToDelete = []; List<string> objectsToDelete = [];
foreach (var file in fileGroup) foreach (var replica in poolGroup)
{ {
objectsToDelete.Add(file.StorageId ?? file.Id); var file = files.First(f => f.ObjectId == replica.ObjectId);
if (file.HasCompression) objectsToDelete.Add(file.StorageId ?? file.Id + ".compressed"); objectsToDelete.Add(replica.StorageId);
if (file.HasThumbnail) objectsToDelete.Add(file.StorageId ?? file.Id + ".thumbnail"); if (file.HasCompression) objectsToDelete.Add(replica.StorageId + ".compressed");
if (file.HasThumbnail) objectsToDelete.Add(replica.StorageId + ".thumbnail");
} }
await client.RemoveObjectsAsync( await client.RemoveObjectsAsync(
new RemoveObjectsArgs().WithBucket(dest.Bucket).WithObjects(objectsToDelete) new RemoveObjectsArgs().WithBucket(dest.Bucket).WithObjects(objectsToDelete)
); );
db.FileReplicas.RemoveRange(poolGroup);
} }
var fileObjects = await db.FileObjects
.Where(fo => objectIds.Contains(fo.Id))
.ToListAsync();
db.FileObjects.RemoveRange(fileObjects);
await db.SaveChangesAsync();
} }
private async Task<SnFileBundle?> GetBundleAsync(Guid id, Guid accountId) private async Task<SnFileBundle?> GetBundleAsync(Guid id, Guid accountId)
@@ -654,6 +732,8 @@ public class FileService(
{ {
var dbFiles = await db.Files var dbFiles = await db.Files
.Where(f => uncachedIds.Contains(f.Id)) .Where(f => uncachedIds.Contains(f.Id))
.Include(f => f.Object)
.ThenInclude(o => o.FileReplicas)
.ToListAsync(); .ToListAsync();
foreach (var file in dbFiles) foreach (var file in dbFiles)
@@ -674,15 +754,21 @@ public class FileService(
public async Task<int> GetReferenceCountAsync(string fileId) public async Task<int> GetReferenceCountAsync(string fileId)
{ {
return await db.FileReferences var file = await db.Files.FirstOrDefaultAsync(f => f.Id == fileId);
.Where(r => r.FileId == fileId) if (file == null || file.ObjectId == null) return 0;
return await db.Files
.Where(f => f.ObjectId == file.ObjectId && f.Id != fileId)
.CountAsync(); .CountAsync();
} }
public async Task<bool> IsReferencedAsync(string fileId) public async Task<bool> IsReferencedAsync(string fileId)
{ {
return await db.FileReferences var file = await db.Files.FirstOrDefaultAsync(f => f.Id == fileId);
.Where(r => r.FileId == fileId) if (file == null || file.ObjectId == null) return false;
return await db.Files
.Where(f => f.ObjectId == file.ObjectId && f.Id != fileId)
.AnyAsync(); .AnyAsync();
} }
@@ -709,8 +795,6 @@ public class FileService(
.Where(f => f.AccountId == accountId && f.IsMarkedRecycle) .Where(f => f.AccountId == accountId && f.IsMarkedRecycle)
.ToListAsync(); .ToListAsync();
var count = files.Count; var count = files.Count;
var tasks = files.Select(f => DeleteFileDataAsync(f, true));
await Task.WhenAll(tasks);
var fileIds = files.Select(f => f.Id).ToList(); var fileIds = files.Select(f => f.Id).ToList();
await _PurgeCacheRangeAsync(fileIds); await _PurgeCacheRangeAsync(fileIds);
db.RemoveRange(files); db.RemoveRange(files);
@@ -724,8 +808,6 @@ public class FileService(
.Where(f => f.AccountId == accountId && fileIds.Contains(f.Id)) .Where(f => f.AccountId == accountId && fileIds.Contains(f.Id))
.ToListAsync(); .ToListAsync();
var count = files.Count; var count = files.Count;
var tasks = files.Select(f => DeleteFileDataAsync(f, true));
await Task.WhenAll(tasks);
var fileIdsList = files.Select(f => f.Id).ToList(); var fileIdsList = files.Select(f => f.Id).ToList();
await _PurgeCacheRangeAsync(fileIdsList); await _PurgeCacheRangeAsync(fileIdsList);
db.RemoveRange(files); db.RemoveRange(files);
@@ -739,8 +821,6 @@ public class FileService(
.Where(f => f.PoolId == poolId && f.IsMarkedRecycle) .Where(f => f.PoolId == poolId && f.IsMarkedRecycle)
.ToListAsync(); .ToListAsync();
var count = files.Count; var count = files.Count;
var tasks = files.Select(f => DeleteFileDataAsync(f, true));
await Task.WhenAll(tasks);
var fileIds = files.Select(f => f.Id).ToList(); var fileIds = files.Select(f => f.Id).ToList();
await _PurgeCacheRangeAsync(fileIds); await _PurgeCacheRangeAsync(fileIds);
db.RemoveRange(files); db.RemoveRange(files);
@@ -754,8 +834,6 @@ public class FileService(
.Where(f => f.IsMarkedRecycle) .Where(f => f.IsMarkedRecycle)
.ToListAsync(); .ToListAsync();
var count = files.Count; var count = files.Count;
var tasks = files.Select(f => DeleteFileDataAsync(f, true));
await Task.WhenAll(tasks);
var fileIds = files.Select(f => f.Id).ToList(); var fileIds = files.Select(f => f.Id).ToList();
await _PurgeCacheRangeAsync(fileIds); await _PurgeCacheRangeAsync(fileIds);
db.RemoveRange(files); db.RemoveRange(files);

35
DysonNetwork.Shared/Models/CloudFile.cs
View File

@@ -60,9 +60,7 @@ public class SnCloudFile : ModelBase, ICloudFile, IIdentifiedResource
[JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)]
public string? FastUploadLink { get; set; } public string? FastUploadLink { get; set; }
public List<SnCloudFileReference> References { get; set; } = new List<SnCloudFileReference>(); public Guid AccountId { get; set; }
public Guid AccountId { get; set; }
public SnCloudFileReferenceObject ToReferenceObject() public SnCloudFileReferenceObject ToReferenceObject()
{ {
@@ -112,34 +110,3 @@ public class SnCloudFile : ModelBase, ICloudFile, IIdentifiedResource
return proto; return proto;
} }
} }
public class SnCloudFileReference : ModelBase
{
public Guid Id { get; set; } = Guid.NewGuid();
[MaxLength(32)] public string FileId { get; set; } = null!;
[JsonIgnore] public SnCloudFile File { get; set; } = null!;
[MaxLength(1024)] public string Usage { get; set; } = null!;
[MaxLength(1024)] public string ResourceId { get; set; } = null!;
/// <summary>
/// Optional expiration date for the file reference
/// </summary>
public Instant? ExpiredAt { get; set; }
/// <summary>
/// Converts the SnCloudFileReference to a protobuf message
/// </summary>
/// <returns>The protobuf message representation of this object</returns>
public CloudFileReference ToProtoValue()
{
return new CloudFileReference
{
Id = Id.ToString(),
FileId = FileId,
File = File?.ToProtoValue(),
Usage = Usage,
ResourceId = ResourceId,
ExpiredAt = ExpiredAt?.ToTimestamp()
};
}
}

1
DysonNetwork.Shared/Models/CloudFileObject.cs
View File

@@ -6,6 +6,7 @@ namespace DysonNetwork.Shared.Models;
public class SnFileObject : ModelBase public class SnFileObject : ModelBase
{ {
[MaxLength(32)] public string Id { get; set; } [MaxLength(32)] public string Id { get; set; }
public Guid AccountId { get; set; }
public long Size { get; set; } public long Size { get; set; }

29
DysonNetwork.Sphere/Post/PostService.cs
View File

@@ -25,7 +25,6 @@ public partial class PostService(
ICacheService cache, ICacheService cache,
ILogger<PostService> logger, ILogger<PostService> logger,
FileService.FileServiceClient files, FileService.FileServiceClient files,
FileReferenceService.FileReferenceServiceClient fileRefs,
Publisher.PublisherService ps, Publisher.PublisherService ps,
RemoteWebReaderService reader, RemoteWebReaderService reader,
AccountService.AccountServiceClient accounts, AccountService.AccountServiceClient accounts,
@@ -194,18 +193,6 @@ public partial class PostService(
db.Posts.Add(post); db.Posts.Add(post);
await db.SaveChangesAsync(); await db.SaveChangesAsync();
// Create file references for each attachment
if (post.Attachments.Count != 0)
{
var request = new CreateReferenceBatchRequest
{
Usage = PostFileUsageIdentifier,
ResourceId = post.ResourceIdentifier,
};
request.FilesId.AddRange(post.Attachments.Select(a => a.Id));
await fileRefs.CreateReferenceBatchAsync(request);
}
if (post.PublishedAt is not null && post.PublishedAt.Value.ToDateTimeUtc() <= DateTime.UtcNow) if (post.PublishedAt is not null && post.PublishedAt.Value.ToDateTimeUtc() <= DateTime.UtcNow)
_ = Task.Run(async () => _ = Task.Run(async () =>
{ {
@@ -306,17 +293,6 @@ public partial class PostService(
if (attachments is not null) if (attachments is not null)
{ {
var postResourceId = $"post:{post.Id}";
// Update resource references using the new file list
var request = new UpdateResourceFilesRequest
{
ResourceId = postResourceId,
Usage = PostFileUsageIdentifier,
};
request.FileIds.AddRange(attachments);
await fileRefs.UpdateResourceFilesAsync(request);
// Update post attachments by getting files from database // Update post attachments by getting files from database
var queryRequest = new GetFileBatchRequest(); var queryRequest = new GetFileBatchRequest();
queryRequest.Ids.AddRange(attachments); queryRequest.Ids.AddRange(attachments);
@@ -475,11 +451,6 @@ public partial class PostService(
public async Task DeletePostAsync(SnPost post) public async Task DeletePostAsync(SnPost post)
{ {
// Delete all file references for this post
await fileRefs.DeleteResourceReferencesAsync(
new DeleteResourceReferencesRequest { ResourceId = post.ResourceIdentifier }
);
var now = SystemClock.Instance.GetCurrentInstant(); var now = SystemClock.Instance.GetCurrentInstant();
await using var transaction = await db.Database.BeginTransactionAsync(); await using var transaction = await db.Database.BeginTransactionAsync();
try try

44
DysonNetwork.Sphere/Publisher/PublisherController.cs
View File

@@ -19,7 +19,6 @@ public class PublisherController(
PublisherService ps, PublisherService ps,
AccountService.AccountServiceClient accounts, AccountService.AccountServiceClient accounts,
FileService.FileServiceClient files, FileService.FileServiceClient files,
FileReferenceService.FileReferenceServiceClient fileRefs,
ActionLogService.ActionLogServiceClient als, ActionLogService.ActionLogServiceClient als,
RemoteRealmService remoteRealmService, RemoteRealmService remoteRealmService,
IServiceScopeFactory factory IServiceScopeFactory factory
@@ -569,25 +568,7 @@ public class PublisherController(
); );
var picture = SnCloudFileReferenceObject.FromProtoValue(queryResult); var picture = SnCloudFileReferenceObject.FromProtoValue(queryResult);
// Remove old references for the publisher picture
if (publisher.Picture is not null)
await fileRefs.DeleteResourceReferencesAsync(
new DeleteResourceReferencesRequest
{
ResourceId = publisher.ResourceIdentifier,
}
);
publisher.Picture = picture; publisher.Picture = picture;
await fileRefs.CreateReferenceAsync(
new CreateReferenceRequest
{
FileId = picture.Id,
Usage = "publisher.picture",
ResourceId = publisher.ResourceIdentifier,
}
);
} }
if (request.BackgroundId is not null) if (request.BackgroundId is not null)
@@ -601,27 +582,7 @@ public class PublisherController(
); );
var background = SnCloudFileReferenceObject.FromProtoValue(queryResult); var background = SnCloudFileReferenceObject.FromProtoValue(queryResult);
// Remove old references for the publisher background
if (publisher.Background is not null)
{
await fileRefs.DeleteResourceReferencesAsync(
new DeleteResourceReferencesRequest
{
ResourceId = publisher.ResourceIdentifier,
}
);
}
publisher.Background = background; publisher.Background = background;
await fileRefs.CreateReferenceAsync(
new CreateReferenceRequest
{
FileId = background.Id,
Usage = "publisher.background",
ResourceId = publisher.ResourceIdentifier,
}
);
} }
db.Update(publisher); db.Update(publisher);
@@ -717,11 +678,6 @@ public class PublisherController(
var publisherResourceId = $"publisher:{publisher.Id}"; var publisherResourceId = $"publisher:{publisher.Id}";
// Delete all file references for this publisher
await fileRefs.DeleteResourceReferencesAsync(
new DeleteResourceReferencesRequest { ResourceId = publisherResourceId }
);
db.Publishers.Remove(publisher); db.Publishers.Remove(publisher);
await db.SaveChangesAsync(); await db.SaveChangesAsync();

49
DysonNetwork.Sphere/Publisher/PublisherService.cs
View File

@@ -22,7 +22,6 @@ public class FediverseStatus
public class PublisherService( public class PublisherService(
AppDatabase db, AppDatabase db,
FileReferenceService.FileReferenceServiceClient fileRefs,
SocialCreditService.SocialCreditServiceClient socialCredits, SocialCreditService.SocialCreditServiceClient socialCredits,
ExperienceService.ExperienceServiceClient experiences, ExperienceService.ExperienceServiceClient experiences,
ICacheService cache, ICacheService cache,
@@ -210,30 +209,6 @@ public class PublisherService(
db.Publishers.Add(publisher); db.Publishers.Add(publisher);
await db.SaveChangesAsync(); await db.SaveChangesAsync();
if (publisher.Picture is not null)
{
await fileRefs.CreateReferenceAsync(
new CreateReferenceRequest
{
FileId = publisher.Picture.Id,
Usage = "publisher.picture",
ResourceId = publisher.ResourceIdentifier,
}
);
}
if (publisher.Background is not null)
{
await fileRefs.CreateReferenceAsync(
new CreateReferenceRequest
{
FileId = publisher.Background.Id,
Usage = "publisher.background",
ResourceId = publisher.ResourceIdentifier,
}
);
}
return publisher; return publisher;
} }
@@ -270,30 +245,6 @@ public class PublisherService(
db.Publishers.Add(publisher); db.Publishers.Add(publisher);
await db.SaveChangesAsync(); await db.SaveChangesAsync();
if (publisher.Picture is not null)
{
await fileRefs.CreateReferenceAsync(
new CreateReferenceRequest
{
FileId = publisher.Picture.Id,
Usage = "publisher.picture",
ResourceId = publisher.ResourceIdentifier,
}
);
}
if (publisher.Background is not null)
{
await fileRefs.CreateReferenceAsync(
new CreateReferenceRequest
{
FileId = publisher.Background.Id,
Usage = "publisher.background",
ResourceId = publisher.ResourceIdentifier,
}
);
}
return publisher; return publisher;
} }