Solar/Swarm
2
1
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

✨ File pool instead of destination configuration

Browse Source
This commit is contained in:
LittleSheep
2025-07-26 00:41:47 +08:00
parent 123dce564c
commit 081f3f609e
10 changed files with 485 additions and 40 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
DysonNetwork.Drive/AppDatabase.cs
View File

@@ -15,6 +15,8 @@ public class AppDatabase(
IConfiguration configuration
) : DbContext(options)
{
public DbSet<FilePool> Pools { get; set; } = null!;
public DbSet<CloudFile> Files { get; set; } = null!;
public DbSet<CloudFileReference> FileReferences { get; set; } = null!;

253
DysonNetwork.Drive/Migrations/20250725163615_AddCloudFilePool.Designer.cs generated Normal file
View File

@@ -0,0 +1,253 @@
// <auto-generated />
using System;
using System.Collections.Generic;
using DysonNetwork.Drive;
using DysonNetwork.Drive.Storage;
using DysonNetwork.Shared.Data;
using Microsoft.EntityFrameworkCore;
using Microsoft.EntityFrameworkCore.Infrastructure;
using Microsoft.EntityFrameworkCore.Migrations;
using Microsoft.EntityFrameworkCore.Storage.ValueConversion;
using NodaTime;
using Npgsql.EntityFrameworkCore.PostgreSQL.Metadata;
#nullable disable
namespace DysonNetwork.Drive.Migrations
{
[DbContext(typeof(AppDatabase))]
[Migration("20250725163615_AddCloudFilePool")]
partial class AddCloudFilePool
{
/// <inheritdoc />
protected override void BuildTargetModel(ModelBuilder modelBuilder)
{
#pragma warning disable 612, 618
modelBuilder
.HasAnnotation("ProductVersion", "9.0.7")
.HasAnnotation("Relational:MaxIdentifierLength", 63);
NpgsqlModelBuilderExtensions.HasPostgresExtension(modelBuilder, "postgis");
NpgsqlModelBuilderExtensions.UseIdentityByDefaultColumns(modelBuilder);
modelBuilder.Entity("DysonNetwork.Drive.Storage.CloudFile", b =>
{
b.Property<string>("Id")
.HasMaxLength(32)
.HasColumnType("character varying(32)")
.HasColumnName("id");
b.Property<Guid>("AccountId")
.HasColumnType("uuid")
.HasColumnName("account_id");
b.Property<Instant>("CreatedAt")
.HasColumnType("timestamp with time zone")
.HasColumnName("created_at");
b.Property<Instant?>("DeletedAt")
.HasColumnType("timestamp with time zone")
.HasColumnName("deleted_at");
b.Property<string>("Description")
.HasMaxLength(4096)
.HasColumnType("character varying(4096)")
.HasColumnName("description");
b.Property<Dictionary<string, object>>("FileMeta")
.IsRequired()
.HasColumnType("jsonb")
.HasColumnName("file_meta");
b.Property<bool>("HasCompression")
.HasColumnType("boolean")
.HasColumnName("has_compression");
b.Property<bool>("HasThumbnail")
.HasColumnType("boolean")
.HasColumnName("has_thumbnail");
b.Property<string>("Hash")
.HasMaxLength(256)
.HasColumnType("character varying(256)")
.HasColumnName("hash");
b.Property<bool>("IsMarkedRecycle")
.HasColumnType("boolean")
.HasColumnName("is_marked_recycle");
b.Property<string>("MimeType")
.HasMaxLength(256)
.HasColumnType("character varying(256)")
.HasColumnName("mime_type");
b.Property<string>("Name")
.IsRequired()
.HasMaxLength(1024)
.HasColumnType("character varying(1024)")
.HasColumnName("name");
b.Property<Guid?>("PoolId")
.HasColumnType("uuid")
.HasColumnName("pool_id");
b.Property<List<ContentSensitiveMark>>("SensitiveMarks")
.HasColumnType("jsonb")
.HasColumnName("sensitive_marks");
b.Property<long>("Size")
.HasColumnType("bigint")
.HasColumnName("size");
b.Property<string>("StorageId")
.HasMaxLength(32)
.HasColumnType("character varying(32)")
.HasColumnName("storage_id");
b.Property<string>("StorageUrl")
.HasMaxLength(4096)
.HasColumnType("character varying(4096)")
.HasColumnName("storage_url");
b.Property<Instant>("UpdatedAt")
.HasColumnType("timestamp with time zone")
.HasColumnName("updated_at");
b.Property<Instant?>("UploadedAt")
.HasColumnType("timestamp with time zone")
.HasColumnName("uploaded_at");
b.Property<string>("UploadedTo")
.HasMaxLength(128)
.HasColumnType("character varying(128)")
.HasColumnName("uploaded_to");
b.Property<Dictionary<string, object>>("UserMeta")
.HasColumnType("jsonb")
.HasColumnName("user_meta");
b.HasKey("Id")
.HasName("pk_files");
b.HasIndex("PoolId")
.HasDatabaseName("ix_files_pool_id");
b.ToTable("files", (string)null);
});
modelBuilder.Entity("DysonNetwork.Drive.Storage.CloudFileReference", b =>
{
b.Property<Guid>("Id")
.ValueGeneratedOnAdd()
.HasColumnType("uuid")
.HasColumnName("id");
b.Property<Instant>("CreatedAt")
.HasColumnType("timestamp with time zone")
.HasColumnName("created_at");
b.Property<Instant?>("DeletedAt")
.HasColumnType("timestamp with time zone")
.HasColumnName("deleted_at");
b.Property<Instant?>("ExpiredAt")
.HasColumnType("timestamp with time zone")
.HasColumnName("expired_at");
b.Property<string>("FileId")
.IsRequired()
.HasMaxLength(32)
.HasColumnType("character varying(32)")
.HasColumnName("file_id");
b.Property<string>("ResourceId")
.IsRequired()
.HasMaxLength(1024)
.HasColumnType("character varying(1024)")
.HasColumnName("resource_id");
b.Property<Instant>("UpdatedAt")
.HasColumnType("timestamp with time zone")
.HasColumnName("updated_at");
b.Property<string>("Usage")
.IsRequired()
.HasMaxLength(1024)
.HasColumnType("character varying(1024)")
.HasColumnName("usage");
b.HasKey("Id")
.HasName("pk_file_references");
b.HasIndex("FileId")
.HasDatabaseName("ix_file_references_file_id");
b.ToTable("file_references", (string)null);
});
modelBuilder.Entity("DysonNetwork.Drive.Storage.FilePool", b =>
{
b.Property<Guid>("Id")
.ValueGeneratedOnAdd()
.HasColumnType("uuid")
.HasColumnName("id");
b.Property<BillingConfig>("BillingConfig")
.IsRequired()
.HasColumnType("jsonb")
.HasColumnName("billing_config");
b.Property<Instant>("CreatedAt")
.HasColumnType("timestamp with time zone")
.HasColumnName("created_at");
b.Property<Instant?>("DeletedAt")
.HasColumnType("timestamp with time zone")
.HasColumnName("deleted_at");
b.Property<string>("Name")
.IsRequired()
.HasMaxLength(1024)
.HasColumnType("character varying(1024)")
.HasColumnName("name");
b.Property<RemoteStorageConfig>("StorageConfig")
.IsRequired()
.HasColumnType("jsonb")
.HasColumnName("storage_config");
b.Property<Instant>("UpdatedAt")
.HasColumnType("timestamp with time zone")
.HasColumnName("updated_at");
b.HasKey("Id")
.HasName("pk_pools");
b.ToTable("pools", (string)null);
});
modelBuilder.Entity("DysonNetwork.Drive.Storage.CloudFile", b =>
{
b.HasOne("DysonNetwork.Drive.Storage.FilePool", "Pool")
.WithMany()
.HasForeignKey("PoolId")
.HasConstraintName("fk_files_pools_pool_id");
b.Navigation("Pool");
});
modelBuilder.Entity("DysonNetwork.Drive.Storage.CloudFileReference", b =>
{
b.HasOne("DysonNetwork.Drive.Storage.CloudFile", "File")
.WithMany()
.HasForeignKey("FileId")
.OnDelete(DeleteBehavior.Cascade)
.IsRequired()
.HasConstraintName("fk_file_references_files_file_id");
b.Navigation("File");
});
#pragma warning restore 612, 618
}
}
}

71
DysonNetwork.Drive/Migrations/20250725163615_AddCloudFilePool.cs Normal file
View File

@@ -0,0 +1,71 @@
using System;
using DysonNetwork.Drive.Storage;
using Microsoft.EntityFrameworkCore.Migrations;
using NodaTime;
#nullable disable
namespace DysonNetwork.Drive.Migrations
{
/// <inheritdoc />
public partial class AddCloudFilePool : Migration
{
/// <inheritdoc />
protected override void Up(MigrationBuilder migrationBuilder)
{
migrationBuilder.AddColumn<Guid>(
name: "pool_id",
table: "files",
type: "uuid",
nullable: true);
migrationBuilder.CreateTable(
name: "pools",
columns: table => new
{
id = table.Column<Guid>(type: "uuid", nullable: false),
name = table.Column<string>(type: "character varying(1024)", maxLength: 1024, nullable: false),
storage_config = table.Column<RemoteStorageConfig>(type: "jsonb", nullable: false),
billing_config = table.Column<BillingConfig>(type: "jsonb", nullable: false),
created_at = table.Column<Instant>(type: "timestamp with time zone", nullable: false),
updated_at = table.Column<Instant>(type: "timestamp with time zone", nullable: false),
deleted_at = table.Column<Instant>(type: "timestamp with time zone", nullable: true)
},
constraints: table =>
{
table.PrimaryKey("pk_pools", x => x.id);
});
migrationBuilder.CreateIndex(
name: "ix_files_pool_id",
table: "files",
column: "pool_id");
migrationBuilder.AddForeignKey(
name: "fk_files_pools_pool_id",
table: "files",
column: "pool_id",
principalTable: "pools",
principalColumn: "id");
}
/// <inheritdoc />
protected override void Down(MigrationBuilder migrationBuilder)
{
migrationBuilder.DropForeignKey(
name: "fk_files_pools_pool_id",
table: "files");
migrationBuilder.DropTable(
name: "pools");
migrationBuilder.DropIndex(
name: "ix_files_pool_id",
table: "files");
migrationBuilder.DropColumn(
name: "pool_id",
table: "files");
}
}
}

59
DysonNetwork.Drive/Migrations/AppDatabaseModelSnapshot.cs
View File

@@ -2,6 +2,7 @@
using System;
using System.Collections.Generic;
using DysonNetwork.Drive;
using DysonNetwork.Drive.Storage;
using DysonNetwork.Shared.Data;
using Microsoft.EntityFrameworkCore;
using Microsoft.EntityFrameworkCore.Infrastructure;
@@ -83,6 +84,10 @@ namespace DysonNetwork.Drive.Migrations
.HasColumnType("character varying(1024)")
.HasColumnName("name");
b.Property<Guid?>("PoolId")
.HasColumnType("uuid")
.HasColumnName("pool_id");
b.Property<List<ContentSensitiveMark>>("SensitiveMarks")
.HasColumnType("jsonb")
.HasColumnName("sensitive_marks");
@@ -121,6 +126,9 @@ namespace DysonNetwork.Drive.Migrations
b.HasKey("Id")
.HasName("pk_files");
b.HasIndex("PoolId")
.HasDatabaseName("ix_files_pool_id");
b.ToTable("files", (string)null);
});
@@ -174,6 +182,57 @@ namespace DysonNetwork.Drive.Migrations
b.ToTable("file_references", (string)null);
});
modelBuilder.Entity("DysonNetwork.Drive.Storage.FilePool", b =>
{
b.Property<Guid>("Id")
.ValueGeneratedOnAdd()
.HasColumnType("uuid")
.HasColumnName("id");
b.Property<BillingConfig>("BillingConfig")
.IsRequired()
.HasColumnType("jsonb")
.HasColumnName("billing_config");
b.Property<Instant>("CreatedAt")
.HasColumnType("timestamp with time zone")
.HasColumnName("created_at");
b.Property<Instant?>("DeletedAt")
.HasColumnType("timestamp with time zone")
.HasColumnName("deleted_at");
b.Property<string>("Name")
.IsRequired()
.HasMaxLength(1024)
.HasColumnType("character varying(1024)")
.HasColumnName("name");
b.Property<RemoteStorageConfig>("StorageConfig")
.IsRequired()
.HasColumnType("jsonb")
.HasColumnName("storage_config");
b.Property<Instant>("UpdatedAt")
.HasColumnType("timestamp with time zone")
.HasColumnName("updated_at");
b.HasKey("Id")
.HasName("pk_pools");
b.ToTable("pools", (string)null);
});
modelBuilder.Entity("DysonNetwork.Drive.Storage.CloudFile", b =>
{
b.HasOne("DysonNetwork.Drive.Storage.FilePool", "Pool")
.WithMany()
.HasForeignKey("PoolId")
.HasConstraintName("fk_files_pools_pool_id");
b.Navigation("Pool");
});
modelBuilder.Entity("DysonNetwork.Drive.Storage.CloudFileReference", b =>
{
b.HasOne("DysonNetwork.Drive.Storage.CloudFile", "File")

24
DysonNetwork.Drive/Storage/CloudFile.cs
View File

@@ -3,26 +3,12 @@ using System.ComponentModel.DataAnnotations.Schema;
using DysonNetwork.Shared.Data;
using DysonNetwork.Shared.Proto;
using Google.Protobuf;
using Newtonsoft.Json;
using NodaTime;
using NodaTime.Serialization.Protobuf;
namespace DysonNetwork.Drive.Storage;
public class RemoteStorageConfig
{
public string Id { get; set; } = string.Empty;
public string Label { get; set; } = string.Empty;
public string Region { get; set; } = string.Empty;
public string Bucket { get; set; } = string.Empty;
public string Endpoint { get; set; } = string.Empty;
public string SecretId { get; set; } = string.Empty;
public string SecretKey { get; set; } = string.Empty;
public bool EnableSigned { get; set; }
public bool EnableSsl { get; set; }
public string? ImageProxy { get; set; }
public string? AccessProxy { get; set; }
}
/// <summary>
/// The class that used in jsonb columns which referenced the cloud file.
/// The aim of this class is to store some properties that won't change to a file to reduce the database load.
@@ -54,10 +40,16 @@ public class CloudFile : ModelBase, ICloudFile, IIdentifiedResource
[MaxLength(256)] public string? Hash { get; set; }
public long Size { get; set; }
public Instant? UploadedAt { get; set; }
[MaxLength(128)] public string? UploadedTo { get; set; }
public bool HasCompression { get; set; } = false;
public bool HasThumbnail { get; set; } = false;
[JsonIgnore] public FilePool? Pool { get; set; }
public Guid? PoolId { get; set; }
[Obsolete("Deprecated, use PoolId instead. For database migration only.")]
[MaxLength(128)]
public string? UploadedTo { get; set; }
/// <summary>
/// The field is set to true if the recycling job plans to delete the file.
/// Due to the unstable of the recycling job, this doesn't really delete the file until a human verifies it.

4
DysonNetwork.Drive/Storage/FileController.cs
View File

@@ -37,7 +37,7 @@ public class FileController(
if (!string.IsNullOrWhiteSpace(file.StorageUrl)) return Redirect(file.StorageUrl);
if (file.UploadedTo is null)
if (!file.PoolId.HasValue)
{
var tusStorePath = configuration.GetValue<string>("Tus:StorePath")!;
var filePath = Path.Combine(env.ContentRootPath, tusStorePath, file.Id);
@@ -45,7 +45,7 @@ public class FileController(
return PhysicalFile(filePath, file.MimeType ?? "application/octet-stream", file.Name);
}
var dest = fs.GetRemoteStorageConfig(file.UploadedTo);
var dest = await fs.GetRemoteStorageConfig(file.PoolId.Value);
var fileName = string.IsNullOrWhiteSpace(file.StorageId) ? file.Id : file.StorageId;
if (!original && file.HasCompression)

37
DysonNetwork.Drive/Storage/FilePool.cs Normal file
View File

@@ -0,0 +1,37 @@
using System.ComponentModel.DataAnnotations;
using System.ComponentModel.DataAnnotations.Schema;
using DysonNetwork.Shared.Data;
using NodaTime;
namespace DysonNetwork.Drive.Storage;
public class RemoteStorageConfig
{
public string Id { get; set; } = string.Empty;
public string Label { get; set; } = string.Empty;
public string Region { get; set; } = string.Empty;
public string Bucket { get; set; } = string.Empty;
public string Endpoint { get; set; } = string.Empty;
public string SecretId { get; set; } = string.Empty;
public string SecretKey { get; set; } = string.Empty;
public bool EnableSigned { get; set; }
public bool EnableSsl { get; set; }
public string? ImageProxy { get; set; }
public string? AccessProxy { get; set; }
public Duration? Expiration { get; set; }
}
public class BillingConfig
{
public double CostMultiplier { get; set; } = 1.0;
}
public class FilePool : ModelBase, IIdentifiedResource
{
public Guid Id { get; set; } = Guid.NewGuid();
[MaxLength(1024)] public string Name { get; set; } = string.Empty;
[Column(TypeName = "jsonb")] public RemoteStorageConfig StorageConfig { get; set; } = new();
[Column(TypeName = "jsonb")] public BillingConfig BillingConfig { get; set; } = new();
public string ResourceIdentifier => $"file-pool/{Id}";
}

15
DysonNetwork.Drive/Storage/FileReferenceService.cs
View File

@@ -7,7 +7,7 @@ namespace DysonNetwork.Drive.Storage;
public class FileReferenceService(AppDatabase db, FileService fileService, ICacheService cache)
{
private const string CacheKeyPrefix = "fileref:";
private const string CacheKeyPrefix = "file:ref:";
private static readonly TimeSpan CacheDuration = TimeSpan.FromMinutes(15);
/// <summary>
@@ -32,6 +32,19 @@ public class FileReferenceService(AppDatabase db, FileService fileService, ICach
if (duration.HasValue)
finalExpiration = SystemClock.Instance.GetCurrentInstant() + duration.Value;
var file = await db.Files
.Where(f => f.Id == fileId)
.Include(f => f.Pool)
.FirstOrDefaultAsync();
if (file is null) throw new InvalidOperationException("File not found");
if (file.Pool?.StorageConfig.Expiration != null)
{
var now = SystemClock.Instance.GetCurrentInstant();
var expectedDuration = finalExpiration - now;
if (finalExpiration == null || expectedDuration > file.Pool.StorageConfig.Expiration)
finalExpiration = now.Plus(file.Pool.StorageConfig.Expiration.Value);
}
var reference = new CloudFileReference
{
FileId = fileId,

58
DysonNetwork.Drive/Storage/FileService.cs
View File

@@ -132,7 +132,7 @@ public class FileService(
file.SensitiveMarks = existingFile.SensitiveMarks;
file.MimeType = existingFile.MimeType;
file.UploadedAt = existingFile.UploadedAt;
file.UploadedTo = existingFile.UploadedTo;
file.PoolId = existingFile.PoolId;
db.Files.Add(file);
await db.SaveChangesAsync();
@@ -342,9 +342,9 @@ public class FileService(
if (uploads.Count > 0)
{
var uploadedTo = configuration.GetValue<string>("Storage:PreferredRemote")!;
var destPool = Guid.Parse(configuration.GetValue<string>("Storage:PreferredRemote")!);
var uploadTasks = uploads.Select(item =>
nfs.UploadFileToRemoteAsync(storageId, uploadedTo, item.FilePath, item.Suffix, item.ContentType,
nfs.UploadFileToRemoteAsync(storageId, destPool, item.FilePath, item.Suffix, item.ContentType,
item.SelfDestruct)
).ToList();
@@ -358,7 +358,7 @@ public class FileService(
var now = SystemClock.Instance.GetCurrentInstant();
await scopedDb.Files.Where(f => f.Id == fileId).ExecuteUpdateAsync(setter => setter
.SetProperty(f => f.UploadedAt, now)
.SetProperty(f => f.UploadedTo, uploadedTo)
.SetProperty(f => f.PoolId, destPool)
.SetProperty(f => f.MimeType, newMimeType)
.SetProperty(f => f.HasCompression, hasCompression)
.SetProperty(f => f.HasThumbnail, hasThumbnail)
@@ -411,7 +411,7 @@ public class FileService(
return Convert.ToHexString(hash).ToLowerInvariant();
}
public async Task UploadFileToRemoteAsync(string storageId, string targetRemote, string filePath,
public async Task UploadFileToRemoteAsync(string storageId, Guid targetRemote, string filePath,
string? suffix = null, string? contentType = null, bool selfDestruct = false)
{
await using var fileStream = File.OpenRead(filePath);
@@ -419,15 +419,15 @@ public class FileService(
if (selfDestruct) File.Delete(filePath);
}
public async Task UploadFileToRemoteAsync(string storageId, string targetRemote, Stream stream,
public async Task UploadFileToRemoteAsync(string storageId, Guid targetRemote, Stream stream,
string? suffix = null, string? contentType = null)
{
var dest = GetRemoteStorageConfig(targetRemote);
var client = CreateMinioClient(dest);
if (client is null)
var dest = await GetRemoteStorageConfig(targetRemote);
if (dest is null)
throw new InvalidOperationException(
$"Failed to configure client for remote destination '{targetRemote}'"
);
var client = CreateMinioClient(dest);
var bucket = dest.Bucket;
contentType ??= "application/octet-stream";
@@ -495,31 +495,32 @@ public class FileService(
public async Task DeleteFileDataAsync(CloudFile file)
{
if (file.StorageId is null) return;
if (file.UploadedTo is null) return;
if (!file.PoolId.HasValue) return;
// Check if any other file with the same storage ID is referenced
var otherFilesWithSameStorageId = await db.Files
var sameOriginFiles = await db.Files
.Where(f => f.StorageId == file.StorageId && f.Id != file.Id)
.Select(f => f.Id)
.ToListAsync();
// Check if any of these files are referenced
var anyReferenced = false;
if (otherFilesWithSameStorageId.Any())
if (sameOriginFiles.Count != 0)
{
anyReferenced = await db.FileReferences
.Where(r => otherFilesWithSameStorageId.Contains(r.FileId))
.Where(r => sameOriginFiles.Contains(r.FileId))
.AnyAsync();
}
// If any other file with the same storage ID is referenced, don't delete the actual file data
if (anyReferenced) return;
var dest = GetRemoteStorageConfig(file.UploadedTo);
var dest = await GetRemoteStorageConfig(file.PoolId.Value);
if (dest is null) throw new InvalidOperationException($"No remote storage configured for pool {file.PoolId}");
var client = CreateMinioClient(dest);
if (client is null)
throw new InvalidOperationException(
$"Failed to configure client for remote destination '{file.UploadedTo}'"
$"Failed to configure client for remote destination '{file.PoolId}'"
);
var bucket = dest.Bucket;
@@ -546,12 +547,29 @@ public class FileService(
}
}
public RemoteStorageConfig GetRemoteStorageConfig(string destination)
public async Task<FilePool?> GetPoolAsync(Guid destination)
{
var destinations = configuration.GetSection("Storage:Remote").Get<List<RemoteStorageConfig>>()!;
var dest = destinations.FirstOrDefault(d => d.Id == destination);
if (dest is null) throw new InvalidOperationException($"Remote destination '{destination}' not found");
return dest;
var cacheKey = $"file:pool:{destination}";
var cachedResult = await cache.GetAsync<FilePool?>(cacheKey);
if (cachedResult != null) return cachedResult;
var pool = await db.Pools.FirstOrDefaultAsync(p => p.Id == destination);
if (pool != null)
await cache.SetAsync(cacheKey, pool);
return pool;
}
public async Task<RemoteStorageConfig?> GetRemoteStorageConfig(Guid destination)
{
var pool = await GetPoolAsync(destination);
return pool?.StorageConfig;
}
public async Task<RemoteStorageConfig?> GetRemoteStorageConfig(string destination)
{
var id = Guid.Parse(destination);
return await GetRemoteStorageConfig(id);
}
public IMinioClient? CreateMinioClient(RemoteStorageConfig dest)