Solar/Swarm
2
1
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

✨ File pool instead of destination configuration

Browse Source
This commit is contained in:
LittleSheep
2025-07-26 00:41:47 +08:00
parent 123dce564c
commit 081f3f609e
10 changed files with 485 additions and 40 deletions
Download Patch File Download Diff File Expand all files Collapse all files

24
DysonNetwork.Drive/Storage/CloudFile.cs
View File

@@ -3,26 +3,12 @@ using System.ComponentModel.DataAnnotations.Schema;
using DysonNetwork.Shared.Data;
using DysonNetwork.Shared.Proto;
using Google.Protobuf;
using Newtonsoft.Json;
using NodaTime;
using NodaTime.Serialization.Protobuf;
namespace DysonNetwork.Drive.Storage;
public class RemoteStorageConfig
{
public string Id { get; set; } = string.Empty;
public string Label { get; set; } = string.Empty;
public string Region { get; set; } = string.Empty;
public string Bucket { get; set; } = string.Empty;
public string Endpoint { get; set; } = string.Empty;
public string SecretId { get; set; } = string.Empty;
public string SecretKey { get; set; } = string.Empty;
public bool EnableSigned { get; set; }
public bool EnableSsl { get; set; }
public string? ImageProxy { get; set; }
public string? AccessProxy { get; set; }
}
/// <summary>
/// The class that used in jsonb columns which referenced the cloud file.
/// The aim of this class is to store some properties that won't change to a file to reduce the database load.
@@ -54,10 +40,16 @@ public class CloudFile : ModelBase, ICloudFile, IIdentifiedResource
[MaxLength(256)] public string? Hash { get; set; }
public long Size { get; set; }
public Instant? UploadedAt { get; set; }
[MaxLength(128)] public string? UploadedTo { get; set; }
public bool HasCompression { get; set; } = false;
public bool HasThumbnail { get; set; } = false;
[JsonIgnore] public FilePool? Pool { get; set; }
public Guid? PoolId { get; set; }
[Obsolete("Deprecated, use PoolId instead. For database migration only.")]
[MaxLength(128)]
public string? UploadedTo { get; set; }
/// <summary>
/// The field is set to true if the recycling job plans to delete the file.
/// Due to the unstable of the recycling job, this doesn't really delete the file until a human verifies it.

4
DysonNetwork.Drive/Storage/FileController.cs
View File

@@ -37,7 +37,7 @@ public class FileController(
if (!string.IsNullOrWhiteSpace(file.StorageUrl)) return Redirect(file.StorageUrl);
if (file.UploadedTo is null)
if (!file.PoolId.HasValue)
{
var tusStorePath = configuration.GetValue<string>("Tus:StorePath")!;
var filePath = Path.Combine(env.ContentRootPath, tusStorePath, file.Id);
@@ -45,7 +45,7 @@ public class FileController(
return PhysicalFile(filePath, file.MimeType ?? "application/octet-stream", file.Name);
}
var dest = fs.GetRemoteStorageConfig(file.UploadedTo);
var dest = await fs.GetRemoteStorageConfig(file.PoolId.Value);
var fileName = string.IsNullOrWhiteSpace(file.StorageId) ? file.Id : file.StorageId;
if (!original && file.HasCompression)

37
DysonNetwork.Drive/Storage/FilePool.cs Normal file
View File

@@ -0,0 +1,37 @@
using System.ComponentModel.DataAnnotations;
using System.ComponentModel.DataAnnotations.Schema;
using DysonNetwork.Shared.Data;
using NodaTime;
namespace DysonNetwork.Drive.Storage;
public class RemoteStorageConfig
{
public string Id { get; set; } = string.Empty;
public string Label { get; set; } = string.Empty;
public string Region { get; set; } = string.Empty;
public string Bucket { get; set; } = string.Empty;
public string Endpoint { get; set; } = string.Empty;
public string SecretId { get; set; } = string.Empty;
public string SecretKey { get; set; } = string.Empty;
public bool EnableSigned { get; set; }
public bool EnableSsl { get; set; }
public string? ImageProxy { get; set; }
public string? AccessProxy { get; set; }
public Duration? Expiration { get; set; }
}
public class BillingConfig
{
public double CostMultiplier { get; set; } = 1.0;
}
public class FilePool : ModelBase, IIdentifiedResource
{
public Guid Id { get; set; } = Guid.NewGuid();
[MaxLength(1024)] public string Name { get; set; } = string.Empty;
[Column(TypeName = "jsonb")] public RemoteStorageConfig StorageConfig { get; set; } = new();
[Column(TypeName = "jsonb")] public BillingConfig BillingConfig { get; set; } = new();
public string ResourceIdentifier => $"file-pool/{Id}";
}

15
DysonNetwork.Drive/Storage/FileReferenceService.cs
View File

@@ -7,7 +7,7 @@ namespace DysonNetwork.Drive.Storage;
public class FileReferenceService(AppDatabase db, FileService fileService, ICacheService cache)
{
private const string CacheKeyPrefix = "fileref:";
private const string CacheKeyPrefix = "file:ref:";
private static readonly TimeSpan CacheDuration = TimeSpan.FromMinutes(15);
/// <summary>
@@ -32,6 +32,19 @@ public class FileReferenceService(AppDatabase db, FileService fileService, ICach
if (duration.HasValue)
finalExpiration = SystemClock.Instance.GetCurrentInstant() + duration.Value;
var file = await db.Files
.Where(f => f.Id == fileId)
.Include(f => f.Pool)
.FirstOrDefaultAsync();
if (file is null) throw new InvalidOperationException("File not found");
if (file.Pool?.StorageConfig.Expiration != null)
{
var now = SystemClock.Instance.GetCurrentInstant();
var expectedDuration = finalExpiration - now;
if (finalExpiration == null || expectedDuration > file.Pool.StorageConfig.Expiration)
finalExpiration = now.Plus(file.Pool.StorageConfig.Expiration.Value);
}
var reference = new CloudFileReference
{
FileId = fileId,

58
DysonNetwork.Drive/Storage/FileService.cs
View File

@@ -132,7 +132,7 @@ public class FileService(
file.SensitiveMarks = existingFile.SensitiveMarks;
file.MimeType = existingFile.MimeType;
file.UploadedAt = existingFile.UploadedAt;
file.UploadedTo = existingFile.UploadedTo;
file.PoolId = existingFile.PoolId;
db.Files.Add(file);
await db.SaveChangesAsync();
@@ -342,9 +342,9 @@ public class FileService(
if (uploads.Count > 0)
{
var uploadedTo = configuration.GetValue<string>("Storage:PreferredRemote")!;
var destPool = Guid.Parse(configuration.GetValue<string>("Storage:PreferredRemote")!);
var uploadTasks = uploads.Select(item =>
nfs.UploadFileToRemoteAsync(storageId, uploadedTo, item.FilePath, item.Suffix, item.ContentType,
nfs.UploadFileToRemoteAsync(storageId, destPool, item.FilePath, item.Suffix, item.ContentType,
item.SelfDestruct)
).ToList();
@@ -358,7 +358,7 @@ public class FileService(
var now = SystemClock.Instance.GetCurrentInstant();
await scopedDb.Files.Where(f => f.Id == fileId).ExecuteUpdateAsync(setter => setter
.SetProperty(f => f.UploadedAt, now)
.SetProperty(f => f.UploadedTo, uploadedTo)
.SetProperty(f => f.PoolId, destPool)
.SetProperty(f => f.MimeType, newMimeType)
.SetProperty(f => f.HasCompression, hasCompression)
.SetProperty(f => f.HasThumbnail, hasThumbnail)
@@ -411,7 +411,7 @@ public class FileService(
return Convert.ToHexString(hash).ToLowerInvariant();
}
public async Task UploadFileToRemoteAsync(string storageId, string targetRemote, string filePath,
public async Task UploadFileToRemoteAsync(string storageId, Guid targetRemote, string filePath,
string? suffix = null, string? contentType = null, bool selfDestruct = false)
{
await using var fileStream = File.OpenRead(filePath);
@@ -419,15 +419,15 @@ public class FileService(
if (selfDestruct) File.Delete(filePath);
}
public async Task UploadFileToRemoteAsync(string storageId, string targetRemote, Stream stream,
public async Task UploadFileToRemoteAsync(string storageId, Guid targetRemote, Stream stream,
string? suffix = null, string? contentType = null)
{
var dest = GetRemoteStorageConfig(targetRemote);
var client = CreateMinioClient(dest);
if (client is null)
var dest = await GetRemoteStorageConfig(targetRemote);
if (dest is null)
throw new InvalidOperationException(
$"Failed to configure client for remote destination '{targetRemote}'"
);
var client = CreateMinioClient(dest);
var bucket = dest.Bucket;
contentType ??= "application/octet-stream";
@@ -495,31 +495,32 @@ public class FileService(
public async Task DeleteFileDataAsync(CloudFile file)
{
if (file.StorageId is null) return;
if (file.UploadedTo is null) return;
if (!file.PoolId.HasValue) return;
// Check if any other file with the same storage ID is referenced
var otherFilesWithSameStorageId = await db.Files
var sameOriginFiles = await db.Files
.Where(f => f.StorageId == file.StorageId && f.Id != file.Id)
.Select(f => f.Id)
.ToListAsync();
// Check if any of these files are referenced
var anyReferenced = false;
if (otherFilesWithSameStorageId.Any())
if (sameOriginFiles.Count != 0)
{
anyReferenced = await db.FileReferences
.Where(r => otherFilesWithSameStorageId.Contains(r.FileId))
.Where(r => sameOriginFiles.Contains(r.FileId))
.AnyAsync();
}
// If any other file with the same storage ID is referenced, don't delete the actual file data
if (anyReferenced) return;
var dest = GetRemoteStorageConfig(file.UploadedTo);
var dest = await GetRemoteStorageConfig(file.PoolId.Value);
if (dest is null) throw new InvalidOperationException($"No remote storage configured for pool {file.PoolId}");
var client = CreateMinioClient(dest);
if (client is null)
throw new InvalidOperationException(
$"Failed to configure client for remote destination '{file.UploadedTo}'"
$"Failed to configure client for remote destination '{file.PoolId}'"
);
var bucket = dest.Bucket;
@@ -546,12 +547,29 @@ public class FileService(
}
}
public RemoteStorageConfig GetRemoteStorageConfig(string destination)
public async Task<FilePool?> GetPoolAsync(Guid destination)
{
var destinations = configuration.GetSection("Storage:Remote").Get<List<RemoteStorageConfig>>()!;
var dest = destinations.FirstOrDefault(d => d.Id == destination);
if (dest is null) throw new InvalidOperationException($"Remote destination '{destination}' not found");
return dest;
var cacheKey = $"file:pool:{destination}";
var cachedResult = await cache.GetAsync<FilePool?>(cacheKey);
if (cachedResult != null) return cachedResult;
var pool = await db.Pools.FirstOrDefaultAsync(p => p.Id == destination);
if (pool != null)
await cache.SetAsync(cacheKey, pool);
return pool;
}
public async Task<RemoteStorageConfig?> GetRemoteStorageConfig(Guid destination)
{
var pool = await GetPoolAsync(destination);
return pool?.StorageConfig;
}
public async Task<RemoteStorageConfig?> GetRemoteStorageConfig(string destination)
{
var id = Guid.Parse(destination);
return await GetRemoteStorageConfig(id);
}
public IMinioClient? CreateMinioClient(RemoteStorageConfig dest)