SolarNetwork/Swarm
2
3
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

♻️ Update the usage counting since the pool id logic changed

Browse Source
This commit is contained in:
LittleSheep
2026-01-13 23:26:09 +08:00
parent 5a99665e4e
commit fc1edf0ea3
13 changed files with 833 additions and 231 deletions
Download Patch File Download Diff File Expand all files Collapse all files

99
DysonNetwork.Drive/Billing/UsageService.cs
View File

@@ -1,5 +1,6 @@
using Microsoft.EntityFrameworkCore;
using NodaTime;
using DysonNetwork.Shared.Models;
namespace DysonNetwork.Drive.Billing;
@@ -29,28 +30,42 @@ public class UsageService(AppDatabase db)
public async Task<TotalUsageDetails> GetTotalUsage(Guid accountId)
{
var now = SystemClock.Instance.GetCurrentInstant();
var fileQuery = db.Files
.Where(f => !f.IsMarkedRecycle)
.Where(f => !f.ExpiredAt.HasValue || f.ExpiredAt > now)
.Where(f => f.AccountId == accountId)
.AsQueryable();
var poolUsages = await db.Pools
.Select(p => new UsageDetails
{
PoolId = p.Id,
PoolName = p.Name,
UsageBytes = fileQuery
.Where(f => f.PoolId == p.Id)
.Include(f => f.Object)
.Sum(f => f.Size),
Cost = fileQuery
.Where(f => f.PoolId == p.Id)
.Include(f => f.Object)
.Sum(f => f.Size) / 1024.0 / 1024.0 *
(p.BillingConfig.CostMultiplier ?? 1.0),
FileCount = fileQuery
.Count(f => f.PoolId == p.Id)
UsageBytes = db.Files
.Where(f => f.AccountId == accountId)
.Where(f => !f.IsMarkedRecycle)
.Where(f => !f.ExpiredAt.HasValue || f.ExpiredAt > now)
.SelectMany(f => f.Object!.FileReplicas
.Where(r => r.PoolId == p.Id && r.Status == SnFileReplicaStatus.Available))
.Join(db.FileObjects,
r => r.ObjectId,
o => o.Id,
(r, o) => o.Size)
.DefaultIfEmpty(0L)
.Sum(),
Cost = db.Files
.Where(f => f.AccountId == accountId)
.Where(f => !f.IsMarkedRecycle)
.Where(f => !f.ExpiredAt.HasValue || f.ExpiredAt > now)
.SelectMany(f => f.Object!.FileReplicas
.Where(r => r.PoolId == p.Id && r.Status == SnFileReplicaStatus.Available))
.Join(db.FileObjects,
r => r.ObjectId,
o => o.Id,
(r, o) => new { Size = o.Size, Multiplier = p.BillingConfig.CostMultiplier ?? 1.0 })
.Sum(x => x.Size * x.Multiplier) / 1024.0 / 1024.0,
FileCount = db.Files
.Where(f => f.AccountId == accountId)
.Where(f => !f.IsMarkedRecycle)
.Where(f => !f.ExpiredAt.HasValue || f.ExpiredAt > now)
.SelectMany(f => f.Object!.FileReplicas
.Where(r => r.PoolId == p.Id && r.Status == SnFileReplicaStatus.Available))
.Count()
})
.ToListAsync();
@@ -75,18 +90,22 @@ public class UsageService(AppDatabase db)
}
var now = SystemClock.Instance.GetCurrentInstant();
var fileQuery = db.Files
.Where(f => !f.IsMarkedRecycle)
.Where(f => f.ExpiredAt.HasValue && f.ExpiredAt > now)
var replicaQuery = db.Files
.Where(f => f.AccountId == accountId)
.AsQueryable();
.Where(f => !f.IsMarkedRecycle)
.Where(f => !f.ExpiredAt.HasValue || f.ExpiredAt > now)
.SelectMany(f => f.Object!.FileReplicas
.Where(r => r.PoolId == poolId && r.Status == SnFileReplicaStatus.Available));
var usageBytes = await fileQuery
.Include(f => f.Object)
.SumAsync(f => f.Size);
var usageBytes = await replicaQuery
.Join(db.FileObjects,
r => r.ObjectId,
o => o.Id,
(r, o) => o.Size)
.DefaultIfEmpty(0L)
.SumAsync();
var fileCount = await fileQuery
.CountAsync();
var fileCount = await replicaQuery.CountAsync();
var cost = usageBytes / 1024.0 / 1024.0 *
(pool.BillingConfig.CostMultiplier ?? 1.0);
@@ -104,22 +123,24 @@ public class UsageService(AppDatabase db)
public async Task<long> GetTotalBillableUsage(Guid accountId)
{
var now = SystemClock.Instance.GetCurrentInstant();
var files = await db.Files
.Where(f => f.AccountId == accountId)
.Where(f => f.PoolId.HasValue)
.Where(f => !f.IsMarkedRecycle)
.Include(f => f.Pool)
.Include(f => f.Object)
.Where(f => !f.ExpiredAt.HasValue || f.ExpiredAt > now)
.Select(f => new
var billingData = await (from f in db.Files
where f.AccountId == accountId
where !f.IsMarkedRecycle
where !f.ExpiredAt.HasValue || f.ExpiredAt > now
from r in f.Object!.FileReplicas
where r.Status == SnFileReplicaStatus.Available
where r.PoolId.HasValue
join p in db.Pools on r.PoolId equals p.Id
join o in db.FileObjects on r.ObjectId equals o.Id
select new
{
f.Size,
Multiplier = f.Pool!.BillingConfig.CostMultiplier ?? 1.0
})
.ToListAsync();
Size = o.Size,
Multiplier = p.BillingConfig.CostMultiplier ?? 1.0
}).ToListAsync();
var totalCost = files.Sum(f => f.Size * f.Multiplier) / 1024.0 / 1024.0;
var totalCost = billingData.Sum(x => x.Size * x.Multiplier) / 1024.0 / 1024.0;
return (long)Math.Ceiling(totalCost);
}
}
}

2
DysonNetwork.Drive/Index/FileIndexController.cs
View File

@@ -230,7 +230,7 @@ public class FileIndexController(
: filesQuery.OrderBy(f => f.CreatedAt)
};
if (pool.HasValue) filesQuery = filesQuery.Where(f => f.PoolId == pool);
if (pool.HasValue) filesQuery = filesQuery.Where(f => f.Object!.FileReplicas.Any(r => r.PoolId == pool.Value));
if (!string.IsNullOrWhiteSpace(query))
{

640
DysonNetwork.Drive/Migrations/20260113152536_RemovePoolFromCloudFile.Designer.cs generated Normal file
View File

@@ -0,0 +1,640 @@
// <auto-generated />
using System;
using System.Collections.Generic;
using DysonNetwork.Drive;
using DysonNetwork.Shared.Models;
using Microsoft.EntityFrameworkCore;
using Microsoft.EntityFrameworkCore.Infrastructure;
using Microsoft.EntityFrameworkCore.Migrations;
using Microsoft.EntityFrameworkCore.Storage.ValueConversion;
using NodaTime;
using Npgsql.EntityFrameworkCore.PostgreSQL.Metadata;
#nullable disable
namespace DysonNetwork.Drive.Migrations
{
[DbContext(typeof(AppDatabase))]
[Migration("20260113152536_RemovePoolFromCloudFile")]
partial class RemovePoolFromCloudFile
{
/// <inheritdoc />
protected override void BuildTargetModel(ModelBuilder modelBuilder)
{
#pragma warning disable 612, 618
modelBuilder
.HasAnnotation("ProductVersion", "10.0.1")
.HasAnnotation("Relational:MaxIdentifierLength", 63);
NpgsqlModelBuilderExtensions.UseIdentityByDefaultColumns(modelBuilder);
modelBuilder.Entity("DysonNetwork.Drive.Billing.QuotaRecord", b =>
{
b.Property<Guid>("Id")
.ValueGeneratedOnAdd()
.HasColumnType("uuid")
.HasColumnName("id");
b.Property<Guid>("AccountId")
.HasColumnType("uuid")
.HasColumnName("account_id");
b.Property<Instant>("CreatedAt")
.HasColumnType("timestamp with time zone")
.HasColumnName("created_at");
b.Property<Instant?>("DeletedAt")
.HasColumnType("timestamp with time zone")
.HasColumnName("deleted_at");
b.Property<string>("Description")
.IsRequired()
.HasColumnType("text")
.HasColumnName("description");
b.Property<Instant?>("ExpiredAt")
.HasColumnType("timestamp with time zone")
.HasColumnName("expired_at");
b.Property<string>("Name")
.IsRequired()
.HasColumnType("text")
.HasColumnName("name");
b.Property<long>("Quota")
.HasColumnType("bigint")
.HasColumnName("quota");
b.Property<Instant>("UpdatedAt")
.HasColumnType("timestamp with time zone")
.HasColumnName("updated_at");
b.HasKey("Id")
.HasName("pk_quota_records");
b.ToTable("quota_records", (string)null);
});
modelBuilder.Entity("DysonNetwork.Drive.Storage.Model.PersistentTask", b =>
{
b.Property<Guid>("Id")
.ValueGeneratedOnAdd()
.HasColumnType("uuid")
.HasColumnName("id");
b.Property<Guid>("AccountId")
.HasColumnType("uuid")
.HasColumnName("account_id");
b.Property<Instant?>("CompletedAt")
.HasColumnType("timestamp with time zone")
.HasColumnName("completed_at");
b.Property<Instant>("CreatedAt")
.HasColumnType("timestamp with time zone")
.HasColumnName("created_at");
b.Property<Instant?>("DeletedAt")
.HasColumnType("timestamp with time zone")
.HasColumnName("deleted_at");
b.Property<string>("Description")
.HasMaxLength(1024)
.HasColumnType("character varying(1024)")
.HasColumnName("description");
b.Property<string>("ErrorMessage")
.HasMaxLength(1024)
.HasColumnType("character varying(1024)")
.HasColumnName("error_message");
b.Property<long?>("EstimatedDurationSeconds")
.HasColumnType("bigint")
.HasColumnName("estimated_duration_seconds");
b.Property<Instant?>("ExpiredAt")
.HasColumnType("timestamp with time zone")
.HasColumnName("expired_at");
b.Property<Instant>("LastActivity")
.HasColumnType("timestamp with time zone")
.HasColumnName("last_activity");
b.Property<string>("Name")
.IsRequired()
.HasMaxLength(256)
.HasColumnType("character varying(256)")
.HasColumnName("name");
b.Property<Dictionary<string, object>>("Parameters")
.IsRequired()
.HasColumnType("jsonb")
.HasColumnName("parameters");
b.Property<int>("Priority")
.HasColumnType("integer")
.HasColumnName("priority");
b.Property<double>("Progress")
.HasColumnType("double precision")
.HasColumnName("progress");
b.Property<Dictionary<string, object>>("Results")
.IsRequired()
.HasColumnType("jsonb")
.HasColumnName("results");
b.Property<Instant?>("StartedAt")
.HasColumnType("timestamp with time zone")
.HasColumnName("started_at");
b.Property<int>("Status")
.HasColumnType("integer")
.HasColumnName("status");
b.Property<string>("TaskId")
.IsRequired()
.HasMaxLength(64)
.HasColumnType("character varying(64)")
.HasColumnName("task_id");
b.Property<int>("Type")
.HasColumnType("integer")
.HasColumnName("type");
b.Property<Instant>("UpdatedAt")
.HasColumnType("timestamp with time zone")
.HasColumnName("updated_at");
b.HasKey("Id")
.HasName("pk_tasks");
b.ToTable("tasks", (string)null);
});
modelBuilder.Entity("DysonNetwork.Shared.Models.FilePool", b =>
{
b.Property<Guid>("Id")
.ValueGeneratedOnAdd()
.HasColumnType("uuid")
.HasColumnName("id");
b.Property<Guid?>("AccountId")
.HasColumnType("uuid")
.HasColumnName("account_id");
b.Property<BillingConfig>("BillingConfig")
.IsRequired()
.HasColumnType("jsonb")
.HasColumnName("billing_config");
b.Property<Instant>("CreatedAt")
.HasColumnType("timestamp with time zone")
.HasColumnName("created_at");
b.Property<Instant?>("DeletedAt")
.HasColumnType("timestamp with time zone")
.HasColumnName("deleted_at");
b.Property<string>("Description")
.IsRequired()
.HasMaxLength(8192)
.HasColumnType("character varying(8192)")
.HasColumnName("description");
b.Property<bool>("IsHidden")
.HasColumnType("boolean")
.HasColumnName("is_hidden");
b.Property<string>("Name")
.IsRequired()
.HasMaxLength(1024)
.HasColumnType("character varying(1024)")
.HasColumnName("name");
b.Property<PolicyConfig>("PolicyConfig")
.IsRequired()
.HasColumnType("jsonb")
.HasColumnName("policy_config");
b.Property<RemoteStorageConfig>("StorageConfig")
.IsRequired()
.HasColumnType("jsonb")
.HasColumnName("storage_config");
b.Property<Instant>("UpdatedAt")
.HasColumnType("timestamp with time zone")
.HasColumnName("updated_at");
b.HasKey("Id")
.HasName("pk_pools");
b.ToTable("pools", (string)null);
});
modelBuilder.Entity("DysonNetwork.Shared.Models.SnCloudFile", b =>
{
b.Property<string>("Id")
.HasMaxLength(32)
.HasColumnType("character varying(32)")
.HasColumnName("id");
b.Property<Guid>("AccountId")
.HasColumnType("uuid")
.HasColumnName("account_id");
b.Property<Guid?>("BundleId")
.HasColumnType("uuid")
.HasColumnName("bundle_id");
b.Property<Instant>("CreatedAt")
.HasColumnType("timestamp with time zone")
.HasColumnName("created_at");
b.Property<Instant?>("DeletedAt")
.HasColumnType("timestamp with time zone")
.HasColumnName("deleted_at");
b.Property<string>("Description")
.HasMaxLength(4096)
.HasColumnType("character varying(4096)")
.HasColumnName("description");
b.Property<Instant?>("ExpiredAt")
.HasColumnType("timestamp with time zone")
.HasColumnName("expired_at");
b.Property<bool>("IsMarkedRecycle")
.HasColumnType("boolean")
.HasColumnName("is_marked_recycle");
b.Property<string>("Name")
.IsRequired()
.HasMaxLength(1024)
.HasColumnType("character varying(1024)")
.HasColumnName("name");
b.Property<string>("ObjectId")
.HasMaxLength(32)
.HasColumnType("character varying(32)")
.HasColumnName("object_id");
b.PrimitiveCollection<string>("SensitiveMarks")
.HasColumnType("jsonb")
.HasColumnName("sensitive_marks");
b.Property<string>("StorageId")
.HasMaxLength(32)
.HasColumnType("character varying(32)")
.HasColumnName("storage_id");
b.Property<string>("StorageUrl")
.HasMaxLength(4096)
.HasColumnType("character varying(4096)")
.HasColumnName("storage_url");
b.Property<Instant>("UpdatedAt")
.HasColumnType("timestamp with time zone")
.HasColumnName("updated_at");
b.Property<Instant?>("UploadedAt")
.HasColumnType("timestamp with time zone")
.HasColumnName("uploaded_at");
b.Property<Dictionary<string, object>>("UserMeta")
.HasColumnType("jsonb")
.HasColumnName("user_meta");
b.HasKey("Id")
.HasName("pk_files");
b.HasIndex("BundleId")
.HasDatabaseName("ix_files_bundle_id");
b.HasIndex("ObjectId")
.HasDatabaseName("ix_files_object_id");
b.ToTable("files", (string)null);
});
modelBuilder.Entity("DysonNetwork.Shared.Models.SnCloudFileIndex", b =>
{
b.Property<Guid>("Id")
.ValueGeneratedOnAdd()
.HasColumnType("uuid")
.HasColumnName("id");
b.Property<Guid>("AccountId")
.HasColumnType("uuid")
.HasColumnName("account_id");
b.Property<Instant>("CreatedAt")
.HasColumnType("timestamp with time zone")
.HasColumnName("created_at");
b.Property<Instant?>("DeletedAt")
.HasColumnType("timestamp with time zone")
.HasColumnName("deleted_at");
b.Property<string>("FileId")
.IsRequired()
.HasMaxLength(32)
.HasColumnType("character varying(32)")
.HasColumnName("file_id");
b.Property<string>("Path")
.IsRequired()
.HasMaxLength(8192)
.HasColumnType("character varying(8192)")
.HasColumnName("path");
b.Property<Instant>("UpdatedAt")
.HasColumnType("timestamp with time zone")
.HasColumnName("updated_at");
b.HasKey("Id")
.HasName("pk_file_indexes");
b.HasIndex("FileId")
.HasDatabaseName("ix_file_indexes_file_id");
b.HasIndex("Path", "AccountId")
.HasDatabaseName("ix_file_indexes_path_account_id");
b.ToTable("file_indexes", (string)null);
});
modelBuilder.Entity("DysonNetwork.Shared.Models.SnFileBundle", b =>
{
b.Property<Guid>("Id")
.ValueGeneratedOnAdd()
.HasColumnType("uuid")
.HasColumnName("id");
b.Property<Guid>("AccountId")
.HasColumnType("uuid")
.HasColumnName("account_id");
b.Property<Instant>("CreatedAt")
.HasColumnType("timestamp with time zone")
.HasColumnName("created_at");
b.Property<Instant?>("DeletedAt")
.HasColumnType("timestamp with time zone")
.HasColumnName("deleted_at");
b.Property<string>("Description")
.HasMaxLength(8192)
.HasColumnType("character varying(8192)")
.HasColumnName("description");
b.Property<Instant?>("ExpiredAt")
.HasColumnType("timestamp with time zone")
.HasColumnName("expired_at");
b.Property<string>("Name")
.IsRequired()
.HasMaxLength(1024)
.HasColumnType("character varying(1024)")
.HasColumnName("name");
b.Property<string>("Passcode")
.HasMaxLength(256)
.HasColumnType("character varying(256)")
.HasColumnName("passcode");
b.Property<string>("Slug")
.IsRequired()
.HasMaxLength(1024)
.HasColumnType("character varying(1024)")
.HasColumnName("slug");
b.Property<Instant>("UpdatedAt")
.HasColumnType("timestamp with time zone")
.HasColumnName("updated_at");
b.HasKey("Id")
.HasName("pk_bundles");
b.HasIndex("Slug")
.IsUnique()
.HasDatabaseName("ix_bundles_slug");
b.ToTable("bundles", (string)null);
});
modelBuilder.Entity("DysonNetwork.Shared.Models.SnFileObject", b =>
{
b.Property<string>("Id")
.HasMaxLength(32)
.HasColumnType("character varying(32)")
.HasColumnName("id");
b.Property<Instant>("CreatedAt")
.HasColumnType("timestamp with time zone")
.HasColumnName("created_at");
b.Property<Instant?>("DeletedAt")
.HasColumnType("timestamp with time zone")
.HasColumnName("deleted_at");
b.Property<bool>("HasCompression")
.HasColumnType("boolean")
.HasColumnName("has_compression");
b.Property<bool>("HasThumbnail")
.HasColumnType("boolean")
.HasColumnName("has_thumbnail");
b.Property<string>("Hash")
.HasMaxLength(256)
.HasColumnType("character varying(256)")
.HasColumnName("hash");
b.Property<Dictionary<string, object>>("Meta")
.HasColumnType("jsonb")
.HasColumnName("meta");
b.Property<string>("MimeType")
.HasMaxLength(256)
.HasColumnType("character varying(256)")
.HasColumnName("mime_type");
b.Property<long>("Size")
.HasColumnType("bigint")
.HasColumnName("size");
b.Property<Instant>("UpdatedAt")
.HasColumnType("timestamp with time zone")
.HasColumnName("updated_at");
b.HasKey("Id")
.HasName("pk_file_objects");
b.ToTable("file_objects", (string)null);
});
modelBuilder.Entity("DysonNetwork.Shared.Models.SnFilePermission", b =>
{
b.Property<Guid>("Id")
.ValueGeneratedOnAdd()
.HasColumnType("uuid")
.HasColumnName("id");
b.Property<Instant>("CreatedAt")
.HasColumnType("timestamp with time zone")
.HasColumnName("created_at");
b.Property<Instant?>("DeletedAt")
.HasColumnType("timestamp with time zone")
.HasColumnName("deleted_at");
b.Property<string>("FileId")
.IsRequired()
.HasColumnType("text")
.HasColumnName("file_id");
b.Property<int>("Permission")
.HasColumnType("integer")
.HasColumnName("permission");
b.Property<string>("SubjectId")
.IsRequired()
.HasColumnType("text")
.HasColumnName("subject_id");
b.Property<int>("SubjectType")
.HasColumnType("integer")
.HasColumnName("subject_type");
b.Property<Instant>("UpdatedAt")
.HasColumnType("timestamp with time zone")
.HasColumnName("updated_at");
b.HasKey("Id")
.HasName("pk_file_permissions");
b.ToTable("file_permissions", (string)null);
});
modelBuilder.Entity("DysonNetwork.Shared.Models.SnFileReplica", b =>
{
b.Property<Guid>("Id")
.ValueGeneratedOnAdd()
.HasColumnType("uuid")
.HasColumnName("id");
b.Property<Instant>("CreatedAt")
.HasColumnType("timestamp with time zone")
.HasColumnName("created_at");
b.Property<Instant?>("DeletedAt")
.HasColumnType("timestamp with time zone")
.HasColumnName("deleted_at");
b.Property<bool>("IsPrimary")
.HasColumnType("boolean")
.HasColumnName("is_primary");
b.Property<string>("ObjectId")
.IsRequired()
.HasMaxLength(32)
.HasColumnType("character varying(32)")
.HasColumnName("object_id");
b.Property<Guid?>("PoolId")
.HasColumnType("uuid")
.HasColumnName("pool_id");
b.Property<int>("Status")
.HasColumnType("integer")
.HasColumnName("status");
b.Property<string>("StorageId")
.IsRequired()
.HasMaxLength(128)
.HasColumnType("character varying(128)")
.HasColumnName("storage_id");
b.Property<Instant>("UpdatedAt")
.HasColumnType("timestamp with time zone")
.HasColumnName("updated_at");
b.HasKey("Id")
.HasName("pk_file_replicas");
b.HasIndex("ObjectId")
.HasDatabaseName("ix_file_replicas_object_id");
b.HasIndex("PoolId")
.HasDatabaseName("ix_file_replicas_pool_id");
b.ToTable("file_replicas", (string)null);
});
modelBuilder.Entity("DysonNetwork.Shared.Models.SnCloudFile", b =>
{
b.HasOne("DysonNetwork.Shared.Models.SnFileBundle", "Bundle")
.WithMany("Files")
.HasForeignKey("BundleId")
.HasConstraintName("fk_files_bundles_bundle_id");
b.HasOne("DysonNetwork.Shared.Models.SnFileObject", "Object")
.WithMany()
.HasForeignKey("ObjectId")
.HasConstraintName("fk_files_file_objects_object_id");
b.Navigation("Bundle");
b.Navigation("Object");
});
modelBuilder.Entity("DysonNetwork.Shared.Models.SnCloudFileIndex", b =>
{
b.HasOne("DysonNetwork.Shared.Models.SnCloudFile", "File")
.WithMany("FileIndexes")
.HasForeignKey("FileId")
.OnDelete(DeleteBehavior.Cascade)
.IsRequired()
.HasConstraintName("fk_file_indexes_files_file_id");
b.Navigation("File");
});
modelBuilder.Entity("DysonNetwork.Shared.Models.SnFileReplica", b =>
{
b.HasOne("DysonNetwork.Shared.Models.SnFileObject", "Object")
.WithMany("FileReplicas")
.HasForeignKey("ObjectId")
.OnDelete(DeleteBehavior.Cascade)
.IsRequired()
.HasConstraintName("fk_file_replicas_file_objects_object_id");
b.HasOne("DysonNetwork.Shared.Models.FilePool", "Pool")
.WithMany()
.HasForeignKey("PoolId")
.HasConstraintName("fk_file_replicas_pools_pool_id");
b.Navigation("Object");
b.Navigation("Pool");
});
modelBuilder.Entity("DysonNetwork.Shared.Models.SnCloudFile", b =>
{
b.Navigation("FileIndexes");
});
modelBuilder.Entity("DysonNetwork.Shared.Models.SnFileBundle", b =>
{
b.Navigation("Files");
});
modelBuilder.Entity("DysonNetwork.Shared.Models.SnFileObject", b =>
{
b.Navigation("FileReplicas");
});
#pragma warning restore 612, 618
}
}
}

49
DysonNetwork.Drive/Migrations/20260113152536_RemovePoolFromCloudFile.cs Normal file
View File

@@ -0,0 +1,49 @@
using System;
using Microsoft.EntityFrameworkCore.Migrations;
#nullable disable
namespace DysonNetwork.Drive.Migrations
{
/// <inheritdoc />
public partial class RemovePoolFromCloudFile : Migration
{
/// <inheritdoc />
protected override void Up(MigrationBuilder migrationBuilder)
{
migrationBuilder.DropForeignKey(
name: "fk_files_pools_pool_id",
table: "files");
migrationBuilder.DropIndex(
name: "ix_files_pool_id",
table: "files");
migrationBuilder.DropColumn(
name: "pool_id",
table: "files");
}
/// <inheritdoc />
protected override void Down(MigrationBuilder migrationBuilder)
{
migrationBuilder.AddColumn<Guid>(
name: "pool_id",
table: "files",
type: "uuid",
nullable: true);
migrationBuilder.CreateIndex(
name: "ix_files_pool_id",
table: "files",
column: "pool_id");
migrationBuilder.AddForeignKey(
name: "fk_files_pools_pool_id",
table: "files",
column: "pool_id",
principalTable: "pools",
principalColumn: "id");
}
}
}

14
DysonNetwork.Drive/Migrations/AppDatabaseModelSnapshot.cs
View File

@@ -276,10 +276,6 @@ namespace DysonNetwork.Drive.Migrations
.HasColumnType("character varying(32)")
.HasColumnName("object_id");
b.Property<Guid?>("PoolId")
.HasColumnType("uuid")
.HasColumnName("pool_id");
b.PrimitiveCollection<string>("SensitiveMarks")
.HasColumnType("jsonb")
.HasColumnName("sensitive_marks");
@@ -315,9 +311,6 @@ namespace DysonNetwork.Drive.Migrations
b.HasIndex("ObjectId")
.HasDatabaseName("ix_files_object_id");
b.HasIndex("PoolId")
.HasDatabaseName("ix_files_pool_id");
b.ToTable("files", (string)null);
});
@@ -588,16 +581,9 @@ namespace DysonNetwork.Drive.Migrations
.HasForeignKey("ObjectId")
.HasConstraintName("fk_files_file_objects_object_id");
b.HasOne("DysonNetwork.Shared.Models.FilePool", "Pool")
.WithMany()
.HasForeignKey("PoolId")
.HasConstraintName("fk_files_pools_pool_id");
b.Navigation("Bundle");
b.Navigation("Object");
b.Navigation("Pool");
});
modelBuilder.Entity("DysonNetwork.Shared.Models.SnCloudFileIndex", b =>

13
DysonNetwork.Drive/Startup/BroadcastEventHandler.cs
View File

@@ -314,9 +314,20 @@ public class BroadcastEventHandler(
logger.LogInformation("Uploaded file {FileId} done!", fileId);
var now = SystemClock.Instance.GetCurrentInstant();
var newReplica = new SnFileReplica
{
Id = Guid.NewGuid(),
ObjectId = fileId,
PoolId = destPool,
StorageId = storageId,
Status = SnFileReplicaStatus.Available,
IsPrimary = false
};
scopedDb.FileReplicas.Add(newReplica);
await scopedDb.Files.Where(f => f.Id == fileId).ExecuteUpdateAsync(setter => setter
.SetProperty(f => f.UploadedAt, now)
.SetProperty(f => f.PoolId, destPool)
);
await scopedDb.FileObjects.Where(fo => fo.Id == fileId).ExecuteUpdateAsync(setter => setter

1
DysonNetwork.Drive/Startup/ServiceCollectionExtensions.cs
View File

@@ -54,7 +54,6 @@ public static class ServiceCollectionExtensions
public IServiceCollection AddAppBusinessServices()
{
services.AddScoped<Storage.FileMigrationService>();
services.AddScoped<Storage.FileService>();
services.AddScoped<Storage.FileReanalysisService>();
services.AddScoped<Storage.PersistentTaskService>();

27
DysonNetwork.Drive/Storage/CloudFileUnusedRecyclingJob.cs
View File

@@ -1,5 +1,6 @@
using Microsoft.EntityFrameworkCore;
using NodaTime;
using DysonNetwork.Shared.Models;
using Quartz;
namespace DysonNetwork.Drive.Storage;
@@ -40,8 +41,10 @@ public class CloudFileUnusedRecyclingJob(
var markedCount = 0;
var totalFiles = await db.Files
.Where(f => f.FileIndexes.Count == 0)
.Where(f => f.PoolId.HasValue && recyclablePools.Contains(f.PoolId.Value))
.Where(f => f.Object!.FileReplicas.Any(r => r.PoolId.HasValue && recyclablePools.Contains(r.PoolId.Value)))
.Where(f => !f.IsMarkedRecycle)
.Include(f => f.Object)
.ThenInclude(o => o.FileReplicas)
.CountAsync();
logger.LogInformation("Found {TotalFiles} files to check for unused status", totalFiles);
@@ -56,17 +59,18 @@ public class CloudFileUnusedRecyclingJob(
while (hasMoreFiles)
{
// Query for the next batch of files using keyset pagination
var filesQuery = db.Files
.Where(f => f.PoolId.HasValue && recyclablePools.Contains(f.PoolId.Value))
IQueryable<SnCloudFile> baseQuery = db.Files
.Where(f => f.Object!.FileReplicas.Any(r => r.PoolId.HasValue && recyclablePools.Contains(r.PoolId.Value)))
.Where(f => !f.IsMarkedRecycle)
.Where(f => f.CreatedAt <= ageThreshold); // Only process older files first
.Where(f => f.CreatedAt <= ageThreshold)
.Include(f => f.Object)
.ThenInclude(o => o.FileReplicas);
if (lastProcessedId != null)
filesQuery = filesQuery.Where(f => string.Compare(f.Id, lastProcessedId) > 0);
baseQuery = baseQuery.Where(f => string.Compare(f.Id, lastProcessedId) > 0);
var fileBatch = await filesQuery
.OrderBy(f => f.Id) // Ensure consistent ordering for pagination
var fileBatch = await baseQuery
.OrderBy(f => f.Id)
.Take(batchSize)
.Select(f => f.Id)
.ToListAsync();
@@ -80,12 +84,11 @@ public class CloudFileUnusedRecyclingJob(
processedCount += fileBatch.Count;
lastProcessedId = fileBatch.Last();
// Optimized query: Find files that have no other cloud files sharing the same object
// A file is considered "unused" if no other SnCloudFile shares its ObjectId
// Optimized query: Find files that have no file object or no replicas
// A file is considered "unused" if its file object has no replicas
var filesToMark = await db.Files
.Where(f => fileBatch.Contains(f.Id))
.Where(f => f.ObjectId == null || // No file object at all
!db.Files.Any(cf => cf.ObjectId == f.ObjectId && cf.Id != f.Id)) // Or no other files share this object
.Where(f => f.Object == null || f.Object.FileReplicas.Count == 0)
.Select(f => f.Id)
.ToListAsync();

8
DysonNetwork.Drive/Storage/FileController.cs
View File

@@ -253,11 +253,12 @@ public class FileController(
string? overrideMimeType
)
{
if (!file.PoolId.HasValue)
var primaryReplica = file.Object?.FileReplicas.FirstOrDefault(r => r.IsPrimary);
if (primaryReplica == null || primaryReplica.PoolId == null)
return StatusCode(StatusCodes.Status500InternalServerError,
"File is in an inconsistent state: uploaded but no pool ID.");
var pool = await fs.GetPoolAsync(file.PoolId.Value);
var pool = await fs.GetPoolAsync(primaryReplica.PoolId.Value);
if (pool is null)
return StatusCode(StatusCodes.Status410Gone, "The pool of the file no longer exists or not accessible.");
@@ -461,11 +462,10 @@ public class FileController(
var filesQuery = db.Files
.Where(e => e.IsMarkedRecycle == recycled)
.Where(e => e.AccountId == accountId)
.Include(e => e.Pool)
.Include(e => e.Object)
.AsQueryable();
if (pool.HasValue) filesQuery = filesQuery.Where(e => e.PoolId == pool);
if (pool.HasValue) filesQuery = filesQuery.Where(e => e.Object!.FileReplicas.Any(r => r.PoolId == pool.Value));
if (!string.IsNullOrWhiteSpace(query))
{

127
DysonNetwork.Drive/Storage/FileMigrationService.cs
View File

@@ -1,127 +0,0 @@
using DysonNetwork.Shared.Models;
using Microsoft.EntityFrameworkCore;
namespace DysonNetwork.Drive.Storage;
public class FileMigrationService(AppDatabase db, ILogger<FileMigrationService> logger)
{
public async Task MigrateCloudFilesAsync()
{
logger.LogInformation("Starting cloud file migration.");
var cloudFiles = await db.Files
.Where(f =>
f.ObjectId == null &&
f.PoolId != null
)
.ToListAsync();
logger.LogDebug("Found {Count} cloud files to migrate.", cloudFiles.Count);
foreach (var cf in cloudFiles)
{
try
{
var ext = Path.GetExtension(cf.Name);
var mimeType = ext != "" && MimeTypes.TryGetMimeType(ext, out var mime) ? mime : "application/octet-stream";
var fileObject = await db.FileObjects.FindAsync(cf.Id);
if (fileObject == null)
{
fileObject = new SnFileObject
{
Id = cf.Id,
MimeType = mimeType,
HasCompression = mimeType.StartsWith("image/"),
HasThumbnail = mimeType.StartsWith("video/")
};
db.FileObjects.Add(fileObject);
}
var replicaExists = await db.FileReplicas.AnyAsync(r =>
r.ObjectId == fileObject.Id &&
r.PoolId == cf.PoolId!.Value);
if (!replicaExists)
{
var fileReplica = new SnFileReplica
{
Id = Guid.NewGuid(),
ObjectId = fileObject.Id,
PoolId = cf.PoolId!.Value,
StorageId = cf.StorageId ?? cf.Id,
Status = SnFileReplicaStatus.Available,
IsPrimary = true
};
fileObject.FileReplicas.Add(fileReplica);
db.FileReplicas.Add(fileReplica);
}
var permissionExists = await db.FilePermissions.AnyAsync(p => p.FileId == cf.Id);
if (!permissionExists)
{
var permission = new SnFilePermission
{
Id = Guid.NewGuid(),
FileId = cf.Id,
SubjectType = SnFilePermissionType.Anyone,
SubjectId = string.Empty,
Permission = SnFilePermissionLevel.Read
};
db.FilePermissions.Add(permission);
}
cf.ObjectId = fileObject.Id;
cf.Object = fileObject;
await db.SaveChangesAsync();
logger.LogInformation("Migrated file {FileId} successfully.", cf.Id);
}
catch (Exception ex)
{
logger.LogError(ex,
"Failed migrating file {FileId}. ObjectId={ObjectId}, PoolId={PoolId}, StorageId={StorageId}",
cf.Id,
cf.ObjectId,
cf.PoolId,
cf.StorageId);
}
}
logger.LogInformation("Cloud file migration completed.");
}
public async Task MigratePermissionsAsync()
{
logger.LogInformation("Starting file permission migration.");
var filesWithoutPermission = await db.Files
.Where(f => !db.FilePermissions.Any(p => p.FileId == f.Id))
.ToListAsync();
logger.LogDebug("Found {Count} files without permissions.", filesWithoutPermission.Count);
foreach (var file in filesWithoutPermission)
{
var permission = new SnFilePermission
{
Id = Guid.NewGuid(),
FileId = file.Id,
SubjectType = SnFilePermissionType.Anyone,
SubjectId = string.Empty,
Permission = SnFilePermissionLevel.Read
};
db.FilePermissions.Add(permission);
}
await db.SaveChangesAsync();
logger.LogInformation("Permission migration completed. Created {Count} permissions.", filesWithoutPermission.Count);
}
}

21
DysonNetwork.Drive/Storage/FileReanalysisService.cs
View File

@@ -23,10 +23,9 @@ public class FileReanalysisService(
var now = SystemClock.Instance.GetCurrentInstant();
var deadline = now.Minus(Duration.FromMinutes(30));
return await db.Files
.Where(f => f.ObjectId != null && f.PoolId != null)
.Where(f => f.ObjectId != null)
.Include(f => f.Object)
.ThenInclude(f => f.FileReplicas)
.Include(f => f.Pool)
.Where(f => f.Object != null && (f.Object.Meta == null || f.Object.Meta.Count == 0))
.Where(f => f.Object!.FileReplicas.Count > 0)
.Where(f => f.CreatedAt <= deadline)
@@ -39,9 +38,9 @@ public class FileReanalysisService(
{
logger.LogInformation("Starting reanalysis for file {FileId}: {FileName}", file.Id, file.Name);
if (file.Object == null || file.Pool == null)
if (file.Object == null)
{
logger.LogWarning("File {FileId} missing object or pool, skipping reanalysis", file.Id);
logger.LogWarning("File {FileId} missing object, skipping reanalysis", file.Id);
return true; // not a failure
}
@@ -147,16 +146,22 @@ public class FileReanalysisService(
private async Task DownloadFileAsync(SnCloudFile file, SnFileReplica replica, string tempPath)
{
var dest = file.Pool!.StorageConfig;
if (dest == null)
if (replica.PoolId == null)
{
throw new InvalidOperationException($"No remote storage configured for pool {file.PoolId}");
throw new InvalidOperationException($"Replica for file {file.Id} has no pool ID");
}
var pool = await db.Pools.FindAsync(replica.PoolId.Value);
if (pool == null)
{
throw new InvalidOperationException($"No remote storage configured for pool {replica.PoolId}");
}
var dest = pool.StorageConfig;
var client = CreateMinioClient(dest);
if (client == null)
{
throw new InvalidOperationException($"Failed to create Minio client for pool {file.PoolId}");
throw new InvalidOperationException($"Failed to create Minio client for pool {replica.PoolId}");
}
await using var fileStream = File.Create(tempPath);

61
DysonNetwork.Drive/Storage/FileService.cs
View File

@@ -38,7 +38,6 @@ public class FileService(
var file = await db.Files
.Where(f => f.Id == fileId)
.Include(f => f.Pool)
.Include(f => f.Bundle)
.Include(f => f.Object)
.ThenInclude(o => o.FileReplicas)
@@ -70,7 +69,7 @@ public class FileService(
{
var dbFiles = await db.Files
.Where(f => uncachedIds.Contains(f.Id))
.Include(f => f.Pool)
.Include(f => f.Bundle)
.Include(f => f.Object)
.ThenInclude(o => o.FileReplicas)
.ToListAsync();
@@ -124,7 +123,7 @@ public class FileService(
fileObject.Hash = await HashFileAsync(processingPath);
await SaveFileToDatabaseAsync(file, fileObject);
await SaveFileToDatabaseAsync(file, fileObject, pool.Id);
await PublishFileUploadedEventAsync(file, pool, processingPath, isTempFile);
@@ -245,13 +244,13 @@ public class FileService(
return Task.FromResult((encryptedPath, true));
}
private async Task SaveFileToDatabaseAsync(SnCloudFile file, SnFileObject fileObject)
private async Task SaveFileToDatabaseAsync(SnCloudFile file, SnFileObject fileObject, Guid poolId)
{
var replica = new SnFileReplica
{
Id = Guid.NewGuid(),
ObjectId = file.Id,
PoolId = file.PoolId,
PoolId = poolId,
StorageId = file.StorageId ?? file.Id,
Status = SnFileReplicaStatus.Available,
IsPrimary = true
@@ -540,7 +539,30 @@ public class FileService(
public async Task DeleteFileDataAsync(SnCloudFile file, bool force = false)
{
if (!file.PoolId.HasValue || file.ObjectId == null) return;
if (file.ObjectId == null) return;
var replicas = await db.FileReplicas
.Where(r => r.ObjectId == file.ObjectId)
.ToListAsync();
if (replicas.Count == 0)
{
logger.LogWarning("No replicas found for file object {ObjectId}", file.ObjectId);
return;
}
var primaryReplica = replicas.FirstOrDefault(r => r.IsPrimary);
if (primaryReplica == null)
{
logger.LogWarning("No primary replica found for file object {ObjectId}", file.ObjectId);
return;
}
if (primaryReplica.PoolId == null)
{
logger.LogWarning("Primary replica has no pool ID for file object {ObjectId}", file.ObjectId);
return;
}
if (!force)
{
@@ -553,23 +575,12 @@ public class FileService(
return;
}
var replicas = await db.FileReplicas
.Where(r => r.ObjectId == file.ObjectId)
.ToListAsync();
if (replicas.Count == 0)
{
logger.LogWarning("No replicas found for file object {ObjectId}", file.ObjectId);
return;
}
var primaryReplica = replicas.First(r => r.IsPrimary);
var dest = await GetRemoteStorageConfig(file.PoolId.Value);
if (dest is null) throw new InvalidOperationException($"No remote storage configured for pool {file.PoolId}");
var dest = await GetRemoteStorageConfig(primaryReplica.PoolId.Value);
if (dest is null) throw new InvalidOperationException($"No remote storage configured for pool {primaryReplica.PoolId}");
var client = CreateMinioClient(dest);
if (client is null)
throw new InvalidOperationException(
$"Failed to configure client for remote destination '{file.PoolId}'"
$"Failed to configure client for remote destination '{primaryReplica.PoolId}'"
);
var bucket = dest.Bucket;
@@ -615,7 +626,7 @@ public class FileService(
public async Task DeleteFileDataBatchAsync(List<SnCloudFile> files)
{
files = files.Where(f => f.PoolId.HasValue && f.ObjectId != null).ToList();
files = files.Where(f => f.ObjectId != null).ToList();
var objectIds = files.Select(f => f.ObjectId).Distinct().ToList();
var replicas = await db.FileReplicas
@@ -759,8 +770,14 @@ public class FileService(
public async Task<int> DeletePoolRecycledFilesAsync(Guid poolId)
{
var fileIdsWithReplicas = await db.FileReplicas
.Where(r => r.PoolId == poolId)
.Select(r => r.ObjectId)
.Distinct()
.ToListAsync();
var files = await db.Files
.Where(f => f.PoolId == poolId && f.IsMarkedRecycle)
.Where(f => fileIdsWithReplicas.Contains(f.Id) && f.IsMarkedRecycle)
.ToListAsync();
var count = files.Count;
var fileIds = files.Select(f => f.Id).ToList();

2
DysonNetwork.Shared/Models/CloudFile.cs
View File

@@ -32,8 +32,6 @@ public class SnCloudFile : ModelBase, ICloudFile, IIdentifiedResource
[MaxLength(32)] public string? ObjectId { get; set; }
public SnFileObject? Object { get; set; }
public FilePool? Pool { get; set; }
public Guid? PoolId { get; set; }
[JsonIgnore] public SnFileBundle? Bundle { get; set; }
public Guid? BundleId { get; set; }
[JsonIgnore] public List<SnCloudFileIndex> FileIndexes { get; set; } = [];