♻️ Update the usage counting since the pool id logic changed
This commit is contained in:
@@ -1,5 +1,6 @@
|
|||||||
using Microsoft.EntityFrameworkCore;
|
using Microsoft.EntityFrameworkCore;
|
||||||
using NodaTime;
|
using NodaTime;
|
||||||
|
using DysonNetwork.Shared.Models;
|
||||||
|
|
||||||
namespace DysonNetwork.Drive.Billing;
|
namespace DysonNetwork.Drive.Billing;
|
||||||
|
|
||||||
@@ -29,28 +30,42 @@ public class UsageService(AppDatabase db)
|
|||||||
public async Task<TotalUsageDetails> GetTotalUsage(Guid accountId)
|
public async Task<TotalUsageDetails> GetTotalUsage(Guid accountId)
|
||||||
{
|
{
|
||||||
var now = SystemClock.Instance.GetCurrentInstant();
|
var now = SystemClock.Instance.GetCurrentInstant();
|
||||||
var fileQuery = db.Files
|
|
||||||
.Where(f => !f.IsMarkedRecycle)
|
|
||||||
.Where(f => !f.ExpiredAt.HasValue || f.ExpiredAt > now)
|
|
||||||
.Where(f => f.AccountId == accountId)
|
|
||||||
.AsQueryable();
|
|
||||||
|
|
||||||
var poolUsages = await db.Pools
|
var poolUsages = await db.Pools
|
||||||
.Select(p => new UsageDetails
|
.Select(p => new UsageDetails
|
||||||
{
|
{
|
||||||
PoolId = p.Id,
|
PoolId = p.Id,
|
||||||
PoolName = p.Name,
|
PoolName = p.Name,
|
||||||
UsageBytes = fileQuery
|
UsageBytes = db.Files
|
||||||
.Where(f => f.PoolId == p.Id)
|
.Where(f => f.AccountId == accountId)
|
||||||
.Include(f => f.Object)
|
.Where(f => !f.IsMarkedRecycle)
|
||||||
.Sum(f => f.Size),
|
.Where(f => !f.ExpiredAt.HasValue || f.ExpiredAt > now)
|
||||||
Cost = fileQuery
|
.SelectMany(f => f.Object!.FileReplicas
|
||||||
.Where(f => f.PoolId == p.Id)
|
.Where(r => r.PoolId == p.Id && r.Status == SnFileReplicaStatus.Available))
|
||||||
.Include(f => f.Object)
|
.Join(db.FileObjects,
|
||||||
.Sum(f => f.Size) / 1024.0 / 1024.0 *
|
r => r.ObjectId,
|
||||||
(p.BillingConfig.CostMultiplier ?? 1.0),
|
o => o.Id,
|
||||||
FileCount = fileQuery
|
(r, o) => o.Size)
|
||||||
.Count(f => f.PoolId == p.Id)
|
.DefaultIfEmpty(0L)
|
||||||
|
.Sum(),
|
||||||
|
Cost = db.Files
|
||||||
|
.Where(f => f.AccountId == accountId)
|
||||||
|
.Where(f => !f.IsMarkedRecycle)
|
||||||
|
.Where(f => !f.ExpiredAt.HasValue || f.ExpiredAt > now)
|
||||||
|
.SelectMany(f => f.Object!.FileReplicas
|
||||||
|
.Where(r => r.PoolId == p.Id && r.Status == SnFileReplicaStatus.Available))
|
||||||
|
.Join(db.FileObjects,
|
||||||
|
r => r.ObjectId,
|
||||||
|
o => o.Id,
|
||||||
|
(r, o) => new { Size = o.Size, Multiplier = p.BillingConfig.CostMultiplier ?? 1.0 })
|
||||||
|
.Sum(x => x.Size * x.Multiplier) / 1024.0 / 1024.0,
|
||||||
|
FileCount = db.Files
|
||||||
|
.Where(f => f.AccountId == accountId)
|
||||||
|
.Where(f => !f.IsMarkedRecycle)
|
||||||
|
.Where(f => !f.ExpiredAt.HasValue || f.ExpiredAt > now)
|
||||||
|
.SelectMany(f => f.Object!.FileReplicas
|
||||||
|
.Where(r => r.PoolId == p.Id && r.Status == SnFileReplicaStatus.Available))
|
||||||
|
.Count()
|
||||||
})
|
})
|
||||||
.ToListAsync();
|
.ToListAsync();
|
||||||
|
|
||||||
@@ -75,18 +90,22 @@ public class UsageService(AppDatabase db)
|
|||||||
}
|
}
|
||||||
|
|
||||||
var now = SystemClock.Instance.GetCurrentInstant();
|
var now = SystemClock.Instance.GetCurrentInstant();
|
||||||
var fileQuery = db.Files
|
var replicaQuery = db.Files
|
||||||
.Where(f => !f.IsMarkedRecycle)
|
|
||||||
.Where(f => f.ExpiredAt.HasValue && f.ExpiredAt > now)
|
|
||||||
.Where(f => f.AccountId == accountId)
|
.Where(f => f.AccountId == accountId)
|
||||||
.AsQueryable();
|
.Where(f => !f.IsMarkedRecycle)
|
||||||
|
.Where(f => !f.ExpiredAt.HasValue || f.ExpiredAt > now)
|
||||||
|
.SelectMany(f => f.Object!.FileReplicas
|
||||||
|
.Where(r => r.PoolId == poolId && r.Status == SnFileReplicaStatus.Available));
|
||||||
|
|
||||||
var usageBytes = await fileQuery
|
var usageBytes = await replicaQuery
|
||||||
.Include(f => f.Object)
|
.Join(db.FileObjects,
|
||||||
.SumAsync(f => f.Size);
|
r => r.ObjectId,
|
||||||
|
o => o.Id,
|
||||||
|
(r, o) => o.Size)
|
||||||
|
.DefaultIfEmpty(0L)
|
||||||
|
.SumAsync();
|
||||||
|
|
||||||
var fileCount = await fileQuery
|
var fileCount = await replicaQuery.CountAsync();
|
||||||
.CountAsync();
|
|
||||||
|
|
||||||
var cost = usageBytes / 1024.0 / 1024.0 *
|
var cost = usageBytes / 1024.0 / 1024.0 *
|
||||||
(pool.BillingConfig.CostMultiplier ?? 1.0);
|
(pool.BillingConfig.CostMultiplier ?? 1.0);
|
||||||
@@ -104,22 +123,24 @@ public class UsageService(AppDatabase db)
|
|||||||
public async Task<long> GetTotalBillableUsage(Guid accountId)
|
public async Task<long> GetTotalBillableUsage(Guid accountId)
|
||||||
{
|
{
|
||||||
var now = SystemClock.Instance.GetCurrentInstant();
|
var now = SystemClock.Instance.GetCurrentInstant();
|
||||||
var files = await db.Files
|
|
||||||
.Where(f => f.AccountId == accountId)
|
var billingData = await (from f in db.Files
|
||||||
.Where(f => f.PoolId.HasValue)
|
where f.AccountId == accountId
|
||||||
.Where(f => !f.IsMarkedRecycle)
|
where !f.IsMarkedRecycle
|
||||||
.Include(f => f.Pool)
|
where !f.ExpiredAt.HasValue || f.ExpiredAt > now
|
||||||
.Include(f => f.Object)
|
from r in f.Object!.FileReplicas
|
||||||
.Where(f => !f.ExpiredAt.HasValue || f.ExpiredAt > now)
|
where r.Status == SnFileReplicaStatus.Available
|
||||||
.Select(f => new
|
where r.PoolId.HasValue
|
||||||
|
join p in db.Pools on r.PoolId equals p.Id
|
||||||
|
join o in db.FileObjects on r.ObjectId equals o.Id
|
||||||
|
select new
|
||||||
{
|
{
|
||||||
f.Size,
|
Size = o.Size,
|
||||||
Multiplier = f.Pool!.BillingConfig.CostMultiplier ?? 1.0
|
Multiplier = p.BillingConfig.CostMultiplier ?? 1.0
|
||||||
})
|
}).ToListAsync();
|
||||||
.ToListAsync();
|
|
||||||
|
|
||||||
var totalCost = files.Sum(f => f.Size * f.Multiplier) / 1024.0 / 1024.0;
|
var totalCost = billingData.Sum(x => x.Size * x.Multiplier) / 1024.0 / 1024.0;
|
||||||
|
|
||||||
return (long)Math.Ceiling(totalCost);
|
return (long)Math.Ceiling(totalCost);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -230,7 +230,7 @@ public class FileIndexController(
|
|||||||
: filesQuery.OrderBy(f => f.CreatedAt)
|
: filesQuery.OrderBy(f => f.CreatedAt)
|
||||||
};
|
};
|
||||||
|
|
||||||
if (pool.HasValue) filesQuery = filesQuery.Where(f => f.PoolId == pool);
|
if (pool.HasValue) filesQuery = filesQuery.Where(f => f.Object!.FileReplicas.Any(r => r.PoolId == pool.Value));
|
||||||
|
|
||||||
if (!string.IsNullOrWhiteSpace(query))
|
if (!string.IsNullOrWhiteSpace(query))
|
||||||
{
|
{
|
||||||
|
|||||||
640
DysonNetwork.Drive/Migrations/20260113152536_RemovePoolFromCloudFile.Designer.cs
generated
Normal file
640
DysonNetwork.Drive/Migrations/20260113152536_RemovePoolFromCloudFile.Designer.cs
generated
Normal file
@@ -0,0 +1,640 @@
|
|||||||
|
// <auto-generated />
|
||||||
|
using System;
|
||||||
|
using System.Collections.Generic;
|
||||||
|
using DysonNetwork.Drive;
|
||||||
|
using DysonNetwork.Shared.Models;
|
||||||
|
using Microsoft.EntityFrameworkCore;
|
||||||
|
using Microsoft.EntityFrameworkCore.Infrastructure;
|
||||||
|
using Microsoft.EntityFrameworkCore.Migrations;
|
||||||
|
using Microsoft.EntityFrameworkCore.Storage.ValueConversion;
|
||||||
|
using NodaTime;
|
||||||
|
using Npgsql.EntityFrameworkCore.PostgreSQL.Metadata;
|
||||||
|
|
||||||
|
#nullable disable
|
||||||
|
|
||||||
|
namespace DysonNetwork.Drive.Migrations
|
||||||
|
{
|
||||||
|
[DbContext(typeof(AppDatabase))]
|
||||||
|
[Migration("20260113152536_RemovePoolFromCloudFile")]
|
||||||
|
partial class RemovePoolFromCloudFile
|
||||||
|
{
|
||||||
|
/// <inheritdoc />
|
||||||
|
protected override void BuildTargetModel(ModelBuilder modelBuilder)
|
||||||
|
{
|
||||||
|
#pragma warning disable 612, 618
|
||||||
|
modelBuilder
|
||||||
|
.HasAnnotation("ProductVersion", "10.0.1")
|
||||||
|
.HasAnnotation("Relational:MaxIdentifierLength", 63);
|
||||||
|
|
||||||
|
NpgsqlModelBuilderExtensions.UseIdentityByDefaultColumns(modelBuilder);
|
||||||
|
|
||||||
|
modelBuilder.Entity("DysonNetwork.Drive.Billing.QuotaRecord", b =>
|
||||||
|
{
|
||||||
|
b.Property<Guid>("Id")
|
||||||
|
.ValueGeneratedOnAdd()
|
||||||
|
.HasColumnType("uuid")
|
||||||
|
.HasColumnName("id");
|
||||||
|
|
||||||
|
b.Property<Guid>("AccountId")
|
||||||
|
.HasColumnType("uuid")
|
||||||
|
.HasColumnName("account_id");
|
||||||
|
|
||||||
|
b.Property<Instant>("CreatedAt")
|
||||||
|
.HasColumnType("timestamp with time zone")
|
||||||
|
.HasColumnName("created_at");
|
||||||
|
|
||||||
|
b.Property<Instant?>("DeletedAt")
|
||||||
|
.HasColumnType("timestamp with time zone")
|
||||||
|
.HasColumnName("deleted_at");
|
||||||
|
|
||||||
|
b.Property<string>("Description")
|
||||||
|
.IsRequired()
|
||||||
|
.HasColumnType("text")
|
||||||
|
.HasColumnName("description");
|
||||||
|
|
||||||
|
b.Property<Instant?>("ExpiredAt")
|
||||||
|
.HasColumnType("timestamp with time zone")
|
||||||
|
.HasColumnName("expired_at");
|
||||||
|
|
||||||
|
b.Property<string>("Name")
|
||||||
|
.IsRequired()
|
||||||
|
.HasColumnType("text")
|
||||||
|
.HasColumnName("name");
|
||||||
|
|
||||||
|
b.Property<long>("Quota")
|
||||||
|
.HasColumnType("bigint")
|
||||||
|
.HasColumnName("quota");
|
||||||
|
|
||||||
|
b.Property<Instant>("UpdatedAt")
|
||||||
|
.HasColumnType("timestamp with time zone")
|
||||||
|
.HasColumnName("updated_at");
|
||||||
|
|
||||||
|
b.HasKey("Id")
|
||||||
|
.HasName("pk_quota_records");
|
||||||
|
|
||||||
|
b.ToTable("quota_records", (string)null);
|
||||||
|
});
|
||||||
|
|
||||||
|
modelBuilder.Entity("DysonNetwork.Drive.Storage.Model.PersistentTask", b =>
|
||||||
|
{
|
||||||
|
b.Property<Guid>("Id")
|
||||||
|
.ValueGeneratedOnAdd()
|
||||||
|
.HasColumnType("uuid")
|
||||||
|
.HasColumnName("id");
|
||||||
|
|
||||||
|
b.Property<Guid>("AccountId")
|
||||||
|
.HasColumnType("uuid")
|
||||||
|
.HasColumnName("account_id");
|
||||||
|
|
||||||
|
b.Property<Instant?>("CompletedAt")
|
||||||
|
.HasColumnType("timestamp with time zone")
|
||||||
|
.HasColumnName("completed_at");
|
||||||
|
|
||||||
|
b.Property<Instant>("CreatedAt")
|
||||||
|
.HasColumnType("timestamp with time zone")
|
||||||
|
.HasColumnName("created_at");
|
||||||
|
|
||||||
|
b.Property<Instant?>("DeletedAt")
|
||||||
|
.HasColumnType("timestamp with time zone")
|
||||||
|
.HasColumnName("deleted_at");
|
||||||
|
|
||||||
|
b.Property<string>("Description")
|
||||||
|
.HasMaxLength(1024)
|
||||||
|
.HasColumnType("character varying(1024)")
|
||||||
|
.HasColumnName("description");
|
||||||
|
|
||||||
|
b.Property<string>("ErrorMessage")
|
||||||
|
.HasMaxLength(1024)
|
||||||
|
.HasColumnType("character varying(1024)")
|
||||||
|
.HasColumnName("error_message");
|
||||||
|
|
||||||
|
b.Property<long?>("EstimatedDurationSeconds")
|
||||||
|
.HasColumnType("bigint")
|
||||||
|
.HasColumnName("estimated_duration_seconds");
|
||||||
|
|
||||||
|
b.Property<Instant?>("ExpiredAt")
|
||||||
|
.HasColumnType("timestamp with time zone")
|
||||||
|
.HasColumnName("expired_at");
|
||||||
|
|
||||||
|
b.Property<Instant>("LastActivity")
|
||||||
|
.HasColumnType("timestamp with time zone")
|
||||||
|
.HasColumnName("last_activity");
|
||||||
|
|
||||||
|
b.Property<string>("Name")
|
||||||
|
.IsRequired()
|
||||||
|
.HasMaxLength(256)
|
||||||
|
.HasColumnType("character varying(256)")
|
||||||
|
.HasColumnName("name");
|
||||||
|
|
||||||
|
b.Property<Dictionary<string, object>>("Parameters")
|
||||||
|
.IsRequired()
|
||||||
|
.HasColumnType("jsonb")
|
||||||
|
.HasColumnName("parameters");
|
||||||
|
|
||||||
|
b.Property<int>("Priority")
|
||||||
|
.HasColumnType("integer")
|
||||||
|
.HasColumnName("priority");
|
||||||
|
|
||||||
|
b.Property<double>("Progress")
|
||||||
|
.HasColumnType("double precision")
|
||||||
|
.HasColumnName("progress");
|
||||||
|
|
||||||
|
b.Property<Dictionary<string, object>>("Results")
|
||||||
|
.IsRequired()
|
||||||
|
.HasColumnType("jsonb")
|
||||||
|
.HasColumnName("results");
|
||||||
|
|
||||||
|
b.Property<Instant?>("StartedAt")
|
||||||
|
.HasColumnType("timestamp with time zone")
|
||||||
|
.HasColumnName("started_at");
|
||||||
|
|
||||||
|
b.Property<int>("Status")
|
||||||
|
.HasColumnType("integer")
|
||||||
|
.HasColumnName("status");
|
||||||
|
|
||||||
|
b.Property<string>("TaskId")
|
||||||
|
.IsRequired()
|
||||||
|
.HasMaxLength(64)
|
||||||
|
.HasColumnType("character varying(64)")
|
||||||
|
.HasColumnName("task_id");
|
||||||
|
|
||||||
|
b.Property<int>("Type")
|
||||||
|
.HasColumnType("integer")
|
||||||
|
.HasColumnName("type");
|
||||||
|
|
||||||
|
b.Property<Instant>("UpdatedAt")
|
||||||
|
.HasColumnType("timestamp with time zone")
|
||||||
|
.HasColumnName("updated_at");
|
||||||
|
|
||||||
|
b.HasKey("Id")
|
||||||
|
.HasName("pk_tasks");
|
||||||
|
|
||||||
|
b.ToTable("tasks", (string)null);
|
||||||
|
});
|
||||||
|
|
||||||
|
modelBuilder.Entity("DysonNetwork.Shared.Models.FilePool", b =>
|
||||||
|
{
|
||||||
|
b.Property<Guid>("Id")
|
||||||
|
.ValueGeneratedOnAdd()
|
||||||
|
.HasColumnType("uuid")
|
||||||
|
.HasColumnName("id");
|
||||||
|
|
||||||
|
b.Property<Guid?>("AccountId")
|
||||||
|
.HasColumnType("uuid")
|
||||||
|
.HasColumnName("account_id");
|
||||||
|
|
||||||
|
b.Property<BillingConfig>("BillingConfig")
|
||||||
|
.IsRequired()
|
||||||
|
.HasColumnType("jsonb")
|
||||||
|
.HasColumnName("billing_config");
|
||||||
|
|
||||||
|
b.Property<Instant>("CreatedAt")
|
||||||
|
.HasColumnType("timestamp with time zone")
|
||||||
|
.HasColumnName("created_at");
|
||||||
|
|
||||||
|
b.Property<Instant?>("DeletedAt")
|
||||||
|
.HasColumnType("timestamp with time zone")
|
||||||
|
.HasColumnName("deleted_at");
|
||||||
|
|
||||||
|
b.Property<string>("Description")
|
||||||
|
.IsRequired()
|
||||||
|
.HasMaxLength(8192)
|
||||||
|
.HasColumnType("character varying(8192)")
|
||||||
|
.HasColumnName("description");
|
||||||
|
|
||||||
|
b.Property<bool>("IsHidden")
|
||||||
|
.HasColumnType("boolean")
|
||||||
|
.HasColumnName("is_hidden");
|
||||||
|
|
||||||
|
b.Property<string>("Name")
|
||||||
|
.IsRequired()
|
||||||
|
.HasMaxLength(1024)
|
||||||
|
.HasColumnType("character varying(1024)")
|
||||||
|
.HasColumnName("name");
|
||||||
|
|
||||||
|
b.Property<PolicyConfig>("PolicyConfig")
|
||||||
|
.IsRequired()
|
||||||
|
.HasColumnType("jsonb")
|
||||||
|
.HasColumnName("policy_config");
|
||||||
|
|
||||||
|
b.Property<RemoteStorageConfig>("StorageConfig")
|
||||||
|
.IsRequired()
|
||||||
|
.HasColumnType("jsonb")
|
||||||
|
.HasColumnName("storage_config");
|
||||||
|
|
||||||
|
b.Property<Instant>("UpdatedAt")
|
||||||
|
.HasColumnType("timestamp with time zone")
|
||||||
|
.HasColumnName("updated_at");
|
||||||
|
|
||||||
|
b.HasKey("Id")
|
||||||
|
.HasName("pk_pools");
|
||||||
|
|
||||||
|
b.ToTable("pools", (string)null);
|
||||||
|
});
|
||||||
|
|
||||||
|
modelBuilder.Entity("DysonNetwork.Shared.Models.SnCloudFile", b =>
|
||||||
|
{
|
||||||
|
b.Property<string>("Id")
|
||||||
|
.HasMaxLength(32)
|
||||||
|
.HasColumnType("character varying(32)")
|
||||||
|
.HasColumnName("id");
|
||||||
|
|
||||||
|
b.Property<Guid>("AccountId")
|
||||||
|
.HasColumnType("uuid")
|
||||||
|
.HasColumnName("account_id");
|
||||||
|
|
||||||
|
b.Property<Guid?>("BundleId")
|
||||||
|
.HasColumnType("uuid")
|
||||||
|
.HasColumnName("bundle_id");
|
||||||
|
|
||||||
|
b.Property<Instant>("CreatedAt")
|
||||||
|
.HasColumnType("timestamp with time zone")
|
||||||
|
.HasColumnName("created_at");
|
||||||
|
|
||||||
|
b.Property<Instant?>("DeletedAt")
|
||||||
|
.HasColumnType("timestamp with time zone")
|
||||||
|
.HasColumnName("deleted_at");
|
||||||
|
|
||||||
|
b.Property<string>("Description")
|
||||||
|
.HasMaxLength(4096)
|
||||||
|
.HasColumnType("character varying(4096)")
|
||||||
|
.HasColumnName("description");
|
||||||
|
|
||||||
|
b.Property<Instant?>("ExpiredAt")
|
||||||
|
.HasColumnType("timestamp with time zone")
|
||||||
|
.HasColumnName("expired_at");
|
||||||
|
|
||||||
|
b.Property<bool>("IsMarkedRecycle")
|
||||||
|
.HasColumnType("boolean")
|
||||||
|
.HasColumnName("is_marked_recycle");
|
||||||
|
|
||||||
|
b.Property<string>("Name")
|
||||||
|
.IsRequired()
|
||||||
|
.HasMaxLength(1024)
|
||||||
|
.HasColumnType("character varying(1024)")
|
||||||
|
.HasColumnName("name");
|
||||||
|
|
||||||
|
b.Property<string>("ObjectId")
|
||||||
|
.HasMaxLength(32)
|
||||||
|
.HasColumnType("character varying(32)")
|
||||||
|
.HasColumnName("object_id");
|
||||||
|
|
||||||
|
b.PrimitiveCollection<string>("SensitiveMarks")
|
||||||
|
.HasColumnType("jsonb")
|
||||||
|
.HasColumnName("sensitive_marks");
|
||||||
|
|
||||||
|
b.Property<string>("StorageId")
|
||||||
|
.HasMaxLength(32)
|
||||||
|
.HasColumnType("character varying(32)")
|
||||||
|
.HasColumnName("storage_id");
|
||||||
|
|
||||||
|
b.Property<string>("StorageUrl")
|
||||||
|
.HasMaxLength(4096)
|
||||||
|
.HasColumnType("character varying(4096)")
|
||||||
|
.HasColumnName("storage_url");
|
||||||
|
|
||||||
|
b.Property<Instant>("UpdatedAt")
|
||||||
|
.HasColumnType("timestamp with time zone")
|
||||||
|
.HasColumnName("updated_at");
|
||||||
|
|
||||||
|
b.Property<Instant?>("UploadedAt")
|
||||||
|
.HasColumnType("timestamp with time zone")
|
||||||
|
.HasColumnName("uploaded_at");
|
||||||
|
|
||||||
|
b.Property<Dictionary<string, object>>("UserMeta")
|
||||||
|
.HasColumnType("jsonb")
|
||||||
|
.HasColumnName("user_meta");
|
||||||
|
|
||||||
|
b.HasKey("Id")
|
||||||
|
.HasName("pk_files");
|
||||||
|
|
||||||
|
b.HasIndex("BundleId")
|
||||||
|
.HasDatabaseName("ix_files_bundle_id");
|
||||||
|
|
||||||
|
b.HasIndex("ObjectId")
|
||||||
|
.HasDatabaseName("ix_files_object_id");
|
||||||
|
|
||||||
|
b.ToTable("files", (string)null);
|
||||||
|
});
|
||||||
|
|
||||||
|
modelBuilder.Entity("DysonNetwork.Shared.Models.SnCloudFileIndex", b =>
|
||||||
|
{
|
||||||
|
b.Property<Guid>("Id")
|
||||||
|
.ValueGeneratedOnAdd()
|
||||||
|
.HasColumnType("uuid")
|
||||||
|
.HasColumnName("id");
|
||||||
|
|
||||||
|
b.Property<Guid>("AccountId")
|
||||||
|
.HasColumnType("uuid")
|
||||||
|
.HasColumnName("account_id");
|
||||||
|
|
||||||
|
b.Property<Instant>("CreatedAt")
|
||||||
|
.HasColumnType("timestamp with time zone")
|
||||||
|
.HasColumnName("created_at");
|
||||||
|
|
||||||
|
b.Property<Instant?>("DeletedAt")
|
||||||
|
.HasColumnType("timestamp with time zone")
|
||||||
|
.HasColumnName("deleted_at");
|
||||||
|
|
||||||
|
b.Property<string>("FileId")
|
||||||
|
.IsRequired()
|
||||||
|
.HasMaxLength(32)
|
||||||
|
.HasColumnType("character varying(32)")
|
||||||
|
.HasColumnName("file_id");
|
||||||
|
|
||||||
|
b.Property<string>("Path")
|
||||||
|
.IsRequired()
|
||||||
|
.HasMaxLength(8192)
|
||||||
|
.HasColumnType("character varying(8192)")
|
||||||
|
.HasColumnName("path");
|
||||||
|
|
||||||
|
b.Property<Instant>("UpdatedAt")
|
||||||
|
.HasColumnType("timestamp with time zone")
|
||||||
|
.HasColumnName("updated_at");
|
||||||
|
|
||||||
|
b.HasKey("Id")
|
||||||
|
.HasName("pk_file_indexes");
|
||||||
|
|
||||||
|
b.HasIndex("FileId")
|
||||||
|
.HasDatabaseName("ix_file_indexes_file_id");
|
||||||
|
|
||||||
|
b.HasIndex("Path", "AccountId")
|
||||||
|
.HasDatabaseName("ix_file_indexes_path_account_id");
|
||||||
|
|
||||||
|
b.ToTable("file_indexes", (string)null);
|
||||||
|
});
|
||||||
|
|
||||||
|
modelBuilder.Entity("DysonNetwork.Shared.Models.SnFileBundle", b =>
|
||||||
|
{
|
||||||
|
b.Property<Guid>("Id")
|
||||||
|
.ValueGeneratedOnAdd()
|
||||||
|
.HasColumnType("uuid")
|
||||||
|
.HasColumnName("id");
|
||||||
|
|
||||||
|
b.Property<Guid>("AccountId")
|
||||||
|
.HasColumnType("uuid")
|
||||||
|
.HasColumnName("account_id");
|
||||||
|
|
||||||
|
b.Property<Instant>("CreatedAt")
|
||||||
|
.HasColumnType("timestamp with time zone")
|
||||||
|
.HasColumnName("created_at");
|
||||||
|
|
||||||
|
b.Property<Instant?>("DeletedAt")
|
||||||
|
.HasColumnType("timestamp with time zone")
|
||||||
|
.HasColumnName("deleted_at");
|
||||||
|
|
||||||
|
b.Property<string>("Description")
|
||||||
|
.HasMaxLength(8192)
|
||||||
|
.HasColumnType("character varying(8192)")
|
||||||
|
.HasColumnName("description");
|
||||||
|
|
||||||
|
b.Property<Instant?>("ExpiredAt")
|
||||||
|
.HasColumnType("timestamp with time zone")
|
||||||
|
.HasColumnName("expired_at");
|
||||||
|
|
||||||
|
b.Property<string>("Name")
|
||||||
|
.IsRequired()
|
||||||
|
.HasMaxLength(1024)
|
||||||
|
.HasColumnType("character varying(1024)")
|
||||||
|
.HasColumnName("name");
|
||||||
|
|
||||||
|
b.Property<string>("Passcode")
|
||||||
|
.HasMaxLength(256)
|
||||||
|
.HasColumnType("character varying(256)")
|
||||||
|
.HasColumnName("passcode");
|
||||||
|
|
||||||
|
b.Property<string>("Slug")
|
||||||
|
.IsRequired()
|
||||||
|
.HasMaxLength(1024)
|
||||||
|
.HasColumnType("character varying(1024)")
|
||||||
|
.HasColumnName("slug");
|
||||||
|
|
||||||
|
b.Property<Instant>("UpdatedAt")
|
||||||
|
.HasColumnType("timestamp with time zone")
|
||||||
|
.HasColumnName("updated_at");
|
||||||
|
|
||||||
|
b.HasKey("Id")
|
||||||
|
.HasName("pk_bundles");
|
||||||
|
|
||||||
|
b.HasIndex("Slug")
|
||||||
|
.IsUnique()
|
||||||
|
.HasDatabaseName("ix_bundles_slug");
|
||||||
|
|
||||||
|
b.ToTable("bundles", (string)null);
|
||||||
|
});
|
||||||
|
|
||||||
|
modelBuilder.Entity("DysonNetwork.Shared.Models.SnFileObject", b =>
|
||||||
|
{
|
||||||
|
b.Property<string>("Id")
|
||||||
|
.HasMaxLength(32)
|
||||||
|
.HasColumnType("character varying(32)")
|
||||||
|
.HasColumnName("id");
|
||||||
|
|
||||||
|
b.Property<Instant>("CreatedAt")
|
||||||
|
.HasColumnType("timestamp with time zone")
|
||||||
|
.HasColumnName("created_at");
|
||||||
|
|
||||||
|
b.Property<Instant?>("DeletedAt")
|
||||||
|
.HasColumnType("timestamp with time zone")
|
||||||
|
.HasColumnName("deleted_at");
|
||||||
|
|
||||||
|
b.Property<bool>("HasCompression")
|
||||||
|
.HasColumnType("boolean")
|
||||||
|
.HasColumnName("has_compression");
|
||||||
|
|
||||||
|
b.Property<bool>("HasThumbnail")
|
||||||
|
.HasColumnType("boolean")
|
||||||
|
.HasColumnName("has_thumbnail");
|
||||||
|
|
||||||
|
b.Property<string>("Hash")
|
||||||
|
.HasMaxLength(256)
|
||||||
|
.HasColumnType("character varying(256)")
|
||||||
|
.HasColumnName("hash");
|
||||||
|
|
||||||
|
b.Property<Dictionary<string, object>>("Meta")
|
||||||
|
.HasColumnType("jsonb")
|
||||||
|
.HasColumnName("meta");
|
||||||
|
|
||||||
|
b.Property<string>("MimeType")
|
||||||
|
.HasMaxLength(256)
|
||||||
|
.HasColumnType("character varying(256)")
|
||||||
|
.HasColumnName("mime_type");
|
||||||
|
|
||||||
|
b.Property<long>("Size")
|
||||||
|
.HasColumnType("bigint")
|
||||||
|
.HasColumnName("size");
|
||||||
|
|
||||||
|
b.Property<Instant>("UpdatedAt")
|
||||||
|
.HasColumnType("timestamp with time zone")
|
||||||
|
.HasColumnName("updated_at");
|
||||||
|
|
||||||
|
b.HasKey("Id")
|
||||||
|
.HasName("pk_file_objects");
|
||||||
|
|
||||||
|
b.ToTable("file_objects", (string)null);
|
||||||
|
});
|
||||||
|
|
||||||
|
modelBuilder.Entity("DysonNetwork.Shared.Models.SnFilePermission", b =>
|
||||||
|
{
|
||||||
|
b.Property<Guid>("Id")
|
||||||
|
.ValueGeneratedOnAdd()
|
||||||
|
.HasColumnType("uuid")
|
||||||
|
.HasColumnName("id");
|
||||||
|
|
||||||
|
b.Property<Instant>("CreatedAt")
|
||||||
|
.HasColumnType("timestamp with time zone")
|
||||||
|
.HasColumnName("created_at");
|
||||||
|
|
||||||
|
b.Property<Instant?>("DeletedAt")
|
||||||
|
.HasColumnType("timestamp with time zone")
|
||||||
|
.HasColumnName("deleted_at");
|
||||||
|
|
||||||
|
b.Property<string>("FileId")
|
||||||
|
.IsRequired()
|
||||||
|
.HasColumnType("text")
|
||||||
|
.HasColumnName("file_id");
|
||||||
|
|
||||||
|
b.Property<int>("Permission")
|
||||||
|
.HasColumnType("integer")
|
||||||
|
.HasColumnName("permission");
|
||||||
|
|
||||||
|
b.Property<string>("SubjectId")
|
||||||
|
.IsRequired()
|
||||||
|
.HasColumnType("text")
|
||||||
|
.HasColumnName("subject_id");
|
||||||
|
|
||||||
|
b.Property<int>("SubjectType")
|
||||||
|
.HasColumnType("integer")
|
||||||
|
.HasColumnName("subject_type");
|
||||||
|
|
||||||
|
b.Property<Instant>("UpdatedAt")
|
||||||
|
.HasColumnType("timestamp with time zone")
|
||||||
|
.HasColumnName("updated_at");
|
||||||
|
|
||||||
|
b.HasKey("Id")
|
||||||
|
.HasName("pk_file_permissions");
|
||||||
|
|
||||||
|
b.ToTable("file_permissions", (string)null);
|
||||||
|
});
|
||||||
|
|
||||||
|
modelBuilder.Entity("DysonNetwork.Shared.Models.SnFileReplica", b =>
|
||||||
|
{
|
||||||
|
b.Property<Guid>("Id")
|
||||||
|
.ValueGeneratedOnAdd()
|
||||||
|
.HasColumnType("uuid")
|
||||||
|
.HasColumnName("id");
|
||||||
|
|
||||||
|
b.Property<Instant>("CreatedAt")
|
||||||
|
.HasColumnType("timestamp with time zone")
|
||||||
|
.HasColumnName("created_at");
|
||||||
|
|
||||||
|
b.Property<Instant?>("DeletedAt")
|
||||||
|
.HasColumnType("timestamp with time zone")
|
||||||
|
.HasColumnName("deleted_at");
|
||||||
|
|
||||||
|
b.Property<bool>("IsPrimary")
|
||||||
|
.HasColumnType("boolean")
|
||||||
|
.HasColumnName("is_primary");
|
||||||
|
|
||||||
|
b.Property<string>("ObjectId")
|
||||||
|
.IsRequired()
|
||||||
|
.HasMaxLength(32)
|
||||||
|
.HasColumnType("character varying(32)")
|
||||||
|
.HasColumnName("object_id");
|
||||||
|
|
||||||
|
b.Property<Guid?>("PoolId")
|
||||||
|
.HasColumnType("uuid")
|
||||||
|
.HasColumnName("pool_id");
|
||||||
|
|
||||||
|
b.Property<int>("Status")
|
||||||
|
.HasColumnType("integer")
|
||||||
|
.HasColumnName("status");
|
||||||
|
|
||||||
|
b.Property<string>("StorageId")
|
||||||
|
.IsRequired()
|
||||||
|
.HasMaxLength(128)
|
||||||
|
.HasColumnType("character varying(128)")
|
||||||
|
.HasColumnName("storage_id");
|
||||||
|
|
||||||
|
b.Property<Instant>("UpdatedAt")
|
||||||
|
.HasColumnType("timestamp with time zone")
|
||||||
|
.HasColumnName("updated_at");
|
||||||
|
|
||||||
|
b.HasKey("Id")
|
||||||
|
.HasName("pk_file_replicas");
|
||||||
|
|
||||||
|
b.HasIndex("ObjectId")
|
||||||
|
.HasDatabaseName("ix_file_replicas_object_id");
|
||||||
|
|
||||||
|
b.HasIndex("PoolId")
|
||||||
|
.HasDatabaseName("ix_file_replicas_pool_id");
|
||||||
|
|
||||||
|
b.ToTable("file_replicas", (string)null);
|
||||||
|
});
|
||||||
|
|
||||||
|
modelBuilder.Entity("DysonNetwork.Shared.Models.SnCloudFile", b =>
|
||||||
|
{
|
||||||
|
b.HasOne("DysonNetwork.Shared.Models.SnFileBundle", "Bundle")
|
||||||
|
.WithMany("Files")
|
||||||
|
.HasForeignKey("BundleId")
|
||||||
|
.HasConstraintName("fk_files_bundles_bundle_id");
|
||||||
|
|
||||||
|
b.HasOne("DysonNetwork.Shared.Models.SnFileObject", "Object")
|
||||||
|
.WithMany()
|
||||||
|
.HasForeignKey("ObjectId")
|
||||||
|
.HasConstraintName("fk_files_file_objects_object_id");
|
||||||
|
|
||||||
|
b.Navigation("Bundle");
|
||||||
|
|
||||||
|
b.Navigation("Object");
|
||||||
|
});
|
||||||
|
|
||||||
|
modelBuilder.Entity("DysonNetwork.Shared.Models.SnCloudFileIndex", b =>
|
||||||
|
{
|
||||||
|
b.HasOne("DysonNetwork.Shared.Models.SnCloudFile", "File")
|
||||||
|
.WithMany("FileIndexes")
|
||||||
|
.HasForeignKey("FileId")
|
||||||
|
.OnDelete(DeleteBehavior.Cascade)
|
||||||
|
.IsRequired()
|
||||||
|
.HasConstraintName("fk_file_indexes_files_file_id");
|
||||||
|
|
||||||
|
b.Navigation("File");
|
||||||
|
});
|
||||||
|
|
||||||
|
modelBuilder.Entity("DysonNetwork.Shared.Models.SnFileReplica", b =>
|
||||||
|
{
|
||||||
|
b.HasOne("DysonNetwork.Shared.Models.SnFileObject", "Object")
|
||||||
|
.WithMany("FileReplicas")
|
||||||
|
.HasForeignKey("ObjectId")
|
||||||
|
.OnDelete(DeleteBehavior.Cascade)
|
||||||
|
.IsRequired()
|
||||||
|
.HasConstraintName("fk_file_replicas_file_objects_object_id");
|
||||||
|
|
||||||
|
b.HasOne("DysonNetwork.Shared.Models.FilePool", "Pool")
|
||||||
|
.WithMany()
|
||||||
|
.HasForeignKey("PoolId")
|
||||||
|
.HasConstraintName("fk_file_replicas_pools_pool_id");
|
||||||
|
|
||||||
|
b.Navigation("Object");
|
||||||
|
|
||||||
|
b.Navigation("Pool");
|
||||||
|
});
|
||||||
|
|
||||||
|
modelBuilder.Entity("DysonNetwork.Shared.Models.SnCloudFile", b =>
|
||||||
|
{
|
||||||
|
b.Navigation("FileIndexes");
|
||||||
|
});
|
||||||
|
|
||||||
|
modelBuilder.Entity("DysonNetwork.Shared.Models.SnFileBundle", b =>
|
||||||
|
{
|
||||||
|
b.Navigation("Files");
|
||||||
|
});
|
||||||
|
|
||||||
|
modelBuilder.Entity("DysonNetwork.Shared.Models.SnFileObject", b =>
|
||||||
|
{
|
||||||
|
b.Navigation("FileReplicas");
|
||||||
|
});
|
||||||
|
#pragma warning restore 612, 618
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -0,0 +1,49 @@
|
|||||||
|
using System;
|
||||||
|
using Microsoft.EntityFrameworkCore.Migrations;
|
||||||
|
|
||||||
|
#nullable disable
|
||||||
|
|
||||||
|
namespace DysonNetwork.Drive.Migrations
|
||||||
|
{
|
||||||
|
/// <inheritdoc />
|
||||||
|
public partial class RemovePoolFromCloudFile : Migration
|
||||||
|
{
|
||||||
|
/// <inheritdoc />
|
||||||
|
protected override void Up(MigrationBuilder migrationBuilder)
|
||||||
|
{
|
||||||
|
migrationBuilder.DropForeignKey(
|
||||||
|
name: "fk_files_pools_pool_id",
|
||||||
|
table: "files");
|
||||||
|
|
||||||
|
migrationBuilder.DropIndex(
|
||||||
|
name: "ix_files_pool_id",
|
||||||
|
table: "files");
|
||||||
|
|
||||||
|
migrationBuilder.DropColumn(
|
||||||
|
name: "pool_id",
|
||||||
|
table: "files");
|
||||||
|
}
|
||||||
|
|
||||||
|
/// <inheritdoc />
|
||||||
|
protected override void Down(MigrationBuilder migrationBuilder)
|
||||||
|
{
|
||||||
|
migrationBuilder.AddColumn<Guid>(
|
||||||
|
name: "pool_id",
|
||||||
|
table: "files",
|
||||||
|
type: "uuid",
|
||||||
|
nullable: true);
|
||||||
|
|
||||||
|
migrationBuilder.CreateIndex(
|
||||||
|
name: "ix_files_pool_id",
|
||||||
|
table: "files",
|
||||||
|
column: "pool_id");
|
||||||
|
|
||||||
|
migrationBuilder.AddForeignKey(
|
||||||
|
name: "fk_files_pools_pool_id",
|
||||||
|
table: "files",
|
||||||
|
column: "pool_id",
|
||||||
|
principalTable: "pools",
|
||||||
|
principalColumn: "id");
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -276,10 +276,6 @@ namespace DysonNetwork.Drive.Migrations
|
|||||||
.HasColumnType("character varying(32)")
|
.HasColumnType("character varying(32)")
|
||||||
.HasColumnName("object_id");
|
.HasColumnName("object_id");
|
||||||
|
|
||||||
b.Property<Guid?>("PoolId")
|
|
||||||
.HasColumnType("uuid")
|
|
||||||
.HasColumnName("pool_id");
|
|
||||||
|
|
||||||
b.PrimitiveCollection<string>("SensitiveMarks")
|
b.PrimitiveCollection<string>("SensitiveMarks")
|
||||||
.HasColumnType("jsonb")
|
.HasColumnType("jsonb")
|
||||||
.HasColumnName("sensitive_marks");
|
.HasColumnName("sensitive_marks");
|
||||||
@@ -315,9 +311,6 @@ namespace DysonNetwork.Drive.Migrations
|
|||||||
b.HasIndex("ObjectId")
|
b.HasIndex("ObjectId")
|
||||||
.HasDatabaseName("ix_files_object_id");
|
.HasDatabaseName("ix_files_object_id");
|
||||||
|
|
||||||
b.HasIndex("PoolId")
|
|
||||||
.HasDatabaseName("ix_files_pool_id");
|
|
||||||
|
|
||||||
b.ToTable("files", (string)null);
|
b.ToTable("files", (string)null);
|
||||||
});
|
});
|
||||||
|
|
||||||
@@ -588,16 +581,9 @@ namespace DysonNetwork.Drive.Migrations
|
|||||||
.HasForeignKey("ObjectId")
|
.HasForeignKey("ObjectId")
|
||||||
.HasConstraintName("fk_files_file_objects_object_id");
|
.HasConstraintName("fk_files_file_objects_object_id");
|
||||||
|
|
||||||
b.HasOne("DysonNetwork.Shared.Models.FilePool", "Pool")
|
|
||||||
.WithMany()
|
|
||||||
.HasForeignKey("PoolId")
|
|
||||||
.HasConstraintName("fk_files_pools_pool_id");
|
|
||||||
|
|
||||||
b.Navigation("Bundle");
|
b.Navigation("Bundle");
|
||||||
|
|
||||||
b.Navigation("Object");
|
b.Navigation("Object");
|
||||||
|
|
||||||
b.Navigation("Pool");
|
|
||||||
});
|
});
|
||||||
|
|
||||||
modelBuilder.Entity("DysonNetwork.Shared.Models.SnCloudFileIndex", b =>
|
modelBuilder.Entity("DysonNetwork.Shared.Models.SnCloudFileIndex", b =>
|
||||||
|
|||||||
@@ -314,9 +314,20 @@ public class BroadcastEventHandler(
|
|||||||
logger.LogInformation("Uploaded file {FileId} done!", fileId);
|
logger.LogInformation("Uploaded file {FileId} done!", fileId);
|
||||||
|
|
||||||
var now = SystemClock.Instance.GetCurrentInstant();
|
var now = SystemClock.Instance.GetCurrentInstant();
|
||||||
|
|
||||||
|
var newReplica = new SnFileReplica
|
||||||
|
{
|
||||||
|
Id = Guid.NewGuid(),
|
||||||
|
ObjectId = fileId,
|
||||||
|
PoolId = destPool,
|
||||||
|
StorageId = storageId,
|
||||||
|
Status = SnFileReplicaStatus.Available,
|
||||||
|
IsPrimary = false
|
||||||
|
};
|
||||||
|
scopedDb.FileReplicas.Add(newReplica);
|
||||||
|
|
||||||
await scopedDb.Files.Where(f => f.Id == fileId).ExecuteUpdateAsync(setter => setter
|
await scopedDb.Files.Where(f => f.Id == fileId).ExecuteUpdateAsync(setter => setter
|
||||||
.SetProperty(f => f.UploadedAt, now)
|
.SetProperty(f => f.UploadedAt, now)
|
||||||
.SetProperty(f => f.PoolId, destPool)
|
|
||||||
);
|
);
|
||||||
|
|
||||||
await scopedDb.FileObjects.Where(fo => fo.Id == fileId).ExecuteUpdateAsync(setter => setter
|
await scopedDb.FileObjects.Where(fo => fo.Id == fileId).ExecuteUpdateAsync(setter => setter
|
||||||
|
|||||||
@@ -54,7 +54,6 @@ public static class ServiceCollectionExtensions
|
|||||||
|
|
||||||
public IServiceCollection AddAppBusinessServices()
|
public IServiceCollection AddAppBusinessServices()
|
||||||
{
|
{
|
||||||
services.AddScoped<Storage.FileMigrationService>();
|
|
||||||
services.AddScoped<Storage.FileService>();
|
services.AddScoped<Storage.FileService>();
|
||||||
services.AddScoped<Storage.FileReanalysisService>();
|
services.AddScoped<Storage.FileReanalysisService>();
|
||||||
services.AddScoped<Storage.PersistentTaskService>();
|
services.AddScoped<Storage.PersistentTaskService>();
|
||||||
|
|||||||
@@ -1,5 +1,6 @@
|
|||||||
using Microsoft.EntityFrameworkCore;
|
using Microsoft.EntityFrameworkCore;
|
||||||
using NodaTime;
|
using NodaTime;
|
||||||
|
using DysonNetwork.Shared.Models;
|
||||||
using Quartz;
|
using Quartz;
|
||||||
|
|
||||||
namespace DysonNetwork.Drive.Storage;
|
namespace DysonNetwork.Drive.Storage;
|
||||||
@@ -40,8 +41,10 @@ public class CloudFileUnusedRecyclingJob(
|
|||||||
var markedCount = 0;
|
var markedCount = 0;
|
||||||
var totalFiles = await db.Files
|
var totalFiles = await db.Files
|
||||||
.Where(f => f.FileIndexes.Count == 0)
|
.Where(f => f.FileIndexes.Count == 0)
|
||||||
.Where(f => f.PoolId.HasValue && recyclablePools.Contains(f.PoolId.Value))
|
.Where(f => f.Object!.FileReplicas.Any(r => r.PoolId.HasValue && recyclablePools.Contains(r.PoolId.Value)))
|
||||||
.Where(f => !f.IsMarkedRecycle)
|
.Where(f => !f.IsMarkedRecycle)
|
||||||
|
.Include(f => f.Object)
|
||||||
|
.ThenInclude(o => o.FileReplicas)
|
||||||
.CountAsync();
|
.CountAsync();
|
||||||
|
|
||||||
logger.LogInformation("Found {TotalFiles} files to check for unused status", totalFiles);
|
logger.LogInformation("Found {TotalFiles} files to check for unused status", totalFiles);
|
||||||
@@ -56,17 +59,18 @@ public class CloudFileUnusedRecyclingJob(
|
|||||||
|
|
||||||
while (hasMoreFiles)
|
while (hasMoreFiles)
|
||||||
{
|
{
|
||||||
// Query for the next batch of files using keyset pagination
|
IQueryable<SnCloudFile> baseQuery = db.Files
|
||||||
var filesQuery = db.Files
|
.Where(f => f.Object!.FileReplicas.Any(r => r.PoolId.HasValue && recyclablePools.Contains(r.PoolId.Value)))
|
||||||
.Where(f => f.PoolId.HasValue && recyclablePools.Contains(f.PoolId.Value))
|
|
||||||
.Where(f => !f.IsMarkedRecycle)
|
.Where(f => !f.IsMarkedRecycle)
|
||||||
.Where(f => f.CreatedAt <= ageThreshold); // Only process older files first
|
.Where(f => f.CreatedAt <= ageThreshold)
|
||||||
|
.Include(f => f.Object)
|
||||||
|
.ThenInclude(o => o.FileReplicas);
|
||||||
|
|
||||||
if (lastProcessedId != null)
|
if (lastProcessedId != null)
|
||||||
filesQuery = filesQuery.Where(f => string.Compare(f.Id, lastProcessedId) > 0);
|
baseQuery = baseQuery.Where(f => string.Compare(f.Id, lastProcessedId) > 0);
|
||||||
|
|
||||||
var fileBatch = await filesQuery
|
var fileBatch = await baseQuery
|
||||||
.OrderBy(f => f.Id) // Ensure consistent ordering for pagination
|
.OrderBy(f => f.Id)
|
||||||
.Take(batchSize)
|
.Take(batchSize)
|
||||||
.Select(f => f.Id)
|
.Select(f => f.Id)
|
||||||
.ToListAsync();
|
.ToListAsync();
|
||||||
@@ -80,12 +84,11 @@ public class CloudFileUnusedRecyclingJob(
|
|||||||
processedCount += fileBatch.Count;
|
processedCount += fileBatch.Count;
|
||||||
lastProcessedId = fileBatch.Last();
|
lastProcessedId = fileBatch.Last();
|
||||||
|
|
||||||
// Optimized query: Find files that have no other cloud files sharing the same object
|
// Optimized query: Find files that have no file object or no replicas
|
||||||
// A file is considered "unused" if no other SnCloudFile shares its ObjectId
|
// A file is considered "unused" if its file object has no replicas
|
||||||
var filesToMark = await db.Files
|
var filesToMark = await db.Files
|
||||||
.Where(f => fileBatch.Contains(f.Id))
|
.Where(f => fileBatch.Contains(f.Id))
|
||||||
.Where(f => f.ObjectId == null || // No file object at all
|
.Where(f => f.Object == null || f.Object.FileReplicas.Count == 0)
|
||||||
!db.Files.Any(cf => cf.ObjectId == f.ObjectId && cf.Id != f.Id)) // Or no other files share this object
|
|
||||||
.Select(f => f.Id)
|
.Select(f => f.Id)
|
||||||
.ToListAsync();
|
.ToListAsync();
|
||||||
|
|
||||||
|
|||||||
@@ -253,11 +253,12 @@ public class FileController(
|
|||||||
string? overrideMimeType
|
string? overrideMimeType
|
||||||
)
|
)
|
||||||
{
|
{
|
||||||
if (!file.PoolId.HasValue)
|
var primaryReplica = file.Object?.FileReplicas.FirstOrDefault(r => r.IsPrimary);
|
||||||
|
if (primaryReplica == null || primaryReplica.PoolId == null)
|
||||||
return StatusCode(StatusCodes.Status500InternalServerError,
|
return StatusCode(StatusCodes.Status500InternalServerError,
|
||||||
"File is in an inconsistent state: uploaded but no pool ID.");
|
"File is in an inconsistent state: uploaded but no pool ID.");
|
||||||
|
|
||||||
var pool = await fs.GetPoolAsync(file.PoolId.Value);
|
var pool = await fs.GetPoolAsync(primaryReplica.PoolId.Value);
|
||||||
if (pool is null)
|
if (pool is null)
|
||||||
return StatusCode(StatusCodes.Status410Gone, "The pool of the file no longer exists or not accessible.");
|
return StatusCode(StatusCodes.Status410Gone, "The pool of the file no longer exists or not accessible.");
|
||||||
|
|
||||||
@@ -461,11 +462,10 @@ public class FileController(
|
|||||||
var filesQuery = db.Files
|
var filesQuery = db.Files
|
||||||
.Where(e => e.IsMarkedRecycle == recycled)
|
.Where(e => e.IsMarkedRecycle == recycled)
|
||||||
.Where(e => e.AccountId == accountId)
|
.Where(e => e.AccountId == accountId)
|
||||||
.Include(e => e.Pool)
|
|
||||||
.Include(e => e.Object)
|
.Include(e => e.Object)
|
||||||
.AsQueryable();
|
.AsQueryable();
|
||||||
|
|
||||||
if (pool.HasValue) filesQuery = filesQuery.Where(e => e.PoolId == pool);
|
if (pool.HasValue) filesQuery = filesQuery.Where(e => e.Object!.FileReplicas.Any(r => r.PoolId == pool.Value));
|
||||||
|
|
||||||
if (!string.IsNullOrWhiteSpace(query))
|
if (!string.IsNullOrWhiteSpace(query))
|
||||||
{
|
{
|
||||||
|
|||||||
@@ -1,127 +0,0 @@
|
|||||||
using DysonNetwork.Shared.Models;
|
|
||||||
using Microsoft.EntityFrameworkCore;
|
|
||||||
|
|
||||||
namespace DysonNetwork.Drive.Storage;
|
|
||||||
|
|
||||||
public class FileMigrationService(AppDatabase db, ILogger<FileMigrationService> logger)
|
|
||||||
{
|
|
||||||
public async Task MigrateCloudFilesAsync()
|
|
||||||
{
|
|
||||||
logger.LogInformation("Starting cloud file migration.");
|
|
||||||
|
|
||||||
var cloudFiles = await db.Files
|
|
||||||
.Where(f =>
|
|
||||||
f.ObjectId == null &&
|
|
||||||
f.PoolId != null
|
|
||||||
)
|
|
||||||
.ToListAsync();
|
|
||||||
|
|
||||||
logger.LogDebug("Found {Count} cloud files to migrate.", cloudFiles.Count);
|
|
||||||
|
|
||||||
foreach (var cf in cloudFiles)
|
|
||||||
{
|
|
||||||
try
|
|
||||||
{
|
|
||||||
var ext = Path.GetExtension(cf.Name);
|
|
||||||
var mimeType = ext != "" && MimeTypes.TryGetMimeType(ext, out var mime) ? mime : "application/octet-stream";
|
|
||||||
|
|
||||||
var fileObject = await db.FileObjects.FindAsync(cf.Id);
|
|
||||||
|
|
||||||
if (fileObject == null)
|
|
||||||
{
|
|
||||||
fileObject = new SnFileObject
|
|
||||||
{
|
|
||||||
Id = cf.Id,
|
|
||||||
MimeType = mimeType,
|
|
||||||
HasCompression = mimeType.StartsWith("image/"),
|
|
||||||
HasThumbnail = mimeType.StartsWith("video/")
|
|
||||||
};
|
|
||||||
|
|
||||||
db.FileObjects.Add(fileObject);
|
|
||||||
}
|
|
||||||
|
|
||||||
var replicaExists = await db.FileReplicas.AnyAsync(r =>
|
|
||||||
r.ObjectId == fileObject.Id &&
|
|
||||||
r.PoolId == cf.PoolId!.Value);
|
|
||||||
|
|
||||||
if (!replicaExists)
|
|
||||||
{
|
|
||||||
var fileReplica = new SnFileReplica
|
|
||||||
{
|
|
||||||
Id = Guid.NewGuid(),
|
|
||||||
ObjectId = fileObject.Id,
|
|
||||||
PoolId = cf.PoolId!.Value,
|
|
||||||
StorageId = cf.StorageId ?? cf.Id,
|
|
||||||
Status = SnFileReplicaStatus.Available,
|
|
||||||
IsPrimary = true
|
|
||||||
};
|
|
||||||
|
|
||||||
fileObject.FileReplicas.Add(fileReplica);
|
|
||||||
db.FileReplicas.Add(fileReplica);
|
|
||||||
}
|
|
||||||
|
|
||||||
var permissionExists = await db.FilePermissions.AnyAsync(p => p.FileId == cf.Id);
|
|
||||||
|
|
||||||
if (!permissionExists)
|
|
||||||
{
|
|
||||||
var permission = new SnFilePermission
|
|
||||||
{
|
|
||||||
Id = Guid.NewGuid(),
|
|
||||||
FileId = cf.Id,
|
|
||||||
SubjectType = SnFilePermissionType.Anyone,
|
|
||||||
SubjectId = string.Empty,
|
|
||||||
Permission = SnFilePermissionLevel.Read
|
|
||||||
};
|
|
||||||
|
|
||||||
db.FilePermissions.Add(permission);
|
|
||||||
}
|
|
||||||
|
|
||||||
cf.ObjectId = fileObject.Id;
|
|
||||||
cf.Object = fileObject;
|
|
||||||
|
|
||||||
await db.SaveChangesAsync();
|
|
||||||
logger.LogInformation("Migrated file {FileId} successfully.", cf.Id);
|
|
||||||
}
|
|
||||||
catch (Exception ex)
|
|
||||||
{
|
|
||||||
logger.LogError(ex,
|
|
||||||
"Failed migrating file {FileId}. ObjectId={ObjectId}, PoolId={PoolId}, StorageId={StorageId}",
|
|
||||||
cf.Id,
|
|
||||||
cf.ObjectId,
|
|
||||||
cf.PoolId,
|
|
||||||
cf.StorageId);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
logger.LogInformation("Cloud file migration completed.");
|
|
||||||
}
|
|
||||||
|
|
||||||
public async Task MigratePermissionsAsync()
|
|
||||||
{
|
|
||||||
logger.LogInformation("Starting file permission migration.");
|
|
||||||
|
|
||||||
var filesWithoutPermission = await db.Files
|
|
||||||
.Where(f => !db.FilePermissions.Any(p => p.FileId == f.Id))
|
|
||||||
.ToListAsync();
|
|
||||||
|
|
||||||
logger.LogDebug("Found {Count} files without permissions.", filesWithoutPermission.Count);
|
|
||||||
|
|
||||||
foreach (var file in filesWithoutPermission)
|
|
||||||
{
|
|
||||||
var permission = new SnFilePermission
|
|
||||||
{
|
|
||||||
Id = Guid.NewGuid(),
|
|
||||||
FileId = file.Id,
|
|
||||||
SubjectType = SnFilePermissionType.Anyone,
|
|
||||||
SubjectId = string.Empty,
|
|
||||||
Permission = SnFilePermissionLevel.Read
|
|
||||||
};
|
|
||||||
|
|
||||||
db.FilePermissions.Add(permission);
|
|
||||||
}
|
|
||||||
|
|
||||||
await db.SaveChangesAsync();
|
|
||||||
|
|
||||||
logger.LogInformation("Permission migration completed. Created {Count} permissions.", filesWithoutPermission.Count);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
@@ -23,10 +23,9 @@ public class FileReanalysisService(
|
|||||||
var now = SystemClock.Instance.GetCurrentInstant();
|
var now = SystemClock.Instance.GetCurrentInstant();
|
||||||
var deadline = now.Minus(Duration.FromMinutes(30));
|
var deadline = now.Minus(Duration.FromMinutes(30));
|
||||||
return await db.Files
|
return await db.Files
|
||||||
.Where(f => f.ObjectId != null && f.PoolId != null)
|
.Where(f => f.ObjectId != null)
|
||||||
.Include(f => f.Object)
|
.Include(f => f.Object)
|
||||||
.ThenInclude(f => f.FileReplicas)
|
.ThenInclude(f => f.FileReplicas)
|
||||||
.Include(f => f.Pool)
|
|
||||||
.Where(f => f.Object != null && (f.Object.Meta == null || f.Object.Meta.Count == 0))
|
.Where(f => f.Object != null && (f.Object.Meta == null || f.Object.Meta.Count == 0))
|
||||||
.Where(f => f.Object!.FileReplicas.Count > 0)
|
.Where(f => f.Object!.FileReplicas.Count > 0)
|
||||||
.Where(f => f.CreatedAt <= deadline)
|
.Where(f => f.CreatedAt <= deadline)
|
||||||
@@ -39,9 +38,9 @@ public class FileReanalysisService(
|
|||||||
{
|
{
|
||||||
logger.LogInformation("Starting reanalysis for file {FileId}: {FileName}", file.Id, file.Name);
|
logger.LogInformation("Starting reanalysis for file {FileId}: {FileName}", file.Id, file.Name);
|
||||||
|
|
||||||
if (file.Object == null || file.Pool == null)
|
if (file.Object == null)
|
||||||
{
|
{
|
||||||
logger.LogWarning("File {FileId} missing object or pool, skipping reanalysis", file.Id);
|
logger.LogWarning("File {FileId} missing object, skipping reanalysis", file.Id);
|
||||||
return true; // not a failure
|
return true; // not a failure
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -147,16 +146,22 @@ public class FileReanalysisService(
|
|||||||
|
|
||||||
private async Task DownloadFileAsync(SnCloudFile file, SnFileReplica replica, string tempPath)
|
private async Task DownloadFileAsync(SnCloudFile file, SnFileReplica replica, string tempPath)
|
||||||
{
|
{
|
||||||
var dest = file.Pool!.StorageConfig;
|
if (replica.PoolId == null)
|
||||||
if (dest == null)
|
|
||||||
{
|
{
|
||||||
throw new InvalidOperationException($"No remote storage configured for pool {file.PoolId}");
|
throw new InvalidOperationException($"Replica for file {file.Id} has no pool ID");
|
||||||
}
|
}
|
||||||
|
|
||||||
|
var pool = await db.Pools.FindAsync(replica.PoolId.Value);
|
||||||
|
if (pool == null)
|
||||||
|
{
|
||||||
|
throw new InvalidOperationException($"No remote storage configured for pool {replica.PoolId}");
|
||||||
|
}
|
||||||
|
var dest = pool.StorageConfig;
|
||||||
|
|
||||||
var client = CreateMinioClient(dest);
|
var client = CreateMinioClient(dest);
|
||||||
if (client == null)
|
if (client == null)
|
||||||
{
|
{
|
||||||
throw new InvalidOperationException($"Failed to create Minio client for pool {file.PoolId}");
|
throw new InvalidOperationException($"Failed to create Minio client for pool {replica.PoolId}");
|
||||||
}
|
}
|
||||||
|
|
||||||
await using var fileStream = File.Create(tempPath);
|
await using var fileStream = File.Create(tempPath);
|
||||||
|
|||||||
@@ -38,7 +38,6 @@ public class FileService(
|
|||||||
|
|
||||||
var file = await db.Files
|
var file = await db.Files
|
||||||
.Where(f => f.Id == fileId)
|
.Where(f => f.Id == fileId)
|
||||||
.Include(f => f.Pool)
|
|
||||||
.Include(f => f.Bundle)
|
.Include(f => f.Bundle)
|
||||||
.Include(f => f.Object)
|
.Include(f => f.Object)
|
||||||
.ThenInclude(o => o.FileReplicas)
|
.ThenInclude(o => o.FileReplicas)
|
||||||
@@ -70,7 +69,7 @@ public class FileService(
|
|||||||
{
|
{
|
||||||
var dbFiles = await db.Files
|
var dbFiles = await db.Files
|
||||||
.Where(f => uncachedIds.Contains(f.Id))
|
.Where(f => uncachedIds.Contains(f.Id))
|
||||||
.Include(f => f.Pool)
|
.Include(f => f.Bundle)
|
||||||
.Include(f => f.Object)
|
.Include(f => f.Object)
|
||||||
.ThenInclude(o => o.FileReplicas)
|
.ThenInclude(o => o.FileReplicas)
|
||||||
.ToListAsync();
|
.ToListAsync();
|
||||||
@@ -124,7 +123,7 @@ public class FileService(
|
|||||||
|
|
||||||
fileObject.Hash = await HashFileAsync(processingPath);
|
fileObject.Hash = await HashFileAsync(processingPath);
|
||||||
|
|
||||||
await SaveFileToDatabaseAsync(file, fileObject);
|
await SaveFileToDatabaseAsync(file, fileObject, pool.Id);
|
||||||
|
|
||||||
await PublishFileUploadedEventAsync(file, pool, processingPath, isTempFile);
|
await PublishFileUploadedEventAsync(file, pool, processingPath, isTempFile);
|
||||||
|
|
||||||
@@ -245,13 +244,13 @@ public class FileService(
|
|||||||
return Task.FromResult((encryptedPath, true));
|
return Task.FromResult((encryptedPath, true));
|
||||||
}
|
}
|
||||||
|
|
||||||
private async Task SaveFileToDatabaseAsync(SnCloudFile file, SnFileObject fileObject)
|
private async Task SaveFileToDatabaseAsync(SnCloudFile file, SnFileObject fileObject, Guid poolId)
|
||||||
{
|
{
|
||||||
var replica = new SnFileReplica
|
var replica = new SnFileReplica
|
||||||
{
|
{
|
||||||
Id = Guid.NewGuid(),
|
Id = Guid.NewGuid(),
|
||||||
ObjectId = file.Id,
|
ObjectId = file.Id,
|
||||||
PoolId = file.PoolId,
|
PoolId = poolId,
|
||||||
StorageId = file.StorageId ?? file.Id,
|
StorageId = file.StorageId ?? file.Id,
|
||||||
Status = SnFileReplicaStatus.Available,
|
Status = SnFileReplicaStatus.Available,
|
||||||
IsPrimary = true
|
IsPrimary = true
|
||||||
@@ -540,7 +539,30 @@ public class FileService(
|
|||||||
|
|
||||||
public async Task DeleteFileDataAsync(SnCloudFile file, bool force = false)
|
public async Task DeleteFileDataAsync(SnCloudFile file, bool force = false)
|
||||||
{
|
{
|
||||||
if (!file.PoolId.HasValue || file.ObjectId == null) return;
|
if (file.ObjectId == null) return;
|
||||||
|
|
||||||
|
var replicas = await db.FileReplicas
|
||||||
|
.Where(r => r.ObjectId == file.ObjectId)
|
||||||
|
.ToListAsync();
|
||||||
|
|
||||||
|
if (replicas.Count == 0)
|
||||||
|
{
|
||||||
|
logger.LogWarning("No replicas found for file object {ObjectId}", file.ObjectId);
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
var primaryReplica = replicas.FirstOrDefault(r => r.IsPrimary);
|
||||||
|
if (primaryReplica == null)
|
||||||
|
{
|
||||||
|
logger.LogWarning("No primary replica found for file object {ObjectId}", file.ObjectId);
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
if (primaryReplica.PoolId == null)
|
||||||
|
{
|
||||||
|
logger.LogWarning("Primary replica has no pool ID for file object {ObjectId}", file.ObjectId);
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
if (!force)
|
if (!force)
|
||||||
{
|
{
|
||||||
@@ -553,23 +575,12 @@ public class FileService(
|
|||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
|
|
||||||
var replicas = await db.FileReplicas
|
var dest = await GetRemoteStorageConfig(primaryReplica.PoolId.Value);
|
||||||
.Where(r => r.ObjectId == file.ObjectId)
|
if (dest is null) throw new InvalidOperationException($"No remote storage configured for pool {primaryReplica.PoolId}");
|
||||||
.ToListAsync();
|
|
||||||
|
|
||||||
if (replicas.Count == 0)
|
|
||||||
{
|
|
||||||
logger.LogWarning("No replicas found for file object {ObjectId}", file.ObjectId);
|
|
||||||
return;
|
|
||||||
}
|
|
||||||
|
|
||||||
var primaryReplica = replicas.First(r => r.IsPrimary);
|
|
||||||
var dest = await GetRemoteStorageConfig(file.PoolId.Value);
|
|
||||||
if (dest is null) throw new InvalidOperationException($"No remote storage configured for pool {file.PoolId}");
|
|
||||||
var client = CreateMinioClient(dest);
|
var client = CreateMinioClient(dest);
|
||||||
if (client is null)
|
if (client is null)
|
||||||
throw new InvalidOperationException(
|
throw new InvalidOperationException(
|
||||||
$"Failed to configure client for remote destination '{file.PoolId}'"
|
$"Failed to configure client for remote destination '{primaryReplica.PoolId}'"
|
||||||
);
|
);
|
||||||
|
|
||||||
var bucket = dest.Bucket;
|
var bucket = dest.Bucket;
|
||||||
@@ -615,7 +626,7 @@ public class FileService(
|
|||||||
|
|
||||||
public async Task DeleteFileDataBatchAsync(List<SnCloudFile> files)
|
public async Task DeleteFileDataBatchAsync(List<SnCloudFile> files)
|
||||||
{
|
{
|
||||||
files = files.Where(f => f.PoolId.HasValue && f.ObjectId != null).ToList();
|
files = files.Where(f => f.ObjectId != null).ToList();
|
||||||
|
|
||||||
var objectIds = files.Select(f => f.ObjectId).Distinct().ToList();
|
var objectIds = files.Select(f => f.ObjectId).Distinct().ToList();
|
||||||
var replicas = await db.FileReplicas
|
var replicas = await db.FileReplicas
|
||||||
@@ -759,8 +770,14 @@ public class FileService(
|
|||||||
|
|
||||||
public async Task<int> DeletePoolRecycledFilesAsync(Guid poolId)
|
public async Task<int> DeletePoolRecycledFilesAsync(Guid poolId)
|
||||||
{
|
{
|
||||||
|
var fileIdsWithReplicas = await db.FileReplicas
|
||||||
|
.Where(r => r.PoolId == poolId)
|
||||||
|
.Select(r => r.ObjectId)
|
||||||
|
.Distinct()
|
||||||
|
.ToListAsync();
|
||||||
|
|
||||||
var files = await db.Files
|
var files = await db.Files
|
||||||
.Where(f => f.PoolId == poolId && f.IsMarkedRecycle)
|
.Where(f => fileIdsWithReplicas.Contains(f.Id) && f.IsMarkedRecycle)
|
||||||
.ToListAsync();
|
.ToListAsync();
|
||||||
var count = files.Count;
|
var count = files.Count;
|
||||||
var fileIds = files.Select(f => f.Id).ToList();
|
var fileIds = files.Select(f => f.Id).ToList();
|
||||||
|
|||||||
@@ -32,8 +32,6 @@ public class SnCloudFile : ModelBase, ICloudFile, IIdentifiedResource
|
|||||||
[MaxLength(32)] public string? ObjectId { get; set; }
|
[MaxLength(32)] public string? ObjectId { get; set; }
|
||||||
public SnFileObject? Object { get; set; }
|
public SnFileObject? Object { get; set; }
|
||||||
|
|
||||||
public FilePool? Pool { get; set; }
|
|
||||||
public Guid? PoolId { get; set; }
|
|
||||||
[JsonIgnore] public SnFileBundle? Bundle { get; set; }
|
[JsonIgnore] public SnFileBundle? Bundle { get; set; }
|
||||||
public Guid? BundleId { get; set; }
|
public Guid? BundleId { get; set; }
|
||||||
[JsonIgnore] public List<SnCloudFileIndex> FileIndexes { get; set; } = [];
|
[JsonIgnore] public List<SnCloudFileIndex> FileIndexes { get; set; } = [];
|
||||||
|
|||||||
Reference in New Issue
Block a user