From c875c82bdc82ed6a043c2603f59c2e16a5501b65 Mon Sep 17 00:00:00 2001 From: LittleSheep Date: Sun, 27 Jul 2025 18:08:39 +0800 Subject: [PATCH] :sparkles: Quota and better drive dashboard --- DysonNetwork.Drive/AppDatabase.cs | 2 + DysonNetwork.Drive/Billing/Quota.cs | 28 ++ DysonNetwork.Drive/Billing/QuotaController.cs | 66 ++++ DysonNetwork.Drive/Billing/QuotaService.cs | 69 ++++ DysonNetwork.Drive/Billing/UsageController.cs | 34 +- DysonNetwork.Drive/Billing/UsageService.cs | 55 +-- .../Client/src/components/FilePoolSelect.vue | 2 +- .../Client/src/layouts/default.vue | 2 +- .../Client/src/views/dashboard/usage.vue | 85 ++++- DysonNetwork.Drive/Client/src/views/index.vue | 14 +- .../20250727092028_AddQuotaRecord.Designer.cs | 322 ++++++++++++++++++ .../20250727092028_AddQuotaRecord.cs | 42 +++ .../Migrations/AppDatabaseModelSnapshot.cs | 47 +++ DysonNetwork.Drive/Program.cs | 1 + .../Startup/ServiceCollectionExtensions.cs | 2 +- DysonNetwork.Drive/Storage/TusService.cs | 33 +- 16 files changed, 758 insertions(+), 46 deletions(-) create mode 100644 DysonNetwork.Drive/Billing/Quota.cs create mode 100644 DysonNetwork.Drive/Billing/QuotaController.cs create mode 100644 DysonNetwork.Drive/Billing/QuotaService.cs create mode 100644 DysonNetwork.Drive/Migrations/20250727092028_AddQuotaRecord.Designer.cs create mode 100644 DysonNetwork.Drive/Migrations/20250727092028_AddQuotaRecord.cs diff --git a/DysonNetwork.Drive/AppDatabase.cs b/DysonNetwork.Drive/AppDatabase.cs index 6986052..c680c58 100644 --- a/DysonNetwork.Drive/AppDatabase.cs +++ b/DysonNetwork.Drive/AppDatabase.cs @@ -1,5 +1,6 @@ using System.Linq.Expressions; using System.Reflection; +using DysonNetwork.Drive.Billing; using DysonNetwork.Drive.Storage; using DysonNetwork.Shared.Data; using Microsoft.EntityFrameworkCore; @@ -16,6 +17,7 @@ public class AppDatabase( ) : DbContext(options) { public DbSet Pools { get; set; } = null!; + public DbSet QuotaRecords { get; set; } = null!; public DbSet Files { get; set; } = null!; public DbSet FileReferences { get; set; } = null!; diff --git a/DysonNetwork.Drive/Billing/Quota.cs b/DysonNetwork.Drive/Billing/Quota.cs new file mode 100644 index 0000000..4a4d611 --- /dev/null +++ b/DysonNetwork.Drive/Billing/Quota.cs @@ -0,0 +1,28 @@ +using DysonNetwork.Shared.Data; +using NodaTime; + +namespace DysonNetwork.Drive.Billing; + +/// +/// The quota record stands for the extra quota that a user has. +/// For normal users, the quota is 1GiB. +/// For stellar program t1 users, the quota is 5GiB +/// For stellar program t2 users, the quota is 10GiB +/// For stellar program t3 users, the quota is 15GiB +/// +/// If users want to increase the quota, they need to pay for it. +/// Each 1NSD they paid for one GiB. +/// +/// But the quota record unit is MiB, the minimal billable unit. +/// +public class QuotaRecord : ModelBase +{ + public Guid Id { get; set; } = Guid.NewGuid(); + public Guid AccountId { get; set; } + public string Name { get; set; } = string.Empty; + public string Description { get; set; } = string.Empty; + + public long Quota { get; set; } + + public Instant? ExpiredAt { get; set; } +} \ No newline at end of file diff --git a/DysonNetwork.Drive/Billing/QuotaController.cs b/DysonNetwork.Drive/Billing/QuotaController.cs new file mode 100644 index 0000000..39cfaa1 --- /dev/null +++ b/DysonNetwork.Drive/Billing/QuotaController.cs @@ -0,0 +1,66 @@ +using DysonNetwork.Shared.Proto; +using Microsoft.AspNetCore.Authorization; +using Microsoft.AspNetCore.Mvc; +using Microsoft.EntityFrameworkCore; +using NodaTime; + +namespace DysonNetwork.Drive.Billing; + +[ApiController] +[Route("/api/billing/quota")] +public class QuotaController(AppDatabase db, QuotaService quota) : ControllerBase +{ + public class QuotaDetails + { + public long BasedQuota { get; set; } + public long ExtraQuota { get; set; } + public long TotalQuota { get; set; } + } + + [HttpGet] + [Authorize] + public async Task> GetQuota() + { + if (HttpContext.Items["CurrentUser"] is not Account currentUser) return Unauthorized(); + var accountId = Guid.Parse(currentUser.Id); + + var (based, extra) = await quota.GetQuotaVerbose(accountId); + return Ok(new QuotaDetails + { + BasedQuota = based, + ExtraQuota = extra, + TotalQuota = based + extra + }); + } + + [HttpGet("records")] + [Authorize] + public async Task>> GetQuotaRecords( + [FromQuery] bool expired = false, + [FromQuery] int offset = 0, + [FromQuery] int take = 20 + ) + { + if (HttpContext.Items["CurrentUser"] is not Account currentUser) return Unauthorized(); + var accountId = Guid.Parse(currentUser.Id); + + var now = SystemClock.Instance.GetCurrentInstant(); + var query = db.QuotaRecords + .Where(r => r.AccountId == accountId) + .AsQueryable(); + if (!expired) + query = query + .Where(r => !r.ExpiredAt.HasValue || r.ExpiredAt > now); + + var total = await query.CountAsync(); + Response.Headers.Append("X-Total", total.ToString()); + + var records = await query + .OrderByDescending(r => r.CreatedAt) + .Skip(offset) + .Take(take) + .ToListAsync(); + + return Ok(records); + } +} \ No newline at end of file diff --git a/DysonNetwork.Drive/Billing/QuotaService.cs b/DysonNetwork.Drive/Billing/QuotaService.cs new file mode 100644 index 0000000..aa91132 --- /dev/null +++ b/DysonNetwork.Drive/Billing/QuotaService.cs @@ -0,0 +1,69 @@ +using DysonNetwork.Shared.Auth; +using DysonNetwork.Shared.Cache; +using DysonNetwork.Shared.Proto; +using Microsoft.EntityFrameworkCore; +using NodaTime; + +namespace DysonNetwork.Drive.Billing; + +public class QuotaService( + AppDatabase db, + UsageService usage, + AccountService.AccountServiceClient accounts, + ICacheService cache +) +{ + public async Task<(bool ok, long billable, long quota)> IsFileAcceptable(Guid accountId, double costMultiplier, long newFileSize) + { + // The billable unit is MiB + var billableUnit = (long)Math.Ceiling(newFileSize / 1024.0 / 1024.0 * costMultiplier); + var totalBillableUsage = await usage.GetTotalBillableUsage(); + var quota = await GetQuota(accountId); + return (totalBillableUsage + billableUnit <= quota, billableUnit, quota); + } + + public async Task GetQuota(Guid accountId) + { + var cacheKey = $"file:quota:{accountId}"; + var cachedResult = await cache.GetAsync(cacheKey); + if (cachedResult.HasValue) return cachedResult.Value; + + var (based, extra) = await GetQuotaVerbose(accountId); + var quota = based + extra; + await cache.SetAsync(cacheKey, quota); + return quota; + } + + public async Task<(long based, long extra)> GetQuotaVerbose(Guid accountId) + { + + + var response = await accounts.GetAccountAsync(new GetAccountRequest { Id = accountId.ToString() }); + var perkSubscription = response.PerkSubscription; + + // The base quota is 1GiB, T1 is 5GiB, T2 is 10GiB, T3 is 15GiB + var basedQuota = 1L; + if (perkSubscription != null) + { + var privilege = PerkSubscriptionPrivilege.GetPrivilegeFromIdentifier(perkSubscription.Identifier); + basedQuota = privilege switch + { + 1 => 5L, + 2 => 10L, + 3 => 15L, + _ => basedQuota + }; + } + + // The based quota is in GiB, we need to convert it to MiB + basedQuota *= 1024L; + + var now = SystemClock.Instance.GetCurrentInstant(); + var extraQuota = await db.QuotaRecords + .Where(e => e.AccountId == accountId) + .Where(e => !e.ExpiredAt.HasValue || e.ExpiredAt > now) + .SumAsync(e => e.Quota); + + return (basedQuota, extraQuota); + } +} \ No newline at end of file diff --git a/DysonNetwork.Drive/Billing/UsageController.cs b/DysonNetwork.Drive/Billing/UsageController.cs index e270118..56426e0 100644 --- a/DysonNetwork.Drive/Billing/UsageController.cs +++ b/DysonNetwork.Drive/Billing/UsageController.cs @@ -1,25 +1,49 @@ +using DysonNetwork.Shared.Cache; +using DysonNetwork.Shared.Proto; +using Microsoft.AspNetCore.Authorization; using Microsoft.AspNetCore.Mvc; namespace DysonNetwork.Drive.Billing; [ApiController] [Route("api/billing/usage")] -public class UsageController(UsageService usageService) : ControllerBase +public class UsageController(UsageService usage, QuotaService quota, ICacheService cache) : ControllerBase { [HttpGet] + [Authorize] public async Task> GetTotalUsage() { - return await usageService.GetTotalUsage(); + if (HttpContext.Items["CurrentUser"] is not Account currentUser) return Unauthorized(); + var accountId = Guid.Parse(currentUser.Id); + + var cacheKey = $"file:usage:{accountId}"; + + // Try to get from cache first + var (found, cachedResult) = await cache.GetAsyncWithStatus(cacheKey); + if (found && cachedResult != null) + return Ok(cachedResult); + + // If not in cache, get from services + var result = await usage.GetTotalUsage(accountId); + var totalQuota = await quota.GetQuota(accountId); + result.TotalQuota = totalQuota; + + // Cache the result for 5 minutes + await cache.SetAsync(cacheKey, result, TimeSpan.FromMinutes(5)); + + return Ok(result); } + [Authorize] [HttpGet("{poolId:guid}")] public async Task> GetPoolUsage(Guid poolId) { - var usageDetails = await usageService.GetPoolUsage(poolId); + if (HttpContext.Items["CurrentUser"] is not Account currentUser) return Unauthorized(); + var accountId = Guid.Parse(currentUser.Id); + + var usageDetails = await usage.GetPoolUsage(poolId, accountId); if (usageDetails == null) - { return NotFound(); - } return usageDetails; } } diff --git a/DysonNetwork.Drive/Billing/UsageService.cs b/DysonNetwork.Drive/Billing/UsageService.cs index b4e2e57..a612752 100644 --- a/DysonNetwork.Drive/Billing/UsageService.cs +++ b/DysonNetwork.Drive/Billing/UsageService.cs @@ -12,27 +12,27 @@ public class UsageDetails public required long FileCount { get; set; } } -public class UsageDetailsWithPercentage : UsageDetails -{ - public required double Percentage { get; set; } -} - public class TotalUsageDetails { - public required List PoolUsages { get; set; } + public required List PoolUsages { get; set; } public required long TotalUsageBytes { get; set; } - public required double TotalCost { get; set; } public required long TotalFileCount { get; set; } + + // Quota, cannot be loaded in the service, cause circular dependency + // Let the controller do the calculation + public long? TotalQuota { get; set; } + public long? UsedQuota { get; set; } } public class UsageService(AppDatabase db) { - public async Task GetTotalUsage() + public async Task GetTotalUsage(Guid accountId) { var now = SystemClock.Instance.GetCurrentInstant(); var fileQuery = db.Files .Where(f => !f.IsMarkedRecycle) .Where(f => !f.ExpiredAt.HasValue || f.ExpiredAt > now) + .Where(f => f.AccountId == accountId) .AsQueryable(); var poolUsages = await db.Pools @@ -56,26 +56,16 @@ public class UsageService(AppDatabase db) var totalCost = poolUsages.Sum(p => p.Cost); var totalFileCount = poolUsages.Sum(p => p.FileCount); - var poolUsagesWithPercentage = poolUsages.Select(p => new UsageDetailsWithPercentage - { - PoolId = p.PoolId, - PoolName = p.PoolName, - UsageBytes = p.UsageBytes, - Cost = p.Cost, - FileCount = p.FileCount, - Percentage = totalUsage > 0 ? (double)p.UsageBytes / totalUsage : 0 - }).ToList(); - return new TotalUsageDetails { - PoolUsages = poolUsagesWithPercentage, + PoolUsages = poolUsages, TotalUsageBytes = totalUsage, - TotalCost = totalCost, - TotalFileCount = totalFileCount + TotalFileCount = totalFileCount, + UsedQuota = await GetTotalBillableUsage() }; } - public async Task GetPoolUsage(Guid poolId) + public async Task GetPoolUsage(Guid poolId, Guid accountId) { var pool = await db.Pools.FindAsync(poolId); if (pool == null) @@ -87,6 +77,7 @@ public class UsageService(AppDatabase db) var fileQuery = db.Files .Where(f => !f.IsMarkedRecycle) .Where(f => f.ExpiredAt.HasValue && f.ExpiredAt > now) + .Where(f => f.AccountId == accountId) .AsQueryable(); var usageBytes = await fileQuery @@ -107,4 +98,24 @@ public class UsageService(AppDatabase db) FileCount = fileCount }; } + + public async Task GetTotalBillableUsage() + { + var now = SystemClock.Instance.GetCurrentInstant(); + var files = await db.Files + .Where(f => f.PoolId.HasValue) + .Where(f => !f.IsMarkedRecycle) + .Include(f => f.Pool) + .Where(f => !f.ExpiredAt.HasValue || f.ExpiredAt > now) + .Select(f => new + { + f.Size, + Multiplier = f.Pool!.BillingConfig.CostMultiplier ?? 1.0 + }) + .ToListAsync(); + + var totalCost = files.Sum(f => f.Size * f.Multiplier) / 1024.0 / 1024.0; + + return (long)Math.Ceiling(totalCost); + } } \ No newline at end of file diff --git a/DysonNetwork.Drive/Client/src/components/FilePoolSelect.vue b/DysonNetwork.Drive/Client/src/components/FilePoolSelect.vue index 43f01f7..01d65b0 100644 --- a/DysonNetwork.Drive/Client/src/components/FilePoolSelect.vue +++ b/DysonNetwork.Drive/Client/src/components/FilePoolSelect.vue @@ -122,7 +122,7 @@ function renderPoolSelectLabel(option: SelectOption & SnFilePool) { ), policy.require_privilege && h('span', `Require ${perkPrivilegeList[policy.require_privilege - 1]} Program`), - h('span', `Cost x${option.billing_config.cost_multiplier.toFixed(1)} NSD`), + h('span', `Cost x${option.billing_config.cost_multiplier.toFixed(1)}`), ] .filter((el) => el) .flatMap((el, idx, arr) => diff --git a/DysonNetwork.Drive/Client/src/layouts/default.vue b/DysonNetwork.Drive/Client/src/layouts/default.vue index cb1d6e8..f683098 100644 --- a/DysonNetwork.Drive/Client/src/layouts/default.vue +++ b/DysonNetwork.Drive/Client/src/layouts/default.vue @@ -64,7 +64,7 @@ const guestOptions = [ const userOptions = computed(() => [ { - label: 'Usage', + label: 'Dashboard', key: 'dashboardUsage', icon: () => h(NIcon, null, { diff --git a/DysonNetwork.Drive/Client/src/views/dashboard/usage.vue b/DysonNetwork.Drive/Client/src/views/dashboard/usage.vue index d4b136e..2bb81e2 100644 --- a/DysonNetwork.Drive/Client/src/views/dashboard/usage.vue +++ b/DysonNetwork.Drive/Client/src/views/dashboard/usage.vue @@ -3,7 +3,16 @@
- + + + +

+ The minimal billable unit is MiB, if your file is not enough 1 MiB it will be counted as + 1 MiB. +

+

The 1 MiB = 1024 KiB = 1,048,576 B

+ + @@ -25,23 +34,45 @@ - - - + + + - - - +
+ + + + + +
- + + + + + + -import { NSpin, NCard, NStatistic, NGrid, NGi, NNumberAnimation } from 'naive-ui' +import { NSpin, NCard, NStatistic, NGrid, NGi, NNumberAnimation, NAlert, NProgress } from 'naive-ui' import { Chart as ChartJS, Title, Tooltip, Legend, ArcElement } from 'chart.js' import { Pie } from 'vue-chartjs' import { computed, onMounted, ref } from 'vue' @@ -66,7 +97,7 @@ ChartJS.register(Title, Tooltip, Legend, ArcElement) const breakpoints = useBreakpoints(breakpointsTailwind) const isDesktop = breakpoints.greaterOrEqual('md') -const chartData = computed(() => ({ +const poolChartData = computed(() => ({ labels: usage.value.pool_usages.map((pool: any) => pool.pool_name), datasets: [ { @@ -74,7 +105,7 @@ const chartData = computed(() => ({ backgroundColor: '#7D80BAFF', data: usage.value.pool_usages.map((pool: any) => pool.usage_bytes), }, - ] + ], })) const usage = ref() @@ -91,6 +122,36 @@ async function fetchUsage() { } onMounted(() => fetchUsage()) +const verboseQuota = ref< + { based_quota: number; extra_quota: number; total_quota: number } | undefined +>() +async function fetchVerboseQuota() { + try { + const response = await fetch('/api/billing/quota') + if (!response.ok) { + throw new Error('Network response was not ok') + } + verboseQuota.value = await response.json() + } catch (error) { + console.error('Failed to fetch verbose data:', error) + } +} +onMounted(() => fetchVerboseQuota()) + +const quotaChartData = computed(() => ({ + labels: ['Base Quota', 'Extra Quota'], + datasets: [ + { + label: 'Verbose Quota', + backgroundColor: '#7D80BAFF', + data: [verboseQuota.value?.based_quota ?? 0, verboseQuota.value?.extra_quota ?? 0], + }, + ], +})) +const quotaUsagePercentage = computed( + () => (usage.value.used_quota / usage.value.total_quota) * 100, +) + function toGigabytes(bytes: number): number { return bytes / (1024 * 1024 * 1024) } diff --git a/DysonNetwork.Drive/Client/src/views/index.vue b/DysonNetwork.Drive/Client/src/views/index.vue index 88d4cbe..833963c 100644 --- a/DysonNetwork.Drive/Client/src/views/index.vue +++ b/DysonNetwork.Drive/Client/src/views/index.vue @@ -1,6 +1,6 @@ + + + You're uploading to a pool which enabled recycle. If the file you uploaded didn't + referenced from the Solar Network. It will be marked and will be deleted some while later. + + +
@@ -107,6 +114,7 @@ import { NSwitch, NCollapseTransition, NDatePicker, + NAlert, type UploadCustomRequestOptions, type UploadSettledFileInfo, type UploadFileInfo, @@ -149,6 +157,10 @@ const currentFilePool = computed(() => { if (!filePool.value) return null return pools.value?.find((pool) => pool.id === filePool.value) ?? null }) +const showRecycleHint = computed(() => { + if (!filePool.value) return true + return currentFilePool.value.policy_config?.enable_recycle || false +}) const messageDisplay = useMessage() diff --git a/DysonNetwork.Drive/Migrations/20250727092028_AddQuotaRecord.Designer.cs b/DysonNetwork.Drive/Migrations/20250727092028_AddQuotaRecord.Designer.cs new file mode 100644 index 0000000..42f0a28 --- /dev/null +++ b/DysonNetwork.Drive/Migrations/20250727092028_AddQuotaRecord.Designer.cs @@ -0,0 +1,322 @@ +// +using System; +using System.Collections.Generic; +using DysonNetwork.Drive; +using DysonNetwork.Drive.Storage; +using DysonNetwork.Shared.Data; +using Microsoft.EntityFrameworkCore; +using Microsoft.EntityFrameworkCore.Infrastructure; +using Microsoft.EntityFrameworkCore.Migrations; +using Microsoft.EntityFrameworkCore.Storage.ValueConversion; +using NodaTime; +using Npgsql.EntityFrameworkCore.PostgreSQL.Metadata; + +#nullable disable + +namespace DysonNetwork.Drive.Migrations +{ + [DbContext(typeof(AppDatabase))] + [Migration("20250727092028_AddQuotaRecord")] + partial class AddQuotaRecord + { + /// + protected override void BuildTargetModel(ModelBuilder modelBuilder) + { +#pragma warning disable 612, 618 + modelBuilder + .HasAnnotation("ProductVersion", "9.0.7") + .HasAnnotation("Relational:MaxIdentifierLength", 63); + + NpgsqlModelBuilderExtensions.HasPostgresExtension(modelBuilder, "postgis"); + NpgsqlModelBuilderExtensions.UseIdentityByDefaultColumns(modelBuilder); + + modelBuilder.Entity("DysonNetwork.Drive.Billing.QuotaRecord", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("uuid") + .HasColumnName("id"); + + b.Property("AccountId") + .HasColumnType("uuid") + .HasColumnName("account_id"); + + b.Property("CreatedAt") + .HasColumnType("timestamp with time zone") + .HasColumnName("created_at"); + + b.Property("DeletedAt") + .HasColumnType("timestamp with time zone") + .HasColumnName("deleted_at"); + + b.Property("Description") + .IsRequired() + .HasColumnType("text") + .HasColumnName("description"); + + b.Property("ExpiredAt") + .HasColumnType("timestamp with time zone") + .HasColumnName("expired_at"); + + b.Property("Name") + .IsRequired() + .HasColumnType("text") + .HasColumnName("name"); + + b.Property("Quota") + .HasColumnType("bigint") + .HasColumnName("quota"); + + b.Property("UpdatedAt") + .HasColumnType("timestamp with time zone") + .HasColumnName("updated_at"); + + b.HasKey("Id") + .HasName("pk_quota_records"); + + b.ToTable("quota_records", (string)null); + }); + + modelBuilder.Entity("DysonNetwork.Drive.Storage.CloudFile", b => + { + b.Property("Id") + .HasMaxLength(32) + .HasColumnType("character varying(32)") + .HasColumnName("id"); + + b.Property("AccountId") + .HasColumnType("uuid") + .HasColumnName("account_id"); + + b.Property("CreatedAt") + .HasColumnType("timestamp with time zone") + .HasColumnName("created_at"); + + b.Property("DeletedAt") + .HasColumnType("timestamp with time zone") + .HasColumnName("deleted_at"); + + b.Property("Description") + .HasMaxLength(4096) + .HasColumnType("character varying(4096)") + .HasColumnName("description"); + + b.Property("ExpiredAt") + .HasColumnType("timestamp with time zone") + .HasColumnName("expired_at"); + + b.Property>("FileMeta") + .HasColumnType("jsonb") + .HasColumnName("file_meta"); + + b.Property("HasCompression") + .HasColumnType("boolean") + .HasColumnName("has_compression"); + + b.Property("HasThumbnail") + .HasColumnType("boolean") + .HasColumnName("has_thumbnail"); + + b.Property("Hash") + .HasMaxLength(256) + .HasColumnType("character varying(256)") + .HasColumnName("hash"); + + b.Property("IsEncrypted") + .HasColumnType("boolean") + .HasColumnName("is_encrypted"); + + b.Property("IsMarkedRecycle") + .HasColumnType("boolean") + .HasColumnName("is_marked_recycle"); + + b.Property("MimeType") + .HasMaxLength(256) + .HasColumnType("character varying(256)") + .HasColumnName("mime_type"); + + b.Property("Name") + .IsRequired() + .HasMaxLength(1024) + .HasColumnType("character varying(1024)") + .HasColumnName("name"); + + b.Property("PoolId") + .HasColumnType("uuid") + .HasColumnName("pool_id"); + + b.Property>("SensitiveMarks") + .HasColumnType("jsonb") + .HasColumnName("sensitive_marks"); + + b.Property("Size") + .HasColumnType("bigint") + .HasColumnName("size"); + + b.Property("StorageId") + .HasMaxLength(32) + .HasColumnType("character varying(32)") + .HasColumnName("storage_id"); + + b.Property("StorageUrl") + .HasMaxLength(4096) + .HasColumnType("character varying(4096)") + .HasColumnName("storage_url"); + + b.Property("UpdatedAt") + .HasColumnType("timestamp with time zone") + .HasColumnName("updated_at"); + + b.Property("UploadedAt") + .HasColumnType("timestamp with time zone") + .HasColumnName("uploaded_at"); + + b.Property("UploadedTo") + .HasMaxLength(128) + .HasColumnType("character varying(128)") + .HasColumnName("uploaded_to"); + + b.Property>("UserMeta") + .HasColumnType("jsonb") + .HasColumnName("user_meta"); + + b.HasKey("Id") + .HasName("pk_files"); + + b.HasIndex("PoolId") + .HasDatabaseName("ix_files_pool_id"); + + b.ToTable("files", (string)null); + }); + + modelBuilder.Entity("DysonNetwork.Drive.Storage.CloudFileReference", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("uuid") + .HasColumnName("id"); + + b.Property("CreatedAt") + .HasColumnType("timestamp with time zone") + .HasColumnName("created_at"); + + b.Property("DeletedAt") + .HasColumnType("timestamp with time zone") + .HasColumnName("deleted_at"); + + b.Property("ExpiredAt") + .HasColumnType("timestamp with time zone") + .HasColumnName("expired_at"); + + b.Property("FileId") + .IsRequired() + .HasMaxLength(32) + .HasColumnType("character varying(32)") + .HasColumnName("file_id"); + + b.Property("ResourceId") + .IsRequired() + .HasMaxLength(1024) + .HasColumnType("character varying(1024)") + .HasColumnName("resource_id"); + + b.Property("UpdatedAt") + .HasColumnType("timestamp with time zone") + .HasColumnName("updated_at"); + + b.Property("Usage") + .IsRequired() + .HasMaxLength(1024) + .HasColumnType("character varying(1024)") + .HasColumnName("usage"); + + b.HasKey("Id") + .HasName("pk_file_references"); + + b.HasIndex("FileId") + .HasDatabaseName("ix_file_references_file_id"); + + b.ToTable("file_references", (string)null); + }); + + modelBuilder.Entity("DysonNetwork.Drive.Storage.FilePool", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("uuid") + .HasColumnName("id"); + + b.Property("AccountId") + .HasColumnType("uuid") + .HasColumnName("account_id"); + + b.Property("BillingConfig") + .IsRequired() + .HasColumnType("jsonb") + .HasColumnName("billing_config"); + + b.Property("CreatedAt") + .HasColumnType("timestamp with time zone") + .HasColumnName("created_at"); + + b.Property("DeletedAt") + .HasColumnType("timestamp with time zone") + .HasColumnName("deleted_at"); + + b.Property("Description") + .IsRequired() + .HasMaxLength(8192) + .HasColumnType("character varying(8192)") + .HasColumnName("description"); + + b.Property("Name") + .IsRequired() + .HasMaxLength(1024) + .HasColumnType("character varying(1024)") + .HasColumnName("name"); + + b.Property("PolicyConfig") + .IsRequired() + .HasColumnType("jsonb") + .HasColumnName("policy_config"); + + b.Property("StorageConfig") + .IsRequired() + .HasColumnType("jsonb") + .HasColumnName("storage_config"); + + b.Property("UpdatedAt") + .HasColumnType("timestamp with time zone") + .HasColumnName("updated_at"); + + b.HasKey("Id") + .HasName("pk_pools"); + + b.ToTable("pools", (string)null); + }); + + modelBuilder.Entity("DysonNetwork.Drive.Storage.CloudFile", b => + { + b.HasOne("DysonNetwork.Drive.Storage.FilePool", "Pool") + .WithMany() + .HasForeignKey("PoolId") + .HasConstraintName("fk_files_pools_pool_id"); + + b.Navigation("Pool"); + }); + + modelBuilder.Entity("DysonNetwork.Drive.Storage.CloudFileReference", b => + { + b.HasOne("DysonNetwork.Drive.Storage.CloudFile", "File") + .WithMany() + .HasForeignKey("FileId") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired() + .HasConstraintName("fk_file_references_files_file_id"); + + b.Navigation("File"); + }); +#pragma warning restore 612, 618 + } + } +} diff --git a/DysonNetwork.Drive/Migrations/20250727092028_AddQuotaRecord.cs b/DysonNetwork.Drive/Migrations/20250727092028_AddQuotaRecord.cs new file mode 100644 index 0000000..537a8bb --- /dev/null +++ b/DysonNetwork.Drive/Migrations/20250727092028_AddQuotaRecord.cs @@ -0,0 +1,42 @@ +using System; +using Microsoft.EntityFrameworkCore.Migrations; +using NodaTime; + +#nullable disable + +namespace DysonNetwork.Drive.Migrations +{ + /// + public partial class AddQuotaRecord : Migration + { + /// + protected override void Up(MigrationBuilder migrationBuilder) + { + migrationBuilder.CreateTable( + name: "quota_records", + columns: table => new + { + id = table.Column(type: "uuid", nullable: false), + account_id = table.Column(type: "uuid", nullable: false), + name = table.Column(type: "text", nullable: false), + description = table.Column(type: "text", nullable: false), + quota = table.Column(type: "bigint", nullable: false), + expired_at = table.Column(type: "timestamp with time zone", nullable: true), + created_at = table.Column(type: "timestamp with time zone", nullable: false), + updated_at = table.Column(type: "timestamp with time zone", nullable: false), + deleted_at = table.Column(type: "timestamp with time zone", nullable: true) + }, + constraints: table => + { + table.PrimaryKey("pk_quota_records", x => x.id); + }); + } + + /// + protected override void Down(MigrationBuilder migrationBuilder) + { + migrationBuilder.DropTable( + name: "quota_records"); + } + } +} diff --git a/DysonNetwork.Drive/Migrations/AppDatabaseModelSnapshot.cs b/DysonNetwork.Drive/Migrations/AppDatabaseModelSnapshot.cs index 1f83209..7c180a3 100644 --- a/DysonNetwork.Drive/Migrations/AppDatabaseModelSnapshot.cs +++ b/DysonNetwork.Drive/Migrations/AppDatabaseModelSnapshot.cs @@ -27,6 +27,53 @@ namespace DysonNetwork.Drive.Migrations NpgsqlModelBuilderExtensions.HasPostgresExtension(modelBuilder, "postgis"); NpgsqlModelBuilderExtensions.UseIdentityByDefaultColumns(modelBuilder); + modelBuilder.Entity("DysonNetwork.Drive.Billing.QuotaRecord", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("uuid") + .HasColumnName("id"); + + b.Property("AccountId") + .HasColumnType("uuid") + .HasColumnName("account_id"); + + b.Property("CreatedAt") + .HasColumnType("timestamp with time zone") + .HasColumnName("created_at"); + + b.Property("DeletedAt") + .HasColumnType("timestamp with time zone") + .HasColumnName("deleted_at"); + + b.Property("Description") + .IsRequired() + .HasColumnType("text") + .HasColumnName("description"); + + b.Property("ExpiredAt") + .HasColumnType("timestamp with time zone") + .HasColumnName("expired_at"); + + b.Property("Name") + .IsRequired() + .HasColumnType("text") + .HasColumnName("name"); + + b.Property("Quota") + .HasColumnType("bigint") + .HasColumnName("quota"); + + b.Property("UpdatedAt") + .HasColumnType("timestamp with time zone") + .HasColumnName("updated_at"); + + b.HasKey("Id") + .HasName("pk_quota_records"); + + b.ToTable("quota_records", (string)null); + }); + modelBuilder.Entity("DysonNetwork.Drive.Storage.CloudFile", b => { b.Property("Id") diff --git a/DysonNetwork.Drive/Program.cs b/DysonNetwork.Drive/Program.cs index 344ab0e..e88b280 100644 --- a/DysonNetwork.Drive/Program.cs +++ b/DysonNetwork.Drive/Program.cs @@ -20,6 +20,7 @@ builder.Services.AddAppRateLimiting(); builder.Services.AddAppAuthentication(); builder.Services.AddAppSwagger(); builder.Services.AddDysonAuth(); +builder.Services.AddAccountService(); builder.Services.AddAppFileStorage(builder.Configuration); diff --git a/DysonNetwork.Drive/Startup/ServiceCollectionExtensions.cs b/DysonNetwork.Drive/Startup/ServiceCollectionExtensions.cs index 4f64952..2fe5450 100644 --- a/DysonNetwork.Drive/Startup/ServiceCollectionExtensions.cs +++ b/DysonNetwork.Drive/Startup/ServiceCollectionExtensions.cs @@ -139,7 +139,7 @@ public static class ServiceCollectionExtensions services.AddScoped(); services.AddScoped(); services.AddScoped(); - services.AddScoped(); + services.AddScoped(); return services; } diff --git a/DysonNetwork.Drive/Storage/TusService.cs b/DysonNetwork.Drive/Storage/TusService.cs index ae46f10..f9a58ba 100644 --- a/DysonNetwork.Drive/Storage/TusService.cs +++ b/DysonNetwork.Drive/Storage/TusService.cs @@ -1,6 +1,7 @@ using System.Net; using System.Text; using System.Text.Json; +using DysonNetwork.Drive.Billing; using DysonNetwork.Shared.Auth; using DysonNetwork.Shared.Proto; using Microsoft.AspNetCore.Mvc; @@ -9,7 +10,6 @@ using NodaTime; using tusdotnet.Interfaces; using tusdotnet.Models; using tusdotnet.Models.Configuration; -using tusdotnet.Stores; namespace DysonNetwork.Drive.Storage; @@ -86,7 +86,6 @@ public abstract class TusService HttpStatusCode.Forbidden, $"You need Stellar Program tier {pool.PolicyConfig.RequirePrivilege} to use this pool, you are tier {privilege}" ); - return; } } }, @@ -140,8 +139,10 @@ public abstract class TusService } catch (Exception ex) { + var logger = services.GetRequiredService>(); eventContext.HttpContext.Response.StatusCode = StatusCodes.Status400BadRequest; await eventContext.HttpContext.Response.WriteAsync(ex.Message); + logger.LogError(ex, "Error handling file upload..."); } finally { @@ -151,6 +152,13 @@ public abstract class TusService }, OnBeforeCreateAsync = async eventContext => { + var httpContext = eventContext.HttpContext; + if (httpContext.Items["CurrentUser"] is not Account currentUser) + { + eventContext.FailRequest(HttpStatusCode.Unauthorized); + return; + } + var filePool = eventContext.HttpContext.Request.Headers["X-FilePool"].FirstOrDefault(); if (string.IsNullOrEmpty(filePool)) filePool = configuration["Storage:PreferredRemote"]; if (!Guid.TryParse(filePool, out _)) @@ -223,6 +231,25 @@ public abstract class TusService } } + if (!rejected) + { + var quotaService = scope.ServiceProvider.GetRequiredService(); + var accountId = Guid.Parse(currentUser.Id); + var (ok, billableUnit, quota) = await quotaService.IsFileAcceptable( + accountId, + pool.BillingConfig.CostMultiplier ?? 1.0, + eventContext.UploadLength + ); + if (!ok) + { + eventContext.FailRequest( + HttpStatusCode.Forbidden, + $"File size {billableUnit} MiB is exceed than the user's quota {quota} MiB" + ); + rejected = true; + } + } + if (rejected) logger.LogInformation("File rejected #{FileId}", eventContext.FileId); }, @@ -230,7 +257,7 @@ public abstract class TusService { var directUpload = eventContext.HttpContext.Request.Headers["X-DirectUpload"].FirstOrDefault(); if (!string.IsNullOrEmpty(directUpload)) return Task.CompletedTask; - + var gatewayUrl = configuration["GatewayUrl"]; if (gatewayUrl is not null) eventContext.SetUploadUrl(new Uri(gatewayUrl + "/drive/tus/" + eventContext.FileId));