From 9cffd8383e32e007ad63fcbbcc46c565a8a01dff Mon Sep 17 00:00:00 2001 From: LittleSheep Date: Wed, 16 Apr 2025 00:18:59 +0800 Subject: [PATCH] :sparkles: Timed tasks :bug: Bug fixes --- .../Account/AccountController.cs | 21 +++----- DysonNetwork.Sphere/AppDatabase.cs | 50 ++++++++++++++++++- .../DysonNetwork.Sphere.csproj | 4 ++ DysonNetwork.Sphere/Program.cs | 23 +++++++++ DysonNetwork.Sphere/Storage/FileController.cs | 2 +- DysonNetwork.Sphere/Storage/FileService.cs | 47 ++++++++++++++++- DysonNetwork.sln.DotSettings.user | 5 ++ 7 files changed, 135 insertions(+), 17 deletions(-) diff --git a/DysonNetwork.Sphere/Account/AccountController.cs b/DysonNetwork.Sphere/Account/AccountController.cs index 38626d8..92dccf3 100644 --- a/DysonNetwork.Sphere/Account/AccountController.cs +++ b/DysonNetwork.Sphere/Account/AccountController.cs @@ -1,4 +1,5 @@ using System.ComponentModel.DataAnnotations; +using DysonNetwork.Sphere.Storage; using Microsoft.AspNetCore.Authorization; using Microsoft.AspNetCore.Mvc; using Microsoft.EntityFrameworkCore; @@ -7,7 +8,7 @@ namespace DysonNetwork.Sphere.Account; [ApiController] [Route("/accounts")] -public class AccountController(AppDatabase db) : ControllerBase +public class AccountController(AppDatabase db, FileService fs) : ControllerBase { [HttpGet("{name}")] [ProducesResponseType(StatusCodes.Status200OK)] @@ -150,14 +151,10 @@ public class AccountController(AppDatabase db) : ControllerBase var picture = await db.Files.Where(f => f.Id == request.PictureId).FirstOrDefaultAsync(); if (picture is null) return BadRequest("Invalid picture id, unable to find the file on cloud."); if (profile.Picture is not null) - { - profile.Picture.UsedCount--; - db.Update(profile.Picture); - } - - picture.UsedCount++; + await fs.MarkUsageAsync(profile.Picture, -1); + profile.Picture = picture; - db.Update(picture); + await fs.MarkUsageAsync(picture, 1); } if (request.BackgroundId is not null) @@ -165,14 +162,10 @@ public class AccountController(AppDatabase db) : ControllerBase var background = await db.Files.Where(f => f.Id == request.BackgroundId).FirstOrDefaultAsync(); if (background is null) return BadRequest("Invalid background id, unable to find the file on cloud."); if (profile.Background is not null) - { - profile.Background.UsedCount--; - db.Update(profile.Background); - } + await fs.MarkUsageAsync(profile.Background, -1); - background.UsedCount++; profile.Background = background; - db.Update(background); + await fs.MarkUsageAsync(background, 1); } db.Update(profile); diff --git a/DysonNetwork.Sphere/AppDatabase.cs b/DysonNetwork.Sphere/AppDatabase.cs index a4701ef..493956b 100644 --- a/DysonNetwork.Sphere/AppDatabase.cs +++ b/DysonNetwork.Sphere/AppDatabase.cs @@ -1,7 +1,9 @@ +using System.Linq.Expressions; using Microsoft.EntityFrameworkCore; using Microsoft.EntityFrameworkCore.Design; using NodaTime; using Npgsql; +using Quartz; namespace DysonNetwork.Sphere; @@ -43,7 +45,7 @@ public class AppDatabase( protected override void OnModelCreating(ModelBuilder modelBuilder) { base.OnModelCreating(modelBuilder); - + modelBuilder.Entity() .HasOne(a => a.Profile) .WithOne(p => p.Account) @@ -100,6 +102,52 @@ public class AppDatabase( } } +public class AppDatabaseRecyclingJob(AppDatabase db, ILogger logger) : IJob +{ + public async Task Execute(IJobExecutionContext context) + { + logger.LogInformation("Deleting soft-deleted records..."); + + var now = SystemClock.Instance.GetCurrentInstant(); + var threshold = now - Duration.FromDays(7); + + var entityTypes = db.Model.GetEntityTypes() + .Where(t => typeof(BaseModel).IsAssignableFrom(t.ClrType) && t.ClrType != typeof(BaseModel)) + .Select(t => t.ClrType); + + foreach (var entityType in entityTypes) + { + var set = (IQueryable)db.GetType().GetMethod(nameof(DbContext.Set), Type.EmptyTypes)! + .MakeGenericMethod(entityType).Invoke(db, null)!; + var parameter = Expression.Parameter(entityType, "e"); + var property = Expression.Property(parameter, nameof(BaseModel.DeletedAt)); + var condition = Expression.LessThan(property, Expression.Constant(threshold, typeof(Instant?))); + var notNull = Expression.NotEqual(property, Expression.Constant(null, typeof(Instant?))); + var finalCondition = Expression.AndAlso(notNull, condition); + var lambda = Expression.Lambda(finalCondition, parameter); + + var queryable = set.Provider.CreateQuery( + Expression.Call( + typeof(Queryable), + "Where", + [entityType], + set.Expression, + Expression.Quote(lambda) + ) + ); + + var toListAsync = typeof(EntityFrameworkQueryableExtensions) + .GetMethod(nameof(EntityFrameworkQueryableExtensions.ToListAsync))! + .MakeGenericMethod(entityType); + + var items = await (dynamic)toListAsync.Invoke(null, [queryable, CancellationToken.None])!; + db.RemoveRange(items); + } + + await db.SaveChangesAsync(); + } +} + public class AppDatabaseFactory : IDesignTimeDbContextFactory { public AppDatabase CreateDbContext(string[] args) diff --git a/DysonNetwork.Sphere/DysonNetwork.Sphere.csproj b/DysonNetwork.Sphere/DysonNetwork.Sphere.csproj index d824a99..c0a3d83 100644 --- a/DysonNetwork.Sphere/DysonNetwork.Sphere.csproj +++ b/DysonNetwork.Sphere/DysonNetwork.Sphere.csproj @@ -21,6 +21,7 @@ all runtime; build; native; contentfiles; analyzers; buildtransitive + all @@ -32,6 +33,9 @@ + + + diff --git a/DysonNetwork.Sphere/Program.cs b/DysonNetwork.Sphere/Program.cs index 1a66c11..f704f61 100644 --- a/DysonNetwork.Sphere/Program.cs +++ b/DysonNetwork.Sphere/Program.cs @@ -17,6 +17,7 @@ using Microsoft.IdentityModel.Tokens; using Microsoft.OpenApi.Models; using NodaTime; using NodaTime.Serialization.SystemTextJson; +using Quartz; using tusdotnet; using tusdotnet.Models; using File = System.IO.File; @@ -117,6 +118,28 @@ builder.Services.AddScoped(); builder.Services.AddScoped(); builder.Services.AddScoped(); +// Timed task + +builder.Services.AddQuartz(q => +{ + var appDatabaseRecyclingJob = new JobKey("AppDatabaseRecycling"); + q.AddJob(opts => opts.WithIdentity(appDatabaseRecyclingJob)); + q.AddTrigger(opts => opts + .ForJob(appDatabaseRecyclingJob) + .WithIdentity("AppDatabaseRecyclingTrigger") + .WithCronSchedule("0 0 0 * * ?")); + + var cloudFilesRecyclingJob = new JobKey("CloudFilesUnusedRecycling"); + q.AddJob(opts => opts.WithIdentity(cloudFilesRecyclingJob)); + q.AddTrigger(opts => opts + .ForJob(cloudFilesRecyclingJob) + .WithIdentity("CloudFilesUnusedRecyclingTrigger") + .WithSimpleSchedule(o => o.WithIntervalInHours(1).RepeatForever()) + ); +}); +builder.Services.AddQuartzHostedService(q => q.WaitForJobsToComplete = true); + + var app = builder.Build(); using (var scope = app.Services.CreateScope()) diff --git a/DysonNetwork.Sphere/Storage/FileController.cs b/DysonNetwork.Sphere/Storage/FileController.cs index 4dab257..c9ab38e 100644 --- a/DysonNetwork.Sphere/Storage/FileController.cs +++ b/DysonNetwork.Sphere/Storage/FileController.cs @@ -93,7 +93,7 @@ public class FileController( .FirstOrDefaultAsync(); if (file is null) return NotFound(); - await fs.DeleteFileDataAsync(file); + await fs.DeleteFileAsync(file); db.Files.Remove(file); await db.SaveChangesAsync(); diff --git a/DysonNetwork.Sphere/Storage/FileService.cs b/DysonNetwork.Sphere/Storage/FileService.cs index 66db4f6..ff9ab6b 100644 --- a/DysonNetwork.Sphere/Storage/FileService.cs +++ b/DysonNetwork.Sphere/Storage/FileService.cs @@ -6,6 +6,7 @@ using Microsoft.EntityFrameworkCore; using Minio; using Minio.DataModel.Args; using NodaTime; +using Quartz; using SixLabors.ImageSharp.PixelFormats; using SixLabors.ImageSharp.Metadata.Profiles.Exif; using ExifTag = SixLabors.ImageSharp.Metadata.Profiles.Exif.ExifTag; @@ -162,6 +163,14 @@ public class FileService(AppDatabase db, IConfiguration configuration) return file; } + public async Task DeleteFileAsync(CloudFile file) + { + await DeleteFileDataAsync(file); + + db.Remove(file); + await db.SaveChangesAsync(); + } + public async Task DeleteFileDataAsync(CloudFile file) { if (file.UploadedTo is null) return; @@ -177,7 +186,8 @@ public class FileService(AppDatabase db, IConfiguration configuration) new RemoveObjectArgs().WithBucket(bucket).WithObject(file.Id) ); - return; + db.Remove(file); + await db.SaveChangesAsync(); } public RemoteStorageConfig GetRemoteStorageConfig(string destination) @@ -198,4 +208,39 @@ public class FileService(AppDatabase db, IConfiguration configuration) return client.Build(); } + + public async Task MarkUsageAsync(CloudFile file, int delta) + { + await db.Files.Where(o => o.Id == file.Id) + .ExecuteUpdateAsync( + setter => setter.SetProperty( + b => b.UsedCount, + b => b.UsedCount + delta + ) + ); + } +} + +public class CloudFileUnusedRecyclingJob(AppDatabase db, FileService fs, ILogger logger) + : IJob +{ + public async Task Execute(IJobExecutionContext context) + { + logger.LogInformation("Deleting unused cloud files..."); + + var cutoff = SystemClock.Instance.GetCurrentInstant() - Duration.FromHours(1); + var files = db.Files + .Where(f => f.UsedCount == 0) + .Where(f => f.CreatedAt < cutoff) + .ToList(); + + logger.LogInformation($"Deleting {files.Count} unused cloud files..."); + + var tasks = files.Select(fs.DeleteFileDataAsync); + await Task.WhenAll(tasks); + + await db.Files + .Where(f => f.UsedCount == 0 && f.CreatedAt < cutoff) + .ExecuteDeleteAsync(); + } } \ No newline at end of file diff --git a/DysonNetwork.sln.DotSettings.user b/DysonNetwork.sln.DotSettings.user index 4093039..a0e769c 100644 --- a/DysonNetwork.sln.DotSettings.user +++ b/DysonNetwork.sln.DotSettings.user @@ -4,6 +4,8 @@ ForceIncluded ForceIncluded ForceIncluded + ForceIncluded + ForceIncluded ForceIncluded ForceIncluded ForceIncluded @@ -17,8 +19,10 @@ ForceIncluded ForceIncluded ForceIncluded + ForceIncluded ForceIncluded ForceIncluded + ForceIncluded ForceIncluded ForceIncluded ForceIncluded @@ -27,6 +31,7 @@ ForceIncluded ForceIncluded ForceIncluded + ForceIncluded ForceIncluded ForceIncluded ForceIncluded