Files
.github
.idx
DysonNetwork.Sphere
Account
Activity
Auth
Chat
Connection
Developer
Email
Localization
Migrations
Pages
Permission
Post
Properties
Publisher
Realm
Resources
Sticker
Storage
Handlers
CacheService.cs
CloudFile.cs
CloudFileUnusedRecyclingJob.cs
FileController.cs
FileExpirationJob.cs
FileReferenceService.cs
FileService.ReferenceMigration.cs
FileService.cs
FlushBufferService.cs
ICloudFile.cs
TextSanitizer.cs
TusService.cs
Wallet
wwwroot
.DS_Store
.gitignore
AppDatabase.cs
Dockerfile
DysonNetwork.Sphere.csproj
DysonNetwork.Sphere.csproj.DotSettings.user
DysonNetwork.Sphere.http
Program.cs
appsettings.json
package.json
postcss.config.js
tailwind.config.js
.dockerignore
.gitignore
DysonNetwork.sln
DysonNetwork.sln.DotSettings.user
compose.yaml
Swarm/DysonNetwork.Sphere/Storage/CloudFileUnusedRecyclingJob.cs

128 lines
4.8 KiB
C#

using Microsoft.EntityFrameworkCore;
using NodaTime;
using Quartz;
namespace DysonNetwork.Sphere.Storage;
public class CloudFileUnusedRecyclingJob(
AppDatabase db,
FileService fs,
FileReferenceService fileRefService,
ILogger<CloudFileUnusedRecyclingJob> logger
)
: IJob
{
public async Task Execute(IJobExecutionContext context)
{
return;
logger.LogInformation("Deleting unused cloud files...");
var cutoff = SystemClock.Instance.GetCurrentInstant() - Duration.FromHours(1);
var now = SystemClock.Instance.GetCurrentInstant();
// Get files that are either expired or created more than an hour ago
var fileIds = await db.Files
.Select(f => f.Id)
.ToListAsync();
// Filter to only include files that have no references or all references have expired
var deletionPlan = new List<string>();
foreach (var batch in fileIds.Chunk(100)) // Process in batches to avoid excessive query size
{
var references = await fileRefService.GetReferencesAsync(batch);
deletionPlan.AddRange(from refer in references
where refer.Value.Count == 0 || refer.Value.All(r => r.ExpiredAt != null && now >= r.ExpiredAt)
select refer.Key);
}
if (deletionPlan.Count == 0)
{
logger.LogInformation("No files to delete");
return;
}
// Get the actual file objects for the files to be deleted
var files = await db.Files
.Where(f => deletionPlan.Contains(f.Id))
.ToListAsync();
logger.LogInformation($"Found {files.Count} files to delete...");
// Group files by StorageId and find which ones are safe to delete
var storageIds = files.Where(f => f.StorageId != null)
.Select(f => f.StorageId!)
.Distinct()
.ToList();
// Check if any other files with the same storage IDs are referenced
var usedStorageIds = new List<string>();
var filesWithSameStorageId = await db.Files
.Where(f => f.StorageId != null &&
storageIds.Contains(f.StorageId) &&
!files.Select(ff => ff.Id).Contains(f.Id))
.ToListAsync();
foreach (var file in filesWithSameStorageId)
{
// Get all references for the file
var references = await fileRefService.GetReferencesAsync(file.Id);
// Check if file has active references (non-expired)
if (references.Any(r => r.ExpiredAt == null || r.ExpiredAt > now) && file.StorageId != null)
{
usedStorageIds.Add(file.StorageId);
}
}
// Group files for deletion
var filesToDelete = files.Where(f => f.StorageId == null || !usedStorageIds.Contains(f.StorageId))
.GroupBy(f => f.UploadedTo)
.ToDictionary(grouping => grouping.Key!, grouping => grouping.ToList());
// Delete files by remote storage
foreach (var group in filesToDelete.Where(group => !string.IsNullOrEmpty(group.Key)))
{
try
{
var dest = fs.GetRemoteStorageConfig(group.Key);
var client = fs.CreateMinioClient(dest);
if (client == null) continue;
// Create delete tasks for each file in the group
// var deleteTasks = group.Value.Select(file =>
// {
// var objectId = file.StorageId ?? file.Id;
// var tasks = new List<Task>
// {
// client.RemoveObjectAsync(new Minio.DataModel.Args.RemoveObjectArgs()
// .WithBucket(dest.Bucket)
// .WithObject(objectId))
// };
//
// if (file.HasCompression)
// {
// tasks.Add(client.RemoveObjectAsync(new Minio.DataModel.Args.RemoveObjectArgs()
// .WithBucket(dest.Bucket)
// .WithObject(objectId + ".compressed")));
// }
//
// return Task.WhenAll(tasks);
// });
//
// await Task.WhenAll(deleteTasks);
}
catch (Exception ex)
{
logger.LogError(ex, "Error deleting files from remote storage {remote}", group.Key);
}
}
// Delete all file records from the database
var fileIdsToDelete = files.Select(f => f.Id).ToList();
await db.Files
.Where(f => fileIdsToDelete.Contains(f.Id))
.ExecuteDeleteAsync();
logger.LogInformation($"Completed deleting {files.Count} files");
}
}