diff --git a/DysonNetwork.Drive/Program.cs b/DysonNetwork.Drive/Program.cs index e88b280..eef5c20 100644 --- a/DysonNetwork.Drive/Program.cs +++ b/DysonNetwork.Drive/Program.cs @@ -5,6 +5,7 @@ using DysonNetwork.Shared.Auth; using DysonNetwork.Shared.Http; using DysonNetwork.Shared.PageData; using DysonNetwork.Shared.Registry; +using DysonNetwork.Shared.Stream; using Microsoft.EntityFrameworkCore; using tusdotnet.Stores; @@ -15,6 +16,7 @@ builder.ConfigureAppKestrel(builder.Configuration, maxRequestBodySize: long.MaxV // Add application services builder.Services.AddRegistryService(builder.Configuration); +builder.Services.AddStreamConnection(builder.Configuration); builder.Services.AddAppServices(builder.Configuration); builder.Services.AddAppRateLimiting(); builder.Services.AddAppAuthentication(); diff --git a/DysonNetwork.Drive/Startup/BroadcastEventHandler.cs b/DysonNetwork.Drive/Startup/BroadcastEventHandler.cs new file mode 100644 index 0000000..77691cc --- /dev/null +++ b/DysonNetwork.Drive/Startup/BroadcastEventHandler.cs @@ -0,0 +1,53 @@ +using System.Text.Json; +using DysonNetwork.Drive.Storage; +using DysonNetwork.Shared.Stream; +using Microsoft.EntityFrameworkCore; +using NATS.Client.Core; + +namespace DysonNetwork.Drive.Startup; + +public class BroadcastEventHandler( + INatsConnection nats, + ILogger logger, + FileService fs, + AppDatabase db +) : BackgroundService +{ + protected override async Task ExecuteAsync(CancellationToken stoppingToken) + { + await foreach (var msg in nats.SubscribeAsync("accounts.deleted", cancellationToken: stoppingToken)) + { + try + { + var evt = JsonSerializer.Deserialize(msg.Data); + if (evt == null) continue; + + logger.LogInformation("Account deleted: {AccountId}", evt.AccountId); + + await using var transaction = await db.Database.BeginTransactionAsync(cancellationToken: stoppingToken); + try + { + var files = await db.Files + .Where(p => p.AccountId == evt.AccountId) + .ToListAsync(cancellationToken: stoppingToken); + + await fs.DeleteFileDataBatchAsync(files); + await db.Files + .Where(p => p.AccountId == evt.AccountId) + .ExecuteDeleteAsync(cancellationToken: stoppingToken); + + await transaction.CommitAsync(cancellationToken: stoppingToken); + } + catch (Exception) + { + await transaction.RollbackAsync(cancellationToken: stoppingToken); + throw; + } + } + catch (Exception ex) + { + logger.LogError(ex, "Error processing AccountDeleted"); + } + } + } +} \ No newline at end of file diff --git a/DysonNetwork.Drive/Startup/ServiceCollectionExtensions.cs b/DysonNetwork.Drive/Startup/ServiceCollectionExtensions.cs index 2fe5450..3479169 100644 --- a/DysonNetwork.Drive/Startup/ServiceCollectionExtensions.cs +++ b/DysonNetwork.Drive/Startup/ServiceCollectionExtensions.cs @@ -140,6 +140,8 @@ public static class ServiceCollectionExtensions services.AddScoped(); services.AddScoped(); services.AddScoped(); + + services.AddHostedService(); return services; } diff --git a/DysonNetwork.Drive/Storage/CloudFile.cs b/DysonNetwork.Drive/Storage/CloudFile.cs index 9f198da..3469006 100644 --- a/DysonNetwork.Drive/Storage/CloudFile.cs +++ b/DysonNetwork.Drive/Storage/CloudFile.cs @@ -33,10 +33,6 @@ public class CloudFile : ModelBase, ICloudFile, IIdentifiedResource [JsonIgnore] public FileBundle? Bundle { get; set; } public Guid? BundleId { get; set; } - [Obsolete("Deprecated, use PoolId instead. For database migration only.")] - [MaxLength(128)] - public string? UploadedTo { get; set; } - /// /// The field is set to true if the recycling job plans to delete the file. /// Due to the unstable of the recycling job, this doesn't really delete the file until a human verifies it. @@ -60,6 +56,8 @@ public class CloudFile : ModelBase, ICloudFile, IIdentifiedResource [NotMapped] [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] public string? FastUploadLink { get; set; } + + public ICollection References { get; set; } = new List(); public Guid AccountId { get; set; } diff --git a/DysonNetwork.Drive/Storage/FileService.cs b/DysonNetwork.Drive/Storage/FileService.cs index 15d1fe3..affc2fb 100644 --- a/DysonNetwork.Drive/Storage/FileService.cs +++ b/DysonNetwork.Drive/Storage/FileService.cs @@ -102,6 +102,7 @@ public class FileService( private static readonly string[] AnimatedImageTypes = ["image/gif", "image/apng", "image/avif"]; + private static readonly string[] AnimatedImageExtensions = [".gif", ".apng", ".avif"]; @@ -278,15 +279,15 @@ public class FileService( s.Rotation }).Where(s => double.IsNormal(s.AvgFrameRate)).ToList(), ["audio_streams"] = mediaInfo.AudioStreams.Select(s => new - { - s.BitRate, - s.Channels, - s.ChannelLayout, - s.CodecName, - s.Duration, - s.Language, - s.SampleRateHz - }) + { + s.BitRate, + s.Channels, + s.ChannelLayout, + s.CodecName, + s.Duration, + s.Language, + s.SampleRateHz + }) .ToList(), }; if (mediaInfo.PrimaryVideoStream is not null) @@ -336,7 +337,8 @@ public class FileService( if (!pool.PolicyConfig.NoOptimization) switch (contentType.Split('/')[0]) { - case "image" when !AnimatedImageTypes.Contains(contentType) && !AnimatedImageExtensions.Contains(fileExtension): + case "image" when !AnimatedImageTypes.Contains(contentType) && + !AnimatedImageExtensions.Contains(fileExtension): newMimeType = "image/webp"; using (var vipsImage = Image.NewFromFile(originalFilePath)) { @@ -643,7 +645,44 @@ public class FileService( } } - public async Task GetBundleAsync(Guid id, Guid accountId) + /// + /// The most efficent way to delete file data (stored files) in batch. + /// But this DO NOT check the storage id, so use with caution! + /// + /// Files to delete + /// Something went wrong + public async Task DeleteFileDataBatchAsync(List files) + { + files = files.Where(f => f.PoolId.HasValue).ToList(); + + foreach (var fileGroup in files.GroupBy(f => f.PoolId!.Value)) + { + // If any other file with the same storage ID is referenced, don't delete the actual file data + var dest = await GetRemoteStorageConfig(fileGroup.Key); + if (dest is null) + throw new InvalidOperationException($"No remote storage configured for pool {fileGroup.Key}"); + var client = CreateMinioClient(dest); + if (client is null) + throw new InvalidOperationException( + $"Failed to configure client for remote destination '{fileGroup.Key}'" + ); + + List objectsToDelete = []; + + foreach (var file in fileGroup) + { + objectsToDelete.Add(file.StorageId ?? file.Id); + if(file.HasCompression) objectsToDelete.Add(file.StorageId ?? file.Id + ".compressed"); + if(file.HasThumbnail) objectsToDelete.Add(file.StorageId ?? file.Id + ".thumbnail"); + } + + await client.RemoveObjectsAsync( + new RemoveObjectsArgs().WithBucket(dest.Bucket).WithObjects(objectsToDelete) + ); + } + } + + private async Task GetBundleAsync(Guid id, Guid accountId) { var bundle = await db.Bundles .Where(e => e.Id == id) @@ -880,4 +919,4 @@ file class UpdatableCloudFile(CloudFile file) .SetProperty(f => f.UserMeta, userMeta!) .SetProperty(f => f.IsMarkedRecycle, IsMarkedRecycle); } -} +} \ No newline at end of file diff --git a/DysonNetwork.Shared/Proto/GrpcClientHelper.cs b/DysonNetwork.Shared/Proto/GrpcClientHelper.cs index 3eab47f..96fdf64 100644 --- a/DysonNetwork.Shared/Proto/GrpcClientHelper.cs +++ b/DysonNetwork.Shared/Proto/GrpcClientHelper.cs @@ -21,7 +21,6 @@ public static class GrpcClientHelper ? X509Certificate2.CreateFromPemFile(clientCertPath, clientKeyPath) : X509Certificate2.CreateFromEncryptedPemFile(clientCertPath, clientCertPassword, clientKeyPath) ); - // TODO: Verify the ca in the future handler.ServerCertificateCustomValidationCallback = (message, cert, chain, errors) => true; var httpClient = new HttpClient(handler); httpClient.DefaultRequestVersion = HttpVersion.Version20; diff --git a/DysonNetwork.Sphere/Startup/BroadcastEventHandler.cs b/DysonNetwork.Sphere/Startup/BroadcastEventHandler.cs index b2473e6..2512f76 100644 --- a/DysonNetwork.Sphere/Startup/BroadcastEventHandler.cs +++ b/DysonNetwork.Sphere/Startup/BroadcastEventHandler.cs @@ -21,6 +21,15 @@ public class BroadcastEventHandler( if (evt == null) continue; logger.LogInformation("Account deleted: {AccountId}", evt.AccountId); + + // TODO: Add empty realm, chat recycler in the db recycle + await db.ChatMembers + .Where(m => m.AccountId == evt.AccountId) + .ExecuteDeleteAsync(cancellationToken: stoppingToken); + + await db.RealmMembers + .Where(m => m.AccountId == evt.AccountId) + .ExecuteDeleteAsync(cancellationToken: stoppingToken); await using var transaction = await db.Database.BeginTransactionAsync(cancellationToken: stoppingToken); try @@ -33,6 +42,11 @@ public class BroadcastEventHandler( await db.Posts .Where(p => p.PublisherId == publisher.Id) .ExecuteDeleteAsync(cancellationToken: stoppingToken); + + var publisherIds = publishers.Select(p => p.Id).ToList(); + await db.Publishers + .Where(p => publisherIds.Contains(p.Id)) + .ExecuteDeleteAsync(cancellationToken: stoppingToken); await transaction.CommitAsync(cancellationToken: stoppingToken); }