✨ Compressed duplication of image files

DysonNetwork.Sphere/Migrations/20250501072931_UnableToNamingThing1.cs

@@ -0,0 +1,63 @@
+using System;
+using Microsoft.EntityFrameworkCore.Migrations;
+using NodaTime;
+
+#nullable disable
+
+namespace DysonNetwork.Sphere.Migrations
+{
+    /// <inheritdoc />
+    public partial class UnableToNamingThing1 : Migration
+    {
+        /// <inheritdoc />
+        protected override void Up(MigrationBuilder migrationBuilder)
+        {
+            migrationBuilder.AddColumn<bool>(
+                name: "has_compression",
+                table: "files",
+                type: "boolean",
+                nullable: false,
+                defaultValue: false);
+
+            migrationBuilder.CreateTable(
+                name: "activities",
+                columns: table => new
+                {
+                    id = table.Column<Guid>(type: "uuid", nullable: false),
+                    type = table.Column<string>(type: "character varying(1024)", maxLength: 1024, nullable: false),
+                    resource_identifier = table.Column<string>(type: "character varying(4096)", maxLength: 4096, nullable: false),
+                    visibility = table.Column<int>(type: "integer", nullable: false),
+                    account_id = table.Column<long>(type: "bigint", nullable: false),
+                    created_at = table.Column<Instant>(type: "timestamp with time zone", nullable: false),
+                    updated_at = table.Column<Instant>(type: "timestamp with time zone", nullable: false),
+                    deleted_at = table.Column<Instant>(type: "timestamp with time zone", nullable: true)
+                },
+                constraints: table =>
+                {
+                    table.PrimaryKey("pk_activities", x => x.id);
+                    table.ForeignKey(
+                        name: "fk_activities_accounts_account_id",
+                        column: x => x.account_id,
+                        principalTable: "accounts",
+                        principalColumn: "id",
+                        onDelete: ReferentialAction.Cascade);
+                });
+
+            migrationBuilder.CreateIndex(
+                name: "ix_activities_account_id",
+                table: "activities",
+                column: "account_id");
+        }
+
+        /// <inheritdoc />
+        protected override void Down(MigrationBuilder migrationBuilder)
+        {
+            migrationBuilder.DropTable(
+                name: "activities");
+
+            migrationBuilder.DropColumn(
+                name: "has_compression",
+                table: "files");
+        }
+    }
+}

DysonNetwork.Sphere/Migrations/AppDatabaseModelSnapshot.cs

@@ -453,6 +453,54 @@ namespace DysonNetwork.Sphere.Migrations
                     b.ToTable("account_relationships", (string)null);
                 });
 
+            modelBuilder.Entity("DysonNetwork.Sphere.Activity.Activity", b =>
+                {
+                    b.Property<Guid>("Id")
+                        .ValueGeneratedOnAdd()
+                        .HasColumnType("uuid")
+                        .HasColumnName("id");
+
+                    b.Property<long>("AccountId")
+                        .HasColumnType("bigint")
+                        .HasColumnName("account_id");
+
+                    b.Property<Instant>("CreatedAt")
+                        .HasColumnType("timestamp with time zone")
+                        .HasColumnName("created_at");
+
+                    b.Property<Instant?>("DeletedAt")
+                        .HasColumnType("timestamp with time zone")
+                        .HasColumnName("deleted_at");
+
+                    b.Property<string>("ResourceIdentifier")
+                        .IsRequired()
+                        .HasMaxLength(4096)
+                        .HasColumnType("character varying(4096)")
+                        .HasColumnName("resource_identifier");
+
+                    b.Property<string>("Type")
+                        .IsRequired()
+                        .HasMaxLength(1024)
+                        .HasColumnType("character varying(1024)")
+                        .HasColumnName("type");
+
+                    b.Property<Instant>("UpdatedAt")
+                        .HasColumnType("timestamp with time zone")
+                        .HasColumnName("updated_at");
+
+                    b.Property<int>("Visibility")
+                        .HasColumnType("integer")
+                        .HasColumnName("visibility");
+
+                    b.HasKey("Id")
+                        .HasName("pk_activities");
+
+                    b.HasIndex("AccountId")
+                        .HasDatabaseName("ix_activities_account_id");
+
+                    b.ToTable("activities", (string)null);
+                });
+
             modelBuilder.Entity("DysonNetwork.Sphere.Auth.Challenge", b =>
                 {
                     b.Property<Guid>("Id")
@@ -1162,6 +1210,10 @@ namespace DysonNetwork.Sphere.Migrations
                         .HasColumnType("jsonb")
                         .HasColumnName("file_meta");
 
+                    b.Property<bool>("HasCompression")
+                        .HasColumnType("boolean")
+                        .HasColumnName("has_compression");
+
                     b.Property<string>("Hash")
                         .HasMaxLength(256)
                         .HasColumnType("character varying(256)")
@@ -1381,6 +1433,18 @@ namespace DysonNetwork.Sphere.Migrations
                     b.Navigation("Related");
                 });
 
+            modelBuilder.Entity("DysonNetwork.Sphere.Activity.Activity", b =>
+                {
+                    b.HasOne("DysonNetwork.Sphere.Account.Account", "Account")
+                        .WithMany()
+                        .HasForeignKey("AccountId")
+                        .OnDelete(DeleteBehavior.Cascade)
+                        .IsRequired()
+                        .HasConstraintName("fk_activities_accounts_account_id");
+
+                    b.Navigation("Account");
+                });
+
             modelBuilder.Entity("DysonNetwork.Sphere.Auth.Challenge", b =>
                 {
                     b.HasOne("DysonNetwork.Sphere.Account.Account", "Account")

DysonNetwork.Sphere/Program.cs

@@ -192,7 +192,7 @@ app.MapRazorPages().RequireRateLimiting("fixed");
 var tusDiskStore = new tusdotnet.Stores.TusDiskStore(
     builder.Configuration.GetSection("Tus").GetValue<string>("StorePath")!
 );
-app.MapTus("/files/tus", (_) => Task.FromResult<DefaultTusConfiguration>(new()
+app.MapTus("/files/tus", _ => Task.FromResult<DefaultTusConfiguration>(new()
 {
     Store = tusDiskStore,
     Events = new Events
@@ -239,28 +239,12 @@ app.MapTus("/files/tus", (_) => Task.FromResult<DefaultTusConfiguration>(new()
 
             var fileService = eventContext.HttpContext.RequestServices.GetRequiredService<FileService>();
 
-            var info = await fileService.AnalyzeFileAsync(user, file.Id, fileStream, fileName, contentType);
+            var info = await fileService.ProcessNewFileAsync(user, file.Id, fileStream, fileName, contentType);
 
             var jsonOptions = httpContext.RequestServices.GetRequiredService<IOptions<JsonOptions>>().Value
                 .JsonSerializerOptions;
             var infoJson = JsonSerializer.Serialize(info, jsonOptions);
             eventContext.HttpContext.Response.Headers.Append("X-FileInfo", infoJson);
-
-#pragma warning disable CS4014
-            Task.Run(async () =>
-            {
-                using var scope = eventContext.HttpContext.RequestServices
-                    .GetRequiredService<IServiceScopeFactory>()
-                    .CreateScope();
-                // Keep the service didn't be disposed
-                var fs = scope.ServiceProvider.GetRequiredService<FileService>();
-                // Keep the file stream opened
-                var fileData = await tusDiskStore.GetFileAsync(file.Id, CancellationToken.None);
-                var newStream = await fileData.GetContentAsync(CancellationToken.None);
-                await fs.UploadFileToRemoteAsync(info, newStream, null);
-                await tusDiskStore.DeleteFileAsync(file.Id, CancellationToken.None);
-            });
-#pragma warning restore CS4014
         },
     }
 }));

DysonNetwork.Sphere/Storage/CloudFile.cs

@@ -34,6 +34,7 @@ public class CloudFile : ModelBase
     public Instant? UploadedAt { get; set; }
     public Instant? ExpiredAt { get; set; }
     [MaxLength(128)] public string? UploadedTo { get; set; }
+    public bool HasCompression { get; set; }= false;
 
     // Metrics
     // When this used count keep zero, it means it's not used by anybody, so it can be recycled

DysonNetwork.Sphere/Storage/FileService.cs

@@ -4,6 +4,7 @@ using System.Security.Cryptography;
 using Blurhash.ImageSharp;
 using Microsoft.EntityFrameworkCore;
 using Minio;
+using Minio.DataModel;
 using Minio.DataModel.Args;
 using NodaTime;
 using Quartz;
@@ -15,9 +16,11 @@ namespace DysonNetwork.Sphere.Storage;
 
 public class FileService(AppDatabase db, IConfiguration configuration)
 {
+    private static readonly string TempFilePrefix = "dyn-cloudfile";
+
     // The analysis file method no longer will remove the GPS EXIF data
     // It should be handled on the client side, and for some specific cases it should be keep
-    public async Task<CloudFile> AnalyzeFileAsync(
+    public async Task<CloudFile> ProcessNewFileAsync(
         Account.Account account,
         string fileId,
         Stream stream,
@@ -25,6 +28,10 @@ public class FileService(AppDatabase db, IConfiguration configuration)
         string? contentType
     )
     {
+        // If this variable present a value means the processor modified the uploaded file
+        // Upload this file to the remote instead
+        var modifiedResult = new List<(string filePath, string suffix)>();
+
         var fileSize = stream.Length;
         var hash = await HashFileAsync(stream, fileSize: fileSize);
         contentType ??= !fileName.Contains('.') ? "application/octet-stream" : MimeTypes.GetMimeType(fileName);
@@ -76,6 +83,24 @@ public class FileService(AppDatabase db, IConfiguration configuration)
                         ["ratio"] = aspectRatio,
                         ["exif"] = exif
                     };
+
+                    var imagePath = Path.Join(Path.GetTempPath(), $"{TempFilePrefix}#{file.Id}");
+                    var ogTask = imageSharp.SaveAsWebpAsync(imagePath);
+                    modifiedResult.Add((imagePath, string.Empty));
+
+                    var compressedClone = imageSharp.Clone();
+                    compressedClone.Mutate(i => i.Resize(new ResizeOptions
+                        {
+                            Mode = ResizeMode.Max,
+                            Size = new Size(1024, 1024),
+                        })
+                    );
+                    var imageCompressedPath = Path.Join(Path.GetTempPath(), $"{TempFilePrefix}#{file.Id}-compressed");
+                    var compressedTask = compressedClone.SaveAsWebpAsync(imagePath);
+                    modifiedResult.Add((imageCompressedPath, ".compressed"));
+                    file.HasCompression = true;
+
+                    await Task.WhenAll(ogTask, compressedTask);
                 }
 
                 break;
@@ -105,6 +130,15 @@ public class FileService(AppDatabase db, IConfiguration configuration)
 
         db.Files.Add(file);
         await db.SaveChangesAsync();
+
+#pragma warning disable CS4014
+        if (modifiedResult.Count > 0)
+            foreach (var result in modifiedResult)
+                UploadFileToRemoteAsync(file, result.filePath, null, result.suffix, true);
+        else
+            UploadFileToRemoteAsync(file, stream, null);
+#pragma warning restore CS4014
+
         return file;
     }
 
@@ -141,7 +175,17 @@ public class FileService(AppDatabase db, IConfiguration configuration)
         return Convert.ToHexString(hash).ToLowerInvariant();
     }
 
-    public async Task<CloudFile> UploadFileToRemoteAsync(CloudFile file, Stream stream, string? targetRemote)
+    public async Task<CloudFile> UploadFileToRemoteAsync(CloudFile file, string filePath, string? targetRemote,
+        string? suffix = null, bool selfDestruct = false)
+    {
+        var fileStream = File.OpenRead(filePath);
+        var result = await UploadFileToRemoteAsync(file, fileStream, targetRemote);
+        if (selfDestruct) File.Delete(filePath);
+        return result;
+    }
+
+    public async Task<CloudFile> UploadFileToRemoteAsync(CloudFile file, Stream stream, string? targetRemote,
+        string? suffix = null)
     {
         if (file.UploadedAt.HasValue) return file;
 
@@ -159,7 +203,7 @@ public class FileService(AppDatabase db, IConfiguration configuration)
 
         await client.PutObjectAsync(new PutObjectArgs()
             .WithBucket(bucket)
-            .WithObject(file.Id)
+            .WithObject(string.IsNullOrWhiteSpace(suffix) ? file.Id : file.Id + suffix)
             .WithStreamData(stream)
             .WithObjectSize(stream.Length)
             .WithContentType(contentType)

DysonNetwork.sln.DotSettings.user

@@ -30,6 +30,7 @@
 	<s:String x:Key="/Default/CodeInspection/ExcludedFiles/FilesAndFoldersToSkip2/=7020124F_002D9FFC_002D4AC3_002D8F3D_002DAAB8E0240759_002Ff_003AITusStore_002Ecs_002Fl_003A_002E_002E_003F_002E_002E_003F_002E_002E_003FLibrary_003FApplication_0020Support_003FJetBrains_003FRider2024_002E3_003Fresharper_002Dhost_003FDecompilerCache_003Fdecompiler_003F8bb08a178b5b43c5bac20a5a54159a5b2a800_003Fb1_003F7e861de5_003FITusStore_002Ecs/@EntryIndexedValue">ForceIncluded</s:String>
 	<s:String x:Key="/Default/CodeInspection/ExcludedFiles/FilesAndFoldersToSkip2/=7020124F_002D9FFC_002D4AC3_002D8F3D_002DAAB8E0240759_002Ff_003AJsonSerializerOptions_002Ecs_002Fl_003A_002E_002E_003F_002E_002E_003F_002E_002E_003FLibrary_003FApplication_0020Support_003FJetBrains_003FRider2025_002E1_003Fresharper_002Dhost_003FDecompilerCache_003Fdecompiler_003F5703920a18f94462b4354fab05326e6519a200_003F35_003F8536fc49_003FJsonSerializerOptions_002Ecs/@EntryIndexedValue">ForceIncluded</s:String>
 	<s:String x:Key="/Default/CodeInspection/ExcludedFiles/FilesAndFoldersToSkip2/=7020124F_002D9FFC_002D4AC3_002D8F3D_002DAAB8E0240759_002Ff_003AKestrelServerLimits_002Ecs_002Fl_003A_002E_002E_003F_002E_002E_003F_002E_002E_003FLibrary_003FApplication_0020Support_003FJetBrains_003FRider2025_002E1_003Fresharper_002Dhost_003FDecompilerCache_003Fdecompiler_003F1e2e5dfcafad4407b569dd5df56a2fbf274e00_003Fa4_003F39445f62_003FKestrelServerLimits_002Ecs/@EntryIndexedValue">ForceIncluded</s:String>
+	<s:String x:Key="/Default/CodeInspection/ExcludedFiles/FilesAndFoldersToSkip2/=7020124F_002D9FFC_002D4AC3_002D8F3D_002DAAB8E0240759_002Ff_003AKnownResamplers_002Ecs_002Fl_003A_002E_002E_003F_002E_002E_003F_002E_002E_003FLibrary_003FApplication_0020Support_003FJetBrains_003FRider2025_002E1_003Fresharper_002Dhost_003FDecompilerCache_003Fdecompiler_003Fef3339e864a448e2b1ec6fa7bbf4c6661fee00_003Fb3_003Fcdb3e080_003FKnownResamplers_002Ecs/@EntryIndexedValue">ForceIncluded</s:String>
 	<s:String x:Key="/Default/CodeInspection/ExcludedFiles/FilesAndFoldersToSkip2/=7020124F_002D9FFC_002D4AC3_002D8F3D_002DAAB8E0240759_002Ff_003AMailboxAddress_002Ecs_002Fl_003A_002E_002E_003F_002E_002E_003F_002E_002E_003FLibrary_003FApplication_0020Support_003FJetBrains_003FRider2025_002E1_003Fresharper_002Dhost_003FDecompilerCache_003Fdecompiler_003F8e03e47c46b7469f97abc40667cbcf9b133000_003Fa6_003F83324248_003FMailboxAddress_002Ecs/@EntryIndexedValue">ForceIncluded</s:String>
 	<s:String x:Key="/Default/CodeInspection/ExcludedFiles/FilesAndFoldersToSkip2/=7020124F_002D9FFC_002D4AC3_002D8F3D_002DAAB8E0240759_002Ff_003AMediaAnalysis_002Ecs_002Fl_003A_002E_002E_003F_002E_002E_003F_002E_002E_003FLibrary_003FApplication_0020Support_003FJetBrains_003FRider2024_002E3_003Fresharper_002Dhost_003FDecompilerCache_003Fdecompiler_003Ffef366b36a224d469ff150d30f9a866d23c00_003Fd7_003F5c138865_003FMediaAnalysis_002Ecs/@EntryIndexedValue">ForceIncluded</s:String>
 	<s:String x:Key="/Default/CodeInspection/ExcludedFiles/FilesAndFoldersToSkip2/=7020124F_002D9FFC_002D4AC3_002D8F3D_002DAAB8E0240759_002Ff_003AMicrosoftDependencyInjectionJobFactory_002Ecs_002Fl_003A_002E_002E_003F_002E_002E_003F_002E_002E_003FLibrary_003FApplication_0020Support_003FJetBrains_003FRider2024_002E3_003Fresharper_002Dhost_003FDecompilerCache_003Fdecompiler_003F1edbd6e24d7b430fabce72177269baa19200_003Fa8_003F91b091de_003FMicrosoftDependencyInjectionJobFactory_002Ecs/@EntryIndexedValue">ForceIncluded</s:String>
@@ -40,6 +41,7 @@
 	<s:String x:Key="/Default/CodeInspection/ExcludedFiles/FilesAndFoldersToSkip2/=7020124F_002D9FFC_002D4AC3_002D8F3D_002DAAB8E0240759_002Ff_003APath_002Ecs_002Fl_003A_002E_002E_003F_002E_002E_003F_002E_002E_003FLibrary_003FApplication_0020Support_003FJetBrains_003FRider2024_002E3_003Fresharper_002Dhost_003FDecompilerCache_003Fdecompiler_003Fb6f0571a6bc744b0b551fd4578292582e54c00_003Fd3_003F7b05b2bd_003FPath_002Ecs/@EntryIndexedValue">ForceIncluded</s:String>
 	<s:String x:Key="/Default/CodeInspection/ExcludedFiles/FilesAndFoldersToSkip2/=7020124F_002D9FFC_002D4AC3_002D8F3D_002DAAB8E0240759_002Ff_003APresignedGetObjectArgs_002Ecs_002Fl_003A_002E_002E_003F_002E_002E_003F_002E_002E_003FLibrary_003FApplication_0020Support_003FJetBrains_003FRider2024_002E3_003Fresharper_002Dhost_003FSourcesCache_003F0df26a9d89e29319e9efcaea0a8489db9e97bc1aedcca3f7e360cc50f8f4ea_003FPresignedGetObjectArgs_002Ecs/@EntryIndexedValue">ForceIncluded</s:String>
 	<s:String x:Key="/Default/CodeInspection/ExcludedFiles/FilesAndFoldersToSkip2/=7020124F_002D9FFC_002D4AC3_002D8F3D_002DAAB8E0240759_002Ff_003AQueryable_002Ecs_002Fl_003A_002E_002E_003F_002E_002E_003F_002E_002E_003FLibrary_003FApplication_0020Support_003FJetBrains_003FRider2025_002E1_003Fresharper_002Dhost_003FDecompilerCache_003Fdecompiler_003F42d8f09d6a294d00a6f49efc989927492fe00_003F4e_003F26d1ee34_003FQueryable_002Ecs/@EntryIndexedValue">ForceIncluded</s:String>
+	<s:String x:Key="/Default/CodeInspection/ExcludedFiles/FilesAndFoldersToSkip2/=7020124F_002D9FFC_002D4AC3_002D8F3D_002DAAB8E0240759_002Ff_003AResizeOptions_002Ecs_002Fl_003A_002E_002E_003F_002E_002E_003F_002E_002E_003FLibrary_003FApplication_0020Support_003FJetBrains_003FRider2025_002E1_003Fresharper_002Dhost_003FDecompilerCache_003Fdecompiler_003Fef3339e864a448e2b1ec6fa7bbf4c6661fee00_003F48_003F0209e410_003FResizeOptions_002Ecs/@EntryIndexedValue">ForceIncluded</s:String>
 	<s:String x:Key="/Default/CodeInspection/ExcludedFiles/FilesAndFoldersToSkip2/=7020124F_002D9FFC_002D4AC3_002D8F3D_002DAAB8E0240759_002Ff_003ASecuritySchemeType_002Ecs_002Fl_003A_002E_002E_003F_002E_002E_003F_002E_002E_003FLibrary_003FApplication_0020Support_003FJetBrains_003FRider2024_002E3_003Fresharper_002Dhost_003FSourcesCache_003F29898ce74e3763a786ac1bd9a6db2152e1af75769440b1e53b9cbdf1dda1bd99_003FSecuritySchemeType_002Ecs/@EntryIndexedValue">ForceIncluded</s:String>
 	<s:String x:Key="/Default/CodeInspection/ExcludedFiles/FilesAndFoldersToSkip2/=7020124F_002D9FFC_002D4AC3_002D8F3D_002DAAB8E0240759_002Ff_003AServiceCollectionContainerBuilderExtensions_002Ecs_002Fl_003A_002E_002E_003F_002E_002E_003F_002E_002E_003FLibrary_003FApplication_0020Support_003FJetBrains_003FRider2024_002E3_003Fresharper_002Dhost_003FDecompilerCache_003Fdecompiler_003Fc0e30e11d8f5456cb7a11b21ebee6c5a35c00_003F60_003F78b485f5_003FServiceCollectionContainerBuilderExtensions_002Ecs/@EntryIndexedValue">ForceIncluded</s:String>
 	<s:String x:Key="/Default/CodeInspection/ExcludedFiles/FilesAndFoldersToSkip2/=7020124F_002D9FFC_002D4AC3_002D8F3D_002DAAB8E0240759_002Ff_003ASetPropertyCalls_002Ecs_002Fl_003A_002E_002E_003F_002E_002E_003F_002E_002E_003FLibrary_003FApplication_0020Support_003FJetBrains_003FRider2024_002E3_003Fresharper_002Dhost_003FSourcesCache_003F458b5f22476b4599b87176214d5e4026c2327b148f4d3f885ee92362b4dac3_003FSetPropertyCalls_002Ecs/@EntryIndexedValue">ForceIncluded</s:String>