DysonNetwork/Swarm
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

✨ Compressed duplication of image files

Browse Source
This commit is contained in:
LittleSheep 2025-05-01 15:29:52 +08:00
parent bf64afd849
commit 0f9e865c0b
7 changed files with 1909 additions and 21 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1730
DysonNetwork.Sphere/Migrations/20250501072931_UnableToNamingThing1.Designer.cs generated Normal file
View File

File diff suppressed because it is too large Load Diff

63
DysonNetwork.Sphere/Migrations/20250501072931_UnableToNamingThing1.cs Normal file
View File

@ -0,0 +1,63 @@
using System;
using Microsoft.EntityFrameworkCore.Migrations;
using NodaTime;
#nullable disable
namespace DysonNetwork.Sphere.Migrations
{
/// <inheritdoc />
public partial class UnableToNamingThing1 : Migration
{
/// <inheritdoc />
protected override void Up(MigrationBuilder migrationBuilder)
{
migrationBuilder.AddColumn<bool>(
name: "has_compression",
table: "files",
type: "boolean",
nullable: false,
defaultValue: false);
migrationBuilder.CreateTable(
name: "activities",
columns: table => new
{
id = table.Column<Guid>(type: "uuid", nullable: false),
type = table.Column<string>(type: "character varying(1024)", maxLength: 1024, nullable: false),
resource_identifier = table.Column<string>(type: "character varying(4096)", maxLength: 4096, nullable: false),
visibility = table.Column<int>(type: "integer", nullable: false),
account_id = table.Column<long>(type: "bigint", nullable: false),
created_at = table.Column<Instant>(type: "timestamp with time zone", nullable: false),
updated_at = table.Column<Instant>(type: "timestamp with time zone", nullable: false),
deleted_at = table.Column<Instant>(type: "timestamp with time zone", nullable: true)
},
constraints: table =>
{
table.PrimaryKey("pk_activities", x => x.id);
table.ForeignKey(
name: "fk_activities_accounts_account_id",
column: x => x.account_id,
principalTable: "accounts",
principalColumn: "id",
onDelete: ReferentialAction.Cascade);
});
migrationBuilder.CreateIndex(
name: "ix_activities_account_id",
table: "activities",
column: "account_id");
}
/// <inheritdoc />
protected override void Down(MigrationBuilder migrationBuilder)
{
migrationBuilder.DropTable(
name: "activities");
migrationBuilder.DropColumn(
name: "has_compression",
table: "files");
}
}
}

64
DysonNetwork.Sphere/Migrations/AppDatabaseModelSnapshot.cs
View File

@ -453,6 +453,54 @@ namespace DysonNetwork.Sphere.Migrations
b.ToTable("account_relationships", (string)null);
});
modelBuilder.Entity("DysonNetwork.Sphere.Activity.Activity", b =>
{
b.Property<Guid>("Id")
.ValueGeneratedOnAdd()
.HasColumnType("uuid")
.HasColumnName("id");
b.Property<long>("AccountId")
.HasColumnType("bigint")
.HasColumnName("account_id");
b.Property<Instant>("CreatedAt")
.HasColumnType("timestamp with time zone")
.HasColumnName("created_at");
b.Property<Instant?>("DeletedAt")
.HasColumnType("timestamp with time zone")
.HasColumnName("deleted_at");
b.Property<string>("ResourceIdentifier")
.IsRequired()
.HasMaxLength(4096)
.HasColumnType("character varying(4096)")
.HasColumnName("resource_identifier");
b.Property<string>("Type")
.IsRequired()
.HasMaxLength(1024)
.HasColumnType("character varying(1024)")
.HasColumnName("type");
b.Property<Instant>("UpdatedAt")
.HasColumnType("timestamp with time zone")
.HasColumnName("updated_at");
b.Property<int>("Visibility")
.HasColumnType("integer")
.HasColumnName("visibility");
b.HasKey("Id")
.HasName("pk_activities");
b.HasIndex("AccountId")
.HasDatabaseName("ix_activities_account_id");
b.ToTable("activities", (string)null);
});
modelBuilder.Entity("DysonNetwork.Sphere.Auth.Challenge", b =>
{
b.Property<Guid>("Id")
@ -1162,6 +1210,10 @@ namespace DysonNetwork.Sphere.Migrations
.HasColumnType("jsonb")
.HasColumnName("file_meta");
b.Property<bool>("HasCompression")
.HasColumnType("boolean")
.HasColumnName("has_compression");
b.Property<string>("Hash")
.HasMaxLength(256)
.HasColumnType("character varying(256)")
@ -1381,6 +1433,18 @@ namespace DysonNetwork.Sphere.Migrations
b.Navigation("Related");
});
modelBuilder.Entity("DysonNetwork.Sphere.Activity.Activity", b =>
{
b.HasOne("DysonNetwork.Sphere.Account.Account", "Account")
.WithMany()
.HasForeignKey("AccountId")
.OnDelete(DeleteBehavior.Cascade)
.IsRequired()
.HasConstraintName("fk_activities_accounts_account_id");
b.Navigation("Account");
});
modelBuilder.Entity("DysonNetwork.Sphere.Auth.Challenge", b =>
{
b.HasOne("DysonNetwork.Sphere.Account.Account", "Account")

20
DysonNetwork.Sphere/Program.cs
View File

@ -192,7 +192,7 @@ app.MapRazorPages().RequireRateLimiting("fixed");
var tusDiskStore = new tusdotnet.Stores.TusDiskStore(
builder.Configuration.GetSection("Tus").GetValue<string>("StorePath")!
);
app.MapTus("/files/tus", (_) => Task.FromResult<DefaultTusConfiguration>(new()
app.MapTus("/files/tus", _ => Task.FromResult<DefaultTusConfiguration>(new()
{
Store = tusDiskStore,
Events = new Events
@ -239,28 +239,12 @@ app.MapTus("/files/tus", (_) => Task.FromResult<DefaultTusConfiguration>(new()
var fileService = eventContext.HttpContext.RequestServices.GetRequiredService<FileService>();
var info = await fileService.AnalyzeFileAsync(user, file.Id, fileStream, fileName, contentType);
var info = await fileService.ProcessNewFileAsync(user, file.Id, fileStream, fileName, contentType);
var jsonOptions = httpContext.RequestServices.GetRequiredService<IOptions<JsonOptions>>().Value
.JsonSerializerOptions;
var infoJson = JsonSerializer.Serialize(info, jsonOptions);
eventContext.HttpContext.Response.Headers.Append("X-FileInfo", infoJson);
#pragma warning disable CS4014
Task.Run(async () =>
{
using var scope = eventContext.HttpContext.RequestServices
.GetRequiredService<IServiceScopeFactory>()
.CreateScope();
// Keep the service didn't be disposed
var fs = scope.ServiceProvider.GetRequiredService<FileService>();
// Keep the file stream opened
var fileData = await tusDiskStore.GetFileAsync(file.Id, CancellationToken.None);
var newStream = await fileData.GetContentAsync(CancellationToken.None);
await fs.UploadFileToRemoteAsync(info, newStream, null);
await tusDiskStore.DeleteFileAsync(file.Id, CancellationToken.None);
});
#pragma warning restore CS4014
},
}
}));

1
DysonNetwork.Sphere/Storage/CloudFile.cs
View File

@ -34,6 +34,7 @@ public class CloudFile : ModelBase
public Instant? UploadedAt { get; set; }
public Instant? ExpiredAt { get; set; }
[MaxLength(128)] public string? UploadedTo { get; set; }
public bool HasCompression { get; set; }= false;
// Metrics
// When this used count keep zero, it means it's not used by anybody, so it can be recycled

50
DysonNetwork.Sphere/Storage/FileService.cs
View File

@ -4,6 +4,7 @@ using System.Security.Cryptography;
using Blurhash.ImageSharp;
using Microsoft.EntityFrameworkCore;
using Minio;
using Minio.DataModel;
using Minio.DataModel.Args;
using NodaTime;
using Quartz;
@ -15,9 +16,11 @@ namespace DysonNetwork.Sphere.Storage;
public class FileService(AppDatabase db, IConfiguration configuration)
{
private static readonly string TempFilePrefix = "dyn-cloudfile";
// The analysis file method no longer will remove the GPS EXIF data
// It should be handled on the client side, and for some specific cases it should be keep
public async Task<CloudFile> AnalyzeFileAsync(
public async Task<CloudFile> ProcessNewFileAsync(
Account.Account account,
string fileId,
Stream stream,
@ -25,6 +28,10 @@ public class FileService(AppDatabase db, IConfiguration configuration)
string? contentType
)
{
// If this variable present a value means the processor modified the uploaded file
// Upload this file to the remote instead
var modifiedResult = new List<(string filePath, string suffix)>();
var fileSize = stream.Length;
var hash = await HashFileAsync(stream, fileSize: fileSize);
contentType ??= !fileName.Contains('.') ? "application/octet-stream" : MimeTypes.GetMimeType(fileName);
@ -76,6 +83,24 @@ public class FileService(AppDatabase db, IConfiguration configuration)
["ratio"] = aspectRatio,
["exif"] = exif
};
var imagePath = Path.Join(Path.GetTempPath(), $"{TempFilePrefix}#{file.Id}");
var ogTask = imageSharp.SaveAsWebpAsync(imagePath);
modifiedResult.Add((imagePath, string.Empty));
var compressedClone = imageSharp.Clone();
compressedClone.Mutate(i => i.Resize(new ResizeOptions
{
Mode = ResizeMode.Max,
Size = new Size(1024, 1024),
})
);
var imageCompressedPath = Path.Join(Path.GetTempPath(), $"{TempFilePrefix}#{file.Id}-compressed");
var compressedTask = compressedClone.SaveAsWebpAsync(imagePath);
modifiedResult.Add((imageCompressedPath, ".compressed"));
file.HasCompression = true;
await Task.WhenAll(ogTask, compressedTask);
}
break;
@ -105,6 +130,15 @@ public class FileService(AppDatabase db, IConfiguration configuration)
db.Files.Add(file);
await db.SaveChangesAsync();
#pragma warning disable CS4014
if (modifiedResult.Count > 0)
foreach (var result in modifiedResult)
UploadFileToRemoteAsync(file, result.filePath, null, result.suffix, true);
else
UploadFileToRemoteAsync(file, stream, null);
#pragma warning restore CS4014
return file;
}
@ -141,7 +175,17 @@ public class FileService(AppDatabase db, IConfiguration configuration)
return Convert.ToHexString(hash).ToLowerInvariant();
}
public async Task<CloudFile> UploadFileToRemoteAsync(CloudFile file, Stream stream, string? targetRemote)
public async Task<CloudFile> UploadFileToRemoteAsync(CloudFile file, string filePath, string? targetRemote,
string? suffix = null, bool selfDestruct = false)
{
var fileStream = File.OpenRead(filePath);
var result = await UploadFileToRemoteAsync(file, fileStream, targetRemote);
if (selfDestruct) File.Delete(filePath);
return result;
}
public async Task<CloudFile> UploadFileToRemoteAsync(CloudFile file, Stream stream, string? targetRemote,
string? suffix = null)
{
if (file.UploadedAt.HasValue) return file;
@ -159,7 +203,7 @@ public class FileService(AppDatabase db, IConfiguration configuration)
await client.PutObjectAsync(new PutObjectArgs()
.WithBucket(bucket)
.WithObject(file.Id)
.WithObject(string.IsNullOrWhiteSpace(suffix) ? file.Id : file.Id + suffix)
.WithStreamData(stream)
.WithObjectSize(stream.Length)
.WithContentType(contentType)

2
DysonNetwork.sln.DotSettings.user
View File

@ -30,6 +30,7 @@
<s:String x:Key="/Default/CodeInspection/ExcludedFiles/FilesAndFoldersToSkip2/=7020124F_002D9FFC_002D4AC3_002D8F3D_002DAAB8E0240759_002Ff_003AITusStore_002Ecs_002Fl_003A_002E_002E_003F_002E_002E_003F_002E_002E_003FLibrary_003FApplication_0020Support_003FJetBrains_003FRider2024_002E3_003Fresharper_002Dhost_003FDecompilerCache_003Fdecompiler_003F8bb08a178b5b43c5bac20a5a54159a5b2a800_003Fb1_003F7e861de5_003FITusStore_002Ecs/@EntryIndexedValue">ForceIncluded</s:String>
<s:String x:Key="/Default/CodeInspection/ExcludedFiles/FilesAndFoldersToSkip2/=7020124F_002D9FFC_002D4AC3_002D8F3D_002DAAB8E0240759_002Ff_003AJsonSerializerOptions_002Ecs_002Fl_003A_002E_002E_003F_002E_002E_003F_002E_002E_003FLibrary_003FApplication_0020Support_003FJetBrains_003FRider2025_002E1_003Fresharper_002Dhost_003FDecompilerCache_003Fdecompiler_003F5703920a18f94462b4354fab05326e6519a200_003F35_003F8536fc49_003FJsonSerializerOptions_002Ecs/@EntryIndexedValue">ForceIncluded</s:String>
<s:String x:Key="/Default/CodeInspection/ExcludedFiles/FilesAndFoldersToSkip2/=7020124F_002D9FFC_002D4AC3_002D8F3D_002DAAB8E0240759_002Ff_003AKestrelServerLimits_002Ecs_002Fl_003A_002E_002E_003F_002E_002E_003F_002E_002E_003FLibrary_003FApplication_0020Support_003FJetBrains_003FRider2025_002E1_003Fresharper_002Dhost_003FDecompilerCache_003Fdecompiler_003F1e2e5dfcafad4407b569dd5df56a2fbf274e00_003Fa4_003F39445f62_003FKestrelServerLimits_002Ecs/@EntryIndexedValue">ForceIncluded</s:String>
<s:String x:Key="/Default/CodeInspection/ExcludedFiles/FilesAndFoldersToSkip2/=7020124F_002D9FFC_002D4AC3_002D8F3D_002DAAB8E0240759_002Ff_003AKnownResamplers_002Ecs_002Fl_003A_002E_002E_003F_002E_002E_003F_002E_002E_003FLibrary_003FApplication_0020Support_003FJetBrains_003FRider2025_002E1_003Fresharper_002Dhost_003FDecompilerCache_003Fdecompiler_003Fef3339e864a448e2b1ec6fa7bbf4c6661fee00_003Fb3_003Fcdb3e080_003FKnownResamplers_002Ecs/@EntryIndexedValue">ForceIncluded</s:String>
<s:String x:Key="/Default/CodeInspection/ExcludedFiles/FilesAndFoldersToSkip2/=7020124F_002D9FFC_002D4AC3_002D8F3D_002DAAB8E0240759_002Ff_003AMailboxAddress_002Ecs_002Fl_003A_002E_002E_003F_002E_002E_003F_002E_002E_003FLibrary_003FApplication_0020Support_003FJetBrains_003FRider2025_002E1_003Fresharper_002Dhost_003FDecompilerCache_003Fdecompiler_003F8e03e47c46b7469f97abc40667cbcf9b133000_003Fa6_003F83324248_003FMailboxAddress_002Ecs/@EntryIndexedValue">ForceIncluded</s:String>
<s:String x:Key="/Default/CodeInspection/ExcludedFiles/FilesAndFoldersToSkip2/=7020124F_002D9FFC_002D4AC3_002D8F3D_002DAAB8E0240759_002Ff_003AMediaAnalysis_002Ecs_002Fl_003A_002E_002E_003F_002E_002E_003F_002E_002E_003FLibrary_003FApplication_0020Support_003FJetBrains_003FRider2024_002E3_003Fresharper_002Dhost_003FDecompilerCache_003Fdecompiler_003Ffef366b36a224d469ff150d30f9a866d23c00_003Fd7_003F5c138865_003FMediaAnalysis_002Ecs/@EntryIndexedValue">ForceIncluded</s:String>
<s:String x:Key="/Default/CodeInspection/ExcludedFiles/FilesAndFoldersToSkip2/=7020124F_002D9FFC_002D4AC3_002D8F3D_002DAAB8E0240759_002Ff_003AMicrosoftDependencyInjectionJobFactory_002Ecs_002Fl_003A_002E_002E_003F_002E_002E_003F_002E_002E_003FLibrary_003FApplication_0020Support_003FJetBrains_003FRider2024_002E3_003Fresharper_002Dhost_003FDecompilerCache_003Fdecompiler_003F1edbd6e24d7b430fabce72177269baa19200_003Fa8_003F91b091de_003FMicrosoftDependencyInjectionJobFactory_002Ecs/@EntryIndexedValue">ForceIncluded</s:String>
@ -40,6 +41,7 @@
<s:String x:Key="/Default/CodeInspection/ExcludedFiles/FilesAndFoldersToSkip2/=7020124F_002D9FFC_002D4AC3_002D8F3D_002DAAB8E0240759_002Ff_003APath_002Ecs_002Fl_003A_002E_002E_003F_002E_002E_003F_002E_002E_003FLibrary_003FApplication_0020Support_003FJetBrains_003FRider2024_002E3_003Fresharper_002Dhost_003FDecompilerCache_003Fdecompiler_003Fb6f0571a6bc744b0b551fd4578292582e54c00_003Fd3_003F7b05b2bd_003FPath_002Ecs/@EntryIndexedValue">ForceIncluded</s:String>
<s:String x:Key="/Default/CodeInspection/ExcludedFiles/FilesAndFoldersToSkip2/=7020124F_002D9FFC_002D4AC3_002D8F3D_002DAAB8E0240759_002Ff_003APresignedGetObjectArgs_002Ecs_002Fl_003A_002E_002E_003F_002E_002E_003F_002E_002E_003FLibrary_003FApplication_0020Support_003FJetBrains_003FRider2024_002E3_003Fresharper_002Dhost_003FSourcesCache_003F0df26a9d89e29319e9efcaea0a8489db9e97bc1aedcca3f7e360cc50f8f4ea_003FPresignedGetObjectArgs_002Ecs/@EntryIndexedValue">ForceIncluded</s:String>
<s:String x:Key="/Default/CodeInspection/ExcludedFiles/FilesAndFoldersToSkip2/=7020124F_002D9FFC_002D4AC3_002D8F3D_002DAAB8E0240759_002Ff_003AQueryable_002Ecs_002Fl_003A_002E_002E_003F_002E_002E_003F_002E_002E_003FLibrary_003FApplication_0020Support_003FJetBrains_003FRider2025_002E1_003Fresharper_002Dhost_003FDecompilerCache_003Fdecompiler_003F42d8f09d6a294d00a6f49efc989927492fe00_003F4e_003F26d1ee34_003FQueryable_002Ecs/@EntryIndexedValue">ForceIncluded</s:String>
<s:String x:Key="/Default/CodeInspection/ExcludedFiles/FilesAndFoldersToSkip2/=7020124F_002D9FFC_002D4AC3_002D8F3D_002DAAB8E0240759_002Ff_003AResizeOptions_002Ecs_002Fl_003A_002E_002E_003F_002E_002E_003F_002E_002E_003FLibrary_003FApplication_0020Support_003FJetBrains_003FRider2025_002E1_003Fresharper_002Dhost_003FDecompilerCache_003Fdecompiler_003Fef3339e864a448e2b1ec6fa7bbf4c6661fee00_003F48_003F0209e410_003FResizeOptions_002Ecs/@EntryIndexedValue">ForceIncluded</s:String>
<s:String x:Key="/Default/CodeInspection/ExcludedFiles/FilesAndFoldersToSkip2/=7020124F_002D9FFC_002D4AC3_002D8F3D_002DAAB8E0240759_002Ff_003ASecuritySchemeType_002Ecs_002Fl_003A_002E_002E_003F_002E_002E_003F_002E_002E_003FLibrary_003FApplication_0020Support_003FJetBrains_003FRider2024_002E3_003Fresharper_002Dhost_003FSourcesCache_003F29898ce74e3763a786ac1bd9a6db2152e1af75769440b1e53b9cbdf1dda1bd99_003FSecuritySchemeType_002Ecs/@EntryIndexedValue">ForceIncluded</s:String>
<s:String x:Key="/Default/CodeInspection/ExcludedFiles/FilesAndFoldersToSkip2/=7020124F_002D9FFC_002D4AC3_002D8F3D_002DAAB8E0240759_002Ff_003AServiceCollectionContainerBuilderExtensions_002Ecs_002Fl_003A_002E_002E_003F_002E_002E_003F_002E_002E_003FLibrary_003FApplication_0020Support_003FJetBrains_003FRider2024_002E3_003Fresharper_002Dhost_003FDecompilerCache_003Fdecompiler_003Fc0e30e11d8f5456cb7a11b21ebee6c5a35c00_003F60_003F78b485f5_003FServiceCollectionContainerBuilderExtensions_002Ecs/@EntryIndexedValue">ForceIncluded</s:String>
<s:String x:Key="/Default/CodeInspection/ExcludedFiles/FilesAndFoldersToSkip2/=7020124F_002D9FFC_002D4AC3_002D8F3D_002DAAB8E0240759_002Ff_003ASetPropertyCalls_002Ecs_002Fl_003A_002E_002E_003F_002E_002E_003F_002E_002E_003FLibrary_003FApplication_0020Support_003FJetBrains_003FRider2024_002E3_003Fresharper_002Dhost_003FSourcesCache_003F458b5f22476b4599b87176214d5e4026c2327b148f4d3f885ee92362b4dac3_003FSetPropertyCalls_002Ecs/@EntryIndexedValue">ForceIncluded</s:String>