From d22a15c42d5948681ce8f83a50526dc966a15cbb Mon Sep 17 00:00:00 2001 From: LittleSheep Date: Sun, 13 Apr 2025 13:50:30 +0800 Subject: [PATCH] :sparkles: File uploading --- DysonNetwork.Sphere/.gitignore | 3 +- DysonNetwork.Sphere/AppDatabase.cs | 1 + .../DysonNetwork.Sphere.csproj | 11 + .../20250412182922_AddCloudFiles.Designer.cs | 454 ++++++++++++++++++ .../20250412182922_AddCloudFiles.cs | 59 +++ .../Migrations/AppDatabaseModelSnapshot.cs | 89 ++++ DysonNetwork.Sphere/Program.cs | 71 ++- DysonNetwork.Sphere/Storage/CloudFile.cs | 59 +++ DysonNetwork.Sphere/Storage/FileService.cs | 142 ++++++ DysonNetwork.Sphere/appsettings.json | 20 + DysonNetwork.sln.DotSettings.user | 12 +- 11 files changed, 918 insertions(+), 3 deletions(-) create mode 100644 DysonNetwork.Sphere/Migrations/20250412182922_AddCloudFiles.Designer.cs create mode 100644 DysonNetwork.Sphere/Migrations/20250412182922_AddCloudFiles.cs create mode 100644 DysonNetwork.Sphere/Storage/CloudFile.cs create mode 100644 DysonNetwork.Sphere/Storage/FileService.cs diff --git a/DysonNetwork.Sphere/.gitignore b/DysonNetwork.Sphere/.gitignore index fa31d04..fd7ee1f 100644 --- a/DysonNetwork.Sphere/.gitignore +++ b/DysonNetwork.Sphere/.gitignore @@ -1 +1,2 @@ -Keys \ No newline at end of file +Keys +Uploads \ No newline at end of file diff --git a/DysonNetwork.Sphere/AppDatabase.cs b/DysonNetwork.Sphere/AppDatabase.cs index 7a23503..abbbc85 100644 --- a/DysonNetwork.Sphere/AppDatabase.cs +++ b/DysonNetwork.Sphere/AppDatabase.cs @@ -22,6 +22,7 @@ public class AppDatabase( public DbSet AccountAuthFactors { get; set; } public DbSet AuthSessions { get; set; } public DbSet AuthChallenges { get; set; } + public DbSet Files { get; set; } protected override void OnConfiguring(DbContextOptionsBuilder optionsBuilder) { diff --git a/DysonNetwork.Sphere/DysonNetwork.Sphere.csproj b/DysonNetwork.Sphere/DysonNetwork.Sphere.csproj index bacff2e..0faf296 100644 --- a/DysonNetwork.Sphere/DysonNetwork.Sphere.csproj +++ b/DysonNetwork.Sphere/DysonNetwork.Sphere.csproj @@ -13,6 +13,7 @@ + @@ -20,6 +21,11 @@ runtime; build; native; contentfiles; analyzers; buildtransitive + + all + runtime; build; native; contentfiles; analyzers; buildtransitive + + @@ -27,6 +33,7 @@ + @@ -35,4 +42,8 @@ + + + + diff --git a/DysonNetwork.Sphere/Migrations/20250412182922_AddCloudFiles.Designer.cs b/DysonNetwork.Sphere/Migrations/20250412182922_AddCloudFiles.Designer.cs new file mode 100644 index 0000000..1b94d9c --- /dev/null +++ b/DysonNetwork.Sphere/Migrations/20250412182922_AddCloudFiles.Designer.cs @@ -0,0 +1,454 @@ +// +using System; +using System.Collections.Generic; +using DysonNetwork.Sphere; +using Microsoft.EntityFrameworkCore; +using Microsoft.EntityFrameworkCore.Infrastructure; +using Microsoft.EntityFrameworkCore.Migrations; +using Microsoft.EntityFrameworkCore.Storage.ValueConversion; +using NodaTime; +using Npgsql.EntityFrameworkCore.PostgreSQL.Metadata; + +#nullable disable + +namespace DysonNetwork.Sphere.Migrations +{ + [DbContext(typeof(AppDatabase))] + [Migration("20250412182922_AddCloudFiles")] + partial class AddCloudFiles + { + /// + protected override void BuildTargetModel(ModelBuilder modelBuilder) + { +#pragma warning disable 612, 618 + modelBuilder + .HasAnnotation("ProductVersion", "9.0.3") + .HasAnnotation("Relational:MaxIdentifierLength", 63); + + NpgsqlModelBuilderExtensions.UseIdentityByDefaultColumns(modelBuilder); + + modelBuilder.Entity("DysonNetwork.Sphere.Account.Account", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("bigint") + .HasColumnName("id"); + + NpgsqlPropertyBuilderExtensions.UseIdentityByDefaultColumn(b.Property("Id")); + + b.Property("CreatedAt") + .HasColumnType("timestamp with time zone") + .HasColumnName("created_at"); + + b.Property("DeletedAt") + .HasColumnType("timestamp with time zone") + .HasColumnName("deleted_at"); + + b.Property("IsSuperuser") + .HasColumnType("boolean") + .HasColumnName("is_superuser"); + + b.Property("Language") + .IsRequired() + .HasMaxLength(32) + .HasColumnType("character varying(32)") + .HasColumnName("language"); + + b.Property("Name") + .IsRequired() + .HasMaxLength(256) + .HasColumnType("character varying(256)") + .HasColumnName("name"); + + b.Property("Nick") + .IsRequired() + .HasMaxLength(256) + .HasColumnType("character varying(256)") + .HasColumnName("nick"); + + b.Property("UpdatedAt") + .HasColumnType("timestamp with time zone") + .HasColumnName("updated_at"); + + b.HasKey("Id") + .HasName("pk_accounts"); + + b.ToTable("accounts", (string)null); + }); + + modelBuilder.Entity("DysonNetwork.Sphere.Account.AccountAuthFactor", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("bigint") + .HasColumnName("id"); + + NpgsqlPropertyBuilderExtensions.UseIdentityByDefaultColumn(b.Property("Id")); + + b.Property("AccountId") + .HasColumnType("bigint") + .HasColumnName("account_id"); + + b.Property("CreatedAt") + .HasColumnType("timestamp with time zone") + .HasColumnName("created_at"); + + b.Property("DeletedAt") + .HasColumnType("timestamp with time zone") + .HasColumnName("deleted_at"); + + b.Property("Secret") + .HasColumnType("text") + .HasColumnName("secret"); + + b.Property("Type") + .HasColumnType("integer") + .HasColumnName("type"); + + b.Property("UpdatedAt") + .HasColumnType("timestamp with time zone") + .HasColumnName("updated_at"); + + b.HasKey("Id") + .HasName("pk_account_auth_factors"); + + b.HasIndex("AccountId") + .HasDatabaseName("ix_account_auth_factors_account_id"); + + b.ToTable("account_auth_factors", (string)null); + }); + + modelBuilder.Entity("DysonNetwork.Sphere.Account.AccountContact", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("bigint") + .HasColumnName("id"); + + NpgsqlPropertyBuilderExtensions.UseIdentityByDefaultColumn(b.Property("Id")); + + b.Property("AccountId") + .HasColumnType("bigint") + .HasColumnName("account_id"); + + b.Property("Content") + .IsRequired() + .HasMaxLength(1024) + .HasColumnType("character varying(1024)") + .HasColumnName("content"); + + b.Property("CreatedAt") + .HasColumnType("timestamp with time zone") + .HasColumnName("created_at"); + + b.Property("DeletedAt") + .HasColumnType("timestamp with time zone") + .HasColumnName("deleted_at"); + + b.Property("Type") + .HasColumnType("integer") + .HasColumnName("type"); + + b.Property("UpdatedAt") + .HasColumnType("timestamp with time zone") + .HasColumnName("updated_at"); + + b.Property("VerifiedAt") + .HasColumnType("timestamp with time zone") + .HasColumnName("verified_at"); + + b.HasKey("Id") + .HasName("pk_account_contacts"); + + b.HasIndex("AccountId") + .HasDatabaseName("ix_account_contacts_account_id"); + + b.ToTable("account_contacts", (string)null); + }); + + modelBuilder.Entity("DysonNetwork.Sphere.Auth.Challenge", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("uuid") + .HasColumnName("id"); + + b.Property("AccountId") + .HasColumnType("bigint") + .HasColumnName("account_id"); + + b.Property>("Audiences") + .IsRequired() + .HasColumnType("jsonb") + .HasColumnName("audiences"); + + b.Property>("BlacklistFactors") + .IsRequired() + .HasColumnType("jsonb") + .HasColumnName("blacklist_factors"); + + b.Property("CreatedAt") + .HasColumnType("timestamp with time zone") + .HasColumnName("created_at"); + + b.Property("DeletedAt") + .HasColumnType("timestamp with time zone") + .HasColumnName("deleted_at"); + + b.Property("DeviceId") + .HasMaxLength(256) + .HasColumnType("character varying(256)") + .HasColumnName("device_id"); + + b.Property("ExpiredAt") + .HasColumnType("timestamp with time zone") + .HasColumnName("expired_at"); + + b.Property("IpAddress") + .HasMaxLength(128) + .HasColumnType("character varying(128)") + .HasColumnName("ip_address"); + + b.Property("Nonce") + .HasMaxLength(1024) + .HasColumnType("character varying(1024)") + .HasColumnName("nonce"); + + b.Property>("Scopes") + .IsRequired() + .HasColumnType("jsonb") + .HasColumnName("scopes"); + + b.Property("StepRemain") + .HasColumnType("integer") + .HasColumnName("step_remain"); + + b.Property("StepTotal") + .HasColumnType("integer") + .HasColumnName("step_total"); + + b.Property("UpdatedAt") + .HasColumnType("timestamp with time zone") + .HasColumnName("updated_at"); + + b.Property("UserAgent") + .HasMaxLength(512) + .HasColumnType("character varying(512)") + .HasColumnName("user_agent"); + + b.HasKey("Id") + .HasName("pk_auth_challenges"); + + b.HasIndex("AccountId") + .HasDatabaseName("ix_auth_challenges_account_id"); + + b.ToTable("auth_challenges", (string)null); + }); + + modelBuilder.Entity("DysonNetwork.Sphere.Auth.Session", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("uuid") + .HasColumnName("id"); + + b.Property("AccountId") + .HasColumnType("bigint") + .HasColumnName("account_id"); + + b.Property("ChallengeId") + .HasColumnType("uuid") + .HasColumnName("challenge_id"); + + b.Property("CreatedAt") + .HasColumnType("timestamp with time zone") + .HasColumnName("created_at"); + + b.Property("DeletedAt") + .HasColumnType("timestamp with time zone") + .HasColumnName("deleted_at"); + + b.Property("ExpiredAt") + .HasColumnType("timestamp with time zone") + .HasColumnName("expired_at"); + + b.Property("LastGrantedAt") + .HasColumnType("timestamp with time zone") + .HasColumnName("last_granted_at"); + + b.Property("UpdatedAt") + .HasColumnType("timestamp with time zone") + .HasColumnName("updated_at"); + + b.HasKey("Id") + .HasName("pk_auth_sessions"); + + b.HasIndex("AccountId") + .HasDatabaseName("ix_auth_sessions_account_id"); + + b.HasIndex("ChallengeId") + .HasDatabaseName("ix_auth_sessions_challenge_id"); + + b.ToTable("auth_sessions", (string)null); + }); + + modelBuilder.Entity("DysonNetwork.Sphere.Storage.CloudFile", b => + { + b.Property("Id") + .HasColumnType("text") + .HasColumnName("id"); + + b.Property("AccountId") + .HasColumnType("bigint") + .HasColumnName("account_id"); + + b.Property("CreatedAt") + .HasColumnType("timestamp with time zone") + .HasColumnName("created_at"); + + b.Property("DeletedAt") + .HasColumnType("timestamp with time zone") + .HasColumnName("deleted_at"); + + b.Property("Description") + .HasMaxLength(4096) + .HasColumnType("character varying(4096)") + .HasColumnName("description"); + + b.Property>("FileMeta") + .HasColumnType("jsonb") + .HasColumnName("file_meta"); + + b.Property("Hash") + .HasMaxLength(256) + .HasColumnType("character varying(256)") + .HasColumnName("hash"); + + b.Property("MimeType") + .HasMaxLength(256) + .HasColumnType("character varying(256)") + .HasColumnName("mime_type"); + + b.Property("Name") + .IsRequired() + .HasMaxLength(1024) + .HasColumnType("character varying(1024)") + .HasColumnName("name"); + + b.Property("Size") + .HasColumnType("bigint") + .HasColumnName("size"); + + b.Property("UpdatedAt") + .HasColumnType("timestamp with time zone") + .HasColumnName("updated_at"); + + b.Property("UploadedAt") + .HasColumnType("timestamp with time zone") + .HasColumnName("uploaded_at"); + + b.Property("UploadedTo") + .HasMaxLength(128) + .HasColumnType("character varying(128)") + .HasColumnName("uploaded_to"); + + b.Property("UsedCount") + .HasColumnType("integer") + .HasColumnName("used_count"); + + b.Property>("UserMeta") + .HasColumnType("jsonb") + .HasColumnName("user_meta"); + + b.HasKey("Id") + .HasName("pk_files"); + + b.HasIndex("AccountId") + .HasDatabaseName("ix_files_account_id"); + + b.ToTable("files", (string)null); + }); + + modelBuilder.Entity("DysonNetwork.Sphere.Account.AccountAuthFactor", b => + { + b.HasOne("DysonNetwork.Sphere.Account.Account", "Account") + .WithMany("AuthFactors") + .HasForeignKey("AccountId") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired() + .HasConstraintName("fk_account_auth_factors_accounts_account_id"); + + b.Navigation("Account"); + }); + + modelBuilder.Entity("DysonNetwork.Sphere.Account.AccountContact", b => + { + b.HasOne("DysonNetwork.Sphere.Account.Account", "Account") + .WithMany("Contacts") + .HasForeignKey("AccountId") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired() + .HasConstraintName("fk_account_contacts_accounts_account_id"); + + b.Navigation("Account"); + }); + + modelBuilder.Entity("DysonNetwork.Sphere.Auth.Challenge", b => + { + b.HasOne("DysonNetwork.Sphere.Account.Account", "Account") + .WithMany("Challenges") + .HasForeignKey("AccountId") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired() + .HasConstraintName("fk_auth_challenges_accounts_account_id"); + + b.Navigation("Account"); + }); + + modelBuilder.Entity("DysonNetwork.Sphere.Auth.Session", b => + { + b.HasOne("DysonNetwork.Sphere.Account.Account", "Account") + .WithMany("Sessions") + .HasForeignKey("AccountId") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired() + .HasConstraintName("fk_auth_sessions_accounts_account_id"); + + b.HasOne("DysonNetwork.Sphere.Auth.Challenge", "Challenge") + .WithMany() + .HasForeignKey("ChallengeId") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired() + .HasConstraintName("fk_auth_sessions_auth_challenges_challenge_id"); + + b.Navigation("Account"); + + b.Navigation("Challenge"); + }); + + modelBuilder.Entity("DysonNetwork.Sphere.Storage.CloudFile", b => + { + b.HasOne("DysonNetwork.Sphere.Account.Account", "Account") + .WithMany() + .HasForeignKey("AccountId") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired() + .HasConstraintName("fk_files_accounts_account_id"); + + b.Navigation("Account"); + }); + + modelBuilder.Entity("DysonNetwork.Sphere.Account.Account", b => + { + b.Navigation("AuthFactors"); + + b.Navigation("Challenges"); + + b.Navigation("Contacts"); + + b.Navigation("Sessions"); + }); +#pragma warning restore 612, 618 + } + } +} diff --git a/DysonNetwork.Sphere/Migrations/20250412182922_AddCloudFiles.cs b/DysonNetwork.Sphere/Migrations/20250412182922_AddCloudFiles.cs new file mode 100644 index 0000000..c4664d4 --- /dev/null +++ b/DysonNetwork.Sphere/Migrations/20250412182922_AddCloudFiles.cs @@ -0,0 +1,59 @@ +using System.Collections.Generic; +using Microsoft.EntityFrameworkCore.Migrations; +using NodaTime; + +#nullable disable + +namespace DysonNetwork.Sphere.Migrations +{ + /// + public partial class AddCloudFiles : Migration + { + /// + protected override void Up(MigrationBuilder migrationBuilder) + { + migrationBuilder.CreateTable( + name: "files", + columns: table => new + { + id = table.Column(type: "text", nullable: false), + name = table.Column(type: "character varying(1024)", maxLength: 1024, nullable: false), + description = table.Column(type: "character varying(4096)", maxLength: 4096, nullable: true), + file_meta = table.Column>(type: "jsonb", nullable: true), + user_meta = table.Column>(type: "jsonb", nullable: true), + mime_type = table.Column(type: "character varying(256)", maxLength: 256, nullable: true), + hash = table.Column(type: "character varying(256)", maxLength: 256, nullable: true), + size = table.Column(type: "bigint", nullable: false), + uploaded_at = table.Column(type: "timestamp with time zone", nullable: true), + uploaded_to = table.Column(type: "character varying(128)", maxLength: 128, nullable: true), + used_count = table.Column(type: "integer", nullable: false), + account_id = table.Column(type: "bigint", nullable: false), + created_at = table.Column(type: "timestamp with time zone", nullable: false), + updated_at = table.Column(type: "timestamp with time zone", nullable: false), + deleted_at = table.Column(type: "timestamp with time zone", nullable: true) + }, + constraints: table => + { + table.PrimaryKey("pk_files", x => x.id); + table.ForeignKey( + name: "fk_files_accounts_account_id", + column: x => x.account_id, + principalTable: "accounts", + principalColumn: "id", + onDelete: ReferentialAction.Cascade); + }); + + migrationBuilder.CreateIndex( + name: "ix_files_account_id", + table: "files", + column: "account_id"); + } + + /// + protected override void Down(MigrationBuilder migrationBuilder) + { + migrationBuilder.DropTable( + name: "files"); + } + } +} diff --git a/DysonNetwork.Sphere/Migrations/AppDatabaseModelSnapshot.cs b/DysonNetwork.Sphere/Migrations/AppDatabaseModelSnapshot.cs index 3f6c0fa..d5e35e9 100644 --- a/DysonNetwork.Sphere/Migrations/AppDatabaseModelSnapshot.cs +++ b/DysonNetwork.Sphere/Migrations/AppDatabaseModelSnapshot.cs @@ -289,6 +289,83 @@ namespace DysonNetwork.Sphere.Migrations b.ToTable("auth_sessions", (string)null); }); + modelBuilder.Entity("DysonNetwork.Sphere.Storage.CloudFile", b => + { + b.Property("Id") + .HasColumnType("text") + .HasColumnName("id"); + + b.Property("AccountId") + .HasColumnType("bigint") + .HasColumnName("account_id"); + + b.Property("CreatedAt") + .HasColumnType("timestamp with time zone") + .HasColumnName("created_at"); + + b.Property("DeletedAt") + .HasColumnType("timestamp with time zone") + .HasColumnName("deleted_at"); + + b.Property("Description") + .HasMaxLength(4096) + .HasColumnType("character varying(4096)") + .HasColumnName("description"); + + b.Property>("FileMeta") + .HasColumnType("jsonb") + .HasColumnName("file_meta"); + + b.Property("Hash") + .HasMaxLength(256) + .HasColumnType("character varying(256)") + .HasColumnName("hash"); + + b.Property("MimeType") + .HasMaxLength(256) + .HasColumnType("character varying(256)") + .HasColumnName("mime_type"); + + b.Property("Name") + .IsRequired() + .HasMaxLength(1024) + .HasColumnType("character varying(1024)") + .HasColumnName("name"); + + b.Property("Size") + .HasColumnType("bigint") + .HasColumnName("size"); + + b.Property("UpdatedAt") + .HasColumnType("timestamp with time zone") + .HasColumnName("updated_at"); + + b.Property("UploadedAt") + .HasColumnType("timestamp with time zone") + .HasColumnName("uploaded_at"); + + b.Property("UploadedTo") + .HasMaxLength(128) + .HasColumnType("character varying(128)") + .HasColumnName("uploaded_to"); + + b.Property("UsedCount") + .HasColumnType("integer") + .HasColumnName("used_count"); + + b.Property>("UserMeta") + .HasColumnType("jsonb") + .HasColumnName("user_meta"); + + b.HasKey("Id") + .HasName("pk_files"); + + b.HasIndex("AccountId") + .HasDatabaseName("ix_files_account_id"); + + b.ToTable("files", (string)null); + }); + modelBuilder.Entity("DysonNetwork.Sphere.Account.AccountAuthFactor", b => { b.HasOne("DysonNetwork.Sphere.Account.Account", "Account") @@ -346,6 +423,18 @@ namespace DysonNetwork.Sphere.Migrations b.Navigation("Challenge"); }); + modelBuilder.Entity("DysonNetwork.Sphere.Storage.CloudFile", b => + { + b.HasOne("DysonNetwork.Sphere.Account.Account", "Account") + .WithMany() + .HasForeignKey("AccountId") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired() + .HasConstraintName("fk_files_accounts_account_id"); + + b.Navigation("Account"); + }); + modelBuilder.Entity("DysonNetwork.Sphere.Account.Account", b => { b.Navigation("AuthFactors"); diff --git a/DysonNetwork.Sphere/Program.cs b/DysonNetwork.Sphere/Program.cs index 639819f..fb1c414 100644 --- a/DysonNetwork.Sphere/Program.cs +++ b/DysonNetwork.Sphere/Program.cs @@ -1,3 +1,4 @@ +using System.Net; using System.Security.Cryptography; using System.Text; using System.Text.Json; @@ -6,6 +7,7 @@ using Casbin.Persist.Adapter.EFCore; using DysonNetwork.Sphere; using DysonNetwork.Sphere.Account; using DysonNetwork.Sphere.Auth; +using DysonNetwork.Sphere.Storage; using Microsoft.AspNetCore.Authorization; using Microsoft.AspNetCore.HttpOverrides; using Microsoft.EntityFrameworkCore; @@ -13,6 +15,9 @@ using Microsoft.IdentityModel.Tokens; using Microsoft.OpenApi.Models; using NodaTime; using NodaTime.Serialization.SystemTextJson; +using tusdotnet; +using tusdotnet.Models; +using File = System.IO.File; var builder = WebApplication.CreateBuilder(args); @@ -106,6 +111,7 @@ builder.Services.AddOpenApi(); builder.Services.AddScoped(); builder.Services.AddScoped(); +builder.Services.AddScoped(); var app = builder.Build(); @@ -129,11 +135,74 @@ app.UseCors(opts => opts.SetIsOriginAllowed(_ => true) .AllowCredentials() .AllowAnyHeader() - .AllowAnyMethod()); + .AllowAnyMethod() +); app.UseHttpsRedirection(); app.UseAuthorization(); app.MapControllers(); +var tusDiskStore = new tusdotnet.Stores.TusDiskStore( + builder.Configuration.GetSection("Tus").GetValue("StorePath")! +); +app.MapTus("/files/tus", (_) => Task.FromResult(new() +{ + Store = tusDiskStore, + Events = new() + { + OnAuthorizeAsync = async eventContext => + { + var httpContext = eventContext.HttpContext; + var user = httpContext.User; + if (!user.Identity?.IsAuthenticated ?? true) + { + eventContext.FailRequest(HttpStatusCode.Unauthorized); + return; + } + + var userId = httpContext.User.FindFirst("user_id")?.Value; + if (userId == null) return; + var isSuperuser = httpContext.User.FindFirst("is_superuser")?.Value == "1"; + if (isSuperuser) userId = "super:" + userId; + + var enforcer = httpContext.RequestServices.GetRequiredService(); + var allowed = await enforcer.EnforceAsync(userId, "global", "files", "create"); + if (!allowed) + { + eventContext.FailRequest(HttpStatusCode.Forbidden); + } + }, + OnFileCompleteAsync = async eventContext => + { + var httpContext = eventContext.HttpContext; + var user = httpContext.User; + var userId = long.Parse(user.FindFirst("user_id")!.Value); + + var db = httpContext.RequestServices.GetRequiredService(); + var account = await db.Accounts.FindAsync(userId); + if (account is null) return; + + var file = await eventContext.GetFileAsync(); + var metadata = await file.GetMetadataAsync(eventContext.CancellationToken); + var fileName = metadata.TryGetValue("filename", out var fn) ? fn.GetString(Encoding.UTF8) : "uploaded_file"; + var contentType = metadata.TryGetValue("content-type", out var ct) ? ct.GetString(Encoding.UTF8) : null; + var fileStream = await file.GetContentAsync(eventContext.CancellationToken); + + var fileService = eventContext.HttpContext.RequestServices.GetRequiredService(); + + var info = await fileService.AnalyzeFileAsync(account, file.Id, fileStream, fileName, contentType); + await fileService.UploadFileToRemoteAsync(info, fileStream, null); + + await tusDiskStore.DeleteFileAsync(file.Id, eventContext.CancellationToken); + }, + OnCreateCompleteAsync = eventContext => + { + // var baseUrl = builder.Configuration.GetValue("Storage:BaseUrl")!; + // eventContext.SetUploadUrl(new Uri($"{baseUrl}/files/{eventContext.FileId}")); + return Task.CompletedTask; + } + } +})); + app.Run(); \ No newline at end of file diff --git a/DysonNetwork.Sphere/Storage/CloudFile.cs b/DysonNetwork.Sphere/Storage/CloudFile.cs new file mode 100644 index 0000000..93aa63f --- /dev/null +++ b/DysonNetwork.Sphere/Storage/CloudFile.cs @@ -0,0 +1,59 @@ +using System.ComponentModel.DataAnnotations; +using System.ComponentModel.DataAnnotations.Schema; +using Newtonsoft.Json; +using NodaTime; + +namespace DysonNetwork.Sphere.Storage; + +public class RemoteStorageConfig +{ + public string Id { get; set; } = string.Empty; + public string Label { get; set; } = string.Empty; + public string Region { get; set; } = string.Empty; + public string Bucket { get; set; } = string.Empty; + public string Endpoint { get; set; } = string.Empty; + public string SecretId { get; set; } = string.Empty; + public string SecretKey { get; set; } = string.Empty; + public bool EnableSigned { get; set; } + public bool EnableSsl { get; set; } + public string? ImageProxy { get; set; } + public string? AccessProxy { get; set; } +} + +public class CloudFile : BaseModel +{ + public string Id { get; set; } = Guid.NewGuid().ToString(); + [MaxLength(1024)] public string Name { get; set; } = string.Empty; + [MaxLength(4096)] public string? Description { get; set; } + [Column(TypeName = "jsonb")] public Dictionary? FileMeta { get; set; } = null!; + [Column(TypeName = "jsonb")] public Dictionary? UserMeta { get; set; } = null!; + [Column(TypeName = "jsonb")] List SensitiveMarks { get; set; } = new(); + [MaxLength(256)] public string? MimeType { get; set; } + [MaxLength(256)] public string? Hash { get; set; } + public long Size { get; set; } + public Instant? UploadedAt { get; set; } + [MaxLength(128)] public string? UploadedTo { get; set; } + + // Metrics + // When this used count keep zero, it means it's not used by anybody, so it can be recycled + public int UsedCount { get; set; } = 0; + + [JsonIgnore] public Account.Account Account { get; set; } = null!; +} + +public enum CloudFileSensitiveMark +{ + Language, + SexualContent, + Violence, + Profanity, + HateSpeech, + Racism, + AdultContent, + DrugAbuse, + AlcoholAbuse, + Gambling, + SelfHarm, + ChildAbuse, + Other +} \ No newline at end of file diff --git a/DysonNetwork.Sphere/Storage/FileService.cs b/DysonNetwork.Sphere/Storage/FileService.cs new file mode 100644 index 0000000..2858a92 --- /dev/null +++ b/DysonNetwork.Sphere/Storage/FileService.cs @@ -0,0 +1,142 @@ +using System.Globalization; +using FFMpegCore; +using System.Security.Cryptography; +using Microsoft.EntityFrameworkCore; +using Minio; +using Minio.DataModel.Args; +using Minio.DataModel.Tags; +using NodaTime; + +namespace DysonNetwork.Sphere.Storage; + +public class FileService(AppDatabase db, IConfiguration configuration) +{ + public async Task AnalyzeFileAsync( + Account.Account account, + string fileId, + Stream stream, + string fileName, + string? contentType + ) + { + var fileSize = stream.Length; + var hash = await HashFileAsync(stream, fileSize: fileSize); + contentType ??= !fileName.Contains('.') ? "application/octet-stream" : MimeTypes.GetMimeType(fileName); + + var existingFile = await db.Files.Where(f => f.Hash == hash).FirstOrDefaultAsync(); + if (existingFile is not null) return existingFile; + + var file = new CloudFile + { + Id = fileId, + Name = fileName, + MimeType = contentType, + Size = fileSize, + Hash = hash, + Account = account, + }; + + switch (contentType.Split('/')[0]) + { + case "video": + case "audio": + var mediaInfo = await FFProbe.AnalyseAsync(stream); + file.FileMeta = new Dictionary + { + ["duration"] = mediaInfo.Duration.TotalSeconds, + ["format_name"] = mediaInfo.Format.FormatName, + ["format_long_name"] = mediaInfo.Format.FormatLongName, + ["start_time"] = mediaInfo.Format.StartTime.ToString(), + ["bit_rate"] = mediaInfo.Format.BitRate.ToString(CultureInfo.InvariantCulture), + ["tags"] = mediaInfo.Format.Tags ?? [], + ["chapters"] = mediaInfo.Chapters, + }; + break; + } + + db.Files.Add(file); + await db.SaveChangesAsync(); + return file; + } + + private static async Task HashFileAsync(Stream stream, int chunkSize = 1024 * 1024, long? fileSize = null) + { + fileSize ??= stream.Length; + if (fileSize > chunkSize * 1024 * 5) + return await HashFastApproximateAsync(stream, chunkSize); + + using var md5 = MD5.Create(); + var hashBytes = await md5.ComputeHashAsync(stream); + return Convert.ToHexString(hashBytes).ToLowerInvariant(); + } + + private static async Task HashFastApproximateAsync(Stream stream, int chunkSize = 1024 * 1024) + { + // Scale the chunk size to kB level + chunkSize *= 1024; + + using var md5 = MD5.Create(); + + var buffer = new byte[chunkSize * 2]; + var fileLength = stream.Length; + + var bytesRead = await stream.ReadAsync(buffer.AsMemory(0, chunkSize)); + + if (fileLength > chunkSize) + { + stream.Seek(-chunkSize, SeekOrigin.End); + bytesRead += await stream.ReadAsync(buffer.AsMemory(chunkSize, chunkSize)); + } + + var hash = md5.ComputeHash(buffer, 0, bytesRead); + return Convert.ToHexString(hash).ToLowerInvariant(); + } + + public async Task UploadFileToRemoteAsync(CloudFile file, Stream stream, string? targetRemote) + { + if (file.UploadedAt.HasValue) return file; + + file.UploadedTo = targetRemote ?? configuration.GetValue("Storage:PreferredRemote")!; + + var dest = GetRemoteStorageConfig(file.UploadedTo); + var client = CreateMinioClient(dest); + if (client is null) + throw new InvalidOperationException( + $"Failed to configure client for remote destination '{file.UploadedTo}'" + ); + + var bucket = dest.Bucket; + var contentType = file.MimeType ?? "application/octet-stream"; + + await client.PutObjectAsync(new PutObjectArgs() + .WithBucket(bucket) + .WithObject(file.Id) + .WithStreamData(stream) + .WithObjectSize(stream.Length) + .WithContentType(contentType) + ); + + file.UploadedAt = Instant.FromDateTimeUtc(DateTime.UtcNow); + await db.SaveChangesAsync(); + return file; + } + + private RemoteStorageConfig GetRemoteStorageConfig(string destination) + { + var destinations = configuration.GetSection("Storage:Remote").Get>()!; + var dest = destinations.FirstOrDefault(d => d.Id == destination); + if (dest is null) throw new InvalidOperationException($"Remote destination '{destination}' not found"); + return dest; + } + + private IMinioClient? CreateMinioClient(RemoteStorageConfig dest) + { + var client = new MinioClient() + .WithEndpoint(dest.Endpoint) + .WithRegion(dest.Region) + .WithCredentials(dest.SecretId, dest.SecretKey); + if (dest.EnableSsl) client = client.WithSSL(); + + return client.Build(); + } +} \ No newline at end of file diff --git a/DysonNetwork.Sphere/appsettings.json b/DysonNetwork.Sphere/appsettings.json index 307b405..3c78ba5 100644 --- a/DysonNetwork.Sphere/appsettings.json +++ b/DysonNetwork.Sphere/appsettings.json @@ -24,5 +24,25 @@ "Jwt": { "PublicKeyPath": "Keys/PublicKey.pem", "PrivateKeyPath": "Keys/PrivateKey.pem" + }, + "Tus": { + "StorePath": "Uploads" + }, + "Storage": { + "BaseUrl": "http://localhost:5071", + "PreferredRemote": "cloudflare", + "Remote": [ + { + "Id": "cloudflare", + "Label": "Cloudflare R2", + "Region": "auto", + "Bucket": "solar-network", + "Endpoint": "0a70a6d1b7128888c823359d0008f4e1.r2.cloudflarestorage.com", + "SecretId": "8ff5d06c7b1639829d60bc6838a542e6", + "SecretKey": "4b000df2b31936e1ceb0aa48bbd4166214945bd7f83b85b26f9d164318587991", + "EnableSigned": true, + "EnableSsl": true + } + ] } } diff --git a/DysonNetwork.sln.DotSettings.user b/DysonNetwork.sln.DotSettings.user index 837c008..43df543 100644 --- a/DysonNetwork.sln.DotSettings.user +++ b/DysonNetwork.sln.DotSettings.user @@ -1,14 +1,24 @@  + ForceIncluded + ForceIncluded ForceIncluded ForceIncluded ForceIncluded + ForceIncluded + ForceIncluded ForceIncluded ForceIncluded + ForceIncluded ForceIncluded + ForceIncluded + ForceIncluded ForceIncluded ForceIncluded ForceIncluded ForceIncluded ForceIncluded ForceIncluded - ForceIncluded \ No newline at end of file + ForceIncluded + ForceIncluded + ForceIncluded + ForceIncluded \ No newline at end of file