From 3c52a6d787505e102ede54d54d9c072684b501c7 Mon Sep 17 00:00:00 2001 From: LittleSheep Date: Mon, 2 Jun 2025 00:49:19 +0800 Subject: [PATCH] :airplane: Better migration to new cloud files reference system --- DysonNetwork.Sphere/Account/Account.cs | 4 + .../Account/AccountCurrentController.cs | 10 +- .../Account/NotificationService.cs | 2 +- DysonNetwork.Sphere/Chat/ChatRoom.cs | 4 + .../Chat/ChatRoomController.cs | 3 - DysonNetwork.Sphere/Chat/Message.cs | 5 +- ...48_RefactorCloudFileReference.Designer.cs} | 72 +++- ...50601142048_RefactorCloudFileReference.cs} | 199 +++-------- .../Migrations/AppDatabaseModelSnapshot.cs | 70 +++- DysonNetwork.Sphere/Post/Post.cs | 3 + DysonNetwork.Sphere/Program.cs | 1 + DysonNetwork.Sphere/Publisher/Publisher.cs | 4 + .../Publisher/PublisherController.cs | 4 - DysonNetwork.Sphere/Realm/Realm.cs | 4 + DysonNetwork.Sphere/Realm/RealmController.cs | 2 - DysonNetwork.Sphere/Sticker/Sticker.cs | 6 +- DysonNetwork.Sphere/Sticker/StickerService.cs | 4 +- DysonNetwork.Sphere/Storage/CloudFile.cs | 5 +- DysonNetwork.Sphere/Storage/FileController.cs | 13 +- .../Storage/FileService.ReferenceMigration.cs | 318 ++++++++++++++++++ DysonNetwork.Sphere/Storage/ICloudFile.cs | 6 + DysonNetwork.Sphere/appsettings.json | 13 +- 22 files changed, 562 insertions(+), 190 deletions(-) rename DysonNetwork.Sphere/Migrations/{20250601111032_RefactorCloudFileReference.Designer.cs => 20250601142048_RefactorCloudFileReference.Designer.cs} (97%) rename DysonNetwork.Sphere/Migrations/{20250601111032_RefactorCloudFileReference.cs => 20250601142048_RefactorCloudFileReference.cs} (73%) create mode 100644 DysonNetwork.Sphere/Storage/FileService.ReferenceMigration.cs diff --git a/DysonNetwork.Sphere/Account/Account.cs b/DysonNetwork.Sphere/Account/Account.cs index 47704d6..345b2ea 100644 --- a/DysonNetwork.Sphere/Account/Account.cs +++ b/DysonNetwork.Sphere/Account/Account.cs @@ -69,6 +69,10 @@ public class Profile : ModelBase (Experience - Leveling.ExperiencePerLevel[Level]) * 100.0 / (Leveling.ExperiencePerLevel[Level + 1] - Leveling.ExperiencePerLevel[Level]); + // Outdated fields, for backward compability + [MaxLength(32)] public string? PictureId { get; set; } + [MaxLength(32)] public string? BackgroundId { get; set; } + [Column(TypeName = "jsonb")] public CloudFileReferenceObject? Picture { get; set; } [Column(TypeName = "jsonb")] public CloudFileReferenceObject? Background { get; set; } diff --git a/DysonNetwork.Sphere/Account/AccountCurrentController.cs b/DysonNetwork.Sphere/Account/AccountCurrentController.cs index debf145..b681b93 100644 --- a/DysonNetwork.Sphere/Account/AccountCurrentController.cs +++ b/DysonNetwork.Sphere/Account/AccountCurrentController.cs @@ -21,8 +21,6 @@ public class AccountCurrentController( AuthService auth ) : ControllerBase { - private const string ProfilePictureFileUsageIdentifier = "profile"; - [HttpGet] [ProducesResponseType(StatusCodes.Status200OK)] public async Task> GetCurrentIdentity() @@ -98,7 +96,7 @@ public class AccountCurrentController( // Remove old references for the profile picture if (profile.Picture is not null) { - var oldPictureRefs = await fileRefService.GetResourceReferencesAsync(profileResourceId, ProfilePictureFileUsageIdentifier); + var oldPictureRefs = await fileRefService.GetResourceReferencesAsync(profileResourceId, "profile.picture"); foreach (var oldRef in oldPictureRefs) { await fileRefService.DeleteReferenceAsync(oldRef.Id); @@ -110,7 +108,7 @@ public class AccountCurrentController( // Create new reference await fileRefService.CreateReferenceAsync( picture.Id, - ProfilePictureFileUsageIdentifier, + "profile.picture", profileResourceId ); } @@ -124,7 +122,7 @@ public class AccountCurrentController( // Remove old references for the profile background if (profile.Background is not null) { - var oldBackgroundRefs = await fileRefService.GetResourceReferencesAsync(profileResourceId, ProfilePictureFileUsageIdentifier); + var oldBackgroundRefs = await fileRefService.GetResourceReferencesAsync(profileResourceId, "profile.background"); foreach (var oldRef in oldBackgroundRefs) { await fileRefService.DeleteReferenceAsync(oldRef.Id); @@ -136,7 +134,7 @@ public class AccountCurrentController( // Create new reference await fileRefService.CreateReferenceAsync( background.Id, - ProfilePictureFileUsageIdentifier, + "profile.background", profileResourceId ); } diff --git a/DysonNetwork.Sphere/Account/NotificationService.cs b/DysonNetwork.Sphere/Account/NotificationService.cs index dd5e5e8..20959eb 100644 --- a/DysonNetwork.Sphere/Account/NotificationService.cs +++ b/DysonNetwork.Sphere/Account/NotificationService.cs @@ -28,7 +28,7 @@ public class NotificationService( { var existingSubscription = await db.NotificationPushSubscriptions .Where(s => s.AccountId == account.Id) - .Where(s => (s.DeviceId == deviceId || s.DeviceToken == deviceToken) && s.Provider == provider) + .Where(s => s.DeviceId == deviceId || s.DeviceToken == deviceToken) .FirstOrDefaultAsync(); if (existingSubscription is not null) diff --git a/DysonNetwork.Sphere/Chat/ChatRoom.cs b/DysonNetwork.Sphere/Chat/ChatRoom.cs index 5172c26..2a457e8 100644 --- a/DysonNetwork.Sphere/Chat/ChatRoom.cs +++ b/DysonNetwork.Sphere/Chat/ChatRoom.cs @@ -21,6 +21,10 @@ public class ChatRoom : ModelBase, IIdentifiedResource public bool IsCommunity { get; set; } public bool IsPublic { get; set; } + // Outdated fields, for backward compability + [MaxLength(32)] public string? PictureId { get; set; } + [MaxLength(32)] public string? BackgroundId { get; set; } + [Column(TypeName = "jsonb")] public CloudFileReferenceObject? Picture { get; set; } [Column(TypeName = "jsonb")] public CloudFileReferenceObject? Background { get; set; } diff --git a/DysonNetwork.Sphere/Chat/ChatRoomController.cs b/DysonNetwork.Sphere/Chat/ChatRoomController.cs index 83ab10a..29efc67 100644 --- a/DysonNetwork.Sphere/Chat/ChatRoomController.cs +++ b/DysonNetwork.Sphere/Chat/ChatRoomController.cs @@ -223,7 +223,6 @@ public class ChatRoomController( var chatRoom = await db.ChatRooms .Where(e => e.Id == id) - .Include(c => c.Picture) .Include(c => c.Background) .FirstOrDefaultAsync(); if (chatRoom is null) return NotFound(); @@ -320,8 +319,6 @@ public class ChatRoomController( var chatRoom = await db.ChatRooms .Where(e => e.Id == id) - .Include(c => c.Picture) - .Include(c => c.Background) .FirstOrDefaultAsync(); if (chatRoom is null) return NotFound(); diff --git a/DysonNetwork.Sphere/Chat/Message.cs b/DysonNetwork.Sphere/Chat/Message.cs index 1f12238..a0ef968 100644 --- a/DysonNetwork.Sphere/Chat/Message.cs +++ b/DysonNetwork.Sphere/Chat/Message.cs @@ -16,8 +16,11 @@ public class Message : ModelBase, IIdentifiedResource [Column(TypeName = "jsonb")] public List? MembersMentioned { get; set; } [MaxLength(36)] public string Nonce { get; set; } = null!; public Instant? EditedAt { get; set; } + + [Column(TypeName = "jsonb")] public List Attachments { get; set; } = []; - [Column(TypeName = "jsonb")] public List Attachments { get; set; } = []; + // Outdated fields, keep for backward compability + public ICollection OutdatedAttachments { get; set; } = new List(); public ICollection Reactions { get; set; } = new List(); public Guid? RepliedMessageId { get; set; } diff --git a/DysonNetwork.Sphere/Migrations/20250601111032_RefactorCloudFileReference.Designer.cs b/DysonNetwork.Sphere/Migrations/20250601142048_RefactorCloudFileReference.Designer.cs similarity index 97% rename from DysonNetwork.Sphere/Migrations/20250601111032_RefactorCloudFileReference.Designer.cs rename to DysonNetwork.Sphere/Migrations/20250601142048_RefactorCloudFileReference.Designer.cs index 0ab0474..7fc73ef 100644 --- a/DysonNetwork.Sphere/Migrations/20250601111032_RefactorCloudFileReference.Designer.cs +++ b/DysonNetwork.Sphere/Migrations/20250601142048_RefactorCloudFileReference.Designer.cs @@ -19,7 +19,7 @@ using NpgsqlTypes; namespace DysonNetwork.Sphere.Migrations { [DbContext(typeof(AppDatabase))] - [Migration("20250601111032_RefactorCloudFileReference")] + [Migration("20250601142048_RefactorCloudFileReference")] partial class RefactorCloudFileReference { /// @@ -540,6 +540,11 @@ namespace DysonNetwork.Sphere.Migrations .HasColumnType("jsonb") .HasColumnName("background"); + b.Property("BackgroundId") + .HasMaxLength(32) + .HasColumnType("character varying(32)") + .HasColumnName("background_id"); + b.Property("Bio") .HasMaxLength(4096) .HasColumnType("character varying(4096)") @@ -589,6 +594,11 @@ namespace DysonNetwork.Sphere.Migrations .HasColumnType("jsonb") .HasColumnName("picture"); + b.Property("PictureId") + .HasMaxLength(32) + .HasColumnType("character varying(32)") + .HasColumnName("picture_id"); + b.Property("Pronouns") .HasMaxLength(1024) .HasColumnType("character varying(1024)") @@ -984,6 +994,11 @@ namespace DysonNetwork.Sphere.Migrations .HasColumnType("jsonb") .HasColumnName("background"); + b.Property("BackgroundId") + .HasMaxLength(32) + .HasColumnType("character varying(32)") + .HasColumnName("background_id"); + b.Property("CreatedAt") .HasColumnType("timestamp with time zone") .HasColumnName("created_at"); @@ -1014,6 +1029,11 @@ namespace DysonNetwork.Sphere.Migrations .HasColumnType("jsonb") .HasColumnName("picture"); + b.Property("PictureId") + .HasMaxLength(32) + .HasColumnType("character varying(32)") + .HasColumnName("picture_id"); + b.Property("RealmId") .HasColumnType("uuid") .HasColumnName("realm_id"); @@ -1767,6 +1787,11 @@ namespace DysonNetwork.Sphere.Migrations .HasColumnType("jsonb") .HasColumnName("background"); + b.Property("BackgroundId") + .HasMaxLength(32) + .HasColumnType("character varying(32)") + .HasColumnName("background_id"); + b.Property("Bio") .HasMaxLength(4096) .HasColumnType("character varying(4096)") @@ -1796,6 +1821,11 @@ namespace DysonNetwork.Sphere.Migrations .HasColumnType("jsonb") .HasColumnName("picture"); + b.Property("PictureId") + .HasMaxLength(32) + .HasColumnType("character varying(32)") + .HasColumnName("picture_id"); + b.Property("RealmId") .HasColumnType("uuid") .HasColumnName("realm_id"); @@ -1967,6 +1997,11 @@ namespace DysonNetwork.Sphere.Migrations .HasColumnType("jsonb") .HasColumnName("background"); + b.Property("BackgroundId") + .HasMaxLength(32) + .HasColumnType("character varying(32)") + .HasColumnName("background_id"); + b.Property("CreatedAt") .HasColumnType("timestamp with time zone") .HasColumnName("created_at"); @@ -1999,6 +2034,11 @@ namespace DysonNetwork.Sphere.Migrations .HasColumnType("jsonb") .HasColumnName("picture"); + b.Property("PictureId") + .HasMaxLength(32) + .HasColumnType("character varying(32)") + .HasColumnName("picture_id"); + b.Property("Slug") .IsRequired() .HasMaxLength(1024) @@ -2090,12 +2130,10 @@ namespace DysonNetwork.Sphere.Migrations .HasColumnName("deleted_at"); b.Property("Image") - .IsRequired() .HasColumnType("jsonb") .HasColumnName("image"); b.Property("ImageId") - .IsRequired() .HasMaxLength(32) .HasColumnType("character varying(32)") .HasColumnName("image_id"); @@ -2217,6 +2255,10 @@ namespace DysonNetwork.Sphere.Migrations .HasColumnType("character varying(256)") .HasColumnName("hash"); + b.Property("MessageId") + .HasColumnType("uuid") + .HasColumnName("message_id"); + b.Property("MimeType") .HasMaxLength(256) .HasColumnType("character varying(256)") @@ -2228,6 +2270,10 @@ namespace DysonNetwork.Sphere.Migrations .HasColumnType("character varying(1024)") .HasColumnName("name"); + b.Property("PostId") + .HasColumnType("uuid") + .HasColumnName("post_id"); + b.Property>("SensitiveMarks") .HasColumnType("jsonb") .HasColumnName("sensitive_marks"); @@ -2269,6 +2315,12 @@ namespace DysonNetwork.Sphere.Migrations b.HasIndex("AccountId") .HasDatabaseName("ix_files_account_id"); + b.HasIndex("MessageId") + .HasDatabaseName("ix_files_message_id"); + + b.HasIndex("PostId") + .HasDatabaseName("ix_files_post_id"); + b.ToTable("files", (string)null); }); @@ -3125,6 +3177,16 @@ namespace DysonNetwork.Sphere.Migrations .IsRequired() .HasConstraintName("fk_files_accounts_account_id"); + b.HasOne("DysonNetwork.Sphere.Chat.Message", null) + .WithMany("OutdatedAttachments") + .HasForeignKey("MessageId") + .HasConstraintName("fk_files_chat_messages_message_id"); + + b.HasOne("DysonNetwork.Sphere.Post.Post", null) + .WithMany("OutdatedAttachments") + .HasForeignKey("PostId") + .HasConstraintName("fk_files_posts_post_id"); + b.Navigation("Account"); }); @@ -3285,6 +3347,8 @@ namespace DysonNetwork.Sphere.Migrations modelBuilder.Entity("DysonNetwork.Sphere.Chat.Message", b => { + b.Navigation("OutdatedAttachments"); + b.Navigation("Reactions"); }); @@ -3297,6 +3361,8 @@ namespace DysonNetwork.Sphere.Migrations modelBuilder.Entity("DysonNetwork.Sphere.Post.Post", b => { + b.Navigation("OutdatedAttachments"); + b.Navigation("Reactions"); }); diff --git a/DysonNetwork.Sphere/Migrations/20250601111032_RefactorCloudFileReference.cs b/DysonNetwork.Sphere/Migrations/20250601142048_RefactorCloudFileReference.cs similarity index 73% rename from DysonNetwork.Sphere/Migrations/20250601111032_RefactorCloudFileReference.cs rename to DysonNetwork.Sphere/Migrations/20250601142048_RefactorCloudFileReference.cs index 1e124db..1be30f6 100644 --- a/DysonNetwork.Sphere/Migrations/20250601111032_RefactorCloudFileReference.cs +++ b/DysonNetwork.Sphere/Migrations/20250601142048_RefactorCloudFileReference.cs @@ -30,14 +30,6 @@ namespace DysonNetwork.Sphere.Migrations name: "fk_chat_rooms_files_picture_id", table: "chat_rooms"); - migrationBuilder.DropForeignKey( - name: "fk_files_chat_messages_message_id", - table: "files"); - - migrationBuilder.DropForeignKey( - name: "fk_files_posts_post_id", - table: "files"); - migrationBuilder.DropForeignKey( name: "fk_posts_posts_threaded_post_id", table: "posts"); @@ -86,14 +78,6 @@ namespace DysonNetwork.Sphere.Migrations name: "ix_posts_threaded_post_id", table: "posts"); - migrationBuilder.DropIndex( - name: "ix_files_message_id", - table: "files"); - - migrationBuilder.DropIndex( - name: "ix_files_post_id", - table: "files"); - migrationBuilder.DropIndex( name: "ix_chat_rooms_background_id", table: "chat_rooms"); @@ -110,68 +94,39 @@ namespace DysonNetwork.Sphere.Migrations name: "ix_account_profiles_picture_id", table: "account_profiles"); - migrationBuilder.DropColumn( - name: "background_id", - table: "realms"); - - migrationBuilder.DropColumn( - name: "picture_id", - table: "realms"); - - migrationBuilder.DropColumn( - name: "background_id", - table: "publishers"); - - migrationBuilder.DropColumn( - name: "picture_id", - table: "publishers"); - migrationBuilder.DropColumn( name: "threaded_post_id", table: "posts"); - // TODO Move these following changes to next migrations after migrated all the attachments data - // migrationBuilder.DropColumn( - // name: "expired_at", - // table: "files"); - // - // migrationBuilder.DropColumn( - // name: "message_id", - // table: "files"); - // - // migrationBuilder.DropColumn( - // name: "post_id", - // table: "files"); - // - // migrationBuilder.DropColumn( - // name: "usage", - // table: "files"); - // - // migrationBuilder.DropColumn( - // name: "used_count", - // table: "files"); - // - // migrationBuilder.DropColumn( - // name: "background_id", - // table: "chat_rooms"); - // - // migrationBuilder.DropColumn( - // name: "picture_id", - // table: "chat_rooms"); - // - // migrationBuilder.DropColumn( - // name: "background_id", - // table: "account_profiles"); - // - // migrationBuilder.DropColumn( - // name: "picture_id", - // table: "account_profiles"); + migrationBuilder.DropColumn( + name: "expired_at", + table: "files"); + + migrationBuilder.DropColumn( + name: "usage", + table: "files"); + + migrationBuilder.DropColumn( + name: "used_count", + table: "files"); + + migrationBuilder.AlterColumn( + name: "image_id", + table: "stickers", + type: "character varying(32)", + maxLength: 32, + nullable: true, + oldClrType: typeof(string), + oldType: "character varying(32)", + oldMaxLength: 32); migrationBuilder.AddColumn( name: "image", table: "stickers", type: "jsonb", - nullable: false); + nullable: true, + defaultValueSql: "'[]'::jsonb" + ); migrationBuilder.AddColumn( name: "background", @@ -201,7 +156,9 @@ namespace DysonNetwork.Sphere.Migrations name: "attachments", table: "posts", type: "jsonb", - nullable: false); + nullable: false, + defaultValueSql: "'[]'::jsonb" + ); migrationBuilder.AddColumn( name: "background", @@ -219,7 +176,9 @@ namespace DysonNetwork.Sphere.Migrations name: "attachments", table: "chat_messages", type: "jsonb", - nullable: false); + nullable: false, + defaultValueSql: "'[]'::jsonb" + ); migrationBuilder.AddColumn( name: "background", @@ -313,33 +272,17 @@ namespace DysonNetwork.Sphere.Migrations name: "picture", table: "account_profiles"); - migrationBuilder.AddColumn( - name: "background_id", - table: "realms", + migrationBuilder.AlterColumn( + name: "image_id", + table: "stickers", type: "character varying(32)", maxLength: 32, - nullable: true); - - migrationBuilder.AddColumn( - name: "picture_id", - table: "realms", - type: "character varying(32)", - maxLength: 32, - nullable: true); - - migrationBuilder.AddColumn( - name: "background_id", - table: "publishers", - type: "character varying(32)", - maxLength: 32, - nullable: true); - - migrationBuilder.AddColumn( - name: "picture_id", - table: "publishers", - type: "character varying(32)", - maxLength: 32, - nullable: true); + nullable: false, + defaultValue: "", + oldClrType: typeof(string), + oldType: "character varying(32)", + oldMaxLength: 32, + oldNullable: true); migrationBuilder.AddColumn( name: "threaded_post_id", @@ -353,18 +296,6 @@ namespace DysonNetwork.Sphere.Migrations type: "timestamp with time zone", nullable: true); - migrationBuilder.AddColumn( - name: "message_id", - table: "files", - type: "uuid", - nullable: true); - - migrationBuilder.AddColumn( - name: "post_id", - table: "files", - type: "uuid", - nullable: true); - migrationBuilder.AddColumn( name: "usage", table: "files", @@ -379,34 +310,6 @@ namespace DysonNetwork.Sphere.Migrations nullable: false, defaultValue: 0); - migrationBuilder.AddColumn( - name: "background_id", - table: "chat_rooms", - type: "character varying(32)", - maxLength: 32, - nullable: true); - - migrationBuilder.AddColumn( - name: "picture_id", - table: "chat_rooms", - type: "character varying(32)", - maxLength: 32, - nullable: true); - - migrationBuilder.AddColumn( - name: "background_id", - table: "account_profiles", - type: "character varying(32)", - maxLength: 32, - nullable: true); - - migrationBuilder.AddColumn( - name: "picture_id", - table: "account_profiles", - type: "character varying(32)", - maxLength: 32, - nullable: true); - migrationBuilder.CreateIndex( name: "ix_stickers_image_id", table: "stickers", @@ -438,16 +341,6 @@ namespace DysonNetwork.Sphere.Migrations column: "threaded_post_id", unique: true); - migrationBuilder.CreateIndex( - name: "ix_files_message_id", - table: "files", - column: "message_id"); - - migrationBuilder.CreateIndex( - name: "ix_files_post_id", - table: "files", - column: "post_id"); - migrationBuilder.CreateIndex( name: "ix_chat_rooms_background_id", table: "chat_rooms", @@ -496,20 +389,6 @@ namespace DysonNetwork.Sphere.Migrations principalTable: "files", principalColumn: "id"); - migrationBuilder.AddForeignKey( - name: "fk_files_chat_messages_message_id", - table: "files", - column: "message_id", - principalTable: "chat_messages", - principalColumn: "id"); - - migrationBuilder.AddForeignKey( - name: "fk_files_posts_post_id", - table: "files", - column: "post_id", - principalTable: "posts", - principalColumn: "id"); - migrationBuilder.AddForeignKey( name: "fk_posts_posts_threaded_post_id", table: "posts", diff --git a/DysonNetwork.Sphere/Migrations/AppDatabaseModelSnapshot.cs b/DysonNetwork.Sphere/Migrations/AppDatabaseModelSnapshot.cs index 5d78e89..41de855 100644 --- a/DysonNetwork.Sphere/Migrations/AppDatabaseModelSnapshot.cs +++ b/DysonNetwork.Sphere/Migrations/AppDatabaseModelSnapshot.cs @@ -537,6 +537,11 @@ namespace DysonNetwork.Sphere.Migrations .HasColumnType("jsonb") .HasColumnName("background"); + b.Property("BackgroundId") + .HasMaxLength(32) + .HasColumnType("character varying(32)") + .HasColumnName("background_id"); + b.Property("Bio") .HasMaxLength(4096) .HasColumnType("character varying(4096)") @@ -586,6 +591,11 @@ namespace DysonNetwork.Sphere.Migrations .HasColumnType("jsonb") .HasColumnName("picture"); + b.Property("PictureId") + .HasMaxLength(32) + .HasColumnType("character varying(32)") + .HasColumnName("picture_id"); + b.Property("Pronouns") .HasMaxLength(1024) .HasColumnType("character varying(1024)") @@ -981,6 +991,11 @@ namespace DysonNetwork.Sphere.Migrations .HasColumnType("jsonb") .HasColumnName("background"); + b.Property("BackgroundId") + .HasMaxLength(32) + .HasColumnType("character varying(32)") + .HasColumnName("background_id"); + b.Property("CreatedAt") .HasColumnType("timestamp with time zone") .HasColumnName("created_at"); @@ -1011,6 +1026,11 @@ namespace DysonNetwork.Sphere.Migrations .HasColumnType("jsonb") .HasColumnName("picture"); + b.Property("PictureId") + .HasMaxLength(32) + .HasColumnType("character varying(32)") + .HasColumnName("picture_id"); + b.Property("RealmId") .HasColumnType("uuid") .HasColumnName("realm_id"); @@ -1764,6 +1784,11 @@ namespace DysonNetwork.Sphere.Migrations .HasColumnType("jsonb") .HasColumnName("background"); + b.Property("BackgroundId") + .HasMaxLength(32) + .HasColumnType("character varying(32)") + .HasColumnName("background_id"); + b.Property("Bio") .HasMaxLength(4096) .HasColumnType("character varying(4096)") @@ -1793,6 +1818,11 @@ namespace DysonNetwork.Sphere.Migrations .HasColumnType("jsonb") .HasColumnName("picture"); + b.Property("PictureId") + .HasMaxLength(32) + .HasColumnType("character varying(32)") + .HasColumnName("picture_id"); + b.Property("RealmId") .HasColumnType("uuid") .HasColumnName("realm_id"); @@ -1964,6 +1994,11 @@ namespace DysonNetwork.Sphere.Migrations .HasColumnType("jsonb") .HasColumnName("background"); + b.Property("BackgroundId") + .HasMaxLength(32) + .HasColumnType("character varying(32)") + .HasColumnName("background_id"); + b.Property("CreatedAt") .HasColumnType("timestamp with time zone") .HasColumnName("created_at"); @@ -1996,6 +2031,11 @@ namespace DysonNetwork.Sphere.Migrations .HasColumnType("jsonb") .HasColumnName("picture"); + b.Property("PictureId") + .HasMaxLength(32) + .HasColumnType("character varying(32)") + .HasColumnName("picture_id"); + b.Property("Slug") .IsRequired() .HasMaxLength(1024) @@ -2087,12 +2127,10 @@ namespace DysonNetwork.Sphere.Migrations .HasColumnName("deleted_at"); b.Property("Image") - .IsRequired() .HasColumnType("jsonb") .HasColumnName("image"); b.Property("ImageId") - .IsRequired() .HasMaxLength(32) .HasColumnType("character varying(32)") .HasColumnName("image_id"); @@ -2214,6 +2252,10 @@ namespace DysonNetwork.Sphere.Migrations .HasColumnType("character varying(256)") .HasColumnName("hash"); + b.Property("MessageId") + .HasColumnType("uuid") + .HasColumnName("message_id"); + b.Property("MimeType") .HasMaxLength(256) .HasColumnType("character varying(256)") @@ -2225,6 +2267,10 @@ namespace DysonNetwork.Sphere.Migrations .HasColumnType("character varying(1024)") .HasColumnName("name"); + b.Property("PostId") + .HasColumnType("uuid") + .HasColumnName("post_id"); + b.Property>("SensitiveMarks") .HasColumnType("jsonb") .HasColumnName("sensitive_marks"); @@ -2266,6 +2312,12 @@ namespace DysonNetwork.Sphere.Migrations b.HasIndex("AccountId") .HasDatabaseName("ix_files_account_id"); + b.HasIndex("MessageId") + .HasDatabaseName("ix_files_message_id"); + + b.HasIndex("PostId") + .HasDatabaseName("ix_files_post_id"); + b.ToTable("files", (string)null); }); @@ -3122,6 +3174,16 @@ namespace DysonNetwork.Sphere.Migrations .IsRequired() .HasConstraintName("fk_files_accounts_account_id"); + b.HasOne("DysonNetwork.Sphere.Chat.Message", null) + .WithMany("OutdatedAttachments") + .HasForeignKey("MessageId") + .HasConstraintName("fk_files_chat_messages_message_id"); + + b.HasOne("DysonNetwork.Sphere.Post.Post", null) + .WithMany("OutdatedAttachments") + .HasForeignKey("PostId") + .HasConstraintName("fk_files_posts_post_id"); + b.Navigation("Account"); }); @@ -3282,6 +3344,8 @@ namespace DysonNetwork.Sphere.Migrations modelBuilder.Entity("DysonNetwork.Sphere.Chat.Message", b => { + b.Navigation("OutdatedAttachments"); + b.Navigation("Reactions"); }); @@ -3294,6 +3358,8 @@ namespace DysonNetwork.Sphere.Migrations modelBuilder.Entity("DysonNetwork.Sphere.Post.Post", b => { + b.Navigation("OutdatedAttachments"); + b.Navigation("Reactions"); }); diff --git a/DysonNetwork.Sphere/Post/Post.cs b/DysonNetwork.Sphere/Post/Post.cs index cb0891b..e84904e 100644 --- a/DysonNetwork.Sphere/Post/Post.cs +++ b/DysonNetwork.Sphere/Post/Post.cs @@ -48,6 +48,9 @@ public class Post : ModelBase, IIdentifiedResource public Post? RepliedPost { get; set; } public Guid? ForwardedPostId { get; set; } public Post? ForwardedPost { get; set; } + + // Outdated fields, keep for backward compability + public ICollection OutdatedAttachments { get; set; } = new List(); [Column(TypeName = "jsonb")] public List Attachments { get; set; } = []; [JsonIgnore] public NpgsqlTsVector SearchVector { get; set; } = null!; diff --git a/DysonNetwork.Sphere/Program.cs b/DysonNetwork.Sphere/Program.cs index bfba11a..cde67e9 100644 --- a/DysonNetwork.Sphere/Program.cs +++ b/DysonNetwork.Sphere/Program.cs @@ -208,6 +208,7 @@ builder.Services.AddScoped(); builder.Services.AddScoped(); builder.Services.AddScoped(); builder.Services.AddScoped(); +builder.Services.AddScoped(); builder.Services.AddScoped(); builder.Services.AddScoped(); builder.Services.AddScoped(); diff --git a/DysonNetwork.Sphere/Publisher/Publisher.cs b/DysonNetwork.Sphere/Publisher/Publisher.cs index 63a3450..ecb6d33 100644 --- a/DysonNetwork.Sphere/Publisher/Publisher.cs +++ b/DysonNetwork.Sphere/Publisher/Publisher.cs @@ -23,6 +23,10 @@ public class Publisher : ModelBase [MaxLength(256)] public string Nick { get; set; } = string.Empty; [MaxLength(4096)] public string? Bio { get; set; } + // Outdated fields, for backward compability + [MaxLength(32)] public string? PictureId { get; set; } + [MaxLength(32)] public string? BackgroundId { get; set; } + [Column(TypeName = "jsonb")] public CloudFileReferenceObject? Picture { get; set; } [Column(TypeName = "jsonb")] public CloudFileReferenceObject? Background { get; set; } diff --git a/DysonNetwork.Sphere/Publisher/PublisherController.cs b/DysonNetwork.Sphere/Publisher/PublisherController.cs index 4b8343c..a513856 100644 --- a/DysonNetwork.Sphere/Publisher/PublisherController.cs +++ b/DysonNetwork.Sphere/Publisher/PublisherController.cs @@ -53,8 +53,6 @@ namespace DysonNetwork.Sphere.Publisher; .Where(m => m.AccountId == userId) .Where(m => m.JoinedAt != null) .Include(e => e.Publisher) - .Include(e => e.Publisher.Picture) - .Include(e => e.Publisher.Background) .ToListAsync(); return members.Select(m => m.Publisher).ToList(); @@ -71,8 +69,6 @@ namespace DysonNetwork.Sphere.Publisher; .Where(m => m.AccountId == userId) .Where(m => m.JoinedAt == null) .Include(e => e.Publisher) - .Include(e => e.Publisher.Picture) - .Include(e => e.Publisher.Background) .ToListAsync(); return members.ToList(); diff --git a/DysonNetwork.Sphere/Realm/Realm.cs b/DysonNetwork.Sphere/Realm/Realm.cs index 1b68fa4..260d4f7 100644 --- a/DysonNetwork.Sphere/Realm/Realm.cs +++ b/DysonNetwork.Sphere/Realm/Realm.cs @@ -19,6 +19,10 @@ public class Realm : ModelBase, IIdentifiedResource public Instant? VerifiedAt { get; set; } public bool IsCommunity { get; set; } public bool IsPublic { get; set; } + + // Outdated fields, for backward compability + [MaxLength(32)] public string? PictureId { get; set; } + [MaxLength(32)] public string? BackgroundId { get; set; } [Column(TypeName = "jsonb")] public CloudFileReferenceObject? Picture { get; set; } [Column(TypeName = "jsonb")] public CloudFileReferenceObject? Background { get; set; } diff --git a/DysonNetwork.Sphere/Realm/RealmController.cs b/DysonNetwork.Sphere/Realm/RealmController.cs index 83b6de4..d50a6b5 100644 --- a/DysonNetwork.Sphere/Realm/RealmController.cs +++ b/DysonNetwork.Sphere/Realm/RealmController.cs @@ -24,8 +24,6 @@ public class RealmController( { var realm = await db.Realms .Where(e => e.Slug == slug) - .Include(e => e.Picture) - .Include(e => e.Background) .FirstOrDefaultAsync(); if (realm is null) return NotFound(); diff --git a/DysonNetwork.Sphere/Sticker/Sticker.cs b/DysonNetwork.Sphere/Sticker/Sticker.cs index 878a0a5..e109cb5 100644 --- a/DysonNetwork.Sphere/Sticker/Sticker.cs +++ b/DysonNetwork.Sphere/Sticker/Sticker.cs @@ -11,8 +11,10 @@ public class Sticker : ModelBase, IIdentifiedResource public Guid Id { get; set; } = Guid.NewGuid(); [MaxLength(128)] public string Slug { get; set; } = null!; - [MaxLength(32)] public string ImageId { get; set; } = null!; - [Column(TypeName = "jsonb")] public CloudFileReferenceObject Image { get; set; } = null!; + // Outdated fields, for backward compability + [MaxLength(32)] public string? ImageId { get; set; } + + [Column(TypeName = "jsonb")] public CloudFileReferenceObject? Image { get; set; } = null!; public Guid PackId { get; set; } public StickerPack Pack { get; set; } = null!; diff --git a/DysonNetwork.Sphere/Sticker/StickerService.cs b/DysonNetwork.Sphere/Sticker/StickerService.cs index 354989e..d5519ce 100644 --- a/DysonNetwork.Sphere/Sticker/StickerService.cs +++ b/DysonNetwork.Sphere/Sticker/StickerService.cs @@ -73,7 +73,6 @@ public class StickerService(AppDatabase db, FileService fs, FileReferenceService public async Task DeleteStickerPackAsync(StickerPack pack) { var stickers = await db.Stickers - .Include(s => s.Image) .Where(s => s.PackId == pack.Id) .ToListAsync(); @@ -110,8 +109,7 @@ public class StickerService(AppDatabase db, FileService fs, FileReferenceService // If not in cache, fetch from the database IQueryable query = db.Stickers - .Include(e => e.Pack) - .Include(e => e.Image); + .Include(e => e.Pack); query = Guid.TryParse(identifier, out var guid) ? query.Where(e => e.Id == guid) : query.Where(e => (e.Pack.Prefix + e.Slug).ToLower() == identifier); diff --git a/DysonNetwork.Sphere/Storage/CloudFile.cs b/DysonNetwork.Sphere/Storage/CloudFile.cs index 8cd1e04..1a3ae46 100644 --- a/DysonNetwork.Sphere/Storage/CloudFile.cs +++ b/DysonNetwork.Sphere/Storage/CloudFile.cs @@ -24,7 +24,7 @@ public class RemoteStorageConfig /// The class that used in jsonb columns which referenced the cloud file. /// The aim of this class is to store some properties that won't change to a file to reduce the database load. /// -public class CloudFileReferenceObject : ICloudFile +public class CloudFileReferenceObject : ModelBase, ICloudFile { public string Id { get; set; } = null!; public string Name { get; set; } = string.Empty; @@ -75,6 +75,9 @@ public class CloudFile : ModelBase, ICloudFile, IIdentifiedResource { return new CloudFileReferenceObject { + CreatedAt = CreatedAt, + UpdatedAt = UpdatedAt, + DeletedAt = DeletedAt, Id = Id, Name = Name, FileMeta = FileMeta, diff --git a/DysonNetwork.Sphere/Storage/FileController.cs b/DysonNetwork.Sphere/Storage/FileController.cs index 5b3784d..7998845 100644 --- a/DysonNetwork.Sphere/Storage/FileController.cs +++ b/DysonNetwork.Sphere/Storage/FileController.cs @@ -1,3 +1,4 @@ +using DysonNetwork.Sphere.Permission; using Microsoft.AspNetCore.Authorization; using Microsoft.AspNetCore.Mvc; using Microsoft.EntityFrameworkCore; @@ -11,7 +12,8 @@ public class FileController( AppDatabase db, FileService fs, IConfiguration configuration, - IWebHostEnvironment env + IWebHostEnvironment env, + FileReferenceMigrationService rms ) : ControllerBase { [HttpGet("{id}")] @@ -107,4 +109,13 @@ public class FileController( return NoContent(); } + + [HttpPost("/maintenance/migrateReferences")] + [Authorize] + [RequiredPermission("maintenance", "files.references")] + public async Task MigrateFileReferences() + { + await rms.ScanAndMigrateReferences(); + return Ok(); + } } \ No newline at end of file diff --git a/DysonNetwork.Sphere/Storage/FileService.ReferenceMigration.cs b/DysonNetwork.Sphere/Storage/FileService.ReferenceMigration.cs new file mode 100644 index 0000000..5476ad9 --- /dev/null +++ b/DysonNetwork.Sphere/Storage/FileService.ReferenceMigration.cs @@ -0,0 +1,318 @@ +using Microsoft.EntityFrameworkCore; + +namespace DysonNetwork.Sphere.Storage; + +public class FileReferenceMigrationService(AppDatabase db) +{ + public async Task ScanAndMigrateReferences() + { + // Scan Posts for file references + await ScanPosts(); + + // Scan Messages for file references + await ScanMessages(); + + // Scan Profiles for file references + await ScanProfiles(); + + // Scan Chat entities for file references + await ScanChatRooms(); + + // Scan Realms for file references + await ScanRealms(); + + // Scan Publishers for file references + await ScanPublishers(); + } + + private async Task ScanPosts() + { + var posts = await db.Posts + .Include(p => p.OutdatedAttachments) + .Where(p => p.OutdatedAttachments.Any()) + .ToListAsync(); + + foreach (var post in posts) + { + var updatedAttachments = new List(); + + foreach (var attachment in post.OutdatedAttachments) + { + var file = await db.Files.FirstOrDefaultAsync(f => f.Id == attachment.Id); + if (file != null) + { + // Create a reference for the file + var reference = new CloudFileReference + { + FileId = file.Id, + File = file, + Usage = "post", + ResourceId = post.Id.ToString() + }; + + await db.FileReferences.AddAsync(reference); + updatedAttachments.Add(file.ToReferenceObject()); + } + else + { + // Keep the existing reference object if file not found + updatedAttachments.Add(attachment.ToReferenceObject()); + } + } + + post.Attachments = updatedAttachments; + db.Posts.Update(post); + } + + await db.SaveChangesAsync(); + } + + private async Task ScanMessages() + { + var messages = await db.ChatMessages + .Include(m => m.OutdatedAttachments) + .Where(m => m.OutdatedAttachments.Any() && m.DeletedAt == null) + .ToListAsync(); + + foreach (var message in messages) + { + var updatedAttachments = new List(); + + foreach (var attachment in message.OutdatedAttachments) + { + var file = await db.Files.FirstOrDefaultAsync(f => f.Id == attachment.Id); + if (file != null) + { + // Create a reference for the file + var reference = new CloudFileReference + { + FileId = file.Id, + File = file, + Usage = "chat", + ResourceId = message.Id.ToString() + }; + + await db.FileReferences.AddAsync(reference); + updatedAttachments.Add(file.ToReferenceObject()); + } + else + { + // Keep the existing reference object if file not found + updatedAttachments.Add(attachment.ToReferenceObject()); + } + } + + message.Attachments = updatedAttachments; + db.ChatMessages.Update(message); + } + + await db.SaveChangesAsync(); + } + + private async Task ScanProfiles() + { + var profiles = await db.AccountProfiles + .Where(p => p.PictureId != null || p.BackgroundId != null) + .ToListAsync(); + + foreach (var profile in profiles) + { + if (profile is { PictureId: not null, Picture: null }) + { + var avatarFile = await db.Files.FirstOrDefaultAsync(f => f.Id == profile.PictureId); + if (avatarFile != null) + { + // Create a reference for the avatar file + var reference = new CloudFileReference + { + FileId = avatarFile.Id, + File = avatarFile, + Usage = "profile.picture", + ResourceId = profile.Id.ToString() + }; + + await db.FileReferences.AddAsync(reference); + profile.Picture = avatarFile.ToReferenceObject(); + db.AccountProfiles.Update(profile); + } + } + + // Also check for the banner if it exists + if (profile is not { BackgroundId: not null, Background: null }) continue; + var bannerFile = await db.Files.FirstOrDefaultAsync(f => f.Id == profile.BackgroundId); + if (bannerFile == null) continue; + { + // Create a reference for the banner file + var reference = new CloudFileReference + { + FileId = bannerFile.Id, + File = bannerFile, + Usage = "profile.background", + ResourceId = profile.Id.ToString() + }; + + await db.FileReferences.AddAsync(reference); + profile.Background = bannerFile.ToReferenceObject(); + db.AccountProfiles.Update(profile); + } + } + + await db.SaveChangesAsync(); + } + + private async Task ScanChatRooms() + { + var chatRooms = await db.ChatRooms + .Where(c => c.PictureId != null || c.BackgroundId != null) + .ToListAsync(); + + foreach (var chatRoom in chatRooms) + { + if (chatRoom is { PictureId: not null, Picture: null }) + { + var avatarFile = await db.Files.FirstOrDefaultAsync(f => f.Id == chatRoom.PictureId); + if (avatarFile != null) + { + // Create a reference for the avatar file + var reference = new CloudFileReference + { + FileId = avatarFile.Id, + File = avatarFile, + Usage = "chatroom.picture", + ResourceId = chatRoom.Id.ToString() + }; + + await db.FileReferences.AddAsync(reference); + chatRoom.Picture = avatarFile.ToReferenceObject(); + db.ChatRooms.Update(chatRoom); + } + } + + if (chatRoom is not { BackgroundId: not null, Background: null }) continue; + var bannerFile = await db.Files.FirstOrDefaultAsync(f => f.Id == chatRoom.BackgroundId); + if (bannerFile == null) continue; + { + // Create a reference for the banner file + var reference = new CloudFileReference + { + FileId = bannerFile.Id, + File = bannerFile, + Usage = "chatroom.background", + ResourceId = chatRoom.Id.ToString() + }; + + await db.FileReferences.AddAsync(reference); + chatRoom.Background = bannerFile.ToReferenceObject(); + db.ChatRooms.Update(chatRoom); + } + } + + await db.SaveChangesAsync(); + } + + private async Task ScanRealms() + { + var realms = await db.Realms + .Where(r => r.PictureId != null && r.BackgroundId != null) + .ToListAsync(); + + foreach (var realm in realms) + { + // Process avatar if it exists + if (realm is { PictureId: not null, Picture: null }) + { + var avatarFile = await db.Files.FirstOrDefaultAsync(f => f.Id == realm.PictureId); + if (avatarFile != null) + { + // Create a reference for the avatar file + var reference = new CloudFileReference + { + FileId = avatarFile.Id, + File = avatarFile, + Usage = "realm.picture", + ResourceId = realm.Id.ToString() + }; + + await db.FileReferences.AddAsync(reference); + realm.Picture = avatarFile.ToReferenceObject(); + } + } + + // Process banner if it exists + if (realm is { BackgroundId: not null, Background: null }) + { + var bannerFile = await db.Files.FirstOrDefaultAsync(f => f.Id == realm.BackgroundId); + if (bannerFile != null) + { + // Create a reference for the banner file + var reference = new CloudFileReference + { + FileId = bannerFile.Id, + File = bannerFile, + Usage = "realm.background", + ResourceId = realm.Id.ToString() + }; + + await db.FileReferences.AddAsync(reference); + realm.Background = bannerFile.ToReferenceObject(); + } + } + + db.Realms.Update(realm); + } + + await db.SaveChangesAsync(); + } + + private async Task ScanPublishers() + { + var publishers = await db.Publishers + .Where(p => p.PictureId != null || p.BackgroundId != null) + .ToListAsync(); + + foreach (var publisher in publishers) + { + if (publisher is { PictureId: not null, Picture: null }) + { + var pictureFile = await db.Files.FirstOrDefaultAsync(f => f.Id == publisher.PictureId); + if (pictureFile != null) + { + // Create a reference for the picture file + var reference = new CloudFileReference + { + FileId = pictureFile.Id, + File = pictureFile, + Usage = "publisher.picture", + ResourceId = publisher.Id.ToString() + }; + + await db.FileReferences.AddAsync(reference); + publisher.Picture = pictureFile.ToReferenceObject(); + } + } + + if (publisher is { BackgroundId: not null, Background: null }) + { + var backgroundFile = await db.Files.FirstOrDefaultAsync(f => f.Id == publisher.BackgroundId); + if (backgroundFile != null) + { + // Create a reference for the background file + var reference = new CloudFileReference + { + FileId = backgroundFile.Id, + File = backgroundFile, + Usage = "publisher.background", + ResourceId = publisher.Id.ToString() + }; + + await db.FileReferences.AddAsync(reference); + publisher.Background = backgroundFile.ToReferenceObject(); + } + } + + db.Publishers.Update(publisher); + } + + await db.SaveChangesAsync(); + } +} \ No newline at end of file diff --git a/DysonNetwork.Sphere/Storage/ICloudFile.cs b/DysonNetwork.Sphere/Storage/ICloudFile.cs index fcdc9e8..4a10851 100644 --- a/DysonNetwork.Sphere/Storage/ICloudFile.cs +++ b/DysonNetwork.Sphere/Storage/ICloudFile.cs @@ -1,3 +1,5 @@ +using NodaTime; + namespace DysonNetwork.Sphere.Storage; /// @@ -7,6 +9,10 @@ namespace DysonNetwork.Sphere.Storage; /// public interface ICloudFile { + public Instant CreatedAt { get; } + public Instant UpdatedAt { get; } + public Instant? DeletedAt { get; } + /// /// Gets the unique identifier of the cloud file. /// diff --git a/DysonNetwork.Sphere/appsettings.json b/DysonNetwork.Sphere/appsettings.json index 6923269..2c337b5 100644 --- a/DysonNetwork.Sphere/appsettings.json +++ b/DysonNetwork.Sphere/appsettings.json @@ -31,8 +31,19 @@ "StorePath": "Uploads" }, "Storage": { - "PreferredRemote": "cloudflare", + "PreferredRemote": "minio", "Remote": [ + { + "Id": "minio", + "Label": "Minio", + "Region": "auto", + "Bucket": "solar-network-development", + "Endpoint": "localhost:9000", + "SecretId": "littlesheep", + "SecretKey": "password", + "EnabledSigned": true, + "EnableSsl": false + }, { "Id": "cloudflare", "Label": "Cloudflare R2",