DysonNetwork/Swarm
1
1
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

💥 ♻️ Refactor cloud files' references, and loading system

Browse Source
This commit is contained in:
LittleSheep 2025-06-01 19:18:23 +08:00
parent 02ae634690
commit 00229fd406
32 changed files with 5204 additions and 582 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

7
DysonNetwork.Sphere/Account/Account.cs
View File

@ -2,6 +2,7 @@ using System.ComponentModel.DataAnnotations;
using System.ComponentModel.DataAnnotations.Schema;
using System.Text.Json.Serialization;
using DysonNetwork.Sphere.Permission;
using DysonNetwork.Sphere.Storage;
using Microsoft.EntityFrameworkCore;
using NodaTime;
@ -68,10 +69,8 @@ public class Profile : ModelBase
(Experience - Leveling.ExperiencePerLevel[Level]) * 100.0 /
(Leveling.ExperiencePerLevel[Level + 1] - Leveling.ExperiencePerLevel[Level]);
[MaxLength(32)] public string? PictureId { get; set; }
public Storage.CloudFile? Picture { get; set; }
[MaxLength(32)] public string? BackgroundId { get; set; }
public Storage.CloudFile? Background { get; set; }
[Column(TypeName = "jsonb")] public CloudFileReferenceObject? Picture { get; set; }
[Column(TypeName = "jsonb")] public CloudFileReferenceObject? Background { get; set; }
public Guid AccountId { get; set; }
[JsonIgnore] public Account Account { get; set; } = null!;

49
DysonNetwork.Sphere/Account/AccountCurrentController.cs
View File

@ -16,10 +16,13 @@ public class AccountCurrentController(
AppDatabase db,
AccountService accounts,
FileService fs,
FileReferenceService fileRefService,
AccountEventService events,
AuthService auth
) : ControllerBase
{
private const string ProfilePictureFileUsageIdentifier = "profile";
[HttpGet]
[ProducesResponseType<Account>(StatusCodes.Status200OK)]
public async Task<ActionResult<Account>> GetCurrentIdentity()
@ -90,22 +93,52 @@ public class AccountCurrentController(
{
var picture = await db.Files.Where(f => f.Id == request.PictureId).FirstOrDefaultAsync();
if (picture is null) return BadRequest("Invalid picture id, unable to find the file on cloud.");
if (profile.Picture is not null)
await fs.MarkUsageAsync(profile.Picture, -1);
profile.Picture = picture;
await fs.MarkUsageAsync(picture, 1);
var profileResourceId = $"profile:{profile.Id}";
// Remove old references for the profile picture
if (profile.Picture is not null) {
var oldPictureRefs = await fileRefService.GetResourceReferencesAsync(profileResourceId, ProfilePictureFileUsageIdentifier);
foreach (var oldRef in oldPictureRefs)
{
await fileRefService.DeleteReferenceAsync(oldRef.Id);
}
}
profile.Picture = picture.ToReferenceObject();
// Create new reference
await fileRefService.CreateReferenceAsync(
picture.Id,
ProfilePictureFileUsageIdentifier,
profileResourceId
);
}
if (request.BackgroundId is not null)
{
var background = await db.Files.Where(f => f.Id == request.BackgroundId).FirstOrDefaultAsync();
if (background is null) return BadRequest("Invalid background id, unable to find the file on cloud.");
if (profile.Background is not null)
await fs.MarkUsageAsync(profile.Background, -1);
profile.Background = background;
await fs.MarkUsageAsync(background, 1);
var profileResourceId = $"profile:{profile.Id}";
// Remove old references for the profile background
if (profile.Background is not null) {
var oldBackgroundRefs = await fileRefService.GetResourceReferencesAsync(profileResourceId, ProfilePictureFileUsageIdentifier);
foreach (var oldRef in oldBackgroundRefs)
{
await fileRefService.DeleteReferenceAsync(oldRef.Id);
}
}
profile.Background = background.ToReferenceObject();
// Create new reference
await fileRefService.CreateReferenceAsync(
background.Id,
ProfilePictureFileUsageIdentifier,
profileResourceId
);
}
db.Update(profile);

2
DysonNetwork.Sphere/Activity/ActivityService.cs
View File

@ -18,9 +18,7 @@ public class ActivityReaderService(AppDatabase db, PostService ps)
if (postsId.Count > 0)
{
var posts = await db.Posts.Where(e => postsId.Contains(e.Id))
.Include(e => e.ThreadedPost)
.Include(e => e.ForwardedPost)
.Include(e => e.Attachments)
.Include(e => e.Categories)
.Include(e => e.Tags)
.FilterWithVisibility(currentUser, userFriends)

10
DysonNetwork.Sphere/AppDatabase.cs
View File

@ -10,6 +10,11 @@ using Quartz;
namespace DysonNetwork.Sphere;
public interface IIdentifiedResource
{
public string ResourceIdentifier { get; }
}
public abstract class ModelBase
{
public Instant CreatedAt { get; set; }
@ -43,6 +48,7 @@ public class AppDatabase(
public DbSet<Auth.Challenge> AuthChallenges { get; set; }
public DbSet<Storage.CloudFile> Files { get; set; }
public DbSet<Storage.CloudFileReference> FileReferences { get; set; }
public DbSet<Activity.Activity> Activities { get; set; }
@ -178,10 +184,6 @@ public class AppDatabase(
.HasGeneratedTsVectorColumn(p => p.SearchVector, "simple", p => new { p.Title, p.Description, p.Content })
.HasIndex(p => p.SearchVector)
.HasMethod("GIN");
modelBuilder.Entity<Post.Post>()
.HasOne(p => p.ThreadedPost)
.WithOne()
.HasForeignKey<Post.Post>(p => p.ThreadedPostId);
modelBuilder.Entity<Post.Post>()
.HasOne(p => p.RepliedPost)
.WithMany()

3
DysonNetwork.Sphere/Chat/ChatController.cs
View File

@ -92,7 +92,6 @@ public partial class ChatController(AppDatabase db, ChatService cs, ChatRoomServ
.Include(m => m.Sender)
.Include(m => m.Sender.Account)
.Include(m => m.Sender.Account.Profile)
.Include(m => m.Attachments)
.Skip(offset)
.Take(take)
.ToListAsync();
@ -169,6 +168,7 @@ public partial class ChatController(AppDatabase db, ChatService cs, ChatRoomServ
.ToListAsync();
message.Attachments = attachments
.OrderBy(f => request.AttachmentsId.IndexOf(f.Id))
.Select(f => f.ToReferenceObject())
.ToList();
}
@ -270,7 +270,6 @@ public partial class ChatController(AppDatabase db, ChatService cs, ChatRoomServ
var message = await db.ChatMessages
.Include(m => m.Sender)
.Include(m => m.ChatRoom)
.Include(m => m.Attachments)
.FirstOrDefaultAsync(m => m.Id == messageId && m.ChatRoomId == roomId);
if (message == null) return NotFound();

10
DysonNetwork.Sphere/Chat/ChatRoom.cs
View File

@ -12,7 +12,7 @@ public enum ChatRoomType
DirectMessage
}
public class ChatRoom : ModelBase
public class ChatRoom : ModelBase, IIdentifiedResource
{
public Guid Id { get; set; }
[MaxLength(1024)] public string? Name { get; set; }
@ -21,10 +21,8 @@ public class ChatRoom : ModelBase
public bool IsCommunity { get; set; }
public bool IsPublic { get; set; }
[MaxLength(32)] public string? PictureId { get; set; }
public CloudFile? Picture { get; set; }
[MaxLength(32)] public string? BackgroundId { get; set; }
public CloudFile? Background { get; set; }
[Column(TypeName = "jsonb")] public CloudFileReferenceObject? Picture { get; set; }
[Column(TypeName = "jsonb")] public CloudFileReferenceObject? Background { get; set; }
[JsonIgnore] public ICollection<ChatMember> Members { get; set; } = new List<ChatMember>();
@ -35,6 +33,8 @@ public class ChatRoom : ModelBase
[JsonPropertyName("members")]
public ICollection<ChatMemberTransmissionObject> DirectMembers { get; set; } =
new List<ChatMemberTransmissionObject>();
public string ResourceIdentifier => $"chatroom/{Id}";
}
public enum ChatMemberRole

70
DysonNetwork.Sphere/Chat/ChatRoomController.cs
View File

@ -17,6 +17,7 @@ namespace DysonNetwork.Sphere.Chat;
public class ChatRoomController(
AppDatabase db,
FileService fs,
FileReferenceService fileRefService,
ChatRoomService crs,
RealmService rs,
ActionLogService als,
@ -176,13 +177,13 @@ public class ChatRoomController(
if (request.PictureId is not null)
{
chatRoom.Picture = await db.Files.FindAsync(request.PictureId);
chatRoom.Picture = (await db.Files.FindAsync(request.PictureId))?.ToReferenceObject();
if (chatRoom.Picture is null) return BadRequest("Invalid picture id, unable to find the file on cloud.");
}
if (request.BackgroundId is not null)
{
chatRoom.Background = await db.Files.FindAsync(request.BackgroundId);
chatRoom.Background = (await db.Files.FindAsync(request.BackgroundId))?.ToReferenceObject();
if (chatRoom.Background is null)
return BadRequest("Invalid background id, unable to find the file on cloud.");
}
@ -190,10 +191,21 @@ public class ChatRoomController(
db.ChatRooms.Add(chatRoom);
await db.SaveChangesAsync();
var chatRoomResourceId = $"chatroom:{chatRoom.Id}";
if (chatRoom.Picture is not null)
await fs.MarkUsageAsync(chatRoom.Picture, 1);
await fileRefService.CreateReferenceAsync(
chatRoom.Picture.Id,
"chat.room.picture",
chatRoomResourceId
);
if (chatRoom.Background is not null)
await fs.MarkUsageAsync(chatRoom.Background, 1);
await fileRefService.CreateReferenceAsync(
chatRoom.Background.Id,
"chat.room.background",
chatRoomResourceId
);
als.CreateActionLogFromRequest(
ActionLogType.ChatroomCreate,
@ -235,22 +247,50 @@ public class ChatRoomController(
chatRoom.RealmId = member.RealmId;
}
var chatRoomResourceId = $"chatroom:{chatRoom.Id}";
if (request.PictureId is not null)
{
var picture = await db.Files.FindAsync(request.PictureId);
if (picture is null) return BadRequest("Invalid picture id, unable to find the file on cloud.");
await fs.MarkUsageAsync(picture, 1);
if (chatRoom.Picture is not null) await fs.MarkUsageAsync(chatRoom.Picture, -1);
chatRoom.Picture = picture;
// Remove old references for pictures
var oldPictureRefs = await fileRefService.GetResourceReferencesAsync(chatRoomResourceId, "chat.room.picture");
foreach (var oldRef in oldPictureRefs)
{
await fileRefService.DeleteReferenceAsync(oldRef.Id);
}
// Add a new reference
await fileRefService.CreateReferenceAsync(
picture.Id,
"chat.room.picture",
chatRoomResourceId
);
chatRoom.Picture = picture.ToReferenceObject();
}
if (request.BackgroundId is not null)
{
var background = await db.Files.FindAsync(request.BackgroundId);
if (background is null) return BadRequest("Invalid background id, unable to find the file on cloud.");
await fs.MarkUsageAsync(background, 1);
if (chatRoom.Background is not null) await fs.MarkUsageAsync(chatRoom.Background, -1);
chatRoom.Background = background;
// Remove old references for backgrounds
var oldBackgroundRefs = await fileRefService.GetResourceReferencesAsync(chatRoomResourceId, "chat.room.background");
foreach (var oldRef in oldBackgroundRefs)
{
await fileRefService.DeleteReferenceAsync(oldRef.Id);
}
// Add a new reference
await fileRefService.CreateReferenceAsync(
background.Id,
"chat.room.background",
chatRoomResourceId
);
chatRoom.Background = background.ToReferenceObject();
}
if (request.Name is not null)
@ -293,14 +333,14 @@ public class ChatRoomController(
else if (!await crs.IsMemberWithRole(chatRoom.Id, currentUser.Id, ChatMemberRole.Owner))
return StatusCode(403, "You need at least be the owner to delete the chat.");
var chatRoomResourceId = $"chatroom:{chatRoom.Id}";
// Delete all file references for this chat room
await fileRefService.DeleteResourceReferencesAsync(chatRoomResourceId);
db.ChatRooms.Remove(chatRoom);
await db.SaveChangesAsync();
if (chatRoom.Picture is not null)
await fs.MarkUsageAsync(chatRoom.Picture, -1);
if (chatRoom.Background is not null)
await fs.MarkUsageAsync(chatRoom.Background, -1);
als.CreateActionLogFromRequest(
ActionLogType.ChatroomDelete,
new Dictionary<string, object> { { "chatroom_id", chatRoom.Id } }, Request

51
DysonNetwork.Sphere/Chat/ChatService.cs
View File

@ -10,6 +10,7 @@ namespace DysonNetwork.Sphere.Chat;
public class ChatService(
AppDatabase db,
FileService fs,
FileReferenceService fileRefService,
IServiceScopeFactory scopeFactory,
IRealtimeService realtime,
ILogger<ChatService> logger
@ -30,9 +31,16 @@ public class ChatService(
var files = message.Attachments.Distinct().ToList();
if (files.Count != 0)
{
await fs.MarkUsageRangeAsync(files, 1);
await fs.SetExpiresRangeAsync(files, Duration.FromDays(30));
await fs.SetUsageRangeAsync(files, ChatFileUsageIdentifier);
var messageResourceId = $"message:{message.Id}";
foreach (var file in files)
{
await fileRefService.CreateReferenceAsync(
file.Id,
ChatFileUsageIdentifier,
messageResourceId,
duration: Duration.FromDays(30)
);
}
}
// Then start the delivery process
@ -64,7 +72,7 @@ public class ChatService(
{
message.Sender = sender;
message.ChatRoom = room;
using var scope = scopeFactory.CreateScope();
var scopedWs = scope.ServiceProvider.GetRequiredService<WebSocketService>();
var scopedNty = scope.ServiceProvider.GetRequiredService<NotificationService>();
@ -87,8 +95,8 @@ public class ChatService(
.Where(a => a.MimeType != null && a.MimeType.StartsWith("image"))
.Select(a => a.Id).ToList()
};
if (sender.Account.Profile is not { PictureId: null })
metaDict["pfp"] = sender.Account.Profile.PictureId;
if (sender.Account.Profile is not { Picture: null })
metaDict["pfp"] = sender.Account.Profile.Picture.Id;
if (!string.IsNullOrEmpty(room.Name))
metaDict["room_name"] = room.Name;
@ -346,9 +354,28 @@ public class ChatService(
if (attachmentsId is not null)
{
message.Attachments = (await fs.DiffAndMarkFilesAsync(attachmentsId, message.Attachments)).current;
await fs.DiffAndSetExpiresAsync(attachmentsId, Duration.FromDays(30), message.Attachments);
await fs.DiffAndSetUsageAsync(attachmentsId, ChatFileUsageIdentifier, message.Attachments);
var messageResourceId = $"message:{message.Id}";
// Delete existing references for this message
await fileRefService.DeleteResourceReferencesAsync(messageResourceId);
// Create new references for each attachment
foreach (var fileId in attachmentsId)
{
await fileRefService.CreateReferenceAsync(
fileId,
ChatFileUsageIdentifier,
messageResourceId,
duration: Duration.FromDays(30)
);
}
// Update message attachments by getting files from database
var files = await db.Files
.Where(f => attachmentsId.Contains(f.Id))
.ToListAsync();
message.Attachments = files.Select(x => x.ToReferenceObject()).ToList();
}
message.EditedAt = SystemClock.Instance.GetCurrentInstant();
@ -371,9 +398,9 @@ public class ChatService(
/// <param name="message">The message to delete</param>
public async Task DeleteMessageAsync(Message message)
{
var files = message.Attachments.Distinct().ToList();
if (files.Count != 0)
await fs.MarkUsageRangeAsync(files, -1);
// Remove all file references for this message
var messageResourceId = $"message:{message.Id}";
await fileRefService.DeleteResourceReferencesAsync(messageResourceId);
db.ChatMessages.Remove(message);
await db.SaveChangesAsync();

32
DysonNetwork.Sphere/Chat/Message.cs
View File

@ -7,7 +7,7 @@ using NodaTime;
namespace DysonNetwork.Sphere.Chat;
public class Message : ModelBase
public class Message : ModelBase, IIdentifiedResource
{
public Guid Id { get; set; } = Guid.NewGuid();
[MaxLength(1024)] public string Type { get; set; } = null!;
@ -17,7 +17,7 @@ public class Message : ModelBase
[MaxLength(36)] public string Nonce { get; set; } = null!;
public Instant? EditedAt { get; set; }
public ICollection<CloudFile> Attachments { get; set; } = new List<CloudFile>();
[Column(TypeName = "jsonb")] public List<CloudFileReferenceObject> Attachments { get; set; } = [];
public ICollection<MessageReaction> Reactions { get; set; } = new List<MessageReaction>();
public Guid? RepliedMessageId { get; set; }
@ -29,32 +29,8 @@ public class Message : ModelBase
public ChatMember Sender { get; set; } = null!;
public Guid ChatRoomId { get; set; }
[JsonIgnore] public ChatRoom ChatRoom { get; set; } = null!;
public Message Clone()
{
return new Message
{
Id = Id,
Content = Content,
Meta = Meta?.ToDictionary(entry => entry.Key, entry => entry.Value),
MembersMentioned = MembersMentioned?.ToList(),
Nonce = Nonce,
EditedAt = EditedAt,
Attachments = new List<CloudFile>(Attachments),
Reactions = new List<MessageReaction>(Reactions),
RepliedMessageId = RepliedMessageId,
RepliedMessage = RepliedMessage?.Clone() as Message,
ForwardedMessageId = ForwardedMessageId,
ForwardedMessage = ForwardedMessage?.Clone() as Message,
SenderId = SenderId,
Sender = Sender,
ChatRoomId = ChatRoomId,
ChatRoom = ChatRoom,
CreatedAt = CreatedAt,
UpdatedAt = UpdatedAt,
DeletedAt = DeletedAt
};
}
public string ResourceIdentifier => $"message/{Id}";
}
public enum MessageReactionAttitude

3328
DysonNetwork.Sphere/Migrations/20250601111032_RefactorCloudFileReference.Designer.cs generated Normal file
View File

File diff suppressed because it is too large Load Diff

557
DysonNetwork.Sphere/Migrations/20250601111032_RefactorCloudFileReference.cs Normal file
View File

@ -0,0 +1,557 @@
using System;
using System.Collections.Generic;
using DysonNetwork.Sphere.Storage;
using Microsoft.EntityFrameworkCore.Migrations;
using NodaTime;
#nullable disable
namespace DysonNetwork.Sphere.Migrations
{
/// <inheritdoc />
public partial class RefactorCloudFileReference : Migration
{
/// <inheritdoc />
protected override void Up(MigrationBuilder migrationBuilder)
{
migrationBuilder.DropForeignKey(
name: "fk_account_profiles_files_background_id",
table: "account_profiles");
migrationBuilder.DropForeignKey(
name: "fk_account_profiles_files_picture_id",
table: "account_profiles");
migrationBuilder.DropForeignKey(
name: "fk_chat_rooms_files_background_id",
table: "chat_rooms");
migrationBuilder.DropForeignKey(
name: "fk_chat_rooms_files_picture_id",
table: "chat_rooms");
migrationBuilder.DropForeignKey(
name: "fk_files_chat_messages_message_id",
table: "files");
migrationBuilder.DropForeignKey(
name: "fk_files_posts_post_id",
table: "files");
migrationBuilder.DropForeignKey(
name: "fk_posts_posts_threaded_post_id",
table: "posts");
migrationBuilder.DropForeignKey(
name: "fk_publishers_files_background_id",
table: "publishers");
migrationBuilder.DropForeignKey(
name: "fk_publishers_files_picture_id",
table: "publishers");
migrationBuilder.DropForeignKey(
name: "fk_realms_files_background_id",
table: "realms");
migrationBuilder.DropForeignKey(
name: "fk_realms_files_picture_id",
table: "realms");
migrationBuilder.DropForeignKey(
name: "fk_stickers_files_image_id",
table: "stickers");
migrationBuilder.DropIndex(
name: "ix_stickers_image_id",
table: "stickers");
migrationBuilder.DropIndex(
name: "ix_realms_background_id",
table: "realms");
migrationBuilder.DropIndex(
name: "ix_realms_picture_id",
table: "realms");
migrationBuilder.DropIndex(
name: "ix_publishers_background_id",
table: "publishers");
migrationBuilder.DropIndex(
name: "ix_publishers_picture_id",
table: "publishers");
migrationBuilder.DropIndex(
name: "ix_posts_threaded_post_id",
table: "posts");
migrationBuilder.DropIndex(
name: "ix_files_message_id",
table: "files");
migrationBuilder.DropIndex(
name: "ix_files_post_id",
table: "files");
migrationBuilder.DropIndex(
name: "ix_chat_rooms_background_id",
table: "chat_rooms");
migrationBuilder.DropIndex(
name: "ix_chat_rooms_picture_id",
table: "chat_rooms");
migrationBuilder.DropIndex(
name: "ix_account_profiles_background_id",
table: "account_profiles");
migrationBuilder.DropIndex(
name: "ix_account_profiles_picture_id",
table: "account_profiles");
migrationBuilder.DropColumn(
name: "background_id",
table: "realms");
migrationBuilder.DropColumn(
name: "picture_id",
table: "realms");
migrationBuilder.DropColumn(
name: "background_id",
table: "publishers");
migrationBuilder.DropColumn(
name: "picture_id",
table: "publishers");
migrationBuilder.DropColumn(
name: "threaded_post_id",
table: "posts");
// TODO Move these following changes to next migrations after migrated all the attachments data
// migrationBuilder.DropColumn(
// name: "expired_at",
// table: "files");
//
// migrationBuilder.DropColumn(
// name: "message_id",
// table: "files");
//
// migrationBuilder.DropColumn(
// name: "post_id",
// table: "files");
//
// migrationBuilder.DropColumn(
// name: "usage",
// table: "files");
//
// migrationBuilder.DropColumn(
// name: "used_count",
// table: "files");
//
// migrationBuilder.DropColumn(
// name: "background_id",
// table: "chat_rooms");
//
// migrationBuilder.DropColumn(
// name: "picture_id",
// table: "chat_rooms");
//
// migrationBuilder.DropColumn(
// name: "background_id",
// table: "account_profiles");
//
// migrationBuilder.DropColumn(
// name: "picture_id",
// table: "account_profiles");
migrationBuilder.AddColumn<CloudFileReferenceObject>(
name: "image",
table: "stickers",
type: "jsonb",
nullable: false);
migrationBuilder.AddColumn<CloudFileReferenceObject>(
name: "background",
table: "realms",
type: "jsonb",
nullable: true);
migrationBuilder.AddColumn<CloudFileReferenceObject>(
name: "picture",
table: "realms",
type: "jsonb",
nullable: true);
migrationBuilder.AddColumn<CloudFileReferenceObject>(
name: "background",
table: "publishers",
type: "jsonb",
nullable: true);
migrationBuilder.AddColumn<CloudFileReferenceObject>(
name: "picture",
table: "publishers",
type: "jsonb",
nullable: true);
migrationBuilder.AddColumn<List<CloudFileReferenceObject>>(
name: "attachments",
table: "posts",
type: "jsonb",
nullable: false);
migrationBuilder.AddColumn<CloudFileReferenceObject>(
name: "background",
table: "chat_rooms",
type: "jsonb",
nullable: true);
migrationBuilder.AddColumn<CloudFileReferenceObject>(
name: "picture",
table: "chat_rooms",
type: "jsonb",
nullable: true);
migrationBuilder.AddColumn<List<CloudFileReferenceObject>>(
name: "attachments",
table: "chat_messages",
type: "jsonb",
nullable: false);
migrationBuilder.AddColumn<CloudFileReferenceObject>(
name: "background",
table: "account_profiles",
type: "jsonb",
nullable: true);
migrationBuilder.AddColumn<CloudFileReferenceObject>(
name: "picture",
table: "account_profiles",
type: "jsonb",
nullable: true);
migrationBuilder.CreateTable(
name: "file_references",
columns: table => new
{
id = table.Column<Guid>(type: "uuid", nullable: false),
file_id = table.Column<string>(type: "character varying(32)", maxLength: 32, nullable: false),
usage = table.Column<string>(type: "character varying(1024)", maxLength: 1024, nullable: false),
resource_id = table.Column<string>(type: "character varying(1024)", maxLength: 1024, nullable: false),
expired_at = table.Column<Instant>(type: "timestamp with time zone", nullable: true),
created_at = table.Column<Instant>(type: "timestamp with time zone", nullable: false),
updated_at = table.Column<Instant>(type: "timestamp with time zone", nullable: false),
deleted_at = table.Column<Instant>(type: "timestamp with time zone", nullable: true)
},
constraints: table =>
{
table.PrimaryKey("pk_file_references", x => x.id);
table.ForeignKey(
name: "fk_file_references_files_file_id",
column: x => x.file_id,
principalTable: "files",
principalColumn: "id",
onDelete: ReferentialAction.Cascade);
});
migrationBuilder.CreateIndex(
name: "ix_file_references_file_id",
table: "file_references",
column: "file_id");
}
/// <inheritdoc />
protected override void Down(MigrationBuilder migrationBuilder)
{
migrationBuilder.DropTable(
name: "file_references");
migrationBuilder.DropColumn(
name: "image",
table: "stickers");
migrationBuilder.DropColumn(
name: "background",
table: "realms");
migrationBuilder.DropColumn(
name: "picture",
table: "realms");
migrationBuilder.DropColumn(
name: "background",
table: "publishers");
migrationBuilder.DropColumn(
name: "picture",
table: "publishers");
migrationBuilder.DropColumn(
name: "attachments",
table: "posts");
migrationBuilder.DropColumn(
name: "background",
table: "chat_rooms");
migrationBuilder.DropColumn(
name: "picture",
table: "chat_rooms");
migrationBuilder.DropColumn(
name: "attachments",
table: "chat_messages");
migrationBuilder.DropColumn(
name: "background",
table: "account_profiles");
migrationBuilder.DropColumn(
name: "picture",
table: "account_profiles");
migrationBuilder.AddColumn<string>(
name: "background_id",
table: "realms",
type: "character varying(32)",
maxLength: 32,
nullable: true);
migrationBuilder.AddColumn<string>(
name: "picture_id",
table: "realms",
type: "character varying(32)",
maxLength: 32,
nullable: true);
migrationBuilder.AddColumn<string>(
name: "background_id",
table: "publishers",
type: "character varying(32)",
maxLength: 32,
nullable: true);
migrationBuilder.AddColumn<string>(
name: "picture_id",
table: "publishers",
type: "character varying(32)",
maxLength: 32,
nullable: true);
migrationBuilder.AddColumn<Guid>(
name: "threaded_post_id",
table: "posts",
type: "uuid",
nullable: true);
migrationBuilder.AddColumn<Instant>(
name: "expired_at",
table: "files",
type: "timestamp with time zone",
nullable: true);
migrationBuilder.AddColumn<Guid>(
name: "message_id",
table: "files",
type: "uuid",
nullable: true);
migrationBuilder.AddColumn<Guid>(
name: "post_id",
table: "files",
type: "uuid",
nullable: true);
migrationBuilder.AddColumn<string>(
name: "usage",
table: "files",
type: "character varying(1024)",
maxLength: 1024,
nullable: true);
migrationBuilder.AddColumn<int>(
name: "used_count",
table: "files",
type: "integer",
nullable: false,
defaultValue: 0);
migrationBuilder.AddColumn<string>(
name: "background_id",
table: "chat_rooms",
type: "character varying(32)",
maxLength: 32,
nullable: true);
migrationBuilder.AddColumn<string>(
name: "picture_id",
table: "chat_rooms",
type: "character varying(32)",
maxLength: 32,
nullable: true);
migrationBuilder.AddColumn<string>(
name: "background_id",
table: "account_profiles",
type: "character varying(32)",
maxLength: 32,
nullable: true);
migrationBuilder.AddColumn<string>(
name: "picture_id",
table: "account_profiles",
type: "character varying(32)",
maxLength: 32,
nullable: true);
migrationBuilder.CreateIndex(
name: "ix_stickers_image_id",
table: "stickers",
column: "image_id");
migrationBuilder.CreateIndex(
name: "ix_realms_background_id",
table: "realms",
column: "background_id");
migrationBuilder.CreateIndex(
name: "ix_realms_picture_id",
table: "realms",
column: "picture_id");
migrationBuilder.CreateIndex(
name: "ix_publishers_background_id",
table: "publishers",
column: "background_id");
migrationBuilder.CreateIndex(
name: "ix_publishers_picture_id",
table: "publishers",
column: "picture_id");
migrationBuilder.CreateIndex(
name: "ix_posts_threaded_post_id",
table: "posts",
column: "threaded_post_id",
unique: true);
migrationBuilder.CreateIndex(
name: "ix_files_message_id",
table: "files",
column: "message_id");
migrationBuilder.CreateIndex(
name: "ix_files_post_id",
table: "files",
column: "post_id");
migrationBuilder.CreateIndex(
name: "ix_chat_rooms_background_id",
table: "chat_rooms",
column: "background_id");
migrationBuilder.CreateIndex(
name: "ix_chat_rooms_picture_id",
table: "chat_rooms",
column: "picture_id");
migrationBuilder.CreateIndex(
name: "ix_account_profiles_background_id",
table: "account_profiles",
column: "background_id");
migrationBuilder.CreateIndex(
name: "ix_account_profiles_picture_id",
table: "account_profiles",
column: "picture_id");
migrationBuilder.AddForeignKey(
name: "fk_account_profiles_files_background_id",
table: "account_profiles",
column: "background_id",
principalTable: "files",
principalColumn: "id");
migrationBuilder.AddForeignKey(
name: "fk_account_profiles_files_picture_id",
table: "account_profiles",
column: "picture_id",
principalTable: "files",
principalColumn: "id");
migrationBuilder.AddForeignKey(
name: "fk_chat_rooms_files_background_id",
table: "chat_rooms",
column: "background_id",
principalTable: "files",
principalColumn: "id");
migrationBuilder.AddForeignKey(
name: "fk_chat_rooms_files_picture_id",
table: "chat_rooms",
column: "picture_id",
principalTable: "files",
principalColumn: "id");
migrationBuilder.AddForeignKey(
name: "fk_files_chat_messages_message_id",
table: "files",
column: "message_id",
principalTable: "chat_messages",
principalColumn: "id");
migrationBuilder.AddForeignKey(
name: "fk_files_posts_post_id",
table: "files",
column: "post_id",
principalTable: "posts",
principalColumn: "id");
migrationBuilder.AddForeignKey(
name: "fk_posts_posts_threaded_post_id",
table: "posts",
column: "threaded_post_id",
principalTable: "posts",
principalColumn: "id");
migrationBuilder.AddForeignKey(
name: "fk_publishers_files_background_id",
table: "publishers",
column: "background_id",
principalTable: "files",
principalColumn: "id");
migrationBuilder.AddForeignKey(
name: "fk_publishers_files_picture_id",
table: "publishers",
column: "picture_id",
principalTable: "files",
principalColumn: "id");
migrationBuilder.AddForeignKey(
name: "fk_realms_files_background_id",
table: "realms",
column: "background_id",
principalTable: "files",
principalColumn: "id");
migrationBuilder.AddForeignKey(
name: "fk_realms_files_picture_id",
table: "realms",
column: "picture_id",
principalTable: "files",
principalColumn: "id");
migrationBuilder.AddForeignKey(
name: "fk_stickers_files_image_id",
table: "stickers",
column: "image_id",
principalTable: "files",
principalColumn: "id",
onDelete: ReferentialAction.Cascade);
}
}
}

281
DysonNetwork.Sphere/Migrations/AppDatabaseModelSnapshot.cs
View File

@ -533,10 +533,9 @@ namespace DysonNetwork.Sphere.Migrations
.HasColumnType("uuid")
.HasColumnName("account_id");
b.Property<string>("BackgroundId")
.HasMaxLength(32)
.HasColumnType("character varying(32)")
.HasColumnName("background_id");
b.Property<CloudFileReferenceObject>("Background")
.HasColumnType("jsonb")
.HasColumnName("background");
b.Property<string>("Bio")
.HasMaxLength(4096)
@ -583,10 +582,9 @@ namespace DysonNetwork.Sphere.Migrations
.HasColumnType("character varying(256)")
.HasColumnName("middle_name");
b.Property<string>("PictureId")
.HasMaxLength(32)
.HasColumnType("character varying(32)")
.HasColumnName("picture_id");
b.Property<CloudFileReferenceObject>("Picture")
.HasColumnType("jsonb")
.HasColumnName("picture");
b.Property<string>("Pronouns")
.HasMaxLength(1024)
@ -604,12 +602,6 @@ namespace DysonNetwork.Sphere.Migrations
.IsUnique()
.HasDatabaseName("ix_account_profiles_account_id");
b.HasIndex("BackgroundId")
.HasDatabaseName("ix_account_profiles_background_id");
b.HasIndex("PictureId")
.HasDatabaseName("ix_account_profiles_picture_id");
b.ToTable("account_profiles", (string)null);
});
@ -985,10 +977,9 @@ namespace DysonNetwork.Sphere.Migrations
.HasColumnType("uuid")
.HasColumnName("id");
b.Property<string>("BackgroundId")
.HasMaxLength(32)
.HasColumnType("character varying(32)")
.HasColumnName("background_id");
b.Property<CloudFileReferenceObject>("Background")
.HasColumnType("jsonb")
.HasColumnName("background");
b.Property<Instant>("CreatedAt")
.HasColumnType("timestamp with time zone")
@ -1016,10 +1007,9 @@ namespace DysonNetwork.Sphere.Migrations
.HasColumnType("character varying(1024)")
.HasColumnName("name");
b.Property<string>("PictureId")
.HasMaxLength(32)
.HasColumnType("character varying(32)")
.HasColumnName("picture_id");
b.Property<CloudFileReferenceObject>("Picture")
.HasColumnType("jsonb")
.HasColumnName("picture");
b.Property<Guid?>("RealmId")
.HasColumnType("uuid")
@ -1036,12 +1026,6 @@ namespace DysonNetwork.Sphere.Migrations
b.HasKey("Id")
.HasName("pk_chat_rooms");
b.HasIndex("BackgroundId")
.HasDatabaseName("ix_chat_rooms_background_id");
b.HasIndex("PictureId")
.HasDatabaseName("ix_chat_rooms_picture_id");
b.HasIndex("RealmId")
.HasDatabaseName("ix_chat_rooms_realm_id");
@ -1055,6 +1039,11 @@ namespace DysonNetwork.Sphere.Migrations
.HasColumnType("uuid")
.HasColumnName("id");
b.Property<List<CloudFileReferenceObject>>("Attachments")
.IsRequired()
.HasColumnType("jsonb")
.HasColumnName("attachments");
b.Property<Guid>("ChatRoomId")
.HasColumnType("uuid")
.HasColumnName("chat_room_id");
@ -1479,6 +1468,11 @@ namespace DysonNetwork.Sphere.Migrations
.HasColumnType("uuid")
.HasColumnName("id");
b.Property<List<CloudFileReferenceObject>>("Attachments")
.IsRequired()
.HasColumnType("jsonb")
.HasColumnName("attachments");
b.Property<string>("Content")
.HasColumnType("text")
.HasColumnName("content");
@ -1537,10 +1531,6 @@ namespace DysonNetwork.Sphere.Migrations
.HasAnnotation("Npgsql:TsVectorConfig", "simple")
.HasAnnotation("Npgsql:TsVectorProperties", new[] { "Title", "Description", "Content" });
b.Property<Guid?>("ThreadedPostId")
.HasColumnType("uuid")
.HasColumnName("threaded_post_id");
b.Property<string>("Title")
.HasMaxLength(1024)
.HasColumnType("character varying(1024)")
@ -1587,10 +1577,6 @@ namespace DysonNetwork.Sphere.Migrations
NpgsqlIndexBuilderExtensions.HasMethod(b.HasIndex("SearchVector"), "GIN");
b.HasIndex("ThreadedPostId")
.IsUnique()
.HasDatabaseName("ix_posts_threaded_post_id");
b.ToTable("posts", (string)null);
});
@ -1774,10 +1760,9 @@ namespace DysonNetwork.Sphere.Migrations
.HasColumnType("uuid")
.HasColumnName("account_id");
b.Property<string>("BackgroundId")
.HasMaxLength(32)
.HasColumnType("character varying(32)")
.HasColumnName("background_id");
b.Property<CloudFileReferenceObject>("Background")
.HasColumnType("jsonb")
.HasColumnName("background");
b.Property<string>("Bio")
.HasMaxLength(4096)
@ -1804,10 +1789,9 @@ namespace DysonNetwork.Sphere.Migrations
.HasColumnType("character varying(256)")
.HasColumnName("nick");
b.Property<string>("PictureId")
.HasMaxLength(32)
.HasColumnType("character varying(32)")
.HasColumnName("picture_id");
b.Property<CloudFileReferenceObject>("Picture")
.HasColumnType("jsonb")
.HasColumnName("picture");
b.Property<Guid?>("RealmId")
.HasColumnType("uuid")
@ -1827,16 +1811,10 @@ namespace DysonNetwork.Sphere.Migrations
b.HasIndex("AccountId")
.HasDatabaseName("ix_publishers_account_id");
b.HasIndex("BackgroundId")
.HasDatabaseName("ix_publishers_background_id");
b.HasIndex("Name")
.IsUnique()
.HasDatabaseName("ix_publishers_name");
b.HasIndex("PictureId")
.HasDatabaseName("ix_publishers_picture_id");
b.HasIndex("RealmId")
.HasDatabaseName("ix_publishers_realm_id");
@ -1982,10 +1960,9 @@ namespace DysonNetwork.Sphere.Migrations
.HasColumnType("uuid")
.HasColumnName("account_id");
b.Property<string>("BackgroundId")
.HasMaxLength(32)
.HasColumnType("character varying(32)")
.HasColumnName("background_id");
b.Property<CloudFileReferenceObject>("Background")
.HasColumnType("jsonb")
.HasColumnName("background");
b.Property<Instant>("CreatedAt")
.HasColumnType("timestamp with time zone")
@ -2015,10 +1992,9 @@ namespace DysonNetwork.Sphere.Migrations
.HasColumnType("character varying(1024)")
.HasColumnName("name");
b.Property<string>("PictureId")
.HasMaxLength(32)
.HasColumnType("character varying(32)")
.HasColumnName("picture_id");
b.Property<CloudFileReferenceObject>("Picture")
.HasColumnType("jsonb")
.HasColumnName("picture");
b.Property<string>("Slug")
.IsRequired()
@ -2045,12 +2021,6 @@ namespace DysonNetwork.Sphere.Migrations
b.HasIndex("AccountId")
.HasDatabaseName("ix_realms_account_id");
b.HasIndex("BackgroundId")
.HasDatabaseName("ix_realms_background_id");
b.HasIndex("PictureId")
.HasDatabaseName("ix_realms_picture_id");
b.HasIndex("Slug")
.IsUnique()
.HasDatabaseName("ix_realms_slug");
@ -2116,6 +2086,11 @@ namespace DysonNetwork.Sphere.Migrations
.HasColumnType("timestamp with time zone")
.HasColumnName("deleted_at");
b.Property<CloudFileReferenceObject>("Image")
.IsRequired()
.HasColumnType("jsonb")
.HasColumnName("image");
b.Property<string>("ImageId")
.IsRequired()
.HasMaxLength(32)
@ -2139,9 +2114,6 @@ namespace DysonNetwork.Sphere.Migrations
b.HasKey("Id")
.HasName("pk_stickers");
b.HasIndex("ImageId")
.HasDatabaseName("ix_stickers_image_id");
b.HasIndex("PackId")
.HasDatabaseName("ix_stickers_pack_id");
@ -2229,10 +2201,6 @@ namespace DysonNetwork.Sphere.Migrations
.HasColumnType("character varying(4096)")
.HasColumnName("description");
b.Property<Instant?>("ExpiredAt")
.HasColumnType("timestamp with time zone")
.HasColumnName("expired_at");
b.Property<Dictionary<string, object>>("FileMeta")
.HasColumnType("jsonb")
.HasColumnName("file_meta");
@ -2246,10 +2214,6 @@ namespace DysonNetwork.Sphere.Migrations
.HasColumnType("character varying(256)")
.HasColumnName("hash");
b.Property<Guid?>("MessageId")
.HasColumnType("uuid")
.HasColumnName("message_id");
b.Property<string>("MimeType")
.HasMaxLength(256)
.HasColumnType("character varying(256)")
@ -2261,10 +2225,6 @@ namespace DysonNetwork.Sphere.Migrations
.HasColumnType("character varying(1024)")
.HasColumnName("name");
b.Property<Guid?>("PostId")
.HasColumnType("uuid")
.HasColumnName("post_id");
b.Property<List<CloudFileSensitiveMark>>("SensitiveMarks")
.HasColumnType("jsonb")
.HasColumnName("sensitive_marks");
@ -2296,15 +2256,6 @@ namespace DysonNetwork.Sphere.Migrations
.HasColumnType("character varying(128)")
.HasColumnName("uploaded_to");
b.Property<string>("Usage")
.HasMaxLength(1024)
.HasColumnType("character varying(1024)")
.HasColumnName("usage");
b.Property<int>("UsedCount")
.HasColumnType("integer")
.HasColumnName("used_count");
b.Property<Dictionary<string, object>>("UserMeta")
.HasColumnType("jsonb")
.HasColumnName("user_meta");
@ -2315,15 +2266,59 @@ namespace DysonNetwork.Sphere.Migrations
b.HasIndex("AccountId")
.HasDatabaseName("ix_files_account_id");
b.HasIndex("MessageId")
.HasDatabaseName("ix_files_message_id");
b.HasIndex("PostId")
.HasDatabaseName("ix_files_post_id");
b.ToTable("files", (string)null);
});
modelBuilder.Entity("DysonNetwork.Sphere.Storage.CloudFileReference", b =>
{
b.Property<Guid>("Id")
.ValueGeneratedOnAdd()
.HasColumnType("uuid")
.HasColumnName("id");
b.Property<Instant>("CreatedAt")
.HasColumnType("timestamp with time zone")
.HasColumnName("created_at");
b.Property<Instant?>("DeletedAt")
.HasColumnType("timestamp with time zone")
.HasColumnName("deleted_at");
b.Property<Instant?>("ExpiredAt")
.HasColumnType("timestamp with time zone")
.HasColumnName("expired_at");
b.Property<string>("FileId")
.IsRequired()
.HasMaxLength(32)
.HasColumnType("character varying(32)")
.HasColumnName("file_id");
b.Property<string>("ResourceId")
.IsRequired()
.HasMaxLength(1024)
.HasColumnType("character varying(1024)")
.HasColumnName("resource_id");
b.Property<Instant>("UpdatedAt")
.HasColumnType("timestamp with time zone")
.HasColumnName("updated_at");
b.Property<string>("Usage")
.IsRequired()
.HasMaxLength(1024)
.HasColumnType("character varying(1024)")
.HasColumnName("usage");
b.HasKey("Id")
.HasName("pk_file_references");
b.HasIndex("FileId")
.HasDatabaseName("ix_file_references_file_id");
b.ToTable("file_references", (string)null);
});
modelBuilder.Entity("DysonNetwork.Sphere.Wallet.Order", b =>
{
b.Property<Guid>("Id")
@ -2692,21 +2687,7 @@ namespace DysonNetwork.Sphere.Migrations
.IsRequired()
.HasConstraintName("fk_account_profiles_accounts_account_id");
b.HasOne("DysonNetwork.Sphere.Storage.CloudFile", "Background")
.WithMany()
.HasForeignKey("BackgroundId")
.HasConstraintName("fk_account_profiles_files_background_id");
b.HasOne("DysonNetwork.Sphere.Storage.CloudFile", "Picture")
.WithMany()
.HasForeignKey("PictureId")
.HasConstraintName("fk_account_profiles_files_picture_id");
b.Navigation("Account");
b.Navigation("Background");
b.Navigation("Picture");
});
modelBuilder.Entity("DysonNetwork.Sphere.Account.Relationship", b =>
@ -2810,25 +2791,11 @@ namespace DysonNetwork.Sphere.Migrations
modelBuilder.Entity("DysonNetwork.Sphere.Chat.ChatRoom", b =>
{
b.HasOne("DysonNetwork.Sphere.Storage.CloudFile", "Background")
.WithMany()
.HasForeignKey("BackgroundId")
.HasConstraintName("fk_chat_rooms_files_background_id");
b.HasOne("DysonNetwork.Sphere.Storage.CloudFile", "Picture")
.WithMany()
.HasForeignKey("PictureId")
.HasConstraintName("fk_chat_rooms_files_picture_id");
b.HasOne("DysonNetwork.Sphere.Realm.Realm", "Realm")
.WithMany("ChatRooms")
.HasForeignKey("RealmId")
.HasConstraintName("fk_chat_rooms_realms_realm_id");
b.Navigation("Background");
b.Navigation("Picture");
b.Navigation("Realm");
});
@ -2978,18 +2945,11 @@ namespace DysonNetwork.Sphere.Migrations
.OnDelete(DeleteBehavior.Restrict)
.HasConstraintName("fk_posts_posts_replied_post_id");
b.HasOne("DysonNetwork.Sphere.Post.Post", "ThreadedPost")
.WithOne()
.HasForeignKey("DysonNetwork.Sphere.Post.Post", "ThreadedPostId")
.HasConstraintName("fk_posts_posts_threaded_post_id");
b.Navigation("ForwardedPost");
b.Navigation("Publisher");
b.Navigation("RepliedPost");
b.Navigation("ThreadedPost");
});
modelBuilder.Entity("DysonNetwork.Sphere.Post.PostCollection", b =>
@ -3032,16 +2992,6 @@ namespace DysonNetwork.Sphere.Migrations
.HasForeignKey("AccountId")
.HasConstraintName("fk_publishers_accounts_account_id");
b.HasOne("DysonNetwork.Sphere.Storage.CloudFile", "Background")
.WithMany()
.HasForeignKey("BackgroundId")
.HasConstraintName("fk_publishers_files_background_id");
b.HasOne("DysonNetwork.Sphere.Storage.CloudFile", "Picture")
.WithMany()
.HasForeignKey("PictureId")
.HasConstraintName("fk_publishers_files_picture_id");
b.HasOne("DysonNetwork.Sphere.Realm.Realm", "Realm")
.WithMany()
.HasForeignKey("RealmId")
@ -3049,10 +2999,6 @@ namespace DysonNetwork.Sphere.Migrations
b.Navigation("Account");
b.Navigation("Background");
b.Navigation("Picture");
b.Navigation("Realm");
});
@ -3119,21 +3065,7 @@ namespace DysonNetwork.Sphere.Migrations
.IsRequired()
.HasConstraintName("fk_realms_accounts_account_id");
b.HasOne("DysonNetwork.Sphere.Storage.CloudFile", "Background")
.WithMany()
.HasForeignKey("BackgroundId")
.HasConstraintName("fk_realms_files_background_id");
b.HasOne("DysonNetwork.Sphere.Storage.CloudFile", "Picture")
.WithMany()
.HasForeignKey("PictureId")
.HasConstraintName("fk_realms_files_picture_id");
b.Navigation("Account");
b.Navigation("Background");
b.Navigation("Picture");
});
modelBuilder.Entity("DysonNetwork.Sphere.Realm.RealmMember", b =>
@ -3159,13 +3091,6 @@ namespace DysonNetwork.Sphere.Migrations
modelBuilder.Entity("DysonNetwork.Sphere.Sticker.Sticker", b =>
{
b.HasOne("DysonNetwork.Sphere.Storage.CloudFile", "Image")
.WithMany()
.HasForeignKey("ImageId")
.OnDelete(DeleteBehavior.Cascade)
.IsRequired()
.HasConstraintName("fk_stickers_files_image_id");
b.HasOne("DysonNetwork.Sphere.Sticker.StickerPack", "Pack")
.WithMany()
.HasForeignKey("PackId")
@ -3173,8 +3098,6 @@ namespace DysonNetwork.Sphere.Migrations
.IsRequired()
.HasConstraintName("fk_stickers_sticker_packs_pack_id");
b.Navigation("Image");
b.Navigation("Pack");
});
@ -3199,19 +3122,21 @@ namespace DysonNetwork.Sphere.Migrations
.IsRequired()
.HasConstraintName("fk_files_accounts_account_id");
b.HasOne("DysonNetwork.Sphere.Chat.Message", null)
.WithMany("Attachments")
.HasForeignKey("MessageId")
.HasConstraintName("fk_files_chat_messages_message_id");
b.HasOne("DysonNetwork.Sphere.Post.Post", null)
.WithMany("Attachments")
.HasForeignKey("PostId")
.HasConstraintName("fk_files_posts_post_id");
b.Navigation("Account");
});
modelBuilder.Entity("DysonNetwork.Sphere.Storage.CloudFileReference", b =>
{
b.HasOne("DysonNetwork.Sphere.Storage.CloudFile", "File")
.WithMany()
.HasForeignKey("FileId")
.OnDelete(DeleteBehavior.Cascade)
.IsRequired()
.HasConstraintName("fk_file_references_files_file_id");
b.Navigation("File");
});
modelBuilder.Entity("DysonNetwork.Sphere.Wallet.Order", b =>
{
b.HasOne("DysonNetwork.Sphere.Developer.CustomApp", "IssuerApp")
@ -3357,8 +3282,6 @@ namespace DysonNetwork.Sphere.Migrations
modelBuilder.Entity("DysonNetwork.Sphere.Chat.Message", b =>
{
b.Navigation("Attachments");
b.Navigation("Reactions");
});
@ -3371,8 +3294,6 @@ namespace DysonNetwork.Sphere.Migrations
modelBuilder.Entity("DysonNetwork.Sphere.Post.Post", b =>
{
b.Navigation("Attachments");
b.Navigation("Reactions");
});

12
DysonNetwork.Sphere/Post/Post.cs
View File

@ -22,7 +22,7 @@ public enum PostVisibility
Private
}
public class Post : ModelBase
public class Post : ModelBase, IIdentifiedResource
{
public Guid Id { get; set; }
[MaxLength(1024)] public string? Title { get; set; }
@ -31,7 +31,7 @@ public class Post : ModelBase
public Instant? EditedAt { get; set; }
public Instant? PublishedAt { get; set; }
public PostVisibility Visibility { get; set; } = PostVisibility.Public;
// ReSharper disable once EntityFramework.ModelValidation.UnlimitedStringLength
public string? Content { get; set; }
@ -44,19 +44,17 @@ public class Post : ModelBase
public int Downvotes { get; set; }
[NotMapped] public Dictionary<string, int> ReactionsCount { get; set; } = new();
public Guid? ThreadedPostId { get; set; }
public Post? ThreadedPost { get; set; }
public Guid? RepliedPostId { get; set; }
public Post? RepliedPost { get; set; }
public Guid? ForwardedPostId { get; set; }
public Post? ForwardedPost { get; set; }
public ICollection<CloudFile> Attachments { get; set; } = new List<CloudFile>();
[Column(TypeName = "jsonb")] public List<CloudFileReferenceObject> Attachments { get; set; } = [];
[JsonIgnore] public NpgsqlTsVector SearchVector { get; set; } = null!;
public Guid PublisherId { get; set; }
public Publisher.Publisher Publisher { get; set; } = null!;
public ICollection<PostReaction> Reactions { get; set; } = new List<PostReaction>();
public ICollection<PostTag> Tags { get; set; } = new List<PostTag>();
public ICollection<PostCategory> Categories { get; set; } = new List<PostCategory>();
@ -64,6 +62,8 @@ public class Post : ModelBase
[JsonIgnore] public bool Empty => Content == null && Attachments.Count == 0 && ForwardedPostId == null;
[NotMapped] public bool IsTruncated = false;
public string ResourceIdentifier => $"post/{Id}";
}
public class PostTag : ModelBase

5
DysonNetwork.Sphere/Post/PostController.cs
View File

@ -41,9 +41,7 @@ public class PostController(
.CountAsync();
var posts = await query
.Include(e => e.RepliedPost)
.Include(e => e.ThreadedPost)
.Include(e => e.ForwardedPost)
.Include(e => e.Attachments)
.Include(e => e.Categories)
.Include(e => e.Tags)
.Where(e => e.RepliedPostId == null)
@ -107,9 +105,7 @@ public class PostController(
.CountAsync();
var posts = await db.Posts
.Where(e => e.RepliedPostId == id)
.Include(e => e.ThreadedPost)
.Include(e => e.ForwardedPost)
.Include(e => e.Attachments)
.Include(e => e.Categories)
.Include(e => e.Tags)
.FilterWithVisibility(currentUser, userFriends, isListing: true)
@ -351,7 +347,6 @@ public class PostController(
var post = await db.Posts
.Where(e => e.Id == id)
.Include(e => e.Publisher)
.Include(e => e.Attachments)
.FirstOrDefaultAsync();
if (post is null) return NotFound();

53
DysonNetwork.Sphere/Post/PostService.cs
View File

@ -1,4 +1,3 @@
using System.Text.Json;
using DysonNetwork.Sphere.Account;
using DysonNetwork.Sphere.Activity;
using DysonNetwork.Sphere.Localization;
@ -13,6 +12,7 @@ namespace DysonNetwork.Sphere.Post;
public class PostService(
AppDatabase db,
FileService fs,
FileReferenceService fileRefService,
ActivityService act,
IStringLocalizer<NotificationResource> localizer,
NotificationService nty,
@ -57,7 +57,8 @@ public class PostService(
if (attachments is not null)
{
post.Attachments = await db.Files.Where(e => attachments.Contains(e.Id)).ToListAsync();
post.Attachments = (await db.Files.Where(e => attachments.Contains(e.Id)).ToListAsync())
.Select(x => x.ToReferenceObject()).ToList();
// Re-order the list to match the id list places
post.Attachments = attachments
.Select(id => post.Attachments.First(a => a.Id == id))
@ -91,8 +92,20 @@ public class PostService(
db.Posts.Add(post);
await db.SaveChangesAsync();
await fs.MarkUsageRangeAsync(post.Attachments, 1);
await fs.SetUsageRangeAsync(post.Attachments, PostFileUsageIdentifier);
// Create file references for each attachment
if (post.Attachments.Any())
{
var postResourceId = $"post:{post.Id}";
foreach (var file in post.Attachments)
{
await fileRefService.CreateReferenceAsync(
file.Id,
PostFileUsageIdentifier,
postResourceId
);
}
}
await act.CreateNewPostActivity(user, post);
@ -131,8 +144,21 @@ public class PostService(
if (attachments is not null)
{
post.Attachments = (await fs.DiffAndMarkFilesAsync(attachments, post.Attachments)).current;
await fs.DiffAndSetUsageAsync(attachments, PostFileUsageIdentifier);
var postResourceId = $"post:{post.Id}";
// Update resource references using the new file list
await fileRefService.UpdateResourceFilesAsync(
postResourceId,
attachments,
PostFileUsageIdentifier
);
// Update post attachments by getting files from database
var files = await db.Files
.Where(f => attachments.Contains(f.Id))
.ToListAsync();
post.Attachments = files.Select(x => x.ToReferenceObject()).ToList();
}
if (tags is not null)
@ -168,9 +194,13 @@ public class PostService(
public async Task DeletePostAsync(Post post)
{
var postResourceId = $"post:{post.Id}";
// Delete all file references for this post
await fileRefService.DeleteResourceReferencesAsync(postResourceId);
db.Posts.Remove(post);
await db.SaveChangesAsync();
await fs.MarkUsageRangeAsync(post.Attachments, -1);
}
/// <summary>
@ -279,8 +309,7 @@ public class PostService(
[
e.PublisherId,
e.RepliedPost?.PublisherId,
e.ForwardedPost?.PublisherId,
e.ThreadedPost?.PublisherId
e.ForwardedPost?.PublisherId
])
.Where(e => e != null)
.Distinct()
@ -300,13 +329,9 @@ public class PostService(
publishers.TryGetValue(post.RepliedPost.PublisherId, out var repliedPublisher))
post.RepliedPost.Publisher = repliedPublisher;
if (post.ForwardedPost?.PublisherId != null &&
if (post.ForwardedPost?.PublisherId != null &&
publishers.TryGetValue(post.ForwardedPost.PublisherId, out var forwardedPublisher))
post.ForwardedPost.Publisher = forwardedPublisher;
if (post.ThreadedPost?.PublisherId != null &&
publishers.TryGetValue(post.ThreadedPost.PublisherId, out var threadedPublisher))
post.ThreadedPost.Publisher = threadedPublisher;
}
return posts;

1
DysonNetwork.Sphere/Program.cs
View File

@ -207,6 +207,7 @@ builder.Services.AddScoped<MagicSpellService>();
builder.Services.AddScoped<NotificationService>();
builder.Services.AddScoped<AuthService>();
builder.Services.AddScoped<FileService>();
builder.Services.AddScoped<FileReferenceService>();
builder.Services.AddScoped<PublisherService>();
builder.Services.AddScoped<PublisherSubscriptionService>();
builder.Services.AddScoped<ActivityService>();

7
DysonNetwork.Sphere/Publisher/Publisher.cs
View File

@ -1,4 +1,5 @@
using System.ComponentModel.DataAnnotations;
using System.ComponentModel.DataAnnotations.Schema;
using System.Text.Json.Serialization;
using DysonNetwork.Sphere.Post;
using DysonNetwork.Sphere.Storage;
@ -22,10 +23,8 @@ public class Publisher : ModelBase
[MaxLength(256)] public string Nick { get; set; } = string.Empty;
[MaxLength(4096)] public string? Bio { get; set; }
[MaxLength(32)] public string? PictureId { get; set; }
public CloudFile? Picture { get; set; }
[MaxLength(32)] public string? BackgroundId { get; set; }
public CloudFile? Background { get; set; }
[Column(TypeName = "jsonb")] public CloudFileReferenceObject? Picture { get; set; }
[Column(TypeName = "jsonb")] public CloudFileReferenceObject? Background { get; set; }
[JsonIgnore] public ICollection<Post.Post> Posts { get; set; } = new List<Post.Post>();
[JsonIgnore] public ICollection<PostCollection> Collections { get; set; } = new List<PostCollection>();

54
DysonNetwork.Sphere/Publisher/PublisherController.cs
View File

@ -13,7 +13,7 @@ namespace DysonNetwork.Sphere.Publisher;
[ApiController]
[Route("/publishers")]
public class PublisherController(AppDatabase db, PublisherService ps, FileService fs, ActionLogService als)
public class PublisherController(AppDatabase db, PublisherService ps, FileService fs, FileReferenceService fileRefService, ActionLogService als)
: ControllerBase
{
[HttpGet("{name}")]
@ -356,20 +356,52 @@ public class PublisherController(AppDatabase db, PublisherService ps, FileServic
{
var picture = await db.Files.Where(f => f.Id == request.PictureId).FirstOrDefaultAsync();
if (picture is null) return BadRequest("Invalid picture id.");
if (publisher.Picture is not null) await fs.MarkUsageAsync(publisher.Picture, -1);
publisher.Picture = picture;
await fs.MarkUsageAsync(picture, 1);
var publisherResourceId = $"publisher:{publisher.Id}";
// Remove old references for the publisher picture
if (publisher.Picture is not null) {
var oldPictureRefs = await fileRefService.GetResourceReferencesAsync(publisherResourceId, "publisher.picture");
foreach (var oldRef in oldPictureRefs)
{
await fileRefService.DeleteReferenceAsync(oldRef.Id);
}
}
publisher.Picture = picture.ToReferenceObject();
// Create a new reference
await fileRefService.CreateReferenceAsync(
picture.Id,
"publisher.picture",
publisherResourceId
);
}
if (request.BackgroundId is not null)
{
var background = await db.Files.Where(f => f.Id == request.BackgroundId).FirstOrDefaultAsync();
if (background is null) return BadRequest("Invalid background id.");
if (publisher.Background is not null) await fs.MarkUsageAsync(publisher.Background, -1);
publisher.Background = background;
await fs.MarkUsageAsync(background, 1);
var publisherResourceId = $"publisher:{publisher.Id}";
// Remove old references for the publisher background
if (publisher.Background is not null) {
var oldBackgroundRefs = await fileRefService.GetResourceReferencesAsync(publisherResourceId, "publisher.background");
foreach (var oldRef in oldBackgroundRefs)
{
await fileRefService.DeleteReferenceAsync(oldRef.Id);
}
}
publisher.Background = background.ToReferenceObject();
// Create a new reference
await fileRefService.CreateReferenceAsync(
background.Id,
"publisher.background",
publisherResourceId
);
}
db.Update(publisher);
@ -405,10 +437,10 @@ public class PublisherController(AppDatabase db, PublisherService ps, FileServic
if (member.Role < PublisherMemberRole.Owner)
return StatusCode(403, "You need to be the owner to delete the publisher.");
if (publisher.Picture is not null)
await fs.MarkUsageAsync(publisher.Picture, -1);
if (publisher.Background is not null)
await fs.MarkUsageAsync(publisher.Background, -1);
var publisherResourceId = $"publisher:{publisher.Id}";
// Delete all file references for this publisher
await fileRefService.DeleteResourceReferencesAsync(publisherResourceId);
db.Publishers.Remove(publisher);
await db.SaveChangesAsync();

48
DysonNetwork.Sphere/Publisher/PublisherService.cs
View File

@ -6,7 +6,7 @@ using NodaTime;
namespace DysonNetwork.Sphere.Publisher;
public class PublisherService(AppDatabase db, FileService fs, ICacheService cache)
public class PublisherService(AppDatabase db, FileService fs, FileReferenceService fileRefService, ICacheService cache)
{
public async Task<Publisher> CreateIndividualPublisher(
Account.Account account,
@ -23,8 +23,8 @@ public class PublisherService(AppDatabase db, FileService fs, ICacheService cach
Name = name ?? account.Name,
Nick = nick ?? account.Nick,
Bio = bio ?? account.Profile.Bio,
Picture = picture ?? account.Profile.Picture,
Background = background ?? account.Profile.Background,
Picture = picture?.ToReferenceObject() ?? account.Profile.Picture,
Background = background?.ToReferenceObject() ?? account.Profile.Background,
AccountId = account.Id,
Members = new List<PublisherMember>
{
@ -40,8 +40,23 @@ public class PublisherService(AppDatabase db, FileService fs, ICacheService cach
db.Publishers.Add(publisher);
await db.SaveChangesAsync();
if (publisher.Picture is not null) await fs.MarkUsageAsync(publisher.Picture, 1);
if (publisher.Background is not null) await fs.MarkUsageAsync(publisher.Background, 1);
var publisherResourceId = $"publisher:{publisher.Id}";
if (publisher.Picture is not null) {
await fileRefService.CreateReferenceAsync(
publisher.Picture.Id,
"publisher.picture",
publisherResourceId
);
}
if (publisher.Background is not null) {
await fileRefService.CreateReferenceAsync(
publisher.Background.Id,
"publisher.background",
publisherResourceId
);
}
return publisher;
}
@ -62,8 +77,8 @@ public class PublisherService(AppDatabase db, FileService fs, ICacheService cach
Name = name ?? realm.Slug,
Nick = nick ?? realm.Name,
Bio = bio ?? realm.Description,
Picture = picture ?? realm.Picture,
Background = background ?? realm.Background,
Picture = picture?.ToReferenceObject() ?? realm.Picture,
Background = background?.ToReferenceObject() ?? realm.Background,
RealmId = realm.Id,
Members = new List<PublisherMember>
{
@ -79,8 +94,23 @@ public class PublisherService(AppDatabase db, FileService fs, ICacheService cach
db.Publishers.Add(publisher);
await db.SaveChangesAsync();
if (publisher.Picture is not null) await fs.MarkUsageAsync(publisher.Picture, 1);
if (publisher.Background is not null) await fs.MarkUsageAsync(publisher.Background, 1);
var publisherResourceId = $"publisher:{publisher.Id}";
if (publisher.Picture is not null) {
await fileRefService.CreateReferenceAsync(
publisher.Picture.Id,
"publisher.picture",
publisherResourceId
);
}
if (publisher.Background is not null) {
await fileRefService.CreateReferenceAsync(
publisher.Background.Id,
"publisher.background",
publisherResourceId
);
}
return publisher;
}

11
DysonNetwork.Sphere/Realm/Realm.cs
View File

@ -1,4 +1,5 @@
using System.ComponentModel.DataAnnotations;
using System.ComponentModel.DataAnnotations.Schema;
using System.Text.Json.Serialization;
using DysonNetwork.Sphere.Chat;
using DysonNetwork.Sphere.Storage;
@ -8,7 +9,7 @@ using NodaTime;
namespace DysonNetwork.Sphere.Realm;
[Index(nameof(Slug), IsUnique = true)]
public class Realm : ModelBase
public class Realm : ModelBase, IIdentifiedResource
{
public Guid Id { get; set; }
[MaxLength(1024)] public string Slug { get; set; } = string.Empty;
@ -19,16 +20,16 @@ public class Realm : ModelBase
public bool IsCommunity { get; set; }
public bool IsPublic { get; set; }
[MaxLength(32)] public string? PictureId { get; set; }
public CloudFile? Picture { get; set; }
[MaxLength(32)] public string? BackgroundId { get; set; }
public CloudFile? Background { get; set; }
[Column(TypeName = "jsonb")] public CloudFileReferenceObject? Picture { get; set; }
[Column(TypeName = "jsonb")] public CloudFileReferenceObject? Background { get; set; }
[JsonIgnore] public ICollection<RealmMember> Members { get; set; } = new List<RealmMember>();
[JsonIgnore] public ICollection<ChatRoom> ChatRooms { get; set; } = new List<ChatRoom>();
public Guid AccountId { get; set; }
[JsonIgnore] public Account.Account Account { get; set; } = null!;
public string ResourceIdentifier => $"realm/{Id}";
}
public enum RealmMemberRole

88
DysonNetwork.Sphere/Realm/RealmController.cs
View File

@ -10,7 +10,14 @@ namespace DysonNetwork.Sphere.Realm;
[ApiController]
[Route("/realms")]
public class RealmController(AppDatabase db, RealmService rs, FileService fs, RelationshipService rels, ActionLogService als) : Controller
public class RealmController(
AppDatabase db,
RealmService rs,
FileService fs,
FileReferenceService fileRefService,
RelationshipService rels,
ActionLogService als
) : Controller
{
[HttpGet("{slug}")]
public async Task<ActionResult<Realm>> GetRealm(string slug)
@ -298,13 +305,13 @@ public class RealmController(AppDatabase db, RealmService rs, FileService fs, Re
if (request.PictureId is not null)
{
realm.Picture = await db.Files.FindAsync(request.PictureId);
realm.Picture = (await db.Files.FindAsync(request.PictureId))?.ToReferenceObject();
if (realm.Picture is null) return BadRequest("Invalid picture id, unable to find the file on cloud.");
}
if (request.BackgroundId is not null)
{
realm.Background = await db.Files.FindAsync(request.BackgroundId);
realm.Background = (await db.Files.FindAsync(request.BackgroundId))?.ToReferenceObject();
if (realm.Background is null) return BadRequest("Invalid background id, unable to find the file on cloud.");
}
@ -316,8 +323,25 @@ public class RealmController(AppDatabase db, RealmService rs, FileService fs, Re
new Dictionary<string, object> { { "realm_id", realm.Id } }, Request
);
if (realm.Picture is not null) await fs.MarkUsageAsync(realm.Picture, 1);
if (realm.Background is not null) await fs.MarkUsageAsync(realm.Background, 1);
var realmResourceId = $"realm:{realm.Id}";
if (realm.Picture is not null)
{
await fileRefService.CreateReferenceAsync(
realm.Picture.Id,
"realm.picture",
realmResourceId
);
}
if (realm.Background is not null)
{
await fileRefService.CreateReferenceAsync(
realm.Background.Id,
"realm.background",
realmResourceId
);
}
return Ok(realm);
}
@ -357,22 +381,57 @@ public class RealmController(AppDatabase db, RealmService rs, FileService fs, Re
if (request.IsPublic is not null)
realm.IsPublic = request.IsPublic.Value;
var realmResourceId = $"realm:{realm.Id}";
if (request.PictureId is not null)
{
var picture = await db.Files.FindAsync(request.PictureId);
if (picture is null) return BadRequest("Invalid picture id, unable to find the file on cloud.");
await fs.MarkUsageAsync(picture, 1);
if (realm.Picture is not null) await fs.MarkUsageAsync(realm.Picture, -1);
realm.Picture = picture;
// Remove old references for the realm picture
if (realm.Picture is not null)
{
var oldPictureRefs = await fileRefService.GetResourceReferencesAsync(realmResourceId, "realm.picture");
foreach (var oldRef in oldPictureRefs)
{
await fileRefService.DeleteReferenceAsync(oldRef.Id);
}
}
realm.Picture = picture.ToReferenceObject();
// Create a new reference
await fileRefService.CreateReferenceAsync(
picture.Id,
"realm.picture",
realmResourceId
);
}
if (request.BackgroundId is not null)
{
var background = await db.Files.FindAsync(request.BackgroundId);
if (background is null) return BadRequest("Invalid background id, unable to find the file on cloud.");
await fs.MarkUsageAsync(background, 1);
if (realm.Background is not null) await fs.MarkUsageAsync(realm.Background, -1);
realm.Background = background;
// Remove old references for the realm background
if (realm.Background is not null)
{
var oldBackgroundRefs =
await fileRefService.GetResourceReferencesAsync(realmResourceId, "realm.background");
foreach (var oldRef in oldBackgroundRefs)
{
await fileRefService.DeleteReferenceAsync(oldRef.Id);
}
}
realm.Background = background.ToReferenceObject();
// Create a new reference
await fileRefService.CreateReferenceAsync(
background.Id,
"realm.background",
realmResourceId
);
}
db.Realms.Update(realm);
@ -517,10 +576,9 @@ public class RealmController(AppDatabase db, RealmService rs, FileService fs, Re
new Dictionary<string, object> { { "realm_id", realm.Id } }, Request
);
if (realm.Picture is not null)
await fs.MarkUsageAsync(realm.Picture, -1);
if (realm.Background is not null)
await fs.MarkUsageAsync(realm.Background, -1);
// Delete all file references for this realm
var realmResourceId = $"realm:{realm.Id}";
await fileRefService.DeleteResourceReferencesAsync(realmResourceId);
return NoContent();
}

7
DysonNetwork.Sphere/Sticker/Sticker.cs
View File

@ -1,20 +1,23 @@
using System.ComponentModel.DataAnnotations;
using System.ComponentModel.DataAnnotations.Schema;
using DysonNetwork.Sphere.Storage;
using Microsoft.EntityFrameworkCore;
namespace DysonNetwork.Sphere.Sticker;
[Index(nameof(Slug))] // The slug index shouldn't be unique, the sticker slug can be repeated across packs.
public class Sticker : ModelBase
public class Sticker : ModelBase, IIdentifiedResource
{
public Guid Id { get; set; } = Guid.NewGuid();
[MaxLength(128)] public string Slug { get; set; } = null!;
[MaxLength(32)] public string ImageId { get; set; } = null!;
public CloudFile Image { get; set; } = null!;
[Column(TypeName = "jsonb")] public CloudFileReferenceObject Image { get; set; } = null!;
public Guid PackId { get; set; }
public StickerPack Pack { get; set; } = null!;
public string ResourceIdentifier => $"sticker/{Id}";
}
[Index(nameof(Prefix), IsUnique = true)]

2
DysonNetwork.Sphere/Sticker/StickerController.cs
View File

@ -294,7 +294,7 @@ public class StickerController(AppDatabase db, StickerService st) : ControllerBa
{
Slug = request.Slug,
ImageId = image.Id,
Image = image,
Image = image.ToReferenceObject(),
Pack = pack
};

52
DysonNetwork.Sphere/Sticker/StickerService.cs
View File

@ -3,8 +3,10 @@ using Microsoft.EntityFrameworkCore;
namespace DysonNetwork.Sphere.Sticker;
public class StickerService(AppDatabase db, FileService fs, ICacheService cache)
public class StickerService(AppDatabase db, FileService fs, FileReferenceService fileRefService, ICacheService cache)
{
public const string StickerFileUsageIdentifier = "sticker";
private static readonly TimeSpan CacheDuration = TimeSpan.FromMinutes(15);
public async Task<Sticker> CreateStickerAsync(Sticker sticker)
@ -12,18 +14,37 @@ public class StickerService(AppDatabase db, FileService fs, ICacheService cache)
db.Stickers.Add(sticker);
await db.SaveChangesAsync();
await fs.MarkUsageAsync(sticker.Image, 1);
var stickerResourceId = $"sticker:{sticker.Id}";
await fileRefService.CreateReferenceAsync(
sticker.Image.Id,
StickerFileUsageIdentifier,
stickerResourceId
);
return sticker;
}
public async Task<Sticker> UpdateStickerAsync(Sticker sticker, CloudFile? newImage)
{
if (newImage != null)
if (newImage is not null)
{
await fs.MarkUsageAsync(sticker.Image, -1);
sticker.Image = newImage;
await fs.MarkUsageAsync(sticker.Image, 1);
var stickerResourceId = $"sticker:{sticker.Id}";
// Delete old references
var oldRefs = await fileRefService.GetResourceReferencesAsync(stickerResourceId, StickerFileUsageIdentifier);
foreach (var oldRef in oldRefs)
{
await fileRefService.DeleteReferenceAsync(oldRef.Id);
}
sticker.Image = newImage.ToReferenceObject();
// Create new reference
await fileRefService.CreateReferenceAsync(
newImage.Id,
StickerFileUsageIdentifier,
stickerResourceId
);
}
db.Stickers.Update(sticker);
@ -37,9 +58,13 @@ public class StickerService(AppDatabase db, FileService fs, ICacheService cache)
public async Task DeleteStickerAsync(Sticker sticker)
{
var stickerResourceId = $"sticker:{sticker.Id}";
// Delete all file references for this sticker
await fileRefService.DeleteResourceReferencesAsync(stickerResourceId);
db.Stickers.Remove(sticker);
await db.SaveChangesAsync();
await fs.MarkUsageAsync(sticker.Image, -1);
// Invalidate cache for this sticker
await PurgeStickerCache(sticker);
@ -54,12 +79,21 @@ public class StickerService(AppDatabase db, FileService fs, ICacheService cache)
var images = stickers.Select(s => s.Image).ToList();
// Delete all file references for each sticker in the pack
foreach (var sticker in stickers)
{
var stickerResourceId = $"sticker:{sticker.Id}";
await fileRefService.DeleteResourceReferencesAsync(stickerResourceId);
}
// Delete any references for the pack itself
var packResourceId = $"stickerpack:{pack.Id}";
await fileRefService.DeleteResourceReferencesAsync(packResourceId);
db.Stickers.RemoveRange(stickers);
db.StickerPacks.Remove(pack);
await db.SaveChangesAsync();
await fs.MarkUsageRangeAsync(images, -1);
// Invalidate cache for all stickers in this pack
foreach (var sticker in stickers)
await PurgeStickerCache(sticker);

6
DysonNetwork.Sphere/Storage/CacheService.cs
View File

@ -215,6 +215,7 @@ public class CacheServiceRedis : ICacheService
public async Task<bool> SetAsync<T>(string key, T value, TimeSpan? expiry = null)
{
key = $"{GlobalKeyPrefix}{key}";
if (string.IsNullOrEmpty(key))
throw new ArgumentException("Key cannot be null or empty", nameof(key));
@ -224,6 +225,7 @@ public class CacheServiceRedis : ICacheService
public async Task<T?> GetAsync<T>(string key)
{
key = $"{GlobalKeyPrefix}{key}";
if (string.IsNullOrEmpty(key))
throw new ArgumentException("Key cannot be null or empty", nameof(key));
@ -238,6 +240,7 @@ public class CacheServiceRedis : ICacheService
public async Task<(bool found, T? value)> GetAsyncWithStatus<T>(string key)
{
key = $"{GlobalKeyPrefix}{key}";
if (string.IsNullOrEmpty(key))
throw new ArgumentException("Key cannot be null or empty", nameof(key));
@ -252,6 +255,7 @@ public class CacheServiceRedis : ICacheService
public async Task<bool> RemoveAsync(string key)
{
key = $"{GlobalKeyPrefix}{key}";
if (string.IsNullOrEmpty(key))
throw new ArgumentException("Key cannot be null or empty", nameof(key));
@ -281,6 +285,7 @@ public class CacheServiceRedis : ICacheService
throw new ArgumentException(@"Group cannot be null or empty.", nameof(group));
var groupKey = $"{GroupKeyPrefix}{group}";
key = $"{GlobalKeyPrefix}{key}";
await _database.SetAddAsync(groupKey, key);
}
@ -319,6 +324,7 @@ public class CacheServiceRedis : ICacheService
public async Task<bool> SetWithGroupsAsync<T>(string key, T value, IEnumerable<string>? groups = null,
TimeSpan? expiry = null)
{
key = $"{GlobalKeyPrefix}{key}";
// First, set the value in the cache
var setResult = await SetAsync(key, value, expiry);

71
DysonNetwork.Sphere/Storage/CloudFile.cs
View File

@ -20,10 +20,28 @@ public class RemoteStorageConfig
public string? AccessProxy { get; set; }
}
public class CloudFile : ModelBase
/// <summary>
/// The class that used in jsonb columns which referenced the cloud file.
/// The aim of this class is to store some properties that won't change to a file to reduce the database load.
/// </summary>
public class CloudFileReferenceObject : ICloudFile
{
public string Id { get; set; } = null!;
public string Name { get; set; } = string.Empty;
public Dictionary<string, object>? FileMeta { get; set; } = null!;
public Dictionary<string, object>? UserMeta { get; set; } = null!;
public string? MimeType { get; set; }
public string? Hash { get; set; }
public long Size { get; set; }
public bool HasCompression { get; set; } = false;
}
public class CloudFile : ModelBase, ICloudFile, IIdentifiedResource
{
/// The id generated by TuS, basically just UUID remove the dash lines
[MaxLength(32)] public string Id { get; set; } = Guid.NewGuid().ToString();
[MaxLength(32)]
public string Id { get; set; } = Guid.NewGuid().ToString();
[MaxLength(1024)] public string Name { get; set; } = string.Empty;
[MaxLength(4096)] public string? Description { get; set; }
[Column(TypeName = "jsonb")] public Dictionary<string, object>? FileMeta { get; set; } = null!;
@ -33,29 +51,42 @@ public class CloudFile : ModelBase
[MaxLength(256)] public string? Hash { get; set; }
public long Size { get; set; }
public Instant? UploadedAt { get; set; }
public Instant? ExpiredAt { get; set; }
[MaxLength(128)] public string? UploadedTo { get; set; }
public bool HasCompression { get; set; }= false;
public bool HasCompression { get; set; } = false;
/// The object name which stored remotely,
/// multiple cloud file may have same storage id to indicate they are the same file
///
/// If the storage id was null and the uploaded at is not null, means it is an embedding file,
/// The embedding file means the file is store on another site,
/// or it is a webpage (based on mimetype)
[MaxLength(32)] public string? StorageId { get; set; }
[MaxLength(32)]
public string? StorageId { get; set; }
/// This field should be null when the storage id is filled
/// Indicates the off-site accessible url of the file
[MaxLength(4096)] public string? StorageUrl { get; set; }
/// Metrics
/// When this used count keep zero, it means it's not used by anybody, so it can be recycled
public int UsedCount { get; set; } = 0;
/// An optional package identifier that indicates the cloud file's usage
[MaxLength(1024)] public string? Usage { get; set; }
[MaxLength(4096)]
public string? StorageUrl { get; set; }
[JsonIgnore] public Account.Account Account { get; set; } = null!;
public Guid AccountId { get; set; }
public CloudFileReferenceObject ToReferenceObject()
{
return new CloudFileReferenceObject
{
Id = Id,
Name = Name,
FileMeta = FileMeta,
UserMeta = UserMeta,
MimeType = MimeType,
Hash = Hash,
Size = Size,
HasCompression = HasCompression
};
}
public string ResourceIdentifier => $"file/{Id}";
}
public enum CloudFileSensitiveMark
@ -73,4 +104,18 @@ public enum CloudFileSensitiveMark
SelfHarm,
ChildAbuse,
Other
}
public class CloudFileReference : ModelBase
{
public Guid Id { get; set; } = Guid.NewGuid();
[MaxLength(32)] public string FileId { get; set; } = null!;
public CloudFile File { get; set; } = null!;
[MaxLength(1024)] public string Usage { get; set; } = null!;
[MaxLength(1024)] public string ResourceId { get; set; } = null!;
/// <summary>
/// Optional expiration date for the file reference
/// </summary>
public Instant? ExpiredAt { get; set; }
}

127
DysonNetwork.Sphere/Storage/CloudFileUnusedRecyclingJob.cs Normal file
View File

@ -0,0 +1,127 @@
using Microsoft.EntityFrameworkCore;
using NodaTime;
using Quartz;
namespace DysonNetwork.Sphere.Storage;
public class CloudFileUnusedRecyclingJob(
AppDatabase db,
FileService fs,
FileReferenceService fileRefService,
ILogger<CloudFileUnusedRecyclingJob> logger
)
: IJob
{
public async Task Execute(IJobExecutionContext context)
{
logger.LogInformation("Deleting unused cloud files...");
var cutoff = SystemClock.Instance.GetCurrentInstant() - Duration.FromHours(1);
var now = SystemClock.Instance.GetCurrentInstant();
// Get files that are either expired or created more than an hour ago
var fileIds = await db.Files
.Select(f => f.Id)
.ToListAsync();
// Filter to only include files that have no references or all references have expired
var deletionPlan = new List<string>();
foreach (var batch in fileIds.Chunk(100)) // Process in batches to avoid excessive query size
{
var references = await fileRefService.GetReferencesAsync(batch);
deletionPlan.AddRange(from refer in references
where refer.Value.Count == 0 || refer.Value.All(r => r.ExpiredAt != null && now >= r.ExpiredAt)
select refer.Key);
}
if (deletionPlan.Count == 0)
{
logger.LogInformation("No files to delete");
return;
}
// Get the actual file objects for the files to be deleted
var files = await db.Files
.Where(f => deletionPlan.Contains(f.Id))
.ToListAsync();
logger.LogInformation($"Found {files.Count} files to delete...");
// Group files by StorageId and find which ones are safe to delete
var storageIds = files.Where(f => f.StorageId != null)
.Select(f => f.StorageId!)
.Distinct()
.ToList();
// Check if any other files with the same storage IDs are referenced
var usedStorageIds = new List<string>();
var filesWithSameStorageId = await db.Files
.Where(f => f.StorageId != null &&
storageIds.Contains(f.StorageId) &&
!files.Select(ff => ff.Id).Contains(f.Id))
.ToListAsync();
foreach (var file in filesWithSameStorageId)
{
// Get all references for the file
var references = await fileRefService.GetReferencesAsync(file.Id);
// Check if file has active references (non-expired)
if (references.Any(r => r.ExpiredAt == null || r.ExpiredAt > now) && file.StorageId != null)
{
usedStorageIds.Add(file.StorageId);
}
}
// Group files for deletion
var filesToDelete = files.Where(f => f.StorageId == null || !usedStorageIds.Contains(f.StorageId))
.GroupBy(f => f.UploadedTo)
.ToDictionary(grouping => grouping.Key!, grouping => grouping.ToList());
// Delete files by remote storage
foreach (var group in filesToDelete.Where(group => !string.IsNullOrEmpty(group.Key)))
{
try
{
var dest = fs.GetRemoteStorageConfig(group.Key);
var client = fs.CreateMinioClient(dest);
if (client == null) continue;
// Create delete tasks for each file in the group
var deleteTasks = group.Value.Select(file =>
{
var objectId = file.StorageId ?? file.Id;
var tasks = new List<Task>
{
client.RemoveObjectAsync(new Minio.DataModel.Args.RemoveObjectArgs()
.WithBucket(dest.Bucket)
.WithObject(objectId))
};
if (file.HasCompression)
{
tasks.Add(client.RemoveObjectAsync(new Minio.DataModel.Args.RemoveObjectArgs()
.WithBucket(dest.Bucket)
.WithObject(objectId + ".compressed")));
}
return Task.WhenAll(tasks);
});
await Task.WhenAll(deleteTasks);
}
catch (Exception ex)
{
logger.LogError(ex, "Error deleting files from remote storage {remote}", group.Key);
}
}
// Delete all file records from the database
var fileIdsToDelete = files.Select(f => f.Id).ToList();
await db.Files
.Where(f => fileIdsToDelete.Contains(f.Id))
.ExecuteDeleteAsync();
logger.LogInformation($"Completed deleting {files.Count} files");
}
}

66
DysonNetwork.Sphere/Storage/FileExpirationJob.cs Normal file
View File

@ -0,0 +1,66 @@
using Microsoft.EntityFrameworkCore;
using NodaTime;
using Quartz;
namespace DysonNetwork.Sphere.Storage;
/// <summary>
/// Job responsible for cleaning up expired file references
/// </summary>
public class FileExpirationJob(AppDatabase db, FileService fileService, ILogger<FileExpirationJob> logger) : IJob
{
public async Task Execute(IJobExecutionContext context)
{
var now = SystemClock.Instance.GetCurrentInstant();
logger.LogInformation("Running file reference expiration job at {now}", now);
// Find all expired references
var expiredReferences = await db.FileReferences
.Where(r => r.ExpiredAt < now && r.ExpiredAt != null)
.ToListAsync();
if (!expiredReferences.Any())
{
logger.LogInformation("No expired file references found");
return;
}
logger.LogInformation("Found {count} expired file references", expiredReferences.Count);
// Get unique file IDs
var fileIds = expiredReferences.Select(r => r.FileId).Distinct().ToList();
var filesAndReferenceCount = new Dictionary<string, int>();
// Delete expired references
db.FileReferences.RemoveRange(expiredReferences);
await db.SaveChangesAsync();
// Check remaining references for each file
foreach (var fileId in fileIds)
{
var remainingReferences = await db.FileReferences
.Where(r => r.FileId == fileId)
.CountAsync();
filesAndReferenceCount[fileId] = remainingReferences;
// If no references remain, delete the file
if (remainingReferences == 0)
{
var file = await db.Files.FirstOrDefaultAsync(f => f.Id == fileId);
if (file != null)
{
logger.LogInformation("Deleting file {fileId} as all references have expired", fileId);
await fileService.DeleteFileAsync(file);
}
}
else
{
// Just purge the cache
await fileService._PurgeCacheAsync(fileId);
}
}
logger.LogInformation("Completed file reference expiration job");
}
}

407
DysonNetwork.Sphere/Storage/FileReferenceService.cs Normal file
View File

@ -0,0 +1,407 @@
using Microsoft.EntityFrameworkCore;
using NodaTime;
namespace DysonNetwork.Sphere.Storage;
public class FileReferenceService(AppDatabase db, FileService fileService, ICacheService cache)
{
private const string CacheKeyPrefix = "fileref:";
private static readonly TimeSpan CacheDuration = TimeSpan.FromMinutes(15);
/// <summary>
/// Creates a new reference to a file for a specific resource
/// </summary>
/// <param name="fileId">The ID of the file to reference</param>
/// <param name="usage">The usage context (e.g., "avatar", "post-attachment")</param>
/// <param name="resourceId">The ID of the resource using the file</param>
/// <param name="expiredAt">Optional expiration time for the file</param>
/// <param name="duration">Optional duration after which the file expires (alternative to expiredAt)</param>
/// <returns>The created file reference</returns>
public async Task<CloudFileReference> CreateReferenceAsync(
string fileId,
string usage,
string resourceId,
Instant? expiredAt = null,
Duration? duration = null)
{
// Calculate expiration time if needed
Instant? finalExpiration = expiredAt;
if (duration.HasValue)
{
finalExpiration = SystemClock.Instance.GetCurrentInstant() + duration.Value;
}
var reference = new CloudFileReference
{
FileId = fileId,
Usage = usage,
ResourceId = resourceId,
ExpiredAt = finalExpiration
};
db.FileReferences.Add(reference);
await db.SaveChangesAsync();
// Purge cache for the file since its usage count has effectively changed
await fileService._PurgeCacheAsync(fileId);
return reference;
}
/// <summary>
/// Gets all references to a file
/// </summary>
/// <param name="fileId">The ID of the file</param>
/// <returns>A list of all references to the file</returns>
public async Task<List<CloudFileReference>> GetReferencesAsync(string fileId)
{
var cacheKey = $"{CacheKeyPrefix}list:{fileId}";
var cachedReferences = await cache.GetAsync<List<CloudFileReference>>(cacheKey);
if (cachedReferences is not null)
return cachedReferences;
var references = await db.FileReferences
.Where(r => r.FileId == fileId)
.ToListAsync();
await cache.SetAsync(cacheKey, references, CacheDuration);
return references;
}
public async Task<Dictionary<string, List<CloudFileReference>>> GetReferencesAsync(IEnumerable<string> fileId)
{
var references = await db.FileReferences
.Where(r => fileId.Contains(r.FileId))
.GroupBy(r => r.FileId)
.ToDictionaryAsync(r => r.Key, r => r.ToList());
return references;
}
/// <summary>
/// Gets the number of references to a file
/// </summary>
/// <param name="fileId">The ID of the file</param>
/// <returns>The number of references to the file</returns>
public async Task<int> GetReferenceCountAsync(string fileId)
{
var cacheKey = $"{CacheKeyPrefix}count:{fileId}";
var cachedCount = await cache.GetAsync<int?>(cacheKey);
if (cachedCount.HasValue)
return cachedCount.Value;
var count = await db.FileReferences
.Where(r => r.FileId == fileId)
.CountAsync();
await cache.SetAsync(cacheKey, count, CacheDuration);
return count;
}
/// <summary>
/// Gets all references for a specific resource
/// </summary>
/// <param name="resourceId">The ID of the resource</param>
/// <returns>A list of file references associated with the resource</returns>
public async Task<List<CloudFileReference>> GetResourceReferencesAsync(string resourceId)
{
var cacheKey = $"{CacheKeyPrefix}resource:{resourceId}";
var cachedReferences = await cache.GetAsync<List<CloudFileReference>>(cacheKey);
if (cachedReferences is not null)
return cachedReferences;
var references = await db.FileReferences
.Where(r => r.ResourceId == resourceId)
.ToListAsync();
await cache.SetAsync(cacheKey, references, CacheDuration);
return references;
}
/// <summary>
/// Gets all file references for a specific usage context
/// </summary>
/// <param name="usage">The usage context</param>
/// <returns>A list of file references with the specified usage</returns>
public async Task<List<CloudFileReference>> GetUsageReferencesAsync(string usage)
{
return await db.FileReferences
.Where(r => r.Usage == usage)
.ToListAsync();
}
/// <summary>
/// Deletes references for a specific resource
/// </summary>
/// <param name="resourceId">The ID of the resource</param>
/// <returns>The number of deleted references</returns>
public async Task<int> DeleteResourceReferencesAsync(string resourceId)
{
var references = await db.FileReferences
.Where(r => r.ResourceId == resourceId)
.ToListAsync();
var fileIds = references.Select(r => r.FileId).Distinct().ToList();
db.FileReferences.RemoveRange(references);
var deletedCount = await db.SaveChangesAsync();
// Purge caches
var tasks = fileIds.Select(fileService._PurgeCacheAsync).ToList();
tasks.Add(PurgeCacheForResourceAsync(resourceId));
await Task.WhenAll(tasks);
return deletedCount;
}
/// <summary>
/// Deletes a specific file reference
/// </summary>
/// <param name="referenceId">The ID of the reference to delete</param>
/// <returns>True if the reference was deleted, false otherwise</returns>
public async Task<bool> DeleteReferenceAsync(Guid referenceId)
{
var reference = await db.FileReferences
.FirstOrDefaultAsync(r => r.Id == referenceId);
if (reference == null)
return false;
db.FileReferences.Remove(reference);
await db.SaveChangesAsync();
// Purge caches
await fileService._PurgeCacheAsync(reference.FileId);
await PurgeCacheForResourceAsync(reference.ResourceId);
await PurgeCacheForFileAsync(reference.FileId);
return true;
}
/// <summary>
/// Updates the files referenced by a resource
/// </summary>
/// <param name="resourceId">The ID of the resource</param>
/// <param name="newFileIds">The new list of file IDs</param>
/// <param name="usage">The usage context</param>
/// <param name="expiredAt">Optional expiration time for newly added files</param>
/// <param name="duration">Optional duration after which newly added files expire</param>
/// <returns>A list of the updated file references</returns>
public async Task<List<CloudFileReference>> UpdateResourceFilesAsync(
string resourceId,
IEnumerable<string>? newFileIds,
string usage,
Instant? expiredAt = null,
Duration? duration = null)
{
if (newFileIds == null)
return new List<CloudFileReference>();
var existingReferences = await db.FileReferences
.Where(r => r.ResourceId == resourceId && r.Usage == usage)
.ToListAsync();
var existingFileIds = existingReferences.Select(r => r.FileId).ToHashSet();
var newFileIdsList = newFileIds.ToList();
var newFileIdsSet = newFileIdsList.ToHashSet();
// Files to remove
var toRemove = existingReferences
.Where(r => !newFileIdsSet.Contains(r.FileId))
.ToList();
// Files to add
var toAdd = newFileIdsList
.Where(id => !existingFileIds.Contains(id))
.Select(id => new CloudFileReference
{
FileId = id,
Usage = usage,
ResourceId = resourceId
})
.ToList();
// Apply changes
if (toRemove.Any())
db.FileReferences.RemoveRange(toRemove);
if (toAdd.Any())
db.FileReferences.AddRange(toAdd);
await db.SaveChangesAsync();
// Update expiration for newly added references if specified
if ((expiredAt.HasValue || duration.HasValue) && toAdd.Any())
{
var finalExpiration = expiredAt;
if (duration.HasValue)
{
finalExpiration = SystemClock.Instance.GetCurrentInstant() + duration.Value;
}
// Update newly added references with the expiration time
var referenceIds = await db.FileReferences
.Where(r => toAdd.Select(a => a.FileId).Contains(r.FileId) &&
r.ResourceId == resourceId &&
r.Usage == usage)
.Select(r => r.Id)
.ToListAsync();
await db.FileReferences
.Where(r => referenceIds.Contains(r.Id))
.ExecuteUpdateAsync(setter => setter.SetProperty(
r => r.ExpiredAt,
_ => finalExpiration
));
}
// Purge caches
var allFileIds = existingFileIds.Union(newFileIdsSet).ToList();
var tasks = allFileIds.Select(fileService._PurgeCacheAsync).ToList();
tasks.Add(PurgeCacheForResourceAsync(resourceId));
await Task.WhenAll(tasks);
// Return updated references
return await db.FileReferences
.Where(r => r.ResourceId == resourceId && r.Usage == usage)
.ToListAsync();
}
/// <summary>
/// Gets all files referenced by a resource
/// </summary>
/// <param name="resourceId">The ID of the resource</param>
/// <param name="usage">Optional filter by usage context</param>
/// <returns>A list of files referenced by the resource</returns>
public async Task<List<CloudFile>> GetResourceFilesAsync(string resourceId, string? usage = null)
{
var query = db.FileReferences.Where(r => r.ResourceId == resourceId);
if (usage != null)
query = query.Where(r => r.Usage == usage);
var references = await query.ToListAsync();
var fileIds = references.Select(r => r.FileId).ToList();
return await db.Files
.Where(f => fileIds.Contains(f.Id))
.ToListAsync();
}
/// <summary>
/// Purges all caches related to a resource
/// </summary>
private async Task PurgeCacheForResourceAsync(string resourceId)
{
var cacheKey = $"{CacheKeyPrefix}resource:{resourceId}";
await cache.RemoveAsync(cacheKey);
}
/// <summary>
/// Purges all caches related to a file
/// </summary>
private async Task PurgeCacheForFileAsync(string fileId)
{
var cacheKeys = new[]
{
$"{CacheKeyPrefix}list:{fileId}",
$"{CacheKeyPrefix}count:{fileId}"
};
var tasks = cacheKeys.Select(cache.RemoveAsync);
await Task.WhenAll(tasks);
}
/// <summary>
/// Updates the expiration time for a file reference
/// </summary>
/// <param name="referenceId">The ID of the reference</param>
/// <param name="expiredAt">The new expiration time, or null to remove expiration</param>
/// <returns>True if the reference was found and updated, false otherwise</returns>
public async Task<bool> SetReferenceExpirationAsync(Guid referenceId, Instant? expiredAt)
{
var reference = await db.FileReferences
.FirstOrDefaultAsync(r => r.Id == referenceId);
if (reference == null)
return false;
reference.ExpiredAt = expiredAt;
await db.SaveChangesAsync();
await PurgeCacheForFileAsync(reference.FileId);
await PurgeCacheForResourceAsync(reference.ResourceId);
return true;
}
/// <summary>
/// Updates the expiration time for all references to a file
/// </summary>
/// <param name="fileId">The ID of the file</param>
/// <param name="expiredAt">The new expiration time, or null to remove expiration</param>
/// <returns>The number of references updated</returns>
public async Task<int> SetFileReferencesExpirationAsync(string fileId, Instant? expiredAt)
{
var rowsAffected = await db.FileReferences
.Where(r => r.FileId == fileId)
.ExecuteUpdateAsync(setter => setter.SetProperty(
r => r.ExpiredAt,
_ => expiredAt
));
if (rowsAffected > 0)
{
await fileService._PurgeCacheAsync(fileId);
await PurgeCacheForFileAsync(fileId);
}
return rowsAffected;
}
/// <summary>
/// Get all file references for a specific resource and usage type
/// </summary>
/// <param name="resourceId">The resource ID</param>
/// <param name="usageType">The usage type</param>
/// <returns>List of file references</returns>
public async Task<List<CloudFileReference>> GetResourceReferencesAsync(string resourceId, string usageType)
{
return await db.FileReferences
.Where(r => r.ResourceId == resourceId && r.Usage == usageType)
.ToListAsync();
}
/// <summary>
/// Check if a file has any references
/// </summary>
/// <param name="fileId">The file ID to check</param>
/// <returns>True if the file has references, false otherwise</returns>
public async Task<bool> HasFileReferencesAsync(string fileId)
{
return await db.FileReferences.AnyAsync(r => r.FileId == fileId);
}
/// <summary>
/// Updates the expiration time for a file reference using a duration from now
/// </summary>
/// <param name="referenceId">The ID of the reference</param>
/// <param name="duration">The duration after which the reference expires, or null to remove expiration</param>
/// <returns>True if the reference was found and updated, false otherwise</returns>
public async Task<bool> SetReferenceExpirationDurationAsync(Guid referenceId, Duration? duration)
{
Instant? expiredAt = null;
if (duration.HasValue)
{
expiredAt = SystemClock.Instance.GetCurrentInstant() + duration.Value;
}
return await SetReferenceExpirationAsync(referenceId, expiredAt);
}
}

319
DysonNetwork.Sphere/Storage/FileService.cs
View File

@ -39,7 +39,10 @@ public class FileService(
if (cachedFile is not null)
return cachedFile;
var file = await db.Files.FirstOrDefaultAsync(f => f.Id == fileId);
var file = await db.Files
.Include(f => f.Account)
.Where(f => f.Id == fileId)
.FirstOrDefaultAsync();
if (file != null)
await cache.SetAsync(cacheKey, file, CacheDuration);
@ -208,8 +211,9 @@ public class FileService(
if (result.Count > 0)
{
List<Task<CloudFile>> tasks = [];
tasks.AddRange(result.Select(result =>
nfs.UploadFileToRemoteAsync(file, result.filePath, null, result.suffix, true)));
tasks.AddRange(result.Select(item =>
nfs.UploadFileToRemoteAsync(file, item.filePath, null, item.suffix, true))
);
await Task.WhenAll(tasks);
file = await tasks.First();
@ -326,10 +330,23 @@ public class FileService(
if (file.StorageId is null) return;
if (file.UploadedTo is null) return;
var repeatedStorageId = await db.Files
.Where(f => f.StorageId == file.StorageId && f.Id != file.Id && f.UsedCount > 0)
.AnyAsync();
if (repeatedStorageId) return;
// Check if any other file with the same storage ID is referenced
var otherFilesWithSameStorageId = await db.Files
.Where(f => f.StorageId == file.StorageId && f.Id != file.Id)
.Select(f => f.Id)
.ToListAsync();
// Check if any of these files are referenced
var anyReferenced = false;
if (otherFilesWithSameStorageId.Any())
{
anyReferenced = await db.FileReferences
.Where(r => otherFilesWithSameStorageId.Contains(r.FileId))
.AnyAsync();
}
// If any other file with the same storage ID is referenced, don't delete the actual file data
if (anyReferenced) return;
var dest = GetRemoteStorageConfig(file.UploadedTo);
var client = CreateMinioClient(dest);
@ -380,242 +397,88 @@ public class FileService(
return client.Build();
}
public async Task MarkUsageAsync(CloudFile file, int delta)
{
await db.Files.Where(o => o.Id == file.Id)
.ExecuteUpdateAsync(setter => setter.SetProperty(
b => b.UsedCount,
b => b.UsedCount + delta
)
);
await _PurgeCacheAsync(file.Id);
}
public async Task MarkUsageRangeAsync(ICollection<CloudFile> files, int delta)
{
var ids = files.Select(f => f.Id).ToArray();
await db.Files.Where(o => ids.Contains(o.Id))
.ExecuteUpdateAsync(setter => setter.SetProperty(
b => b.UsedCount,
b => b.UsedCount + delta
)
);
await _PurgeCacheRangeAsync(files.Select(x => x.Id).ToList());
}
public async Task SetExpiresRangeAsync(ICollection<CloudFile> files, Duration? duration)
{
var ids = files.Select(f => f.Id).ToArray();
await db.Files.Where(o => ids.Contains(o.Id))
.ExecuteUpdateAsync(setter => setter.SetProperty(
b => b.ExpiredAt,
duration.HasValue
? b => SystemClock.Instance.GetCurrentInstant() + duration.Value
: _ => null
)
);
await _PurgeCacheRangeAsync(files.Select(x => x.Id).ToList());
}
public async Task SetUsageAsync(CloudFile file, string? usage)
{
await db.Files.Where(o => o.Id == file.Id)
.ExecuteUpdateAsync(setter => setter.SetProperty(
b => b.Usage,
_ => usage
)
);
await _PurgeCacheAsync(file.Id);
}
public async Task SetUsageRangeAsync(ICollection<CloudFile> files, string? usage)
{
var ids = files.Select(f => f.Id).ToArray();
await db.Files.Where(o => ids.Contains(o.Id))
.ExecuteUpdateAsync(setter => setter.SetProperty(
b => b.Usage,
_ => usage
)
);
await _PurgeCacheRangeAsync(files.Select(x => x.Id).ToList());
}
public async Task<(ICollection<CloudFile> current, ICollection<CloudFile> added, ICollection<CloudFile> removed)>
DiffAndSetUsageAsync(
ICollection<string>? newFileIds,
string? usage,
ICollection<CloudFile>? previousFiles = null
)
{
if (newFileIds == null) return ([], [], previousFiles ?? []);
var records = await db.Files.Where(f => newFileIds.Contains(f.Id)).ToListAsync();
var previous = previousFiles?.ToDictionary(f => f.Id) ?? new Dictionary<string, CloudFile>();
var current = records.ToDictionary(f => f.Id);
var added = current.Keys.Except(previous.Keys).Select(id => current[id]).ToList();
var removed = previous.Keys.Except(current.Keys).Select(id => previous[id]).ToList();
if (added.Count > 0) await SetUsageRangeAsync(added, usage);
if (removed.Count > 0) await SetUsageRangeAsync(removed, null);
return (newFileIds.Select(id => current[id]).ToList(), added, removed);
}
public async Task<(ICollection<CloudFile> current, ICollection<CloudFile> added, ICollection<CloudFile> removed)>
DiffAndMarkFilesAsync(
ICollection<string>? newFileIds,
ICollection<CloudFile>? previousFiles = null
)
{
if (newFileIds == null) return ([], [], previousFiles ?? []);
var records = await db.Files.Where(f => newFileIds.Contains(f.Id)).ToListAsync();
var previous = previousFiles?.ToDictionary(f => f.Id) ?? new Dictionary<string, CloudFile>();
var current = records.ToDictionary(f => f.Id);
var added = current.Keys.Except(previous.Keys).Select(id => current[id]).ToList();
var removed = previous.Keys.Except(current.Keys).Select(id => previous[id]).ToList();
if (added.Count > 0) await MarkUsageRangeAsync(added, 1);
if (removed.Count > 0) await MarkUsageRangeAsync(removed, -1);
return (newFileIds.Select(id => current[id]).ToList(), added, removed);
}
public async Task<(ICollection<CloudFile> current, ICollection<CloudFile> added, ICollection<CloudFile> removed)>
DiffAndSetExpiresAsync(
ICollection<string>? newFileIds,
Duration? duration,
ICollection<CloudFile>? previousFiles = null
)
{
if (newFileIds == null) return ([], [], previousFiles ?? []);
var records = await db.Files.Where(f => newFileIds.Contains(f.Id)).ToListAsync();
var previous = previousFiles?.ToDictionary(f => f.Id) ?? new Dictionary<string, CloudFile>();
var current = records.ToDictionary(f => f.Id);
var added = current.Keys.Except(previous.Keys).Select(id => current[id]).ToList();
var removed = previous.Keys.Except(current.Keys).Select(id => previous[id]).ToList();
if (added.Count > 0) await SetExpiresRangeAsync(added, duration);
if (removed.Count > 0) await SetExpiresRangeAsync(removed, null);
return (newFileIds.Select(id => current[id]).ToList(), added, removed);
}
// Add this helper method to purge the cache for a specific file
private async Task _PurgeCacheAsync(string fileId)
// Helper method to purge the cache for a specific file
// Made internal to allow FileReferenceService to use it
internal async Task _PurgeCacheAsync(string fileId)
{
var cacheKey = $"{CacheKeyPrefix}{fileId}";
await cache.RemoveAsync(cacheKey);
}
// Add this helper method to purge cache for multiple files
private async Task _PurgeCacheRangeAsync(ICollection<string> fileIds)
// Helper method to purge cache for multiple files
internal async Task _PurgeCacheRangeAsync(IEnumerable<string> fileIds)
{
var tasks = fileIds.Select(_PurgeCacheAsync);
await Task.WhenAll(tasks);
}
}
public class CloudFileUnusedRecyclingJob(AppDatabase db, FileService fs, ILogger<CloudFileUnusedRecyclingJob> logger)
: IJob
{
public async Task Execute(IJobExecutionContext context)
public async Task<List<CloudFile?>> LoadFromReference(List<CloudFileReferenceObject> references)
{
logger.LogInformation("Deleting unused cloud files...");
var cachedFiles = new Dictionary<string, CloudFile>();
var uncachedIds = new List<string>();
var cutoff = SystemClock.Instance.GetCurrentInstant() - Duration.FromHours(1);
var now = SystemClock.Instance.GetCurrentInstant();
// Get files to delete along with their storage IDs
var files = await db.Files
.Where(f =>
(f.ExpiredAt == null && f.UsedCount == 0 && f.CreatedAt < cutoff) ||
(f.ExpiredAt != null && now >= f.ExpiredAt)
)
.ToListAsync();
if (files.Count == 0)
// Check cache first
foreach (var reference in references)
{
logger.LogInformation("No files to delete");
return;
var cacheKey = $"{CacheKeyPrefix}{reference.Id}";
var cachedFile = await cache.GetAsync<CloudFile>(cacheKey);
if (cachedFile != null)
{
cachedFiles[reference.Id] = cachedFile;
}
else
{
uncachedIds.Add(reference.Id);
}
}
logger.LogInformation($"Found {files.Count} files to process...");
// Load uncached files from database
if (uncachedIds.Count > 0)
{
var dbFiles = await db.Files
.Include(f => f.Account)
.Where(f => uncachedIds.Contains(f.Id))
.ToListAsync();
// Group files by StorageId and find which ones are safe to delete
var storageIds = files.Where(f => f.StorageId != null)
.Select(f => f.StorageId!)
.Distinct()
// Add to cache
foreach (var file in dbFiles)
{
var cacheKey = $"{CacheKeyPrefix}{file.Id}";
await cache.SetAsync(cacheKey, file, CacheDuration);
cachedFiles[file.Id] = file;
}
}
// Preserve original order
return references
.Select(r => cachedFiles.GetValueOrDefault(r.Id))
.Where(f => f != null)
.ToList();
var usedStorageIds = await db.Files
.Where(f => f.StorageId != null &&
storageIds.Contains(f.StorageId) &&
!files.Select(ff => ff.Id).Contains(f.Id))
.Select(f => f.StorageId!)
.Distinct()
.ToListAsync();
// Group files for deletion
var filesToDelete = files.Where(f => f.StorageId == null || !usedStorageIds.Contains(f.StorageId))
.GroupBy(f => f.UploadedTo)
.ToDictionary(grouping => grouping.Key!, grouping => grouping.ToList());
// Delete files by remote storage
foreach (var group in filesToDelete.Where(group => !string.IsNullOrEmpty(group.Key)))
{
try
{
var dest = fs.GetRemoteStorageConfig(group.Key);
var client = fs.CreateMinioClient(dest);
if (client == null) continue;
// Create delete tasks for each file in the group
var deleteTasks = group.Value.Select(file =>
{
var objectId = file.StorageId ?? file.Id;
var tasks = new List<Task>
{
client.RemoveObjectAsync(new RemoveObjectArgs()
.WithBucket(dest.Bucket)
.WithObject(objectId))
};
if (file.HasCompression)
{
tasks.Add(client.RemoveObjectAsync(new RemoveObjectArgs()
.WithBucket(dest.Bucket)
.WithObject(objectId + ".compressed")));
}
return Task.WhenAll(tasks);
});
await Task.WhenAll(deleteTasks);
}
catch (Exception ex)
{
logger.LogError(ex, "Error deleting files from remote storage {remote}", group.Key);
}
}
// Delete all file records from the database
var fileIds = files.Select(f => f.Id).ToList();
await db.Files
.Where(f => fileIds.Contains(f.Id))
.ExecuteDeleteAsync();
logger.LogInformation($"Completed deleting {files.Count} files");
}
}
/// <summary>
/// Gets the number of references to a file based on CloudFileReference records
/// </summary>
/// <param name="fileId">The ID of the file</param>
/// <returns>The number of references to the file</returns>
public async Task<int> GetReferenceCountAsync(string fileId)
{
return await db.FileReferences
.Where(r => r.FileId == fileId)
.CountAsync();
}
/// <summary>
/// Checks if a file is referenced by any resource
/// </summary>
/// <param name="fileId">The ID of the file to check</param>
/// <returns>True if the file is referenced, false otherwise</returns>
public async Task<bool> IsReferencedAsync(string fileId)
{
return await db.FileReferences
.Where(r => r.FileId == fileId)
.AnyAsync();
}
}

49
DysonNetwork.Sphere/Storage/ICloudFile.cs Normal file
View File

@ -0,0 +1,49 @@
namespace DysonNetwork.Sphere.Storage;
/// <summary>
/// Common interface for cloud file entities that can be used in file operations.
/// This interface exposes the essential properties needed for file operations
/// and is implemented by both CloudFile and CloudFileReferenceObject.
/// </summary>
public interface ICloudFile
{
/// <summary>
/// Gets the unique identifier of the cloud file.
/// </summary>
string Id { get; }
/// <summary>
/// Gets the name of the cloud file.
/// </summary>
string Name { get; }
/// <summary>
/// Gets the file metadata dictionary.
/// </summary>
Dictionary<string, object>? FileMeta { get; }
/// <summary>
/// Gets the user metadata dictionary.
/// </summary>
Dictionary<string, object>? UserMeta { get; }
/// <summary>
/// Gets the MIME type of the file.
/// </summary>
string? MimeType { get; }
/// <summary>
/// Gets the hash of the file content.
/// </summary>
string? Hash { get; }
/// <summary>
/// Gets the size of the file in bytes.
/// </summary>
long Size { get; }
/// <summary>
/// Gets whether the file has a compressed version available.
/// </summary>
bool HasCompression { get; }
}

1
DysonNetwork.sln.DotSettings.user
View File

@ -62,6 +62,7 @@
<s:String x:Key="/Default/CodeInspection/ExcludedFiles/FilesAndFoldersToSkip2/=7020124F_002D9FFC_002D4AC3_002D8F3D_002DAAB8E0240759_002Ff_003APresignedGetObjectArgs_002Ecs_002Fl_003A_002E_002E_003F_002E_002E_003F_002E_002E_003FLibrary_003FApplication_0020Support_003FJetBrains_003FRider2024_002E3_003Fresharper_002Dhost_003FSourcesCache_003F0df26a9d89e29319e9efcaea0a8489db9e97bc1aedcca3f7e360cc50f8f4ea_003FPresignedGetObjectArgs_002Ecs/@EntryIndexedValue">ForceIncluded</s:String>
<s:String x:Key="/Default/CodeInspection/ExcludedFiles/FilesAndFoldersToSkip2/=7020124F_002D9FFC_002D4AC3_002D8F3D_002DAAB8E0240759_002Ff_003APutObjectArgs_002Ecs_002Fl_003A_002E_002E_003F_002E_002E_003F_002E_002E_003FLibrary_003FApplication_0020Support_003FJetBrains_003FRider2025_002E1_003Fresharper_002Dhost_003FSourcesCache_003F6efe388c7585d5dd5587416a55298550b030c2a107edf45f988791297c3ffa_003FPutObjectArgs_002Ecs/@EntryIndexedValue">ForceIncluded</s:String>
<s:String x:Key="/Default/CodeInspection/ExcludedFiles/FilesAndFoldersToSkip2/=7020124F_002D9FFC_002D4AC3_002D8F3D_002DAAB8E0240759_002Ff_003AQueryable_002Ecs_002Fl_003A_002E_002E_003F_002E_002E_003F_002E_002E_003FLibrary_003FApplication_0020Support_003FJetBrains_003FRider2025_002E1_003Fresharper_002Dhost_003FDecompilerCache_003Fdecompiler_003F42d8f09d6a294d00a6f49efc989927492fe00_003F4e_003F26d1ee34_003FQueryable_002Ecs/@EntryIndexedValue">ForceIncluded</s:String>
<s:String x:Key="/Default/CodeInspection/ExcludedFiles/FilesAndFoldersToSkip2/=7020124F_002D9FFC_002D4AC3_002D8F3D_002DAAB8E0240759_002Ff_003AQueryable_002Ecs_002Fl_003A_002E_002E_003F_002E_002E_003F_002E_002E_003FLibrary_003FApplication_0020Support_003FJetBrains_003FRider2025_002E1_003Fresharper_002Dhost_003FDecompilerCache_003Fdecompiler_003Fcbafb95b4df34952928f87356db00c8f2fe00_003F9b_003F8ba036bb_003FQueryable_002Ecs/@EntryIndexedValue">ForceIncluded</s:String>
<s:String x:Key="/Default/CodeInspection/ExcludedFiles/FilesAndFoldersToSkip2/=7020124F_002D9FFC_002D4AC3_002D8F3D_002DAAB8E0240759_002Ff_003AResizeOptions_002Ecs_002Fl_003A_002E_002E_003F_002E_002E_003F_002E_002E_003FLibrary_003FApplication_0020Support_003FJetBrains_003FRider2025_002E1_003Fresharper_002Dhost_003FDecompilerCache_003Fdecompiler_003Fef3339e864a448e2b1ec6fa7bbf4c6661fee00_003F48_003F0209e410_003FResizeOptions_002Ecs/@EntryIndexedValue">ForceIncluded</s:String>
<s:String x:Key="/Default/CodeInspection/ExcludedFiles/FilesAndFoldersToSkip2/=7020124F_002D9FFC_002D4AC3_002D8F3D_002DAAB8E0240759_002Ff_003AResourceManagerStringLocalizerFactory_002Ecs_002Fl_003A_002E_002E_003F_002E_002E_003F_002E_002E_003FLibrary_003FApplication_0020Support_003FJetBrains_003FRider2025_002E1_003Fresharper_002Dhost_003FDecompilerCache_003Fdecompiler_003Fb62f365d06c44ad695ff75960cdf97a2a800_003Fe4_003Ff6ba93b7_003FResourceManagerStringLocalizerFactory_002Ecs/@EntryIndexedValue">ForceIncluded</s:String>
<s:String x:Key="/Default/CodeInspection/ExcludedFiles/FilesAndFoldersToSkip2/=7020124F_002D9FFC_002D4AC3_002D8F3D_002DAAB8E0240759_002Ff_003ARSA_002Ecs_002Fl_003A_002E_002E_003F_002E_002E_003F_002E_002E_003FLibrary_003FApplication_0020Support_003FJetBrains_003FRider2025_002E1_003Fresharper_002Dhost_003FDecompilerCache_003Fdecompiler_003Fee4f989f6b8042b59b2654fdc188e287243600_003F8b_003F44e5f855_003FRSA_002Ecs/@EntryIndexedValue">ForceIncluded</s:String>