From e2b2bdd262c9b15402c1e5d3f8e014ee0730d7d2 Mon Sep 17 00:00:00 2001 From: LittleSheep Date: Wed, 12 Nov 2025 22:09:13 +0800 Subject: [PATCH] :sparkles: File index --- .../Properties/launchSettings.json | 2 +- DysonNetwork.Control/aspire-manifest.json | 357 ++++++++++ DysonNetwork.Drive/AppDatabase.cs | 1 + .../Index/FileIndexController.cs | 411 ++++++++++++ DysonNetwork.Drive/Index/FileIndexService.cs | 187 ++++++ DysonNetwork.Drive/Index/README.md | 341 ++++++++++ .../20251112135535_AddFileIndex.Designer.cs | 632 ++++++++++++++++++ .../Migrations/20251112135535_AddFileIndex.cs | 66 ++ .../Migrations/AppDatabaseModelSnapshot.cs | 65 ++ .../Startup/ServiceCollectionExtensions.cs | 2 + .../Storage/CloudFileUnusedRecyclingJob.cs | 3 +- DysonNetwork.Drive/Storage/FileController.cs | 2 +- .../Storage/FileUploadController.cs | 22 +- .../Storage/Model/FileUploadModels.cs | 15 +- .../Storage/PersistentTaskService.cs | 12 +- DysonNetwork.Drive/appsettings.json | 3 - DysonNetwork.Shared/Models/CloudFile.cs | 1 + DysonNetwork.Shared/Models/CloudFileIndex.cs | 30 + DysonNetwork.Shared/Models/Subscription.cs | 99 ++- 19 files changed, 2189 insertions(+), 62 deletions(-) create mode 100644 DysonNetwork.Control/aspire-manifest.json create mode 100644 DysonNetwork.Drive/Index/FileIndexController.cs create mode 100644 DysonNetwork.Drive/Index/FileIndexService.cs create mode 100644 DysonNetwork.Drive/Index/README.md create mode 100644 DysonNetwork.Drive/Migrations/20251112135535_AddFileIndex.Designer.cs create mode 100644 DysonNetwork.Drive/Migrations/20251112135535_AddFileIndex.cs create mode 100644 DysonNetwork.Shared/Models/CloudFileIndex.cs diff --git a/DysonNetwork.Control/Properties/launchSettings.json b/DysonNetwork.Control/Properties/launchSettings.json index 169f45b..b445d76 100644 --- a/DysonNetwork.Control/Properties/launchSettings.json +++ b/DysonNetwork.Control/Properties/launchSettings.json @@ -5,7 +5,7 @@ "commandName": "Project", "dotnetRunMessages": true, "launchBrowser": true, - "applicationUrl": "https://localhost:17025;http://localhost:15057", + "applicationUrl": "https://localhost:17169;http://localhost:15057", "environmentVariables": { "ASPNETCORE_ENVIRONMENT": "Development", "DOTNET_ENVIRONMENT": "Development", diff --git a/DysonNetwork.Control/aspire-manifest.json b/DysonNetwork.Control/aspire-manifest.json new file mode 100644 index 0000000..69ad8b5 --- /dev/null +++ b/DysonNetwork.Control/aspire-manifest.json @@ -0,0 +1,357 @@ +{ + "$schema": "https://json.schemastore.org/aspire-8.0.json", + "resources": { + "cache": { + "type": "container.v1", + "connectionString": "{cache.bindings.tcp.host}:{cache.bindings.tcp.port},password={cache-password.value}", + "image": "docker.io/library/redis:8.2", + "entrypoint": "/bin/sh", + "args": [ + "-c", + "redis-server --requirepass $REDIS_PASSWORD" + ], + "env": { + "REDIS_PASSWORD": "{cache-password.value}" + }, + "bindings": { + "tcp": { + "scheme": "tcp", + "protocol": "tcp", + "transport": "tcp", + "targetPort": 6379 + } + } + }, + "queue": { + "type": "container.v1", + "connectionString": "nats://nats:{queue-password.value}@{queue.bindings.tcp.host}:{queue.bindings.tcp.port}", + "image": "docker.io/library/nats:2.11", + "args": [ + "--user", + "nats", + "--pass", + "{queue-password.value}", + "-js" + ], + "bindings": { + "tcp": { + "scheme": "tcp", + "protocol": "tcp", + "transport": "tcp", + "targetPort": 4222 + } + } + }, + "ring": { + "type": "project.v1", + "path": "../DysonNetwork.Ring/DysonNetwork.Ring.csproj", + "env": { + "OTEL_DOTNET_EXPERIMENTAL_OTLP_EMIT_EXCEPTION_LOG_ATTRIBUTES": "true", + "OTEL_DOTNET_EXPERIMENTAL_OTLP_EMIT_EVENT_LOG_ATTRIBUTES": "true", + "OTEL_DOTNET_EXPERIMENTAL_OTLP_RETRY": "in_memory", + "ASPNETCORE_FORWARDEDHEADERS_ENABLED": "true", + "HTTP_PORTS": "8001", + "HTTPS_PORTS": "{ring.bindings.grpc.targetPort}", + "ConnectionStrings__cache": "{cache.connectionString}", + "ConnectionStrings__queue": "{queue.connectionString}", + "GRPC_PORT": "7002", + "services__pass__http__0": "{pass.bindings.http.url}", + "services__pass__grpc__0": "{pass.bindings.grpc.url}", + "OTEL_EXPORTER_OTLP_ENDPOINT": "{docker-compose-dashboard.bindings.otlp-grpc.url}", + "OTEL_EXPORTER_OTLP_PROTOCOL": "grpc", + "OTEL_SERVICE_NAME": "ring" + }, + "bindings": { + "http": { + "scheme": "http", + "protocol": "tcp", + "transport": "http", + "targetPort": 8001 + }, + "grpc": { + "scheme": "https", + "protocol": "tcp", + "transport": "http", + "targetPort": 7002 + } + } + }, + "pass": { + "type": "project.v1", + "path": "../DysonNetwork.Pass/DysonNetwork.Pass.csproj", + "env": { + "OTEL_DOTNET_EXPERIMENTAL_OTLP_EMIT_EXCEPTION_LOG_ATTRIBUTES": "true", + "OTEL_DOTNET_EXPERIMENTAL_OTLP_EMIT_EVENT_LOG_ATTRIBUTES": "true", + "OTEL_DOTNET_EXPERIMENTAL_OTLP_RETRY": "in_memory", + "ASPNETCORE_FORWARDEDHEADERS_ENABLED": "true", + "HTTP_PORTS": "8002", + "HTTPS_PORTS": "{pass.bindings.grpc.targetPort}", + "services__ring__http__0": "{ring.bindings.http.url}", + "services__ring__grpc__0": "{ring.bindings.grpc.url}", + "services__develop__http__0": "{develop.bindings.http.url}", + "services__develop__grpc__0": "{develop.bindings.grpc.url}", + "services__drive__http__0": "{drive.bindings.http.url}", + "services__drive__grpc__0": "{drive.bindings.grpc.url}", + "ConnectionStrings__cache": "{cache.connectionString}", + "ConnectionStrings__queue": "{queue.connectionString}", + "GRPC_PORT": "7003", + "OTEL_EXPORTER_OTLP_ENDPOINT": "{docker-compose-dashboard.bindings.otlp-grpc.url}", + "OTEL_EXPORTER_OTLP_PROTOCOL": "grpc", + "OTEL_SERVICE_NAME": "pass" + }, + "bindings": { + "http": { + "scheme": "http", + "protocol": "tcp", + "transport": "http", + "targetPort": 8002 + }, + "grpc": { + "scheme": "https", + "protocol": "tcp", + "transport": "http", + "targetPort": 7003 + } + } + }, + "drive": { + "type": "project.v1", + "path": "../DysonNetwork.Drive/DysonNetwork.Drive.csproj", + "env": { + "OTEL_DOTNET_EXPERIMENTAL_OTLP_EMIT_EXCEPTION_LOG_ATTRIBUTES": "true", + "OTEL_DOTNET_EXPERIMENTAL_OTLP_EMIT_EVENT_LOG_ATTRIBUTES": "true", + "OTEL_DOTNET_EXPERIMENTAL_OTLP_RETRY": "in_memory", + "ASPNETCORE_FORWARDEDHEADERS_ENABLED": "true", + "HTTP_PORTS": "8003", + "HTTPS_PORTS": "{drive.bindings.grpc.targetPort}", + "services__pass__http__0": "{pass.bindings.http.url}", + "services__pass__grpc__0": "{pass.bindings.grpc.url}", + "services__ring__http__0": "{ring.bindings.http.url}", + "services__ring__grpc__0": "{ring.bindings.grpc.url}", + "ConnectionStrings__cache": "{cache.connectionString}", + "ConnectionStrings__queue": "{queue.connectionString}", + "GRPC_PORT": "7004", + "OTEL_EXPORTER_OTLP_ENDPOINT": "{docker-compose-dashboard.bindings.otlp-grpc.url}", + "OTEL_EXPORTER_OTLP_PROTOCOL": "grpc", + "OTEL_SERVICE_NAME": "drive" + }, + "bindings": { + "http": { + "scheme": "http", + "protocol": "tcp", + "transport": "http", + "targetPort": 8003 + }, + "grpc": { + "scheme": "https", + "protocol": "tcp", + "transport": "http", + "targetPort": 7004 + } + } + }, + "sphere": { + "type": "project.v1", + "path": "../DysonNetwork.Sphere/DysonNetwork.Sphere.csproj", + "env": { + "OTEL_DOTNET_EXPERIMENTAL_OTLP_EMIT_EXCEPTION_LOG_ATTRIBUTES": "true", + "OTEL_DOTNET_EXPERIMENTAL_OTLP_EMIT_EVENT_LOG_ATTRIBUTES": "true", + "OTEL_DOTNET_EXPERIMENTAL_OTLP_RETRY": "in_memory", + "ASPNETCORE_FORWARDEDHEADERS_ENABLED": "true", + "HTTP_PORTS": "8004", + "HTTPS_PORTS": "{sphere.bindings.grpc.targetPort}", + "services__pass__http__0": "{pass.bindings.http.url}", + "services__pass__grpc__0": "{pass.bindings.grpc.url}", + "services__ring__http__0": "{ring.bindings.http.url}", + "services__ring__grpc__0": "{ring.bindings.grpc.url}", + "services__drive__http__0": "{drive.bindings.http.url}", + "services__drive__grpc__0": "{drive.bindings.grpc.url}", + "ConnectionStrings__cache": "{cache.connectionString}", + "ConnectionStrings__queue": "{queue.connectionString}", + "GRPC_PORT": "7005", + "OTEL_EXPORTER_OTLP_ENDPOINT": "{docker-compose-dashboard.bindings.otlp-grpc.url}", + "OTEL_EXPORTER_OTLP_PROTOCOL": "grpc", + "OTEL_SERVICE_NAME": "sphere" + }, + "bindings": { + "http": { + "scheme": "http", + "protocol": "tcp", + "transport": "http", + "targetPort": 8004 + }, + "grpc": { + "scheme": "https", + "protocol": "tcp", + "transport": "http", + "targetPort": 7005 + } + } + }, + "develop": { + "type": "project.v1", + "path": "../DysonNetwork.Develop/DysonNetwork.Develop.csproj", + "env": { + "OTEL_DOTNET_EXPERIMENTAL_OTLP_EMIT_EXCEPTION_LOG_ATTRIBUTES": "true", + "OTEL_DOTNET_EXPERIMENTAL_OTLP_EMIT_EVENT_LOG_ATTRIBUTES": "true", + "OTEL_DOTNET_EXPERIMENTAL_OTLP_RETRY": "in_memory", + "ASPNETCORE_FORWARDEDHEADERS_ENABLED": "true", + "HTTP_PORTS": "8005", + "HTTPS_PORTS": "{develop.bindings.grpc.targetPort}", + "services__pass__http__0": "{pass.bindings.http.url}", + "services__pass__grpc__0": "{pass.bindings.grpc.url}", + "services__ring__http__0": "{ring.bindings.http.url}", + "services__ring__grpc__0": "{ring.bindings.grpc.url}", + "services__sphere__http__0": "{sphere.bindings.http.url}", + "services__sphere__grpc__0": "{sphere.bindings.grpc.url}", + "ConnectionStrings__cache": "{cache.connectionString}", + "ConnectionStrings__queue": "{queue.connectionString}", + "GRPC_PORT": "7006", + "OTEL_EXPORTER_OTLP_ENDPOINT": "{docker-compose-dashboard.bindings.otlp-grpc.url}", + "OTEL_EXPORTER_OTLP_PROTOCOL": "grpc", + "OTEL_SERVICE_NAME": "develop" + }, + "bindings": { + "http": { + "scheme": "http", + "protocol": "tcp", + "transport": "http", + "targetPort": 8005 + }, + "grpc": { + "scheme": "https", + "protocol": "tcp", + "transport": "http", + "targetPort": 7006 + } + } + }, + "insight": { + "type": "project.v1", + "path": "../DysonNetwork.Insight/DysonNetwork.Insight.csproj", + "env": { + "OTEL_DOTNET_EXPERIMENTAL_OTLP_EMIT_EXCEPTION_LOG_ATTRIBUTES": "true", + "OTEL_DOTNET_EXPERIMENTAL_OTLP_EMIT_EVENT_LOG_ATTRIBUTES": "true", + "OTEL_DOTNET_EXPERIMENTAL_OTLP_RETRY": "in_memory", + "ASPNETCORE_FORWARDEDHEADERS_ENABLED": "true", + "HTTP_PORTS": "8006", + "HTTPS_PORTS": "{insight.bindings.grpc.targetPort}", + "services__pass__http__0": "{pass.bindings.http.url}", + "services__pass__grpc__0": "{pass.bindings.grpc.url}", + "services__ring__http__0": "{ring.bindings.http.url}", + "services__ring__grpc__0": "{ring.bindings.grpc.url}", + "services__sphere__http__0": "{sphere.bindings.http.url}", + "services__sphere__grpc__0": "{sphere.bindings.grpc.url}", + "services__develop__http__0": "{develop.bindings.http.url}", + "services__develop__grpc__0": "{develop.bindings.grpc.url}", + "ConnectionStrings__cache": "{cache.connectionString}", + "ConnectionStrings__queue": "{queue.connectionString}", + "GRPC_PORT": "7007", + "OTEL_EXPORTER_OTLP_ENDPOINT": "{docker-compose-dashboard.bindings.otlp-grpc.url}", + "OTEL_EXPORTER_OTLP_PROTOCOL": "grpc", + "OTEL_SERVICE_NAME": "insight" + }, + "bindings": { + "http": { + "scheme": "http", + "protocol": "tcp", + "transport": "http", + "targetPort": 8006 + }, + "grpc": { + "scheme": "https", + "protocol": "tcp", + "transport": "http", + "targetPort": 7007 + } + } + }, + "gateway": { + "type": "project.v1", + "path": "../DysonNetwork.Gateway/DysonNetwork.Gateway.csproj", + "env": { + "OTEL_DOTNET_EXPERIMENTAL_OTLP_EMIT_EXCEPTION_LOG_ATTRIBUTES": "true", + "OTEL_DOTNET_EXPERIMENTAL_OTLP_EMIT_EVENT_LOG_ATTRIBUTES": "true", + "OTEL_DOTNET_EXPERIMENTAL_OTLP_RETRY": "in_memory", + "ASPNETCORE_FORWARDEDHEADERS_ENABLED": "true", + "HTTP_PORTS": "5001", + "services__ring__http__0": "{ring.bindings.http.url}", + "services__ring__grpc__0": "{ring.bindings.grpc.url}", + "services__pass__http__0": "{pass.bindings.http.url}", + "services__pass__grpc__0": "{pass.bindings.grpc.url}", + "services__drive__http__0": "{drive.bindings.http.url}", + "services__drive__grpc__0": "{drive.bindings.grpc.url}", + "services__sphere__http__0": "{sphere.bindings.http.url}", + "services__sphere__grpc__0": "{sphere.bindings.grpc.url}", + "services__develop__http__0": "{develop.bindings.http.url}", + "services__develop__grpc__0": "{develop.bindings.grpc.url}", + "services__insight__http__0": "{insight.bindings.http.url}", + "services__insight__grpc__0": "{insight.bindings.grpc.url}", + "OTEL_EXPORTER_OTLP_ENDPOINT": "{docker-compose-dashboard.bindings.otlp-grpc.url}", + "OTEL_EXPORTER_OTLP_PROTOCOL": "grpc", + "OTEL_SERVICE_NAME": "gateway" + }, + "bindings": { + "http": { + "scheme": "http", + "protocol": "tcp", + "transport": "http", + "targetPort": 5001 + } + } + }, + "docker-compose": { + "error": "This resource does not support generation in the manifest." + }, + "cache-password": { + "type": "parameter.v0", + "value": "{cache-password.inputs.value}", + "inputs": { + "value": { + "type": "string", + "secret": true, + "default": { + "generate": { + "minLength": 22, + "special": false + } + } + } + } + }, + "queue-password": { + "type": "parameter.v0", + "value": "{queue-password.inputs.value}", + "inputs": { + "value": { + "type": "string", + "secret": true, + "default": { + "generate": { + "minLength": 22, + "special": false + } + } + } + } + }, + "docker-compose-dashboard": { + "type": "container.v1", + "image": "mcr.microsoft.com/dotnet/nightly/aspire-dashboard:latest", + "bindings": { + "http": { + "scheme": "http", + "protocol": "tcp", + "transport": "http", + "targetPort": 18888 + }, + "otlp-grpc": { + "scheme": "http", + "protocol": "tcp", + "transport": "http", + "targetPort": 18889 + } + } + } + } +} \ No newline at end of file diff --git a/DysonNetwork.Drive/AppDatabase.cs b/DysonNetwork.Drive/AppDatabase.cs index e3dbfd4..da5176d 100644 --- a/DysonNetwork.Drive/AppDatabase.cs +++ b/DysonNetwork.Drive/AppDatabase.cs @@ -25,6 +25,7 @@ public class AppDatabase( public DbSet Files { get; set; } = null!; public DbSet FileReferences { get; set; } = null!; + public DbSet FileIndexes { get; set; } public DbSet Tasks { get; set; } = null!; public DbSet UploadTasks { get; set; } = null!; // Backward compatibility diff --git a/DysonNetwork.Drive/Index/FileIndexController.cs b/DysonNetwork.Drive/Index/FileIndexController.cs new file mode 100644 index 0000000..c66bce9 --- /dev/null +++ b/DysonNetwork.Drive/Index/FileIndexController.cs @@ -0,0 +1,411 @@ +using System.ComponentModel.DataAnnotations; +using DysonNetwork.Drive.Storage; +using DysonNetwork.Shared.Auth; +using DysonNetwork.Shared.Http; +using DysonNetwork.Shared.Models; +using DysonNetwork.Shared.Proto; +using Microsoft.AspNetCore.Authorization; +using Microsoft.AspNetCore.Mvc; +using Microsoft.EntityFrameworkCore; + +namespace DysonNetwork.Drive.Index; + +[ApiController] +[Route("/api/index")] +[Authorize] +public class FileIndexController( + FileIndexService fileIndexService, + AppDatabase db, + ILogger logger +) : ControllerBase +{ + /// + /// Gets files in a specific path for the current user + /// + /// The path to browse (defaults to root "/") + /// List of files in the specified path + [HttpGet("browse")] + public async Task BrowseFiles([FromQuery] string path = "/") + { + if (HttpContext.Items["CurrentUser"] is not Account currentUser) + return new ObjectResult(ApiError.Unauthorized()) { StatusCode = 401 }; + + var accountId = Guid.Parse(currentUser.Id); + + try + { + var fileIndexes = await fileIndexService.GetByPathAsync(accountId, path); + + return Ok(new + { + Path = path, + Files = fileIndexes, + TotalCount = fileIndexes.Count + }); + } + catch (Exception ex) + { + logger.LogError(ex, "Failed to browse files for account {AccountId} at path {Path}", accountId, path); + return new ObjectResult(new ApiError + { + Code = "BROWSE_FAILED", + Message = "Failed to browse files", + Status = 500 + }) { StatusCode = 500 }; + } + } + + /// + /// Gets all files for the current user (across all paths) + /// + /// List of all files for the user + [HttpGet("all")] + public async Task GetAllFiles() + { + if (HttpContext.Items["CurrentUser"] is not Account currentUser) + return new ObjectResult(ApiError.Unauthorized()) { StatusCode = 401 }; + + var accountId = Guid.Parse(currentUser.Id); + + try + { + var fileIndexes = await fileIndexService.GetByAccountIdAsync(accountId); + + return Ok(new + { + Files = fileIndexes, + TotalCount = fileIndexes.Count() + }); + } + catch (Exception ex) + { + logger.LogError(ex, "Failed to get all files for account {AccountId}", accountId); + return new ObjectResult(new ApiError + { + Code = "GET_ALL_FAILED", + Message = "Failed to get files", + Status = 500 + }) { StatusCode = 500 }; + } + } + + /// + /// Moves a file to a new path + /// + /// The file index ID + /// The new path + /// The updated file index + [HttpPost("move/{indexId}")] + public async Task MoveFile(Guid indexId, [FromBody] MoveFileRequest request) + { + if (HttpContext.Items["CurrentUser"] is not Account currentUser) + return new ObjectResult(ApiError.Unauthorized()) { StatusCode = 401 }; + + var accountId = Guid.Parse(currentUser.Id); + + try + { + // Verify ownership + var existingIndex = await db.FileIndexes + .Include(fi => fi.File) + .FirstOrDefaultAsync(fi => fi.Id == indexId && fi.AccountId == accountId); + + if (existingIndex == null) + return new ObjectResult(ApiError.NotFound("File index")) { StatusCode = 404 }; + + var updatedIndex = await fileIndexService.UpdateAsync(indexId, request.NewPath); + + if (updatedIndex == null) + return new ObjectResult(ApiError.NotFound("File index")) { StatusCode = 404 }; + + return Ok(new + { + updatedIndex.FileId, + IndexId = updatedIndex.Id, + OldPath = existingIndex.Path, + NewPath = updatedIndex.Path, + Message = "File moved successfully" + }); + } + catch (Exception ex) + { + logger.LogError(ex, "Failed to move file index {IndexId} for account {AccountId}", indexId, accountId); + return new ObjectResult(new ApiError + { + Code = "MOVE_FAILED", + Message = "Failed to move file", + Status = 500 + }) { StatusCode = 500 }; + } + } + + /// + /// Removes a file index (does not delete the actual file by default) + /// + /// The file index ID + /// Whether to also delete the actual file data + /// Success message + [HttpDelete("remove/{indexId}")] + public async Task RemoveFileIndex(Guid indexId, [FromQuery] bool deleteFile = false) + { + if (HttpContext.Items["CurrentUser"] is not Account currentUser) + return new ObjectResult(ApiError.Unauthorized()) { StatusCode = 401 }; + + var accountId = Guid.Parse(currentUser.Id); + + try + { + // Verify ownership + var existingIndex = await db.FileIndexes + .Include(fi => fi.File) + .FirstOrDefaultAsync(fi => fi.Id == indexId && fi.AccountId == accountId); + + if (existingIndex == null) + return new ObjectResult(ApiError.NotFound("File index")) { StatusCode = 404 }; + + var fileId = existingIndex.FileId; + var fileName = existingIndex.File.Name; + var filePath = existingIndex.Path; + + // Remove the index + var removed = await fileIndexService.RemoveAsync(indexId); + + if (!removed) + return new ObjectResult(ApiError.NotFound("File index")) { StatusCode = 404 }; + + // Optionally delete the actual file + if (!deleteFile) + return Ok(new + { + Message = deleteFile + ? "File index and file data removed successfully" + : "File index removed successfully", + FileId = fileId, + FileName = fileName, + Path = filePath, + FileDataDeleted = deleteFile + }); + try + { + // Check if there are any other indexes for this file + var remainingIndexes = await fileIndexService.GetByFileIdAsync(fileId); + if (remainingIndexes.Count == 0) + { + // No other indexes exist, safe to delete the file + var file = await db.Files.FirstOrDefaultAsync(f => f.Id == fileId.ToString()); + if (file != null) + { + db.Files.Remove(file); + await db.SaveChangesAsync(); + logger.LogInformation("Deleted file {FileId} ({FileName}) as requested", fileId, fileName); + } + } + } + catch (Exception ex) + { + logger.LogWarning(ex, "Failed to delete file {FileId} while removing index", fileId); + // Continue even if file deletion fails + } + + return Ok(new + { + Message = deleteFile ? "File index and file data removed successfully" : "File index removed successfully", + FileId = fileId, + FileName = fileName, + Path = filePath, + FileDataDeleted = deleteFile + }); + } + catch (Exception ex) + { + logger.LogError(ex, "Failed to remove file index {IndexId} for account {AccountId}", indexId, accountId); + return new ObjectResult(new ApiError + { + Code = "REMOVE_FAILED", + Message = "Failed to remove file", + Status = 500 + }) { StatusCode = 500 }; + } + } + + /// + /// Removes all file indexes in a specific path + /// + /// The path to clear + /// Whether to also delete the actual file data + /// Success message with count of removed items + [HttpDelete("clear-path")] + public async Task ClearPath([FromQuery] string path = "/", [FromQuery] bool deleteFiles = false) + { + if (HttpContext.Items["CurrentUser"] is not Account currentUser) + return new ObjectResult(ApiError.Unauthorized()) { StatusCode = 401 }; + + var accountId = Guid.Parse(currentUser.Id); + + try + { + var removedCount = await fileIndexService.RemoveByPathAsync(accountId, path); + + if (!deleteFiles || removedCount <= 0) + return Ok(new + { + Message = deleteFiles + ? $"Cleared {removedCount} file indexes from path and deleted orphaned files" + : $"Cleared {removedCount} file indexes from path", + Path = path, + RemovedCount = removedCount, + FilesDeleted = deleteFiles + }); + // Get the files that were in this path and check if they have other indexes + var filesInPath = await fileIndexService.GetByPathAsync(accountId, path); + var fileIdsToCheck = filesInPath.Select(fi => fi.FileId).Distinct().ToList(); + + foreach (var fileId in fileIdsToCheck) + { + var remainingIndexes = await fileIndexService.GetByFileIdAsync(fileId); + if (remainingIndexes.Count != 0) continue; + // No other indexes exist, safe to delete the file + var file = await db.Files.FirstOrDefaultAsync(f => f.Id == fileId.ToString()); + if (file == null) continue; + db.Files.Remove(file); + logger.LogInformation("Deleted orphaned file {FileId} after clearing path {Path}", fileId, path); + } + await db.SaveChangesAsync(); + + return Ok(new + { + Message = deleteFiles ? + $"Cleared {removedCount} file indexes from path and deleted orphaned files" : + $"Cleared {removedCount} file indexes from path", + Path = path, + RemovedCount = removedCount, + FilesDeleted = deleteFiles + }); + } + catch (Exception ex) + { + logger.LogError(ex, "Failed to clear path {Path} for account {AccountId}", path, accountId); + return new ObjectResult(new ApiError + { + Code = "CLEAR_PATH_FAILED", + Message = "Failed to clear path", + Status = 500 + }) { StatusCode = 500 }; + } + } + + /// + /// Creates a new file index (useful for adding existing files to a path) + /// + /// The create index request + /// The created file index + [HttpPost("create")] + public async Task CreateFileIndex([FromBody] CreateFileIndexRequest request) + { + if (HttpContext.Items["CurrentUser"] is not Account currentUser) + return new ObjectResult(ApiError.Unauthorized()) { StatusCode = 401 }; + + var accountId = Guid.Parse(currentUser.Id); + + try + { + // Verify the file exists and belongs to the user + var file = await db.Files.FirstOrDefaultAsync(f => f.Id == request.FileId); + if (file == null) + return new ObjectResult(ApiError.NotFound("File")) { StatusCode = 404 }; + + if (file.AccountId != accountId) + return new ObjectResult(ApiError.Unauthorized(forbidden: true)) { StatusCode = 403 }; + + // Check if index already exists for this file and path + var existingIndex = await db.FileIndexes + .FirstOrDefaultAsync(fi => fi.FileId == request.FileId && fi.Path == request.Path && fi.AccountId == accountId); + + if (existingIndex != null) + return new ObjectResult(ApiError.Validation(new Dictionary + { + { "fileId", ["File index already exists for this path"] } + })) { StatusCode = 400 }; + + var fileIndex = await fileIndexService.CreateAsync(request.Path, request.FileId, accountId); + + return Ok(new + { + IndexId = fileIndex.Id, + fileIndex.FileId, + fileIndex.Path, + Message = "File index created successfully" + }); + } + catch (Exception ex) + { + logger.LogError(ex, "Failed to create file index for file {FileId} at path {Path} for account {AccountId}", + request.FileId, request.Path, accountId); + return new ObjectResult(new ApiError + { + Code = "CREATE_INDEX_FAILED", + Message = "Failed to create file index", + Status = 500 + }) { StatusCode = 500 }; + } + } + + /// + /// Searches for files by name or metadata + /// + /// The search query + /// Optional path to limit search to + /// Matching files + [HttpGet("search")] + public async Task SearchFiles([FromQuery] string query, [FromQuery] string? path = null) + { + if (HttpContext.Items["CurrentUser"] is not Account currentUser) + return new ObjectResult(ApiError.Unauthorized()) { StatusCode = 401 }; + + var accountId = Guid.Parse(currentUser.Id); + + try + { + // Build the query with all conditions at once + var searchTerm = query.ToLower(); + var fileIndexes = await db.FileIndexes + .Where(fi => fi.AccountId == accountId) + .Include(fi => fi.File) + .Where(fi => + (string.IsNullOrEmpty(path) || fi.Path == FileIndexService.NormalizePath(path)) && + (fi.File.Name.ToLower().Contains(searchTerm) || + (fi.File.Description != null && fi.File.Description.ToLower().Contains(searchTerm)) || + (fi.File.MimeType != null && fi.File.MimeType.ToLower().Contains(searchTerm)))) + .ToListAsync(); + + return Ok(new + { + Query = query, + Path = path, + Results = fileIndexes, + TotalCount = fileIndexes.Count() + }); + } + catch (Exception ex) + { + logger.LogError(ex, "Failed to search files for account {AccountId} with query {Query}", accountId, query); + return new ObjectResult(new ApiError + { + Code = "SEARCH_FAILED", + Message = "Failed to search files", + Status = 500 + }) { StatusCode = 500 }; + } + } +} + +public class MoveFileRequest +{ + public string NewPath { get; set; } = null!; +} + +public class CreateFileIndexRequest +{ + [MaxLength(32)] public string FileId { get; set; } = null!; + public string Path { get; set; } = null!; +} diff --git a/DysonNetwork.Drive/Index/FileIndexService.cs b/DysonNetwork.Drive/Index/FileIndexService.cs new file mode 100644 index 0000000..0707e92 --- /dev/null +++ b/DysonNetwork.Drive/Index/FileIndexService.cs @@ -0,0 +1,187 @@ +using DysonNetwork.Shared.Models; +using Microsoft.EntityFrameworkCore; + +namespace DysonNetwork.Drive.Index; + +public class FileIndexService(AppDatabase db) +{ + /// + /// Creates a new file index entry + /// + /// The parent folder path with trailing slash + /// The file ID + /// The account ID + /// The created file index + public async Task CreateAsync(string path, string fileId, Guid accountId) + { + // Ensure a path has trailing slash and is query-safe + var normalizedPath = NormalizePath(path); + + var fileIndex = new SnCloudFileIndex + { + Path = normalizedPath, + FileId = fileId, + AccountId = accountId + }; + + db.FileIndexes.Add(fileIndex); + await db.SaveChangesAsync(); + + return fileIndex; + } + + /// + /// Updates an existing file index entry by removing the old one and creating a new one + /// + /// The file index ID + /// The new parent folder path with trailing slash + /// The updated file index + public async Task UpdateAsync(Guid id, string newPath) + { + var fileIndex = await db.FileIndexes.FindAsync(id); + if (fileIndex == null) + return null; + + // Since properties are init-only, we need to remove the old index and create a new one + db.FileIndexes.Remove(fileIndex); + + var newFileIndex = new SnCloudFileIndex + { + Path = NormalizePath(newPath), + FileId = fileIndex.FileId, + AccountId = fileIndex.AccountId + }; + + db.FileIndexes.Add(newFileIndex); + await db.SaveChangesAsync(); + + return newFileIndex; + } + + /// + /// Removes a file index entry by ID + /// + /// The file index ID + /// True if the index was found and removed, false otherwise + public async Task RemoveAsync(Guid id) + { + var fileIndex = await db.FileIndexes.FindAsync(id); + if (fileIndex == null) + return false; + + db.FileIndexes.Remove(fileIndex); + await db.SaveChangesAsync(); + + return true; + } + + /// + /// Removes file index entries by file ID + /// + /// The file ID + /// The number of indexes removed + public async Task RemoveByFileIdAsync(string fileId) + { + var indexes = await db.FileIndexes + .Where(fi => fi.FileId == fileId) + .ToListAsync(); + + if (indexes.Count == 0) + return 0; + + db.FileIndexes.RemoveRange(indexes); + await db.SaveChangesAsync(); + + return indexes.Count; + } + + /// + /// Removes file index entries by account ID and path + /// + /// The account ID + /// The parent folder path + /// The number of indexes removed + public async Task RemoveByPathAsync(Guid accountId, string path) + { + var normalizedPath = NormalizePath(path); + + var indexes = await db.FileIndexes + .Where(fi => fi.AccountId == accountId && fi.Path == normalizedPath) + .ToListAsync(); + + if (!indexes.Any()) + return 0; + + db.FileIndexes.RemoveRange(indexes); + await db.SaveChangesAsync(); + + return indexes.Count; + } + + /// + /// Gets file indexes by account ID and path + /// + /// The account ID + /// The parent folder path + /// List of file indexes + public async Task> GetByPathAsync(Guid accountId, string path) + { + var normalizedPath = NormalizePath(path); + + return await db.FileIndexes + .Where(fi => fi.AccountId == accountId && fi.Path == normalizedPath) + .Include(fi => fi.File) + .ToListAsync(); + } + + /// + /// Gets file indexes by file ID + /// + /// The file ID + /// List of file indexes + public async Task> GetByFileIdAsync(string fileId) + { + return await db.FileIndexes + .Where(fi => fi.FileId == fileId) + .Include(fi => fi.File) + .ToListAsync(); + } + + /// + /// Gets all file indexes for an account + /// + /// The account ID + /// List of file indexes + public async Task> GetByAccountIdAsync(Guid accountId) + { + return await db.FileIndexes + .Where(fi => fi.AccountId == accountId) + .Include(fi => fi.File) + .ToListAsync(); + } + + /// + /// Normalizes the path to ensure it has a trailing slash and is query-safe + /// + /// The original path + /// The normalized path + public static string NormalizePath(string path) + { + if (string.IsNullOrEmpty(path)) + return "/"; + + // Ensure the path starts with a slash + if (!path.StartsWith('/')) + path = "/" + path; + + // Ensure the path ends with a slash (unless it's just the root) + if (path != "/" && !path.EndsWith('/')) + path += "/"; + + // Make path query-safe by removing problematic characters + // This is a basic implementation - you might want to add more robust validation + path = path.Replace("%", "").Replace("'", "").Replace("\"", ""); + + return path; + } +} diff --git a/DysonNetwork.Drive/Index/README.md b/DysonNetwork.Drive/Index/README.md new file mode 100644 index 0000000..2a72155 --- /dev/null +++ b/DysonNetwork.Drive/Index/README.md @@ -0,0 +1,341 @@ +# File Indexing System Documentation + +## Overview + +The File Indexing System provides a hierarchical file organization layer on top of the existing file storage system in DysonNetwork Drive. It allows users to organize their files in folders and paths while maintaining the underlying file storage capabilities. + +When using with the gateway, replace the `/api` with the `/drive` in the path. +And all the arguments will be transformed into snake case via the gateway. + +## Architecture + +### Core Components + +1. **SnCloudFileIndex Model** - Represents the file-to-path mapping +2. **FileIndexService** - Business logic for file index operations +3. **FileIndexController** - REST API endpoints for file management +4. **FileUploadController Integration** - Automatic index creation during upload + +### Database Schema + +```sql +-- File Indexes table +CREATE TABLE "FileIndexes" ( + "Id" uuid NOT NULL DEFAULT gen_random_uuid(), + "Path" character varying(8192) NOT NULL, + "FileId" uuid NOT NULL, + "AccountId" uuid NOT NULL, + "CreatedAt" timestamp with time zone NOT NULL DEFAULT (now() at time zone 'utc'), + "UpdatedAt" timestamp with time zone NOT NULL DEFAULT (now() at time zone 'utc'), + CONSTRAINT "PK_FileIndexes" PRIMARY KEY ("Id"), + CONSTRAINT "FK_FileIndexes_Files_FileId" FOREIGN KEY ("FileId") REFERENCES "Files" ("Id") ON DELETE CASCADE, + INDEX "IX_FileIndexes_Path_AccountId" ("Path", "AccountId") +); +``` + +## API Endpoints + +### Browse Files +**GET** `/api/index/browse?path=/documents/` + +Browse files in a specific path. + +**Query Parameters:** +- `path` (optional, default: "/") - The path to browse + +**Response:** +```json +{ + "path": "/documents/", + "files": [ + { + "id": "guid", + "path": "/documents/", + "fileId": "guid", + "accountId": "guid", + "createdAt": "2024-01-01T00:00:00Z", + "updatedAt": "2024-01-01T00:00:00Z", + "file": { + "id": "string", + "name": "document.pdf", + "size": 1024, + "mimeType": "application/pdf", + "hash": "sha256-hash", + "uploadedAt": "2024-01-01T00:00:00Z", + "expiredAt": null, + "hasCompression": false, + "hasThumbnail": true, + "isEncrypted": false, + "description": null + } + } + ], + "totalCount": 1 +} +``` + +### Get All Files +**GET** `/api/index/all` + +Get all files for the current user across all paths. + +**Response:** +```json +{ + "files": [ + // Same structure as browse endpoint + ], + "totalCount": 10 +} +``` + +### Move File +**POST** `/api/index/move/{indexId}` + +Move a file to a new path. + +**Path Parameters:** +- `indexId` - The file index ID + +**Request Body:** +```json +{ + "newPath": "/archived/" +} +``` + +**Response:** +```json +{ + "fileId": "guid", + "indexId": "guid", + "oldPath": "/documents/", + "newPath": "/archived/", + "message": "File moved successfully" +} +``` + +### Remove File Index +**DELETE** `/api/index/remove/{indexId}?deleteFile=false` + +Remove a file index. Optionally delete the actual file data. + +**Path Parameters:** +- `indexId` - The file index ID + +**Query Parameters:** +- `deleteFile` (optional, default: false) - Whether to also delete the file data + +**Response:** +```json +{ + "message": "File index removed successfully", + "fileId": "guid", + "fileName": "document.pdf", + "path": "/documents/", + "fileDataDeleted": false +} +``` + +### Clear Path +**DELETE** `/api/index/clear-path?path=/temp/&deleteFiles=false` + +Remove all file indexes in a specific path. + +**Query Parameters:** +- `path` (optional, default: "/") - The path to clear +- `deleteFiles` (optional, default: false) - Whether to also delete orphaned files + +**Response:** +```json +{ + "message": "Cleared 5 file indexes from path", + "path": "/temp/", + "removedCount": 5, + "filesDeleted": false +} +``` + +### Create File Index +**POST** `/api/index/create` + +Create a new file index for an existing file. + +**Request Body:** +```json +{ + "fileId": "guid", + "path": "/documents/" +} +``` + +**Response:** +```json +{ + "indexId": "guid", + "fileId": "guid", + "path": "/documents/", + "message": "File index created successfully" +} +``` + +### Search Files +**GET** `/api/index/search?query=report&path=/documents/` + +Search for files by name or metadata. + +**Query Parameters:** +- `query` (required) - The search query +- `path` (optional) - Limit search to specific path + +**Response:** +```json +{ + "query": "report", + "path": "/documents/", + "results": [ + // Same structure as browse endpoint + ], + "totalCount": 3 +} +``` + +## Path Normalization + +The system automatically normalizes paths to ensure consistency: + +- **Trailing Slash**: All paths end with `/` +- **Root Path**: User home folder is represented as `/` +- **Query Safety**: Paths are validated to avoid SQL injection +- **Examples**: + - `/documents/` ✅ (correct) + - `/documents` → `/documents/` ✅ (normalized) + - `/documents/reports/` ✅ (correct) + - `/documents/reports` → `/documents/reports/` ✅ (normalized) + +## File Upload Integration + +When uploading files with the `FileUploadController`, you can specify a path to automatically create file indexes: + +**Create Upload Task Request:** +```json +{ + "fileName": "document.pdf", + "fileSize": 1024, + "contentType": "application/pdf", + "hash": "sha256-hash", + "path": "/documents/" // New field for file indexing +} +``` + +The system will automatically create a file index when the upload completes successfully. + +## Service Methods + +### FileIndexService + +```csharp +public class FileIndexService +{ + // Create a new file index + Task CreateAsync(string path, Guid fileId, Guid accountId); + + // Get files by path + Task> GetByPathAsync(Guid accountId, string path); + + // Get all files for account + Task> GetByAccountIdAsync(Guid accountId); + + // Get indexes for specific file + Task> GetByFileIdAsync(Guid fileId); + + // Move file to new path + Task UpdateAsync(Guid indexId, string newPath); + + // Remove file index + Task RemoveAsync(Guid indexId); + + // Remove all indexes in path + Task RemoveByPathAsync(Guid accountId, string path); + + // Normalize path format + public static string NormalizePath(string path); +} +``` + +## Error Handling + +The API returns appropriate HTTP status codes and error messages: + +- **400 Bad Request**: Invalid input parameters +- **401 Unauthorized**: User not authenticated +- **403 Forbidden**: User lacks permission +- **404 Not Found**: Resource not found +- **500 Internal Server Error**: Server-side error + +**Error Response Format:** +```json +{ + "code": "BROWSE_FAILED", + "message": "Failed to browse files", + "status": 500 +} +``` + +## Security Considerations + +1. **Ownership Verification**: All operations verify that the user owns the file indexes +2. **Path Validation**: Paths are normalized and validated +3. **Cascade Deletion**: File indexes are automatically removed when files are deleted +4. **Safe File Deletion**: Files are only deleted when no other indexes reference them + +## Usage Examples + +### Upload File to Specific Path +```bash +# Create upload task with path +curl -X POST /api/files/upload/create \ + -H "Authorization: Bearer {token}" \ + -H "Content-Type: application/json" \ + -d '{ + "fileName": "report.pdf", + "fileSize": 2048, + "contentType": "application/pdf", + "path": "/documents/reports/" + }' +``` + +### Browse Files +```bash +curl -X GET "/api/index/browse?path=/documents/reports/" \ + -H "Authorization: Bearer {token}" +``` + +### Move File +```bash +curl -X POST "/api/index/move/{indexId}" \ + -H "Authorization: Bearer {token}" \ + -H "Content-Type: application/json" \ + -d '{"newPath": "/archived/"}' +``` + +### Search Files +```bash +curl -X GET "/api/index/search?query=invoice&path=/documents/" \ + -H "Authorization: Bearer {token}" +``` + +## Best Practices + +1. **Use Trailing Slashes**: Always include trailing slashes in paths +2. **Organize Hierarchically**: Use meaningful folder structures +3. **Search Efficiently**: Use the search endpoint instead of client-side filtering +4. **Clean Up**: Use the clear-path endpoint for temporary directories +5. **Monitor Usage**: Check total file counts for quota management + +## Integration Notes + +- The file indexing system works alongside the existing file storage +- Files can exist in multiple paths (hard links) +- File deletion is optional and only removes data when safe +- The system maintains referential integrity between files and indexes diff --git a/DysonNetwork.Drive/Migrations/20251112135535_AddFileIndex.Designer.cs b/DysonNetwork.Drive/Migrations/20251112135535_AddFileIndex.Designer.cs new file mode 100644 index 0000000..3d5ea74 --- /dev/null +++ b/DysonNetwork.Drive/Migrations/20251112135535_AddFileIndex.Designer.cs @@ -0,0 +1,632 @@ +// +using System; +using System.Collections.Generic; +using DysonNetwork.Drive; +using DysonNetwork.Shared.Models; +using Microsoft.EntityFrameworkCore; +using Microsoft.EntityFrameworkCore.Infrastructure; +using Microsoft.EntityFrameworkCore.Migrations; +using Microsoft.EntityFrameworkCore.Storage.ValueConversion; +using NodaTime; +using Npgsql.EntityFrameworkCore.PostgreSQL.Metadata; + +#nullable disable + +namespace DysonNetwork.Drive.Migrations +{ + [DbContext(typeof(AppDatabase))] + [Migration("20251112135535_AddFileIndex")] + partial class AddFileIndex + { + /// + protected override void BuildTargetModel(ModelBuilder modelBuilder) + { +#pragma warning disable 612, 618 + modelBuilder + .HasAnnotation("ProductVersion", "9.0.10") + .HasAnnotation("Relational:MaxIdentifierLength", 63); + + NpgsqlModelBuilderExtensions.UseIdentityByDefaultColumns(modelBuilder); + + modelBuilder.Entity("DysonNetwork.Drive.Billing.QuotaRecord", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("uuid") + .HasColumnName("id"); + + b.Property("AccountId") + .HasColumnType("uuid") + .HasColumnName("account_id"); + + b.Property("CreatedAt") + .HasColumnType("timestamp with time zone") + .HasColumnName("created_at"); + + b.Property("DeletedAt") + .HasColumnType("timestamp with time zone") + .HasColumnName("deleted_at"); + + b.Property("Description") + .IsRequired() + .HasColumnType("text") + .HasColumnName("description"); + + b.Property("ExpiredAt") + .HasColumnType("timestamp with time zone") + .HasColumnName("expired_at"); + + b.Property("Name") + .IsRequired() + .HasColumnType("text") + .HasColumnName("name"); + + b.Property("Quota") + .HasColumnType("bigint") + .HasColumnName("quota"); + + b.Property("UpdatedAt") + .HasColumnType("timestamp with time zone") + .HasColumnName("updated_at"); + + b.HasKey("Id") + .HasName("pk_quota_records"); + + b.ToTable("quota_records", (string)null); + }); + + modelBuilder.Entity("DysonNetwork.Drive.Storage.Model.PersistentTask", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("uuid") + .HasColumnName("id"); + + b.Property("AccountId") + .HasColumnType("uuid") + .HasColumnName("account_id"); + + b.Property("CompletedAt") + .HasColumnType("timestamp with time zone") + .HasColumnName("completed_at"); + + b.Property("CreatedAt") + .HasColumnType("timestamp with time zone") + .HasColumnName("created_at"); + + b.Property("DeletedAt") + .HasColumnType("timestamp with time zone") + .HasColumnName("deleted_at"); + + b.Property("Description") + .HasMaxLength(1024) + .HasColumnType("character varying(1024)") + .HasColumnName("description"); + + b.Property("Discriminator") + .IsRequired() + .HasMaxLength(21) + .HasColumnType("character varying(21)") + .HasColumnName("discriminator"); + + b.Property("ErrorMessage") + .HasMaxLength(1024) + .HasColumnType("character varying(1024)") + .HasColumnName("error_message"); + + b.Property("EstimatedDurationSeconds") + .HasColumnType("bigint") + .HasColumnName("estimated_duration_seconds"); + + b.Property("ExpiredAt") + .HasColumnType("timestamp with time zone") + .HasColumnName("expired_at"); + + b.Property("LastActivity") + .HasColumnType("timestamp with time zone") + .HasColumnName("last_activity"); + + b.Property("Name") + .IsRequired() + .HasMaxLength(256) + .HasColumnType("character varying(256)") + .HasColumnName("name"); + + b.Property>("Parameters") + .IsRequired() + .HasColumnType("jsonb") + .HasColumnName("parameters"); + + b.Property("Priority") + .HasColumnType("integer") + .HasColumnName("priority"); + + b.Property("Progress") + .HasColumnType("double precision") + .HasColumnName("progress"); + + b.Property>("Results") + .IsRequired() + .HasColumnType("jsonb") + .HasColumnName("results"); + + b.Property("StartedAt") + .HasColumnType("timestamp with time zone") + .HasColumnName("started_at"); + + b.Property("Status") + .HasColumnType("integer") + .HasColumnName("status"); + + b.Property("TaskId") + .IsRequired() + .HasMaxLength(64) + .HasColumnType("character varying(64)") + .HasColumnName("task_id"); + + b.Property("Type") + .HasColumnType("integer") + .HasColumnName("type"); + + b.Property("UpdatedAt") + .HasColumnType("timestamp with time zone") + .HasColumnName("updated_at"); + + b.HasKey("Id") + .HasName("pk_tasks"); + + b.ToTable("tasks", (string)null); + + b.HasDiscriminator().HasValue("PersistentTask"); + + b.UseTphMappingStrategy(); + }); + + modelBuilder.Entity("DysonNetwork.Shared.Models.CloudFileReference", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("uuid") + .HasColumnName("id"); + + b.Property("CreatedAt") + .HasColumnType("timestamp with time zone") + .HasColumnName("created_at"); + + b.Property("DeletedAt") + .HasColumnType("timestamp with time zone") + .HasColumnName("deleted_at"); + + b.Property("ExpiredAt") + .HasColumnType("timestamp with time zone") + .HasColumnName("expired_at"); + + b.Property("FileId") + .IsRequired() + .HasMaxLength(32) + .HasColumnType("character varying(32)") + .HasColumnName("file_id"); + + b.Property("ResourceId") + .IsRequired() + .HasMaxLength(1024) + .HasColumnType("character varying(1024)") + .HasColumnName("resource_id"); + + b.Property("UpdatedAt") + .HasColumnType("timestamp with time zone") + .HasColumnName("updated_at"); + + b.Property("Usage") + .IsRequired() + .HasMaxLength(1024) + .HasColumnType("character varying(1024)") + .HasColumnName("usage"); + + b.HasKey("Id") + .HasName("pk_file_references"); + + b.HasIndex("FileId") + .HasDatabaseName("ix_file_references_file_id"); + + b.ToTable("file_references", (string)null); + }); + + modelBuilder.Entity("DysonNetwork.Shared.Models.FilePool", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("uuid") + .HasColumnName("id"); + + b.Property("AccountId") + .HasColumnType("uuid") + .HasColumnName("account_id"); + + b.Property("BillingConfig") + .IsRequired() + .HasColumnType("jsonb") + .HasColumnName("billing_config"); + + b.Property("CreatedAt") + .HasColumnType("timestamp with time zone") + .HasColumnName("created_at"); + + b.Property("DeletedAt") + .HasColumnType("timestamp with time zone") + .HasColumnName("deleted_at"); + + b.Property("Description") + .IsRequired() + .HasMaxLength(8192) + .HasColumnType("character varying(8192)") + .HasColumnName("description"); + + b.Property("IsHidden") + .HasColumnType("boolean") + .HasColumnName("is_hidden"); + + b.Property("Name") + .IsRequired() + .HasMaxLength(1024) + .HasColumnType("character varying(1024)") + .HasColumnName("name"); + + b.Property("PolicyConfig") + .IsRequired() + .HasColumnType("jsonb") + .HasColumnName("policy_config"); + + b.Property("StorageConfig") + .IsRequired() + .HasColumnType("jsonb") + .HasColumnName("storage_config"); + + b.Property("UpdatedAt") + .HasColumnType("timestamp with time zone") + .HasColumnName("updated_at"); + + b.HasKey("Id") + .HasName("pk_pools"); + + b.ToTable("pools", (string)null); + }); + + modelBuilder.Entity("DysonNetwork.Shared.Models.SnCloudFile", b => + { + b.Property("Id") + .HasMaxLength(32) + .HasColumnType("character varying(32)") + .HasColumnName("id"); + + b.Property("AccountId") + .HasColumnType("uuid") + .HasColumnName("account_id"); + + b.Property("BundleId") + .HasColumnType("uuid") + .HasColumnName("bundle_id"); + + b.Property("CreatedAt") + .HasColumnType("timestamp with time zone") + .HasColumnName("created_at"); + + b.Property("DeletedAt") + .HasColumnType("timestamp with time zone") + .HasColumnName("deleted_at"); + + b.Property("Description") + .HasMaxLength(4096) + .HasColumnType("character varying(4096)") + .HasColumnName("description"); + + b.Property("ExpiredAt") + .HasColumnType("timestamp with time zone") + .HasColumnName("expired_at"); + + b.Property>("FileMeta") + .HasColumnType("jsonb") + .HasColumnName("file_meta"); + + b.Property("HasCompression") + .HasColumnType("boolean") + .HasColumnName("has_compression"); + + b.Property("HasThumbnail") + .HasColumnType("boolean") + .HasColumnName("has_thumbnail"); + + b.Property("Hash") + .HasMaxLength(256) + .HasColumnType("character varying(256)") + .HasColumnName("hash"); + + b.Property("IsEncrypted") + .HasColumnType("boolean") + .HasColumnName("is_encrypted"); + + b.Property("IsMarkedRecycle") + .HasColumnType("boolean") + .HasColumnName("is_marked_recycle"); + + b.Property("MimeType") + .HasMaxLength(256) + .HasColumnType("character varying(256)") + .HasColumnName("mime_type"); + + b.Property("Name") + .IsRequired() + .HasMaxLength(1024) + .HasColumnType("character varying(1024)") + .HasColumnName("name"); + + b.Property("PoolId") + .HasColumnType("uuid") + .HasColumnName("pool_id"); + + b.Property>("SensitiveMarks") + .HasColumnType("jsonb") + .HasColumnName("sensitive_marks"); + + b.Property("Size") + .HasColumnType("bigint") + .HasColumnName("size"); + + b.Property("StorageId") + .HasMaxLength(32) + .HasColumnType("character varying(32)") + .HasColumnName("storage_id"); + + b.Property("StorageUrl") + .HasMaxLength(4096) + .HasColumnType("character varying(4096)") + .HasColumnName("storage_url"); + + b.Property("UpdatedAt") + .HasColumnType("timestamp with time zone") + .HasColumnName("updated_at"); + + b.Property("UploadedAt") + .HasColumnType("timestamp with time zone") + .HasColumnName("uploaded_at"); + + b.Property>("UserMeta") + .HasColumnType("jsonb") + .HasColumnName("user_meta"); + + b.HasKey("Id") + .HasName("pk_files"); + + b.HasIndex("BundleId") + .HasDatabaseName("ix_files_bundle_id"); + + b.HasIndex("PoolId") + .HasDatabaseName("ix_files_pool_id"); + + b.ToTable("files", (string)null); + }); + + modelBuilder.Entity("DysonNetwork.Shared.Models.SnCloudFileIndex", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("uuid") + .HasColumnName("id"); + + b.Property("AccountId") + .HasColumnType("uuid") + .HasColumnName("account_id"); + + b.Property("CreatedAt") + .HasColumnType("timestamp with time zone") + .HasColumnName("created_at"); + + b.Property("DeletedAt") + .HasColumnType("timestamp with time zone") + .HasColumnName("deleted_at"); + + b.Property("FileId") + .IsRequired() + .HasMaxLength(32) + .HasColumnType("character varying(32)") + .HasColumnName("file_id"); + + b.Property("Path") + .IsRequired() + .HasMaxLength(8192) + .HasColumnType("character varying(8192)") + .HasColumnName("path"); + + b.Property("UpdatedAt") + .HasColumnType("timestamp with time zone") + .HasColumnName("updated_at"); + + b.HasKey("Id") + .HasName("pk_file_indexes"); + + b.HasIndex("FileId") + .HasDatabaseName("ix_file_indexes_file_id"); + + b.HasIndex("Path", "AccountId") + .HasDatabaseName("ix_file_indexes_path_account_id"); + + b.ToTable("file_indexes", (string)null); + }); + + modelBuilder.Entity("DysonNetwork.Shared.Models.SnFileBundle", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("uuid") + .HasColumnName("id"); + + b.Property("AccountId") + .HasColumnType("uuid") + .HasColumnName("account_id"); + + b.Property("CreatedAt") + .HasColumnType("timestamp with time zone") + .HasColumnName("created_at"); + + b.Property("DeletedAt") + .HasColumnType("timestamp with time zone") + .HasColumnName("deleted_at"); + + b.Property("Description") + .HasMaxLength(8192) + .HasColumnType("character varying(8192)") + .HasColumnName("description"); + + b.Property("ExpiredAt") + .HasColumnType("timestamp with time zone") + .HasColumnName("expired_at"); + + b.Property("Name") + .IsRequired() + .HasMaxLength(1024) + .HasColumnType("character varying(1024)") + .HasColumnName("name"); + + b.Property("Passcode") + .HasMaxLength(256) + .HasColumnType("character varying(256)") + .HasColumnName("passcode"); + + b.Property("Slug") + .IsRequired() + .HasMaxLength(1024) + .HasColumnType("character varying(1024)") + .HasColumnName("slug"); + + b.Property("UpdatedAt") + .HasColumnType("timestamp with time zone") + .HasColumnName("updated_at"); + + b.HasKey("Id") + .HasName("pk_bundles"); + + b.HasIndex("Slug") + .IsUnique() + .HasDatabaseName("ix_bundles_slug"); + + b.ToTable("bundles", (string)null); + }); + + modelBuilder.Entity("DysonNetwork.Drive.Storage.Model.PersistentUploadTask", b => + { + b.HasBaseType("DysonNetwork.Drive.Storage.Model.PersistentTask"); + + b.Property("BundleId") + .HasColumnType("uuid") + .HasColumnName("bundle_id"); + + b.Property("ChunkSize") + .HasColumnType("bigint") + .HasColumnName("chunk_size"); + + b.Property("ChunksCount") + .HasColumnType("integer") + .HasColumnName("chunks_count"); + + b.Property("ChunksUploaded") + .HasColumnType("integer") + .HasColumnName("chunks_uploaded"); + + b.Property("ContentType") + .IsRequired() + .HasMaxLength(128) + .HasColumnType("character varying(128)") + .HasColumnName("content_type"); + + b.Property("EncryptPassword") + .HasMaxLength(256) + .HasColumnType("character varying(256)") + .HasColumnName("encrypt_password"); + + b.Property("FileName") + .IsRequired() + .HasMaxLength(256) + .HasColumnType("character varying(256)") + .HasColumnName("file_name"); + + b.Property("FileSize") + .HasColumnType("bigint") + .HasColumnName("file_size"); + + b.Property("Hash") + .IsRequired() + .HasColumnType("text") + .HasColumnName("hash"); + + b.Property("Path") + .HasColumnType("text") + .HasColumnName("path"); + + b.Property("PoolId") + .HasColumnType("uuid") + .HasColumnName("pool_id"); + + b.PrimitiveCollection>("UploadedChunks") + .IsRequired() + .HasColumnType("integer[]") + .HasColumnName("uploaded_chunks"); + + b.HasDiscriminator().HasValue("PersistentUploadTask"); + }); + + modelBuilder.Entity("DysonNetwork.Shared.Models.CloudFileReference", b => + { + b.HasOne("DysonNetwork.Shared.Models.SnCloudFile", "File") + .WithMany("References") + .HasForeignKey("FileId") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired() + .HasConstraintName("fk_file_references_files_file_id"); + + b.Navigation("File"); + }); + + modelBuilder.Entity("DysonNetwork.Shared.Models.SnCloudFile", b => + { + b.HasOne("DysonNetwork.Shared.Models.SnFileBundle", "Bundle") + .WithMany("Files") + .HasForeignKey("BundleId") + .HasConstraintName("fk_files_bundles_bundle_id"); + + b.HasOne("DysonNetwork.Shared.Models.FilePool", "Pool") + .WithMany() + .HasForeignKey("PoolId") + .HasConstraintName("fk_files_pools_pool_id"); + + b.Navigation("Bundle"); + + b.Navigation("Pool"); + }); + + modelBuilder.Entity("DysonNetwork.Shared.Models.SnCloudFileIndex", b => + { + b.HasOne("DysonNetwork.Shared.Models.SnCloudFile", "File") + .WithMany("FileIndexes") + .HasForeignKey("FileId") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired() + .HasConstraintName("fk_file_indexes_files_file_id"); + + b.Navigation("File"); + }); + + modelBuilder.Entity("DysonNetwork.Shared.Models.SnCloudFile", b => + { + b.Navigation("FileIndexes"); + + b.Navigation("References"); + }); + + modelBuilder.Entity("DysonNetwork.Shared.Models.SnFileBundle", b => + { + b.Navigation("Files"); + }); +#pragma warning restore 612, 618 + } + } +} diff --git a/DysonNetwork.Drive/Migrations/20251112135535_AddFileIndex.cs b/DysonNetwork.Drive/Migrations/20251112135535_AddFileIndex.cs new file mode 100644 index 0000000..5c9625c --- /dev/null +++ b/DysonNetwork.Drive/Migrations/20251112135535_AddFileIndex.cs @@ -0,0 +1,66 @@ +using System; +using Microsoft.EntityFrameworkCore.Migrations; +using NodaTime; + +#nullable disable + +namespace DysonNetwork.Drive.Migrations +{ + /// + public partial class AddFileIndex : Migration + { + /// + protected override void Up(MigrationBuilder migrationBuilder) + { + migrationBuilder.AddColumn( + name: "path", + table: "tasks", + type: "text", + nullable: true); + + migrationBuilder.CreateTable( + name: "file_indexes", + columns: table => new + { + id = table.Column(type: "uuid", nullable: false), + path = table.Column(type: "character varying(8192)", maxLength: 8192, nullable: false), + file_id = table.Column(type: "character varying(32)", maxLength: 32, nullable: false), + account_id = table.Column(type: "uuid", nullable: false), + created_at = table.Column(type: "timestamp with time zone", nullable: false), + updated_at = table.Column(type: "timestamp with time zone", nullable: false), + deleted_at = table.Column(type: "timestamp with time zone", nullable: true) + }, + constraints: table => + { + table.PrimaryKey("pk_file_indexes", x => x.id); + table.ForeignKey( + name: "fk_file_indexes_files_file_id", + column: x => x.file_id, + principalTable: "files", + principalColumn: "id", + onDelete: ReferentialAction.Cascade); + }); + + migrationBuilder.CreateIndex( + name: "ix_file_indexes_file_id", + table: "file_indexes", + column: "file_id"); + + migrationBuilder.CreateIndex( + name: "ix_file_indexes_path_account_id", + table: "file_indexes", + columns: new[] { "path", "account_id" }); + } + + /// + protected override void Down(MigrationBuilder migrationBuilder) + { + migrationBuilder.DropTable( + name: "file_indexes"); + + migrationBuilder.DropColumn( + name: "path", + table: "tasks"); + } + } +} diff --git a/DysonNetwork.Drive/Migrations/AppDatabaseModelSnapshot.cs b/DysonNetwork.Drive/Migrations/AppDatabaseModelSnapshot.cs index eba7db8..824af79 100644 --- a/DysonNetwork.Drive/Migrations/AppDatabaseModelSnapshot.cs +++ b/DysonNetwork.Drive/Migrations/AppDatabaseModelSnapshot.cs @@ -403,6 +403,53 @@ namespace DysonNetwork.Drive.Migrations b.ToTable("files", (string)null); }); + modelBuilder.Entity("DysonNetwork.Shared.Models.SnCloudFileIndex", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("uuid") + .HasColumnName("id"); + + b.Property("AccountId") + .HasColumnType("uuid") + .HasColumnName("account_id"); + + b.Property("CreatedAt") + .HasColumnType("timestamp with time zone") + .HasColumnName("created_at"); + + b.Property("DeletedAt") + .HasColumnType("timestamp with time zone") + .HasColumnName("deleted_at"); + + b.Property("FileId") + .IsRequired() + .HasMaxLength(32) + .HasColumnType("character varying(32)") + .HasColumnName("file_id"); + + b.Property("Path") + .IsRequired() + .HasMaxLength(8192) + .HasColumnType("character varying(8192)") + .HasColumnName("path"); + + b.Property("UpdatedAt") + .HasColumnType("timestamp with time zone") + .HasColumnName("updated_at"); + + b.HasKey("Id") + .HasName("pk_file_indexes"); + + b.HasIndex("FileId") + .HasDatabaseName("ix_file_indexes_file_id"); + + b.HasIndex("Path", "AccountId") + .HasDatabaseName("ix_file_indexes_path_account_id"); + + b.ToTable("file_indexes", (string)null); + }); + modelBuilder.Entity("DysonNetwork.Shared.Models.SnFileBundle", b => { b.Property("Id") @@ -508,6 +555,10 @@ namespace DysonNetwork.Drive.Migrations .HasColumnType("text") .HasColumnName("hash"); + b.Property("Path") + .HasColumnType("text") + .HasColumnName("path"); + b.Property("PoolId") .HasColumnType("uuid") .HasColumnName("pool_id"); @@ -549,8 +600,22 @@ namespace DysonNetwork.Drive.Migrations b.Navigation("Pool"); }); + modelBuilder.Entity("DysonNetwork.Shared.Models.SnCloudFileIndex", b => + { + b.HasOne("DysonNetwork.Shared.Models.SnCloudFile", "File") + .WithMany("FileIndexes") + .HasForeignKey("FileId") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired() + .HasConstraintName("fk_file_indexes_files_file_id"); + + b.Navigation("File"); + }); + modelBuilder.Entity("DysonNetwork.Shared.Models.SnCloudFile", b => { + b.Navigation("FileIndexes"); + b.Navigation("References"); }); diff --git a/DysonNetwork.Drive/Startup/ServiceCollectionExtensions.cs b/DysonNetwork.Drive/Startup/ServiceCollectionExtensions.cs index af8b39e..3fbd280 100644 --- a/DysonNetwork.Drive/Startup/ServiceCollectionExtensions.cs +++ b/DysonNetwork.Drive/Startup/ServiceCollectionExtensions.cs @@ -1,5 +1,6 @@ using System.Text.Json; using System.Text.Json.Serialization; +using DysonNetwork.Drive.Index; using DysonNetwork.Shared.Cache; using NodaTime; using NodaTime.Serialization.SystemTextJson; @@ -56,6 +57,7 @@ public static class ServiceCollectionExtensions services.AddScoped(); services.AddScoped(); services.AddScoped(); + services.AddScoped(); services.AddScoped(); services.AddScoped(); diff --git a/DysonNetwork.Drive/Storage/CloudFileUnusedRecyclingJob.cs b/DysonNetwork.Drive/Storage/CloudFileUnusedRecyclingJob.cs index e3af918..e66dd50 100644 --- a/DysonNetwork.Drive/Storage/CloudFileUnusedRecyclingJob.cs +++ b/DysonNetwork.Drive/Storage/CloudFileUnusedRecyclingJob.cs @@ -14,7 +14,7 @@ public class CloudFileUnusedRecyclingJob( public async Task Execute(IJobExecutionContext context) { logger.LogInformation("Cleaning tus cloud files..."); - var storePath = configuration["Tus:StorePath"]; + var storePath = configuration["Storage:Uploads"]; if (Directory.Exists(storePath)) { var oneHourAgo = SystemClock.Instance.GetCurrentInstant() - Duration.FromHours(1); @@ -39,6 +39,7 @@ public class CloudFileUnusedRecyclingJob( var processedCount = 0; var markedCount = 0; var totalFiles = await db.Files + .Where(f => f.FileIndexes.Count == 0) .Where(f => f.PoolId.HasValue && recyclablePools.Contains(f.PoolId.Value)) .Where(f => !f.IsMarkedRecycle) .CountAsync(); diff --git a/DysonNetwork.Drive/Storage/FileController.cs b/DysonNetwork.Drive/Storage/FileController.cs index 1a9097d..812fbf6 100644 --- a/DysonNetwork.Drive/Storage/FileController.cs +++ b/DysonNetwork.Drive/Storage/FileController.cs @@ -76,7 +76,7 @@ public class FileController( } // Fallback for tus uploads - var tusStorePath = configuration.GetValue("Tus:StorePath"); + var tusStorePath = configuration.GetValue("Storage:Uploads"); if (!string.IsNullOrEmpty(tusStorePath)) { var tusFilePath = Path.Combine(env.ContentRootPath, tusStorePath, file.Id); diff --git a/DysonNetwork.Drive/Storage/FileUploadController.cs b/DysonNetwork.Drive/Storage/FileUploadController.cs index 5067c5f..2d6738f 100644 --- a/DysonNetwork.Drive/Storage/FileUploadController.cs +++ b/DysonNetwork.Drive/Storage/FileUploadController.cs @@ -1,5 +1,6 @@ using System.ComponentModel.DataAnnotations; using DysonNetwork.Drive.Billing; +using DysonNetwork.Drive.Index; using DysonNetwork.Drive.Storage.Model; using DysonNetwork.Shared.Auth; using DysonNetwork.Shared.Http; @@ -24,6 +25,7 @@ public class FileUploadController( PermissionService.PermissionServiceClient permission, QuotaService quotaService, PersistentTaskService persistentTaskService, + FileIndexService fileIndexService, ILogger logger ) : ControllerBase @@ -260,10 +262,26 @@ public class FileUploadController( persistentTask.ExpiredAt ); - // Update task status to "processing" - background processing is now happening + // Create the file index if a path is provided + if (!string.IsNullOrEmpty(persistentTask.Path)) + { + try + { + var accountId = Guid.Parse(currentUser.Id); + await fileIndexService.CreateAsync(persistentTask.Path, fileId, accountId); + logger.LogInformation("Created file index for file {FileId} at path {Path}", fileId, persistentTask.Path); + } + catch (Exception ex) + { + logger.LogWarning(ex, "Failed to create file index for file {FileId} at path {Path}", fileId, persistentTask.Path); + // Don't fail the upload if index creation fails, just log it + } + } + + // Update the task status to "processing" - background processing is now happening await persistentTaskService.UpdateTaskProgressAsync(taskId, 0.95, "Processing file in background..."); - // Send upload completion notification (file is uploaded, but processing continues) + // Send upload completion notification (a file is uploaded, but processing continues) await persistentTaskService.SendUploadCompletedNotificationAsync(persistentTask, fileId); return Ok(cloudFile); diff --git a/DysonNetwork.Drive/Storage/Model/FileUploadModels.cs b/DysonNetwork.Drive/Storage/Model/FileUploadModels.cs index d3c932e..c2a2f80 100644 --- a/DysonNetwork.Drive/Storage/Model/FileUploadModels.cs +++ b/DysonNetwork.Drive/Storage/Model/FileUploadModels.cs @@ -22,6 +22,7 @@ public class FileUploadParameters public string? EncryptPassword { get; set; } public string Hash { get; set; } = string.Empty; public List UploadedChunks { get; set; } = []; + public string? Path { get; set; } } // File Move Task Parameters @@ -93,6 +94,7 @@ public class CreateUploadTaskRequest public string? EncryptPassword { get; set; } public Instant? ExpiredAt { get; set; } public long? ChunkSize { get; set; } + public string? Path { get; set; } } public class CreateUploadTaskResponse @@ -301,6 +303,17 @@ public class PersistentUploadTask : PersistentTask TypedParameters = parameters; } } + + public string? Path + { + get => TypedParameters.Path; + set + { + var parameters = TypedParameters; + parameters.Path = value; + TypedParameters = parameters; + } + } } public enum TaskType @@ -654,4 +667,4 @@ public enum UploadTaskStatus Completed = TaskStatus.Completed, Failed = TaskStatus.Failed, Expired = TaskStatus.Expired -} \ No newline at end of file +} diff --git a/DysonNetwork.Drive/Storage/PersistentTaskService.cs b/DysonNetwork.Drive/Storage/PersistentTaskService.cs index 1b3676f..24be133 100644 --- a/DysonNetwork.Drive/Storage/PersistentTaskService.cs +++ b/DysonNetwork.Drive/Storage/PersistentTaskService.cs @@ -615,7 +615,14 @@ public class PersistentTaskService( var chunkSize = request.ChunkSize ?? 1024 * 1024 * 5; // 5MB default var chunksCount = (int)Math.Ceiling((double)request.FileSize / chunkSize); - // Use the default pool if no pool is specified, or find first available pool + // If the second chunk is too small (less than 1MB), merge it with the first chunk + if (chunksCount == 2 && (request.FileSize - chunkSize) < 1024 * 1024) + { + chunksCount = 1; + chunkSize = request.FileSize; + } + + // Use the default pool if no pool is specified, or find the first available pool var poolId = request.PoolId ?? await GetFirstAvailablePoolIdAsync(); var uploadTask = new PersistentUploadTask @@ -632,6 +639,7 @@ public class PersistentTaskService( EncryptPassword = request.EncryptPassword, ExpiredAt = request.ExpiredAt, Hash = request.Hash, + Path = request.Path, AccountId = accountId, Status = TaskStatus.InProgress, UploadedChunks = [], @@ -1132,4 +1140,4 @@ public class RecentActivity public double Progress { get; set; } } -#endregion \ No newline at end of file +#endregion diff --git a/DysonNetwork.Drive/appsettings.json b/DysonNetwork.Drive/appsettings.json index ca65a37..8e12c21 100644 --- a/DysonNetwork.Drive/appsettings.json +++ b/DysonNetwork.Drive/appsettings.json @@ -27,9 +27,6 @@ "PublicKeyPath": "Keys/PublicKey.pem", "PrivateKeyPath": "Keys/PrivateKey.pem" }, - "Tus": { - "StorePath": "Uploads" - }, "Storage": { "Uploads": "Uploads", "PreferredRemote": "c53136a6-9152-4ecb-9f88-43c41438c23e", diff --git a/DysonNetwork.Shared/Models/CloudFile.cs b/DysonNetwork.Shared/Models/CloudFile.cs index 94b3295..b3f8971 100644 --- a/DysonNetwork.Shared/Models/CloudFile.cs +++ b/DysonNetwork.Shared/Models/CloudFile.cs @@ -31,6 +31,7 @@ public class SnCloudFile : ModelBase, ICloudFile, IIdentifiedResource public Guid? PoolId { get; set; } [JsonIgnore] public SnFileBundle? Bundle { get; set; } public Guid? BundleId { get; set; } + [JsonIgnore] public List FileIndexes { get; set; } = []; /// /// The field is set to true if the recycling job plans to delete the file. diff --git a/DysonNetwork.Shared/Models/CloudFileIndex.cs b/DysonNetwork.Shared/Models/CloudFileIndex.cs new file mode 100644 index 0000000..9e64e09 --- /dev/null +++ b/DysonNetwork.Shared/Models/CloudFileIndex.cs @@ -0,0 +1,30 @@ +using System.ComponentModel.DataAnnotations; +using System.ComponentModel.DataAnnotations.Schema; +using Microsoft.EntityFrameworkCore; + +namespace DysonNetwork.Shared.Models; + +[Index(nameof(Path), nameof(AccountId))] +public class SnCloudFileIndex : ModelBase +{ + public Guid Id { get; init; } = Guid.NewGuid(); + + /// + /// The path of the file, + /// only store the parent folder. + /// With the trailing slash. + /// + /// Like /hello/here/ not /hello/here/text.txt or /hello/here + /// Or the user home folder files, store as / + /// + /// Besides, the folder name should be all query-safe, not contains % or + /// other special characters that will mess up the pgsql query + /// + [MaxLength(8192)] + public string Path { get; init; } = null!; + + [MaxLength(32)] public string FileId { get; init; } = null!; + public SnCloudFile File { get; init; } = null!; + public Guid AccountId { get; init; } + [NotMapped] public SnAccount? Account { get; init; } +} diff --git a/DysonNetwork.Shared/Models/Subscription.cs b/DysonNetwork.Shared/Models/Subscription.cs index b6a5348..1fdc8b7 100644 --- a/DysonNetwork.Shared/Models/Subscription.cs +++ b/DysonNetwork.Shared/Models/Subscription.cs @@ -185,58 +185,55 @@ public class SnWalletGift : ModelBase } } - // TODO: Uncomment once protobuf files are regenerated - /* - public Proto.Gift ToProtoValue() => new() - { - Id = Id.ToString(), - GifterId = GifterId.ToString(), - RecipientId = RecipientId?.ToString(), - GiftCode = GiftCode, - Message = Message, - SubscriptionIdentifier = SubscriptionIdentifier, - BasePrice = BasePrice.ToString(CultureInfo.InvariantCulture), - FinalPrice = FinalPrice.ToString(CultureInfo.InvariantCulture), - Status = (Proto.GiftStatus)Status, - RedeemedAt = RedeemedAt?.ToTimestamp(), - RedeemerId = RedeemerId?.ToString(), - SubscriptionId = SubscriptionId?.ToString(), - ExpiresAt = ExpiresAt.ToTimestamp(), - IsOpenGift = IsOpenGift, - PaymentMethod = PaymentMethod, - PaymentDetails = PaymentDetails.ToProtoValue(), - CouponId = CouponId?.ToString(), - Coupon = Coupon?.ToProtoValue(), - IsRedeemable = IsRedeemable, - IsExpired = IsExpired, - CreatedAt = CreatedAt.ToTimestamp(), - UpdatedAt = UpdatedAt.ToTimestamp() - }; + public Proto.Gift ToProtoValue() => new() + { + Id = Id.ToString(), + GifterId = GifterId.ToString(), + RecipientId = RecipientId?.ToString(), + GiftCode = GiftCode, + Message = Message, + SubscriptionIdentifier = SubscriptionIdentifier, + BasePrice = BasePrice.ToString(CultureInfo.InvariantCulture), + FinalPrice = FinalPrice.ToString(CultureInfo.InvariantCulture), + Status = (Proto.GiftStatus)Status, + RedeemedAt = RedeemedAt?.ToTimestamp(), + RedeemerId = RedeemerId?.ToString(), + SubscriptionId = SubscriptionId?.ToString(), + ExpiresAt = ExpiresAt.ToTimestamp(), + IsOpenGift = IsOpenGift, + PaymentMethod = PaymentMethod, + PaymentDetails = PaymentDetails.ToProtoValue(), + CouponId = CouponId?.ToString(), + Coupon = Coupon?.ToProtoValue(), + IsRedeemable = IsRedeemable, + IsExpired = IsExpired, + CreatedAt = CreatedAt.ToTimestamp(), + UpdatedAt = UpdatedAt.ToTimestamp() + }; - public static SnWalletGift FromProtoValue(Proto.Gift proto) => new() - { - Id = Guid.Parse(proto.Id), - GifterId = Guid.Parse(proto.GifterId), - RecipientId = proto.HasRecipientId ? Guid.Parse(proto.RecipientId) : null, - GiftCode = proto.GiftCode, - Message = proto.Message, - SubscriptionIdentifier = proto.SubscriptionIdentifier, - BasePrice = decimal.Parse(proto.BasePrice), - FinalPrice = decimal.Parse(proto.FinalPrice), - Status = (GiftStatus)proto.Status, - RedeemedAt = proto.RedeemedAt?.ToInstant(), - RedeemerId = proto.HasRedeemerId ? Guid.Parse(proto.RedeemerId) : null, - SubscriptionId = proto.HasSubscriptionId ? Guid.Parse(proto.SubscriptionId) : null, - ExpiresAt = proto.ExpiresAt.ToInstant(), - IsOpenGift = proto.IsOpenGift, - PaymentMethod = proto.PaymentMethod, - PaymentDetails = SnPaymentDetails.FromProtoValue(proto.PaymentDetails), - CouponId = proto.HasCouponId ? Guid.Parse(proto.CouponId) : null, - Coupon = proto.Coupon is not null ? SnWalletCoupon.FromProtoValue(proto.Coupon) : null, - CreatedAt = proto.CreatedAt.ToInstant(), - UpdatedAt = proto.UpdatedAt.ToInstant() - }; - */ + public static SnWalletGift FromProtoValue(Proto.Gift proto) => new() + { + Id = Guid.Parse(proto.Id), + GifterId = Guid.Parse(proto.GifterId), + RecipientId = proto.HasRecipientId ? Guid.Parse(proto.RecipientId) : null, + GiftCode = proto.GiftCode, + Message = proto.Message, + SubscriptionIdentifier = proto.SubscriptionIdentifier, + BasePrice = decimal.Parse(proto.BasePrice), + FinalPrice = decimal.Parse(proto.FinalPrice), + Status = (GiftStatus)proto.Status, + RedeemedAt = proto.RedeemedAt?.ToInstant(), + RedeemerId = proto.HasRedeemerId ? Guid.Parse(proto.RedeemerId) : null, + SubscriptionId = proto.HasSubscriptionId ? Guid.Parse(proto.SubscriptionId) : null, + ExpiresAt = proto.ExpiresAt.ToInstant(), + IsOpenGift = proto.IsOpenGift, + PaymentMethod = proto.PaymentMethod, + PaymentDetails = SnPaymentDetails.FromProtoValue(proto.PaymentDetails), + CouponId = proto.HasCouponId ? Guid.Parse(proto.CouponId) : null, + Coupon = proto.Coupon is not null ? SnWalletCoupon.FromProtoValue(proto.Coupon) : null, + CreatedAt = proto.CreatedAt.ToInstant(), + UpdatedAt = proto.UpdatedAt.ToInstant() + }; } public abstract class SubscriptionType