🐛 Fix wrong inbox method

🔊 Add more logging during activitypub process for debug

.aspire/settings.json

@@ -1,3 +1,4 @@
 {
   "appHostPath": "../DysonNetwork.Control/DysonNetwork.Control.csproj"
 }
+

.editorconfig

@@ -0,0 +1,5 @@
+root = true
+
+[*]
+indent_style = space
+indent_size = 4

.env

@@ -33,3 +33,6 @@ SPHERE_IMAGE=sphere:latest
 
 # Container image name for develop
 DEVELOP_IMAGE=develop:latest
+
+# Container image name for gateway
+GATEWAY_IMAGE=gateway:latest

.env.testing.example

@@ -0,0 +1,43 @@
+# ActivityPub Testing Environment Variables
+
+# Solar Network Configuration
+SOLAR_DOMAIN=solar.local
+SOLAR_PORT=5000
+SOLAR_URL=http://solar.local:5000
+
+# Mastodon (Self-Hosted Test Instance)
+MASTODON_DOMAIN=mastodon.local
+MASTODON_PORT=3001
+MASTODON_STREAMING_PORT=4000
+MASTODON_URL=http://mastodon.local:3001
+
+# Database
+DB_CONNECTION_STRING=Host=localhost;Port=5432;Database=dyson_network;Username=postgres;Password=postgres
+
+# Test Accounts
+SOLAR_TEST_USERNAME=solaruser
+MASTODON_TEST_USERNAME=testuser
+MASTODON_TEST_PASSWORD=TestPassword123!
+
+# ActivityPub Settings
+ACTIVITYPUB_DOMAIN=solar.local
+ACTIVITYPUB_ENABLE_FEDERATION=true
+ACTIVITYPUB_SIGNATURE_ALGORITHM=rsa-sha256
+
+# HTTP Settings
+HTTP_TIMEOUT=30
+HTTP_MAX_RETRIES=3
+
+# Logging
+LOG_LEVEL=Debug
+ACTIVITYPUB_LOG_LEVEL=Trace
+
+# Testing
+TEST_SKIP_DATABASE_RESET=false
+TEST_SKIP_MASTODON_SETUP=false
+TEST_AUTO_ACCEPT_FOLLOWS=false
+
+# Development (only in dev environment)
+DEV_DISABLE_SIGNATURE_VERIFICATION=false
+DEV_LOG_HTTP_BODIES=false
+DEV_DISABLE_CORS=false

.github/workflows/docker-build.yml

@@ -7,25 +7,69 @@ on:
   workflow_dispatch:
 
 jobs:
+  determine-changes:
+    runs-on: ubuntu-latest
+    outputs:
+      matrix: ${{ steps.changes.outputs.matrix }}
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v4
+        with:
+          fetch-depth: 0
+
+      - name: Get changed files
+        id: changed-files
+        run: |
+          echo "files=$(git diff --name-only ${{ github.event.before }} ${{ github.sha }} | xargs)" >> $GITHUB_OUTPUT
+
+      - name: Determine changed services
+        id: changes
+        run: |
+          files="${{ steps.changed-files.outputs.files }}"
+          matrix="{\"include\":[]}"
+          services=("Sphere" "Pass" "Ring" "Drive" "Develop" "Gateway" "Insight" "Zone")
+          images=("sphere" "pass" "ring" "drive" "develop" "gateway" "insight" "zone")
+          changed_services=()
+
+          for file in $files; do
+            if [[ "$file" == DysonNetwork.Shared/* ]]; then
+              changed_services=("${services[@]}")
+              break
+            fi
+            for i in "${!services[@]}"; do
+              if [[ "$file" == DysonNetwork.${services[$i]}/* ]]; then
+                # check if service is already in changed_services
+                if [[ ! " ${changed_services[@]} " =~ " ${services[$i]} " ]]; then
+                  changed_services+=("${services[$i]}")
+                fi
+              fi
+            done
+          done
+
+          if [ ${#changed_services[@]} -gt 0 ]; then
+            json_objects=""
+            for service in "${changed_services[@]}"; do
+              for i in "${!services[@]}"; do
+                if [[ "${services[$i]}" == "$service" ]]; then
+                  image="${images[$i]}"
+                  break
+                fi
+              done
+              json_objects+="{\"service\":\"$service\",\"image\":\"$image\"},"
+            done
+            matrix="{\"include\":[${json_objects%,}]}"
+          fi
+          echo "matrix=$matrix" >> $GITHUB_OUTPUT
+
   build-and-push:
+    needs: determine-changes
+    if: ${{ needs.determine-changes.outputs.matrix != '{"include":[]}' }}
     runs-on: ubuntu-latest
     permissions:
       contents: read
       packages: write
-
     strategy:
-      matrix:
-        include:
-          - service: Sphere
-            image: sphere
-          - service: Pass
-            image: pass
-          - service: Ring
-            image: ring
-          - service: Drive
-            image: drive
-          - service: Develop
-            image: develop
+      matrix: ${{ fromJson(needs.determine-changes.outputs.matrix) }}
 
     steps:
       - name: Checkout repository

DysonNetwork.Control/AppHost.cs

@@ -1,77 +1,76 @@
-using Aspire.Hosting.Yarp.Transforms;
+using Microsoft.Extensions.Hosting;
 
 var builder = DistributedApplication.CreateBuilder(args);
 
-// Database was configured separately in each service.
-// var database = builder.AddPostgres("database");
+var isDev = builder.Environment.IsDevelopment();
 
-var cache = builder.AddRedis("cache");
-var queue = builder.AddNats("queue").WithJetStream();
+var cache = builder.AddRedis("Cache");
+var queue = builder.AddNats("Queue").WithJetStream();
 
-var ringService = builder.AddProject<Projects.DysonNetwork_Ring>("ring")
-    .WithReference(queue)
-    .WithHttpHealthCheck()
-    .WithEndpoint(5001, 5001, "https", name: "grpc");
+var ringService = builder.AddProject<Projects.DysonNetwork_Ring>("ring");
 var passService = builder.AddProject<Projects.DysonNetwork_Pass>("pass")
-    .WithReference(cache)
-    .WithReference(queue)
-    .WithReference(ringService)
-    .WithHttpHealthCheck()
-    .WithEndpoint(5001, 5001, "https", name: "grpc");
+    .WithReference(ringService);
 var driveService = builder.AddProject<Projects.DysonNetwork_Drive>("drive")
-    .WithReference(cache)
-    .WithReference(queue)
     .WithReference(passService)
-    .WithReference(ringService)
-    .WithHttpHealthCheck()
-    .WithEndpoint(5001, 5001, "https", name: "grpc");
+    .WithReference(ringService);
 var sphereService = builder.AddProject<Projects.DysonNetwork_Sphere>("sphere")
-    .WithReference(cache)
-    .WithReference(queue)
     .WithReference(passService)
     .WithReference(ringService)
-    .WithReference(driveService)
-    .WithHttpHealthCheck()
-    .WithEndpoint(5001, 5001, "https", name: "grpc");
+    .WithReference(driveService);
 var developService = builder.AddProject<Projects.DysonNetwork_Develop>("develop")
-    .WithReference(cache)
     .WithReference(passService)
     .WithReference(ringService)
-    .WithHttpHealthCheck()
-    .WithEndpoint(5001, 5001, "https", name: "grpc");
+    .WithReference(sphereService);
+var insightService = builder.AddProject<Projects.DysonNetwork_Insight>("insight")
+    .WithReference(passService)
+    .WithReference(ringService)
+    .WithReference(sphereService)
+    .WithReference(developService);
+var zoneService = builder.AddProject<Projects.DysonNetwork_Zone>("zone")
+    .WithReference(passService)
+    .WithReference(ringService)
+    .WithReference(sphereService)
+    .WithReference(developService)
+    .WithReference(insightService);
+
+passService.WithReference(developService).WithReference(driveService);
+
+List<IResourceBuilder<ProjectResource>> services =
+    [ringService, passService, driveService, sphereService, developService, insightService, zoneService];
+
+for (var idx = 0; idx < services.Count; idx++)
+{
+    var service = services[idx];
+
+    service.WithReference(cache).WithReference(queue);
+
+    var grpcPort = 7002 + idx;
+
+    if (isDev)
+    {
+        service.WithEnvironment("GRPC_PORT", grpcPort.ToString());
+
+        var httpPort = 8001 + idx;
+        service.WithEnvironment("HTTP_PORTS", httpPort.ToString());
+        service.WithHttpEndpoint(httpPort, targetPort: null, isProxied: false, name: "http");
+    }
+    else
+    {
+        service.WithHttpEndpoint(8080, targetPort: null, isProxied: false, name: "http");
+    }
+
+    service.WithEndpoint(isDev ? grpcPort : 7001, isDev ? null : 7001, "https", name: "grpc", isProxied: false);
+}
 
 // Extra double-ended references
 ringService.WithReference(passService);
 
-builder.AddYarp("gateway")
-    .WithHostPort(5000)
-    .WithConfiguration(yarp =>
-    {
-        var ringCluster = yarp.AddCluster(ringService.GetEndpoint("http"));
-        yarp.AddRoute("/ws", ringCluster);
-        yarp.AddRoute("/ring/{**catch-all}", ringCluster)
-            .WithTransformPathRemovePrefix("/ring")
-            .WithTransformPathPrefix("/api");
-        var passCluster = yarp.AddCluster(passService.GetEndpoint("http"));
-        yarp.AddRoute("/.well-known/openid-configuration", passCluster);
-        yarp.AddRoute("/.well-known/jwks", passCluster);
-        yarp.AddRoute("/id/{**catch-all}", passCluster)
-            .WithTransformPathRemovePrefix("/id")
-            .WithTransformPathPrefix("/api");
-        var driveCluster = yarp.AddCluster(driveService.GetEndpoint("http"));
-        yarp.AddRoute("/api/tus", driveCluster);
-        yarp.AddRoute("/drive/{**catch-all}", driveCluster)
-            .WithTransformPathRemovePrefix("/drive")
-            .WithTransformPathPrefix("/api");
-        var sphereCluster = yarp.AddCluster(sphereService.GetEndpoint("http"));
-        yarp.AddRoute("/sphere/{**catch-all}", sphereCluster)
-            .WithTransformPathRemovePrefix("/sphere")
-            .WithTransformPathPrefix("/api");
-        var developCluster = yarp.AddCluster(developService.GetEndpoint("http"));
-        yarp.AddRoute("/develop/{**catch-all}", developCluster)
-            .WithTransformPathRemovePrefix("/develop")
-            .WithTransformPathPrefix("/api");
-    });
+var gateway = builder.AddProject<Projects.DysonNetwork_Gateway>("gateway")
+    .WithEnvironment("HTTP_PORTS", "5001")
+    .WithHttpEndpoint(port: 5001, targetPort: null, isProxied: false, name: "http");
+
+foreach (var service in services)
+    gateway.WithReference(service);
 
 builder.AddDockerComposeEnvironment("docker-compose");
 

DysonNetwork.Control/DysonNetwork.Control.csproj

@@ -1,10 +1,9 @@
 <Project Sdk="Microsoft.NET.Sdk">
-
-    <Sdk Name="Aspire.AppHost.Sdk" Version="9.4.2"/>
+    <Sdk Name="Aspire.AppHost.Sdk" Version="13.1.0"/>
 
     <PropertyGroup>
         <OutputType>Exe</OutputType>
-        <TargetFramework>net9.0</TargetFramework>
+        <TargetFramework>net10.0</TargetFramework>
         <ImplicitUsings>enable</ImplicitUsings>
         <Nullable>enable</Nullable>
         <UserSecretsId>a68b3195-a00d-40c2-b5ed-d675356b7cde</UserSecretsId>
@@ -12,19 +11,19 @@
     </PropertyGroup>
 
     <ItemGroup>
-        <PackageReference Include="Aspire.Hosting.AppHost" Version="9.4.2"/>
-        <PackageReference Include="Aspire.Hosting.Docker" Version="9.4.2-preview.1.25428.12" />
-        <PackageReference Include="Aspire.Hosting.Nats" Version="9.4.2" />
-        <PackageReference Include="Aspire.Hosting.Redis" Version="9.4.2" />
-        <PackageReference Include="Aspire.Hosting.Yarp" Version="9.4.2-preview.1.25428.12" />
+        <PackageReference Include="Aspire.Hosting.AppHost" Version="13.1.0" />
+        <PackageReference Include="Aspire.Hosting.Docker" Version="13.0.0-preview.1.25560.3"/>
+        <PackageReference Include="Aspire.Hosting.Nats" Version="13.1.0"/>
+        <PackageReference Include="Aspire.Hosting.Redis" Version="13.1.0"/>
     </ItemGroup>
-
     <ItemGroup>
-      <ProjectReference Include="..\DysonNetwork.Develop\DysonNetwork.Develop.csproj" />
-      <ProjectReference Include="..\DysonNetwork.Drive\DysonNetwork.Drive.csproj" />
-      <ProjectReference Include="..\DysonNetwork.Pass\DysonNetwork.Pass.csproj" />
-      <ProjectReference Include="..\DysonNetwork.Ring\DysonNetwork.Ring.csproj" />
-      <ProjectReference Include="..\DysonNetwork.Sphere\DysonNetwork.Sphere.csproj" />
+        <ProjectReference Include="..\DysonNetwork.Develop\DysonNetwork.Develop.csproj"/>
+        <ProjectReference Include="..\DysonNetwork.Drive\DysonNetwork.Drive.csproj"/>
+        <ProjectReference Include="..\DysonNetwork.Pass\DysonNetwork.Pass.csproj"/>
+        <ProjectReference Include="..\DysonNetwork.Ring\DysonNetwork.Ring.csproj"/>
+        <ProjectReference Include="..\DysonNetwork.Sphere\DysonNetwork.Sphere.csproj"/>
+        <ProjectReference Include="..\DysonNetwork.Gateway\DysonNetwork.Gateway.csproj"/>
+        <ProjectReference Include="..\DysonNetwork.Insight\DysonNetwork.Insight.csproj"/>
+        <ProjectReference Include="..\DysonNetwork.Zone\DysonNetwork.Zone.csproj"/>
     </ItemGroup>
-
 </Project>

DysonNetwork.Control/Properties/launchSettings.json

@@ -5,12 +5,14 @@
       "commandName": "Project",
       "dotnetRunMessages": true,
       "launchBrowser": true,
-      "applicationUrl": "https://localhost:17025;http://localhost:15057",
+      "applicationUrl": "https://localhost:17169;http://localhost:15057",
       "environmentVariables": {
         "ASPNETCORE_ENVIRONMENT": "Development",
         "DOTNET_ENVIRONMENT": "Development",
         "ASPIRE_DASHBOARD_OTLP_ENDPOINT_URL": "https://localhost:21175",
-        "ASPIRE_RESOURCE_SERVICE_ENDPOINT_URL": "https://localhost:22189"
+        "ASPIRE_RESOURCE_SERVICE_ENDPOINT_URL": "https://localhost:22189",
+        "DOTNET_DASHBOARD_OTLP_ENDPOINT_URL": "https://localhost:21260",
+        "DOTNET_RESOURCE_SERVICE_ENDPOINT_URL": "https://localhost:22052"
       }
     },
     "http": {
@@ -22,7 +24,8 @@
         "ASPNETCORE_ENVIRONMENT": "Development",
         "DOTNET_ENVIRONMENT": "Development",
         "ASPIRE_DASHBOARD_OTLP_ENDPOINT_URL": "http://localhost:19163",
-        "ASPIRE_RESOURCE_SERVICE_ENDPOINT_URL": "http://localhost:20185"
+        "ASPIRE_RESOURCE_SERVICE_ENDPOINT_URL": "http://localhost:20185",
+        "DOTNET_RESOURCE_SERVICE_ENDPOINT_URL": "http://localhost:22108"
       }
     }
   }

DysonNetwork.Control/aspire-manifest.json

@@ -0,0 +1,357 @@
+{
+  "$schema": "https://json.schemastore.org/aspire-8.0.json",
+  "resources": {
+    "cache": {
+      "type": "container.v1",
+      "connectionString": "{cache.bindings.tcp.host}:{cache.bindings.tcp.port},password={cache-password.value}",
+      "image": "docker.io/library/redis:8.2",
+      "entrypoint": "/bin/sh",
+      "args": [
+        "-c",
+        "redis-server --requirepass $REDIS_PASSWORD"
+      ],
+      "env": {
+        "REDIS_PASSWORD": "{cache-password.value}"
+      },
+      "bindings": {
+        "tcp": {
+          "scheme": "tcp",
+          "protocol": "tcp",
+          "transport": "tcp",
+          "targetPort": 6379
+        }
+      }
+    },
+    "queue": {
+      "type": "container.v1",
+      "connectionString": "nats://nats:{queue-password.value}@{queue.bindings.tcp.host}:{queue.bindings.tcp.port}",
+      "image": "docker.io/library/nats:2.11",
+      "args": [
+        "--user",
+        "nats",
+        "--pass",
+        "{queue-password.value}",
+        "-js"
+      ],
+      "bindings": {
+        "tcp": {
+          "scheme": "tcp",
+          "protocol": "tcp",
+          "transport": "tcp",
+          "targetPort": 4222
+        }
+      }
+    },
+    "ring": {
+      "type": "project.v1",
+      "path": "../DysonNetwork.Ring/DysonNetwork.Ring.csproj",
+      "env": {
+        "OTEL_DOTNET_EXPERIMENTAL_OTLP_EMIT_EXCEPTION_LOG_ATTRIBUTES": "true",
+        "OTEL_DOTNET_EXPERIMENTAL_OTLP_EMIT_EVENT_LOG_ATTRIBUTES": "true",
+        "OTEL_DOTNET_EXPERIMENTAL_OTLP_RETRY": "in_memory",
+        "ASPNETCORE_FORWARDEDHEADERS_ENABLED": "true",
+        "HTTP_PORTS": "8001",
+        "HTTPS_PORTS": "{ring.bindings.grpc.targetPort}",
+        "ConnectionStrings__cache": "{cache.connectionString}",
+        "ConnectionStrings__queue": "{queue.connectionString}",
+        "GRPC_PORT": "7002",
+        "services__pass__http__0": "{pass.bindings.http.url}",
+        "services__pass__grpc__0": "{pass.bindings.grpc.url}",
+        "OTEL_EXPORTER_OTLP_ENDPOINT": "{docker-compose-dashboard.bindings.otlp-grpc.url}",
+        "OTEL_EXPORTER_OTLP_PROTOCOL": "grpc",
+        "OTEL_SERVICE_NAME": "ring"
+      },
+      "bindings": {
+        "http": {
+          "scheme": "http",
+          "protocol": "tcp",
+          "transport": "http",
+          "targetPort": 8001
+        },
+        "grpc": {
+          "scheme": "https",
+          "protocol": "tcp",
+          "transport": "http",
+          "targetPort": 7002
+        }
+      }
+    },
+    "pass": {
+      "type": "project.v1",
+      "path": "../DysonNetwork.Pass/DysonNetwork.Pass.csproj",
+      "env": {
+        "OTEL_DOTNET_EXPERIMENTAL_OTLP_EMIT_EXCEPTION_LOG_ATTRIBUTES": "true",
+        "OTEL_DOTNET_EXPERIMENTAL_OTLP_EMIT_EVENT_LOG_ATTRIBUTES": "true",
+        "OTEL_DOTNET_EXPERIMENTAL_OTLP_RETRY": "in_memory",
+        "ASPNETCORE_FORWARDEDHEADERS_ENABLED": "true",
+        "HTTP_PORTS": "8002",
+        "HTTPS_PORTS": "{pass.bindings.grpc.targetPort}",
+        "services__ring__http__0": "{ring.bindings.http.url}",
+        "services__ring__grpc__0": "{ring.bindings.grpc.url}",
+        "services__develop__http__0": "{develop.bindings.http.url}",
+        "services__develop__grpc__0": "{develop.bindings.grpc.url}",
+        "services__drive__http__0": "{drive.bindings.http.url}",
+        "services__drive__grpc__0": "{drive.bindings.grpc.url}",
+        "ConnectionStrings__cache": "{cache.connectionString}",
+        "ConnectionStrings__queue": "{queue.connectionString}",
+        "GRPC_PORT": "7003",
+        "OTEL_EXPORTER_OTLP_ENDPOINT": "{docker-compose-dashboard.bindings.otlp-grpc.url}",
+        "OTEL_EXPORTER_OTLP_PROTOCOL": "grpc",
+        "OTEL_SERVICE_NAME": "pass"
+      },
+      "bindings": {
+        "http": {
+          "scheme": "http",
+          "protocol": "tcp",
+          "transport": "http",
+          "targetPort": 8002
+        },
+        "grpc": {
+          "scheme": "https",
+          "protocol": "tcp",
+          "transport": "http",
+          "targetPort": 7003
+        }
+      }
+    },
+    "drive": {
+      "type": "project.v1",
+      "path": "../DysonNetwork.Drive/DysonNetwork.Drive.csproj",
+      "env": {
+        "OTEL_DOTNET_EXPERIMENTAL_OTLP_EMIT_EXCEPTION_LOG_ATTRIBUTES": "true",
+        "OTEL_DOTNET_EXPERIMENTAL_OTLP_EMIT_EVENT_LOG_ATTRIBUTES": "true",
+        "OTEL_DOTNET_EXPERIMENTAL_OTLP_RETRY": "in_memory",
+        "ASPNETCORE_FORWARDEDHEADERS_ENABLED": "true",
+        "HTTP_PORTS": "8003",
+        "HTTPS_PORTS": "{drive.bindings.grpc.targetPort}",
+        "services__pass__http__0": "{pass.bindings.http.url}",
+        "services__pass__grpc__0": "{pass.bindings.grpc.url}",
+        "services__ring__http__0": "{ring.bindings.http.url}",
+        "services__ring__grpc__0": "{ring.bindings.grpc.url}",
+        "ConnectionStrings__cache": "{cache.connectionString}",
+        "ConnectionStrings__queue": "{queue.connectionString}",
+        "GRPC_PORT": "7004",
+        "OTEL_EXPORTER_OTLP_ENDPOINT": "{docker-compose-dashboard.bindings.otlp-grpc.url}",
+        "OTEL_EXPORTER_OTLP_PROTOCOL": "grpc",
+        "OTEL_SERVICE_NAME": "drive"
+      },
+      "bindings": {
+        "http": {
+          "scheme": "http",
+          "protocol": "tcp",
+          "transport": "http",
+          "targetPort": 8003
+        },
+        "grpc": {
+          "scheme": "https",
+          "protocol": "tcp",
+          "transport": "http",
+          "targetPort": 7004
+        }
+      }
+    },
+    "sphere": {
+      "type": "project.v1",
+      "path": "../DysonNetwork.Sphere/DysonNetwork.Sphere.csproj",
+      "env": {
+        "OTEL_DOTNET_EXPERIMENTAL_OTLP_EMIT_EXCEPTION_LOG_ATTRIBUTES": "true",
+        "OTEL_DOTNET_EXPERIMENTAL_OTLP_EMIT_EVENT_LOG_ATTRIBUTES": "true",
+        "OTEL_DOTNET_EXPERIMENTAL_OTLP_RETRY": "in_memory",
+        "ASPNETCORE_FORWARDEDHEADERS_ENABLED": "true",
+        "HTTP_PORTS": "8004",
+        "HTTPS_PORTS": "{sphere.bindings.grpc.targetPort}",
+        "services__pass__http__0": "{pass.bindings.http.url}",
+        "services__pass__grpc__0": "{pass.bindings.grpc.url}",
+        "services__ring__http__0": "{ring.bindings.http.url}",
+        "services__ring__grpc__0": "{ring.bindings.grpc.url}",
+        "services__drive__http__0": "{drive.bindings.http.url}",
+        "services__drive__grpc__0": "{drive.bindings.grpc.url}",
+        "ConnectionStrings__cache": "{cache.connectionString}",
+        "ConnectionStrings__queue": "{queue.connectionString}",
+        "GRPC_PORT": "7005",
+        "OTEL_EXPORTER_OTLP_ENDPOINT": "{docker-compose-dashboard.bindings.otlp-grpc.url}",
+        "OTEL_EXPORTER_OTLP_PROTOCOL": "grpc",
+        "OTEL_SERVICE_NAME": "sphere"
+      },
+      "bindings": {
+        "http": {
+          "scheme": "http",
+          "protocol": "tcp",
+          "transport": "http",
+          "targetPort": 8004
+        },
+        "grpc": {
+          "scheme": "https",
+          "protocol": "tcp",
+          "transport": "http",
+          "targetPort": 7005
+        }
+      }
+    },
+    "develop": {
+      "type": "project.v1",
+      "path": "../DysonNetwork.Develop/DysonNetwork.Develop.csproj",
+      "env": {
+        "OTEL_DOTNET_EXPERIMENTAL_OTLP_EMIT_EXCEPTION_LOG_ATTRIBUTES": "true",
+        "OTEL_DOTNET_EXPERIMENTAL_OTLP_EMIT_EVENT_LOG_ATTRIBUTES": "true",
+        "OTEL_DOTNET_EXPERIMENTAL_OTLP_RETRY": "in_memory",
+        "ASPNETCORE_FORWARDEDHEADERS_ENABLED": "true",
+        "HTTP_PORTS": "8005",
+        "HTTPS_PORTS": "{develop.bindings.grpc.targetPort}",
+        "services__pass__http__0": "{pass.bindings.http.url}",
+        "services__pass__grpc__0": "{pass.bindings.grpc.url}",
+        "services__ring__http__0": "{ring.bindings.http.url}",
+        "services__ring__grpc__0": "{ring.bindings.grpc.url}",
+        "services__sphere__http__0": "{sphere.bindings.http.url}",
+        "services__sphere__grpc__0": "{sphere.bindings.grpc.url}",
+        "ConnectionStrings__cache": "{cache.connectionString}",
+        "ConnectionStrings__queue": "{queue.connectionString}",
+        "GRPC_PORT": "7006",
+        "OTEL_EXPORTER_OTLP_ENDPOINT": "{docker-compose-dashboard.bindings.otlp-grpc.url}",
+        "OTEL_EXPORTER_OTLP_PROTOCOL": "grpc",
+        "OTEL_SERVICE_NAME": "develop"
+      },
+      "bindings": {
+        "http": {
+          "scheme": "http",
+          "protocol": "tcp",
+          "transport": "http",
+          "targetPort": 8005
+        },
+        "grpc": {
+          "scheme": "https",
+          "protocol": "tcp",
+          "transport": "http",
+          "targetPort": 7006
+        }
+      }
+    },
+    "insight": {
+      "type": "project.v1",
+      "path": "../DysonNetwork.Insight/DysonNetwork.Insight.csproj",
+      "env": {
+        "OTEL_DOTNET_EXPERIMENTAL_OTLP_EMIT_EXCEPTION_LOG_ATTRIBUTES": "true",
+        "OTEL_DOTNET_EXPERIMENTAL_OTLP_EMIT_EVENT_LOG_ATTRIBUTES": "true",
+        "OTEL_DOTNET_EXPERIMENTAL_OTLP_RETRY": "in_memory",
+        "ASPNETCORE_FORWARDEDHEADERS_ENABLED": "true",
+        "HTTP_PORTS": "8006",
+        "HTTPS_PORTS": "{insight.bindings.grpc.targetPort}",
+        "services__pass__http__0": "{pass.bindings.http.url}",
+        "services__pass__grpc__0": "{pass.bindings.grpc.url}",
+        "services__ring__http__0": "{ring.bindings.http.url}",
+        "services__ring__grpc__0": "{ring.bindings.grpc.url}",
+        "services__sphere__http__0": "{sphere.bindings.http.url}",
+        "services__sphere__grpc__0": "{sphere.bindings.grpc.url}",
+        "services__develop__http__0": "{develop.bindings.http.url}",
+        "services__develop__grpc__0": "{develop.bindings.grpc.url}",
+        "ConnectionStrings__cache": "{cache.connectionString}",
+        "ConnectionStrings__queue": "{queue.connectionString}",
+        "GRPC_PORT": "7007",
+        "OTEL_EXPORTER_OTLP_ENDPOINT": "{docker-compose-dashboard.bindings.otlp-grpc.url}",
+        "OTEL_EXPORTER_OTLP_PROTOCOL": "grpc",
+        "OTEL_SERVICE_NAME": "insight"
+      },
+      "bindings": {
+        "http": {
+          "scheme": "http",
+          "protocol": "tcp",
+          "transport": "http",
+          "targetPort": 8006
+        },
+        "grpc": {
+          "scheme": "https",
+          "protocol": "tcp",
+          "transport": "http",
+          "targetPort": 7007
+        }
+      }
+    },
+    "gateway": {
+      "type": "project.v1",
+      "path": "../DysonNetwork.Gateway/DysonNetwork.Gateway.csproj",
+      "env": {
+        "OTEL_DOTNET_EXPERIMENTAL_OTLP_EMIT_EXCEPTION_LOG_ATTRIBUTES": "true",
+        "OTEL_DOTNET_EXPERIMENTAL_OTLP_EMIT_EVENT_LOG_ATTRIBUTES": "true",
+        "OTEL_DOTNET_EXPERIMENTAL_OTLP_RETRY": "in_memory",
+        "ASPNETCORE_FORWARDEDHEADERS_ENABLED": "true",
+        "HTTP_PORTS": "5001",
+        "services__ring__http__0": "{ring.bindings.http.url}",
+        "services__ring__grpc__0": "{ring.bindings.grpc.url}",
+        "services__pass__http__0": "{pass.bindings.http.url}",
+        "services__pass__grpc__0": "{pass.bindings.grpc.url}",
+        "services__drive__http__0": "{drive.bindings.http.url}",
+        "services__drive__grpc__0": "{drive.bindings.grpc.url}",
+        "services__sphere__http__0": "{sphere.bindings.http.url}",
+        "services__sphere__grpc__0": "{sphere.bindings.grpc.url}",
+        "services__develop__http__0": "{develop.bindings.http.url}",
+        "services__develop__grpc__0": "{develop.bindings.grpc.url}",
+        "services__insight__http__0": "{insight.bindings.http.url}",
+        "services__insight__grpc__0": "{insight.bindings.grpc.url}",
+        "OTEL_EXPORTER_OTLP_ENDPOINT": "{docker-compose-dashboard.bindings.otlp-grpc.url}",
+        "OTEL_EXPORTER_OTLP_PROTOCOL": "grpc",
+        "OTEL_SERVICE_NAME": "gateway"
+      },
+      "bindings": {
+        "http": {
+          "scheme": "http",
+          "protocol": "tcp",
+          "transport": "http",
+          "targetPort": 5001
+        }
+      }
+    },
+    "docker-compose": {
+      "error": "This resource does not support generation in the manifest."
+    },
+    "cache-password": {
+      "type": "parameter.v0",
+      "value": "{cache-password.inputs.value}",
+      "inputs": {
+        "value": {
+          "type": "string",
+          "secret": true,
+          "default": {
+            "generate": {
+              "minLength": 22,
+              "special": false
+            }
+          }
+        }
+      }
+    },
+    "queue-password": {
+      "type": "parameter.v0",
+      "value": "{queue-password.inputs.value}",
+      "inputs": {
+        "value": {
+          "type": "string",
+          "secret": true,
+          "default": {
+            "generate": {
+              "minLength": 22,
+              "special": false
+            }
+          }
+        }
+      }
+    },
+    "docker-compose-dashboard": {
+      "type": "container.v1",
+      "image": "mcr.microsoft.com/dotnet/nightly/aspire-dashboard:latest",
+      "bindings": {
+        "http": {
+          "scheme": "http",
+          "protocol": "tcp",
+          "transport": "http",
+          "targetPort": 18888
+        },
+        "otlp-grpc": {
+          "scheme": "http",
+          "protocol": "tcp",
+          "transport": "http",
+          "targetPort": 18889
+        }
+      }
+    }
+  }
+}

DysonNetwork.Develop/AppDatabase.cs

@@ -1,8 +1,8 @@
-using System.Text.Json;
-using DysonNetwork.Develop.Identity;
-using DysonNetwork.Develop.Project;
+using DysonNetwork.Shared.Data;
+using DysonNetwork.Shared.Models;
 using Microsoft.EntityFrameworkCore;
 using Microsoft.EntityFrameworkCore.Design;
+using NodaTime;
 
 namespace DysonNetwork.Develop;
 
@@ -11,13 +11,13 @@ public class AppDatabase(
     IConfiguration configuration
 ) : DbContext(options)
 {
-    public DbSet<Developer> Developers { get; set; } = null!;
+    public DbSet<SnDeveloper> Developers { get; set; } = null!;
 
-    public DbSet<DevProject> DevProjects { get; set; } = null!;
+    public DbSet<SnDevProject> DevProjects { get; set; } = null!;
     
-    public DbSet<CustomApp> CustomApps { get; set; } = null!;
-    public DbSet<CustomAppSecret> CustomAppSecrets { get; set; } = null!;
-    public DbSet<BotAccount> BotAccounts { get; set; } = null!;
+    public DbSet<SnCustomApp> CustomApps { get; set; } = null!;
+    public DbSet<SnCustomAppSecret> CustomAppSecrets { get; set; } = null!;
+    public DbSet<SnBotAccount> BotAccounts { get; set; } = null!;
 
     protected override void OnConfiguring(DbContextOptionsBuilder optionsBuilder)
     {
@@ -32,9 +32,17 @@ public class AppDatabase(
         base.OnConfiguring(optionsBuilder);
     }
     
+    public override async Task<int> SaveChangesAsync(CancellationToken cancellationToken = default)
+    {
+        this.ApplyAuditableAndSoftDelete();
+        return await base.SaveChangesAsync(cancellationToken);
+    }
+
     protected override void OnModelCreating(ModelBuilder modelBuilder)
     {
         base.OnModelCreating(modelBuilder);
+        
+        modelBuilder.ApplySoftDeleteFilters();
     }
 }
 

DysonNetwork.Develop/Dockerfile

@@ -1,10 +1,15 @@
-FROM mcr.microsoft.com/dotnet/aspnet:9.0 AS base
+FROM mcr.microsoft.com/dotnet/aspnet:10.0 AS base
+RUN apt-get update && \
+    apt-get install -y --no-install-recommends \
+        libkrb5-3 \
+        libgssapi-krb5-2 \
+    && rm -rf /var/lib/apt/lists/*
 USER $APP_UID
 WORKDIR /app
 EXPOSE 8080
 EXPOSE 8081
 
-FROM mcr.microsoft.com/dotnet/sdk:9.0 AS build
+FROM mcr.microsoft.com/dotnet/sdk:10.0 AS build
 ARG BUILD_CONFIGURATION=Release
 WORKDIR /src
 COPY ["DysonNetwork.Develop/DysonNetwork.Develop.csproj", "DysonNetwork.Develop/"]

DysonNetwork.Develop/DysonNetwork.Develop.csproj

@@ -1,27 +1,22 @@
 <Project Sdk="Microsoft.NET.Sdk.Web">
 
     <PropertyGroup>
-        <TargetFramework>net9.0</TargetFramework>
+        <TargetFramework>net10.0</TargetFramework>
         <Nullable>enable</Nullable>
         <ImplicitUsings>enable</ImplicitUsings>
         <DockerDefaultTargetOS>Linux</DockerDefaultTargetOS>
     </PropertyGroup>
 
     <ItemGroup>
-        <PackageReference Include="EFCore.NamingConventions" Version="9.0.0" />
-        <PackageReference Include="Microsoft.AspNetCore.OpenApi" Version="9.0.7"/>
-        <PackageReference Include="Microsoft.EntityFrameworkCore.Design" Version="9.0.7">
-            <PrivateAssets>all</PrivateAssets>
-            <IncludeAssets>runtime; build; native; contentfiles; analyzers; buildtransitive</IncludeAssets>
+        <PackageReference Include="Microsoft.AspNetCore.OpenApi" Version="10.0.1" />
+        <PackageReference Include="Microsoft.EntityFrameworkCore.Design" Version="10.0.1">
+          <PrivateAssets>all</PrivateAssets>
+          <IncludeAssets>runtime; build; native; contentfiles; analyzers; buildtransitive</IncludeAssets>
         </PackageReference>
         <PackageReference Include="NodaTime.Serialization.Protobuf" Version="2.0.2" />
-        <PackageReference Include="Npgsql.EntityFrameworkCore.PostgreSQL" Version="9.0.4"/>
-        <PackageReference Include="Npgsql.EntityFrameworkCore.PostgreSQL.NodaTime" Version="9.0.4" />
-        <PackageReference Include="prometheus-net.AspNetCore" Version="8.2.1"/>
-        <PackageReference Include="Swashbuckle.AspNetCore" Version="9.0.3"/>
-        <PackageReference Include="NodaTime" Version="3.2.2"/>
+        <PackageReference Include="NodaTime" Version="3.2.3" />
         <PackageReference Include="NodaTime.Serialization.SystemTextJson" Version="1.3.0"/>
-        <PackageReference Include="Grpc.AspNetCore.Server" Version="2.71.0"/>
+        <PackageReference Include="Grpc.AspNetCore.Server" Version="2.76.0" />
   </ItemGroup>
  
   <ItemGroup>
@@ -31,7 +26,6 @@
     </ItemGroup>
 
   <ItemGroup>
-    <ProjectReference Include="..\DysonNetwork.ServiceDefaults\DysonNetwork.ServiceDefaults.csproj" />
     <ProjectReference Include="..\DysonNetwork.Shared\DysonNetwork.Shared.csproj" />
   </ItemGroup>
  

DysonNetwork.Develop/Identity/BotAccountController.cs

@@ -1,6 +1,6 @@
 using System.ComponentModel.DataAnnotations;
 using DysonNetwork.Develop.Project;
-using DysonNetwork.Shared.Data;
+using DysonNetwork.Shared.Models;
 using DysonNetwork.Shared.Proto;
 using DysonNetwork.Shared.Registry;
 using Grpc.Core;
@@ -16,10 +16,10 @@ namespace DysonNetwork.Develop.Identity;
 [Authorize]
 public class BotAccountController(
     BotAccountService botService,
-    DeveloperService developerService,
+    DeveloperService ds,
     DevProjectService projectService,
     ILogger<BotAccountController> logger,
-    AccountClientHelper accounts,
+    RemoteAccountService remoteAccounts,
     BotAccountReceiverService.BotAccountReceiverServiceClient accountsReceiver
 )
     : ControllerBase
@@ -50,9 +50,9 @@ public class BotAccountController(
         ]
         public string Name { get; set; } = string.Empty;
 
-        [Required] [MaxLength(256)] public string Nick { get; set; } = string.Empty;
+        [Required][MaxLength(256)] public string Nick { get; set; } = string.Empty;
 
-        [Required] [MaxLength(1024)] public string Slug { get; set; } = string.Empty;
+        [Required][MaxLength(1024)] public string Slug { get; set; } = string.Empty;
 
         [MaxLength(128)] public string Language { get; set; } = "en-us";
     }
@@ -68,7 +68,7 @@ public class BotAccountController(
 
         [MaxLength(256)] public string? Nick { get; set; } = string.Empty;
 
-        [Required] [MaxLength(1024)] public string? Slug { get; set; } = string.Empty;
+        [Required][MaxLength(1024)] public string? Slug { get; set; } = string.Empty;
 
         [MaxLength(128)] public string? Language { get; set; }
 
@@ -83,12 +83,12 @@ public class BotAccountController(
         if (HttpContext.Items["CurrentUser"] is not Account currentUser)
             return Unauthorized();
 
-        var developer = await developerService.GetDeveloperByName(pubName);
+        var developer = await ds.GetDeveloperByName(pubName);
         if (developer is null)
             return NotFound("Developer not found");
 
-        if (!await developerService.IsMemberWithRole(developer.PublisherId, Guid.Parse(currentUser.Id),
-                PublisherMemberRole.Viewer))
+        if (!await ds.IsMemberWithRole(developer.PublisherId, Guid.Parse(currentUser.Id),
+                Shared.Proto.PublisherMemberRole.Viewer))
             return StatusCode(403, "You must be an viewer of the developer to list bots");
 
         var project = await projectService.GetProjectAsync(projectId, developer.Id);
@@ -108,12 +108,12 @@ public class BotAccountController(
         if (HttpContext.Items["CurrentUser"] is not Account currentUser)
             return Unauthorized();
 
-        var developer = await developerService.GetDeveloperByName(pubName);
+        var developer = await ds.GetDeveloperByName(pubName);
         if (developer is null)
             return NotFound("Developer not found");
 
-        if (!await developerService.IsMemberWithRole(developer.PublisherId, Guid.Parse(currentUser.Id),
-                PublisherMemberRole.Viewer))
+        if (!await ds.IsMemberWithRole(developer.PublisherId, Guid.Parse(currentUser.Id),
+                Shared.Proto.PublisherMemberRole.Viewer))
             return StatusCode(403, "You must be an viewer of the developer to view bot details");
 
         var project = await projectService.GetProjectAsync(projectId, developer.Id);
@@ -137,12 +137,12 @@ public class BotAccountController(
         if (HttpContext.Items["CurrentUser"] is not Account currentUser)
             return Unauthorized();
 
-        var developer = await developerService.GetDeveloperByName(pubName);
+        var developer = await ds.GetDeveloperByName(pubName);
         if (developer is null)
             return NotFound("Developer not found");
 
-        if (!await developerService.IsMemberWithRole(developer.PublisherId, Guid.Parse(currentUser.Id),
-                PublisherMemberRole.Editor))
+        if (!await ds.IsMemberWithRole(developer.PublisherId, Guid.Parse(currentUser.Id),
+                Shared.Proto.PublisherMemberRole.Editor))
             return StatusCode(403, "You must be an editor of the developer to create a bot");
 
         var project = await projectService.GetProjectAsync(projectId, developer.Id);
@@ -206,12 +206,12 @@ public class BotAccountController(
         if (HttpContext.Items["CurrentUser"] is not Account currentUser)
             return Unauthorized();
 
-        var developer = await developerService.GetDeveloperByName(pubName);
+        var developer = await ds.GetDeveloperByName(pubName);
         if (developer is null)
             return NotFound("Developer not found");
 
-        if (!await developerService.IsMemberWithRole(developer.PublisherId, Guid.Parse(currentUser.Id),
-                PublisherMemberRole.Editor))
+        if (!await ds.IsMemberWithRole(developer.PublisherId, Guid.Parse(currentUser.Id),
+                Shared.Proto.PublisherMemberRole.Editor))
             return StatusCode(403, "You must be an editor of the developer to update a bot");
 
         var project = await projectService.GetProjectAsync(projectId, developer.Id);
@@ -222,7 +222,7 @@ public class BotAccountController(
         if (bot is null || bot.ProjectId != projectId)
             return NotFound("Bot not found");
 
-        var botAccount = await accounts.GetBotAccount(bot.Id);
+        var botAccount = await remoteAccounts.GetBotAccount(bot.Id);
 
         if (request.Name is not null) botAccount.Name = request.Name;
         if (request.Nick is not null) botAccount.Nick = request.Nick;
@@ -267,12 +267,12 @@ public class BotAccountController(
         if (HttpContext.Items["CurrentUser"] is not Account currentUser)
             return Unauthorized();
 
-        var developer = await developerService.GetDeveloperByName(pubName);
+        var developer = await ds.GetDeveloperByName(pubName);
         if (developer is null)
             return NotFound("Developer not found");
 
-        if (!await developerService.IsMemberWithRole(developer.PublisherId, Guid.Parse(currentUser.Id),
-                PublisherMemberRole.Editor))
+        if (!await ds.IsMemberWithRole(developer.PublisherId, Guid.Parse(currentUser.Id),
+                Shared.Proto.PublisherMemberRole.Editor))
             return StatusCode(403, "You must be an editor of the developer to delete a bot");
 
         var project = await projectService.GetProjectAsync(projectId, developer.Id);
@@ -296,7 +296,7 @@ public class BotAccountController(
     }
 
     [HttpGet("{botId:guid}/keys")]
-    public async Task<ActionResult<List<ApiKeyReference>>> ListBotKeys(
+    public async Task<ActionResult<List<SnApiKey>>> ListBotKeys(
         [FromRoute] string pubName,
         [FromRoute] Guid projectId,
         [FromRoute] Guid botId
@@ -305,7 +305,7 @@ public class BotAccountController(
         if (HttpContext.Items["CurrentUser"] is not Account currentUser)
             return Unauthorized();
 
-        var (developer, project, bot) = await ValidateBotAccess(pubName, projectId, botId, currentUser, PublisherMemberRole.Viewer);
+        var (developer, project, bot) = await ValidateBotAccess(pubName, projectId, botId, currentUser, Shared.Proto.PublisherMemberRole.Viewer);
         if (developer == null) return NotFound("Developer not found");
         if (project == null) return NotFound("Project not found or you don't have access");
         if (bot == null) return NotFound("Bot not found");
@@ -314,13 +314,13 @@ public class BotAccountController(
         {
             AutomatedId = bot.Id.ToString()
         });
-        var data = keys.Data.Select(ApiKeyReference.FromProtoValue).ToList();
+        var data = keys.Data.Select(SnApiKey.FromProtoValue).ToList();
 
         return Ok(data);
     }
 
     [HttpGet("{botId:guid}/keys/{keyId:guid}")]
-    public async Task<ActionResult<ApiKeyReference>> GetBotKey(
+    public async Task<ActionResult<SnApiKey>> GetBotKey(
         [FromRoute] string pubName,
         [FromRoute] Guid projectId,
         [FromRoute] Guid botId,
@@ -329,7 +329,7 @@ public class BotAccountController(
         if (HttpContext.Items["CurrentUser"] is not Account currentUser)
             return Unauthorized();
 
-        var (developer, project, bot) = await ValidateBotAccess(pubName, projectId, botId, currentUser, PublisherMemberRole.Viewer);
+        var (developer, project, bot) = await ValidateBotAccess(pubName, projectId, botId, currentUser, Shared.Proto.PublisherMemberRole.Viewer);
         if (developer == null) return NotFound("Developer not found");
         if (project == null) return NotFound("Project not found or you don't have access");
         if (bot == null) return NotFound("Bot not found");
@@ -338,7 +338,7 @@ public class BotAccountController(
         {
             var key = await accountsReceiver.GetApiKeyAsync(new GetApiKeyRequest { Id = keyId.ToString() });
             if (key == null) return NotFound("API key not found");
-            return Ok(ApiKeyReference.FromProtoValue(key));
+            return Ok(SnApiKey.FromProtoValue(key));
         }
         catch (RpcException ex) when (ex.StatusCode == Grpc.Core.StatusCode.NotFound)
         {
@@ -353,7 +353,7 @@ public class BotAccountController(
     }
 
     [HttpPost("{botId:guid}/keys")]
-    public async Task<ActionResult<ApiKeyReference>> CreateBotKey(
+    public async Task<ActionResult<SnApiKey>> CreateBotKey(
         [FromRoute] string pubName,
         [FromRoute] Guid projectId,
         [FromRoute] Guid botId,
@@ -362,7 +362,7 @@ public class BotAccountController(
         if (HttpContext.Items["CurrentUser"] is not Account currentUser)
             return Unauthorized();
 
-        var (developer, project, bot) = await ValidateBotAccess(pubName, projectId, botId, currentUser, PublisherMemberRole.Editor);
+        var (developer, project, bot) = await ValidateBotAccess(pubName, projectId, botId, currentUser, Shared.Proto.PublisherMemberRole.Editor);
         if (developer == null) return NotFound("Developer not found");
         if (project == null) return NotFound("Project not found or you don't have access");
         if (bot == null) return NotFound("Bot not found");
@@ -376,7 +376,7 @@ public class BotAccountController(
             };
 
             var createdKey = await accountsReceiver.CreateApiKeyAsync(newKey);
-            return Ok(ApiKeyReference.FromProtoValue(createdKey));
+            return Ok(SnApiKey.FromProtoValue(createdKey));
         }
         catch (RpcException ex) when (ex.StatusCode == Grpc.Core.StatusCode.InvalidArgument)
         {
@@ -385,7 +385,7 @@ public class BotAccountController(
     }
 
     [HttpPost("{botId:guid}/keys/{keyId:guid}/rotate")]
-    public async Task<ActionResult<ApiKeyReference>> RotateBotKey(
+    public async Task<ActionResult<SnApiKey>> RotateBotKey(
         [FromRoute] string pubName,
         [FromRoute] Guid projectId,
         [FromRoute] Guid botId,
@@ -394,7 +394,7 @@ public class BotAccountController(
         if (HttpContext.Items["CurrentUser"] is not Account currentUser)
             return Unauthorized();
 
-        var (developer, project, bot) = await ValidateBotAccess(pubName, projectId, botId, currentUser, PublisherMemberRole.Editor);
+        var (developer, project, bot) = await ValidateBotAccess(pubName, projectId, botId, currentUser, Shared.Proto.PublisherMemberRole.Editor);
         if (developer == null) return NotFound("Developer not found");
         if (project == null) return NotFound("Project not found or you don't have access");
         if (bot == null) return NotFound("Bot not found");
@@ -402,7 +402,7 @@ public class BotAccountController(
         try
         {
             var rotatedKey = await accountsReceiver.RotateApiKeyAsync(new GetApiKeyRequest { Id = keyId.ToString() });
-            return Ok(ApiKeyReference.FromProtoValue(rotatedKey));
+            return Ok(SnApiKey.FromProtoValue(rotatedKey));
         }
         catch (RpcException ex) when (ex.StatusCode == Grpc.Core.StatusCode.NotFound)
         {
@@ -420,7 +420,7 @@ public class BotAccountController(
         if (HttpContext.Items["CurrentUser"] is not Account currentUser)
             return Unauthorized();
 
-        var (developer, project, bot) = await ValidateBotAccess(pubName, projectId, botId, currentUser, PublisherMemberRole.Editor);
+        var (developer, project, bot) = await ValidateBotAccess(pubName, projectId, botId, currentUser, Shared.Proto.PublisherMemberRole.Editor);
         if (developer == null) return NotFound("Developer not found");
         if (project == null) return NotFound("Project not found or you don't have access");
         if (bot == null) return NotFound("Bot not found");
@@ -436,17 +436,17 @@ public class BotAccountController(
         }
     }
 
-    private async Task<(Developer?, DevProject?, BotAccount?)> ValidateBotAccess(
+    private async Task<(SnDeveloper?, SnDevProject?, SnBotAccount?)> ValidateBotAccess(
         string pubName,
         Guid projectId,
         Guid botId,
         Account currentUser,
-        PublisherMemberRole requiredRole)
+        Shared.Proto.PublisherMemberRole requiredRole)
     {
-        var developer = await developerService.GetDeveloperByName(pubName);
+        var developer = await ds.GetDeveloperByName(pubName);
         if (developer == null) return (null, null, null);
 
-        if (!await developerService.IsMemberWithRole(developer.PublisherId, Guid.Parse(currentUser.Id), requiredRole))
+        if (!await ds.IsMemberWithRole(developer.PublisherId, Guid.Parse(currentUser.Id), requiredRole))
             return (null, null, null);
 
         var project = await projectService.GetProjectAsync(projectId, developer.Id);

DysonNetwork.Develop/Identity/BotAccountPublicController.cs

@@ -1,3 +1,4 @@
+using DysonNetwork.Shared.Models;
 using Microsoft.AspNetCore.Mvc;
 
 namespace DysonNetwork.Develop.Identity;
@@ -7,7 +8,7 @@ namespace DysonNetwork.Develop.Identity;
 public class BotAccountPublicController(BotAccountService botService, DeveloperService developerService) : ControllerBase
 {
     [HttpGet("{botId:guid}")]
-    public async Task<ActionResult<BotAccount>> GetBotTransparentInfo([FromRoute] Guid botId)
+    public async Task<ActionResult<SnBotAccount>> GetBotTransparentInfo([FromRoute] Guid botId)
     {
         var bot = await botService.GetBotByIdAsync(botId);
         if (bot is null) return NotFound("Bot not found");
@@ -21,7 +22,7 @@ public class BotAccountPublicController(BotAccountService botService, DeveloperS
     }
 
     [HttpGet("{botId:guid}/developer")]
-    public async Task<ActionResult<Developer>> GetBotDeveloper([FromRoute] Guid botId)
+    public async Task<ActionResult<SnDeveloper>> GetBotDeveloper([FromRoute] Guid botId)
     {
         var bot = await botService.GetBotByIdAsync(botId);
         if (bot is null) return NotFound("Bot not found");

DysonNetwork.Develop/Identity/BotAccountService.cs

@@ -1,5 +1,4 @@
-using DysonNetwork.Develop.Project;
-using DysonNetwork.Shared.Data;
+using DysonNetwork.Shared.Models;
 using DysonNetwork.Shared.Proto;
 using DysonNetwork.Shared.Registry;
 using Grpc.Core;
@@ -11,25 +10,25 @@ namespace DysonNetwork.Develop.Identity;
 public class BotAccountService(
     AppDatabase db,
     BotAccountReceiverService.BotAccountReceiverServiceClient accountReceiver,
-    AccountClientHelper accounts
+    RemoteAccountService remoteAccounts
 )
 {
-    public async Task<BotAccount?> GetBotByIdAsync(Guid id)
+    public async Task<SnBotAccount?> GetBotByIdAsync(Guid id)
     {
         return await db.BotAccounts
             .Include(b => b.Project)
             .FirstOrDefaultAsync(b => b.Id == id);
     }
 
-    public async Task<IEnumerable<BotAccount>> GetBotsByProjectAsync(Guid projectId)
+    public async Task<List<SnBotAccount>> GetBotsByProjectAsync(Guid projectId)
     {
         return await db.BotAccounts
             .Where(b => b.ProjectId == projectId)
             .ToListAsync();
     }
 
-    public async Task<BotAccount> CreateBotAsync(
-        DevProject project,
+    public async Task<SnBotAccount> CreateBotAsync(
+        SnDevProject project,
         string slug,
         Account account,
         string? pictureId,
@@ -58,7 +57,7 @@ public class BotAccountService(
             var botAccount = createResponse.Bot;
 
             // Then create the local bot account
-            var bot = new BotAccount
+            var bot = new SnBotAccount
             {
                 Id = automatedId,
                 Slug = slug,
@@ -89,8 +88,8 @@ public class BotAccountService(
         }
     }
 
-    public async Task<BotAccount> UpdateBotAsync(
-        BotAccount bot,
+    public async Task<SnBotAccount> UpdateBotAsync(
+        SnBotAccount bot,
         Account account,
         string? pictureId,
         string? backgroundId
@@ -130,7 +129,7 @@ public class BotAccountService(
         return bot;
     }
 
-    public async Task DeleteBotAsync(BotAccount bot)
+    public async Task DeleteBotAsync(SnBotAccount bot)
     {
         try
         {
@@ -153,22 +152,21 @@ public class BotAccountService(
         await db.SaveChangesAsync();
     }
 
-    public async Task<BotAccount?> LoadBotAccountAsync(BotAccount bot) =>
+    public async Task<SnBotAccount?> LoadBotAccountAsync(SnBotAccount bot) =>
         (await LoadBotsAccountAsync([bot])).FirstOrDefault();
 
-    public async Task<List<BotAccount>> LoadBotsAccountAsync(IEnumerable<BotAccount> bots)
+    public async Task<List<SnBotAccount>> LoadBotsAccountAsync(List<SnBotAccount> bots)
     {
-        bots = bots.ToList();
         var automatedIds = bots.Select(b => b.Id).ToList();
-        var data = await accounts.GetBotAccountBatch(automatedIds);
+        var data = await remoteAccounts.GetBotAccountBatch(automatedIds);
 
         foreach (var bot in bots)
         {
             bot.Account = data
-                .Select(AccountReference.FromProtoValue)
+                .Select(SnAccount.FromProtoValue)
                 .FirstOrDefault(e => e.AutomatedId == bot.Id);
         }
 
-        return bots as List<BotAccount> ?? [];
+        return bots;
     }
 }

DysonNetwork.Develop/Identity/CustomAppController.cs

@@ -1,5 +1,6 @@
 using System.ComponentModel.DataAnnotations;
 using DysonNetwork.Develop.Project;
+using DysonNetwork.Shared.Models;
 using DysonNetwork.Shared.Proto;
 using Microsoft.AspNetCore.Authorization;
 using Microsoft.AspNetCore.Mvc;
@@ -18,9 +19,9 @@ public class CustomAppController(CustomAppService customApps, DeveloperService d
         [MaxLength(4096)] string? Description,
         string? PictureId,
         string? BackgroundId,
-        CustomAppStatus? Status,
-        CustomAppLinks? Links,
-        CustomAppOauthConfig? OauthConfig
+        Shared.Models.CustomAppStatus? Status,
+        SnCustomAppLinks? Links,
+        SnCustomAppOauthConfig? OauthConfig
     );
 
     public record CreateSecretRequest(
@@ -50,7 +51,7 @@ public class CustomAppController(CustomAppService customApps, DeveloperService d
         if (developer is null) return NotFound();
 
         var accountId = Guid.Parse(currentUser.Id);
-        if (!await ds.IsMemberWithRole(developer.PublisherId, accountId, PublisherMemberRole.Viewer))
+        if (!await ds.IsMemberWithRole(developer.PublisherId, accountId, Shared.Proto.PublisherMemberRole.Viewer))
             return StatusCode(403, "You must be a viewer of the developer to list custom apps");
 
         var project = await projectService.GetProjectAsync(projectId, developer.Id);
@@ -72,7 +73,7 @@ public class CustomAppController(CustomAppService customApps, DeveloperService d
         if (developer is null) return NotFound();
         
         var accountId = Guid.Parse(currentUser.Id);
-        if (!await ds.IsMemberWithRole(developer.PublisherId, accountId, PublisherMemberRole.Viewer))
+        if (!await ds.IsMemberWithRole(developer.PublisherId, accountId, Shared.Proto.PublisherMemberRole.Viewer))
             return StatusCode(403, "You must be a viewer of the developer to list custom apps");
 
         var project = await projectService.GetProjectAsync(projectId, developer.Id);
@@ -99,7 +100,7 @@ public class CustomAppController(CustomAppService customApps, DeveloperService d
         if (developer is null)
             return NotFound("Developer not found");
 
-        if (!await ds.IsMemberWithRole(developer.PublisherId, Guid.Parse(currentUser.Id), PublisherMemberRole.Editor))
+        if (!await ds.IsMemberWithRole(developer.PublisherId, Guid.Parse(currentUser.Id), Shared.Proto.PublisherMemberRole.Editor))
             return StatusCode(403, "You must be an editor of the developer to create a custom app");
 
         var project = await projectService.GetProjectAsync(projectId, developer.Id);
@@ -143,7 +144,7 @@ public class CustomAppController(CustomAppService customApps, DeveloperService d
         if (developer is null)
             return NotFound("Developer not found");
 
-        if (!await ds.IsMemberWithRole(developer.PublisherId, Guid.Parse(currentUser.Id), PublisherMemberRole.Editor))
+        if (!await ds.IsMemberWithRole(developer.PublisherId, Guid.Parse(currentUser.Id), Shared.Proto.PublisherMemberRole.Editor))
             return StatusCode(403, "You must be an editor of the developer to update a custom app");
 
         var project = await projectService.GetProjectAsync(projectId, developer.Id);
@@ -180,7 +181,7 @@ public class CustomAppController(CustomAppService customApps, DeveloperService d
         if (developer is null)
             return NotFound("Developer not found");
 
-        if (!await ds.IsMemberWithRole(developer.PublisherId, Guid.Parse(currentUser.Id), PublisherMemberRole.Editor))
+        if (!await ds.IsMemberWithRole(developer.PublisherId, Guid.Parse(currentUser.Id), Shared.Proto.PublisherMemberRole.Editor))
             return StatusCode(403, "You must be an editor of the developer to delete a custom app");
 
         var project = await projectService.GetProjectAsync(projectId, developer.Id);
@@ -212,7 +213,7 @@ public class CustomAppController(CustomAppService customApps, DeveloperService d
         if (developer is null)
             return NotFound("Developer not found");
 
-        if (!await ds.IsMemberWithRole(developer.PublisherId, Guid.Parse(currentUser.Id), PublisherMemberRole.Editor))
+        if (!await ds.IsMemberWithRole(developer.PublisherId, Guid.Parse(currentUser.Id), Shared.Proto.PublisherMemberRole.Editor))
             return StatusCode(403, "You must be an editor of the developer to view app secrets");
 
         var project = await projectService.GetProjectAsync(projectId, developer.Id);
@@ -250,7 +251,7 @@ public class CustomAppController(CustomAppService customApps, DeveloperService d
         if (developer is null)
             return NotFound("Developer not found");
 
-        if (!await ds.IsMemberWithRole(developer.PublisherId, Guid.Parse(currentUser.Id), PublisherMemberRole.Editor))
+        if (!await ds.IsMemberWithRole(developer.PublisherId, Guid.Parse(currentUser.Id), Shared.Proto.PublisherMemberRole.Editor))
             return StatusCode(403, "You must be an editor of the developer to create app secrets");
 
         var project = await projectService.GetProjectAsync(projectId, developer.Id);
@@ -263,7 +264,7 @@ public class CustomAppController(CustomAppService customApps, DeveloperService d
 
         try
         {
-            var secret = await customApps.CreateAppSecretAsync(new CustomAppSecret
+            var secret = await customApps.CreateAppSecretAsync(new SnCustomAppSecret
             {
                 AppId = appId,
                 Description = request.Description,
@@ -309,7 +310,7 @@ public class CustomAppController(CustomAppService customApps, DeveloperService d
         if (developer is null)
             return NotFound("Developer not found");
 
-        if (!await ds.IsMemberWithRole(developer.PublisherId, Guid.Parse(currentUser.Id), PublisherMemberRole.Editor))
+        if (!await ds.IsMemberWithRole(developer.PublisherId, Guid.Parse(currentUser.Id), Shared.Proto.PublisherMemberRole.Editor))
             return StatusCode(403, "You must be an editor of the developer to view app secrets");
 
         var project = await projectService.GetProjectAsync(projectId, developer.Id);
@@ -350,7 +351,7 @@ public class CustomAppController(CustomAppService customApps, DeveloperService d
         if (developer is null)
             return NotFound("Developer not found");
 
-        if (!await ds.IsMemberWithRole(developer.PublisherId, Guid.Parse(currentUser.Id), PublisherMemberRole.Editor))
+        if (!await ds.IsMemberWithRole(developer.PublisherId, Guid.Parse(currentUser.Id), Shared.Proto.PublisherMemberRole.Editor))
             return StatusCode(403, "You must be an editor of the developer to delete app secrets");
 
         var project = await projectService.GetProjectAsync(projectId, developer.Id);
@@ -388,7 +389,7 @@ public class CustomAppController(CustomAppService customApps, DeveloperService d
         if (developer is null)
             return NotFound("Developer not found");
 
-        if (!await ds.IsMemberWithRole(developer.PublisherId, Guid.Parse(currentUser.Id), PublisherMemberRole.Editor))
+        if (!await ds.IsMemberWithRole(developer.PublisherId, Guid.Parse(currentUser.Id), Shared.Proto.PublisherMemberRole.Editor))
             return StatusCode(403, "You must be an editor of the developer to rotate app secrets");
 
         var project = await projectService.GetProjectAsync(projectId, developer.Id);
@@ -401,7 +402,7 @@ public class CustomAppController(CustomAppService customApps, DeveloperService d
 
         try
         {
-            var secret = await customApps.RotateAppSecretAsync(new CustomAppSecret
+            var secret = await customApps.RotateAppSecretAsync(new SnCustomAppSecret
             {
                 Id = secretId,
                 AppId = appId,

DysonNetwork.Develop/Identity/CustomAppService.cs

@@ -1,5 +1,4 @@
-using DysonNetwork.Develop.Project;
-using DysonNetwork.Shared.Data;
+using DysonNetwork.Shared.Models;
 using DysonNetwork.Shared.Proto;
 using Microsoft.EntityFrameworkCore;
 using System.Security.Cryptography;
@@ -13,7 +12,7 @@ public class CustomAppService(
     FileService.FileServiceClient files
 )
 {
-    public async Task<CustomApp?> CreateAppAsync(
+    public async Task<SnCustomApp?> CreateAppAsync(
         Guid projectId,
         CustomAppController.CustomAppRequest request
     )
@@ -25,12 +24,12 @@ public class CustomAppService(
         if (project == null)
             return null;
             
-        var app = new CustomApp
+        var app = new SnCustomApp
         {
             Slug = request.Slug!,
             Name = request.Name!,
             Description = request.Description,
-            Status = request.Status ?? CustomAppStatus.Developing,
+            Status = request.Status ?? Shared.Models.CustomAppStatus.Developing,
             Links = request.Links,
             OauthConfig = request.OauthConfig,
             ProjectId = projectId
@@ -46,7 +45,7 @@ public class CustomAppService(
             );
             if (picture is null)
                 throw new InvalidOperationException("Invalid picture id, unable to find the file on cloud.");
-            app.Picture = CloudFileReferenceObject.FromProtoValue(picture);
+            app.Picture = SnCloudFileReferenceObject.FromProtoValue(picture);
 
             // Create a new reference
             await fileRefs.CreateReferenceAsync(
@@ -65,7 +64,7 @@ public class CustomAppService(
             );
             if (background is null)
                 throw new InvalidOperationException("Invalid picture id, unable to find the file on cloud.");
-            app.Background = CloudFileReferenceObject.FromProtoValue(background);
+            app.Background = SnCloudFileReferenceObject.FromProtoValue(background);
 
             // Create a new reference
             await fileRefs.CreateReferenceAsync(
@@ -84,7 +83,7 @@ public class CustomAppService(
         return app;
     }
 
-    public async Task<CustomApp?> GetAppAsync(Guid id, Guid? projectId = null)
+    public async Task<SnCustomApp?> GetAppAsync(Guid id, Guid? projectId = null)
     {
         var query = db.CustomApps.AsQueryable();
         
@@ -96,7 +95,7 @@ public class CustomAppService(
         return await query.FirstOrDefaultAsync(a => a.Id == id);
     }
 
-    public async Task<List<CustomAppSecret>> GetAppSecretsAsync(Guid appId)
+    public async Task<List<SnCustomAppSecret>> GetAppSecretsAsync(Guid appId)
     {
         return await db.CustomAppSecrets
             .Where(s => s.AppId == appId)
@@ -104,13 +103,13 @@ public class CustomAppService(
             .ToListAsync();
     }
 
-    public async Task<CustomAppSecret?> GetAppSecretAsync(Guid secretId, Guid appId)
+    public async Task<SnCustomAppSecret?> GetAppSecretAsync(Guid secretId, Guid appId)
     {
         return await db.CustomAppSecrets
             .FirstOrDefaultAsync(s => s.Id == secretId && s.AppId == appId);
     }
 
-    public async Task<CustomAppSecret> CreateAppSecretAsync(CustomAppSecret secret)
+    public async Task<SnCustomAppSecret> CreateAppSecretAsync(SnCustomAppSecret secret)
     {
         if (string.IsNullOrWhiteSpace(secret.Secret))
         {
@@ -141,7 +140,7 @@ public class CustomAppService(
         return true;
     }
 
-    public async Task<CustomAppSecret> RotateAppSecretAsync(CustomAppSecret secretUpdate)
+    public async Task<SnCustomAppSecret> RotateAppSecretAsync(SnCustomAppSecret secretUpdate)
     {
         var existingSecret = await db.CustomAppSecrets
             .FirstOrDefaultAsync(s => s.Id == secretUpdate.Id && s.AppId == secretUpdate.AppId);
@@ -177,14 +176,14 @@ public class CustomAppService(
         return res.ToString();
     }
 
-    public async Task<List<CustomApp>> GetAppsByProjectAsync(Guid projectId)
+    public async Task<List<SnCustomApp>> GetAppsByProjectAsync(Guid projectId)
     {
         return await db.CustomApps
             .Where(a => a.ProjectId == projectId)
             .ToListAsync();
     }
 
-    public async Task<CustomApp?> UpdateAppAsync(CustomApp app, CustomAppController.CustomAppRequest request)
+    public async Task<SnCustomApp?> UpdateAppAsync(SnCustomApp app, CustomAppController.CustomAppRequest request)
     {
         if (request.Slug is not null)
             app.Slug = request.Slug;
@@ -209,7 +208,7 @@ public class CustomAppService(
             );
             if (picture is null)
                 throw new InvalidOperationException("Invalid picture id, unable to find the file on cloud.");
-            app.Picture = CloudFileReferenceObject.FromProtoValue(picture);
+            app.Picture = SnCloudFileReferenceObject.FromProtoValue(picture);
 
             // Create a new reference
             await fileRefs.CreateReferenceAsync(
@@ -228,7 +227,7 @@ public class CustomAppService(
             );
             if (background is null)
                 throw new InvalidOperationException("Invalid picture id, unable to find the file on cloud.");
-            app.Background = CloudFileReferenceObject.FromProtoValue(background);
+            app.Background = SnCloudFileReferenceObject.FromProtoValue(background);
 
             // Create a new reference
             await fileRefs.CreateReferenceAsync(

DysonNetwork.Develop/Identity/CustomAppServiceGrpc.cs

@@ -1,3 +1,4 @@
+using DysonNetwork.Shared.Models;
 using DysonNetwork.Shared.Proto;
 using Grpc.Core;
 using Microsoft.EntityFrameworkCore;
@@ -37,7 +38,7 @@ public class CustomAppServiceGrpc(AppDatabase db) : Shared.Proto.CustomAppServic
         if (string.IsNullOrEmpty(request.Secret))
             throw new RpcException(new Status(StatusCode.InvalidArgument, "secret required"));
 
-        IQueryable<CustomAppSecret> q = db.CustomAppSecrets;
+        IQueryable<SnCustomAppSecret> q = db.CustomAppSecrets;
         switch (request.SecretIdentifierCase)
         {
             case CheckCustomAppSecretRequest.SecretIdentifierOneofCase.SecretId:

DysonNetwork.Develop/Identity/Developer.cs

@@ -1,79 +0,0 @@
-using System.ComponentModel.DataAnnotations.Schema;
-using System.Text.Json.Serialization;
-using DysonNetwork.Develop.Project;
-using DysonNetwork.Shared.Proto;
-using DysonNetwork.Shared.Data;
-using VerificationMark = DysonNetwork.Shared.Data.VerificationMark;
-
-namespace DysonNetwork.Develop.Identity;
-
-public class Developer
-{
-    public Guid Id { get; set; } = Guid.NewGuid();
-    public Guid PublisherId { get; set; }
-    
-    [JsonIgnore] public List<DevProject> Projects { get; set; } = [];
-    
-    [NotMapped] public PublisherInfo? Publisher { get; set; }
-}
-
-public class PublisherInfo
-{
-    public Guid Id { get; set; }
-    public PublisherType Type { get; set; }
-    public string Name { get; set; } = string.Empty;
-    public string Nick { get; set; } = string.Empty;
-    public string? Bio { get; set; }
-
-    public CloudFileReferenceObject? Picture { get; set; }
-    public CloudFileReferenceObject? Background { get; set; }
-
-    public VerificationMark? Verification { get; set; }
-    public Guid? AccountId { get; set; }
-    public Guid? RealmId { get; set; }
-
-    public static PublisherInfo FromProto(Publisher proto)
-    {
-        var info = new PublisherInfo
-        {
-            Id = Guid.Parse(proto.Id),
-            Type = proto.Type == PublisherType.PubIndividual
-                ? PublisherType.PubIndividual
-                : PublisherType.PubOrganizational,
-            Name = proto.Name,
-            Nick = proto.Nick,
-            Bio = string.IsNullOrEmpty(proto.Bio) ? null : proto.Bio,
-            Verification = proto.VerificationMark is not null
-                ? VerificationMark.FromProtoValue(proto.VerificationMark)
-                : null,
-            AccountId = string.IsNullOrEmpty(proto.AccountId) ? null : Guid.Parse(proto.AccountId),
-            RealmId = string.IsNullOrEmpty(proto.RealmId) ? null : Guid.Parse(proto.RealmId)
-        };
-
-        if (proto.Picture != null)
-        {
-            info.Picture = new CloudFileReferenceObject
-            {
-                Id = proto.Picture.Id,
-                Name = proto.Picture.Name,
-                MimeType = proto.Picture.MimeType,
-                Hash = proto.Picture.Hash,
-                Size = proto.Picture.Size
-            };
-        }
-
-        if (proto.Background != null)
-        {
-            info.Background = new CloudFileReferenceObject
-            {
-                Id = proto.Background.Id,
-                Name = proto.Background.Name,
-                MimeType = proto.Background.MimeType,
-                Hash = proto.Background.Hash,
-                Size = (long)proto.Background.Size
-            };
-        }
-
-        return info;
-    }
-}

DysonNetwork.Develop/Identity/DeveloperController.cs

@@ -1,4 +1,5 @@
 using DysonNetwork.Shared.Auth;
+using DysonNetwork.Shared.Models;
 using DysonNetwork.Shared.Proto;
 using Grpc.Core;
 using Microsoft.AspNetCore.Authorization;
@@ -18,7 +19,7 @@ public class DeveloperController(
     : ControllerBase
 {
     [HttpGet("{name}")]
-    public async Task<ActionResult<Developer>> GetDeveloper(string name)
+    public async Task<ActionResult<SnDeveloper>> GetDeveloper(string name)
     {
         var developer = await ds.GetDeveloperByName(name);
         if (developer is null) return NotFound();
@@ -47,10 +48,9 @@ public class DeveloperController(
 
     [HttpGet]
     [Authorize]
-    public async Task<ActionResult<List<Developer>>> ListJoinedDevelopers()
+    public async Task<ActionResult<List<SnDeveloper>>> ListJoinedDevelopers()
     {
         if (HttpContext.Items["CurrentUser"] is not Account currentUser) return Unauthorized();
-        var accountId = Guid.Parse(currentUser.Id);
         
         var pubResponse = await ps.ListPublishersAsync(new ListPublishersRequest { AccountId = currentUser.Id });
         var pubIds = pubResponse.Publishers.Select(p => p.Id).Select(Guid.Parse).ToList();
@@ -69,17 +69,17 @@ public class DeveloperController(
 
     [HttpPost("{name}/enroll")]
     [Authorize]
-    [RequiredPermission("global", "developers.create")]
-    public async Task<ActionResult<Developer>> EnrollDeveloperProgram(string name)
+    [AskPermission("developers.create")]
+    public async Task<ActionResult<SnDeveloper>> EnrollDeveloperProgram(string name)
     {
         if (HttpContext.Items["CurrentUser"] is not Account currentUser) return Unauthorized();
         var accountId = Guid.Parse(currentUser.Id);
 
-        PublisherInfo? pub;
+        SnPublisher? pub;
         try
         {
             var pubResponse = await ps.GetPublisherAsync(new GetPublisherRequest { Name = name });
-            pub = PublisherInfo.FromProto(pubResponse.Publisher);
+            pub = SnPublisher.FromProtoValue(pubResponse.Publisher);
         } catch (RpcException ex)
         {
             return NotFound(ex.Status.Detail);
@@ -90,14 +90,14 @@ public class DeveloperController(
         {
             PublisherId = pub.Id.ToString(),
             AccountId = currentUser.Id,
-            Role = PublisherMemberRole.Owner
+            Role = Shared.Proto.PublisherMemberRole.Owner
         });
         if (!permResponse.Valid) return StatusCode(403, "You must be the owner of the publisher to join the developer program");
 
         var hasDeveloper = await db.Developers.AnyAsync(d => d.PublisherId == pub.Id);
         if (hasDeveloper) return BadRequest("Publisher is already in the developer program");
         
-        var developer = new Developer
+        var developer = new SnDeveloper
         {
             Id = Guid.NewGuid(),
             PublisherId = pub.Id

DysonNetwork.Develop/Identity/DeveloperService.cs

@@ -1,3 +1,4 @@
+using DysonNetwork.Shared.Models;
 using DysonNetwork.Shared.Proto;
 using Grpc.Core;
 using Microsoft.EntityFrameworkCore;
@@ -9,22 +10,22 @@ public class DeveloperService(
     PublisherService.PublisherServiceClient ps,
     ILogger<DeveloperService> logger)
 {
-    public async Task<Developer> LoadDeveloperPublisher(Developer developer)
+    public async Task<SnDeveloper> LoadDeveloperPublisher(SnDeveloper developer)
     {
         var pubResponse = await ps.GetPublisherAsync(new GetPublisherRequest { Id = developer.PublisherId.ToString() });
-        developer.Publisher = PublisherInfo.FromProto(pubResponse.Publisher);
+        developer.Publisher = SnPublisher.FromProtoValue(pubResponse.Publisher);
         return developer;
     }
 
 
-    public async Task<IEnumerable<Developer>> LoadDeveloperPublisher(IEnumerable<Developer> developers)
+    public async Task<IEnumerable<SnDeveloper>> LoadDeveloperPublisher(IEnumerable<SnDeveloper> developers)
     {
         var enumerable = developers.ToList();
         var pubIds = enumerable.Select(d => d.PublisherId).ToList();
         var pubRequest = new GetPublisherBatchRequest();
         pubIds.ForEach(x => pubRequest.Ids.Add(x.ToString()));
         var pubResponse = await ps.GetPublisherBatchAsync(pubRequest);
-        var pubs = pubResponse.Publishers.ToDictionary(p => Guid.Parse(p.Id), PublisherInfo.FromProto);
+        var pubs = pubResponse.Publishers.ToDictionary(p => Guid.Parse(p.Id), SnPublisher.FromProtoValue);
 
         return enumerable.Select(d =>
         {
@@ -33,7 +34,7 @@ public class DeveloperService(
         });
     }
 
-    public async Task<Developer?> GetDeveloperByName(string name)
+    public async Task<SnDeveloper?> GetDeveloperByName(string name)
     {
         try
         {
@@ -50,12 +51,12 @@ public class DeveloperService(
         }
     }
 
-    public async Task<Developer?> GetDeveloperById(Guid id)
+    public async Task<SnDeveloper?> GetDeveloperById(Guid id)
     {
         return await db.Developers.FirstOrDefaultAsync(d => d.Id == id);
     }
 
-    public async Task<bool> IsMemberWithRole(Guid pubId, Guid accountId, PublisherMemberRole role)
+    public async Task<bool> IsMemberWithRole(Guid pubId, Guid accountId, Shared.Proto.PublisherMemberRole role)
     {
         try
         {

DysonNetwork.Develop/Migrations/20250807133702_InitialMigration.Designer.cs

@@ -1,8 +1,7 @@
 // <auto-generated />
 using System;
 using DysonNetwork.Develop;
-using DysonNetwork.Develop.Identity;
-using DysonNetwork.Shared.Data;
+using DysonNetwork.Shared.Models;
 using Microsoft.EntityFrameworkCore;
 using Microsoft.EntityFrameworkCore.Infrastructure;
 using Microsoft.EntityFrameworkCore.Migrations;
@@ -35,7 +34,7 @@ namespace DysonNetwork.Develop.Migrations
                         .HasColumnType("uuid")
                         .HasColumnName("id");
 
-                    b.Property<CloudFileReferenceObject>("Background")
+                    b.Property<SnCloudFileReferenceObject>("Background")
                         .HasColumnType("jsonb")
                         .HasColumnName("background");
 
@@ -56,7 +55,7 @@ namespace DysonNetwork.Develop.Migrations
                         .HasColumnType("uuid")
                         .HasColumnName("developer_id");
 
-                    b.Property<CustomAppLinks>("Links")
+                    b.Property<SnCustomAppLinks>("Links")
                         .HasColumnType("jsonb")
                         .HasColumnName("links");
 
@@ -66,11 +65,11 @@ namespace DysonNetwork.Develop.Migrations
                         .HasColumnType("character varying(1024)")
                         .HasColumnName("name");
 
-                    b.Property<CustomAppOauthConfig>("OauthConfig")
+                    b.Property<SnCustomAppOauthConfig>("OauthConfig")
                         .HasColumnType("jsonb")
                         .HasColumnName("oauth_config");
 
-                    b.Property<CloudFileReferenceObject>("Picture")
+                    b.Property<SnCloudFileReferenceObject>("Picture")
                         .HasColumnType("jsonb")
                         .HasColumnName("picture");
 
@@ -88,7 +87,7 @@ namespace DysonNetwork.Develop.Migrations
                         .HasColumnType("timestamp with time zone")
                         .HasColumnName("updated_at");
 
-                    b.Property<VerificationMark>("Verification")
+                    b.Property<SnVerificationMark>("Verification")
                         .HasColumnType("jsonb")
                         .HasColumnName("verification");
 

DysonNetwork.Develop/Migrations/20250807133702_InitialMigration.cs

@@ -1,6 +1,4 @@
-using System;
-using DysonNetwork.Develop.Identity;
-using DysonNetwork.Shared.Data;
+using DysonNetwork.Shared.Models;
 using Microsoft.EntityFrameworkCore.Migrations;
 using NodaTime;
 
@@ -35,11 +33,11 @@ namespace DysonNetwork.Develop.Migrations
                     name = table.Column<string>(type: "character varying(1024)", maxLength: 1024, nullable: false),
                     description = table.Column<string>(type: "character varying(4096)", maxLength: 4096, nullable: true),
                     status = table.Column<int>(type: "integer", nullable: false),
-                    picture = table.Column<CloudFileReferenceObject>(type: "jsonb", nullable: true),
-                    background = table.Column<CloudFileReferenceObject>(type: "jsonb", nullable: true),
-                    verification = table.Column<VerificationMark>(type: "jsonb", nullable: true),
-                    oauth_config = table.Column<CustomAppOauthConfig>(type: "jsonb", nullable: true),
-                    links = table.Column<CustomAppLinks>(type: "jsonb", nullable: true),
+                    picture = table.Column<SnCloudFileReferenceObject>(type: "jsonb", nullable: true),
+                    background = table.Column<SnCloudFileReferenceObject>(type: "jsonb", nullable: true),
+                    verification = table.Column<SnVerificationMark>(type: "jsonb", nullable: true),
+                    oauth_config = table.Column<SnCustomAppOauthConfig>(type: "jsonb", nullable: true),
+                    links = table.Column<SnCustomAppLinks>(type: "jsonb", nullable: true),
                     developer_id = table.Column<Guid>(type: "uuid", nullable: false),
                     created_at = table.Column<Instant>(type: "timestamp with time zone", nullable: false),
                     updated_at = table.Column<Instant>(type: "timestamp with time zone", nullable: false),

DysonNetwork.Develop/Migrations/20250818124844_AddDevProject.Designer.cs

@@ -1,8 +1,7 @@
 // <auto-generated />
 using System;
 using DysonNetwork.Develop;
-using DysonNetwork.Develop.Identity;
-using DysonNetwork.Shared.Data;
+using DysonNetwork.Shared.Models;
 using Microsoft.EntityFrameworkCore;
 using Microsoft.EntityFrameworkCore.Infrastructure;
 using Microsoft.EntityFrameworkCore.Migrations;
@@ -35,7 +34,7 @@ namespace DysonNetwork.Develop.Migrations
                         .HasColumnType("uuid")
                         .HasColumnName("id");
 
-                    b.Property<CloudFileReferenceObject>("Background")
+                    b.Property<SnCloudFileReferenceObject>("Background")
                         .HasColumnType("jsonb")
                         .HasColumnName("background");
 
@@ -52,7 +51,7 @@ namespace DysonNetwork.Develop.Migrations
                         .HasColumnType("character varying(4096)")
                         .HasColumnName("description");
 
-                    b.Property<CustomAppLinks>("Links")
+                    b.Property<SnCustomAppLinks>("Links")
                         .HasColumnType("jsonb")
                         .HasColumnName("links");
 
@@ -62,11 +61,11 @@ namespace DysonNetwork.Develop.Migrations
                         .HasColumnType("character varying(1024)")
                         .HasColumnName("name");
 
-                    b.Property<CustomAppOauthConfig>("OauthConfig")
+                    b.Property<SnCustomAppOauthConfig>("OauthConfig")
                         .HasColumnType("jsonb")
                         .HasColumnName("oauth_config");
 
-                    b.Property<CloudFileReferenceObject>("Picture")
+                    b.Property<SnCloudFileReferenceObject>("Picture")
                         .HasColumnType("jsonb")
                         .HasColumnName("picture");
 
@@ -88,7 +87,7 @@ namespace DysonNetwork.Develop.Migrations
                         .HasColumnType("timestamp with time zone")
                         .HasColumnName("updated_at");
 
-                    b.Property<VerificationMark>("Verification")
+                    b.Property<SnVerificationMark>("Verification")
                         .HasColumnType("jsonb")
                         .HasColumnName("verification");
 

DysonNetwork.Develop/Migrations/20250818124844_AddDevProject.cs

@@ -1,5 +1,4 @@
-using System;
-using Microsoft.EntityFrameworkCore.Migrations;
+using Microsoft.EntityFrameworkCore.Migrations;
 using NodaTime;
 
 #nullable disable

DysonNetwork.Develop/Migrations/20250819163227_AddBotAccount.Designer.cs

@@ -1,8 +1,7 @@
 // <auto-generated />
 using System;
 using DysonNetwork.Develop;
-using DysonNetwork.Develop.Identity;
-using DysonNetwork.Shared.Data;
+using DysonNetwork.Shared.Models;
 using Microsoft.EntityFrameworkCore;
 using Microsoft.EntityFrameworkCore.Infrastructure;
 using Microsoft.EntityFrameworkCore.Migrations;
@@ -77,7 +76,7 @@ namespace DysonNetwork.Develop.Migrations
                         .HasColumnType("uuid")
                         .HasColumnName("id");
 
-                    b.Property<CloudFileReferenceObject>("Background")
+                    b.Property<SnCloudFileReferenceObject>("Background")
                         .HasColumnType("jsonb")
                         .HasColumnName("background");
 
@@ -94,7 +93,7 @@ namespace DysonNetwork.Develop.Migrations
                         .HasColumnType("character varying(4096)")
                         .HasColumnName("description");
 
-                    b.Property<CustomAppLinks>("Links")
+                    b.Property<SnCustomAppLinks>("Links")
                         .HasColumnType("jsonb")
                         .HasColumnName("links");
 
@@ -104,11 +103,11 @@ namespace DysonNetwork.Develop.Migrations
                         .HasColumnType("character varying(1024)")
                         .HasColumnName("name");
 
-                    b.Property<CustomAppOauthConfig>("OauthConfig")
+                    b.Property<SnCustomAppOauthConfig>("OauthConfig")
                         .HasColumnType("jsonb")
                         .HasColumnName("oauth_config");
 
-                    b.Property<CloudFileReferenceObject>("Picture")
+                    b.Property<SnCloudFileReferenceObject>("Picture")
                         .HasColumnType("jsonb")
                         .HasColumnName("picture");
 
@@ -130,7 +129,7 @@ namespace DysonNetwork.Develop.Migrations
                         .HasColumnType("timestamp with time zone")
                         .HasColumnName("updated_at");
 
-                    b.Property<VerificationMark>("Verification")
+                    b.Property<SnVerificationMark>("Verification")
                         .HasColumnType("jsonb")
                         .HasColumnName("verification");
 

DysonNetwork.Develop/Migrations/20250819163227_AddBotAccount.cs

@@ -1,5 +1,4 @@
-using System;
-using Microsoft.EntityFrameworkCore.Migrations;
+using Microsoft.EntityFrameworkCore.Migrations;
 using NodaTime;
 
 #nullable disable

DysonNetwork.Develop/Migrations/AppDatabaseModelSnapshot.cs

@@ -1,8 +1,7 @@
 // <auto-generated />
 using System;
 using DysonNetwork.Develop;
-using DysonNetwork.Develop.Identity;
-using DysonNetwork.Shared.Data;
+using DysonNetwork.Shared.Models;
 using Microsoft.EntityFrameworkCore;
 using Microsoft.EntityFrameworkCore.Infrastructure;
 using Microsoft.EntityFrameworkCore.Storage.ValueConversion;
@@ -74,7 +73,7 @@ namespace DysonNetwork.Develop.Migrations
                         .HasColumnType("uuid")
                         .HasColumnName("id");
 
-                    b.Property<CloudFileReferenceObject>("Background")
+                    b.Property<SnCloudFileReferenceObject>("Background")
                         .HasColumnType("jsonb")
                         .HasColumnName("background");
 
@@ -91,7 +90,7 @@ namespace DysonNetwork.Develop.Migrations
                         .HasColumnType("character varying(4096)")
                         .HasColumnName("description");
 
-                    b.Property<CustomAppLinks>("Links")
+                    b.Property<SnCustomAppLinks>("Links")
                         .HasColumnType("jsonb")
                         .HasColumnName("links");
 
@@ -101,11 +100,11 @@ namespace DysonNetwork.Develop.Migrations
                         .HasColumnType("character varying(1024)")
                         .HasColumnName("name");
 
-                    b.Property<CustomAppOauthConfig>("OauthConfig")
+                    b.Property<SnCustomAppOauthConfig>("OauthConfig")
                         .HasColumnType("jsonb")
                         .HasColumnName("oauth_config");
 
-                    b.Property<CloudFileReferenceObject>("Picture")
+                    b.Property<SnCloudFileReferenceObject>("Picture")
                         .HasColumnType("jsonb")
                         .HasColumnName("picture");
 
@@ -127,7 +126,7 @@ namespace DysonNetwork.Develop.Migrations
                         .HasColumnType("timestamp with time zone")
                         .HasColumnName("updated_at");
 
-                    b.Property<VerificationMark>("Verification")
+                    b.Property<SnVerificationMark>("Verification")
                         .HasColumnType("jsonb")
                         .HasColumnName("verification");
 

DysonNetwork.Develop/Program.cs

@@ -13,12 +13,16 @@ builder.ConfigureAppKestrel(builder.Configuration);
 
 builder.Services.AddAppServices(builder.Configuration);
 builder.Services.AddAppAuthentication();
-builder.Services.AddAppSwagger();
 builder.Services.AddDysonAuth();
-builder.Services.AddPublisherService();
+builder.Services.AddSphereService();
 builder.Services.AddAccountService();
 builder.Services.AddDriveService();
 
+builder.AddSwaggerManifest(
+    "DysonNetwork.Develop",
+    "The developer portal in the Solar Network."
+);
+
 var app = builder.Build();
 
 app.MapDefaultEndpoints();
@@ -31,4 +35,6 @@ using (var scope = app.Services.CreateScope())
 
 app.ConfigureAppMiddleware(builder.Configuration);
 
+app.UseSwaggerManifest("DysonNetwork.Develop");
+
 app.Run();

DysonNetwork.Develop/Project/DevProjectController.cs

@@ -8,7 +8,7 @@ namespace DysonNetwork.Develop.Project;
 
 [ApiController]
 [Route("/api/developers/{pubName}/projects")]
-public class DevProjectController(DevProjectService projectService, DeveloperService developerService) : ControllerBase
+public class DevProjectController(DevProjectService ps, DeveloperService ds) : ControllerBase
 {
     public record DevProjectRequest(
         [MaxLength(1024)] string? Slug,
@@ -19,20 +19,20 @@ public class DevProjectController(DevProjectService projectService, DeveloperSer
     [HttpGet]
     public async Task<IActionResult> ListProjects([FromRoute] string pubName)
     {
-        var developer = await developerService.GetDeveloperByName(pubName);
+        var developer = await ds.GetDeveloperByName(pubName);
         if (developer is null) return NotFound();
         
-        var projects = await projectService.GetProjectsByDeveloperAsync(developer.Id);
+        var projects = await ps.GetProjectsByDeveloperAsync(developer.Id);
         return Ok(projects);
     }
 
     [HttpGet("{id:guid}")]
     public async Task<IActionResult> GetProject([FromRoute] string pubName, Guid id)
     {
-        var developer = await developerService.GetDeveloperByName(pubName);
+        var developer = await ds.GetDeveloperByName(pubName);
         if (developer is null) return NotFound();
 
-        var project = await projectService.GetProjectAsync(id, developer.Id);
+        var project = await ps.GetProjectAsync(id, developer.Id);
         if (project is null) return NotFound();
 
         return Ok(project);
@@ -45,17 +45,17 @@ public class DevProjectController(DevProjectService projectService, DeveloperSer
         if (HttpContext.Items["CurrentUser"] is not Account currentUser) 
             return Unauthorized();
 
-        var developer = await developerService.GetDeveloperByName(pubName);
+        var developer = await ds.GetDeveloperByName(pubName);
         if (developer is null)
             return NotFound("Developer not found");
             
-        if (!await developerService.IsMemberWithRole(developer.PublisherId, Guid.Parse(currentUser.Id), PublisherMemberRole.Editor))
+        if (!await ds.IsMemberWithRole(developer.PublisherId, Guid.Parse(currentUser.Id), PublisherMemberRole.Editor))
             return StatusCode(403, "You must be an editor of the developer to create a project");
 
         if (string.IsNullOrWhiteSpace(request.Slug) || string.IsNullOrWhiteSpace(request.Name))
             return BadRequest("Slug and Name are required");
 
-        var project = await projectService.CreateProjectAsync(developer, request);
+        var project = await ps.CreateProjectAsync(developer, request);
         return CreatedAtAction(
             nameof(GetProject), 
             new { pubName, id = project.Id },
@@ -74,12 +74,15 @@ public class DevProjectController(DevProjectService projectService, DeveloperSer
         if (HttpContext.Items["CurrentUser"] is not Account currentUser) 
             return Unauthorized();
 
-        var developer = await developerService.GetDeveloperByName(pubName);
+        var developer = await ds.GetDeveloperByName(pubName);
         var accountId = Guid.Parse(currentUser.Id);
-        if (developer is null || developer.Id != accountId)
-            return Forbid();
         
-        var project = await projectService.UpdateProjectAsync(id, developer.Id, request);
+        if (developer is null)
+            return Forbid();
+        if (!await ds.IsMemberWithRole(developer.PublisherId, accountId, PublisherMemberRole.Manager))
+            return StatusCode(403, "You must be an manager of the developer to update a project");
+
+        var project = await ps.UpdateProjectAsync(id, developer.Id, request);
         if (project is null)
             return NotFound();
 
@@ -93,12 +96,14 @@ public class DevProjectController(DevProjectService projectService, DeveloperSer
         if (HttpContext.Items["CurrentUser"] is not Account currentUser) 
             return Unauthorized();
 
-        var developer = await developerService.GetDeveloperByName(pubName);
+        var developer = await ds.GetDeveloperByName(pubName);
         var accountId = Guid.Parse(currentUser.Id);
-        if (developer is null || developer.Id != accountId)
+        if (developer is null)
             return Forbid();
+        if (!await ds.IsMemberWithRole(developer.PublisherId, accountId, PublisherMemberRole.Manager))
+            return StatusCode(403, "You must be an manager of the developer to delete a project");
 
-        var success = await projectService.DeleteProjectAsync(id, developer.Id);
+        var success = await ps.DeleteProjectAsync(id, developer.Id);
         if (!success)
             return NotFound();
 

DysonNetwork.Develop/Project/DevProjectService.cs

@@ -1,21 +1,17 @@
-using DysonNetwork.Develop.Identity;
 using Microsoft.EntityFrameworkCore;
 using DysonNetwork.Shared.Proto;
+using DysonNetwork.Shared.Models;
 
 namespace DysonNetwork.Develop.Project;
 
-public class DevProjectService(
-    AppDatabase db,
-    FileReferenceService.FileReferenceServiceClient fileRefs,
-    FileService.FileServiceClient files
-)
+public class DevProjectService(AppDatabase db )
 {
-    public async Task<DevProject> CreateProjectAsync(
-        Developer developer,
+    public async Task<SnDevProject> CreateProjectAsync(
+        SnDeveloper developer,
         DevProjectController.DevProjectRequest request
     )
     {
-        var project = new DevProject
+        var project = new SnDevProject
         {
             Slug = request.Slug!,
             Name = request.Name!,
@@ -29,7 +25,7 @@ public class DevProjectService(
         return project;
     }
 
-    public async Task<DevProject?> GetProjectAsync(Guid id, Guid? developerId = null)
+    public async Task<SnDevProject?> GetProjectAsync(Guid id, Guid? developerId = null)
     {
         var query = db.DevProjects.AsQueryable();
 
@@ -41,14 +37,14 @@ public class DevProjectService(
         return await query.FirstOrDefaultAsync(p => p.Id == id);
     }
 
-    public async Task<List<DevProject>> GetProjectsByDeveloperAsync(Guid developerId)
+    public async Task<List<SnDevProject>> GetProjectsByDeveloperAsync(Guid developerId)
     {
         return await db.DevProjects
             .Where(p => p.DeveloperId == developerId)
             .ToListAsync();
     }
 
-    public async Task<DevProject?> UpdateProjectAsync(
+    public async Task<SnDevProject?> UpdateProjectAsync(
         Guid id,
         Guid developerId,
         DevProjectController.DevProjectRequest request

DysonNetwork.Develop/Properties/launchSettings.json

@@ -5,7 +5,6 @@
       "commandName": "Project",
       "dotnetRunMessages": true,
       "launchBrowser": false,
-      "applicationUrl": "http://localhost:5156",
       "environmentVariables": {
         "ASPNETCORE_ENVIRONMENT": "Development"
       }
@@ -14,7 +13,6 @@
       "commandName": "Project",
       "dotnetRunMessages": true,
       "launchBrowser": false,
-      "applicationUrl": "https://localhost:7192;http://localhost:5156",
       "environmentVariables": {
         "ASPNETCORE_ENVIRONMENT": "Development"
       }

DysonNetwork.Develop/Startup/ApplicationConfiguration.cs

@@ -1,9 +1,6 @@
-using System.Net;
 using DysonNetwork.Develop.Identity;
 using DysonNetwork.Shared.Auth;
 using DysonNetwork.Shared.Http;
-using Microsoft.AspNetCore.HttpOverrides;
-using Prometheus;
 
 namespace DysonNetwork.Develop.Startup;
 
@@ -11,23 +8,20 @@ public static class ApplicationConfiguration
 {
     public static WebApplication ConfigureAppMiddleware(this WebApplication app, IConfiguration configuration)
     {
-        app.MapMetrics();
         app.MapOpenApi();
 
-        app.UseSwagger();
-        app.UseSwaggerUI();
-        
         app.UseRequestLocalization();
 
         app.ConfigureForwardedHeaders(configuration);
 
         app.UseAuthentication();
         app.UseAuthorization();
-        app.UseMiddleware<PermissionMiddleware>();
+        app.UseMiddleware<RemotePermissionMiddleware>();
 
         app.MapControllers();
         
         app.MapGrpcService<CustomAppServiceGrpc>();
+        app.MapGrpcReflectionService();
 
         return app;
     }

DysonNetwork.Develop/Startup/ServiceCollectionExtensions.cs

@@ -1,5 +1,4 @@
 using System.Globalization;
-using Microsoft.OpenApi.Models;
 using NodaTime;
 using NodaTime.Serialization.SystemTextJson;
 using System.Text.Json;
@@ -7,7 +6,6 @@ using System.Text.Json.Serialization;
 using DysonNetwork.Develop.Identity;
 using DysonNetwork.Develop.Project;
 using DysonNetwork.Shared.Cache;
-using StackExchange.Redis;
 
 namespace DysonNetwork.Develop.Startup;
 
@@ -18,9 +16,7 @@ public static class ServiceCollectionExtensions
         services.AddLocalization();
 
         services.AddDbContext<AppDatabase>();
-        services.AddSingleton<IClock>(SystemClock.Instance);
         services.AddHttpContextAccessor();
-        services.AddSingleton<ICacheService, CacheServiceRedis>();
 
         services.AddHttpClient();
 
@@ -34,6 +30,7 @@ public static class ServiceCollectionExtensions
         });
 
         services.AddGrpc(options => { options.EnableDetailedErrors = true; });
+        services.AddGrpcReflection();
 
         services.Configure<RequestLocalizationOptions>(options =>
         {
@@ -57,23 +54,7 @@ public static class ServiceCollectionExtensions
 
     public static IServiceCollection AddAppAuthentication(this IServiceCollection services)
     {
-        services.AddCors();
         services.AddAuthorization();
         return services;
     }
-
-    public static IServiceCollection AddAppSwagger(this IServiceCollection services)
-    {
-        services.AddEndpointsApiExplorer();
-        services.AddSwaggerGen(options =>
-        {
-            options.SwaggerDoc("v1", new OpenApiInfo
-            {
-                Version = "v1",
-                Title = "Develop API",
-            });
-        });
-        services.AddOpenApi();
-        return services;
-    }
 }

DysonNetwork.Develop/appsettings.json

@@ -1,26 +1,25 @@
 {
-  "Debug": true,
-  "BaseUrl": "http://localhost:5071",
-  "SiteUrl": "https://solian.app",
-  "Logging": {
-    "LogLevel": {
-      "Default": "Information",
-      "Microsoft.AspNetCore": "Warning"
+    "Debug": true,
+    "BaseUrl": "http://localhost:5071",
+    "SiteUrl": "https://solian.app",
+    "Logging": {
+        "LogLevel": {
+            "Default": "Information",
+            "Microsoft.AspNetCore": "Warning"
+        }
+    },
+    "AllowedHosts": "*",
+    "ConnectionStrings": {
+        "App": "Host=localhost;Port=5432;Database=dyson_develop;Username=postgres;Password=postgres;Include Error Detail=True;Maximum Pool Size=20;Connection Idle Lifetime=60"
+    },
+    "KnownProxies": [
+        "127.0.0.1",
+        "::1"
+    ],
+    "Swagger": {
+        "PublicBasePath": "/develop"
+    },
+    "Cache": {
+        "Serializer": "MessagePack"
     }
-  },
-  "AllowedHosts": "*",
-  "ConnectionStrings": {
-    "App": "Host=localhost;Port=5432;Database=dyson_network_dev;Username=postgres;Password=postgres;Include Error Detail=True;Maximum Pool Size=20;Connection Idle Lifetime=60"
-  },
-  "KnownProxies": [
-    "127.0.0.1",
-    "::1"
-  ],
-  "Etcd": {
-    "Insecure": true
-  },
-  "Service": {
-    "Name": "DysonNetwork.Develop",
-    "Url": "https://localhost:7192"
-  }
 }

DysonNetwork.Drive/AppDatabase.cs

@@ -1,13 +1,14 @@
 using System.Linq.Expressions;
-using System.Reflection;
 using DysonNetwork.Drive.Billing;
 using DysonNetwork.Drive.Storage;
+using DysonNetwork.Drive.Storage.Model;
 using DysonNetwork.Shared.Data;
+using DysonNetwork.Shared.Models;
 using Microsoft.EntityFrameworkCore;
 using Microsoft.EntityFrameworkCore.Design;
-using Microsoft.EntityFrameworkCore.Query;
 using NodaTime;
 using Quartz;
+using TaskStatus = DysonNetwork.Drive.Storage.Model.TaskStatus;
 
 namespace DysonNetwork.Drive;
 
@@ -17,12 +18,16 @@ public class AppDatabase(
 ) : DbContext(options)
 {
     public DbSet<FilePool> Pools { get; set; } = null!;
-    public DbSet<FileBundle> Bundles { get; set; } = null!;
+    public DbSet<SnFileBundle> Bundles { get; set; } = null!;
     
     public DbSet<QuotaRecord> QuotaRecords { get; set; } = null!;
     
-    public DbSet<CloudFile> Files { get; set; } = null!;
-    public DbSet<CloudFileReference> FileReferences { get; set; } = null!;
+    public DbSet<SnCloudFile> Files { get; set; } = null!;
+    public DbSet<SnCloudFileReference> FileReferences { get; set; } = null!;
+    public DbSet<SnCloudFileIndex> FileIndexes { get; set; }
+
+    public DbSet<PersistentTask> Tasks { get; set; } = null!;
+    public DbSet<PersistentUploadTask> UploadTasks { get; set; } = null!; // Backward compatibility
     
     protected override void OnConfiguring(DbContextOptionsBuilder optionsBuilder)
     {
@@ -40,52 +45,12 @@ public class AppDatabase(
     protected override void OnModelCreating(ModelBuilder modelBuilder)
     {
         base.OnModelCreating(modelBuilder);
-
-        // Automatically apply soft-delete filter to all entities inheriting BaseModel
-        foreach (var entityType in modelBuilder.Model.GetEntityTypes())
-        {
-            if (!typeof(ModelBase).IsAssignableFrom(entityType.ClrType)) continue;
-            var method = typeof(AppDatabase)
-                .GetMethod(nameof(SetSoftDeleteFilter),
-                    BindingFlags.NonPublic | BindingFlags.Static)!
-                .MakeGenericMethod(entityType.ClrType);
-
-            method.Invoke(null, [modelBuilder]);
-        }
-    }
-
-    private static void SetSoftDeleteFilter<TEntity>(ModelBuilder modelBuilder)
-        where TEntity : ModelBase
-    {
-        modelBuilder.Entity<TEntity>().HasQueryFilter(e => e.DeletedAt == null);
+        modelBuilder.ApplySoftDeleteFilters();
     }
 
     public override async Task<int> SaveChangesAsync(CancellationToken cancellationToken = default)
     {
-        var now = SystemClock.Instance.GetCurrentInstant();
-
-        foreach (var entry in ChangeTracker.Entries<ModelBase>())
-        {
-            switch (entry.State)
-            {
-                case EntityState.Added:
-                    entry.Entity.CreatedAt = now;
-                    entry.Entity.UpdatedAt = now;
-                    break;
-                case EntityState.Modified:
-                    entry.Entity.UpdatedAt = now;
-                    break;
-                case EntityState.Deleted:
-                    entry.State = EntityState.Modified;
-                    entry.Entity.DeletedAt = now;
-                    break;
-                case EntityState.Detached:
-                case EntityState.Unchanged:
-                default:
-                    break;
-            }
-        }
-
+        this.ApplyAuditableAndSoftDelete();
         return await base.SaveChangesAsync(cancellationToken);
     }
 }
@@ -137,6 +102,45 @@ public class AppDatabaseRecyclingJob(AppDatabase db, ILogger<AppDatabaseRecyclin
     }
 }
 
+public class PersistentTaskCleanupJob(
+    IServiceProvider serviceProvider,
+    ILogger<PersistentTaskCleanupJob> logger
+) : IJob
+{
+    public async Task Execute(IJobExecutionContext context)
+    {
+        logger.LogInformation("Cleaning up stale persistent tasks...");
+
+        // Get the PersistentTaskService from DI
+        using var scope = serviceProvider.CreateScope();
+        var persistentTaskService = scope.ServiceProvider.GetService(typeof(PersistentTaskService));
+
+        if (persistentTaskService is PersistentTaskService service)
+        {
+            // Clean up tasks for all users (you might want to add user-specific logic here)
+            // For now, we'll clean up tasks older than 30 days for all users
+            var cutoff = SystemClock.Instance.GetCurrentInstant() - Duration.FromDays(30);
+            var tasksToClean = await service.GetUserTasksAsync(
+                Guid.Empty, // This would need to be adjusted for multi-user cleanup
+                status: TaskStatus.Completed | TaskStatus.Failed | TaskStatus.Cancelled | TaskStatus.Expired
+            );
+
+            var cleanedCount = 0;
+            foreach (var task in tasksToClean.Items.Where(t => t.UpdatedAt < cutoff))
+            {
+                await service.CancelTaskAsync(task.TaskId); // Or implement a proper cleanup method
+                cleanedCount++;
+            }
+
+            logger.LogInformation("Cleaned up {Count} stale persistent tasks", cleanedCount);
+        }
+        else
+        {
+            logger.LogWarning("PersistentTaskService not found in DI container");
+        }
+    }
+}
+
 public class AppDatabaseFactory : IDesignTimeDbContextFactory<AppDatabase>
 {
     public AppDatabase CreateDbContext(string[] args)
@@ -150,35 +154,3 @@ public class AppDatabaseFactory : IDesignTimeDbContextFactory<AppDatabase>
         return new AppDatabase(optionsBuilder.Options, configuration);
     }
 }
-
-public static class OptionalQueryExtensions
-{
-    public static IQueryable<T> If<T>(
-        this IQueryable<T> source,
-        bool condition,
-        Func<IQueryable<T>, IQueryable<T>> transform
-    )
-    {
-        return condition ? transform(source) : source;
-    }
-
-    public static IQueryable<T> If<T, TP>(
-        this IIncludableQueryable<T, TP> source,
-        bool condition,
-        Func<IIncludableQueryable<T, TP>, IQueryable<T>> transform
-    )
-        where T : class
-    {
-        return condition ? transform(source) : source;
-    }
-
-    public static IQueryable<T> If<T, TP>(
-        this IIncludableQueryable<T, IEnumerable<TP>> source,
-        bool condition,
-        Func<IIncludableQueryable<T, IEnumerable<TP>>, IQueryable<T>> transform
-    )
-        where T : class
-    {
-        return condition ? transform(source) : source;
-    }
-}

DysonNetwork.Drive/Billing/Quota.cs

@@ -1,4 +1,4 @@
-using DysonNetwork.Shared.Data;
+using DysonNetwork.Shared.Models;
 using NodaTime;
 
 namespace DysonNetwork.Drive.Billing;

DysonNetwork.Drive/Dockerfile

@@ -1,4 +1,4 @@
-FROM mcr.microsoft.com/dotnet/aspnet:9.0 AS base
+FROM mcr.microsoft.com/dotnet/aspnet:10.0 AS base
 WORKDIR /app
 EXPOSE 8080
 EXPOSE 8081
@@ -20,7 +20,7 @@ RUN apt-get update && apt-get install -y --no-install-recommends \
 USER $APP_UID
 
 # Stage 2: Build .NET application
-FROM mcr.microsoft.com/dotnet/sdk:9.0 AS build
+FROM mcr.microsoft.com/dotnet/sdk:10.0 AS build
 ARG BUILD_CONFIGURATION=Release
 WORKDIR /src
 COPY ["DysonNetwork.Drive/DysonNetwork.Drive.csproj", "DysonNetwork.Drive/"]

DysonNetwork.Drive/DysonNetwork.Drive.csproj

@@ -1,7 +1,7 @@
 <Project Sdk="Microsoft.NET.Sdk.Web">
 
     <PropertyGroup>
-        <TargetFramework>net9.0</TargetFramework>
+        <TargetFramework>net10.0</TargetFramework>
         <Nullable>enable</Nullable>
         <ImplicitUsings>enable</ImplicitUsings>
         <DockerDefaultTargetOS>Linux</DockerDefaultTargetOS>
@@ -10,53 +10,34 @@
     <ItemGroup>
         <PackageReference Include="BCrypt.Net-Next" Version="4.0.3" />
         <PackageReference Include="BlurHashSharp.SkiaSharp" Version="1.3.4" />
-        <PackageReference Include="FFMpegCore" Version="5.2.0" />
-        <PackageReference Include="Grpc.AspNetCore.Server" Version="2.71.0" />
-        <PackageReference Include="Microsoft.AspNetCore.OpenApi" Version="9.0.7" />
-        <PackageReference Include="Microsoft.EntityFrameworkCore.Design" Version="9.0.7">
-          <IncludeAssets>runtime; build; native; contentfiles; analyzers; buildtransitive</IncludeAssets>
+        <PackageReference Include="FFMpegCore" Version="5.4.0" />
+        <PackageReference Include="Grpc.AspNetCore.Server" Version="2.76.0" />
+        <PackageReference Include="Microsoft.AspNetCore.OpenApi" Version="10.0.1" />
+        <PackageReference Include="Microsoft.EntityFrameworkCore.Design" Version="10.0.1">
           <PrivateAssets>all</PrivateAssets>
+          <IncludeAssets>runtime; build; native; contentfiles; analyzers; buildtransitive</IncludeAssets>
         </PackageReference>
+        <PackageReference Include="MimeKit" Version="4.14.0" />
         <PackageReference Include="MimeTypes" Version="2.5.2">
           <PrivateAssets>all</PrivateAssets>
           <IncludeAssets>runtime; build; native; contentfiles; analyzers; buildtransitive</IncludeAssets>
         </PackageReference>
-        <PackageReference Include="Minio" Version="6.0.5" />
-        <PackageReference Include="Nerdbank.GitVersioning" Version="3.7.115">
-          <PrivateAssets>all</PrivateAssets>
-          <IncludeAssets>runtime; build; native; contentfiles; analyzers; buildtransitive</IncludeAssets>
-        </PackageReference>
+        <PackageReference Include="Minio" Version="7.0.0" />
+        <PackageReference Include="Nanoid" Version="3.1.0" />
         <PackageReference Include="NetVips" Version="3.1.0" />
-        <PackageReference Include="NetVips.Native.linux-x64" Version="8.17.1" />
-        <PackageReference Include="NetVips.Native.osx-arm64" Version="8.17.1" />
-        <PackageReference Include="NodaTime" Version="3.2.2" />
+        <PackageReference Include="NetVips.Native.linux-x64" Version="8.17.3" />
+        <PackageReference Include="NetVips.Native.osx-arm64" Version="8.17.3" />
+        <PackageReference Include="NodaTime" Version="3.2.3" />
         <PackageReference Include="NodaTime.Serialization.JsonNet" Version="3.2.0" />
         <PackageReference Include="NodaTime.Serialization.Protobuf" Version="2.0.2" />
         <PackageReference Include="NodaTime.Serialization.SystemTextJson" Version="1.3.0" />
-        <PackageReference Include="Npgsql.EntityFrameworkCore.PostgreSQL" Version="9.0.4" />
-        <PackageReference Include="Npgsql.EntityFrameworkCore.PostgreSQL.Design" Version="1.1.0" />
-        <PackageReference Include="Npgsql.EntityFrameworkCore.PostgreSQL.NodaTime" Version="9.0.4" />
-        <PackageReference Include="OpenTelemetry.Exporter.OpenTelemetryProtocol" Version="1.12.0" />
-        <PackageReference Include="OpenTelemetry.Extensions.Hosting" Version="1.12.0" />
-        <PackageReference Include="OpenTelemetry.Instrumentation.AspNetCore" Version="1.12.0" />
-        <PackageReference Include="OpenTelemetry.Instrumentation.Http" Version="1.12.0" />
-        <PackageReference Include="OpenTelemetry.Instrumentation.Runtime" Version="1.12.0" />
-        <PackageReference Include="prometheus-net.AspNetCore" Version="8.2.1" />
-        <PackageReference Include="prometheus-net.AspNetCore.HealthChecks" Version="8.2.1" />
-        <PackageReference Include="prometheus-net.DotNetRuntime" Version="4.4.1" />
-        <PackageReference Include="prometheus-net.EntityFramework" Version="0.9.5" />
-        <PackageReference Include="prometheus-net.SystemMetrics" Version="3.1.0" />
-        <PackageReference Include="Quartz" Version="3.14.0" />
-        <PackageReference Include="Quartz.AspNetCore" Version="3.14.0" />
-        <PackageReference Include="Quartz.Extensions.Hosting" Version="3.14.0" />
-        <PackageReference Include="EFCore.BulkExtensions" Version="9.0.1" />
-        <PackageReference Include="EFCore.BulkExtensions.PostgreSql" Version="9.0.1" />
-        <PackageReference Include="EFCore.NamingConventions" Version="9.0.0" />
+        <PackageReference Include="Quartz" Version="3.15.1" />
+        <PackageReference Include="Quartz.AspNetCore" Version="3.15.1" />
+        <PackageReference Include="Quartz.Extensions.Hosting" Version="3.15.1" />
+        <!-- Pin the SkiaSharp version at the 2.88.9 due to the BlurHash need this specific version -->
+        <PackageReference Include="SkiaSharp" Version="2.88.9" />
         <PackageReference Include="SkiaSharp.NativeAssets.Linux" Version="2.88.9" />
         <PackageReference Include="SkiaSharp.NativeAssets.Linux.NoDependencies" Version="2.88.9" />
-        <PackageReference Include="Swashbuckle.AspNetCore" Version="9.0.3" />
-        <PackageReference Include="Swashbuckle.AspNetCore.SwaggerUI" Version="9.0.3" />
-        <PackageReference Include="tusdotnet" Version="2.10.0" />
     </ItemGroup>
 
     <ItemGroup>
@@ -66,7 +47,6 @@
     </ItemGroup>
 
     <ItemGroup>
-      <ProjectReference Include="..\DysonNetwork.ServiceDefaults\DysonNetwork.ServiceDefaults.csproj" />
       <ProjectReference Include="..\DysonNetwork.Shared\DysonNetwork.Shared.csproj" />
     </ItemGroup>
 </Project>

DysonNetwork.Drive/Index/FileIndexController.cs

@@ -0,0 +1,585 @@
+using System.ComponentModel.DataAnnotations;
+using DysonNetwork.Drive.Storage;
+using DysonNetwork.Shared.Auth;
+using DysonNetwork.Shared.Http;
+using DysonNetwork.Shared.Models;
+using DysonNetwork.Shared.Proto;
+using Microsoft.AspNetCore.Authorization;
+using Microsoft.AspNetCore.Mvc;
+using Microsoft.EntityFrameworkCore;
+
+namespace DysonNetwork.Drive.Index;
+
+[ApiController]
+[Route("/api/index")]
+[Authorize]
+public class FileIndexController(
+    FileIndexService fileIndexService,
+    AppDatabase db,
+    ILogger<FileIndexController> logger
+) : ControllerBase
+{
+    /// <summary>
+    /// Gets files in a specific path for the current user
+    /// </summary>
+    /// <param name="path">The path to browse (defaults to root "/")</param>
+    /// <param name="query">Optional query to filter files by name</param>
+    /// <param name="order">The field to order by (date, size, name - defaults to date)</param>
+    /// <param name="orderDesc">Whether to order in descending order (defaults to true)</param>
+    /// <returns>List of files in the specified path</returns>
+    [HttpGet("browse")]
+    public async Task<IActionResult> BrowseFiles(
+        [FromQuery] string path = "/",
+        [FromQuery] string? query = null,
+        [FromQuery] string order = "date",
+        [FromQuery] bool orderDesc = true
+    )
+    {
+        if (HttpContext.Items["CurrentUser"] is not Account currentUser)
+            return new ObjectResult(ApiError.Unauthorized()) { StatusCode = 401 };
+
+        var accountId = Guid.Parse(currentUser.Id);
+
+        try
+        {
+            var fileIndexes = await fileIndexService.GetByPathAsync(accountId, path);
+
+            if (!string.IsNullOrWhiteSpace(query))
+            {
+                fileIndexes = fileIndexes
+                    .Where(fi => fi.File.Name.Contains(query, StringComparison.OrdinalIgnoreCase))
+                    .ToList();
+            }
+
+            // Apply sorting
+            fileIndexes = order.ToLower() switch
+            {
+                "name" => orderDesc ? fileIndexes.OrderByDescending(fi => fi.File.Name).ToList()
+                                   : fileIndexes.OrderBy(fi => fi.File.Name).ToList(),
+                "size" => orderDesc ? fileIndexes.OrderByDescending(fi => fi.File.Size).ToList()
+                                   : fileIndexes.OrderBy(fi => fi.File.Size).ToList(),
+                _ => orderDesc ? fileIndexes.OrderByDescending(fi => fi.File.CreatedAt).ToList()
+                              : fileIndexes.OrderBy(fi => fi.File.CreatedAt).ToList()
+            };
+
+            // Get all file indexes for this account to extract child folders
+            var allFileIndexes = await fileIndexService.GetByAccountIdAsync(accountId);
+
+            // Extract unique child folder paths
+            var childFolders = ExtractChildFolders(allFileIndexes, path);
+
+            return Ok(new
+            {
+                Path = path,
+                Files = fileIndexes,
+                Folders = childFolders,
+                TotalCount = fileIndexes.Count
+            });
+        }
+        catch (Exception ex)
+        {
+            logger.LogError(ex, "Failed to browse files for account {AccountId} at path {Path}", accountId, path);
+            return new ObjectResult(new ApiError
+            {
+                Code = "BROWSE_FAILED",
+                Message = "Failed to browse files",
+                Status = 500
+            }) { StatusCode = 500 };
+        }
+    }
+
+    /// <summary>
+    /// Extracts unique child folder paths from all file indexes for a given parent path
+    /// </summary>
+    /// <param name="allFileIndexes">All file indexes for the account</param>
+    /// <param name="parentPath">The parent path to find children for</param>
+    /// <returns>List of unique child folder names</returns>
+    private List<string> ExtractChildFolders(List<SnCloudFileIndex> allFileIndexes, string parentPath)
+    {
+        var normalizedParentPath = FileIndexService.NormalizePath(parentPath);
+        var childFolders = new HashSet<string>();
+
+        foreach (var index in allFileIndexes)
+        {
+            var normalizedIndexPath = FileIndexService.NormalizePath(index.Path);
+
+            // Check if this path is a direct child of the parent path
+            if (normalizedIndexPath.StartsWith(normalizedParentPath) &&
+                normalizedIndexPath != normalizedParentPath)
+            {
+                // Remove the parent path prefix to get the relative path
+                var relativePath = normalizedIndexPath.Substring(normalizedParentPath.Length);
+
+                // Extract the first folder name (direct child)
+                var firstSlashIndex = relativePath.IndexOf('/');
+                if (firstSlashIndex > 0)
+                {
+                    var folderName = relativePath.Substring(0, firstSlashIndex);
+                    childFolders.Add(folderName);
+                }
+            }
+        }
+
+        return childFolders.OrderBy(f => f).ToList();
+    }
+
+    /// <summary>
+    /// Gets all files for the current user (across all paths)
+    /// </summary>
+    /// <param name="query">Optional query to filter files by name</param>
+    /// <param name="order">The field to order by (date, size, name - defaults to date)</param>
+    /// <param name="orderDesc">Whether to order in descending order (defaults to true)</param>
+    /// <returns>List of all files for the user</returns>
+    [HttpGet("all")]
+    public async Task<IActionResult> GetAllFiles(
+        [FromQuery] string? query = null,
+        [FromQuery] string order = "date",
+        [FromQuery] bool orderDesc = true
+    )
+    {
+        if (HttpContext.Items["CurrentUser"] is not Account currentUser)
+            return new ObjectResult(ApiError.Unauthorized()) { StatusCode = 401 };
+
+        var accountId = Guid.Parse(currentUser.Id);
+
+        try
+        {
+            var fileIndexes = await fileIndexService.GetByAccountIdAsync(accountId);
+
+            if (!string.IsNullOrWhiteSpace(query))
+            {
+                fileIndexes = fileIndexes
+                    .Where(fi => fi.File.Name.Contains(query, StringComparison.OrdinalIgnoreCase))
+                    .ToList();
+            }
+
+            // Apply sorting
+            fileIndexes = order.ToLower() switch
+            {
+                "name" => orderDesc ? fileIndexes.OrderByDescending(fi => fi.File.Name).ToList()
+                                   : fileIndexes.OrderBy(fi => fi.File.Name).ToList(),
+                "size" => orderDesc ? fileIndexes.OrderByDescending(fi => fi.File.Size).ToList()
+                                   : fileIndexes.OrderBy(fi => fi.File.Size).ToList(),
+                _ => orderDesc ? fileIndexes.OrderByDescending(fi => fi.File.CreatedAt).ToList()
+                              : fileIndexes.OrderBy(fi => fi.File.CreatedAt).ToList()
+            };
+
+            return Ok(new
+            {
+                Files = fileIndexes,
+                TotalCount = fileIndexes.Count()
+            });
+        }
+        catch (Exception ex)
+        {
+            logger.LogError(ex, "Failed to get all files for account {AccountId}", accountId);
+            return new ObjectResult(new ApiError
+            {
+                Code = "GET_ALL_FAILED",
+                Message = "Failed to get files",
+                Status = 500
+            }) { StatusCode = 500 };
+        }
+    }
+
+    /// <summary>
+    /// Gets files that have not been indexed for the current user.
+    /// </summary>
+    /// <param name="recycled">Shows recycled files or not</param>
+    /// <param name="offset">The number of files to skip</param>
+    /// <param name="take">The number of files to return</param>
+    /// <param name="pool">The pool ID of those files</param>
+    /// <param name="query">Optional query to filter files by name</param>
+    /// <param name="order">The field to order by (date, size, name - defaults to date)</param>
+    /// <param name="orderDesc">Whether to order in descending order (defaults to true)</param>
+    /// <returns>List of unindexed files</returns>
+    [HttpGet("unindexed")]
+    public async Task<IActionResult> GetUnindexedFiles(
+        [FromQuery] Guid? pool,
+        [FromQuery] bool recycled = false,
+        [FromQuery] int offset = 0,
+        [FromQuery] int take = 20,
+        [FromQuery] string? query = null,
+        [FromQuery] string order = "date",
+        [FromQuery] bool orderDesc = true
+    )
+    {
+        if (HttpContext.Items["CurrentUser"] is not Account currentUser)
+            return new ObjectResult(ApiError.Unauthorized()) { StatusCode = 401 };
+
+        var accountId = Guid.Parse(currentUser.Id);
+
+        try
+        {
+            var filesQuery = db.Files
+                .Where(f => f.AccountId == accountId
+                            && f.IsMarkedRecycle == recycled
+                            && !db.FileIndexes.Any(fi => fi.FileId == f.Id && fi.AccountId == accountId)
+                )
+                .AsQueryable();
+
+            // Apply sorting
+            filesQuery = order.ToLower() switch
+            {
+                "name" => orderDesc ? filesQuery.OrderByDescending(f => f.Name)
+                                   : filesQuery.OrderBy(f => f.Name),
+                "size" => orderDesc ? filesQuery.OrderByDescending(f => f.Size)
+                                   : filesQuery.OrderBy(f => f.Size),
+                _ => orderDesc ? filesQuery.OrderByDescending(f => f.CreatedAt)
+                              : filesQuery.OrderBy(f => f.CreatedAt)
+            };
+
+            if (pool.HasValue) filesQuery = filesQuery.Where(f => f.PoolId == pool);
+
+            if (!string.IsNullOrWhiteSpace(query))
+            {
+                filesQuery = filesQuery.Where(f => f.Name.Contains(query));
+            }
+
+            var totalCount = await filesQuery.CountAsync();
+
+            Response.Headers.Append("X-Total", totalCount.ToString());
+
+            var unindexedFiles = await filesQuery
+                .Skip(offset)
+                .Take(take)
+                .ToListAsync();
+
+            return Ok(unindexedFiles);
+        }
+        catch (Exception ex)
+        {
+            logger.LogError(ex, "Failed to get unindexed files for account {AccountId}", accountId);
+            return new ObjectResult(new ApiError
+            {
+                Code = "GET_UNINDEXED_FAILED",
+                Message = "Failed to get unindexed files",
+                Status = 500
+            }) { StatusCode = 500 };
+        }
+    }
+
+    /// <summary>
+    /// Moves a file to a new path
+    /// </summary>
+    /// <param name="indexId">The file index ID</param>
+    /// <param name="newPath">The new path</param>
+    /// <returns>The updated file index</returns>
+    [HttpPost("move/{indexId}")]
+    public async Task<IActionResult> MoveFile(Guid indexId, [FromBody] MoveFileRequest request)
+    {
+        if (HttpContext.Items["CurrentUser"] is not Account currentUser)
+            return new ObjectResult(ApiError.Unauthorized()) { StatusCode = 401 };
+
+        var accountId = Guid.Parse(currentUser.Id);
+
+        try
+        {
+            // Verify ownership
+            var existingIndex = await db.FileIndexes
+                .Include(fi => fi.File)
+                .FirstOrDefaultAsync(fi => fi.Id == indexId && fi.AccountId == accountId);
+
+            if (existingIndex == null)
+                return new ObjectResult(ApiError.NotFound("File index")) { StatusCode = 404 };
+
+            var updatedIndex = await fileIndexService.UpdateAsync(indexId, request.NewPath);
+
+            if (updatedIndex == null)
+                return new ObjectResult(ApiError.NotFound("File index")) { StatusCode = 404 };
+
+            return Ok(new
+            {
+                updatedIndex.FileId,
+                IndexId = updatedIndex.Id,
+                OldPath = existingIndex.Path,
+                NewPath = updatedIndex.Path,
+                Message = "File moved successfully"
+            });
+        }
+        catch (Exception ex)
+        {
+            logger.LogError(ex, "Failed to move file index {IndexId} for account {AccountId}", indexId, accountId);
+            return new ObjectResult(new ApiError
+            {
+                Code = "MOVE_FAILED",
+                Message = "Failed to move file",
+                Status = 500
+            }) { StatusCode = 500 };
+        }
+    }
+
+    /// <summary>
+    /// Removes a file index (does not delete the actual file by default)
+    /// </summary>
+    /// <param name="indexId">The file index ID</param>
+    /// <param name="deleteFile">Whether to also delete the actual file data</param>
+    /// <returns>Success message</returns>
+    [HttpDelete("remove/{indexId}")]
+    public async Task<IActionResult> RemoveFileIndex(Guid indexId, [FromQuery] bool deleteFile = false)
+    {
+        if (HttpContext.Items["CurrentUser"] is not Account currentUser)
+            return new ObjectResult(ApiError.Unauthorized()) { StatusCode = 401 };
+
+        var accountId = Guid.Parse(currentUser.Id);
+
+        try
+        {
+            // Verify ownership
+            var existingIndex = await db.FileIndexes
+                .Include(fi => fi.File)
+                .FirstOrDefaultAsync(fi => fi.Id == indexId && fi.AccountId == accountId);
+
+            if (existingIndex == null)
+                return new ObjectResult(ApiError.NotFound("File index")) { StatusCode = 404 };
+
+            var fileId = existingIndex.FileId;
+            var fileName = existingIndex.File.Name;
+            var filePath = existingIndex.Path;
+
+            // Remove the index
+            var removed = await fileIndexService.RemoveAsync(indexId);
+
+            if (!removed)
+                return new ObjectResult(ApiError.NotFound("File index")) { StatusCode = 404 };
+
+            // Optionally delete the actual file
+            if (!deleteFile)
+                return Ok(new
+                {
+                    Message = deleteFile
+                        ? "File index and file data removed successfully"
+                        : "File index removed successfully",
+                    FileId = fileId,
+                    FileName = fileName,
+                    Path = filePath,
+                    FileDataDeleted = deleteFile
+                });
+            try
+            {
+                // Check if there are any other indexes for this file
+                var remainingIndexes = await fileIndexService.GetByFileIdAsync(fileId);
+                if (remainingIndexes.Count == 0)
+                {
+                    // No other indexes exist, safe to delete the file
+                    var file = await db.Files.FirstOrDefaultAsync(f => f.Id == fileId.ToString());
+                    if (file != null)
+                    {
+                        db.Files.Remove(file);
+                        await db.SaveChangesAsync();
+                        logger.LogInformation("Deleted file {FileId} ({FileName}) as requested", fileId, fileName);
+                    }
+                }
+            }
+            catch (Exception ex)
+            {
+                logger.LogWarning(ex, "Failed to delete file {FileId} while removing index", fileId);
+                // Continue even if file deletion fails
+            }
+
+            return Ok(new
+            {
+                Message = deleteFile
+                    ? "File index and file data removed successfully"
+                    : "File index removed successfully",
+                FileId = fileId,
+                FileName = fileName,
+                Path = filePath,
+                FileDataDeleted = deleteFile
+            });
+        }
+        catch (Exception ex)
+        {
+            logger.LogError(ex, "Failed to remove file index {IndexId} for account {AccountId}", indexId, accountId);
+            return new ObjectResult(new ApiError
+            {
+                Code = "REMOVE_FAILED",
+                Message = "Failed to remove file",
+                Status = 500
+            }) { StatusCode = 500 };
+        }
+    }
+
+    /// <summary>
+    /// Removes all file indexes in a specific path
+    /// </summary>
+    /// <param name="path">The path to clear</param>
+    /// <param name="deleteFiles">Whether to also delete the actual file data</param>
+    /// <returns>Success message with count of removed items</returns>
+    [HttpDelete("clear-path")]
+    public async Task<IActionResult> ClearPath([FromQuery] string path = "/", [FromQuery] bool deleteFiles = false)
+    {
+        if (HttpContext.Items["CurrentUser"] is not Account currentUser)
+            return new ObjectResult(ApiError.Unauthorized()) { StatusCode = 401 };
+
+        var accountId = Guid.Parse(currentUser.Id);
+
+        try
+        {
+            var removedCount = await fileIndexService.RemoveByPathAsync(accountId, path);
+
+            if (!deleteFiles || removedCount <= 0)
+                return Ok(new
+                {
+                    Message = deleteFiles
+                        ? $"Cleared {removedCount} file indexes from path and deleted orphaned files"
+                        : $"Cleared {removedCount} file indexes from path",
+                    Path = path,
+                    RemovedCount = removedCount,
+                    FilesDeleted = deleteFiles
+                });
+            // Get the files that were in this path and check if they have other indexes
+            var filesInPath = await fileIndexService.GetByPathAsync(accountId, path);
+            var fileIdsToCheck = filesInPath.Select(fi => fi.FileId).Distinct().ToList();
+
+            foreach (var fileId in fileIdsToCheck)
+            {
+                var remainingIndexes = await fileIndexService.GetByFileIdAsync(fileId);
+                if (remainingIndexes.Count != 0) continue;
+                // No other indexes exist, safe to delete the file
+                var file = await db.Files.FirstOrDefaultAsync(f => f.Id == fileId.ToString());
+                if (file == null) continue;
+                db.Files.Remove(file);
+                logger.LogInformation("Deleted orphaned file {FileId} after clearing path {Path}", fileId, path);
+            }
+
+            await db.SaveChangesAsync();
+
+            return Ok(new
+            {
+                Message = deleteFiles
+                    ? $"Cleared {removedCount} file indexes from path and deleted orphaned files"
+                    : $"Cleared {removedCount} file indexes from path",
+                Path = path,
+                RemovedCount = removedCount,
+                FilesDeleted = deleteFiles
+            });
+        }
+        catch (Exception ex)
+        {
+            logger.LogError(ex, "Failed to clear path {Path} for account {AccountId}", path, accountId);
+            return new ObjectResult(new ApiError
+            {
+                Code = "CLEAR_PATH_FAILED",
+                Message = "Failed to clear path",
+                Status = 500
+            }) { StatusCode = 500 };
+        }
+    }
+
+    /// <summary>
+    /// Creates a new file index (useful for adding existing files to a path)
+    /// </summary>
+    /// <param name="request">The create index request</param>
+    /// <returns>The created file index</returns>
+    [HttpPost("create")]
+    public async Task<IActionResult> CreateFileIndex([FromBody] CreateFileIndexRequest request)
+    {
+        if (HttpContext.Items["CurrentUser"] is not Account currentUser)
+            return new ObjectResult(ApiError.Unauthorized()) { StatusCode = 401 };
+
+        var accountId = Guid.Parse(currentUser.Id);
+
+        try
+        {
+            // Verify the file exists and belongs to the user
+            var file = await db.Files.FirstOrDefaultAsync(f => f.Id == request.FileId);
+            if (file == null)
+                return new ObjectResult(ApiError.NotFound("File")) { StatusCode = 404 };
+
+            if (file.AccountId != accountId)
+                return new ObjectResult(ApiError.Unauthorized(forbidden: true)) { StatusCode = 403 };
+
+            // Check if index already exists for this file and path
+            var existingIndex = await db.FileIndexes
+                .FirstOrDefaultAsync(fi =>
+                    fi.FileId == request.FileId && fi.Path == request.Path && fi.AccountId == accountId);
+
+            if (existingIndex != null)
+                return new ObjectResult(ApiError.Validation(new Dictionary<string, string[]>
+                {
+                    { "fileId", ["File index already exists for this path"] }
+                })) { StatusCode = 400 };
+
+            var fileIndex = await fileIndexService.CreateAsync(request.Path, request.FileId, accountId);
+
+            return Ok(new
+            {
+                IndexId = fileIndex.Id,
+                fileIndex.FileId,
+                fileIndex.Path,
+                Message = "File index created successfully"
+            });
+        }
+        catch (Exception ex)
+        {
+            logger.LogError(ex, "Failed to create file index for file {FileId} at path {Path} for account {AccountId}",
+                request.FileId, request.Path, accountId);
+            return new ObjectResult(new ApiError
+            {
+                Code = "CREATE_INDEX_FAILED",
+                Message = "Failed to create file index",
+                Status = 500
+            }) { StatusCode = 500 };
+        }
+    }
+
+    /// <summary>
+    /// Searches for files by name or metadata
+    /// </summary>
+    /// <param name="query">The search query</param>
+    /// <param name="path">Optional path to limit search to</param>
+    /// <returns>Matching files</returns>
+    [HttpGet("search")]
+    public async Task<IActionResult> SearchFiles([FromQuery] string query, [FromQuery] string? path = null)
+    {
+        if (HttpContext.Items["CurrentUser"] is not Account currentUser)
+            return new ObjectResult(ApiError.Unauthorized()) { StatusCode = 401 };
+
+        var accountId = Guid.Parse(currentUser.Id);
+
+        try
+        {
+            // Build the query with all conditions at once
+            var searchTerm = query.ToLower();
+            var fileIndexes = await db.FileIndexes
+                .Where(fi => fi.AccountId == accountId)
+                .Include(fi => fi.File)
+                .Where(fi =>
+                    (string.IsNullOrEmpty(path) || fi.Path == FileIndexService.NormalizePath(path)) &&
+                    (fi.File.Name.ToLower().Contains(searchTerm) ||
+                     (fi.File.Description != null && fi.File.Description.ToLower().Contains(searchTerm)) ||
+                     (fi.File.MimeType != null && fi.File.MimeType.ToLower().Contains(searchTerm))))
+                .ToListAsync();
+
+            return Ok(new
+            {
+                Query = query,
+                Path = path,
+                Results = fileIndexes,
+                TotalCount = fileIndexes.Count()
+            });
+        }
+        catch (Exception ex)
+        {
+            logger.LogError(ex, "Failed to search files for account {AccountId} with query {Query}", accountId, query);
+            return new ObjectResult(new ApiError
+            {
+                Code = "SEARCH_FAILED",
+                Message = "Failed to search files",
+                Status = 500
+            }) { StatusCode = 500 };
+        }
+    }
+}
+
+public class MoveFileRequest
+{
+    public string NewPath { get; set; } = null!;
+}
+
+public class CreateFileIndexRequest
+{
+    [MaxLength(32)] public string FileId { get; set; } = null!;
+    public string Path { get; set; } = null!;
+}

DysonNetwork.Drive/Index/FileIndexService.cs

@@ -0,0 +1,197 @@
+using DysonNetwork.Shared.Models;
+using Microsoft.EntityFrameworkCore;
+
+namespace DysonNetwork.Drive.Index;
+
+public class FileIndexService(AppDatabase db)
+{
+    /// <summary>
+    /// Creates a new file index entry
+    /// </summary>
+    /// <param name="path">The parent folder path with a trailing slash</param>
+    /// <param name="fileId">The file ID</param>
+    /// <param name="accountId">The account ID</param>
+    /// <returns>The created file index</returns>
+    public async Task<SnCloudFileIndex> CreateAsync(string path, string fileId, Guid accountId)
+    {
+        // Ensure a path has a trailing slash and is query-safe
+        var normalizedPath = NormalizePath(path);
+
+        // Check if a file with the same name already exists in the same path for this account
+        var existingFileIndex = await db.FileIndexes
+            .FirstOrDefaultAsync(fi => fi.AccountId == accountId && fi.Path == normalizedPath && fi.FileId == fileId);
+
+        if (existingFileIndex != null)
+        {
+            throw new InvalidOperationException(
+                $"A file with ID '{fileId}' already exists in path '{normalizedPath}' for account '{accountId}'");
+        }
+
+        var fileIndex = new SnCloudFileIndex
+        {
+            Path = normalizedPath,
+            FileId = fileId,
+            AccountId = accountId
+        };
+
+        db.FileIndexes.Add(fileIndex);
+        await db.SaveChangesAsync();
+
+        return fileIndex;
+    }
+
+    /// <summary>
+    /// Updates an existing file index entry by removing the old one and creating a new one
+    /// </summary>
+    /// <param name="id">The file index ID</param>
+    /// <param name="newPath">The new parent folder path with trailing slash</param>
+    /// <returns>The updated file index</returns>
+    public async Task<SnCloudFileIndex?> UpdateAsync(Guid id, string newPath)
+    {
+        var fileIndex = await db.FileIndexes.FindAsync(id);
+        if (fileIndex == null)
+            return null;
+
+        // Since properties are init-only, we need to remove the old index and create a new one
+        db.FileIndexes.Remove(fileIndex);
+
+        var newFileIndex = new SnCloudFileIndex
+        {
+            Path = NormalizePath(newPath),
+            FileId = fileIndex.FileId,
+            AccountId = fileIndex.AccountId
+        };
+
+        db.FileIndexes.Add(newFileIndex);
+        await db.SaveChangesAsync();
+
+        return newFileIndex;
+    }
+
+    /// <summary>
+    /// Removes a file index entry by ID
+    /// </summary>
+    /// <param name="id">The file index ID</param>
+    /// <returns>True if the index was found and removed, false otherwise</returns>
+    public async Task<bool> RemoveAsync(Guid id)
+    {
+        var fileIndex = await db.FileIndexes.FindAsync(id);
+        if (fileIndex == null)
+            return false;
+
+        db.FileIndexes.Remove(fileIndex);
+        await db.SaveChangesAsync();
+
+        return true;
+    }
+
+    /// <summary>
+    /// Removes file index entries by file ID
+    /// </summary>
+    /// <param name="fileId">The file ID</param>
+    /// <returns>The number of indexes removed</returns>
+    public async Task<int> RemoveByFileIdAsync(string fileId)
+    {
+        var indexes = await db.FileIndexes
+            .Where(fi => fi.FileId == fileId)
+            .ToListAsync();
+
+        if (indexes.Count == 0)
+            return 0;
+
+        db.FileIndexes.RemoveRange(indexes);
+        await db.SaveChangesAsync();
+
+        return indexes.Count;
+    }
+
+    /// <summary>
+    /// Removes file index entries by account ID and path
+    /// </summary>
+    /// <param name="accountId">The account ID</param>
+    /// <param name="path">The parent folder path</param>
+    /// <returns>The number of indexes removed</returns>
+    public async Task<int> RemoveByPathAsync(Guid accountId, string path)
+    {
+        var normalizedPath = NormalizePath(path);
+
+        var indexes = await db.FileIndexes
+            .Where(fi => fi.AccountId == accountId && fi.Path == normalizedPath)
+            .ToListAsync();
+
+        if (!indexes.Any())
+            return 0;
+
+        db.FileIndexes.RemoveRange(indexes);
+        await db.SaveChangesAsync();
+
+        return indexes.Count;
+    }
+
+    /// <summary>
+    /// Gets file indexes by account ID and path
+    /// </summary>
+    /// <param name="accountId">The account ID</param>
+    /// <param name="path">The parent folder path</param>
+    /// <returns>List of file indexes</returns>
+    public async Task<List<SnCloudFileIndex>> GetByPathAsync(Guid accountId, string path)
+    {
+        var normalizedPath = NormalizePath(path);
+
+        return await db.FileIndexes
+            .Where(fi => fi.AccountId == accountId && fi.Path == normalizedPath)
+            .Include(fi => fi.File)
+            .ToListAsync();
+    }
+
+    /// <summary>
+    /// Gets file indexes by file ID
+    /// </summary>
+    /// <param name="fileId">The file ID</param>
+    /// <returns>List of file indexes</returns>
+    public async Task<List<SnCloudFileIndex>> GetByFileIdAsync(string fileId)
+    {
+        return await db.FileIndexes
+            .Where(fi => fi.FileId == fileId)
+            .Include(fi => fi.File)
+            .ToListAsync();
+    }
+
+    /// <summary>
+    /// Gets all file indexes for an account
+    /// </summary>
+    /// <param name="accountId">The account ID</param>
+    /// <returns>List of file indexes</returns>
+    public async Task<List<SnCloudFileIndex>> GetByAccountIdAsync(Guid accountId)
+    {
+        return await db.FileIndexes
+            .Where(fi => fi.AccountId == accountId)
+            .Include(fi => fi.File)
+            .ToListAsync();
+    }
+
+    /// <summary>
+    /// Normalizes the path to ensure it has a trailing slash and is query-safe
+    /// </summary>
+    /// <param name="path">The original path</param>
+    /// <returns>The normalized path</returns>
+    public static string NormalizePath(string path)
+    {
+        if (string.IsNullOrEmpty(path))
+            return "/";
+
+        // Ensure the path starts with a slash
+        if (!path.StartsWith('/'))
+            path = "/" + path;
+
+        // Ensure the path ends with a slash (unless it's just the root)
+        if (path != "/" && !path.EndsWith('/'))
+            path += "/";
+
+        // Make path query-safe by removing problematic characters
+        // This is a basic implementation - you might want to add more robust validation
+        path = path.Replace("%", "").Replace("'", "").Replace("\"", "");
+
+        return path;
+    }
+}

DysonNetwork.Drive/Index/README.md

@@ -0,0 +1,341 @@
+# File Indexing System Documentation
+
+## Overview
+
+The File Indexing System provides a hierarchical file organization layer on top of the existing file storage system in DysonNetwork Drive. It allows users to organize their files in folders and paths while maintaining the underlying file storage capabilities.
+
+When using with the gateway, replace the `/api` with the `/drive` in the path.
+And all the arguments will be transformed into snake case via the gateway.
+
+## Architecture
+
+### Core Components
+
+1. **SnCloudFileIndex Model** - Represents the file-to-path mapping
+2. **FileIndexService** - Business logic for file index operations
+3. **FileIndexController** - REST API endpoints for file management
+4. **FileUploadController Integration** - Automatic index creation during upload
+
+### Database Schema
+
+```sql
+-- File Indexes table
+CREATE TABLE "FileIndexes" (
+    "Id" uuid NOT NULL DEFAULT gen_random_uuid(),
+    "Path" character varying(8192) NOT NULL,
+    "FileId" uuid NOT NULL,
+    "AccountId" uuid NOT NULL,
+    "CreatedAt" timestamp with time zone NOT NULL DEFAULT (now() at time zone 'utc'),
+    "UpdatedAt" timestamp with time zone NOT NULL DEFAULT (now() at time zone 'utc'),
+    CONSTRAINT "PK_FileIndexes" PRIMARY KEY ("Id"),
+    CONSTRAINT "FK_FileIndexes_Files_FileId" FOREIGN KEY ("FileId") REFERENCES "Files" ("Id") ON DELETE CASCADE,
+    INDEX "IX_FileIndexes_Path_AccountId" ("Path", "AccountId")
+);
+```
+
+## API Endpoints
+
+### Browse Files
+**GET** `/api/index/browse?path=/documents/`
+
+Browse files in a specific path.
+
+**Query Parameters:**
+- `path` (optional, default: "/") - The path to browse
+
+**Response:**
+```json
+{
+  "path": "/documents/",
+  "files": [
+    {
+      "id": "guid",
+      "path": "/documents/",
+      "fileId": "guid",
+      "accountId": "guid",
+      "createdAt": "2024-01-01T00:00:00Z",
+      "updatedAt": "2024-01-01T00:00:00Z",
+      "file": {
+        "id": "string",
+        "name": "document.pdf",
+        "size": 1024,
+        "mimeType": "application/pdf",
+        "hash": "sha256-hash",
+        "uploadedAt": "2024-01-01T00:00:00Z",
+        "expiredAt": null,
+        "hasCompression": false,
+        "hasThumbnail": true,
+        "isEncrypted": false,
+        "description": null
+      }
+    }
+  ],
+  "totalCount": 1
+}
+```
+
+### Get All Files
+**GET** `/api/index/all`
+
+Get all files for the current user across all paths.
+
+**Response:**
+```json
+{
+  "files": [
+    // Same structure as browse endpoint
+  ],
+  "totalCount": 10
+}
+```
+
+### Move File
+**POST** `/api/index/move/{indexId}`
+
+Move a file to a new path.
+
+**Path Parameters:**
+- `indexId` - The file index ID
+
+**Request Body:**
+```json
+{
+  "newPath": "/archived/"
+}
+```
+
+**Response:**
+```json
+{
+  "fileId": "guid",
+  "indexId": "guid",
+  "oldPath": "/documents/",
+  "newPath": "/archived/",
+  "message": "File moved successfully"
+}
+```
+
+### Remove File Index
+**DELETE** `/api/index/remove/{indexId}?deleteFile=false`
+
+Remove a file index. Optionally delete the actual file data.
+
+**Path Parameters:**
+- `indexId` - The file index ID
+
+**Query Parameters:**
+- `deleteFile` (optional, default: false) - Whether to also delete the file data
+
+**Response:**
+```json
+{
+  "message": "File index removed successfully",
+  "fileId": "guid",
+  "fileName": "document.pdf",
+  "path": "/documents/",
+  "fileDataDeleted": false
+}
+```
+
+### Clear Path
+**DELETE** `/api/index/clear-path?path=/temp/&deleteFiles=false`
+
+Remove all file indexes in a specific path.
+
+**Query Parameters:**
+- `path` (optional, default: "/") - The path to clear
+- `deleteFiles` (optional, default: false) - Whether to also delete orphaned files
+
+**Response:**
+```json
+{
+  "message": "Cleared 5 file indexes from path",
+  "path": "/temp/",
+  "removedCount": 5,
+  "filesDeleted": false
+}
+```
+
+### Create File Index
+**POST** `/api/index/create`
+
+Create a new file index for an existing file.
+
+**Request Body:**
+```json
+{
+  "fileId": "guid",
+  "path": "/documents/"
+}
+```
+
+**Response:**
+```json
+{
+  "indexId": "guid",
+  "fileId": "guid",
+  "path": "/documents/",
+  "message": "File index created successfully"
+}
+```
+
+### Search Files
+**GET** `/api/index/search?query=report&path=/documents/`
+
+Search for files by name or metadata.
+
+**Query Parameters:**
+- `query` (required) - The search query
+- `path` (optional) - Limit search to specific path
+
+**Response:**
+```json
+{
+  "query": "report",
+  "path": "/documents/",
+  "results": [
+    // Same structure as browse endpoint
+  ],
+  "totalCount": 3
+}
+```
+
+## Path Normalization
+
+The system automatically normalizes paths to ensure consistency:
+
+- **Trailing Slash**: All paths end with `/`
+- **Root Path**: User home folder is represented as `/`
+- **Query Safety**: Paths are validated to avoid SQL injection
+- **Examples**:
+  - `/documents/` ✅ (correct)
+  - `/documents` → `/documents/` ✅ (normalized)
+  - `/documents/reports/` ✅ (correct)
+  - `/documents/reports` → `/documents/reports/` ✅ (normalized)
+
+## File Upload Integration
+
+When uploading files with the `FileUploadController`, you can specify a path to automatically create file indexes:
+
+**Create Upload Task Request:**
+```json
+{
+  "fileName": "document.pdf",
+  "fileSize": 1024,
+  "contentType": "application/pdf",
+  "hash": "sha256-hash",
+  "path": "/documents/"  // New field for file indexing
+}
+```
+
+The system will automatically create a file index when the upload completes successfully.
+
+## Service Methods
+
+### FileIndexService
+
+```csharp
+public class FileIndexService
+{
+    // Create a new file index
+    Task<SnCloudFileIndex> CreateAsync(string path, Guid fileId, Guid accountId);
+    
+    // Get files by path
+    Task<List<SnCloudFileIndex>> GetByPathAsync(Guid accountId, string path);
+    
+    // Get all files for account
+    Task<List<SnCloudFileIndex>> GetByAccountIdAsync(Guid accountId);
+    
+    // Get indexes for specific file
+    Task<List<SnCloudFileIndex>> GetByFileIdAsync(Guid fileId);
+    
+    // Move file to new path
+    Task<SnCloudFileIndex?> UpdateAsync(Guid indexId, string newPath);
+    
+    // Remove file index
+    Task<bool> RemoveAsync(Guid indexId);
+    
+    // Remove all indexes in path
+    Task<int> RemoveByPathAsync(Guid accountId, string path);
+    
+    // Normalize path format
+    public static string NormalizePath(string path);
+}
+```
+
+## Error Handling
+
+The API returns appropriate HTTP status codes and error messages:
+
+- **400 Bad Request**: Invalid input parameters
+- **401 Unauthorized**: User not authenticated
+- **403 Forbidden**: User lacks permission
+- **404 Not Found**: Resource not found
+- **500 Internal Server Error**: Server-side error
+
+**Error Response Format:**
+```json
+{
+  "code": "BROWSE_FAILED",
+  "message": "Failed to browse files",
+  "status": 500
+}
+```
+
+## Security Considerations
+
+1. **Ownership Verification**: All operations verify that the user owns the file indexes
+2. **Path Validation**: Paths are normalized and validated
+3. **Cascade Deletion**: File indexes are automatically removed when files are deleted
+4. **Safe File Deletion**: Files are only deleted when no other indexes reference them
+
+## Usage Examples
+
+### Upload File to Specific Path
+```bash
+# Create upload task with path
+curl -X POST /api/files/upload/create \
+  -H "Authorization: Bearer {token}" \
+  -H "Content-Type: application/json" \
+  -d '{
+    "fileName": "report.pdf",
+    "fileSize": 2048,
+    "contentType": "application/pdf",
+    "path": "/documents/reports/"
+  }'
+```
+
+### Browse Files
+```bash
+curl -X GET "/api/index/browse?path=/documents/reports/" \
+  -H "Authorization: Bearer {token}"
+```
+
+### Move File
+```bash
+curl -X POST "/api/index/move/{indexId}" \
+  -H "Authorization: Bearer {token}" \
+  -H "Content-Type: application/json" \
+  -d '{"newPath": "/archived/"}'
+```
+
+### Search Files
+```bash
+curl -X GET "/api/index/search?query=invoice&path=/documents/" \
+  -H "Authorization: Bearer {token}"
+```
+
+## Best Practices
+
+1. **Use Trailing Slashes**: Always include trailing slashes in paths
+2. **Organize Hierarchically**: Use meaningful folder structures
+3. **Search Efficiently**: Use the search endpoint instead of client-side filtering
+4. **Clean Up**: Use the clear-path endpoint for temporary directories
+5. **Monitor Usage**: Check total file counts for quota management
+
+## Integration Notes
+
+- The file indexing system works alongside the existing file storage
+- Files can exist in multiple paths (hard links)
+- File deletion is optional and only removes data when safe
+- The system maintains referential integrity between files and indexes

DysonNetwork.Drive/Migrations/20250713121317_InitialMigration.Designer.cs

@@ -3,7 +3,7 @@ using System;
 using System.Collections.Generic;
 using DysonNetwork.Drive;
 using DysonNetwork.Drive.Storage;
-using DysonNetwork.Shared.Data;
+using DysonNetwork.Shared.Models;
 using Microsoft.EntityFrameworkCore;
 using Microsoft.EntityFrameworkCore.Infrastructure;
 using Microsoft.EntityFrameworkCore.Migrations;

DysonNetwork.Drive/Migrations/20250713121317_InitialMigration.cs

@@ -1,7 +1,4 @@
-using System;
-using System.Collections.Generic;
-using DysonNetwork.Drive.Storage;
-using DysonNetwork.Shared.Data;
+using DysonNetwork.Shared.Models;
 using Microsoft.EntityFrameworkCore.Migrations;
 using NodaTime;
 

DysonNetwork.Drive/Migrations/20250715080004_ReinitalMigration.Designer.cs

@@ -2,7 +2,7 @@
 using System;
 using System.Collections.Generic;
 using DysonNetwork.Drive;
-using DysonNetwork.Shared.Data;
+using DysonNetwork.Shared.Models;
 using Microsoft.EntityFrameworkCore;
 using Microsoft.EntityFrameworkCore.Infrastructure;
 using Microsoft.EntityFrameworkCore.Migrations;

DysonNetwork.Drive/Migrations/20250715080004_ReinitalMigration.cs

@@ -1,5 +1,4 @@
-using System.Collections.Generic;
-using Microsoft.EntityFrameworkCore.Migrations;
+using Microsoft.EntityFrameworkCore.Migrations;
 
 #nullable disable
 

DysonNetwork.Drive/Migrations/20250726103203_AddCloudFilePool.Designer.cs

@@ -2,8 +2,7 @@
 using System;
 using System.Collections.Generic;
 using DysonNetwork.Drive;
-using DysonNetwork.Drive.Storage;
-using DysonNetwork.Shared.Data;
+using DysonNetwork.Shared.Models;
 using Microsoft.EntityFrameworkCore;
 using Microsoft.EntityFrameworkCore.Infrastructure;
 using Microsoft.EntityFrameworkCore.Migrations;

DysonNetwork.Drive/Migrations/20250726103203_AddCloudFilePool.cs

@@ -1,6 +1,4 @@
-using System;
-using System.Collections.Generic;
-using DysonNetwork.Drive.Storage;
+using DysonNetwork.Shared.Models;
 using Microsoft.EntityFrameworkCore.Migrations;
 using NodaTime;
 

DysonNetwork.Drive/Migrations/20250726120323_AddFilePoolDescription.Designer.cs

@@ -2,8 +2,7 @@
 using System;
 using System.Collections.Generic;
 using DysonNetwork.Drive;
-using DysonNetwork.Drive.Storage;
-using DysonNetwork.Shared.Data;
+using DysonNetwork.Shared.Models;
 using Microsoft.EntityFrameworkCore;
 using Microsoft.EntityFrameworkCore.Infrastructure;
 using Microsoft.EntityFrameworkCore.Migrations;

DysonNetwork.Drive/Migrations/20250726172039_AddCloudFileExpiration.Designer.cs

@@ -2,8 +2,7 @@
 using System;
 using System.Collections.Generic;
 using DysonNetwork.Drive;
-using DysonNetwork.Drive.Storage;
-using DysonNetwork.Shared.Data;
+using DysonNetwork.Shared.Models;
 using Microsoft.EntityFrameworkCore;
 using Microsoft.EntityFrameworkCore.Infrastructure;
 using Microsoft.EntityFrameworkCore.Migrations;

DysonNetwork.Drive/Migrations/20250727092028_AddQuotaRecord.Designer.cs

@@ -2,8 +2,7 @@
 using System;
 using System.Collections.Generic;
 using DysonNetwork.Drive;
-using DysonNetwork.Drive.Storage;
-using DysonNetwork.Shared.Data;
+using DysonNetwork.Shared.Models;
 using Microsoft.EntityFrameworkCore;
 using Microsoft.EntityFrameworkCore.Infrastructure;
 using Microsoft.EntityFrameworkCore.Migrations;

DysonNetwork.Drive/Migrations/20250727092028_AddQuotaRecord.cs

@@ -1,5 +1,4 @@
-using System;
-using Microsoft.EntityFrameworkCore.Migrations;
+using Microsoft.EntityFrameworkCore.Migrations;
 using NodaTime;
 
 #nullable disable

DysonNetwork.Drive/Migrations/20250727130951_AddFileBundle.Designer.cs

@@ -2,8 +2,7 @@
 using System;
 using System.Collections.Generic;
 using DysonNetwork.Drive;
-using DysonNetwork.Drive.Storage;
-using DysonNetwork.Shared.Data;
+using DysonNetwork.Shared.Models;
 using Microsoft.EntityFrameworkCore;
 using Microsoft.EntityFrameworkCore.Infrastructure;
 using Microsoft.EntityFrameworkCore.Migrations;

DysonNetwork.Drive/Migrations/20250727130951_AddFileBundle.cs

@@ -1,5 +1,4 @@
-using System;
-using Microsoft.EntityFrameworkCore.Migrations;
+using Microsoft.EntityFrameworkCore.Migrations;
 using NodaTime;
 
 #nullable disable

DysonNetwork.Drive/Migrations/20250808170904_AddHiddenPool.Designer.cs

@@ -2,8 +2,7 @@
 using System;
 using System.Collections.Generic;
 using DysonNetwork.Drive;
-using DysonNetwork.Drive.Storage;
-using DysonNetwork.Shared.Data;
+using DysonNetwork.Shared.Models;
 using Microsoft.EntityFrameworkCore;
 using Microsoft.EntityFrameworkCore.Infrastructure;
 using Microsoft.EntityFrameworkCore.Migrations;

DysonNetwork.Drive/Migrations/20250819164302_RemoveUploadedTo.Designer.cs

@@ -2,8 +2,7 @@
 using System;
 using System.Collections.Generic;
 using DysonNetwork.Drive;
-using DysonNetwork.Drive.Storage;
-using DysonNetwork.Shared.Data;
+using DysonNetwork.Shared.Models;
 using Microsoft.EntityFrameworkCore;
 using Microsoft.EntityFrameworkCore.Infrastructure;
 using Microsoft.EntityFrameworkCore.Migrations;

DysonNetwork.Drive/Migrations/20250907070034_RemoveNetTopo.Designer.cs

@@ -2,8 +2,7 @@
 using System;
 using System.Collections.Generic;
 using DysonNetwork.Drive;
-using DysonNetwork.Drive.Storage;
-using DysonNetwork.Shared.Data;
+using DysonNetwork.Shared.Models;
 using Microsoft.EntityFrameworkCore;
 using Microsoft.EntityFrameworkCore.Infrastructure;
 using Microsoft.EntityFrameworkCore.Migrations;

DysonNetwork.Drive/Migrations/20251108191230_AddPersistentTask.Designer.cs

@@ -0,0 +1,567 @@
+// <auto-generated />
+using System;
+using System.Collections.Generic;
+using DysonNetwork.Drive;
+using DysonNetwork.Shared.Models;
+using Microsoft.EntityFrameworkCore;
+using Microsoft.EntityFrameworkCore.Infrastructure;
+using Microsoft.EntityFrameworkCore.Migrations;
+using Microsoft.EntityFrameworkCore.Storage.ValueConversion;
+using NodaTime;
+using Npgsql.EntityFrameworkCore.PostgreSQL.Metadata;
+
+#nullable disable
+
+namespace DysonNetwork.Drive.Migrations
+{
+    [DbContext(typeof(AppDatabase))]
+    [Migration("20251108191230_AddPersistentTask")]
+    partial class AddPersistentTask
+    {
+        /// <inheritdoc />
+        protected override void BuildTargetModel(ModelBuilder modelBuilder)
+        {
+#pragma warning disable 612, 618
+            modelBuilder
+                .HasAnnotation("ProductVersion", "9.0.10")
+                .HasAnnotation("Relational:MaxIdentifierLength", 63);
+
+            NpgsqlModelBuilderExtensions.UseIdentityByDefaultColumns(modelBuilder);
+
+            modelBuilder.Entity("DysonNetwork.Drive.Billing.QuotaRecord", b =>
+                {
+                    b.Property<Guid>("Id")
+                        .ValueGeneratedOnAdd()
+                        .HasColumnType("uuid")
+                        .HasColumnName("id");
+
+                    b.Property<Guid>("AccountId")
+                        .HasColumnType("uuid")
+                        .HasColumnName("account_id");
+
+                    b.Property<Instant>("CreatedAt")
+                        .HasColumnType("timestamp with time zone")
+                        .HasColumnName("created_at");
+
+                    b.Property<Instant?>("DeletedAt")
+                        .HasColumnType("timestamp with time zone")
+                        .HasColumnName("deleted_at");
+
+                    b.Property<string>("Description")
+                        .IsRequired()
+                        .HasColumnType("text")
+                        .HasColumnName("description");
+
+                    b.Property<Instant?>("ExpiredAt")
+                        .HasColumnType("timestamp with time zone")
+                        .HasColumnName("expired_at");
+
+                    b.Property<string>("Name")
+                        .IsRequired()
+                        .HasColumnType("text")
+                        .HasColumnName("name");
+
+                    b.Property<long>("Quota")
+                        .HasColumnType("bigint")
+                        .HasColumnName("quota");
+
+                    b.Property<Instant>("UpdatedAt")
+                        .HasColumnType("timestamp with time zone")
+                        .HasColumnName("updated_at");
+
+                    b.HasKey("Id")
+                        .HasName("pk_quota_records");
+
+                    b.ToTable("quota_records", (string)null);
+                });
+
+            modelBuilder.Entity("DysonNetwork.Drive.Storage.Model.PersistentTask", b =>
+                {
+                    b.Property<Guid>("Id")
+                        .ValueGeneratedOnAdd()
+                        .HasColumnType("uuid")
+                        .HasColumnName("id");
+
+                    b.Property<Guid>("AccountId")
+                        .HasColumnType("uuid")
+                        .HasColumnName("account_id");
+
+                    b.Property<Instant?>("CompletedAt")
+                        .HasColumnType("timestamp with time zone")
+                        .HasColumnName("completed_at");
+
+                    b.Property<Instant>("CreatedAt")
+                        .HasColumnType("timestamp with time zone")
+                        .HasColumnName("created_at");
+
+                    b.Property<Instant?>("DeletedAt")
+                        .HasColumnType("timestamp with time zone")
+                        .HasColumnName("deleted_at");
+
+                    b.Property<string>("Description")
+                        .HasMaxLength(1024)
+                        .HasColumnType("character varying(1024)")
+                        .HasColumnName("description");
+
+                    b.Property<string>("Discriminator")
+                        .IsRequired()
+                        .HasMaxLength(21)
+                        .HasColumnType("character varying(21)")
+                        .HasColumnName("discriminator");
+
+                    b.Property<string>("ErrorMessage")
+                        .HasMaxLength(1024)
+                        .HasColumnType("character varying(1024)")
+                        .HasColumnName("error_message");
+
+                    b.Property<long?>("EstimatedDurationSeconds")
+                        .HasColumnType("bigint")
+                        .HasColumnName("estimated_duration_seconds");
+
+                    b.Property<Instant?>("ExpiredAt")
+                        .HasColumnType("timestamp with time zone")
+                        .HasColumnName("expired_at");
+
+                    b.Property<Instant>("LastActivity")
+                        .HasColumnType("timestamp with time zone")
+                        .HasColumnName("last_activity");
+
+                    b.Property<string>("Name")
+                        .IsRequired()
+                        .HasMaxLength(256)
+                        .HasColumnType("character varying(256)")
+                        .HasColumnName("name");
+
+                    b.Property<Dictionary<string, object>>("Parameters")
+                        .IsRequired()
+                        .HasColumnType("jsonb")
+                        .HasColumnName("parameters");
+
+                    b.Property<int>("Priority")
+                        .HasColumnType("integer")
+                        .HasColumnName("priority");
+
+                    b.Property<double>("Progress")
+                        .HasColumnType("double precision")
+                        .HasColumnName("progress");
+
+                    b.Property<Dictionary<string, object>>("Results")
+                        .IsRequired()
+                        .HasColumnType("jsonb")
+                        .HasColumnName("results");
+
+                    b.Property<Instant?>("StartedAt")
+                        .HasColumnType("timestamp with time zone")
+                        .HasColumnName("started_at");
+
+                    b.Property<int>("Status")
+                        .HasColumnType("integer")
+                        .HasColumnName("status");
+
+                    b.Property<string>("TaskId")
+                        .IsRequired()
+                        .HasMaxLength(64)
+                        .HasColumnType("character varying(64)")
+                        .HasColumnName("task_id");
+
+                    b.Property<int>("Type")
+                        .HasColumnType("integer")
+                        .HasColumnName("type");
+
+                    b.Property<Instant>("UpdatedAt")
+                        .HasColumnType("timestamp with time zone")
+                        .HasColumnName("updated_at");
+
+                    b.HasKey("Id")
+                        .HasName("pk_tasks");
+
+                    b.ToTable("tasks", (string)null);
+
+                    b.HasDiscriminator().HasValue("PersistentTask");
+
+                    b.UseTphMappingStrategy();
+                });
+
+            modelBuilder.Entity("DysonNetwork.Shared.Models.CloudFileReference", b =>
+                {
+                    b.Property<Guid>("Id")
+                        .ValueGeneratedOnAdd()
+                        .HasColumnType("uuid")
+                        .HasColumnName("id");
+
+                    b.Property<Instant>("CreatedAt")
+                        .HasColumnType("timestamp with time zone")
+                        .HasColumnName("created_at");
+
+                    b.Property<Instant?>("DeletedAt")
+                        .HasColumnType("timestamp with time zone")
+                        .HasColumnName("deleted_at");
+
+                    b.Property<Instant?>("ExpiredAt")
+                        .HasColumnType("timestamp with time zone")
+                        .HasColumnName("expired_at");
+
+                    b.Property<string>("FileId")
+                        .IsRequired()
+                        .HasMaxLength(32)
+                        .HasColumnType("character varying(32)")
+                        .HasColumnName("file_id");
+
+                    b.Property<string>("ResourceId")
+                        .IsRequired()
+                        .HasMaxLength(1024)
+                        .HasColumnType("character varying(1024)")
+                        .HasColumnName("resource_id");
+
+                    b.Property<Instant>("UpdatedAt")
+                        .HasColumnType("timestamp with time zone")
+                        .HasColumnName("updated_at");
+
+                    b.Property<string>("Usage")
+                        .IsRequired()
+                        .HasMaxLength(1024)
+                        .HasColumnType("character varying(1024)")
+                        .HasColumnName("usage");
+
+                    b.HasKey("Id")
+                        .HasName("pk_file_references");
+
+                    b.HasIndex("FileId")
+                        .HasDatabaseName("ix_file_references_file_id");
+
+                    b.ToTable("file_references", (string)null);
+                });
+
+            modelBuilder.Entity("DysonNetwork.Shared.Models.FilePool", b =>
+                {
+                    b.Property<Guid>("Id")
+                        .ValueGeneratedOnAdd()
+                        .HasColumnType("uuid")
+                        .HasColumnName("id");
+
+                    b.Property<Guid?>("AccountId")
+                        .HasColumnType("uuid")
+                        .HasColumnName("account_id");
+
+                    b.Property<BillingConfig>("BillingConfig")
+                        .IsRequired()
+                        .HasColumnType("jsonb")
+                        .HasColumnName("billing_config");
+
+                    b.Property<Instant>("CreatedAt")
+                        .HasColumnType("timestamp with time zone")
+                        .HasColumnName("created_at");
+
+                    b.Property<Instant?>("DeletedAt")
+                        .HasColumnType("timestamp with time zone")
+                        .HasColumnName("deleted_at");
+
+                    b.Property<string>("Description")
+                        .IsRequired()
+                        .HasMaxLength(8192)
+                        .HasColumnType("character varying(8192)")
+                        .HasColumnName("description");
+
+                    b.Property<bool>("IsHidden")
+                        .HasColumnType("boolean")
+                        .HasColumnName("is_hidden");
+
+                    b.Property<string>("Name")
+                        .IsRequired()
+                        .HasMaxLength(1024)
+                        .HasColumnType("character varying(1024)")
+                        .HasColumnName("name");
+
+                    b.Property<PolicyConfig>("PolicyConfig")
+                        .IsRequired()
+                        .HasColumnType("jsonb")
+                        .HasColumnName("policy_config");
+
+                    b.Property<RemoteStorageConfig>("StorageConfig")
+                        .IsRequired()
+                        .HasColumnType("jsonb")
+                        .HasColumnName("storage_config");
+
+                    b.Property<Instant>("UpdatedAt")
+                        .HasColumnType("timestamp with time zone")
+                        .HasColumnName("updated_at");
+
+                    b.HasKey("Id")
+                        .HasName("pk_pools");
+
+                    b.ToTable("pools", (string)null);
+                });
+
+            modelBuilder.Entity("DysonNetwork.Shared.Models.SnCloudFile", b =>
+                {
+                    b.Property<string>("Id")
+                        .HasMaxLength(32)
+                        .HasColumnType("character varying(32)")
+                        .HasColumnName("id");
+
+                    b.Property<Guid>("AccountId")
+                        .HasColumnType("uuid")
+                        .HasColumnName("account_id");
+
+                    b.Property<Guid?>("BundleId")
+                        .HasColumnType("uuid")
+                        .HasColumnName("bundle_id");
+
+                    b.Property<Instant>("CreatedAt")
+                        .HasColumnType("timestamp with time zone")
+                        .HasColumnName("created_at");
+
+                    b.Property<Instant?>("DeletedAt")
+                        .HasColumnType("timestamp with time zone")
+                        .HasColumnName("deleted_at");
+
+                    b.Property<string>("Description")
+                        .HasMaxLength(4096)
+                        .HasColumnType("character varying(4096)")
+                        .HasColumnName("description");
+
+                    b.Property<Instant?>("ExpiredAt")
+                        .HasColumnType("timestamp with time zone")
+                        .HasColumnName("expired_at");
+
+                    b.Property<Dictionary<string, object>>("FileMeta")
+                        .HasColumnType("jsonb")
+                        .HasColumnName("file_meta");
+
+                    b.Property<bool>("HasCompression")
+                        .HasColumnType("boolean")
+                        .HasColumnName("has_compression");
+
+                    b.Property<bool>("HasThumbnail")
+                        .HasColumnType("boolean")
+                        .HasColumnName("has_thumbnail");
+
+                    b.Property<string>("Hash")
+                        .HasMaxLength(256)
+                        .HasColumnType("character varying(256)")
+                        .HasColumnName("hash");
+
+                    b.Property<bool>("IsEncrypted")
+                        .HasColumnType("boolean")
+                        .HasColumnName("is_encrypted");
+
+                    b.Property<bool>("IsMarkedRecycle")
+                        .HasColumnType("boolean")
+                        .HasColumnName("is_marked_recycle");
+
+                    b.Property<string>("MimeType")
+                        .HasMaxLength(256)
+                        .HasColumnType("character varying(256)")
+                        .HasColumnName("mime_type");
+
+                    b.Property<string>("Name")
+                        .IsRequired()
+                        .HasMaxLength(1024)
+                        .HasColumnType("character varying(1024)")
+                        .HasColumnName("name");
+
+                    b.Property<Guid?>("PoolId")
+                        .HasColumnType("uuid")
+                        .HasColumnName("pool_id");
+
+                    b.Property<List<ContentSensitiveMark>>("SensitiveMarks")
+                        .HasColumnType("jsonb")
+                        .HasColumnName("sensitive_marks");
+
+                    b.Property<long>("Size")
+                        .HasColumnType("bigint")
+                        .HasColumnName("size");
+
+                    b.Property<string>("StorageId")
+                        .HasMaxLength(32)
+                        .HasColumnType("character varying(32)")
+                        .HasColumnName("storage_id");
+
+                    b.Property<string>("StorageUrl")
+                        .HasMaxLength(4096)
+                        .HasColumnType("character varying(4096)")
+                        .HasColumnName("storage_url");
+
+                    b.Property<Instant>("UpdatedAt")
+                        .HasColumnType("timestamp with time zone")
+                        .HasColumnName("updated_at");
+
+                    b.Property<Instant?>("UploadedAt")
+                        .HasColumnType("timestamp with time zone")
+                        .HasColumnName("uploaded_at");
+
+                    b.Property<Dictionary<string, object>>("UserMeta")
+                        .HasColumnType("jsonb")
+                        .HasColumnName("user_meta");
+
+                    b.HasKey("Id")
+                        .HasName("pk_files");
+
+                    b.HasIndex("BundleId")
+                        .HasDatabaseName("ix_files_bundle_id");
+
+                    b.HasIndex("PoolId")
+                        .HasDatabaseName("ix_files_pool_id");
+
+                    b.ToTable("files", (string)null);
+                });
+
+            modelBuilder.Entity("DysonNetwork.Shared.Models.SnFileBundle", b =>
+                {
+                    b.Property<Guid>("Id")
+                        .ValueGeneratedOnAdd()
+                        .HasColumnType("uuid")
+                        .HasColumnName("id");
+
+                    b.Property<Guid>("AccountId")
+                        .HasColumnType("uuid")
+                        .HasColumnName("account_id");
+
+                    b.Property<Instant>("CreatedAt")
+                        .HasColumnType("timestamp with time zone")
+                        .HasColumnName("created_at");
+
+                    b.Property<Instant?>("DeletedAt")
+                        .HasColumnType("timestamp with time zone")
+                        .HasColumnName("deleted_at");
+
+                    b.Property<string>("Description")
+                        .HasMaxLength(8192)
+                        .HasColumnType("character varying(8192)")
+                        .HasColumnName("description");
+
+                    b.Property<Instant?>("ExpiredAt")
+                        .HasColumnType("timestamp with time zone")
+                        .HasColumnName("expired_at");
+
+                    b.Property<string>("Name")
+                        .IsRequired()
+                        .HasMaxLength(1024)
+                        .HasColumnType("character varying(1024)")
+                        .HasColumnName("name");
+
+                    b.Property<string>("Passcode")
+                        .HasMaxLength(256)
+                        .HasColumnType("character varying(256)")
+                        .HasColumnName("passcode");
+
+                    b.Property<string>("Slug")
+                        .IsRequired()
+                        .HasMaxLength(1024)
+                        .HasColumnType("character varying(1024)")
+                        .HasColumnName("slug");
+
+                    b.Property<Instant>("UpdatedAt")
+                        .HasColumnType("timestamp with time zone")
+                        .HasColumnName("updated_at");
+
+                    b.HasKey("Id")
+                        .HasName("pk_bundles");
+
+                    b.HasIndex("Slug")
+                        .IsUnique()
+                        .HasDatabaseName("ix_bundles_slug");
+
+                    b.ToTable("bundles", (string)null);
+                });
+
+            modelBuilder.Entity("DysonNetwork.Drive.Storage.Model.PersistentUploadTask", b =>
+                {
+                    b.HasBaseType("DysonNetwork.Drive.Storage.Model.PersistentTask");
+
+                    b.Property<Guid?>("BundleId")
+                        .HasColumnType("uuid")
+                        .HasColumnName("bundle_id");
+
+                    b.Property<long>("ChunkSize")
+                        .HasColumnType("bigint")
+                        .HasColumnName("chunk_size");
+
+                    b.Property<int>("ChunksCount")
+                        .HasColumnType("integer")
+                        .HasColumnName("chunks_count");
+
+                    b.Property<int>("ChunksUploaded")
+                        .HasColumnType("integer")
+                        .HasColumnName("chunks_uploaded");
+
+                    b.Property<string>("ContentType")
+                        .IsRequired()
+                        .HasMaxLength(128)
+                        .HasColumnType("character varying(128)")
+                        .HasColumnName("content_type");
+
+                    b.Property<string>("EncryptPassword")
+                        .HasMaxLength(256)
+                        .HasColumnType("character varying(256)")
+                        .HasColumnName("encrypt_password");
+
+                    b.Property<string>("FileName")
+                        .IsRequired()
+                        .HasMaxLength(256)
+                        .HasColumnType("character varying(256)")
+                        .HasColumnName("file_name");
+
+                    b.Property<long>("FileSize")
+                        .HasColumnType("bigint")
+                        .HasColumnName("file_size");
+
+                    b.Property<string>("Hash")
+                        .IsRequired()
+                        .HasColumnType("text")
+                        .HasColumnName("hash");
+
+                    b.Property<Guid>("PoolId")
+                        .HasColumnType("uuid")
+                        .HasColumnName("pool_id");
+
+                    b.PrimitiveCollection<List<int>>("UploadedChunks")
+                        .IsRequired()
+                        .HasColumnType("integer[]")
+                        .HasColumnName("uploaded_chunks");
+
+                    b.HasDiscriminator().HasValue("PersistentUploadTask");
+                });
+
+            modelBuilder.Entity("DysonNetwork.Shared.Models.CloudFileReference", b =>
+                {
+                    b.HasOne("DysonNetwork.Shared.Models.SnCloudFile", "File")
+                        .WithMany("References")
+                        .HasForeignKey("FileId")
+                        .OnDelete(DeleteBehavior.Cascade)
+                        .IsRequired()
+                        .HasConstraintName("fk_file_references_files_file_id");
+
+                    b.Navigation("File");
+                });
+
+            modelBuilder.Entity("DysonNetwork.Shared.Models.SnCloudFile", b =>
+                {
+                    b.HasOne("DysonNetwork.Shared.Models.SnFileBundle", "Bundle")
+                        .WithMany("Files")
+                        .HasForeignKey("BundleId")
+                        .HasConstraintName("fk_files_bundles_bundle_id");
+
+                    b.HasOne("DysonNetwork.Shared.Models.FilePool", "Pool")
+                        .WithMany()
+                        .HasForeignKey("PoolId")
+                        .HasConstraintName("fk_files_pools_pool_id");
+
+                    b.Navigation("Bundle");
+
+                    b.Navigation("Pool");
+                });
+
+            modelBuilder.Entity("DysonNetwork.Shared.Models.SnCloudFile", b =>
+                {
+                    b.Navigation("References");
+                });
+
+            modelBuilder.Entity("DysonNetwork.Shared.Models.SnFileBundle", b =>
+                {
+                    b.Navigation("Files");
+                });
+#pragma warning restore 612, 618
+        }
+    }
+}

DysonNetwork.Drive/Migrations/20251108191230_AddPersistentTask.cs

@@ -0,0 +1,66 @@
+using System;
+using System.Collections.Generic;
+using Microsoft.EntityFrameworkCore.Migrations;
+using NodaTime;
+
+#nullable disable
+
+namespace DysonNetwork.Drive.Migrations
+{
+    /// <inheritdoc />
+    public partial class AddPersistentTask : Migration
+    {
+        /// <inheritdoc />
+        protected override void Up(MigrationBuilder migrationBuilder)
+        {
+            migrationBuilder.CreateTable(
+                name: "tasks",
+                columns: table => new
+                {
+                    id = table.Column<Guid>(type: "uuid", nullable: false),
+                    task_id = table.Column<string>(type: "character varying(64)", maxLength: 64, nullable: false),
+                    name = table.Column<string>(type: "character varying(256)", maxLength: 256, nullable: false),
+                    description = table.Column<string>(type: "character varying(1024)", maxLength: 1024, nullable: true),
+                    type = table.Column<int>(type: "integer", nullable: false),
+                    status = table.Column<int>(type: "integer", nullable: false),
+                    account_id = table.Column<Guid>(type: "uuid", nullable: false),
+                    progress = table.Column<double>(type: "double precision", nullable: false),
+                    parameters = table.Column<Dictionary<string, object>>(type: "jsonb", nullable: false),
+                    results = table.Column<Dictionary<string, object>>(type: "jsonb", nullable: false),
+                    error_message = table.Column<string>(type: "character varying(1024)", maxLength: 1024, nullable: true),
+                    started_at = table.Column<Instant>(type: "timestamp with time zone", nullable: true),
+                    completed_at = table.Column<Instant>(type: "timestamp with time zone", nullable: true),
+                    expired_at = table.Column<Instant>(type: "timestamp with time zone", nullable: true),
+                    last_activity = table.Column<Instant>(type: "timestamp with time zone", nullable: false),
+                    priority = table.Column<int>(type: "integer", nullable: false),
+                    estimated_duration_seconds = table.Column<long>(type: "bigint", nullable: true),
+                    discriminator = table.Column<string>(type: "character varying(21)", maxLength: 21, nullable: false),
+                    file_name = table.Column<string>(type: "character varying(256)", maxLength: 256, nullable: true),
+                    file_size = table.Column<long>(type: "bigint", nullable: true),
+                    content_type = table.Column<string>(type: "character varying(128)", maxLength: 128, nullable: true),
+                    chunk_size = table.Column<long>(type: "bigint", nullable: true),
+                    chunks_count = table.Column<int>(type: "integer", nullable: true),
+                    chunks_uploaded = table.Column<int>(type: "integer", nullable: true),
+                    pool_id = table.Column<Guid>(type: "uuid", nullable: true),
+                    bundle_id = table.Column<Guid>(type: "uuid", nullable: true),
+                    encrypt_password = table.Column<string>(type: "character varying(256)", maxLength: 256, nullable: true),
+                    hash = table.Column<string>(type: "text", nullable: true),
+                    uploaded_chunks = table.Column<List<int>>(type: "integer[]", nullable: true),
+                    created_at = table.Column<Instant>(type: "timestamp with time zone", nullable: false),
+                    updated_at = table.Column<Instant>(type: "timestamp with time zone", nullable: false),
+                    deleted_at = table.Column<Instant>(type: "timestamp with time zone", nullable: true)
+                },
+                constraints: table =>
+                {
+                    table.PrimaryKey("pk_tasks", x => x.id);
+                });
+        }
+
+        /// <inheritdoc />
+        protected override void Down(MigrationBuilder migrationBuilder)
+        {
+            migrationBuilder.DropTable(
+                name: "tasks");
+        }
+    }
+}

DysonNetwork.Drive/Migrations/20251112135535_AddFileIndex.Designer.cs

@@ -0,0 +1,632 @@
+// <auto-generated />
+using System;
+using System.Collections.Generic;
+using DysonNetwork.Drive;
+using DysonNetwork.Shared.Models;
+using Microsoft.EntityFrameworkCore;
+using Microsoft.EntityFrameworkCore.Infrastructure;
+using Microsoft.EntityFrameworkCore.Migrations;
+using Microsoft.EntityFrameworkCore.Storage.ValueConversion;
+using NodaTime;
+using Npgsql.EntityFrameworkCore.PostgreSQL.Metadata;
+
+#nullable disable
+
+namespace DysonNetwork.Drive.Migrations
+{
+    [DbContext(typeof(AppDatabase))]
+    [Migration("20251112135535_AddFileIndex")]
+    partial class AddFileIndex
+    {
+        /// <inheritdoc />
+        protected override void BuildTargetModel(ModelBuilder modelBuilder)
+        {
+#pragma warning disable 612, 618
+            modelBuilder
+                .HasAnnotation("ProductVersion", "9.0.10")
+                .HasAnnotation("Relational:MaxIdentifierLength", 63);
+
+            NpgsqlModelBuilderExtensions.UseIdentityByDefaultColumns(modelBuilder);
+
+            modelBuilder.Entity("DysonNetwork.Drive.Billing.QuotaRecord", b =>
+                {
+                    b.Property<Guid>("Id")
+                        .ValueGeneratedOnAdd()
+                        .HasColumnType("uuid")
+                        .HasColumnName("id");
+
+                    b.Property<Guid>("AccountId")
+                        .HasColumnType("uuid")
+                        .HasColumnName("account_id");
+
+                    b.Property<Instant>("CreatedAt")
+                        .HasColumnType("timestamp with time zone")
+                        .HasColumnName("created_at");
+
+                    b.Property<Instant?>("DeletedAt")
+                        .HasColumnType("timestamp with time zone")
+                        .HasColumnName("deleted_at");
+
+                    b.Property<string>("Description")
+                        .IsRequired()
+                        .HasColumnType("text")
+                        .HasColumnName("description");
+
+                    b.Property<Instant?>("ExpiredAt")
+                        .HasColumnType("timestamp with time zone")
+                        .HasColumnName("expired_at");
+
+                    b.Property<string>("Name")
+                        .IsRequired()
+                        .HasColumnType("text")
+                        .HasColumnName("name");
+
+                    b.Property<long>("Quota")
+                        .HasColumnType("bigint")
+                        .HasColumnName("quota");
+
+                    b.Property<Instant>("UpdatedAt")
+                        .HasColumnType("timestamp with time zone")
+                        .HasColumnName("updated_at");
+
+                    b.HasKey("Id")
+                        .HasName("pk_quota_records");
+
+                    b.ToTable("quota_records", (string)null);
+                });
+
+            modelBuilder.Entity("DysonNetwork.Drive.Storage.Model.PersistentTask", b =>
+                {
+                    b.Property<Guid>("Id")
+                        .ValueGeneratedOnAdd()
+                        .HasColumnType("uuid")
+                        .HasColumnName("id");
+
+                    b.Property<Guid>("AccountId")
+                        .HasColumnType("uuid")
+                        .HasColumnName("account_id");
+
+                    b.Property<Instant?>("CompletedAt")
+                        .HasColumnType("timestamp with time zone")
+                        .HasColumnName("completed_at");
+
+                    b.Property<Instant>("CreatedAt")
+                        .HasColumnType("timestamp with time zone")
+                        .HasColumnName("created_at");
+
+                    b.Property<Instant?>("DeletedAt")
+                        .HasColumnType("timestamp with time zone")
+                        .HasColumnName("deleted_at");
+
+                    b.Property<string>("Description")
+                        .HasMaxLength(1024)
+                        .HasColumnType("character varying(1024)")
+                        .HasColumnName("description");
+
+                    b.Property<string>("Discriminator")
+                        .IsRequired()
+                        .HasMaxLength(21)
+                        .HasColumnType("character varying(21)")
+                        .HasColumnName("discriminator");
+
+                    b.Property<string>("ErrorMessage")
+                        .HasMaxLength(1024)
+                        .HasColumnType("character varying(1024)")
+                        .HasColumnName("error_message");
+
+                    b.Property<long?>("EstimatedDurationSeconds")
+                        .HasColumnType("bigint")
+                        .HasColumnName("estimated_duration_seconds");
+
+                    b.Property<Instant?>("ExpiredAt")
+                        .HasColumnType("timestamp with time zone")
+                        .HasColumnName("expired_at");
+
+                    b.Property<Instant>("LastActivity")
+                        .HasColumnType("timestamp with time zone")
+                        .HasColumnName("last_activity");
+
+                    b.Property<string>("Name")
+                        .IsRequired()
+                        .HasMaxLength(256)
+                        .HasColumnType("character varying(256)")
+                        .HasColumnName("name");
+
+                    b.Property<Dictionary<string, object>>("Parameters")
+                        .IsRequired()
+                        .HasColumnType("jsonb")
+                        .HasColumnName("parameters");
+
+                    b.Property<int>("Priority")
+                        .HasColumnType("integer")
+                        .HasColumnName("priority");
+
+                    b.Property<double>("Progress")
+                        .HasColumnType("double precision")
+                        .HasColumnName("progress");
+
+                    b.Property<Dictionary<string, object>>("Results")
+                        .IsRequired()
+                        .HasColumnType("jsonb")
+                        .HasColumnName("results");
+
+                    b.Property<Instant?>("StartedAt")
+                        .HasColumnType("timestamp with time zone")
+                        .HasColumnName("started_at");
+
+                    b.Property<int>("Status")
+                        .HasColumnType("integer")
+                        .HasColumnName("status");
+
+                    b.Property<string>("TaskId")
+                        .IsRequired()
+                        .HasMaxLength(64)
+                        .HasColumnType("character varying(64)")
+                        .HasColumnName("task_id");
+
+                    b.Property<int>("Type")
+                        .HasColumnType("integer")
+                        .HasColumnName("type");
+
+                    b.Property<Instant>("UpdatedAt")
+                        .HasColumnType("timestamp with time zone")
+                        .HasColumnName("updated_at");
+
+                    b.HasKey("Id")
+                        .HasName("pk_tasks");
+
+                    b.ToTable("tasks", (string)null);
+
+                    b.HasDiscriminator().HasValue("PersistentTask");
+
+                    b.UseTphMappingStrategy();
+                });
+
+            modelBuilder.Entity("DysonNetwork.Shared.Models.CloudFileReference", b =>
+                {
+                    b.Property<Guid>("Id")
+                        .ValueGeneratedOnAdd()
+                        .HasColumnType("uuid")
+                        .HasColumnName("id");
+
+                    b.Property<Instant>("CreatedAt")
+                        .HasColumnType("timestamp with time zone")
+                        .HasColumnName("created_at");
+
+                    b.Property<Instant?>("DeletedAt")
+                        .HasColumnType("timestamp with time zone")
+                        .HasColumnName("deleted_at");
+
+                    b.Property<Instant?>("ExpiredAt")
+                        .HasColumnType("timestamp with time zone")
+                        .HasColumnName("expired_at");
+
+                    b.Property<string>("FileId")
+                        .IsRequired()
+                        .HasMaxLength(32)
+                        .HasColumnType("character varying(32)")
+                        .HasColumnName("file_id");
+
+                    b.Property<string>("ResourceId")
+                        .IsRequired()
+                        .HasMaxLength(1024)
+                        .HasColumnType("character varying(1024)")
+                        .HasColumnName("resource_id");
+
+                    b.Property<Instant>("UpdatedAt")
+                        .HasColumnType("timestamp with time zone")
+                        .HasColumnName("updated_at");
+
+                    b.Property<string>("Usage")
+                        .IsRequired()
+                        .HasMaxLength(1024)
+                        .HasColumnType("character varying(1024)")
+                        .HasColumnName("usage");
+
+                    b.HasKey("Id")
+                        .HasName("pk_file_references");
+
+                    b.HasIndex("FileId")
+                        .HasDatabaseName("ix_file_references_file_id");
+
+                    b.ToTable("file_references", (string)null);
+                });
+
+            modelBuilder.Entity("DysonNetwork.Shared.Models.FilePool", b =>
+                {
+                    b.Property<Guid>("Id")
+                        .ValueGeneratedOnAdd()
+                        .HasColumnType("uuid")
+                        .HasColumnName("id");
+
+                    b.Property<Guid?>("AccountId")
+                        .HasColumnType("uuid")
+                        .HasColumnName("account_id");
+
+                    b.Property<BillingConfig>("BillingConfig")
+                        .IsRequired()
+                        .HasColumnType("jsonb")
+                        .HasColumnName("billing_config");
+
+                    b.Property<Instant>("CreatedAt")
+                        .HasColumnType("timestamp with time zone")
+                        .HasColumnName("created_at");
+
+                    b.Property<Instant?>("DeletedAt")
+                        .HasColumnType("timestamp with time zone")
+                        .HasColumnName("deleted_at");
+
+                    b.Property<string>("Description")
+                        .IsRequired()
+                        .HasMaxLength(8192)
+                        .HasColumnType("character varying(8192)")
+                        .HasColumnName("description");
+
+                    b.Property<bool>("IsHidden")
+                        .HasColumnType("boolean")
+                        .HasColumnName("is_hidden");
+
+                    b.Property<string>("Name")
+                        .IsRequired()
+                        .HasMaxLength(1024)
+                        .HasColumnType("character varying(1024)")
+                        .HasColumnName("name");
+
+                    b.Property<PolicyConfig>("PolicyConfig")
+                        .IsRequired()
+                        .HasColumnType("jsonb")
+                        .HasColumnName("policy_config");
+
+                    b.Property<RemoteStorageConfig>("StorageConfig")
+                        .IsRequired()
+                        .HasColumnType("jsonb")
+                        .HasColumnName("storage_config");
+
+                    b.Property<Instant>("UpdatedAt")
+                        .HasColumnType("timestamp with time zone")
+                        .HasColumnName("updated_at");
+
+                    b.HasKey("Id")
+                        .HasName("pk_pools");
+
+                    b.ToTable("pools", (string)null);
+                });
+
+            modelBuilder.Entity("DysonNetwork.Shared.Models.SnCloudFile", b =>
+                {
+                    b.Property<string>("Id")
+                        .HasMaxLength(32)
+                        .HasColumnType("character varying(32)")
+                        .HasColumnName("id");
+
+                    b.Property<Guid>("AccountId")
+                        .HasColumnType("uuid")
+                        .HasColumnName("account_id");
+
+                    b.Property<Guid?>("BundleId")
+                        .HasColumnType("uuid")
+                        .HasColumnName("bundle_id");
+
+                    b.Property<Instant>("CreatedAt")
+                        .HasColumnType("timestamp with time zone")
+                        .HasColumnName("created_at");
+
+                    b.Property<Instant?>("DeletedAt")
+                        .HasColumnType("timestamp with time zone")
+                        .HasColumnName("deleted_at");
+
+                    b.Property<string>("Description")
+                        .HasMaxLength(4096)
+                        .HasColumnType("character varying(4096)")
+                        .HasColumnName("description");
+
+                    b.Property<Instant?>("ExpiredAt")
+                        .HasColumnType("timestamp with time zone")
+                        .HasColumnName("expired_at");
+
+                    b.Property<Dictionary<string, object>>("FileMeta")
+                        .HasColumnType("jsonb")
+                        .HasColumnName("file_meta");
+
+                    b.Property<bool>("HasCompression")
+                        .HasColumnType("boolean")
+                        .HasColumnName("has_compression");
+
+                    b.Property<bool>("HasThumbnail")
+                        .HasColumnType("boolean")
+                        .HasColumnName("has_thumbnail");
+
+                    b.Property<string>("Hash")
+                        .HasMaxLength(256)
+                        .HasColumnType("character varying(256)")
+                        .HasColumnName("hash");
+
+                    b.Property<bool>("IsEncrypted")
+                        .HasColumnType("boolean")
+                        .HasColumnName("is_encrypted");
+
+                    b.Property<bool>("IsMarkedRecycle")
+                        .HasColumnType("boolean")
+                        .HasColumnName("is_marked_recycle");
+
+                    b.Property<string>("MimeType")
+                        .HasMaxLength(256)
+                        .HasColumnType("character varying(256)")
+                        .HasColumnName("mime_type");
+
+                    b.Property<string>("Name")
+                        .IsRequired()
+                        .HasMaxLength(1024)
+                        .HasColumnType("character varying(1024)")
+                        .HasColumnName("name");
+
+                    b.Property<Guid?>("PoolId")
+                        .HasColumnType("uuid")
+                        .HasColumnName("pool_id");
+
+                    b.Property<List<ContentSensitiveMark>>("SensitiveMarks")
+                        .HasColumnType("jsonb")
+                        .HasColumnName("sensitive_marks");
+
+                    b.Property<long>("Size")
+                        .HasColumnType("bigint")
+                        .HasColumnName("size");
+
+                    b.Property<string>("StorageId")
+                        .HasMaxLength(32)
+                        .HasColumnType("character varying(32)")
+                        .HasColumnName("storage_id");
+
+                    b.Property<string>("StorageUrl")
+                        .HasMaxLength(4096)
+                        .HasColumnType("character varying(4096)")
+                        .HasColumnName("storage_url");
+
+                    b.Property<Instant>("UpdatedAt")
+                        .HasColumnType("timestamp with time zone")
+                        .HasColumnName("updated_at");
+
+                    b.Property<Instant?>("UploadedAt")
+                        .HasColumnType("timestamp with time zone")
+                        .HasColumnName("uploaded_at");
+
+                    b.Property<Dictionary<string, object>>("UserMeta")
+                        .HasColumnType("jsonb")
+                        .HasColumnName("user_meta");
+
+                    b.HasKey("Id")
+                        .HasName("pk_files");
+
+                    b.HasIndex("BundleId")
+                        .HasDatabaseName("ix_files_bundle_id");
+
+                    b.HasIndex("PoolId")
+                        .HasDatabaseName("ix_files_pool_id");
+
+                    b.ToTable("files", (string)null);
+                });
+
+            modelBuilder.Entity("DysonNetwork.Shared.Models.SnCloudFileIndex", b =>
+                {
+                    b.Property<Guid>("Id")
+                        .ValueGeneratedOnAdd()
+                        .HasColumnType("uuid")
+                        .HasColumnName("id");
+
+                    b.Property<Guid>("AccountId")
+                        .HasColumnType("uuid")
+                        .HasColumnName("account_id");
+
+                    b.Property<Instant>("CreatedAt")
+                        .HasColumnType("timestamp with time zone")
+                        .HasColumnName("created_at");
+
+                    b.Property<Instant?>("DeletedAt")
+                        .HasColumnType("timestamp with time zone")
+                        .HasColumnName("deleted_at");
+
+                    b.Property<string>("FileId")
+                        .IsRequired()
+                        .HasMaxLength(32)
+                        .HasColumnType("character varying(32)")
+                        .HasColumnName("file_id");
+
+                    b.Property<string>("Path")
+                        .IsRequired()
+                        .HasMaxLength(8192)
+                        .HasColumnType("character varying(8192)")
+                        .HasColumnName("path");
+
+                    b.Property<Instant>("UpdatedAt")
+                        .HasColumnType("timestamp with time zone")
+                        .HasColumnName("updated_at");
+
+                    b.HasKey("Id")
+                        .HasName("pk_file_indexes");
+
+                    b.HasIndex("FileId")
+                        .HasDatabaseName("ix_file_indexes_file_id");
+
+                    b.HasIndex("Path", "AccountId")
+                        .HasDatabaseName("ix_file_indexes_path_account_id");
+
+                    b.ToTable("file_indexes", (string)null);
+                });
+
+            modelBuilder.Entity("DysonNetwork.Shared.Models.SnFileBundle", b =>
+                {
+                    b.Property<Guid>("Id")
+                        .ValueGeneratedOnAdd()
+                        .HasColumnType("uuid")
+                        .HasColumnName("id");
+
+                    b.Property<Guid>("AccountId")
+                        .HasColumnType("uuid")
+                        .HasColumnName("account_id");
+
+                    b.Property<Instant>("CreatedAt")
+                        .HasColumnType("timestamp with time zone")
+                        .HasColumnName("created_at");
+
+                    b.Property<Instant?>("DeletedAt")
+                        .HasColumnType("timestamp with time zone")
+                        .HasColumnName("deleted_at");
+
+                    b.Property<string>("Description")
+                        .HasMaxLength(8192)
+                        .HasColumnType("character varying(8192)")
+                        .HasColumnName("description");
+
+                    b.Property<Instant?>("ExpiredAt")
+                        .HasColumnType("timestamp with time zone")
+                        .HasColumnName("expired_at");
+
+                    b.Property<string>("Name")
+                        .IsRequired()
+                        .HasMaxLength(1024)
+                        .HasColumnType("character varying(1024)")
+                        .HasColumnName("name");
+
+                    b.Property<string>("Passcode")
+                        .HasMaxLength(256)
+                        .HasColumnType("character varying(256)")
+                        .HasColumnName("passcode");
+
+                    b.Property<string>("Slug")
+                        .IsRequired()
+                        .HasMaxLength(1024)
+                        .HasColumnType("character varying(1024)")
+                        .HasColumnName("slug");
+
+                    b.Property<Instant>("UpdatedAt")
+                        .HasColumnType("timestamp with time zone")
+                        .HasColumnName("updated_at");
+
+                    b.HasKey("Id")
+                        .HasName("pk_bundles");
+
+                    b.HasIndex("Slug")
+                        .IsUnique()
+                        .HasDatabaseName("ix_bundles_slug");
+
+                    b.ToTable("bundles", (string)null);
+                });
+
+            modelBuilder.Entity("DysonNetwork.Drive.Storage.Model.PersistentUploadTask", b =>
+                {
+                    b.HasBaseType("DysonNetwork.Drive.Storage.Model.PersistentTask");
+
+                    b.Property<Guid?>("BundleId")
+                        .HasColumnType("uuid")
+                        .HasColumnName("bundle_id");
+
+                    b.Property<long>("ChunkSize")
+                        .HasColumnType("bigint")
+                        .HasColumnName("chunk_size");
+
+                    b.Property<int>("ChunksCount")
+                        .HasColumnType("integer")
+                        .HasColumnName("chunks_count");
+
+                    b.Property<int>("ChunksUploaded")
+                        .HasColumnType("integer")
+                        .HasColumnName("chunks_uploaded");
+
+                    b.Property<string>("ContentType")
+                        .IsRequired()
+                        .HasMaxLength(128)
+                        .HasColumnType("character varying(128)")
+                        .HasColumnName("content_type");
+
+                    b.Property<string>("EncryptPassword")
+                        .HasMaxLength(256)
+                        .HasColumnType("character varying(256)")
+                        .HasColumnName("encrypt_password");
+
+                    b.Property<string>("FileName")
+                        .IsRequired()
+                        .HasMaxLength(256)
+                        .HasColumnType("character varying(256)")
+                        .HasColumnName("file_name");
+
+                    b.Property<long>("FileSize")
+                        .HasColumnType("bigint")
+                        .HasColumnName("file_size");
+
+                    b.Property<string>("Hash")
+                        .IsRequired()
+                        .HasColumnType("text")
+                        .HasColumnName("hash");
+
+                    b.Property<string>("Path")
+                        .HasColumnType("text")
+                        .HasColumnName("path");
+
+                    b.Property<Guid>("PoolId")
+                        .HasColumnType("uuid")
+                        .HasColumnName("pool_id");
+
+                    b.PrimitiveCollection<List<int>>("UploadedChunks")
+                        .IsRequired()
+                        .HasColumnType("integer[]")
+                        .HasColumnName("uploaded_chunks");
+
+                    b.HasDiscriminator().HasValue("PersistentUploadTask");
+                });
+
+            modelBuilder.Entity("DysonNetwork.Shared.Models.CloudFileReference", b =>
+                {
+                    b.HasOne("DysonNetwork.Shared.Models.SnCloudFile", "File")
+                        .WithMany("References")
+                        .HasForeignKey("FileId")
+                        .OnDelete(DeleteBehavior.Cascade)
+                        .IsRequired()
+                        .HasConstraintName("fk_file_references_files_file_id");
+
+                    b.Navigation("File");
+                });
+
+            modelBuilder.Entity("DysonNetwork.Shared.Models.SnCloudFile", b =>
+                {
+                    b.HasOne("DysonNetwork.Shared.Models.SnFileBundle", "Bundle")
+                        .WithMany("Files")
+                        .HasForeignKey("BundleId")
+                        .HasConstraintName("fk_files_bundles_bundle_id");
+
+                    b.HasOne("DysonNetwork.Shared.Models.FilePool", "Pool")
+                        .WithMany()
+                        .HasForeignKey("PoolId")
+                        .HasConstraintName("fk_files_pools_pool_id");
+
+                    b.Navigation("Bundle");
+
+                    b.Navigation("Pool");
+                });
+
+            modelBuilder.Entity("DysonNetwork.Shared.Models.SnCloudFileIndex", b =>
+                {
+                    b.HasOne("DysonNetwork.Shared.Models.SnCloudFile", "File")
+                        .WithMany("FileIndexes")
+                        .HasForeignKey("FileId")
+                        .OnDelete(DeleteBehavior.Cascade)
+                        .IsRequired()
+                        .HasConstraintName("fk_file_indexes_files_file_id");
+
+                    b.Navigation("File");
+                });
+
+            modelBuilder.Entity("DysonNetwork.Shared.Models.SnCloudFile", b =>
+                {
+                    b.Navigation("FileIndexes");
+
+                    b.Navigation("References");
+                });
+
+            modelBuilder.Entity("DysonNetwork.Shared.Models.SnFileBundle", b =>
+                {
+                    b.Navigation("Files");
+                });
+#pragma warning restore 612, 618
+        }
+    }
+}

DysonNetwork.Drive/Migrations/20251112135535_AddFileIndex.cs

@@ -0,0 +1,66 @@
+using System;
+using Microsoft.EntityFrameworkCore.Migrations;
+using NodaTime;
+
+#nullable disable
+
+namespace DysonNetwork.Drive.Migrations
+{
+    /// <inheritdoc />
+    public partial class AddFileIndex : Migration
+    {
+        /// <inheritdoc />
+        protected override void Up(MigrationBuilder migrationBuilder)
+        {
+            migrationBuilder.AddColumn<string>(
+                name: "path",
+                table: "tasks",
+                type: "text",
+                nullable: true);
+
+            migrationBuilder.CreateTable(
+                name: "file_indexes",
+                columns: table => new
+                {
+                    id = table.Column<Guid>(type: "uuid", nullable: false),
+                    path = table.Column<string>(type: "character varying(8192)", maxLength: 8192, nullable: false),
+                    file_id = table.Column<string>(type: "character varying(32)", maxLength: 32, nullable: false),
+                    account_id = table.Column<Guid>(type: "uuid", nullable: false),
+                    created_at = table.Column<Instant>(type: "timestamp with time zone", nullable: false),
+                    updated_at = table.Column<Instant>(type: "timestamp with time zone", nullable: false),
+                    deleted_at = table.Column<Instant>(type: "timestamp with time zone", nullable: true)
+                },
+                constraints: table =>
+                {
+                    table.PrimaryKey("pk_file_indexes", x => x.id);
+                    table.ForeignKey(
+                        name: "fk_file_indexes_files_file_id",
+                        column: x => x.file_id,
+                        principalTable: "files",
+                        principalColumn: "id",
+                        onDelete: ReferentialAction.Cascade);
+                });
+
+            migrationBuilder.CreateIndex(
+                name: "ix_file_indexes_file_id",
+                table: "file_indexes",
+                column: "file_id");
+
+            migrationBuilder.CreateIndex(
+                name: "ix_file_indexes_path_account_id",
+                table: "file_indexes",
+                columns: new[] { "path", "account_id" });
+        }
+
+        /// <inheritdoc />
+        protected override void Down(MigrationBuilder migrationBuilder)
+        {
+            migrationBuilder.DropTable(
+                name: "file_indexes");
+
+            migrationBuilder.DropColumn(
+                name: "path",
+                table: "tasks");
+        }
+    }
+}

DysonNetwork.Drive/Migrations/AppDatabaseModelSnapshot.cs

@@ -2,8 +2,7 @@
 using System;
 using System.Collections.Generic;
 using DysonNetwork.Drive;
-using DysonNetwork.Drive.Storage;
-using DysonNetwork.Shared.Data;
+using DysonNetwork.Shared.Models;
 using Microsoft.EntityFrameworkCore;
 using Microsoft.EntityFrameworkCore.Infrastructure;
 using Microsoft.EntityFrameworkCore.Storage.ValueConversion;
@@ -21,7 +20,7 @@ namespace DysonNetwork.Drive.Migrations
         {
 #pragma warning disable 612, 618
             modelBuilder
-                .HasAnnotation("ProductVersion", "9.0.7")
+                .HasAnnotation("ProductVersion", "9.0.10")
                 .HasAnnotation("Relational:MaxIdentifierLength", 63);
 
             NpgsqlModelBuilderExtensions.UseIdentityByDefaultColumns(modelBuilder);
@@ -73,7 +72,224 @@ namespace DysonNetwork.Drive.Migrations
                     b.ToTable("quota_records", (string)null);
                 });
 
-            modelBuilder.Entity("DysonNetwork.Drive.Storage.CloudFile", b =>
+            modelBuilder.Entity("DysonNetwork.Drive.Storage.Model.PersistentTask", b =>
+                {
+                    b.Property<Guid>("Id")
+                        .ValueGeneratedOnAdd()
+                        .HasColumnType("uuid")
+                        .HasColumnName("id");
+
+                    b.Property<Guid>("AccountId")
+                        .HasColumnType("uuid")
+                        .HasColumnName("account_id");
+
+                    b.Property<Instant?>("CompletedAt")
+                        .HasColumnType("timestamp with time zone")
+                        .HasColumnName("completed_at");
+
+                    b.Property<Instant>("CreatedAt")
+                        .HasColumnType("timestamp with time zone")
+                        .HasColumnName("created_at");
+
+                    b.Property<Instant?>("DeletedAt")
+                        .HasColumnType("timestamp with time zone")
+                        .HasColumnName("deleted_at");
+
+                    b.Property<string>("Description")
+                        .HasMaxLength(1024)
+                        .HasColumnType("character varying(1024)")
+                        .HasColumnName("description");
+
+                    b.Property<string>("Discriminator")
+                        .IsRequired()
+                        .HasMaxLength(21)
+                        .HasColumnType("character varying(21)")
+                        .HasColumnName("discriminator");
+
+                    b.Property<string>("ErrorMessage")
+                        .HasMaxLength(1024)
+                        .HasColumnType("character varying(1024)")
+                        .HasColumnName("error_message");
+
+                    b.Property<long?>("EstimatedDurationSeconds")
+                        .HasColumnType("bigint")
+                        .HasColumnName("estimated_duration_seconds");
+
+                    b.Property<Instant?>("ExpiredAt")
+                        .HasColumnType("timestamp with time zone")
+                        .HasColumnName("expired_at");
+
+                    b.Property<Instant>("LastActivity")
+                        .HasColumnType("timestamp with time zone")
+                        .HasColumnName("last_activity");
+
+                    b.Property<string>("Name")
+                        .IsRequired()
+                        .HasMaxLength(256)
+                        .HasColumnType("character varying(256)")
+                        .HasColumnName("name");
+
+                    b.Property<Dictionary<string, object>>("Parameters")
+                        .IsRequired()
+                        .HasColumnType("jsonb")
+                        .HasColumnName("parameters");
+
+                    b.Property<int>("Priority")
+                        .HasColumnType("integer")
+                        .HasColumnName("priority");
+
+                    b.Property<double>("Progress")
+                        .HasColumnType("double precision")
+                        .HasColumnName("progress");
+
+                    b.Property<Dictionary<string, object>>("Results")
+                        .IsRequired()
+                        .HasColumnType("jsonb")
+                        .HasColumnName("results");
+
+                    b.Property<Instant?>("StartedAt")
+                        .HasColumnType("timestamp with time zone")
+                        .HasColumnName("started_at");
+
+                    b.Property<int>("Status")
+                        .HasColumnType("integer")
+                        .HasColumnName("status");
+
+                    b.Property<string>("TaskId")
+                        .IsRequired()
+                        .HasMaxLength(64)
+                        .HasColumnType("character varying(64)")
+                        .HasColumnName("task_id");
+
+                    b.Property<int>("Type")
+                        .HasColumnType("integer")
+                        .HasColumnName("type");
+
+                    b.Property<Instant>("UpdatedAt")
+                        .HasColumnType("timestamp with time zone")
+                        .HasColumnName("updated_at");
+
+                    b.HasKey("Id")
+                        .HasName("pk_tasks");
+
+                    b.ToTable("tasks", (string)null);
+
+                    b.HasDiscriminator().HasValue("PersistentTask");
+
+                    b.UseTphMappingStrategy();
+                });
+
+            modelBuilder.Entity("DysonNetwork.Shared.Models.SnCloudFileReference", b =>
+                {
+                    b.Property<Guid>("Id")
+                        .ValueGeneratedOnAdd()
+                        .HasColumnType("uuid")
+                        .HasColumnName("id");
+
+                    b.Property<Instant>("CreatedAt")
+                        .HasColumnType("timestamp with time zone")
+                        .HasColumnName("created_at");
+
+                    b.Property<Instant?>("DeletedAt")
+                        .HasColumnType("timestamp with time zone")
+                        .HasColumnName("deleted_at");
+
+                    b.Property<Instant?>("ExpiredAt")
+                        .HasColumnType("timestamp with time zone")
+                        .HasColumnName("expired_at");
+
+                    b.Property<string>("FileId")
+                        .IsRequired()
+                        .HasMaxLength(32)
+                        .HasColumnType("character varying(32)")
+                        .HasColumnName("file_id");
+
+                    b.Property<string>("ResourceId")
+                        .IsRequired()
+                        .HasMaxLength(1024)
+                        .HasColumnType("character varying(1024)")
+                        .HasColumnName("resource_id");
+
+                    b.Property<Instant>("UpdatedAt")
+                        .HasColumnType("timestamp with time zone")
+                        .HasColumnName("updated_at");
+
+                    b.Property<string>("Usage")
+                        .IsRequired()
+                        .HasMaxLength(1024)
+                        .HasColumnType("character varying(1024)")
+                        .HasColumnName("usage");
+
+                    b.HasKey("Id")
+                        .HasName("pk_file_references");
+
+                    b.HasIndex("FileId")
+                        .HasDatabaseName("ix_file_references_file_id");
+
+                    b.ToTable("file_references", (string)null);
+                });
+
+            modelBuilder.Entity("DysonNetwork.Shared.Models.FilePool", b =>
+                {
+                    b.Property<Guid>("Id")
+                        .ValueGeneratedOnAdd()
+                        .HasColumnType("uuid")
+                        .HasColumnName("id");
+
+                    b.Property<Guid?>("AccountId")
+                        .HasColumnType("uuid")
+                        .HasColumnName("account_id");
+
+                    b.Property<BillingConfig>("BillingConfig")
+                        .IsRequired()
+                        .HasColumnType("jsonb")
+                        .HasColumnName("billing_config");
+
+                    b.Property<Instant>("CreatedAt")
+                        .HasColumnType("timestamp with time zone")
+                        .HasColumnName("created_at");
+
+                    b.Property<Instant?>("DeletedAt")
+                        .HasColumnType("timestamp with time zone")
+                        .HasColumnName("deleted_at");
+
+                    b.Property<string>("Description")
+                        .IsRequired()
+                        .HasMaxLength(8192)
+                        .HasColumnType("character varying(8192)")
+                        .HasColumnName("description");
+
+                    b.Property<bool>("IsHidden")
+                        .HasColumnType("boolean")
+                        .HasColumnName("is_hidden");
+
+                    b.Property<string>("Name")
+                        .IsRequired()
+                        .HasMaxLength(1024)
+                        .HasColumnType("character varying(1024)")
+                        .HasColumnName("name");
+
+                    b.Property<PolicyConfig>("PolicyConfig")
+                        .IsRequired()
+                        .HasColumnType("jsonb")
+                        .HasColumnName("policy_config");
+
+                    b.Property<RemoteStorageConfig>("StorageConfig")
+                        .IsRequired()
+                        .HasColumnType("jsonb")
+                        .HasColumnName("storage_config");
+
+                    b.Property<Instant>("UpdatedAt")
+                        .HasColumnType("timestamp with time zone")
+                        .HasColumnName("updated_at");
+
+                    b.HasKey("Id")
+                        .HasName("pk_pools");
+
+                    b.ToTable("pools", (string)null);
+                });
+
+            modelBuilder.Entity("DysonNetwork.Shared.Models.SnCloudFile", b =>
                 {
                     b.Property<string>("Id")
                         .HasMaxLength(32)
@@ -187,13 +403,17 @@ namespace DysonNetwork.Drive.Migrations
                     b.ToTable("files", (string)null);
                 });
 
-            modelBuilder.Entity("DysonNetwork.Drive.Storage.CloudFileReference", b =>
+            modelBuilder.Entity("DysonNetwork.Shared.Models.SnCloudFileIndex", b =>
                 {
                     b.Property<Guid>("Id")
                         .ValueGeneratedOnAdd()
                         .HasColumnType("uuid")
                         .HasColumnName("id");
 
+                    b.Property<Guid>("AccountId")
+                        .HasColumnType("uuid")
+                        .HasColumnName("account_id");
+
                     b.Property<Instant>("CreatedAt")
                         .HasColumnType("timestamp with time zone")
                         .HasColumnName("created_at");
@@ -202,42 +422,35 @@ namespace DysonNetwork.Drive.Migrations
                         .HasColumnType("timestamp with time zone")
                         .HasColumnName("deleted_at");
 
-                    b.Property<Instant?>("ExpiredAt")
-                        .HasColumnType("timestamp with time zone")
-                        .HasColumnName("expired_at");
-
                     b.Property<string>("FileId")
                         .IsRequired()
                         .HasMaxLength(32)
                         .HasColumnType("character varying(32)")
                         .HasColumnName("file_id");
 
-                    b.Property<string>("ResourceId")
+                    b.Property<string>("Path")
                         .IsRequired()
-                        .HasMaxLength(1024)
-                        .HasColumnType("character varying(1024)")
-                        .HasColumnName("resource_id");
+                        .HasMaxLength(8192)
+                        .HasColumnType("character varying(8192)")
+                        .HasColumnName("path");
 
                     b.Property<Instant>("UpdatedAt")
                         .HasColumnType("timestamp with time zone")
                         .HasColumnName("updated_at");
 
-                    b.Property<string>("Usage")
-                        .IsRequired()
-                        .HasMaxLength(1024)
-                        .HasColumnType("character varying(1024)")
-                        .HasColumnName("usage");
-
                     b.HasKey("Id")
-                        .HasName("pk_file_references");
+                        .HasName("pk_file_indexes");
 
                     b.HasIndex("FileId")
-                        .HasDatabaseName("ix_file_references_file_id");
+                        .HasDatabaseName("ix_file_indexes_file_id");
 
-                    b.ToTable("file_references", (string)null);
+                    b.HasIndex("Path", "AccountId")
+                        .HasDatabaseName("ix_file_indexes_path_account_id");
+
+                    b.ToTable("file_indexes", (string)null);
                 });
 
-            modelBuilder.Entity("DysonNetwork.Drive.Storage.FileBundle", b =>
+            modelBuilder.Entity("DysonNetwork.Shared.Models.SnFileBundle", b =>
                 {
                     b.Property<Guid>("Id")
                         .ValueGeneratedOnAdd()
@@ -296,86 +509,71 @@ namespace DysonNetwork.Drive.Migrations
                     b.ToTable("bundles", (string)null);
                 });
 
-            modelBuilder.Entity("DysonNetwork.Drive.Storage.FilePool", b =>
+            modelBuilder.Entity("DysonNetwork.Drive.Storage.Model.PersistentUploadTask", b =>
                 {
-                    b.Property<Guid>("Id")
-                        .ValueGeneratedOnAdd()
+                    b.HasBaseType("DysonNetwork.Drive.Storage.Model.PersistentTask");
+
+                    b.Property<Guid?>("BundleId")
                         .HasColumnType("uuid")
-                        .HasColumnName("id");
+                        .HasColumnName("bundle_id");
 
-                    b.Property<Guid?>("AccountId")
+                    b.Property<long>("ChunkSize")
+                        .HasColumnType("bigint")
+                        .HasColumnName("chunk_size");
+
+                    b.Property<int>("ChunksCount")
+                        .HasColumnType("integer")
+                        .HasColumnName("chunks_count");
+
+                    b.Property<int>("ChunksUploaded")
+                        .HasColumnType("integer")
+                        .HasColumnName("chunks_uploaded");
+
+                    b.Property<string>("ContentType")
+                        .IsRequired()
+                        .HasMaxLength(128)
+                        .HasColumnType("character varying(128)")
+                        .HasColumnName("content_type");
+
+                    b.Property<string>("EncryptPassword")
+                        .HasMaxLength(256)
+                        .HasColumnType("character varying(256)")
+                        .HasColumnName("encrypt_password");
+
+                    b.Property<string>("FileName")
+                        .IsRequired()
+                        .HasMaxLength(256)
+                        .HasColumnType("character varying(256)")
+                        .HasColumnName("file_name");
+
+                    b.Property<long>("FileSize")
+                        .HasColumnType("bigint")
+                        .HasColumnName("file_size");
+
+                    b.Property<string>("Hash")
+                        .IsRequired()
+                        .HasColumnType("text")
+                        .HasColumnName("hash");
+
+                    b.Property<string>("Path")
+                        .HasColumnType("text")
+                        .HasColumnName("path");
+
+                    b.Property<Guid>("PoolId")
                         .HasColumnType("uuid")
-                        .HasColumnName("account_id");
+                        .HasColumnName("pool_id");
 
-                    b.Property<BillingConfig>("BillingConfig")
+                    b.PrimitiveCollection<List<int>>("UploadedChunks")
                         .IsRequired()
-                        .HasColumnType("jsonb")
-                        .HasColumnName("billing_config");
+                        .HasColumnType("integer[]")
+                        .HasColumnName("uploaded_chunks");
 
-                    b.Property<Instant>("CreatedAt")
-                        .HasColumnType("timestamp with time zone")
-                        .HasColumnName("created_at");
-
-                    b.Property<Instant?>("DeletedAt")
-                        .HasColumnType("timestamp with time zone")
-                        .HasColumnName("deleted_at");
-
-                    b.Property<string>("Description")
-                        .IsRequired()
-                        .HasMaxLength(8192)
-                        .HasColumnType("character varying(8192)")
-                        .HasColumnName("description");
-
-                    b.Property<bool>("IsHidden")
-                        .HasColumnType("boolean")
-                        .HasColumnName("is_hidden");
-
-                    b.Property<string>("Name")
-                        .IsRequired()
-                        .HasMaxLength(1024)
-                        .HasColumnType("character varying(1024)")
-                        .HasColumnName("name");
-
-                    b.Property<PolicyConfig>("PolicyConfig")
-                        .IsRequired()
-                        .HasColumnType("jsonb")
-                        .HasColumnName("policy_config");
-
-                    b.Property<RemoteStorageConfig>("StorageConfig")
-                        .IsRequired()
-                        .HasColumnType("jsonb")
-                        .HasColumnName("storage_config");
-
-                    b.Property<Instant>("UpdatedAt")
-                        .HasColumnType("timestamp with time zone")
-                        .HasColumnName("updated_at");
-
-                    b.HasKey("Id")
-                        .HasName("pk_pools");
-
-                    b.ToTable("pools", (string)null);
+                    b.HasDiscriminator().HasValue("PersistentUploadTask");
                 });
 
-            modelBuilder.Entity("DysonNetwork.Drive.Storage.CloudFile", b =>
+            modelBuilder.Entity("DysonNetwork.Shared.Models.SnCloudFileReference", b =>
                 {
-                    b.HasOne("DysonNetwork.Drive.Storage.FileBundle", "Bundle")
-                        .WithMany("Files")
-                        .HasForeignKey("BundleId")
-                        .HasConstraintName("fk_files_bundles_bundle_id");
-
-                    b.HasOne("DysonNetwork.Drive.Storage.FilePool", "Pool")
-                        .WithMany()
-                        .HasForeignKey("PoolId")
-                        .HasConstraintName("fk_files_pools_pool_id");
-
-                    b.Navigation("Bundle");
-
-                    b.Navigation("Pool");
-                });
-
-            modelBuilder.Entity("DysonNetwork.Drive.Storage.CloudFileReference", b =>
-                {
-                    b.HasOne("DysonNetwork.Drive.Storage.CloudFile", "File")
+                    b.HasOne("DysonNetwork.Shared.Models.SnCloudFile", "File")
                         .WithMany("References")
                         .HasForeignKey("FileId")
                         .OnDelete(DeleteBehavior.Cascade)
@@ -385,12 +583,43 @@ namespace DysonNetwork.Drive.Migrations
                     b.Navigation("File");
                 });
 
-            modelBuilder.Entity("DysonNetwork.Drive.Storage.CloudFile", b =>
+            modelBuilder.Entity("DysonNetwork.Shared.Models.SnCloudFile", b =>
                 {
+                    b.HasOne("DysonNetwork.Shared.Models.SnFileBundle", "Bundle")
+                        .WithMany("Files")
+                        .HasForeignKey("BundleId")
+                        .HasConstraintName("fk_files_bundles_bundle_id");
+
+                    b.HasOne("DysonNetwork.Shared.Models.FilePool", "Pool")
+                        .WithMany()
+                        .HasForeignKey("PoolId")
+                        .HasConstraintName("fk_files_pools_pool_id");
+
+                    b.Navigation("Bundle");
+
+                    b.Navigation("Pool");
+                });
+
+            modelBuilder.Entity("DysonNetwork.Shared.Models.SnCloudFileIndex", b =>
+                {
+                    b.HasOne("DysonNetwork.Shared.Models.SnCloudFile", "File")
+                        .WithMany("FileIndexes")
+                        .HasForeignKey("FileId")
+                        .OnDelete(DeleteBehavior.Cascade)
+                        .IsRequired()
+                        .HasConstraintName("fk_file_indexes_files_file_id");
+
+                    b.Navigation("File");
+                });
+
+            modelBuilder.Entity("DysonNetwork.Shared.Models.SnCloudFile", b =>
+                {
+                    b.Navigation("FileIndexes");
+
                     b.Navigation("References");
                 });
 
-            modelBuilder.Entity("DysonNetwork.Drive.Storage.FileBundle", b =>
+            modelBuilder.Entity("DysonNetwork.Shared.Models.SnFileBundle", b =>
                 {
                     b.Navigation("Files");
                 });

DysonNetwork.Drive/Program.cs

@@ -4,7 +4,6 @@ using DysonNetwork.Shared.Auth;
 using DysonNetwork.Shared.Http;
 using DysonNetwork.Shared.Registry;
 using Microsoft.EntityFrameworkCore;
-using tusdotnet.Stores;
 
 var builder = WebApplication.CreateBuilder(args);
 
@@ -16,23 +15,20 @@ builder.ConfigureAppKestrel(builder.Configuration, maxRequestBodySize: long.MaxV
 // Add application services
 
 builder.Services.AddAppServices(builder.Configuration);
-builder.Services.AddAppRateLimiting();
 builder.Services.AddAppAuthentication();
-builder.Services.AddAppSwagger();
 builder.Services.AddDysonAuth();
+builder.Services.AddRingService();
 builder.Services.AddAccountService();
 
-builder.Services.AddAppFileStorage(builder.Configuration);
-
-// Add flush handlers and websocket handlers
 builder.Services.AddAppFlushHandlers();
-
-// Add business services
 builder.Services.AddAppBusinessServices();
-
-// Add scheduled jobs
 builder.Services.AddAppScheduledJobs();
 
+builder.AddSwaggerManifest(
+    "DysonNetwork.Drive",
+    "The file upload and storage service in the Solar Network."
+);
+
 var app = builder.Build();
 
 app.MapDefaultEndpoints();
@@ -44,10 +40,11 @@ using (var scope = app.Services.CreateScope())
     await db.Database.MigrateAsync();
 }
 
-var tusDiskStore = app.Services.GetRequiredService<TusDiskStore>();
-app.ConfigureAppMiddleware(tusDiskStore);
+app.ConfigureAppMiddleware();
 
 // Configure gRPC
 app.ConfigureGrpcServices();
 
+app.UseSwaggerManifest("DysonNetwork.Drive");
+
 app.Run();

DysonNetwork.Drive/Properties/launchSettings.json

@@ -5,7 +5,6 @@
       "commandName": "Project",
       "dotnetRunMessages": true,
       "launchBrowser": false,
-      "applicationUrl": "http://localhost:5090",
       "environmentVariables": {
         "ASPNETCORE_ENVIRONMENT": "Development"
       }
@@ -14,7 +13,6 @@
       "commandName": "Project",
       "dotnetRunMessages": true,
       "launchBrowser": false,
-      "applicationUrl": "https://localhost:7092;http://localhost:5090",
       "environmentVariables": {
         "ASPNETCORE_ENVIRONMENT": "Development"
       }

DysonNetwork.Drive/Startup/ApplicationBuilderExtensions.cs

@@ -1,25 +1,14 @@
 using DysonNetwork.Drive.Storage;
-using tusdotnet;
-using tusdotnet.Interfaces;
 
 namespace DysonNetwork.Drive.Startup;
 
 public static class ApplicationBuilderExtensions
 {
-    public static WebApplication ConfigureAppMiddleware(this WebApplication app, ITusStore tusStore)
+    public static WebApplication ConfigureAppMiddleware(this WebApplication app)
     {
-        // Configure the HTTP request pipeline.
-        if (app.Environment.IsDevelopment())
-        {
-            app.UseSwagger();
-            app.UseSwaggerUI();
-        }
-
         app.UseAuthorization();
         app.MapControllers();
 
-        app.MapTus("/api/tus", _ => Task.FromResult(TusService.BuildConfiguration(tusStore, app.Configuration)));
-
         return app;
     }
 
@@ -28,6 +17,7 @@ public static class ApplicationBuilderExtensions
         // Map your gRPC services here
         app.MapGrpcService<FileServiceGrpc>();
         app.MapGrpcService<FileReferenceServiceGrpc>();
+        app.MapGrpcReflectionService();
 
         return app;
     }

DysonNetwork.Drive/Startup/BroadcastEventHandler.cs

@@ -1,10 +1,18 @@
 using System.Text.Json;
 using DysonNetwork.Drive.Storage;
-using DysonNetwork.Shared.Stream;
+using DysonNetwork.Drive.Storage.Model;
+using DysonNetwork.Shared.Models;
+using DysonNetwork.Shared.Proto;
+using DysonNetwork.Shared.Queue;
+using FFMpegCore;
 using Microsoft.EntityFrameworkCore;
 using NATS.Client.Core;
+using NATS.Client.JetStream;
 using NATS.Client.JetStream.Models;
 using NATS.Net;
+using NetVips;
+using NodaTime;
+using FileService = DysonNetwork.Drive.Storage.FileService;
 
 namespace DysonNetwork.Drive.Startup;
 
@@ -14,20 +22,74 @@ public class BroadcastEventHandler(
     IServiceProvider serviceProvider
 ) : BackgroundService
 {
+    private const string TempFileSuffix = "dypart";
+
+    private static readonly string[] AnimatedImageTypes =
+        ["image/gif", "image/apng", "image/avif"];
+
+    private static readonly string[] AnimatedImageExtensions =
+        [".gif", ".apng", ".avif"];
+
+
     protected override async Task ExecuteAsync(CancellationToken stoppingToken)
     {
         var js = nats.CreateJetStreamContext();
 
         await js.EnsureStreamCreated("account_events", [AccountDeletedEvent.Type]);
 
-        var consumer = await js.CreateOrUpdateConsumerAsync("account_events",
+        var accountEventConsumer = await js.CreateOrUpdateConsumerAsync("account_events",
             new ConsumerConfig("drive_account_deleted_handler"), cancellationToken: stoppingToken);
 
+        await js.EnsureStreamCreated("file_events", [FileUploadedEvent.Type]);
+        var fileUploadedConsumer = await js.CreateOrUpdateConsumerAsync("file_events",
+            new ConsumerConfig("drive_file_uploaded_handler") { MaxDeliver = 3 }, cancellationToken: stoppingToken);
+
+        var accountDeletedTask = HandleAccountDeleted(accountEventConsumer, stoppingToken);
+        var fileUploadedTask = HandleFileUploaded(fileUploadedConsumer, stoppingToken);
+
+        await Task.WhenAll(accountDeletedTask, fileUploadedTask);
+    }
+
+    private async Task HandleFileUploaded(INatsJSConsumer consumer, CancellationToken stoppingToken)
+    {
+        await foreach (var msg in consumer.ConsumeAsync<byte[]>(cancellationToken: stoppingToken))
+        {
+            var payload =
+                JsonSerializer.Deserialize<FileUploadedEventPayload>(msg.Data, GrpcTypeHelper.SerializerOptions);
+            if (payload == null)
+            {
+                await msg.AckAsync(cancellationToken: stoppingToken);
+                continue;
+            }
+
+            try
+            {
+                await ProcessAndUploadInBackgroundAsync(
+                    payload.FileId,
+                    payload.RemoteId,
+                    payload.StorageId,
+                    payload.ContentType,
+                    payload.ProcessingFilePath,
+                    payload.IsTempFile
+                );
+
+                await msg.AckAsync(cancellationToken: stoppingToken);
+            }
+            catch (Exception ex)
+            {
+                logger.LogError(ex, "Error processing FileUploadedEvent for file {FileId}", payload.FileId);
+                await msg.NakAsync(cancellationToken: stoppingToken, delay: TimeSpan.FromSeconds(60));
+            }
+        }
+    }
+
+    private async Task HandleAccountDeleted(INatsJSConsumer consumer, CancellationToken stoppingToken)
+    {
         await foreach (var msg in consumer.ConsumeAsync<byte[]>(cancellationToken: stoppingToken))
         {
             try
             {
-                var evt = JsonSerializer.Deserialize<AccountDeletedEvent>(msg.Data);
+                var evt = JsonSerializer.Deserialize<AccountDeletedEvent>(msg.Data, GrpcTypeHelper.SerializerOptions);
                 if (evt == null)
                 {
                     await msg.AckAsync(cancellationToken: stoppingToken);
@@ -69,4 +131,221 @@ public class BroadcastEventHandler(
             }
         }
     }
+
+    private async Task ProcessAndUploadInBackgroundAsync(
+        string fileId,
+        Guid remoteId,
+        string storageId,
+        string contentType,
+        string processingFilePath,
+        bool isTempFile
+    )
+    {
+        using var scope = serviceProvider.CreateScope();
+        var fs = scope.ServiceProvider.GetRequiredService<FileService>();
+        var scopedDb = scope.ServiceProvider.GetRequiredService<AppDatabase>();
+        var persistentTaskService = scope.ServiceProvider.GetRequiredService<PersistentTaskService>();
+
+        var pool = await fs.GetPoolAsync(remoteId);
+        if (pool is null) return;
+
+        var uploads = new List<(string FilePath, string Suffix, string ContentType, bool SelfDestruct)>();
+        var newMimeType = contentType;
+        var hasCompression = false;
+        var hasThumbnail = false;
+
+        logger.LogInformation("Processing file {FileId} in background...", fileId);
+
+        var fileToUpdate = await scopedDb.Files.AsNoTracking().FirstAsync(f => f.Id == fileId);
+
+        // Find the upload task associated with this file
+        var uploadTask = await scopedDb.Tasks
+            .OfType<PersistentUploadTask>()
+            .FirstOrDefaultAsync(t => t.FileName == fileToUpdate.Name && t.FileSize == fileToUpdate.Size);
+
+        if (fileToUpdate.IsEncrypted)
+        {
+            uploads.Add((processingFilePath, string.Empty, contentType, false));
+        }
+        else if (!pool.PolicyConfig.NoOptimization)
+        {
+            var fileExtension = Path.GetExtension(processingFilePath);
+            switch (contentType.Split('/')[0])
+            {
+                case "image":
+                    if (AnimatedImageTypes.Contains(contentType) || AnimatedImageExtensions.Contains(fileExtension))
+                    {
+                        logger.LogInformation("Skip optimize file {FileId} due to it is animated...", fileId);
+                        uploads.Add((processingFilePath, string.Empty, contentType, false));
+                        break;
+                    }
+
+                    try
+                    {
+                        newMimeType = "image/webp";
+                        using var vipsImage = Image.NewFromFile(processingFilePath);
+                        var imageToWrite = vipsImage;
+
+                        if (vipsImage.Interpretation is Enums.Interpretation.Scrgb or Enums.Interpretation.Xyz)
+                        {
+                            imageToWrite = vipsImage.Colourspace(Enums.Interpretation.Srgb);
+                        }
+
+                        var webpPath = Path.Join(Path.GetTempPath(), $"{fileId}.{TempFileSuffix}.webp");
+                        imageToWrite.Autorot().WriteToFile(webpPath,
+                            new VOption { { "lossless", true }, { "strip", true } });
+                        uploads.Add((webpPath, string.Empty, newMimeType, true));
+
+                        if (imageToWrite.Width * imageToWrite.Height >= 1024 * 1024)
+                        {
+                            var scale = 1024.0 / Math.Max(imageToWrite.Width, imageToWrite.Height);
+                            var compressedPath =
+                                Path.Join(Path.GetTempPath(), $"{fileId}.{TempFileSuffix}.compressed.webp");
+                            using var compressedImage = imageToWrite.Resize(scale);
+                            compressedImage.Autorot().WriteToFile(compressedPath,
+                                new VOption { { "Q", 80 }, { "strip", true } });
+                            uploads.Add((compressedPath, ".compressed", newMimeType, true));
+                            hasCompression = true;
+                        }
+
+                        if (!ReferenceEquals(imageToWrite, vipsImage))
+                        {
+                            imageToWrite.Dispose();
+                        }
+                    }
+                    catch (Exception ex)
+                    {
+                        logger.LogError(ex, "Failed to optimize image {FileId}, uploading original", fileId);
+                        uploads.Add((processingFilePath, string.Empty, contentType, false));
+                        newMimeType = contentType;
+                    }
+
+                    break;
+
+                case "video":
+                    uploads.Add((processingFilePath, string.Empty, contentType, false));
+
+                    var thumbnailPath = Path.Join(Path.GetTempPath(), $"{fileId}.{TempFileSuffix}.thumbnail.jpg");
+                    try
+                    {
+                        await FFMpegArguments
+                            .FromFileInput(processingFilePath, verifyExists: true)
+                            .OutputToFile(thumbnailPath, overwrite: true, options => options
+                                .Seek(TimeSpan.FromSeconds(0))
+                                .WithFrameOutputCount(1)
+                                .WithCustomArgument("-q:v 2")
+                            )
+                            .NotifyOnOutput(line => logger.LogInformation("[FFmpeg] {Line}", line))
+                            .NotifyOnError(line => logger.LogWarning("[FFmpeg] {Line}", line))
+                            .ProcessAsynchronously();
+
+                        if (File.Exists(thumbnailPath))
+                        {
+                            uploads.Add((thumbnailPath, ".thumbnail", "image/jpeg", true));
+                            hasThumbnail = true;
+                        }
+                        else
+                        {
+                            logger.LogWarning("FFMpeg did not produce thumbnail for video {FileId}", fileId);
+                        }
+                    }
+                    catch (Exception ex)
+                    {
+                        logger.LogError(ex, "Failed to generate thumbnail for video {FileId}", fileId);
+                    }
+
+                    break;
+
+                default:
+                    uploads.Add((processingFilePath, string.Empty, contentType, false));
+                    break;
+            }
+        }
+        else
+        {
+            uploads.Add((processingFilePath, string.Empty, contentType, false));
+        }
+
+        logger.LogInformation("Optimized file {FileId}, now uploading...", fileId);
+
+        if (uploads.Count > 0)
+        {
+            var destPool = remoteId;
+            var uploadTasks = uploads.Select(item =>
+                fs.UploadFileToRemoteAsync(
+                    storageId,
+                    destPool,
+                    item.FilePath,
+                    item.Suffix,
+                    item.ContentType,
+                    item.SelfDestruct
+                )
+            ).ToList();
+
+            await Task.WhenAll(uploadTasks);
+
+            logger.LogInformation("Uploaded file {FileId} done!", fileId);
+
+            var now = SystemClock.Instance.GetCurrentInstant();
+            await scopedDb.Files.Where(f => f.Id == fileId).ExecuteUpdateAsync(setter => setter
+                .SetProperty(f => f.UploadedAt, now)
+                .SetProperty(f => f.PoolId, destPool)
+                .SetProperty(f => f.MimeType, newMimeType)
+                .SetProperty(f => f.HasCompression, hasCompression)
+                .SetProperty(f => f.HasThumbnail, hasThumbnail)
+            );
+
+            // Only delete temp file after successful upload and db update
+            if (isTempFile)
+                File.Delete(processingFilePath);
+        }
+
+        await fs._PurgeCacheAsync(fileId);
+
+        // Complete the upload task if found
+        if (uploadTask != null)
+        {
+            await persistentTaskService.MarkTaskCompletedAsync(uploadTask.TaskId, new Dictionary<string, object?>
+            {
+                { "FileId", fileId },
+                { "FileName", fileToUpdate.Name },
+                { "FileInfo", fileToUpdate },
+                { "FileSize", fileToUpdate.Size },
+                { "MimeType", newMimeType },
+                { "HasCompression", hasCompression },
+                { "HasThumbnail", hasThumbnail }
+            });
+
+            // Send push notification for large files (>5MB) that took longer to process
+            if (fileToUpdate.Size > 5 * 1024 * 1024) // 5MB threshold
+                await SendLargeFileProcessingCompleteNotificationAsync(uploadTask, fileToUpdate);
+        }
+    }
+
+    private async Task SendLargeFileProcessingCompleteNotificationAsync(PersistentUploadTask task, SnCloudFile file)
+    {
+        try
+        {
+            var ringService = serviceProvider.GetRequiredService<RingService.RingServiceClient>();
+
+            var pushNotification = new PushNotification
+            {
+                Topic = "drive.tasks.upload",
+                Title = "File Processing Complete",
+                Subtitle = file.Name,
+                Body = $"Your file '{file.Name}' has finished processing and is now available.",
+                IsSavable = true
+            };
+
+            await ringService.SendPushNotificationToUserAsync(new SendPushNotificationToUserRequest
+            {
+                UserId = task.AccountId.ToString(),
+                Notification = pushNotification
+            });
+        }
+        catch (Exception ex)
+        {
+            logger.LogWarning(ex, "Failed to send large file processing notification for task {TaskId}", task.TaskId);
+        }
+    }
 }

DysonNetwork.Drive/Startup/ScheduledJobsConfiguration.cs

@@ -22,6 +22,13 @@ public static class ScheduledJobsConfiguration
                 .ForJob(cloudFileUnusedRecyclingJob)
                 .WithIdentity("CloudFileUnusedRecyclingTrigger")
                 .WithCronSchedule("0 0 0 * * ?"));
+            
+            var persistentTaskCleanupJob = new JobKey("PersistentTaskCleanup");
+            q.AddJob<PersistentTaskCleanupJob>(opts => opts.WithIdentity(persistentTaskCleanupJob));
+            q.AddTrigger(opts => opts
+                .ForJob(persistentTaskCleanupJob)
+                .WithIdentity("PersistentTaskCleanupTrigger")
+                .WithCronSchedule("0 0 2 * * ?")); // Run daily at 2 AM
         });
         services.AddQuartzHostedService(q => q.WaitForJobsToComplete = true);
 

DysonNetwork.Drive/Startup/ServiceCollectionExtensions.cs

@@ -1,14 +1,9 @@
 using System.Text.Json;
 using System.Text.Json.Serialization;
-using System.Threading.RateLimiting;
+using DysonNetwork.Drive.Index;
 using DysonNetwork.Shared.Cache;
-using Microsoft.AspNetCore.RateLimiting;
-using Microsoft.OpenApi.Models;
 using NodaTime;
 using NodaTime.Serialization.SystemTextJson;
-using StackExchange.Redis;
-using DysonNetwork.Shared.Proto;
-using tusdotnet.Stores;
 
 namespace DysonNetwork.Drive.Startup;
 
@@ -17,9 +12,7 @@ public static class ServiceCollectionExtensions
     public static IServiceCollection AddAppServices(this IServiceCollection services, IConfiguration configuration)
     {
         services.AddDbContext<AppDatabase>(); // Assuming you'll have an AppDatabase
-        services.AddSingleton<IClock>(SystemClock.Instance);
         services.AddHttpContextAccessor();
-        services.AddSingleton<ICacheService, CacheServiceRedis>(); // Uncomment if you have CacheServiceRedis
 
         services.AddHttpClient();
 
@@ -30,9 +23,7 @@ public static class ServiceCollectionExtensions
             options.MaxReceiveMessageSize = 16 * 1024 * 1024; // 16MB
             options.MaxSendMessageSize = 16 * 1024 * 1024; // 16MB
         });
-
-        // Register gRPC reflection for service discovery
-        services.AddGrpc();
+        services.AddGrpcReflection();
 
         services.AddControllers().AddJsonOptions(options =>
         {
@@ -46,24 +37,9 @@ public static class ServiceCollectionExtensions
         return services;
     }
 
-    public static IServiceCollection AddAppRateLimiting(this IServiceCollection services)
-    {
-        services.AddRateLimiter(o => o.AddFixedWindowLimiter(policyName: "fixed", opts =>
-        {
-            opts.Window = TimeSpan.FromMinutes(1);
-            opts.PermitLimit = 120;
-            opts.QueueLimit = 2;
-            opts.QueueProcessingOrder = QueueProcessingOrder.OldestFirst;
-        }));
-
-        return services;
-    }
-
     public static IServiceCollection AddAppAuthentication(this IServiceCollection services)
     {
-        services.AddCors();
         services.AddAuthorization();
-
         return services;
     }
 
@@ -74,67 +50,12 @@ public static class ServiceCollectionExtensions
         return services;
     }
 
-    public static IServiceCollection AddAppSwagger(this IServiceCollection services)
-    {
-        services.AddEndpointsApiExplorer();
-        services.AddSwaggerGen(options =>
-        {
-            options.SwaggerDoc("v1", new OpenApiInfo
-            {
-                Version = "v1",
-                Title = "Dyson Drive",
-                Description =
-                    "The file service of the Dyson Network. Mainly handling file storage and sharing. Also provide image processing and media analysis. Powered the Solar Network Drive as well.",
-                TermsOfService = new Uri("https://solsynth.dev/terms"), // Update with actual terms
-                License = new OpenApiLicense
-                {
-                    Name = "APGLv3", // Update with actual license
-                    Url = new Uri("https://www.gnu.org/licenses/agpl-3.0.html")
-                }
-            });
-            options.AddSecurityDefinition("Bearer", new OpenApiSecurityScheme
-            {
-                In = ParameterLocation.Header,
-                Description = "Please enter a valid token",
-                Name = "Authorization",
-                Type = SecuritySchemeType.Http,
-                BearerFormat = "JWT",
-                Scheme = "Bearer"
-            });
-            options.AddSecurityRequirement(new OpenApiSecurityRequirement
-            {
-                {
-                    new OpenApiSecurityScheme
-                    {
-                        Reference = new OpenApiReference
-                        {
-                            Type = ReferenceType.SecurityScheme,
-                            Id = "Bearer"
-                        }
-                    },
-                    []
-                }
-            });
-        });
-
-        return services;
-    }
-    
-    public static IServiceCollection AddAppFileStorage(this IServiceCollection services, IConfiguration configuration)
-    {
-        var tusStorePath = configuration.GetSection("Tus").GetValue<string>("StorePath")!;
-        Directory.CreateDirectory(tusStorePath);
-        var tusDiskStore = new TusDiskStore(tusStorePath);
-
-        services.AddSingleton(tusDiskStore);
-
-        return services;
-    }
-
     public static IServiceCollection AddAppBusinessServices(this IServiceCollection services)
     {
         services.AddScoped<Storage.FileService>();
         services.AddScoped<Storage.FileReferenceService>();
+        services.AddScoped<Storage.PersistentTaskService>();
+        services.AddScoped<FileIndexService>();
         services.AddScoped<Billing.UsageService>();
         services.AddScoped<Billing.QuotaService>();
 

DysonNetwork.Drive/Storage/BundleController.cs

@@ -1,4 +1,5 @@
 using System.ComponentModel.DataAnnotations;
+using DysonNetwork.Shared.Models;
 using DysonNetwork.Shared.Proto;
 using Microsoft.AspNetCore.Authorization;
 using Microsoft.AspNetCore.Mvc;
@@ -22,7 +23,7 @@ public class BundleController(AppDatabase db) : ControllerBase
     }
 
     [HttpGet("{id:guid}")]
-    public async Task<ActionResult<FileBundle>> GetBundle([FromRoute] Guid id, [FromQuery] string? passcode)
+    public async Task<ActionResult<SnFileBundle>> GetBundle([FromRoute] Guid id, [FromQuery] string? passcode)
     {
         var bundle = await db.Bundles
             .Where(e => e.Id == id)
@@ -36,7 +37,7 @@ public class BundleController(AppDatabase db) : ControllerBase
 
     [HttpGet("me")]
     [Authorize]
-    public async Task<ActionResult<List<FileBundle>>> ListBundles(
+    public async Task<ActionResult<List<SnFileBundle>>> ListBundles(
         [FromQuery] string? term,
         [FromQuery] int offset = 0,
         [FromQuery] int take = 20
@@ -65,7 +66,7 @@ public class BundleController(AppDatabase db) : ControllerBase
 
     [HttpPost]
     [Authorize]
-    public async Task<ActionResult<FileBundle>> CreateBundle([FromBody] BundleRequest request)
+    public async Task<ActionResult<SnFileBundle>> CreateBundle([FromBody] BundleRequest request)
     {
         if (HttpContext.Items["CurrentUser"] is not Account currentUser) return Unauthorized();
         var accountId = Guid.Parse(currentUser.Id);
@@ -77,7 +78,7 @@ public class BundleController(AppDatabase db) : ControllerBase
         if (string.IsNullOrEmpty(request.Name))
             request.Name = "Unnamed Bundle";
 
-        var bundle = new FileBundle
+        var bundle = new SnFileBundle
         {
             Slug = request.Slug,
             Name = request.Name,
@@ -95,7 +96,7 @@ public class BundleController(AppDatabase db) : ControllerBase
 
     [HttpPut("{id:guid}")]
     [Authorize]
-    public async Task<ActionResult<FileBundle>> UpdateBundle([FromRoute] Guid id, [FromBody] BundleRequest request)
+    public async Task<ActionResult<SnFileBundle>> UpdateBundle([FromRoute] Guid id, [FromBody] BundleRequest request)
     {
         if (HttpContext.Items["CurrentUser"] is not Account currentUser) return Unauthorized();
         var accountId = Guid.Parse(currentUser.Id);

DysonNetwork.Drive/Storage/CloudFileUnusedRecyclingJob.cs

@@ -6,7 +6,6 @@ namespace DysonNetwork.Drive.Storage;
 
 public class CloudFileUnusedRecyclingJob(
     AppDatabase db,
-    FileReferenceService fileRefService,
     ILogger<CloudFileUnusedRecyclingJob> logger,
     IConfiguration configuration
 )
@@ -15,7 +14,7 @@ public class CloudFileUnusedRecyclingJob(
     public async Task Execute(IJobExecutionContext context)
     {
         logger.LogInformation("Cleaning tus cloud files...");
-        var storePath = configuration["Tus:StorePath"];
+        var storePath = configuration["Storage:Uploads"];
         if (Directory.Exists(storePath))
         {
             var oneHourAgo = SystemClock.Instance.GetCurrentInstant() - Duration.FromHours(1);
@@ -40,6 +39,7 @@ public class CloudFileUnusedRecyclingJob(
         var processedCount = 0;
         var markedCount = 0;
         var totalFiles = await db.Files
+            .Where(f => f.FileIndexes.Count == 0)
             .Where(f => f.PoolId.HasValue && recyclablePools.Contains(f.PoolId.Value))
             .Where(f => !f.IsMarkedRecycle)
             .CountAsync();
@@ -80,15 +80,15 @@ public class CloudFileUnusedRecyclingJob(
             processedCount += fileBatch.Count;
             lastProcessedId = fileBatch.Last();
 
-            // Get all relevant file references for this batch
-            var fileReferences = await fileRefService.GetReferencesAsync(fileBatch);
-
-            // Filter to find files that have no references or all expired references
-            var filesToMark = fileBatch.Where(fileId =>
-                !fileReferences.TryGetValue(fileId, out var references) ||
-                references.Count == 0 ||
-                references.All(r => r.ExpiredAt.HasValue && r.ExpiredAt.Value <= now)
-            ).ToList();
+            // Optimized query: Find files that have no references OR all references are expired
+            // This replaces the memory-intensive approach of loading all references
+            var filesToMark = await db.Files
+                .Where(f => fileBatch.Contains(f.Id))
+                .Where(f => !db.FileReferences.Any(r => r.FileId == f.Id) || // No references at all
+                           !db.FileReferences.Any(r => r.FileId == f.Id && // OR has references but all are expired
+                                                     (r.ExpiredAt == null || r.ExpiredAt > now)))
+                .Select(f => f.Id)
+                .ToListAsync();
 
             if (filesToMark.Count > 0)
             {

DysonNetwork.Drive/Storage/FileController.cs

@@ -1,6 +1,5 @@
-using DysonNetwork.Drive.Billing;
 using DysonNetwork.Shared.Auth;
-using DysonNetwork.Shared.Data;
+using DysonNetwork.Shared.Models;
 using DysonNetwork.Shared.Proto;
 using Microsoft.AspNetCore.Authorization;
 using Microsoft.AspNetCore.Mvc;
@@ -14,9 +13,9 @@ namespace DysonNetwork.Drive.Storage;
 public class FileController(
     AppDatabase db,
     FileService fs,
-    QuotaService qs,
     IConfiguration configuration,
-    IWebHostEnvironment env
+    IWebHostEnvironment env,
+    FileReferenceService fileReferenceService
 ) : ControllerBase
 {
     [HttpGet("{id}")]
@@ -29,119 +28,202 @@ public class FileController(
         [FromQuery] string? passcode = null
     )
     {
-        // Support the file extension for client side data recognize
-        string? fileExtension = null;
-        if (id.Contains('.'))
-        {
-            var splitId = id.Split('.');
-            id = splitId.First();
-            fileExtension = splitId.Last();
-        }
-
-        var file = await fs.GetFileAsync(id);
+        var (fileId, fileExtension) = ParseFileId(id);
+        var file = await fs.GetFileAsync(fileId);
         if (file is null) return NotFound("File not found.");
 
+        var accessResult = await ValidateFileAccess(file, passcode);
+        if (accessResult is not null) return accessResult;
+
+        // Handle direct storage URL redirect
+        if (!string.IsNullOrWhiteSpace(file.StorageUrl))
+            return Redirect(file.StorageUrl);
+
+        // Handle files not yet uploaded to remote storage
+        if (file.UploadedAt is null)
+            return await ServeLocalFile(file);
+
+        // Handle uploaded files
+        return await ServeRemoteFile(file, fileExtension, download, original, thumbnail, overrideMimeType);
+    }
+
+    private (string fileId, string? extension) ParseFileId(string id)
+    {
+        if (!id.Contains('.')) return (id, null);
+
+        var parts = id.Split('.');
+        return (parts.First(), parts.Last());
+    }
+
+    private async Task<ActionResult?> ValidateFileAccess(SnCloudFile file, string? passcode)
+    {
         if (file.Bundle is not null && !file.Bundle.VerifyPasscode(passcode))
             return StatusCode(StatusCodes.Status403Forbidden, "The passcode is incorrect.");
+        return null;
+    }
 
-        if (!string.IsNullOrWhiteSpace(file.StorageUrl)) return Redirect(file.StorageUrl);
-
-        if (!file.PoolId.HasValue)
+    private Task<ActionResult> ServeLocalFile(SnCloudFile file)
+    {
+        // Try temp storage first
+        var tempFilePath = Path.Combine(Path.GetTempPath(), file.Id);
+        if (System.IO.File.Exists(tempFilePath))
         {
-            var tusStorePath = configuration.GetValue<string>("Tus:StorePath")!;
-            var filePath = Path.Combine(env.ContentRootPath, tusStorePath, file.Id);
-            if (!System.IO.File.Exists(filePath)) return new NotFoundResult();
-            return PhysicalFile(filePath, file.MimeType ?? "application/octet-stream", file.Name);
+            if (file.IsEncrypted)
+                return Task.FromResult<ActionResult>(StatusCode(StatusCodes.Status403Forbidden,
+                    "Encrypted files cannot be accessed before they are processed and stored."));
+
+            return Task.FromResult<ActionResult>(PhysicalFile(tempFilePath, file.MimeType ?? "application/octet-stream",
+                file.Name, enableRangeProcessing: true));
         }
 
+        // Fallback for tus uploads
+        var tusStorePath = configuration.GetValue<string>("Storage:Uploads");
+        if (string.IsNullOrEmpty(tusStorePath))
+            return Task.FromResult<ActionResult>(StatusCode(StatusCodes.Status400BadRequest,
+                "File is being processed. Please try again later."));
+        var tusFilePath = Path.Combine(env.ContentRootPath, tusStorePath, file.Id);
+        return System.IO.File.Exists(tusFilePath)
+            ? Task.FromResult<ActionResult>(PhysicalFile(tusFilePath, file.MimeType ?? "application/octet-stream",
+                file.Name, enableRangeProcessing: true))
+            : Task.FromResult<ActionResult>(StatusCode(StatusCodes.Status400BadRequest,
+                "File is being processed. Please try again later."));
+    }
+
+    private async Task<ActionResult> ServeRemoteFile(
+        SnCloudFile file,
+        string? fileExtension,
+        bool download,
+        bool original,
+        bool thumbnail,
+        string? overrideMimeType
+    )
+    {
+        if (!file.PoolId.HasValue)
+            return StatusCode(StatusCodes.Status500InternalServerError,
+                "File is in an inconsistent state: uploaded but no pool ID.");
+
         var pool = await fs.GetPoolAsync(file.PoolId.Value);
         if (pool is null)
             return StatusCode(StatusCodes.Status410Gone, "The pool of the file no longer exists or not accessible.");
+
+        if (!pool.PolicyConfig.AllowAnonymous && HttpContext.Items["CurrentUser"] is not Account)
+            return Unauthorized();
+
         var dest = pool.StorageConfig;
+        var fileName = BuildRemoteFileName(file, original, thumbnail);
 
-        if (!pool.PolicyConfig.AllowAnonymous)
-            if (HttpContext.Items["CurrentUser"] is not Account currentUser)
-                return Unauthorized();
-        // TODO: Provide ability to add access log
-
-        var fileName = string.IsNullOrWhiteSpace(file.StorageId) ? file.Id : file.StorageId;
-
-        switch (thumbnail)
-        {
-            case true when file.HasThumbnail:
-                fileName += ".thumbnail";
-                break;
-            case true when !file.HasThumbnail:
-                return NotFound();
-        }
-
-        if (!original && file.HasCompression)
-            fileName += ".compressed";
-
-        if (dest.ImageProxy is not null && (file.MimeType?.StartsWith("image/") ?? false))
-        {
-            var proxyUrl = dest.ImageProxy;
-            var baseUri = new Uri(proxyUrl.EndsWith('/') ? proxyUrl : $"{proxyUrl}/");
-            var fullUri = new Uri(baseUri, fileName);
-            return Redirect(fullUri.ToString());
-        }
-
-        if (dest.AccessProxy is not null)
-        {
-            var proxyUrl = dest.AccessProxy;
-            var baseUri = new Uri(proxyUrl.EndsWith('/') ? proxyUrl : $"{proxyUrl}/");
-            var fullUri = new Uri(baseUri, fileName);
-            return Redirect(fullUri.ToString());
-        }
+        // Try proxy redirects first
+        var proxyResult = TryProxyRedirect(file, dest, fileName);
+        if (proxyResult is not null) return proxyResult;
 
+        // Handle signed URLs
         if (dest.EnableSigned)
-        {
-            var client = fs.CreateMinioClient(dest);
-            if (client is null)
-                return BadRequest(
-                    "Failed to configure client for remote destination, file got an invalid storage remote."
-                );
+            return await CreateSignedUrl(file, dest, fileName, fileExtension, download, overrideMimeType);
 
-            var headers = new Dictionary<string, string>();
-            if (fileExtension is not null)
-            {
-                if (MimeTypes.TryGetMimeType(fileExtension, out var mimeType))
-                    headers.Add("Response-Content-Type", mimeType);
-            }
-            else if (overrideMimeType is not null)
-            {
-                headers.Add("Response-Content-Type", overrideMimeType);
-            }
-            else if (file.MimeType is not null && !file.MimeType!.EndsWith("unknown"))
-            {
-                headers.Add("Response-Content-Type", file.MimeType);
-            }
-
-            if (download)
-            {
-                headers.Add("Response-Content-Disposition", $"attachment; filename=\"{file.Name}\"");
-            }
-
-            var bucket = dest.Bucket;
-            var openUrl = await client.PresignedGetObjectAsync(
-                new PresignedGetObjectArgs()
-                    .WithBucket(bucket)
-                    .WithObject(fileName)
-                    .WithExpiry(3600)
-                    .WithHeaders(headers)
-            );
-
-            return Redirect(openUrl);
-        }
-
-        // Fallback redirect to the S3 endpoint (public read)
+        // Fallback to direct S3 endpoint
         var protocol = dest.EnableSsl ? "https" : "http";
-        // Use the path bucket lookup mode
         return Redirect($"{protocol}://{dest.Endpoint}/{dest.Bucket}/{fileName}");
     }
 
+    private string BuildRemoteFileName(SnCloudFile file, bool original, bool thumbnail)
+    {
+        var fileName = string.IsNullOrWhiteSpace(file.StorageId) ? file.Id : file.StorageId;
+
+        if (thumbnail)
+        {
+            if (!file.HasThumbnail) throw new InvalidOperationException("Thumbnail not available");
+            fileName += ".thumbnail";
+        }
+        else if (!original && file.HasCompression)
+        {
+            fileName += ".compressed";
+        }
+
+        return fileName;
+    }
+
+    private ActionResult? TryProxyRedirect(SnCloudFile file, RemoteStorageConfig dest, string fileName)
+    {
+        if (dest.ImageProxy is not null && (file.MimeType?.StartsWith("image/") ?? false))
+            return Redirect(BuildProxyUrl(dest.ImageProxy, fileName));
+
+        return dest.AccessProxy is not null ? Redirect(BuildProxyUrl(dest.AccessProxy, fileName)) : null;
+    }
+
+    private static string BuildProxyUrl(string proxyUrl, string fileName)
+    {
+        var baseUri = new Uri(proxyUrl.EndsWith('/') ? proxyUrl : $"{proxyUrl}/");
+        var fullUri = new Uri(baseUri, fileName);
+        return fullUri.ToString();
+    }
+
+    private async Task<ActionResult> CreateSignedUrl(
+        SnCloudFile file,
+        RemoteStorageConfig dest,
+        string fileName,
+        string? fileExtension,
+        bool download,
+        string? overrideMimeType
+    )
+    {
+        var client = fs.CreateMinioClient(dest);
+        if (client is null)
+            return BadRequest("Failed to configure client for remote destination, file got an invalid storage remote.");
+
+        var headers = BuildSignedUrlHeaders(file, fileExtension, overrideMimeType, download);
+
+        var openUrl = await client.PresignedGetObjectAsync(
+            new PresignedGetObjectArgs()
+                .WithBucket(dest.Bucket)
+                .WithObject(fileName)
+                .WithExpiry(3600)
+                .WithHeaders(headers)
+        );
+
+        if (dest.AccessEndpoint is not null)
+            openUrl = openUrl.Replace($"{dest.Endpoint}/{dest.Bucket}", dest.AccessEndpoint);
+
+        return Redirect(openUrl);
+    }
+
+    private static Dictionary<string, string> BuildSignedUrlHeaders(
+        SnCloudFile file,
+        string? fileExtension,
+        string? overrideMimeType,
+        bool download
+    )
+    {
+        var headers = new Dictionary<string, string>();
+
+        string? contentType = null;
+        if (fileExtension is not null && MimeTypes.TryGetMimeType(fileExtension, out var mimeType))
+        {
+            contentType = mimeType;
+        }
+        else if (overrideMimeType is not null)
+        {
+            contentType = overrideMimeType;
+        }
+        else if (file.MimeType is not null && !file.MimeType.EndsWith("unknown"))
+        {
+            contentType = file.MimeType;
+        }
+
+        if (contentType is not null)
+        {
+            headers.Add("Response-Content-Type", contentType);
+        }
+
+        if (download)
+        {
+            headers.Add("Response-Content-Disposition", $"attachment; filename=\"{file.Name}\"");
+        }
+
+        return headers;
+    }
+
     [HttpGet("{id}/info")]
-    public async Task<ActionResult<CloudFile>> GetFileInfo(string id)
+    public async Task<ActionResult<SnCloudFile>> GetFileInfo(string id)
     {
         var file = await fs.GetFileAsync(id);
         if (file is null) return NotFound("File not found.");
@@ -149,78 +231,102 @@ public class FileController(
         return file;
     }
 
+    [HttpGet("{id}/references")]
+    public async Task<ActionResult<List<Shared.Models.SnCloudFileReference>>> GetFileReferences(string id)
+    {
+        var file = await fs.GetFileAsync(id);
+        if (file is null) return NotFound("File not found.");
+
+        // Check if user has access to the file
+        var accessResult = await ValidateFileAccess(file, null);
+        if (accessResult is not null) return accessResult;
+
+        // Get references using the injected FileReferenceService
+        var references = await fileReferenceService.GetReferencesAsync(id);
+        return Ok(references);
+    }
+
     [Authorize]
     [HttpPatch("{id}/name")]
-    public async Task<ActionResult<CloudFile>> UpdateFileName(string id, [FromBody] string name)
+    public async Task<ActionResult<SnCloudFile>> UpdateFileName(string id, [FromBody] string name)
     {
-        if (HttpContext.Items["CurrentUser"] is not Account currentUser) return Unauthorized();
-        var accountId = Guid.Parse(currentUser.Id);
-        var file = await db.Files.FirstOrDefaultAsync(f => f.Id == id && f.AccountId == accountId);
-        if (file is null) return NotFound();
-        file.Name = name;
-        await db.SaveChangesAsync();
-        await fs._PurgeCacheAsync(file.Id);
-        return file;
+        return await UpdateFileProperty(id, file => file.Name = name);
     }
 
     public class MarkFileRequest
     {
-        public List<ContentSensitiveMark>? SensitiveMarks { get; set; }
+        public List<Shared.Models.ContentSensitiveMark>? SensitiveMarks { get; set; }
     }
 
     [Authorize]
     [HttpPut("{id}/marks")]
-    public async Task<ActionResult<CloudFile>> MarkFile(string id, [FromBody] MarkFileRequest request)
+    public async Task<ActionResult<SnCloudFile>> MarkFile(string id, [FromBody] MarkFileRequest request)
     {
-        if (HttpContext.Items["CurrentUser"] is not Account currentUser) return Unauthorized();
-        var accountId = Guid.Parse(currentUser.Id);
-        var file = await db.Files.FirstOrDefaultAsync(f => f.Id == id && f.AccountId == accountId);
-        if (file is null) return NotFound();
-        file.SensitiveMarks = request.SensitiveMarks;
-        await db.SaveChangesAsync();
-        await fs._PurgeCacheAsync(file.Id);
-        return file;
+        return await UpdateFileProperty(id, file => file.SensitiveMarks = request.SensitiveMarks);
     }
 
     [Authorize]
     [HttpPut("{id}/meta")]
-    public async Task<ActionResult<CloudFile>> UpdateFileMeta(string id, [FromBody] Dictionary<string, object?> meta)
+    public async Task<ActionResult<SnCloudFile>> UpdateFileMeta(string id, [FromBody] Dictionary<string, object?> meta)
+    {
+        return await UpdateFileProperty(id, file => file.UserMeta = meta);
+    }
+
+    private async Task<ActionResult<SnCloudFile>> UpdateFileProperty(string fileId, Action<SnCloudFile> updateAction)
     {
         if (HttpContext.Items["CurrentUser"] is not Account currentUser) return Unauthorized();
         var accountId = Guid.Parse(currentUser.Id);
-        var file = await db.Files.FirstOrDefaultAsync(f => f.Id == id && f.AccountId == accountId);
+
+        var file = await db.Files.FirstOrDefaultAsync(f => f.Id == fileId && f.AccountId == accountId);
         if (file is null) return NotFound();
-        file.UserMeta = meta;
+
+        updateAction(file);
         await db.SaveChangesAsync();
         await fs._PurgeCacheAsync(file.Id);
+
         return file;
     }
 
     [Authorize]
     [HttpGet("me")]
-    public async Task<ActionResult<List<CloudFile>>> GetMyFiles(
+    public async Task<ActionResult<List<SnCloudFile>>> GetMyFiles(
         [FromQuery] Guid? pool,
         [FromQuery] bool recycled = false,
         [FromQuery] int offset = 0,
-        [FromQuery] int take = 20
+        [FromQuery] int take = 20,
+        [FromQuery] string? query = null,
+        [FromQuery] string order = "date",
+        [FromQuery] bool orderDesc = true
     )
     {
         if (HttpContext.Items["CurrentUser"] is not Account currentUser) return Unauthorized();
         var accountId = Guid.Parse(currentUser.Id);
 
-        var query = db.Files
+        var filesQuery = db.Files
             .Where(e => e.IsMarkedRecycle == recycled)
             .Where(e => e.AccountId == accountId)
             .Include(e => e.Pool)
-            .OrderByDescending(e => e.CreatedAt)
             .AsQueryable();
 
-        if (pool.HasValue) query = query.Where(e => e.PoolId == pool);
+        if (pool.HasValue) filesQuery = filesQuery.Where(e => e.PoolId == pool);
 
-        var total = await query.CountAsync();
+        if (!string.IsNullOrWhiteSpace(query))
+        {
+            filesQuery = filesQuery.Where(e => e.Name.Contains(query));
+        }
+
+        filesQuery = order.ToLower() switch
+        {
+            "date" => orderDesc ? filesQuery.OrderByDescending(e => e.CreatedAt) : filesQuery.OrderBy(e => e.CreatedAt),
+            "size" => orderDesc ? filesQuery.OrderByDescending(e => e.Size) : filesQuery.OrderBy(e => e.Size),
+            "name" => orderDesc ? filesQuery.OrderByDescending(e => e.Name) : filesQuery.OrderBy(e => e.Name),
+            _ => filesQuery.OrderByDescending(e => e.CreatedAt)
+        };
+
+        var total = await filesQuery.CountAsync();
         Response.Headers.Append("X-Total", total.ToString());
 
-        var files = await query
+        var files = await filesQuery
             .Skip(offset)
             .Take(take)
             .ToListAsync();
@@ -228,9 +334,25 @@ public class FileController(
         return Ok(files);
     }
 
+    public class FileBatchDeletionRequest
+    {
+        public List<string> FileIds { get; set; } = [];
+    }
+
+    [Authorize]
+    [HttpPost("batches/delete")]
+    public async Task<ActionResult> DeleteFileBatch([FromBody] FileBatchDeletionRequest request)
+    {
+        if (HttpContext.Items["CurrentUser"] is not Account currentUser) return Unauthorized();
+        var userId = Guid.Parse(currentUser.Id);
+
+        var count = await fs.DeleteAccountFileBatchAsync(userId, request.FileIds);
+        return Ok(new { Count = count });
+    }
+
     [Authorize]
     [HttpDelete("{id}")]
-    public async Task<ActionResult> DeleteFile(string id)
+    public async Task<ActionResult<SnCloudFile>> DeleteFile(string id)
     {
         if (HttpContext.Items["CurrentUser"] is not Account currentUser) return Unauthorized();
         var userId = Guid.Parse(currentUser.Id);
@@ -242,9 +364,9 @@ public class FileController(
         if (file is null) return NotFound();
 
         await fs.DeleteFileDataAsync(file, force: true);
-        await fs.DeleteFileAsync(file);
+        await fs.DeleteFileAsync(file, skipData: true);
 
-        return NoContent();
+        return Ok(file);
     }
 
     [Authorize]
@@ -260,116 +382,10 @@ public class FileController(
 
     [Authorize]
     [HttpDelete("recycle")]
-    [RequiredPermission("maintenance", "files.delete.recycle")]
+    [AskPermission("files.delete.recycle")]
     public async Task<ActionResult> DeleteAllRecycledFiles()
     {
         var count = await fs.DeleteAllRecycledFilesAsync();
         return Ok(new { Count = count });
     }
-
-    public class CreateFastFileRequest
-    {
-        public string Name { get; set; } = null!;
-        public long Size { get; set; }
-        public string Hash { get; set; } = null!;
-        public string? MimeType { get; set; }
-        public string? Description { get; set; }
-        public Dictionary<string, object?>? UserMeta { get; set; }
-        public Dictionary<string, object?>? FileMeta { get; set; }
-        public List<ContentSensitiveMark>? SensitiveMarks { get; set; }
-        public Guid PoolId { get; set; }
-    }
-
-    [Authorize]
-    [HttpPost("fast")]
-    [RequiredPermission("global", "files.create")]
-    public async Task<ActionResult<CloudFile>> CreateFastFile([FromBody] CreateFastFileRequest request)
-    {
-        if (HttpContext.Items["CurrentUser"] is not Account currentUser) return Unauthorized();
-        var accountId = Guid.Parse(currentUser.Id);
-
-        var pool = await db.Pools.FirstOrDefaultAsync(p => p.Id == request.PoolId);
-        if (pool is null) return BadRequest();
-        if (!currentUser.IsSuperuser && pool.AccountId != accountId)
-            return StatusCode(403, "You don't have permission to create files in this pool.");
-
-        if (!pool.PolicyConfig.EnableFastUpload)
-            return StatusCode(
-                403,
-                "This pool does not allow fast upload"
-            );
-
-        if (pool.PolicyConfig.RequirePrivilege > 0)
-        {
-            if (currentUser.PerkSubscription is null)
-            {
-                return StatusCode(
-                    403,
-                    $"You need to have join the Stellar Program to use this pool"
-                );
-            }
-
-            var privilege =
-                PerkSubscriptionPrivilege.GetPrivilegeFromIdentifier(currentUser.PerkSubscription.Identifier);
-            if (privilege < pool.PolicyConfig.RequirePrivilege)
-            {
-                return StatusCode(
-                    403,
-                    $"You need Stellar Program tier {pool.PolicyConfig.RequirePrivilege} to use this pool, you are tier {privilege}"
-                );
-            }
-        }
-
-        if (request.Size > pool.PolicyConfig.MaxFileSize)
-        {
-            return StatusCode(
-                403,
-                $"File size {request.Size} is larger than the pool's maximum file size {pool.PolicyConfig.MaxFileSize}"
-            );
-        }
-
-        var (ok, billableUnit, quota) = await qs.IsFileAcceptable(
-            accountId,
-            pool.BillingConfig.CostMultiplier ?? 1.0,
-            request.Size
-        );
-        if (!ok)
-        {
-            return StatusCode(
-                403,
-                $"File size {billableUnit} is larger than the user's quota {quota}"
-            );
-        }
-
-        await using var transaction = await db.Database.BeginTransactionAsync();
-        try
-        {
-            var file = new CloudFile
-            {
-                Name = request.Name,
-                Size = request.Size,
-                Hash = request.Hash,
-                MimeType = request.MimeType,
-                Description = request.Description,
-                AccountId = accountId,
-                UserMeta = request.UserMeta,
-                FileMeta = request.FileMeta,
-                SensitiveMarks = request.SensitiveMarks,
-                PoolId = request.PoolId
-            };
-            db.Files.Add(file);
-            await db.SaveChangesAsync();
-            await fs._PurgeCacheAsync(file.Id);
-            await transaction.CommitAsync();
-
-            file.FastUploadLink = await fs.CreateFastUploadLinkAsync(file);
-
-            return file;
-        }
-        catch (Exception)
-        {
-            await transaction.RollbackAsync();
-            throw;
-        }
-    }
 }

DysonNetwork.Drive/Storage/FileExpirationJob.cs

@@ -14,49 +14,55 @@ public class FileExpirationJob(AppDatabase db, FileService fileService, ILogger<
         var now = SystemClock.Instance.GetCurrentInstant();
         logger.LogInformation("Running file reference expiration job at {now}", now);
 
-        // Find all expired references
-        var expiredReferences = await db.FileReferences
+        // Delete expired references in bulk and get affected file IDs
+        var affectedFileIds = await db.FileReferences
             .Where(r => r.ExpiredAt < now && r.ExpiredAt != null)
+            .Select(r => r.FileId)
+            .Distinct()
             .ToListAsync();
 
-        if (!expiredReferences.Any())
+        if (!affectedFileIds.Any())
         {
             logger.LogInformation("No expired file references found");
             return;
         }
 
-        logger.LogInformation("Found {count} expired file references", expiredReferences.Count);
+        logger.LogInformation("Found expired references for {count} files", affectedFileIds.Count);
 
-        // Get unique file IDs
-        var fileIds = expiredReferences.Select(r => r.FileId).Distinct().ToList();
-        var filesAndReferenceCount = new Dictionary<string, int>();
+        // Delete expired references in bulk
+        var deletedReferencesCount = await db.FileReferences
+            .Where(r => r.ExpiredAt < now && r.ExpiredAt != null)
+            .ExecuteDeleteAsync();
 
-        // Delete expired references
-        db.FileReferences.RemoveRange(expiredReferences);
-        await db.SaveChangesAsync();
+        logger.LogInformation("Deleted {count} expired file references", deletedReferencesCount);
 
-        // Check remaining references for each file
-        foreach (var fileId in fileIds)
+        // Find files that now have no remaining references (bulk operation)
+        var filesToDelete = await db.Files
+            .Where(f => affectedFileIds.Contains(f.Id))
+            .Where(f => !db.FileReferences.Any(r => r.FileId == f.Id))
+            .Select(f => f.Id)
+            .ToListAsync();
+
+        if (filesToDelete.Any())
         {
-            var remainingReferences = await db.FileReferences
-                .Where(r => r.FileId == fileId)
-                .CountAsync();
+            logger.LogInformation("Deleting {count} files that have no remaining references", filesToDelete.Count);
 
-            filesAndReferenceCount[fileId] = remainingReferences;
+            // Get files for deletion
+            var files = await db.Files
+                .Where(f => filesToDelete.Contains(f.Id))
+                .ToListAsync();
 
-            // If no references remain, delete the file
-            if (remainingReferences == 0)
-            {
-                var file = await db.Files.FirstOrDefaultAsync(f => f.Id == fileId);
-                if (file == null) continue;
-                logger.LogInformation("Deleting file {fileId} as all references have expired", fileId);
-                await fileService.DeleteFileAsync(file);
-            }
-            else
-            {
-                // Just purge the cache
-                await fileService._PurgeCacheAsync(fileId);
-            }
+            // Delete files and their data in parallel
+            var deleteTasks = files.Select(f => fileService.DeleteFileAsync(f));
+            await Task.WhenAll(deleteTasks);
+        }
+
+        // Purge cache for files that still have references
+        var filesWithRemainingRefs = affectedFileIds.Except(filesToDelete).ToList();
+        if (filesWithRemainingRefs.Any())
+        {
+            var cachePurgeTasks = filesWithRemainingRefs.Select(fileService._PurgeCacheAsync);
+            await Task.WhenAll(cachePurgeTasks);
         }
 
         logger.LogInformation("Completed file reference expiration job");

DysonNetwork.Drive/Storage/FilePoolController.cs

@@ -1,3 +1,4 @@
+using DysonNetwork.Shared.Models;
 using DysonNetwork.Shared.Proto;
 using Microsoft.AspNetCore.Authorization;
 using Microsoft.AspNetCore.Mvc;
@@ -19,6 +20,7 @@ public class FilePoolController(AppDatabase db, FileService fs) : ControllerBase
         var pools = await db.Pools
             .Where(p => p.PolicyConfig.PublicUsable || p.AccountId == accountId)
             .Where(p => !p.IsHidden || p.AccountId == accountId)
+            .OrderBy(p => p.CreatedAt)
             .ToListAsync();
         pools = pools.Select(p =>
         {

DysonNetwork.Drive/Storage/FileReferenceService.cs

@@ -1,5 +1,6 @@
 using DysonNetwork.Shared.Cache;
-using EFCore.BulkExtensions;
+using DysonNetwork.Shared.Data;
+using DysonNetwork.Shared.Models;
 using Microsoft.EntityFrameworkCore;
 using NodaTime;
 
@@ -19,7 +20,7 @@ public class FileReferenceService(AppDatabase db, FileService fileService, ICach
     /// <param name="expiredAt">Optional expiration time for the file</param>
     /// <param name="duration">Optional duration after which the file expires (alternative to expiredAt)</param>
     /// <returns>The created file reference</returns>
-    public async Task<CloudFileReference> CreateReferenceAsync(
+    public async Task<SnCloudFileReference> CreateReferenceAsync(
         string fileId,
         string usage,
         string resourceId,
@@ -32,7 +33,7 @@ public class FileReferenceService(AppDatabase db, FileService fileService, ICach
         if (duration.HasValue)
             finalExpiration = SystemClock.Instance.GetCurrentInstant() + duration.Value;
 
-        var reference = new CloudFileReference
+        var reference = new SnCloudFileReference
         {
             FileId = fileId,
             Usage = usage,
@@ -48,7 +49,7 @@ public class FileReferenceService(AppDatabase db, FileService fileService, ICach
         return reference;
     }
 
-    public async Task<List<CloudFileReference>> CreateReferencesAsync(
+    public async Task<List<SnCloudFileReference>> CreateReferencesAsync(
         List<string> fileId,
         string usage,
         string resourceId,
@@ -56,14 +57,26 @@ public class FileReferenceService(AppDatabase db, FileService fileService, ICach
         Duration? duration = null
     )
     {
-        var data = fileId.Select(id => new CloudFileReference
+        var now = SystemClock.Instance.GetCurrentInstant();
+        var finalExpiredAt = expiredAt;
+        if (finalExpiredAt == null && duration.HasValue)
         {
-            FileId = id,
-            Usage = usage,
-            ResourceId = resourceId,
-            ExpiredAt = expiredAt ?? SystemClock.Instance.GetCurrentInstant() + duration
-        }).ToList();
-        await db.BulkInsertAsync(data);
+            finalExpiredAt = now + duration.Value;
+        }
+
+        var data = fileId.Select(id => new SnCloudFileReference
+            {
+                FileId = id,
+                Usage = usage,
+                ResourceId = resourceId,
+                ExpiredAt = finalExpiredAt,
+                CreatedAt = now,
+                UpdatedAt = now
+            })
+            .ToList();
+
+        db.FileReferences.AddRange(data);
+        await db.SaveChangesAsync();
         return data;
     }
 
@@ -72,11 +85,11 @@ public class FileReferenceService(AppDatabase db, FileService fileService, ICach
     /// </summary>
     /// <param name="fileId">The ID of the file</param>
     /// <returns>A list of all references to the file</returns>
-    public async Task<List<CloudFileReference>> GetReferencesAsync(string fileId)
+    public async Task<List<SnCloudFileReference>> GetReferencesAsync(string fileId)
     {
         var cacheKey = $"{CacheKeyPrefix}list:{fileId}";
 
-        var cachedReferences = await cache.GetAsync<List<CloudFileReference>>(cacheKey);
+        var cachedReferences = await cache.GetAsync<List<SnCloudFileReference>>(cacheKey);
         if (cachedReferences is not null)
             return cachedReferences;
 
@@ -89,13 +102,45 @@ public class FileReferenceService(AppDatabase db, FileService fileService, ICach
         return references;
     }
 
-    public async Task<Dictionary<string, List<CloudFileReference>>> GetReferencesAsync(IEnumerable<string> fileId)
+    public async Task<Dictionary<string, List<SnCloudFileReference>>> GetReferencesAsync(IEnumerable<string> fileIds)
     {
-        var references = await db.FileReferences
-            .Where(r => fileId.Contains(r.FileId))
-            .GroupBy(r => r.FileId)
-            .ToDictionaryAsync(r => r.Key, r => r.ToList());
-        return references;
+        var fileIdList = fileIds.ToList();
+        var result = new Dictionary<string, List<SnCloudFileReference>>();
+
+        // Check cache for each file ID
+        var uncachedFileIds = new List<string>();
+        foreach (var fileId in fileIdList)
+        {
+            var cacheKey = $"{CacheKeyPrefix}list:{fileId}";
+            var cachedReferences = await cache.GetAsync<List<SnCloudFileReference>>(cacheKey);
+            if (cachedReferences is not null)
+            {
+                result[fileId] = cachedReferences;
+            }
+            else
+            {
+                uncachedFileIds.Add(fileId);
+            }
+        }
+
+        // Fetch uncached references from database
+        if (uncachedFileIds.Any())
+        {
+            var dbReferences = await db.FileReferences
+                .Where(r => uncachedFileIds.Contains(r.FileId))
+                .GroupBy(r => r.FileId)
+                .ToDictionaryAsync(r => r.Key, r => r.ToList());
+
+            // Cache the results
+            foreach (var kvp in dbReferences)
+            {
+                var cacheKey = $"{CacheKeyPrefix}list:{kvp.Key}";
+                await cache.SetAsync(cacheKey, kvp.Value, CacheDuration);
+                result[kvp.Key] = kvp.Value;
+            }
+        }
+
+        return result;
     }
 
     /// <summary>
@@ -125,11 +170,11 @@ public class FileReferenceService(AppDatabase db, FileService fileService, ICach
     /// </summary>
     /// <param name="resourceId">The ID of the resource</param>
     /// <returns>A list of file references associated with the resource</returns>
-    public async Task<List<CloudFileReference>> GetResourceReferencesAsync(string resourceId)
+    public async Task<List<SnCloudFileReference>> GetResourceReferencesAsync(string resourceId)
     {
         var cacheKey = $"{CacheKeyPrefix}resource:{resourceId}";
 
-        var cachedReferences = await cache.GetAsync<List<CloudFileReference>>(cacheKey);
+        var cachedReferences = await cache.GetAsync<List<SnCloudFileReference>>(cacheKey);
         if (cachedReferences is not null)
             return cachedReferences;
 
@@ -147,11 +192,21 @@ public class FileReferenceService(AppDatabase db, FileService fileService, ICach
     /// </summary>
     /// <param name="usage">The usage context</param>
     /// <returns>A list of file references with the specified usage</returns>
-    public async Task<List<CloudFileReference>> GetUsageReferencesAsync(string usage)
+    public async Task<List<SnCloudFileReference>> GetUsageReferencesAsync(string usage)
     {
-        return await db.FileReferences
+        var cacheKey = $"{CacheKeyPrefix}usage:{usage}";
+
+        var cachedReferences = await cache.GetAsync<List<SnCloudFileReference>>(cacheKey);
+        if (cachedReferences is not null)
+            return cachedReferences;
+
+        var references = await db.FileReferences
             .Where(r => r.Usage == usage)
             .ToListAsync();
+
+        await cache.SetAsync(cacheKey, references, CacheDuration);
+
+        return references;
     }
 
     /// <summary>
@@ -208,8 +263,9 @@ public class FileReferenceService(AppDatabase db, FileService fileService, ICach
     
     public async Task<int> DeleteResourceReferencesBatchAsync(IEnumerable<string> resourceIds, string? usage = null)
     {
+        var resourceIdList = resourceIds.ToList();
         var references = await db.FileReferences
-            .Where(r => resourceIds.Contains(r.ResourceId))
+            .Where(r => resourceIdList.Contains(r.ResourceId))
             .If(usage != null, q => q.Where(q => q.Usage == usage))
             .ToListAsync();
 
@@ -221,8 +277,9 @@ public class FileReferenceService(AppDatabase db, FileService fileService, ICach
         db.FileReferences.RemoveRange(references);
         var deletedCount = await db.SaveChangesAsync();
 
-        // Purge caches
+        // Purge caches for files and resources
         var tasks = fileIds.Select(fileService._PurgeCacheAsync).ToList();
+        tasks.AddRange(resourceIdList.Select(PurgeCacheForResourceAsync));
         await Task.WhenAll(tasks);
 
         return deletedCount;
@@ -261,7 +318,7 @@ public class FileReferenceService(AppDatabase db, FileService fileService, ICach
     /// <param name="expiredAt">Optional expiration time for newly added files</param>
     /// <param name="duration">Optional duration after which newly added files expire</param>
     /// <returns>A list of the updated file references</returns>
-    public async Task<List<CloudFileReference>> UpdateResourceFilesAsync(
+    public async Task<List<SnCloudFileReference>> UpdateResourceFilesAsync(
         string resourceId,
         IEnumerable<string>? newFileIds,
         string usage,
@@ -269,7 +326,7 @@ public class FileReferenceService(AppDatabase db, FileService fileService, ICach
         Duration? duration = null)
     {
         if (newFileIds == null)
-            return new List<CloudFileReference>();
+            return new List<SnCloudFileReference>();
 
         var existingReferences = await db.FileReferences
             .Where(r => r.ResourceId == resourceId && r.Usage == usage)
@@ -287,7 +344,7 @@ public class FileReferenceService(AppDatabase db, FileService fileService, ICach
         // Files to add
         var toAdd = newFileIdsList
             .Where(id => !existingFileIds.Contains(id))
-            .Select(id => new CloudFileReference
+            .Select(id => new SnCloudFileReference
             {
                 FileId = id,
                 Usage = usage,
@@ -347,7 +404,7 @@ public class FileReferenceService(AppDatabase db, FileService fileService, ICach
     /// <param name="resourceId">The ID of the resource</param>
     /// <param name="usage">Optional filter by usage context</param>
     /// <returns>A list of files referenced by the resource</returns>
-    public async Task<List<CloudFile>> GetResourceFilesAsync(string resourceId, string? usage = null)
+    public async Task<List<SnCloudFile>> GetResourceFilesAsync(string resourceId, string? usage = null)
     {
         var query = db.FileReferences.Where(r => r.ResourceId == resourceId);
 
@@ -439,7 +496,7 @@ public class FileReferenceService(AppDatabase db, FileService fileService, ICach
     /// <param name="resourceId">The resource ID</param>
     /// <param name="usageType">The usage type</param>
     /// <returns>List of file references</returns>
-    public async Task<List<CloudFileReference>> GetResourceReferencesAsync(string resourceId, string usageType)
+    public async Task<List<SnCloudFileReference>> GetResourceReferencesAsync(string resourceId, string usageType)
     {
         return await db.FileReferences
             .Where(r => r.ResourceId == resourceId && r.Usage == usageType)

DysonNetwork.Drive/Storage/FileReferenceServiceGrpc.cs

@@ -3,173 +3,172 @@ using Grpc.Core;
 using NodaTime;
 using Duration = NodaTime.Duration;
 
-namespace DysonNetwork.Drive.Storage
+namespace DysonNetwork.Drive.Storage;
+
+public class FileReferenceServiceGrpc(FileReferenceService fileReferenceService)
+    : Shared.Proto.FileReferenceService.FileReferenceServiceBase
 {
-    public class FileReferenceServiceGrpc(FileReferenceService fileReferenceService)
-        : Shared.Proto.FileReferenceService.FileReferenceServiceBase
+    public override async Task<Shared.Proto.CloudFileReference> CreateReference(CreateReferenceRequest request,
+        ServerCallContext context)
     {
-        public override async Task<Shared.Proto.CloudFileReference> CreateReference(CreateReferenceRequest request,
-            ServerCallContext context)
-        {
-            Instant? expiredAt = null;
-            if (request.ExpiredAt != null)
-                expiredAt = Instant.FromUnixTimeSeconds(request.ExpiredAt.Seconds);
-            else if (request.Duration != null)
-                expiredAt = SystemClock.Instance.GetCurrentInstant() +
-                            Duration.FromTimeSpan(request.Duration.ToTimeSpan());
+        Instant? expiredAt = null;
+        if (request.ExpiredAt != null)
+            expiredAt = Instant.FromUnixTimeSeconds(request.ExpiredAt.Seconds);
+        else if (request.Duration != null)
+            expiredAt = SystemClock.Instance.GetCurrentInstant() +
+                        Duration.FromTimeSpan(request.Duration.ToTimeSpan());
 
-            var reference = await fileReferenceService.CreateReferenceAsync(
-                request.FileId,
-                request.Usage,
-                request.ResourceId,
-                expiredAt
-            );
-            return reference.ToProtoValue();
+        var reference = await fileReferenceService.CreateReferenceAsync(
+            request.FileId,
+            request.Usage,
+            request.ResourceId,
+            expiredAt
+        );
+        return reference.ToProtoValue();
+    }
+
+    public override async Task<CreateReferenceBatchResponse> CreateReferenceBatch(CreateReferenceBatchRequest request,
+        ServerCallContext context)
+    {
+        Instant? expiredAt = null;
+        if (request.ExpiredAt != null)
+            expiredAt = Instant.FromUnixTimeSeconds(request.ExpiredAt.Seconds);
+        else if (request.Duration != null)
+            expiredAt = SystemClock.Instance.GetCurrentInstant() +
+                        Duration.FromTimeSpan(request.Duration.ToTimeSpan());
+
+        var references = await fileReferenceService.CreateReferencesAsync(
+            request.FilesId.ToList(),
+            request.Usage,
+            request.ResourceId,
+            expiredAt
+        );
+        var response = new CreateReferenceBatchResponse();
+        response.References.AddRange(references.Select(r => r.ToProtoValue()));
+        return response;
+    }
+
+    public override async Task<GetReferencesResponse> GetReferences(GetReferencesRequest request,
+        ServerCallContext context)
+    {
+        var references = await fileReferenceService.GetReferencesAsync(request.FileId);
+        var response = new GetReferencesResponse();
+        response.References.AddRange(references.Select(r => r.ToProtoValue()));
+        return response;
+    }
+
+    public override async Task<GetReferenceCountResponse> GetReferenceCount(GetReferenceCountRequest request,
+        ServerCallContext context)
+    {
+        var count = await fileReferenceService.GetReferenceCountAsync(request.FileId);
+        return new GetReferenceCountResponse { Count = count };
+    }
+
+    public override async Task<GetReferencesResponse> GetResourceReferences(GetResourceReferencesRequest request,
+        ServerCallContext context)
+    {
+        var references = await fileReferenceService.GetResourceReferencesAsync(request.ResourceId, request.Usage);
+        var response = new GetReferencesResponse();
+        response.References.AddRange(references.Select(r => r.ToProtoValue()));
+        return response;
+    }
+
+    public override async Task<GetResourceFilesResponse> GetResourceFiles(GetResourceFilesRequest request,
+        ServerCallContext context)
+    {
+        var files = await fileReferenceService.GetResourceFilesAsync(request.ResourceId, request.Usage);
+        var response = new GetResourceFilesResponse();
+        response.Files.AddRange(files.Select(f => f.ToProtoValue()));
+        return response;
+    }
+
+    public override async Task<DeleteResourceReferencesResponse> DeleteResourceReferences(
+        DeleteResourceReferencesRequest request, ServerCallContext context)
+    {
+        int deletedCount;
+        if (request.Usage is null)
+            deletedCount = await fileReferenceService.DeleteResourceReferencesAsync(request.ResourceId);
+        else
+            deletedCount =
+                await fileReferenceService.DeleteResourceReferencesAsync(request.ResourceId, request.Usage!);
+        return new DeleteResourceReferencesResponse { DeletedCount = deletedCount };
+    }
+        
+    public override async Task<DeleteResourceReferencesResponse> DeleteResourceReferencesBatch(DeleteResourceReferencesBatchRequest request, ServerCallContext context)
+    {
+        var resourceIds = request.ResourceIds.ToList();
+        int deletedCount;
+        if (request.Usage is null)
+            deletedCount = await fileReferenceService.DeleteResourceReferencesBatchAsync(resourceIds);
+        else
+            deletedCount =
+                await fileReferenceService.DeleteResourceReferencesBatchAsync(resourceIds, request.Usage!);
+        return new DeleteResourceReferencesResponse { DeletedCount = deletedCount };
+    }
+
+    public override async Task<DeleteReferenceResponse> DeleteReference(DeleteReferenceRequest request,
+        ServerCallContext context)
+    {
+        var success = await fileReferenceService.DeleteReferenceAsync(Guid.Parse(request.ReferenceId));
+        return new DeleteReferenceResponse { Success = success };
+    }
+
+    public override async Task<UpdateResourceFilesResponse> UpdateResourceFiles(UpdateResourceFilesRequest request,
+        ServerCallContext context)
+    {
+        Instant? expiredAt = null;
+        if (request.ExpiredAt != null)
+        {
+            expiredAt = Instant.FromUnixTimeSeconds(request.ExpiredAt.Seconds);
+        }
+        else if (request.Duration != null)
+        {
+            expiredAt = SystemClock.Instance.GetCurrentInstant() +
+                        Duration.FromTimeSpan(request.Duration.ToTimeSpan());
         }
 
-        public override async Task<CreateReferenceBatchResponse> CreateReferenceBatch(CreateReferenceBatchRequest request,
-            ServerCallContext context)
-        {
-            Instant? expiredAt = null;
-            if (request.ExpiredAt != null)
-                expiredAt = Instant.FromUnixTimeSeconds(request.ExpiredAt.Seconds);
-            else if (request.Duration != null)
-                expiredAt = SystemClock.Instance.GetCurrentInstant() +
-                            Duration.FromTimeSpan(request.Duration.ToTimeSpan());
+        var references = await fileReferenceService.UpdateResourceFilesAsync(
+            request.ResourceId,
+            request.FileIds,
+            request.Usage,
+            expiredAt
+        );
+        var response = new UpdateResourceFilesResponse();
+        response.References.AddRange(references.Select(r => r.ToProtoValue()));
+        return response;
+    }
 
-            var references = await fileReferenceService.CreateReferencesAsync(
-                request.FilesId.ToList(),
-                request.Usage,
-                request.ResourceId,
-                expiredAt
-            );
-            var response = new CreateReferenceBatchResponse();
-            response.References.AddRange(references.Select(r => r.ToProtoValue()));
-            return response;
+    public override async Task<SetReferenceExpirationResponse> SetReferenceExpiration(
+        SetReferenceExpirationRequest request, ServerCallContext context)
+    {
+        Instant? expiredAt = null;
+        if (request.ExpiredAt != null)
+        {
+            expiredAt = Instant.FromUnixTimeSeconds(request.ExpiredAt.Seconds);
+        }
+        else if (request.Duration != null)
+        {
+            expiredAt = SystemClock.Instance.GetCurrentInstant() +
+                        Duration.FromTimeSpan(request.Duration.ToTimeSpan());
         }
 
-        public override async Task<GetReferencesResponse> GetReferences(GetReferencesRequest request,
-            ServerCallContext context)
-        {
-            var references = await fileReferenceService.GetReferencesAsync(request.FileId);
-            var response = new GetReferencesResponse();
-            response.References.AddRange(references.Select(r => r.ToProtoValue()));
-            return response;
-        }
+        var success =
+            await fileReferenceService.SetReferenceExpirationAsync(Guid.Parse(request.ReferenceId), expiredAt);
+        return new SetReferenceExpirationResponse { Success = success };
+    }
 
-        public override async Task<GetReferenceCountResponse> GetReferenceCount(GetReferenceCountRequest request,
-            ServerCallContext context)
-        {
-            var count = await fileReferenceService.GetReferenceCountAsync(request.FileId);
-            return new GetReferenceCountResponse { Count = count };
-        }
+    public override async Task<SetFileReferencesExpirationResponse> SetFileReferencesExpiration(
+        SetFileReferencesExpirationRequest request, ServerCallContext context)
+    {
+        var expiredAt = Instant.FromUnixTimeSeconds(request.ExpiredAt.Seconds);
+        var updatedCount = await fileReferenceService.SetFileReferencesExpirationAsync(request.FileId, expiredAt);
+        return new SetFileReferencesExpirationResponse { UpdatedCount = updatedCount };
+    }
 
-        public override async Task<GetReferencesResponse> GetResourceReferences(GetResourceReferencesRequest request,
-            ServerCallContext context)
-        {
-            var references = await fileReferenceService.GetResourceReferencesAsync(request.ResourceId, request.Usage);
-            var response = new GetReferencesResponse();
-            response.References.AddRange(references.Select(r => r.ToProtoValue()));
-            return response;
-        }
-
-        public override async Task<GetResourceFilesResponse> GetResourceFiles(GetResourceFilesRequest request,
-            ServerCallContext context)
-        {
-            var files = await fileReferenceService.GetResourceFilesAsync(request.ResourceId, request.Usage);
-            var response = new GetResourceFilesResponse();
-            response.Files.AddRange(files.Select(f => f.ToProtoValue()));
-            return response;
-        }
-
-        public override async Task<DeleteResourceReferencesResponse> DeleteResourceReferences(
-            DeleteResourceReferencesRequest request, ServerCallContext context)
-        {
-            int deletedCount;
-            if (request.Usage is null)
-                deletedCount = await fileReferenceService.DeleteResourceReferencesAsync(request.ResourceId);
-            else
-                deletedCount =
-                    await fileReferenceService.DeleteResourceReferencesAsync(request.ResourceId, request.Usage!);
-            return new DeleteResourceReferencesResponse { DeletedCount = deletedCount };
-        }
-        
-        public override async Task<DeleteResourceReferencesResponse> DeleteResourceReferencesBatch(DeleteResourceReferencesBatchRequest request, ServerCallContext context)
-        {
-            var resourceIds = request.ResourceIds.ToList();
-            int deletedCount;
-            if (request.Usage is null)
-                deletedCount = await fileReferenceService.DeleteResourceReferencesBatchAsync(resourceIds);
-            else
-                deletedCount =
-                    await fileReferenceService.DeleteResourceReferencesBatchAsync(resourceIds, request.Usage!);
-            return new DeleteResourceReferencesResponse { DeletedCount = deletedCount };
-        }
-
-        public override async Task<DeleteReferenceResponse> DeleteReference(DeleteReferenceRequest request,
-            ServerCallContext context)
-        {
-            var success = await fileReferenceService.DeleteReferenceAsync(Guid.Parse(request.ReferenceId));
-            return new DeleteReferenceResponse { Success = success };
-        }
-
-        public override async Task<UpdateResourceFilesResponse> UpdateResourceFiles(UpdateResourceFilesRequest request,
-            ServerCallContext context)
-        {
-            Instant? expiredAt = null;
-            if (request.ExpiredAt != null)
-            {
-                expiredAt = Instant.FromUnixTimeSeconds(request.ExpiredAt.Seconds);
-            }
-            else if (request.Duration != null)
-            {
-                expiredAt = SystemClock.Instance.GetCurrentInstant() +
-                            Duration.FromTimeSpan(request.Duration.ToTimeSpan());
-            }
-
-            var references = await fileReferenceService.UpdateResourceFilesAsync(
-                request.ResourceId,
-                request.FileIds,
-                request.Usage,
-                expiredAt
-            );
-            var response = new UpdateResourceFilesResponse();
-            response.References.AddRange(references.Select(r => r.ToProtoValue()));
-            return response;
-        }
-
-        public override async Task<SetReferenceExpirationResponse> SetReferenceExpiration(
-            SetReferenceExpirationRequest request, ServerCallContext context)
-        {
-            Instant? expiredAt = null;
-            if (request.ExpiredAt != null)
-            {
-                expiredAt = Instant.FromUnixTimeSeconds(request.ExpiredAt.Seconds);
-            }
-            else if (request.Duration != null)
-            {
-                expiredAt = SystemClock.Instance.GetCurrentInstant() +
-                            Duration.FromTimeSpan(request.Duration.ToTimeSpan());
-            }
-
-            var success =
-                await fileReferenceService.SetReferenceExpirationAsync(Guid.Parse(request.ReferenceId), expiredAt);
-            return new SetReferenceExpirationResponse { Success = success };
-        }
-
-        public override async Task<SetFileReferencesExpirationResponse> SetFileReferencesExpiration(
-            SetFileReferencesExpirationRequest request, ServerCallContext context)
-        {
-            var expiredAt = Instant.FromUnixTimeSeconds(request.ExpiredAt.Seconds);
-            var updatedCount = await fileReferenceService.SetFileReferencesExpirationAsync(request.FileId, expiredAt);
-            return new SetFileReferencesExpirationResponse { UpdatedCount = updatedCount };
-        }
-
-        public override async Task<HasFileReferencesResponse> HasFileReferences(HasFileReferencesRequest request,
-            ServerCallContext context)
-        {
-            var hasReferences = await fileReferenceService.HasFileReferencesAsync(request.FileId);
-            return new HasFileReferencesResponse { HasReferences = hasReferences };
-        }
+    public override async Task<HasFileReferencesResponse> HasFileReferences(HasFileReferencesRequest request,
+        ServerCallContext context)
+    {
+        var hasReferences = await fileReferenceService.HasFileReferencesAsync(request.FileId);
+        return new HasFileReferencesResponse { HasReferences = hasReferences };
     }
 }

DysonNetwork.Drive/Storage/FileService.cs

@@ -1,46 +1,38 @@
-using System.Drawing;
 using System.Globalization;
 using FFMpegCore;
 using System.Security.Cryptography;
+using DysonNetwork.Drive.Storage.Model;
 using DysonNetwork.Shared.Cache;
 using DysonNetwork.Shared.Proto;
 using Google.Protobuf.WellKnownTypes;
 using Microsoft.EntityFrameworkCore;
 using Minio;
 using Minio.DataModel.Args;
+using NATS.Client.Core;
 using NetVips;
 using NodaTime;
-using tusdotnet.Stores;
 using System.Linq.Expressions;
-using DysonNetwork.Shared.Data;
 using Microsoft.EntityFrameworkCore.Query;
+using NATS.Net;
+using DysonNetwork.Shared.Models;
 
 namespace DysonNetwork.Drive.Storage;
 
 public class FileService(
     AppDatabase db,
-    IConfiguration configuration,
     ILogger<FileService> logger,
-    IServiceScopeFactory scopeFactory,
-    ICacheService cache
+    ICacheService cache,
+    INatsConnection nats
 )
 {
     private const string CacheKeyPrefix = "file:";
     private static readonly TimeSpan CacheDuration = TimeSpan.FromMinutes(15);
 
-    /// <summary>
-    /// The api for getting file meta with cache,
-    /// the best use case is for accessing the file data.
-    ///
-    /// <b>This function won't load uploader's information, only keep minimal file meta</b>
-    /// </summary>
-    /// <param name="fileId">The id of the cloud file requested</param>
-    /// <returns>The minimal file meta</returns>
-    public async Task<CloudFile?> GetFileAsync(string fileId)
+    public async Task<SnCloudFile?> GetFileAsync(string fileId)
     {
         var cacheKey = $"{CacheKeyPrefix}{fileId}";
 
-        var cachedFile = await cache.GetAsync<CloudFile>(cacheKey);
+        var cachedFile = await cache.GetAsync<SnCloudFile>(cacheKey);
         if (cachedFile is not null)
             return cachedFile;
 
@@ -56,16 +48,15 @@ public class FileService(
         return file;
     }
 
-    public async Task<List<CloudFile>> GetFilesAsync(List<string> fileIds)
+    public async Task<List<SnCloudFile>> GetFilesAsync(List<string> fileIds)
     {
-        var cachedFiles = new Dictionary<string, CloudFile>();
+        var cachedFiles = new Dictionary<string, SnCloudFile>();
         var uncachedIds = new List<string>();
 
-        // Check cache first
         foreach (var fileId in fileIds)
         {
             var cacheKey = $"{CacheKeyPrefix}{fileId}";
-            var cachedFile = await cache.GetAsync<CloudFile>(cacheKey);
+            var cachedFile = await cache.GetAsync<SnCloudFile>(cacheKey);
 
             if (cachedFile != null)
                 cachedFiles[fileId] = cachedFile;
@@ -73,7 +64,6 @@ public class FileService(
                 uncachedIds.Add(fileId);
         }
 
-        // Load uncached files from database
         if (uncachedIds.Count > 0)
         {
             var dbFiles = await db.Files
@@ -81,7 +71,6 @@ public class FileService(
                 .Include(f => f.Pool)
                 .ToListAsync();
 
-            // Add to cache
             foreach (var file in dbFiles)
             {
                 var cacheKey = $"{CacheKeyPrefix}{file.Id}";
@@ -90,28 +79,19 @@ public class FileService(
             }
         }
 
-        // Preserve original order
         return fileIds
             .Select(f => cachedFiles.GetValueOrDefault(f))
             .Where(f => f != null)
-            .Cast<CloudFile>()
+            .Cast<SnCloudFile>()
             .ToList();
     }
 
-    private const string TempFilePrefix = "dyn-cloudfile";
-
-    private static readonly string[] AnimatedImageTypes =
-        ["image/gif", "image/apng", "image/avif"];
-
-    private static readonly string[] AnimatedImageExtensions =
-        [".gif", ".apng", ".avif"];
-
-    public async Task<CloudFile> ProcessNewFileAsync(
+    public async Task<SnCloudFile> ProcessNewFileAsync(
         Account account,
         string fileId,
         string filePool,
         string? fileBundleId,
-        Stream stream,
+        string filePath,
         string fileName,
         string? contentType,
         string? encryptPassword,
@@ -119,81 +99,158 @@ public class FileService(
     )
     {
         var accountId = Guid.Parse(account.Id);
+        var pool = await ValidateAndGetPoolAsync(filePool);
+        var bundle = await ValidateAndGetBundleAsync(fileBundleId, accountId);
+        var finalExpiredAt = CalculateFinalExpiration(expiredAt, pool, bundle);
 
+        var (managedTempPath, fileSize, finalContentType) =
+            await PrepareFileAsync(fileId, filePath, fileName, contentType);
+
+        var file = CreateFileObject(fileId, fileName, finalContentType, fileSize, finalExpiredAt, bundle, accountId);
+
+        if (!pool.PolicyConfig.NoMetadata)
+        {
+            await ExtractMetadataAsync(file, managedTempPath);
+        }
+
+        var (processingPath, isTempFile) =
+            await ProcessEncryptionAsync(fileId, managedTempPath, encryptPassword, pool, file);
+
+        file.Hash = await HashFileAsync(processingPath);
+
+        await SaveFileToDatabaseAsync(file);
+
+        await PublishFileUploadedEventAsync(file, pool, processingPath, isTempFile);
+
+        return file;
+    }
+
+    private async Task<FilePool> ValidateAndGetPoolAsync(string filePool)
+    {
         var pool = await GetPoolAsync(Guid.Parse(filePool));
-        if (pool is null) throw new InvalidOperationException("Pool not found");
+        return pool ?? throw new InvalidOperationException("Pool not found: " + filePool);
+    }
 
+    private async Task<SnFileBundle?> ValidateAndGetBundleAsync(string? fileBundleId, Guid accountId)
+    {
+        if (fileBundleId is null) return null;
+
+        var bundle = await GetBundleAsync(Guid.Parse(fileBundleId), accountId);
+        return bundle ?? throw new InvalidOperationException("Bundle not found: " + fileBundleId);
+    }
+
+    private static Instant? CalculateFinalExpiration(Instant? expiredAt, FilePool pool, SnFileBundle? bundle)
+    {
+        var finalExpiredAt = expiredAt;
+
+        // Apply pool expiration policy
         if (pool.StorageConfig.Expiration is not null && expiredAt.HasValue)
         {
             var expectedExpiration = SystemClock.Instance.GetCurrentInstant() - expiredAt.Value;
             var effectiveExpiration = pool.StorageConfig.Expiration < expectedExpiration
                 ? pool.StorageConfig.Expiration
                 : expectedExpiration;
-            expiredAt = SystemClock.Instance.GetCurrentInstant() + effectiveExpiration;
-        }
-
-        var bundle = fileBundleId is not null
-            ? await GetBundleAsync(Guid.Parse(fileBundleId), accountId)
-            : null;
-        if (fileBundleId is not null && bundle is null)
-        {
-            throw new InvalidOperationException("Bundle not found");
+            finalExpiredAt = SystemClock.Instance.GetCurrentInstant() + effectiveExpiration;
         }
 
+        // Bundle expiration takes precedence
         if (bundle?.ExpiredAt != null)
-            expiredAt = bundle.ExpiredAt.Value;
+            finalExpiredAt = bundle.ExpiredAt.Value;
 
-        var ogFilePath = Path.GetFullPath(Path.Join(configuration.GetValue<string>("Tus:StorePath"), fileId));
-        var fileSize = stream.Length;
-        contentType ??= !fileName.Contains('.') ? "application/octet-stream" : MimeTypes.GetMimeType(fileName);
+        return finalExpiredAt;
+    }
 
-        if (!string.IsNullOrWhiteSpace(encryptPassword))
-        {
-            if (!pool.PolicyConfig.AllowEncryption)
-                throw new InvalidOperationException("Encryption is not allowed in this pool");
-            var encryptedPath = Path.Combine(Path.GetTempPath(), $"{fileId}.encrypted");
-            FileEncryptor.EncryptFile(ogFilePath, encryptedPath, encryptPassword);
-            File.Delete(ogFilePath); // Delete original unencrypted
-            File.Move(encryptedPath, ogFilePath); // Replace the original one with encrypted
-            contentType = "application/octet-stream";
-        }
+    private async Task<(string tempPath, long fileSize, string contentType)> PrepareFileAsync(
+        string fileId,
+        string filePath,
+        string fileName,
+        string? contentType
+    )
+    {
+        var managedTempPath = Path.Combine(Path.GetTempPath(), fileId);
+        File.Copy(filePath, managedTempPath, true);
 
-        var hash = await HashFileAsync(ogFilePath);
+        var fileInfo = new FileInfo(managedTempPath);
+        var fileSize = fileInfo.Length;
+        var finalContentType = contentType ??
+                               (!fileName.Contains('.') ? "application/octet-stream" : MimeTypes.GetMimeType(fileName));
 
-        var file = new CloudFile
+        return (managedTempPath, fileSize, finalContentType);
+    }
+
+    private SnCloudFile CreateFileObject(
+        string fileId,
+        string fileName,
+        string contentType,
+        long fileSize,
+        Instant? expiredAt,
+        SnFileBundle? bundle,
+        Guid accountId
+    )
+    {
+        return new SnCloudFile
         {
             Id = fileId,
             Name = fileName,
             MimeType = contentType,
             Size = fileSize,
-            Hash = hash,
             ExpiredAt = expiredAt,
             BundleId = bundle?.Id,
-            AccountId = Guid.Parse(account.Id),
-            IsEncrypted = !string.IsNullOrWhiteSpace(encryptPassword) && pool.PolicyConfig.AllowEncryption
+            AccountId = accountId,
         };
-
-        // Extract metadata on the current thread for a faster initial response
-        if (!pool.PolicyConfig.NoMetadata)
-            await ExtractMetadataAsync(file, ogFilePath, stream);
-
-        db.Files.Add(file);
-        await db.SaveChangesAsync();
-
-        file.StorageId ??= file.Id;
-
-        // Offload optimization (image conversion, thumbnailing) and uploading to a background task
-        _ = Task.Run(() =>
-            ProcessAndUploadInBackgroundAsync(file.Id, filePool, file.StorageId, contentType, ogFilePath, stream));
-
-        return file;
     }
 
-    /// <summary>
-    /// Extracts metadata from the file based on its content type.
-    /// This runs synchronously to ensure the initial database record has basic metadata.
-    /// </summary>
-    private async Task ExtractMetadataAsync(CloudFile file, string filePath, Stream stream)
+    private async Task<(string processingPath, bool isTempFile)> ProcessEncryptionAsync(
+        string fileId,
+        string managedTempPath,
+        string? encryptPassword,
+        FilePool pool,
+        SnCloudFile file
+    )
+    {
+        if (string.IsNullOrWhiteSpace(encryptPassword))
+            return (managedTempPath, true);
+
+        if (!pool.PolicyConfig.AllowEncryption)
+            throw new InvalidOperationException("Encryption is not allowed in this pool");
+
+        var encryptedPath = Path.Combine(Path.GetTempPath(), $"{fileId}.encrypted");
+        FileEncryptor.EncryptFile(managedTempPath, encryptedPath, encryptPassword);
+
+        File.Delete(managedTempPath);
+
+        file.IsEncrypted = true;
+        file.MimeType = "application/octet-stream";
+        file.Size = new FileInfo(encryptedPath).Length;
+
+        return (encryptedPath, true);
+    }
+
+    private async Task SaveFileToDatabaseAsync(SnCloudFile file)
+    {
+        db.Files.Add(file);
+        await db.SaveChangesAsync();
+        file.StorageId ??= file.Id;
+    }
+
+    private async Task PublishFileUploadedEventAsync(SnCloudFile file, FilePool pool, string processingPath,
+        bool isTempFile)
+    {
+        var js = nats.CreateJetStreamContext();
+        await js.PublishAsync(
+            FileUploadedEvent.Type,
+            GrpcTypeHelper.ConvertObjectToByteString(new FileUploadedEventPayload(
+                file.Id,
+                pool.Id,
+                file.StorageId,
+                file.MimeType,
+                processingPath,
+                isTempFile)
+            ).ToByteArray()
+        );
+    }
+
+    private async Task ExtractMetadataAsync(SnCloudFile file, string filePath)
     {
         switch (file.MimeType?.Split('/')[0])
         {
@@ -201,6 +258,7 @@ public class FileService(
                 try
                 {
                     var blurhash = BlurHashSharp.SkiaSharp.BlurHashEncoder.Encode(3, 3, filePath);
+                    await using var stream = new FileStream(filePath, FileMode.Open, FileAccess.Read, FileShare.Read);
                     stream.Position = 0;
 
                     using var vipsImage = Image.NewFromStream(stream);
@@ -265,7 +323,6 @@ public class FileService(
                         ["bit_rate"] = mediaInfo.Format.BitRate.ToString(CultureInfo.InvariantCulture),
                         ["tags"] = mediaInfo.Format.Tags ?? new Dictionary<string, string>(),
                         ["chapters"] = mediaInfo.Chapters,
-                        // Add detailed stream information
                         ["video_streams"] = mediaInfo.VideoStreams.Select(s => new
                         {
                             s.AvgFrameRate,
@@ -303,166 +360,6 @@ public class FileService(
         }
     }
 
-    /// <summary>
-    /// Handles file optimization (image compression, video thumbnail) and uploads to remote storage in the background.
-    /// </summary>
-    private async Task ProcessAndUploadInBackgroundAsync(
-        string fileId,
-        string remoteId,
-        string storageId,
-        string contentType,
-        string originalFilePath,
-        Stream stream
-    )
-    {
-        var pool = await GetPoolAsync(Guid.Parse(remoteId));
-        if (pool is null) return;
-
-        await using var bgStream = stream; // Ensure stream is disposed at the end of this task
-        using var scope = scopeFactory.CreateScope();
-        var nfs = scope.ServiceProvider.GetRequiredService<FileService>();
-        var scopedDb = scope.ServiceProvider.GetRequiredService<AppDatabase>();
-
-        var uploads = new List<(string FilePath, string Suffix, string ContentType, bool SelfDestruct)>();
-        var newMimeType = contentType;
-        var hasCompression = false;
-        var hasThumbnail = false;
-
-        try
-        {
-            logger.LogInformation("Processing file {FileId} in background...", fileId);
-
-            var fileExtension = Path.GetExtension(originalFilePath);
-
-            if (!pool.PolicyConfig.NoOptimization)
-                switch (contentType.Split('/')[0])
-                {
-                    case "image":
-                        if (AnimatedImageTypes.Contains(contentType) || AnimatedImageExtensions.Contains(fileExtension))
-                        {
-                            logger.LogInformation("Skip optimize file {FileId} due to it is animated...", fileId);
-                            uploads.Add((originalFilePath, string.Empty, contentType, false));
-                            break;
-                        }
-
-                        newMimeType = "image/webp";
-                        using (var vipsImage = Image.NewFromFile(originalFilePath))
-                        {
-                            var imageToWrite = vipsImage;
-
-                            if (vipsImage.Interpretation is Enums.Interpretation.Scrgb or Enums.Interpretation.Xyz)
-                            {
-                                imageToWrite = vipsImage.Colourspace(Enums.Interpretation.Srgb);
-                            }
-
-                            var webpPath = Path.Join(Path.GetTempPath(), $"{TempFilePrefix}#{fileId}.webp");
-                            imageToWrite.Autorot().WriteToFile(webpPath,
-                                new VOption { { "lossless", true }, { "strip", true } });
-                            uploads.Add((webpPath, string.Empty, newMimeType, true));
-
-                            if (imageToWrite.Width * imageToWrite.Height >= 1024 * 1024)
-                            {
-                                var scale = 1024.0 / Math.Max(imageToWrite.Width, imageToWrite.Height);
-                                var compressedPath =
-                                    Path.Join(Path.GetTempPath(), $"{TempFilePrefix}#{fileId}-compressed.webp");
-                                using var compressedImage = imageToWrite.Resize(scale);
-                                compressedImage.Autorot().WriteToFile(compressedPath,
-                                    new VOption { { "Q", 80 }, { "strip", true } });
-                                uploads.Add((compressedPath, ".compressed", newMimeType, true));
-                                hasCompression = true;
-                            }
-
-                            if (!ReferenceEquals(imageToWrite, vipsImage))
-                            {
-                                imageToWrite.Dispose(); // Clean up manually created colourspace-converted image
-                            }
-                        }
-
-                        break;
-
-                    case "video":
-                        uploads.Add((originalFilePath, string.Empty, contentType, false));
-
-                        var thumbnailPath = Path.Join(Path.GetTempPath(), $"{TempFilePrefix}#{fileId}.thumbnail.jpg");
-                        try
-                        {
-                            await FFMpegArguments
-                                .FromFileInput(originalFilePath, verifyExists: true)
-                                .OutputToFile(thumbnailPath, overwrite: true, options => options
-                                    .Seek(TimeSpan.FromSeconds(0))
-                                    .WithFrameOutputCount(1)
-                                    .WithCustomArgument("-q:v 2")
-                                )
-                                .NotifyOnOutput(line => logger.LogInformation("[FFmpeg] {Line}", line))
-                                .NotifyOnError(line => logger.LogWarning("[FFmpeg] {Line}", line))
-                                .ProcessAsynchronously();
-
-                            if (File.Exists(thumbnailPath))
-                            {
-                                uploads.Add((thumbnailPath, ".thumbnail", "image/jpeg", true));
-                                hasThumbnail = true;
-                            }
-                            else
-                            {
-                                logger.LogWarning("FFMpeg did not produce thumbnail for video {FileId}", fileId);
-                            }
-                        }
-                        catch (Exception ex)
-                        {
-                            logger.LogError(ex, "Failed to generate thumbnail for video {FileId}", fileId);
-                        }
-
-                        break;
-
-                    default:
-                        uploads.Add((originalFilePath, string.Empty, contentType, false));
-                        break;
-                }
-            else uploads.Add((originalFilePath, string.Empty, contentType, false));
-
-            logger.LogInformation("Optimized file {FileId}, now uploading...", fileId);
-
-            if (uploads.Count > 0)
-            {
-                var destPool = Guid.Parse(remoteId!);
-                var uploadTasks = uploads.Select(item =>
-                    nfs.UploadFileToRemoteAsync(
-                        storageId,
-                        destPool,
-                        item.FilePath,
-                        item.Suffix,
-                        item.ContentType,
-                        item.SelfDestruct
-                    )
-                ).ToList();
-
-                await Task.WhenAll(uploadTasks);
-
-                logger.LogInformation("Uploaded file {FileId} done!", fileId);
-
-                var fileToUpdate = await scopedDb.Files.FirstAsync(f => f.Id == fileId);
-                if (hasThumbnail) fileToUpdate.HasThumbnail = true;
-
-                var now = SystemClock.Instance.GetCurrentInstant();
-                await scopedDb.Files.Where(f => f.Id == fileId).ExecuteUpdateAsync(setter => setter
-                    .SetProperty(f => f.UploadedAt, now)
-                    .SetProperty(f => f.PoolId, destPool)
-                    .SetProperty(f => f.MimeType, newMimeType)
-                    .SetProperty(f => f.HasCompression, hasCompression)
-                    .SetProperty(f => f.HasThumbnail, hasThumbnail)
-                );
-            }
-        }
-        catch (Exception err)
-        {
-            logger.LogError(err, "Failed to process and upload {FileId}", fileId);
-        }
-        finally
-        {
-            await nfs._PurgeCacheAsync(fileId);
-        }
-    }
-
     private static async Task<string> HashFileAsync(string filePath, int chunkSize = 1024 * 1024)
     {
         var fileInfo = new FileInfo(filePath);
@@ -491,11 +388,11 @@ public class FileService(
         }
 
         var hash = MD5.HashData(buffer.AsSpan(0, bytesRead));
-        stream.Position = 0; // Reset stream position
+        stream.Position = 0;
         return Convert.ToHexString(hash).ToLowerInvariant();
     }
 
-    private async Task UploadFileToRemoteAsync(
+    public async Task UploadFileToRemoteAsync(
         string storageId,
         Guid targetRemote,
         string filePath,
@@ -536,7 +433,7 @@ public class FileService(
         );
     }
 
-    public async Task<CloudFile> UpdateFileAsync(CloudFile file, FieldMask updateMask)
+    public async Task<SnCloudFile> UpdateFileAsync(SnCloudFile file, FieldMask updateMask)
     {
         var existingFile = await db.Files.FirstOrDefaultAsync(f => f.Id == file.Id);
         if (existingFile == null)
@@ -574,37 +471,34 @@ public class FileService(
         await db.Files.Where(f => f.Id == file.Id).ExecuteUpdateAsync(updatable.ToSetPropertyCalls());
 
         await _PurgeCacheAsync(file.Id);
-        // Re-fetch the file to return the updated state
         return await db.Files.AsNoTracking().FirstAsync(f => f.Id == file.Id);
     }
 
-    public async Task DeleteFileAsync(CloudFile file)
+    public async Task DeleteFileAsync(SnCloudFile file, bool skipData = false)
     {
         db.Remove(file);
         await db.SaveChangesAsync();
         await _PurgeCacheAsync(file.Id);
 
-        await DeleteFileDataAsync(file);
+        if (!skipData)
+            await DeleteFileDataAsync(file);
     }
 
-    public async Task DeleteFileDataAsync(CloudFile file, bool force = false)
+    public async Task DeleteFileDataAsync(SnCloudFile file, bool force = false)
     {
         if (!file.PoolId.HasValue) return;
 
         if (!force)
         {
-            // Check if any other file with the same storage ID is referenced
             var sameOriginFiles = await db.Files
                 .Where(f => f.StorageId == file.StorageId && f.Id != file.Id)
                 .Select(f => f.Id)
                 .ToListAsync();
 
-            // Check if any of these files are referenced
             if (sameOriginFiles.Count != 0)
                 return;
         }
 
-        // If any other file with the same storage ID is referenced, don't delete the actual file data
         var dest = await GetRemoteStorageConfig(file.PoolId.Value);
         if (dest is null) throw new InvalidOperationException($"No remote storage configured for pool {file.PoolId}");
         var client = CreateMinioClient(dest);
@@ -614,7 +508,7 @@ public class FileService(
             );
 
         var bucket = dest.Bucket;
-        var objectId = file.StorageId ?? file.Id; // Use StorageId if available, otherwise fall back to Id
+        var objectId = file.StorageId ?? file.Id;
 
         await client.RemoveObjectAsync(
             new RemoveObjectArgs().WithBucket(bucket).WithObject(objectId)
@@ -630,7 +524,6 @@ public class FileService(
             }
             catch
             {
-                // Ignore errors when deleting compressed version
                 logger.LogWarning("Failed to delete compressed version of file {fileId}", file.Id);
             }
         }
@@ -645,25 +538,17 @@ public class FileService(
             }
             catch
             {
-                // Ignore errors when deleting thumbnail
                 logger.LogWarning("Failed to delete thumbnail of file {fileId}", file.Id);
             }
         }
     }
 
-    /// <summary>
-    /// The most efficent way to delete file data (stored files) in batch.
-    /// But this DO NOT check the storage id, so use with caution!
-    /// </summary>
-    /// <param name="files">Files to delete</param>
-    /// <exception cref="InvalidOperationException">Something went wrong</exception>
-    public async Task DeleteFileDataBatchAsync(List<CloudFile> files)
+    public async Task DeleteFileDataBatchAsync(List<SnCloudFile> files)
     {
         files = files.Where(f => f.PoolId.HasValue).ToList();
 
         foreach (var fileGroup in files.GroupBy(f => f.PoolId!.Value))
         {
-            // If any other file with the same storage ID is referenced, don't delete the actual file data
             var dest = await GetRemoteStorageConfig(fileGroup.Key);
             if (dest is null)
                 throw new InvalidOperationException($"No remote storage configured for pool {fileGroup.Key}");
@@ -688,7 +573,7 @@ public class FileService(
         }
     }
 
-    private async Task<FileBundle?> GetBundleAsync(Guid id, Guid accountId)
+    private async Task<SnFileBundle?> GetBundleAsync(Guid id, Guid accountId)
     {
         var bundle = await db.Bundles
             .Where(e => e.Id == id)
@@ -733,31 +618,27 @@ public class FileService(
         return client.Build();
     }
 
-    // Helper method to purge the cache for a specific file
-    // Made internal to allow FileReferenceService to use it
     internal async Task _PurgeCacheAsync(string fileId)
     {
         var cacheKey = $"{CacheKeyPrefix}{fileId}";
         await cache.RemoveAsync(cacheKey);
     }
 
-    // Helper method to purge cache for multiple files
     internal async Task _PurgeCacheRangeAsync(IEnumerable<string> fileIds)
     {
         var tasks = fileIds.Select(_PurgeCacheAsync);
         await Task.WhenAll(tasks);
     }
 
-    public async Task<List<CloudFile?>> LoadFromReference(List<CloudFileReferenceObject> references)
+    public async Task<List<SnCloudFile?>> LoadFromReference(List<SnCloudFileReferenceObject> references)
     {
-        var cachedFiles = new Dictionary<string, CloudFile>();
+        var cachedFiles = new Dictionary<string, SnCloudFile>();
         var uncachedIds = new List<string>();
 
-        // Check cache first
         foreach (var reference in references)
         {
             var cacheKey = $"{CacheKeyPrefix}{reference.Id}";
-            var cachedFile = await cache.GetAsync<CloudFile>(cacheKey);
+            var cachedFile = await cache.GetAsync<SnCloudFile>(cacheKey);
 
             if (cachedFile != null)
             {
@@ -769,14 +650,12 @@ public class FileService(
             }
         }
 
-        // Load uncached files from database
         if (uncachedIds.Count > 0)
         {
             var dbFiles = await db.Files
                 .Where(f => uncachedIds.Contains(f.Id))
                 .ToListAsync();
 
-            // Add to cache
             foreach (var file in dbFiles)
             {
                 var cacheKey = $"{CacheKeyPrefix}{file.Id}";
@@ -785,18 +664,14 @@ public class FileService(
             }
         }
 
-        // Preserve original order
-        return references
-            .Select(r => cachedFiles.GetValueOrDefault(r.Id))
-            .Where(f => f != null)
-            .ToList();
+        return
+        [
+            .. references
+                .Select(r => cachedFiles.GetValueOrDefault(r.Id))
+                .Where(f => f != null)
+        ];
     }
 
-    /// <summary>
-    /// Gets the number of references to a file based on CloudFileReference records
-    /// </summary>
-    /// <param name="fileId">The ID of the file</param>
-    /// <returns>The number of references to the file</returns>
     public async Task<int> GetReferenceCountAsync(string fileId)
     {
         return await db.FileReferences
@@ -804,11 +679,6 @@ public class FileService(
             .CountAsync();
     }
 
-    /// <summary>
-    /// Checks if a file is referenced by any resource
-    /// </summary>
-    /// <param name="fileId">The ID of the file to check</param>
-    /// <returns>True if the file is referenced, false otherwise</returns>
     public async Task<bool> IsReferencedAsync(string fileId)
     {
         return await db.FileReferences
@@ -816,12 +686,8 @@ public class FileService(
             .AnyAsync();
     }
 
-    /// <summary>
-    /// Checks if an EXIF field should be ignored (e.g., GPS data).
-    /// </summary>
     private static bool IsIgnoredField(string fieldName)
     {
-        // Common GPS EXIF field names
         var gpsFields = new[]
         {
             "gps-latitude", "gps-longitude", "gps-altitude", "gps-latitude-ref", "gps-longitude-ref",
@@ -852,6 +718,21 @@ public class FileService(
         return count;
     }
 
+    public async Task<int> DeleteAccountFileBatchAsync(Guid accountId, List<string> fileIds)
+    {
+        var files = await db.Files
+            .Where(f => f.AccountId == accountId && fileIds.Contains(f.Id))
+            .ToListAsync();
+        var count = files.Count;
+        var tasks = files.Select(f => DeleteFileDataAsync(f, true));
+        await Task.WhenAll(tasks);
+        var fileIdsList = files.Select(f => f.Id).ToList();
+        await _PurgeCacheRangeAsync(fileIdsList);
+        db.RemoveRange(files);
+        await db.SaveChangesAsync();
+        return count;
+    }
+
     public async Task<int> DeletePoolRecycledFilesAsync(Guid poolId)
     {
         var files = await db.Files
@@ -882,7 +763,7 @@ public class FileService(
         return count;
     }
 
-    public async Task<string> CreateFastUploadLinkAsync(CloudFile file)
+    public async Task<string> CreateFastUploadLinkAsync(SnCloudFile file)
     {
         if (file.PoolId is null) throw new InvalidOperationException("Pool ID is null");
 
@@ -904,10 +785,7 @@ public class FileService(
     }
 }
 
-/// <summary>
-/// A helper class to build an ExecuteUpdateAsync call for CloudFile.
-/// </summary>
-file class UpdatableCloudFile(CloudFile file)
+file class UpdatableCloudFile(SnCloudFile file)
 {
     public string Name { get; set; } = file.Name;
     public string? Description { get; set; } = file.Description;
@@ -915,14 +793,14 @@ file class UpdatableCloudFile(CloudFile file)
     public Dictionary<string, object?>? UserMeta { get; set; } = file.UserMeta;
     public bool IsMarkedRecycle { get; set; } = file.IsMarkedRecycle;
 
-    public Expression<Func<SetPropertyCalls<CloudFile>, SetPropertyCalls<CloudFile>>> ToSetPropertyCalls()
+    public Action<UpdateSettersBuilder<SnCloudFile>> ToSetPropertyCalls()
     {
-        var userMeta = UserMeta ?? new Dictionary<string, object?>();
+        var userMeta = UserMeta ?? [];
         return setter => setter
             .SetProperty(f => f.Name, Name)
             .SetProperty(f => f.Description, Description)
             .SetProperty(f => f.FileMeta, FileMeta)
-            .SetProperty(f => f.UserMeta, userMeta!)
+            .SetProperty(f => f.UserMeta, userMeta)
             .SetProperty(f => f.IsMarkedRecycle, IsMarkedRecycle);
     }
 }

DysonNetwork.Drive/Storage/FileServiceGrpc.cs

@@ -1,4 +1,4 @@
-using DysonNetwork.Shared.Data;
+using DysonNetwork.Shared.Models;
 using DysonNetwork.Shared.Proto;
 using Google.Protobuf.WellKnownTypes;
 using Grpc.Core;
@@ -48,7 +48,7 @@ namespace DysonNetwork.Drive.Storage
         {
             // Assuming CloudFileReferenceObject is a simple class/struct that holds an ID
             // You might need to define this or adjust the LoadFromReference method in FileService
-            var references = request.ReferenceIds.Select(id => new CloudFileReferenceObject { Id = id }).ToList();
+            var references = request.ReferenceIds.Select(id => new SnCloudFileReferenceObject { Id = id }).ToList();
             var files = await fileService.LoadFromReference(references);
             var response = new LoadFromReferenceResponse();
             response.Files.AddRange(files.Where(f => f != null).Select(f => f!.ToProtoValue()));

DysonNetwork.Drive/Storage/FileUploadController.cs

@@ -0,0 +1,687 @@
+using System.ComponentModel.DataAnnotations;
+using DysonNetwork.Drive.Billing;
+using DysonNetwork.Drive.Index;
+using DysonNetwork.Drive.Storage.Model;
+using DysonNetwork.Shared.Auth;
+using DysonNetwork.Shared.Http;
+using DysonNetwork.Shared.Models;
+using DysonNetwork.Shared.Proto;
+using Microsoft.AspNetCore.Authorization;
+using Microsoft.AspNetCore.Mvc;
+using Microsoft.EntityFrameworkCore;
+using NanoidDotNet;
+using NodaTime;
+using TaskStatus = DysonNetwork.Drive.Storage.Model.TaskStatus;
+
+namespace DysonNetwork.Drive.Storage;
+
+[ApiController]
+[Route("/api/files/upload")]
+[Authorize]
+public class FileUploadController(
+    IConfiguration configuration,
+    FileService fileService,
+    AppDatabase db,
+    PermissionService.PermissionServiceClient permission,
+    QuotaService quotaService,
+    PersistentTaskService persistentTaskService,
+    FileIndexService fileIndexService,
+    ILogger<FileUploadController> logger
+)
+    : ControllerBase
+{
+    private readonly string _tempPath =
+        configuration.GetValue<string>("Storage:Uploads") ?? Path.Combine(Path.GetTempPath(), "multipart-uploads");
+
+    private const long DefaultChunkSize = 1024 * 1024 * 5; // 5MB
+
+    [HttpPost("create")]
+    public async Task<IActionResult> CreateUploadTask([FromBody] CreateUploadTaskRequest request)
+    {
+        if (HttpContext.Items["CurrentUser"] is not Account currentUser)
+            return new ObjectResult(ApiError.Unauthorized()) { StatusCode = 401 };
+
+        var permissionCheck = await ValidateUserPermissions(currentUser);
+        if (permissionCheck is not null) return permissionCheck;
+
+        request.PoolId ??= Guid.Parse(configuration["Storage:PreferredRemote"]!);
+
+        var pool = await fileService.GetPoolAsync(request.PoolId.Value);
+        if (pool is null)
+            return new ObjectResult(ApiError.NotFound("Pool")) { StatusCode = 404 };
+
+        var poolValidation = await ValidatePoolAccess(currentUser, pool, request);
+        if (poolValidation is not null) return poolValidation;
+
+        var policyValidation = ValidatePoolPolicy(pool.PolicyConfig, request);
+        if (policyValidation is not null) return policyValidation;
+
+        var quotaValidation = await ValidateQuota(currentUser, pool, request.FileSize);
+        if (quotaValidation is not null) return quotaValidation;
+
+        EnsureTempDirectoryExists();
+
+        var accountId = Guid.Parse(currentUser.Id);
+
+        // Check if a file with the same hash already exists
+        var existingFile = await db.Files.FirstOrDefaultAsync(f => f.Hash == request.Hash);
+        if (existingFile != null)
+        {
+            // Create the file index if a path is provided, even for existing files
+            if (string.IsNullOrEmpty(request.Path))
+                return Ok(new CreateUploadTaskResponse
+                {
+                    FileExists = true,
+                    File = existingFile
+                });
+            try
+            {
+                await fileIndexService.CreateAsync(request.Path, existingFile.Id, accountId);
+                logger.LogInformation("Created file index for existing file {FileId} at path {Path}",
+                    existingFile.Id, request.Path);
+            }
+            catch (Exception ex)
+            {
+                logger.LogWarning(ex, "Failed to create file index for existing file {FileId} at path {Path}",
+                    existingFile.Id, request.Path);
+                // Don't fail the request if index creation fails, just log it
+            }
+
+            return Ok(new CreateUploadTaskResponse
+            {
+                FileExists = true,
+                File = existingFile
+            });
+        }
+
+        var taskId = await Nanoid.GenerateAsync();
+
+        // Create persistent upload task
+        var persistentTask = await persistentTaskService.CreateUploadTaskAsync(taskId, request, accountId);
+
+        return Ok(new CreateUploadTaskResponse
+        {
+            FileExists = false,
+            TaskId = taskId,
+            ChunkSize = persistentTask.ChunkSize,
+            ChunksCount = persistentTask.ChunksCount
+        });
+    }
+
+    private async Task<IActionResult?> ValidateUserPermissions(Account currentUser)
+    {
+        if (currentUser.IsSuperuser) return null;
+
+        var allowed = await permission.HasPermissionAsync(new HasPermissionRequest
+            { Actor = currentUser.Id, Key = "files.create" });
+
+        return allowed.HasPermission
+            ? null
+            : new ObjectResult(ApiError.Unauthorized(forbidden: true)) { StatusCode = 403 };
+    }
+
+    private Task<IActionResult?> ValidatePoolAccess(Account currentUser, FilePool pool, CreateUploadTaskRequest request)
+    {
+        if (pool.PolicyConfig.RequirePrivilege <= 0) return Task.FromResult<IActionResult?>(null);
+
+        var privilege = currentUser.PerkSubscription is null
+            ? 0
+            : PerkSubscriptionPrivilege.GetPrivilegeFromIdentifier(currentUser.PerkSubscription.Identifier);
+
+        if (privilege < pool.PolicyConfig.RequirePrivilege)
+        {
+            return Task.FromResult<IActionResult?>(new ObjectResult(ApiError.Unauthorized(
+                    $"You need Stellar Program tier {pool.PolicyConfig.RequirePrivilege} to use pool {pool.Name}, you are tier {privilege}",
+                    forbidden: true))
+                { StatusCode = 403 });
+        }
+
+        return Task.FromResult<IActionResult?>(null);
+    }
+
+    private static IActionResult? ValidatePoolPolicy(PolicyConfig policy, CreateUploadTaskRequest request)
+    {
+        if (!policy.AllowEncryption && !string.IsNullOrEmpty(request.EncryptPassword))
+        {
+            return new ObjectResult(ApiError.Unauthorized("File encryption is not allowed in this pool", true))
+                { StatusCode = 403 };
+        }
+
+        if (policy.AcceptTypes is { Count: > 0 })
+        {
+            if (string.IsNullOrEmpty(request.ContentType))
+            {
+                return new ObjectResult(ApiError.Validation(new Dictionary<string, string[]>
+                    {
+                        { "contentType", new[] { "Content type is required by the pool's policy" } }
+                    }))
+                    { StatusCode = 400 };
+            }
+
+            var foundMatch = policy.AcceptTypes.Any(acceptType =>
+            {
+                if (!acceptType.EndsWith("/*", StringComparison.OrdinalIgnoreCase))
+                    return acceptType.Equals(request.ContentType, StringComparison.OrdinalIgnoreCase);
+                var type = acceptType[..^2];
+                return request.ContentType.StartsWith($"{type}/", StringComparison.OrdinalIgnoreCase);
+            });
+
+            if (!foundMatch)
+            {
+                return new ObjectResult(
+                        ApiError.Unauthorized($"Content type {request.ContentType} is not allowed by the pool's policy",
+                            true))
+                    { StatusCode = 403 };
+            }
+        }
+
+        if (policy.MaxFileSize is not null && request.FileSize > policy.MaxFileSize)
+        {
+            return new ObjectResult(ApiError.Unauthorized(
+                    $"File size {request.FileSize} is larger than the pool's maximum file size {policy.MaxFileSize}",
+                    true))
+                { StatusCode = 403 };
+        }
+
+        return null;
+    }
+
+    private async Task<IActionResult?> ValidateQuota(Account currentUser, FilePool pool, long fileSize)
+    {
+        var (ok, billableUnit, quota) = await quotaService.IsFileAcceptable(
+            Guid.Parse(currentUser.Id),
+            pool.BillingConfig.CostMultiplier ?? 1.0,
+            fileSize
+        );
+
+        if (!ok)
+        {
+            return new ObjectResult(
+                    ApiError.Unauthorized($"File size {billableUnit} MiB is exceeded the user's quota {quota} MiB",
+                        true))
+                { StatusCode = 403 };
+        }
+
+        return null;
+    }
+
+    private void EnsureTempDirectoryExists()
+    {
+        if (!Directory.Exists(_tempPath))
+        {
+            Directory.CreateDirectory(_tempPath);
+        }
+    }
+
+    public class UploadChunkRequest
+    {
+        [Required] public IFormFile Chunk { get; set; } = null!;
+    }
+
+    [HttpPost("chunk/{taskId}/{chunkIndex:int}")]
+    [RequestSizeLimit(DefaultChunkSize + 1024 * 1024)] // 6MB to be safe
+    [RequestFormLimits(MultipartBodyLengthLimit = DefaultChunkSize + 1024 * 1024)]
+    public async Task<IActionResult> UploadChunk(string taskId, int chunkIndex, [FromForm] UploadChunkRequest request)
+    {
+        var chunk = request.Chunk;
+
+        // Check if chunk is already uploaded (resumable upload)
+        if (await persistentTaskService.IsChunkUploadedAsync(taskId, chunkIndex))
+        {
+            return Ok(new { message = "Chunk already uploaded" });
+        }
+
+        var taskPath = Path.Combine(_tempPath, taskId);
+        if (!Directory.Exists(taskPath))
+        {
+            Directory.CreateDirectory(taskPath);
+        }
+
+        var chunkPath = Path.Combine(taskPath, $"{chunkIndex}.chunk");
+        await using var stream = new FileStream(chunkPath, FileMode.Create);
+        await chunk.CopyToAsync(stream);
+
+        // Update persistent task progress
+        await persistentTaskService.UpdateChunkProgressAsync(taskId, chunkIndex);
+
+        return Ok();
+    }
+
+    [HttpPost("complete/{taskId}")]
+    public async Task<IActionResult> CompleteUpload(string taskId)
+    {
+        // Get persistent task
+        var persistentTask = await persistentTaskService.GetUploadTaskAsync(taskId);
+        if (persistentTask is null)
+            return new ObjectResult(ApiError.NotFound("Upload task")) { StatusCode = 404 };
+
+        var currentUser = HttpContext.Items["CurrentUser"] as Account;
+        if (currentUser is null)
+            return new ObjectResult(ApiError.Unauthorized()) { StatusCode = 401 };
+
+        // Verify ownership
+        if (persistentTask.AccountId != Guid.Parse(currentUser.Id))
+            return new ObjectResult(ApiError.Unauthorized(forbidden: true)) { StatusCode = 403 };
+
+        var taskPath = Path.Combine(_tempPath, taskId);
+        if (!Directory.Exists(taskPath))
+            return new ObjectResult(ApiError.NotFound("Upload task directory")) { StatusCode = 404 };
+
+        var mergedFilePath = Path.Combine(_tempPath, taskId + ".tmp");
+
+        try
+        {
+            await MergeChunks(taskId, taskPath, mergedFilePath, persistentTask.ChunksCount, persistentTaskService);
+
+            var fileId = await Nanoid.GenerateAsync();
+            var cloudFile = await fileService.ProcessNewFileAsync(
+                currentUser,
+                fileId,
+                persistentTask.PoolId.ToString(),
+                persistentTask.BundleId?.ToString(),
+                mergedFilePath,
+                persistentTask.FileName,
+                persistentTask.ContentType,
+                persistentTask.EncryptPassword,
+                persistentTask.ExpiredAt
+            );
+
+            // Create the file index if a path is provided
+            if (!string.IsNullOrEmpty(persistentTask.Path))
+            {
+                try
+                {
+                    var accountId = Guid.Parse(currentUser.Id);
+                    await fileIndexService.CreateAsync(persistentTask.Path, fileId, accountId);
+                    logger.LogInformation("Created file index for file {FileId} at path {Path}", fileId,
+                        persistentTask.Path);
+                }
+                catch (Exception ex)
+                {
+                    logger.LogWarning(ex, "Failed to create file index for file {FileId} at path {Path}", fileId,
+                        persistentTask.Path);
+                    // Don't fail the upload if index creation fails, just log it
+                }
+            }
+
+            // Update the task status to "processing" - background processing is now happening
+            await persistentTaskService.UpdateTaskProgressAsync(taskId, 0.95, "Processing file in background...");
+
+            // Send upload completion notification (a file is uploaded, but processing continues)
+            await persistentTaskService.SendUploadCompletedNotificationAsync(persistentTask, fileId);
+
+            return Ok(cloudFile);
+        }
+        catch (Exception ex)
+        {
+            // Log the actual exception for debugging
+            logger.LogError(ex, "Failed to complete upload for task {TaskId}. Error: {ErrorMessage}", taskId,
+                ex.Message);
+
+            // Mark task as failed
+            await persistentTaskService.MarkTaskFailedAsync(taskId);
+
+            // Send failure notification
+            await persistentTaskService.SendUploadFailedNotificationAsync(persistentTask, ex.Message);
+
+            await CleanupTempFiles(taskPath, mergedFilePath);
+
+            return new ObjectResult(new ApiError
+            {
+                Code = "UPLOAD_FAILED",
+                Message = $"Failed to complete file upload: {ex.Message}",
+                Status = 500
+            }) { StatusCode = 500 };
+        }
+        finally
+        {
+            // Always clean up temp files
+            await CleanupTempFiles(taskPath, mergedFilePath);
+        }
+    }
+
+    private static async Task MergeChunks(
+        string taskId,
+        string taskPath,
+        string mergedFilePath,
+        int chunksCount,
+        PersistentTaskService persistentTaskService)
+    {
+        await using var mergedStream = new FileStream(mergedFilePath, FileMode.Create);
+
+        const double baseProgress = 0.8; // Start from 80% (chunk upload is already at 95%)
+        const double remainingProgress = 0.15; // Remaining 15% progress distributed across chunks
+        var progressPerChunk = remainingProgress / chunksCount;
+
+        for (var i = 0; i < chunksCount; i++)
+        {
+            var chunkPath = Path.Combine(taskPath, i + ".chunk");
+            if (!System.IO.File.Exists(chunkPath))
+                throw new InvalidOperationException("Chunk " + i + " is missing.");
+
+            await using var chunkStream = new FileStream(chunkPath, FileMode.Open);
+            await chunkStream.CopyToAsync(mergedStream);
+
+            // Update progress after each chunk is merged
+            var currentProgress = baseProgress + progressPerChunk * (i + 1);
+            await persistentTaskService.UpdateTaskProgressAsync(
+                taskId,
+                currentProgress,
+                "Merging chunks... (" + (i + 1) + "/" + chunksCount + ")"
+            );
+        }
+    }
+
+    private static Task CleanupTempFiles(string taskPath, string mergedFilePath)
+    {
+        try
+        {
+            if (Directory.Exists(taskPath))
+                Directory.Delete(taskPath, true);
+
+            if (System.IO.File.Exists(mergedFilePath))
+                System.IO.File.Delete(mergedFilePath);
+        }
+        catch
+        {
+            // Ignore cleanup errors to avoid masking the original exception
+        }
+
+        return Task.CompletedTask;
+    }
+
+    // New endpoints for resumable uploads
+
+    [HttpGet("tasks")]
+    public async Task<IActionResult> GetMyUploadTasks(
+        [FromQuery] UploadTaskStatus? status = null,
+        [FromQuery] string? sortBy = "lastActivity",
+        [FromQuery] bool sortDescending = true,
+        [FromQuery] int offset = 0,
+        [FromQuery] int limit = 50
+    )
+    {
+        var currentUser = HttpContext.Items["CurrentUser"] as Account;
+        if (currentUser is null)
+            return new ObjectResult(ApiError.Unauthorized()) { StatusCode = 401 };
+
+        var accountId = Guid.Parse(currentUser.Id);
+        var tasks = await persistentTaskService.GetUserUploadTasksAsync(accountId, status, sortBy, sortDescending,
+            offset, limit);
+
+        Response.Headers.Append("X-Total", tasks.TotalCount.ToString());
+
+        return Ok(tasks.Items.Select(t => new
+        {
+            t.TaskId,
+            t.FileName,
+            t.FileSize,
+            t.ContentType,
+            t.ChunkSize,
+            t.ChunksCount,
+            t.ChunksUploaded,
+            Progress = t.ChunksCount > 0 ? (double)t.ChunksUploaded / t.ChunksCount * 100 : 0,
+            t.Status,
+            t.LastActivity,
+            t.CreatedAt,
+            t.UpdatedAt,
+            t.UploadedChunks,
+            Pool = new { t.PoolId, Name = "Pool Name" }, // Could be expanded to include pool details
+            Bundle = t.BundleId.HasValue ? new { t.BundleId } : null
+        }));
+    }
+
+    [HttpGet("progress/{taskId}")]
+    public async Task<IActionResult> GetUploadProgress(string taskId)
+    {
+        var currentUser = HttpContext.Items["CurrentUser"] as Account;
+        if (currentUser is null)
+            return new ObjectResult(ApiError.Unauthorized()) { StatusCode = 401 };
+
+        var task = await persistentTaskService.GetUploadTaskAsync(taskId);
+        if (task is null)
+            return new ObjectResult(ApiError.NotFound("Upload task")) { StatusCode = 404 };
+
+        // Verify ownership
+        if (task.AccountId != Guid.Parse(currentUser.Id))
+            return new ObjectResult(ApiError.Unauthorized(forbidden: true)) { StatusCode = 403 };
+
+        var progress = await persistentTaskService.GetUploadProgressAsync(taskId);
+
+        return Ok(new
+        {
+            task.TaskId,
+            task.FileName,
+            task.FileSize,
+            task.ChunksCount,
+            task.ChunksUploaded,
+            Progress = progress,
+            task.Status,
+            task.LastActivity,
+            task.UploadedChunks
+        });
+    }
+
+    [HttpGet("resume/{taskId}")]
+    public async Task<IActionResult> ResumeUploadTask(string taskId)
+    {
+        var currentUser = HttpContext.Items["CurrentUser"] as Account;
+        if (currentUser is null)
+            return new ObjectResult(ApiError.Unauthorized()) { StatusCode = 401 };
+
+        var task = await persistentTaskService.GetUploadTaskAsync(taskId);
+        if (task is null)
+            return new ObjectResult(ApiError.NotFound("Upload task")) { StatusCode = 404 };
+
+        // Verify ownership
+        if (task.AccountId != Guid.Parse(currentUser.Id))
+            return new ObjectResult(ApiError.Unauthorized(forbidden: true)) { StatusCode = 403 };
+
+        // Ensure temp directory exists
+        var taskPath = Path.Combine(_tempPath, taskId);
+        if (!Directory.Exists(taskPath))
+        {
+            Directory.CreateDirectory(taskPath);
+        }
+
+        return Ok(new
+        {
+            task.TaskId,
+            task.FileName,
+            task.FileSize,
+            task.ContentType,
+            task.ChunkSize,
+            task.ChunksCount,
+            task.ChunksUploaded,
+            task.UploadedChunks,
+            Progress = task.ChunksCount > 0 ? (double)task.ChunksUploaded / task.ChunksCount * 100 : 0
+        });
+    }
+
+    [HttpDelete("task/{taskId}")]
+    public async Task<IActionResult> CancelUploadTask(string taskId)
+    {
+        var currentUser = HttpContext.Items["CurrentUser"] as Account;
+        if (currentUser is null)
+            return new ObjectResult(ApiError.Unauthorized()) { StatusCode = 401 };
+
+        var task = await persistentTaskService.GetUploadTaskAsync(taskId);
+        if (task is null)
+            return new ObjectResult(ApiError.NotFound("Upload task")) { StatusCode = 404 };
+
+        // Verify ownership
+        if (task.AccountId != Guid.Parse(currentUser.Id))
+            return new ObjectResult(ApiError.Unauthorized(forbidden: true)) { StatusCode = 403 };
+
+        // Mark as failed (cancelled)
+        await persistentTaskService.MarkTaskFailedAsync(taskId);
+
+        // Clean up temp files
+        var taskPath = Path.Combine(_tempPath, taskId);
+        await CleanupTempFiles(taskPath, string.Empty);
+
+        return Ok(new { message = "Upload task cancelled" });
+    }
+
+    [HttpGet("stats")]
+    public async Task<IActionResult> GetUploadStats()
+    {
+        var currentUser = HttpContext.Items["CurrentUser"] as Account;
+        if (currentUser is null)
+            return new ObjectResult(ApiError.Unauthorized()) { StatusCode = 401 };
+
+        var accountId = Guid.Parse(currentUser.Id);
+        var stats = await persistentTaskService.GetUserUploadStatsAsync(accountId);
+
+        return Ok(new
+        {
+            stats.TotalTasks,
+            stats.InProgressTasks,
+            stats.CompletedTasks,
+            stats.FailedTasks,
+            stats.ExpiredTasks,
+            stats.TotalUploadedBytes,
+            stats.AverageProgress,
+            stats.RecentActivity
+        });
+    }
+
+    [HttpDelete("tasks/cleanup")]
+    public async Task<IActionResult> CleanupFailedTasks()
+    {
+        var currentUser = HttpContext.Items["CurrentUser"] as Account;
+        if (currentUser is null)
+            return new ObjectResult(ApiError.Unauthorized()) { StatusCode = 401 };
+
+        var accountId = Guid.Parse(currentUser.Id);
+        var cleanedCount = await persistentTaskService.CleanupUserFailedTasksAsync(accountId);
+
+        return Ok(new { message = $"Cleaned up {cleanedCount} failed tasks" });
+    }
+
+    [HttpGet("tasks/recent")]
+    public async Task<IActionResult> GetRecentTasks([FromQuery] int limit = 10)
+    {
+        var currentUser = HttpContext.Items["CurrentUser"] as Account;
+        if (currentUser is null)
+            return new ObjectResult(ApiError.Unauthorized()) { StatusCode = 401 };
+
+        var accountId = Guid.Parse(currentUser.Id);
+        var tasks = await persistentTaskService.GetRecentUserTasksAsync(accountId, limit);
+
+        return Ok(tasks.Select(t => new
+        {
+            t.TaskId,
+            t.FileName,
+            t.FileSize,
+            t.ContentType,
+            Progress = t.ChunksCount > 0 ? (double)t.ChunksUploaded / t.ChunksCount * 100 : 0,
+            t.Status,
+            t.LastActivity,
+            t.CreatedAt
+        }));
+    }
+
+    [HttpGet("tasks/{taskId}/details")]
+    public async Task<IActionResult> GetTaskDetails(string taskId)
+    {
+        var currentUser = HttpContext.Items["CurrentUser"] as Account;
+        if (currentUser is null)
+            return new ObjectResult(ApiError.Unauthorized()) { StatusCode = 401 };
+
+        var task = await persistentTaskService.GetUploadTaskAsync(taskId);
+        if (task is null)
+            return new ObjectResult(ApiError.NotFound("Upload task")) { StatusCode = 404 };
+
+        // Verify ownership
+        if (task.AccountId != Guid.Parse(currentUser.Id))
+            return new ObjectResult(ApiError.Unauthorized(forbidden: true)) { StatusCode = 403 };
+
+        // Get pool information
+        var pool = await fileService.GetPoolAsync(task.PoolId);
+        var bundle = task.BundleId.HasValue
+            ? await db.Bundles.FirstOrDefaultAsync(b => b.Id == task.BundleId.Value)
+            : null;
+
+        return Ok(new
+        {
+            Task = new
+            {
+                task.TaskId,
+                task.FileName,
+                task.FileSize,
+                task.ContentType,
+                task.ChunkSize,
+                task.ChunksCount,
+                task.ChunksUploaded,
+                Progress = task.ChunksCount > 0 ? (double)task.ChunksUploaded / task.ChunksCount * 100 : 0,
+                task.Status,
+                task.LastActivity,
+                task.CreatedAt,
+                task.UpdatedAt,
+                task.ExpiredAt,
+                task.Hash,
+                task.UploadedChunks
+            },
+            Pool = pool != null
+                ? new
+                {
+                    pool.Id,
+                    pool.Name,
+                    pool.Description
+                }
+                : null,
+            Bundle = bundle != null
+                ? new
+                {
+                    bundle.Id,
+                    bundle.Name,
+                    bundle.Description
+                }
+                : null,
+            EstimatedTimeRemaining = CalculateEstimatedTime(task),
+            UploadSpeed = CalculateUploadSpeed(task)
+        });
+    }
+
+    private static string? CalculateEstimatedTime(PersistentUploadTask task)
+    {
+        if (task.Status != TaskStatus.InProgress || task.ChunksUploaded == 0)
+            return null;
+
+        var elapsed = NodaTime.SystemClock.Instance.GetCurrentInstant() - task.CreatedAt;
+        var elapsedSeconds = elapsed.TotalSeconds;
+        var chunksPerSecond = task.ChunksUploaded / elapsedSeconds;
+        var remainingChunks = task.ChunksCount - task.ChunksUploaded;
+
+        if (chunksPerSecond <= 0)
+            return null;
+
+        var remainingSeconds = remainingChunks / chunksPerSecond;
+
+        return remainingSeconds switch
+        {
+            < 60 => $"{remainingSeconds:F0} seconds",
+            < 3600 => $"{remainingSeconds / 60:F0} minutes",
+            _ => $"{remainingSeconds / 3600:F1} hours"
+        };
+    }
+
+    private static string? CalculateUploadSpeed(PersistentUploadTask task)
+    {
+        if (task.ChunksUploaded == 0)
+            return null;
+
+        var elapsed = SystemClock.Instance.GetCurrentInstant() - task.CreatedAt;
+        var elapsedSeconds = elapsed.TotalSeconds;
+        var bytesUploaded = task.ChunksUploaded * task.ChunkSize;
+        var bytesPerSecond = bytesUploaded / elapsedSeconds;
+
+        return bytesPerSecond switch
+        {
+            < 1024 => $"{bytesPerSecond:F0} B/s",
+            < 1024 * 1024 => $"{bytesPerSecond / 1024:F0} KB/s",
+            _ => $"{bytesPerSecond / (1024 * 1024):F1} MB/s"
+        };
+    }
+}

DysonNetwork.Drive/Storage/Model/Events.cs

@@ -0,0 +1,15 @@
+namespace DysonNetwork.Drive.Storage.Model;
+
+public static class FileUploadedEvent
+{
+    public const string Type = "file_uploaded";
+}
+
+public record FileUploadedEventPayload(
+    string FileId,
+    Guid RemoteId,
+    string StorageId,
+    string ContentType,
+    string ProcessingFilePath,
+    bool IsTempFile
+);

DysonNetwork.Drive/Storage/Model/FileUploadModels.cs

@@ -0,0 +1,670 @@
+using DysonNetwork.Shared.Models;
+using DysonNetwork.Shared.Proto;
+using Google.Protobuf.Collections;
+using NodaTime;
+using System.ComponentModel.DataAnnotations;
+using System.ComponentModel.DataAnnotations.Schema;
+using System.Text.Json;
+
+namespace DysonNetwork.Drive.Storage.Model;
+
+// File Upload Task Parameters
+public class FileUploadParameters
+{
+    public string FileName { get; set; } = string.Empty;
+    public long FileSize { get; set; }
+    public string ContentType { get; set; } = string.Empty;
+    public long ChunkSize { get; set; } = 5242880L;
+    public int ChunksCount { get; set; }
+    public int ChunksUploaded { get; set; }
+    public Guid PoolId { get; set; }
+    public Guid? BundleId { get; set; }
+    public string? EncryptPassword { get; set; }
+    public string Hash { get; set; } = string.Empty;
+    public List<int> UploadedChunks { get; set; } = [];
+    public string? Path { get; set; }
+}
+
+// File Move Task Parameters
+public class FileMoveParameters
+{
+    public List<string> FileIds { get; set; } = [];
+    public Guid TargetPoolId { get; set; }
+    public Guid? TargetBundleId { get; set; }
+    public int FilesProcessed { get; set; }
+}
+
+// File Compression Task Parameters
+public class FileCompressParameters
+{
+    public List<string> FileIds { get; set; } = [];
+    public string CompressionFormat { get; set; } = "zip";
+    public int CompressionLevel { get; set; } = 6;
+    public string? OutputFileName { get; set; }
+    public int FilesProcessed { get; set; }
+    public string? ResultFileId { get; set; }
+}
+
+// Bulk Operation Task Parameters
+public class BulkOperationParameters
+{
+    public string OperationType { get; set; } = string.Empty;
+    public List<string> TargetIds { get; set; } = [];
+    public Dictionary<string, object?> OperationParameters { get; set; } = new();
+    public int ItemsProcessed { get; set; }
+    public Dictionary<string, object?>? OperationResults { get; set; }
+}
+
+// Storage Migration Task Parameters
+public class StorageMigrationParameters
+{
+    public Guid SourcePoolId { get; set; }
+    public Guid TargetPoolId { get; set; }
+    public List<string> FileIds { get; set; } = new();
+    public bool PreserveOriginals { get; set; } = true;
+    public long TotalBytesToTransfer { get; set; }
+    public long BytesTransferred { get; set; }
+    public int FilesMigrated { get; set; }
+}
+
+// Helper class for parameter operations using GrpcTypeHelper
+public static class ParameterHelper
+{
+    public static T? Typed<T>(Dictionary<string, object?> parameters)
+    {
+        var rawParams = GrpcTypeHelper.ConvertObjectToByteString(parameters);
+        return GrpcTypeHelper.ConvertByteStringToObject<T>(rawParams);
+    }
+
+    public static Dictionary<string, object?> Untyped<T>(T parameters)
+    {
+        var rawParams = GrpcTypeHelper.ConvertObjectToByteString(parameters);
+        return GrpcTypeHelper.ConvertByteStringToObject<Dictionary<string, object?>>(rawParams) ?? [];
+    }
+}
+
+public class CreateUploadTaskRequest
+{
+    public string Hash { get; set; } = null!;
+    public string FileName { get; set; } = null!;
+    public long FileSize { get; set; }
+    public string ContentType { get; set; } = null!;
+    public Guid? PoolId { get; set; } = null!;
+    public Guid? BundleId { get; set; }
+    public string? EncryptPassword { get; set; }
+    public Instant? ExpiredAt { get; set; }
+    public long? ChunkSize { get; set; }
+    public string? Path { get; set; }
+}
+
+public class CreateUploadTaskResponse
+{
+    public bool FileExists { get; set; }
+    public SnCloudFile? File { get; set; }
+    public string? TaskId { get; set; }
+    public long? ChunkSize { get; set; }
+    public int? ChunksCount { get; set; }
+}
+
+internal class UploadTask
+{
+    public string TaskId { get; set; } = null!;
+    public string FileName { get; set; } = null!;
+    public long FileSize { get; set; }
+    public string ContentType { get; set; } = null!;
+    public long ChunkSize { get; set; }
+    public int ChunksCount { get; set; }
+    public Guid PoolId { get; set; }
+    public Guid? BundleId { get; set; }
+    public string? EncryptPassword { get; set; }
+    public Instant? ExpiredAt { get; set; }
+    public string Hash { get; set; } = null!;
+}
+
+public class PersistentTask : ModelBase
+{
+    public Guid Id { get; set; } = Guid.NewGuid();
+
+    [MaxLength(64)] public string TaskId { get; set; } = null!;
+
+    [MaxLength(256)] public string Name { get; set; } = null!;
+
+    [MaxLength(1024)] public string? Description { get; set; }
+
+    public TaskType Type { get; set; }
+
+    public TaskStatus Status { get; set; } = TaskStatus.InProgress;
+
+    public Guid AccountId { get; set; }
+
+    // Progress tracking (0-100)
+    public double Progress { get; set; }
+
+    // Task-specific parameters stored as JSON
+    [Column(TypeName = "jsonb")] public Dictionary<string, object?> Parameters { get; set; } = new();
+
+    // Task results/output stored as JSON
+    [Column(TypeName = "jsonb")] public Dictionary<string, object?> Results { get; set; } = new();
+
+    [MaxLength(1024)] public string? ErrorMessage { get; set; }
+
+    public Instant? StartedAt { get; set; }
+    public Instant? CompletedAt { get; set; }
+    public Instant? ExpiredAt { get; set; }
+
+    public Instant LastActivity { get; set; }
+
+    // Priority (higher = more important)
+    public int Priority { get; set; } = 0;
+
+    // Estimated duration in seconds
+    public long? EstimatedDurationSeconds { get; set; }
+}
+
+// Backward compatibility - UploadTask inherits from PersistentTask
+public class PersistentUploadTask : PersistentTask
+{
+    public PersistentUploadTask()
+    {
+        Type = TaskType.FileUpload;
+        Name = "File Upload";
+    }
+
+    // Convenience properties using typed parameters
+    [NotMapped]
+    public FileUploadParameters TypedParameters
+    {
+        get => ParameterHelper.Typed<FileUploadParameters>(Parameters)!;
+        set => Parameters = ParameterHelper.Untyped(value);
+    }
+
+    [MaxLength(256)]
+    public string FileName
+    {
+        get => TypedParameters.FileName;
+        set
+        {
+            var parameters = TypedParameters;
+            parameters.FileName = value;
+            TypedParameters = parameters;
+        }
+    }
+
+    public long FileSize
+    {
+        get => TypedParameters.FileSize;
+        set
+        {
+            var parameters = TypedParameters;
+            parameters.FileSize = value;
+            TypedParameters = parameters;
+        }
+    }
+
+    [MaxLength(128)]
+    public string ContentType
+    {
+        get => TypedParameters.ContentType;
+        set
+        {
+            var parameters = TypedParameters;
+            parameters.ContentType = value;
+            TypedParameters = parameters;
+        }
+    }
+
+    public long ChunkSize
+    {
+        get => TypedParameters.ChunkSize;
+        set
+        {
+            var parameters = TypedParameters;
+            parameters.ChunkSize = value;
+            TypedParameters = parameters;
+        }
+    }
+
+    public int ChunksCount
+    {
+        get => TypedParameters.ChunksCount;
+        set
+        {
+            var parameters = TypedParameters;
+            parameters.ChunksCount = value;
+            TypedParameters = parameters;
+        }
+    }
+
+    public int ChunksUploaded
+    {
+        get => TypedParameters.ChunksUploaded;
+        set
+        {
+            var parameters = TypedParameters;
+            parameters.ChunksUploaded = value;
+            TypedParameters = parameters;
+            Progress = ChunksCount > 0 ? (double)value / ChunksCount * 100 : 0;
+        }
+    }
+
+    public Guid PoolId
+    {
+        get => TypedParameters.PoolId;
+        set
+        {
+            var parameters = TypedParameters;
+            parameters.PoolId = value;
+            TypedParameters = parameters;
+        }
+    }
+
+    public Guid? BundleId
+    {
+        get => TypedParameters.BundleId;
+        set
+        {
+            var parameters = TypedParameters;
+            parameters.BundleId = value;
+            TypedParameters = parameters;
+        }
+    }
+
+    [MaxLength(256)]
+    public string? EncryptPassword
+    {
+        get => TypedParameters.EncryptPassword;
+        set
+        {
+            var parameters = TypedParameters;
+            parameters.EncryptPassword = value;
+            TypedParameters = parameters;
+        }
+    }
+
+    public string Hash
+    {
+        get => TypedParameters.Hash;
+        set
+        {
+            var parameters = TypedParameters;
+            parameters.Hash = value;
+            TypedParameters = parameters;
+        }
+    }
+
+    // JSON array of uploaded chunk indices for resumability
+    public List<int> UploadedChunks
+    {
+        get => TypedParameters.UploadedChunks;
+        set
+        {
+            var parameters = TypedParameters;
+            parameters.UploadedChunks = value;
+            TypedParameters = parameters;
+        }
+    }
+
+    public string? Path
+    {
+        get => TypedParameters.Path;
+        set
+        {
+            var parameters = TypedParameters;
+            parameters.Path = value;
+            TypedParameters = parameters;
+        }
+    }
+}
+
+public enum TaskType
+{
+    FileUpload,
+    FileMove,
+    FileCompress,
+    FileDecompress,
+    FileEncrypt,
+    FileDecrypt,
+    BulkOperation,
+    StorageMigration,
+    FileConversion,
+    Custom
+}
+
+[Flags]
+public enum TaskStatus
+{
+    Pending,
+    InProgress,
+    Paused,
+    Completed,
+    Failed,
+    Cancelled,
+    Expired
+}
+
+// File Move Task
+public class FileMoveTask : PersistentTask
+{
+    public FileMoveTask()
+    {
+        Type = TaskType.FileMove;
+        Name = "Move Files";
+    }
+
+    // Convenience properties using typed parameters
+    public FileMoveParameters TypedParameters
+    {
+        get => ParameterHelper.Typed<FileMoveParameters>(Parameters)!;
+        set => Parameters = ParameterHelper.Untyped(value);
+    }
+
+    public List<string> FileIds
+    {
+        get => TypedParameters.FileIds;
+        set
+        {
+            var parameters = TypedParameters;
+            parameters.FileIds = value;
+            TypedParameters = parameters;
+        }
+    }
+
+    public Guid TargetPoolId
+    {
+        get => TypedParameters.TargetPoolId;
+        set
+        {
+            var parameters = TypedParameters;
+            parameters.TargetPoolId = value;
+            TypedParameters = parameters;
+        }
+    }
+
+    public Guid? TargetBundleId
+    {
+        get => TypedParameters.TargetBundleId;
+        set
+        {
+            var parameters = TypedParameters;
+            parameters.TargetBundleId = value;
+            TypedParameters = parameters;
+        }
+    }
+
+    public int FilesProcessed
+    {
+        get => TypedParameters.FilesProcessed;
+        set
+        {
+            var parameters = TypedParameters;
+            parameters.FilesProcessed = value;
+            TypedParameters = parameters;
+            Progress = FileIds.Count > 0 ? (double)value / FileIds.Count * 100 : 0;
+        }
+    }
+}
+
+// File Compression Task
+public class FileCompressTask : PersistentTask
+{
+    public FileCompressTask()
+    {
+        Type = TaskType.FileCompress;
+        Name = "Compress Files";
+    }
+
+    // Convenience properties using typed parameters
+    public FileCompressParameters TypedParameters
+    {
+        get => ParameterHelper.Typed<FileCompressParameters>(Parameters)!;
+        set => Parameters = ParameterHelper.Untyped(value);
+    }
+
+    public List<string> FileIds
+    {
+        get => TypedParameters.FileIds;
+        set
+        {
+            var parameters = TypedParameters;
+            parameters.FileIds = value;
+            TypedParameters = parameters;
+        }
+    }
+
+    [MaxLength(32)]
+    public string CompressionFormat
+    {
+        get => TypedParameters.CompressionFormat;
+        set
+        {
+            var parameters = TypedParameters;
+            parameters.CompressionFormat = value;
+            TypedParameters = parameters;
+        }
+    }
+
+    public int CompressionLevel
+    {
+        get => TypedParameters.CompressionLevel;
+        set
+        {
+            var parameters = TypedParameters;
+            parameters.CompressionLevel = value;
+            TypedParameters = parameters;
+        }
+    }
+
+    public string? OutputFileName
+    {
+        get => TypedParameters.OutputFileName;
+        set
+        {
+            var parameters = TypedParameters;
+            parameters.OutputFileName = value;
+            TypedParameters = parameters;
+        }
+    }
+
+    public int FilesProcessed
+    {
+        get => TypedParameters.FilesProcessed;
+        set
+        {
+            var parameters = TypedParameters;
+            parameters.FilesProcessed = value;
+            TypedParameters = parameters;
+            Progress = FileIds.Count > 0 ? (double)value / FileIds.Count * 100 : 0;
+        }
+    }
+
+    public string? ResultFileId
+    {
+        get => TypedParameters.ResultFileId;
+        set
+        {
+            var parameters = TypedParameters;
+            parameters.ResultFileId = value;
+            TypedParameters = parameters;
+        }
+    }
+}
+
+// Bulk Operation Task
+public class BulkOperationTask : PersistentTask
+{
+    public BulkOperationTask()
+    {
+        Type = TaskType.BulkOperation;
+        Name = "Bulk Operation";
+    }
+
+    // Convenience properties using typed parameters
+    public BulkOperationParameters TypedParameters
+    {
+        get => ParameterHelper.Typed<BulkOperationParameters>(Parameters)!;
+        set => Parameters = ParameterHelper.Untyped(value);
+    }
+
+    [MaxLength(128)]
+    public string OperationType
+    {
+        get => TypedParameters.OperationType;
+        set
+        {
+            var parameters = TypedParameters;
+            parameters.OperationType = value;
+            TypedParameters = parameters;
+        }
+    }
+
+    public List<string> TargetIds
+    {
+        get => TypedParameters.TargetIds;
+        set
+        {
+            var parameters = TypedParameters;
+            parameters.TargetIds = value;
+            TypedParameters = parameters;
+        }
+    }
+
+    [Column(TypeName = "jsonb")]
+    public Dictionary<string, object?> OperationParameters
+    {
+        get => TypedParameters.OperationParameters;
+        set
+        {
+            var parameters = TypedParameters;
+            parameters.OperationParameters = value;
+            TypedParameters = parameters;
+        }
+    }
+
+    public int ItemsProcessed
+    {
+        get => TypedParameters.ItemsProcessed;
+        set
+        {
+            var parameters = TypedParameters;
+            parameters.ItemsProcessed = value;
+            TypedParameters = parameters;
+            Progress = TargetIds.Count > 0 ? (double)value / TargetIds.Count * 100 : 0;
+        }
+    }
+
+    [Column(TypeName = "jsonb")]
+    public Dictionary<string, object?>? OperationResults
+    {
+        get => TypedParameters.OperationResults;
+        set
+        {
+            var parameters = TypedParameters;
+            parameters.OperationResults = value;
+            TypedParameters = parameters;
+        }
+    }
+}
+
+// Storage Migration Task
+public class StorageMigrationTask : PersistentTask
+{
+    public StorageMigrationTask()
+    {
+        Type = TaskType.StorageMigration;
+        Name = "Storage Migration";
+    }
+
+    // Convenience properties using typed parameters
+    public StorageMigrationParameters TypedParameters
+    {
+        get => ParameterHelper.Typed<StorageMigrationParameters>(Parameters)!;
+        set => Parameters = ParameterHelper.Untyped(value);
+    }
+
+    public Guid SourcePoolId
+    {
+        get => TypedParameters.SourcePoolId;
+        set
+        {
+            var parameters = TypedParameters;
+            parameters.SourcePoolId = value;
+            TypedParameters = parameters;
+        }
+    }
+
+    public Guid TargetPoolId
+    {
+        get => TypedParameters.TargetPoolId;
+        set
+        {
+            var parameters = TypedParameters;
+            parameters.TargetPoolId = value;
+            TypedParameters = parameters;
+        }
+    }
+
+    public List<string> FileIds
+    {
+        get => TypedParameters.FileIds;
+        set
+        {
+            var parameters = TypedParameters;
+            parameters.FileIds = value;
+            TypedParameters = parameters;
+        }
+    }
+
+    public bool PreserveOriginals
+    {
+        get => TypedParameters.PreserveOriginals;
+        set
+        {
+            var parameters = TypedParameters;
+            parameters.PreserveOriginals = value;
+            TypedParameters = parameters;
+        }
+    }
+
+    public long TotalBytesToTransfer
+    {
+        get => TypedParameters.TotalBytesToTransfer;
+        set
+        {
+            var parameters = TypedParameters;
+            parameters.TotalBytesToTransfer = value;
+            TypedParameters = parameters;
+        }
+    }
+
+    public long BytesTransferred
+    {
+        get => TypedParameters.BytesTransferred;
+        set
+        {
+            var parameters = TypedParameters;
+            parameters.BytesTransferred = value;
+            TypedParameters = parameters;
+            Progress = TotalBytesToTransfer > 0 ? (double)value / TotalBytesToTransfer * 100 : 0;
+        }
+    }
+
+    public int FilesMigrated
+    {
+        get => TypedParameters.FilesMigrated;
+        set
+        {
+            var parameters = TypedParameters;
+            parameters.FilesMigrated = value;
+            TypedParameters = parameters;
+        }
+    }
+}
+
+// Legacy enum for backward compatibility
+public enum UploadTaskStatus
+{
+    InProgress = TaskStatus.InProgress,
+    Completed = TaskStatus.Completed,
+    Failed = TaskStatus.Failed,
+    Expired = TaskStatus.Expired
+}

DysonNetwork.Drive/Storage/README.md

@@ -0,0 +1,994 @@
+# DysonNetwork Drive - Persistent Task System
+
+A comprehensive, production-ready generic task system with support for file uploads, background operations, real-time progress tracking, and dynamic notifications powered by RingService.
+
+When using with the Gateway, use the `/drive` to replace `/api`.
+The realtime messages are from the websocket gateway.
+
+## 🚀 Features
+
+### Core Task Features
+- **Generic Task System**: Support for various background operations beyond file uploads
+- **Resumable Uploads**: Pause and resume uploads across app restarts
+- **Chunked Uploads**: Efficient large file handling with configurable chunk sizes
+- **Progress Persistence**: Task state survives server restarts and network interruptions
+- **Duplicate Detection**: Automatic detection of already uploaded files via hash checking
+- **Quota Management**: Integration with user quota and billing systems
+- **Pool-based Storage**: Support for multiple storage pools with different policies
+
+### Real-Time Features
+- **Live Progress Updates**: WebSocket-based real-time progress tracking for all task types
+- **Task Lifecycle Notifications**: Instant notifications for task creation, progress, completion, and failure
+- **Failure Alerts**: Immediate notification of task failures with error details
+- **Push Notifications**: Cross-platform push notifications for mobile/desktop
+- **Smart Throttling**: Optimized update frequency to prevent network spam
+
+### Management Features
+- **Task Listing**: Comprehensive API for listing and filtering all task types
+- **Task Statistics**: Detailed analytics and usage statistics for all operations
+- **Cleanup Operations**: Automatic and manual cleanup of failed/stale tasks
+- **Ownership Verification**: Secure access control for all operations
+- **Detailed Task Info**: Rich metadata including progress, parameters, and results
+- **Task Lifecycle Management**: Full control over task states (pause, resume, cancel)
+
+## 📋 Table of Contents
+
+- [Quick Start](#quick-start)
+- [API Reference](#api-reference)
+- [WebSocket Events](#websocket-events)
+- [Database Schema](#database-schema)
+- [Configuration](#configuration)
+- [Usage Examples](#usage-examples)
+- [Error Handling](#error-handling)
+- [Performance](#performance)
+- [Security](#security)
+- [Troubleshooting](#troubleshooting)
+
+## 🚀 Quick Start
+
+### 1. Create Upload Task
+
+```http
+POST /api/files/upload/create
+Content-Type: application/json
+
+{
+  "fileName": "large-video.mp4",
+  "fileSize": 1073741824,
+  "contentType": "video/mp4",
+  "poolId": "550e8400-e29b-41d4-a716-446655440000",
+  "chunkSize": 8388608
+}
+```
+
+**Response:**
+```json
+{
+  "taskId": "abc123def456ghi789",
+  "chunkSize": 8388608,
+  "chunksCount": 128
+}
+```
+
+### 2. Upload Chunks
+
+```http
+POST /api/files/upload/chunk/abc123def456ghi789/0
+Content-Type: multipart/form-data
+
+(chunk data as form file)
+```
+
+### 3. Complete Upload
+
+```http
+POST /api/files/upload/complete/abc123def456ghi789
+```
+
+## 📚 API Reference
+
+### Upload Task Management
+
+#### `POST /api/files/upload/create`
+Creates a new resumable upload task.
+
+**Request Body:**
+```json
+{
+  "fileName": "string",
+  "fileSize": "long",
+  "contentType": "string",
+  "poolId": "uuid",
+  "bundleId": "uuid",
+  "chunkSize": "long",
+  "encryptPassword": "string",
+  "expiredAt": "datetime",
+  "hash": "string"
+}
+```
+
+**Field Descriptions:**
+- `fileName`: Required - Name of the file
+- `fileSize`: Required - Size in bytes
+- `contentType`: Required - MIME type
+- `poolId`: Optional - Storage pool ID
+- `bundleId`: Optional - File bundle ID
+- `chunkSize`: Optional - Chunk size (default: 5MB)
+- `encryptPassword`: Optional - Encryption password
+- `expiredAt`: Optional - Expiration date
+- `hash`: Required - File hash for deduplication
+
+**Response:**
+```json
+{
+  "fileExists": false,
+  "taskId": "string",
+  "chunkSize": 5242880,
+  "chunksCount": 10
+}
+```
+
+#### `POST /api/files/upload/chunk/{taskId}/{chunkIndex}`
+Uploads a specific chunk of the file.
+
+**Parameters:**
+- `taskId`: Upload task identifier
+- `chunkIndex`: Zero-based chunk index
+
+**Request:** Multipart form data with chunk file
+
+**Response:** `200 OK` or `409 Conflict` (chunk already uploaded)
+
+#### `POST /api/files/upload/complete/{taskId}`
+Completes the upload and processes the file.
+
+**Response:** CloudFile object with file metadata
+
+### Task Information & Management
+
+#### `GET /api/files/upload/tasks`
+Lists user's upload tasks with filtering and pagination.
+
+**Query Parameters:**
+- `status`: Filter by status (`InProgress`, `Completed`, `Failed`, `Expired`)
+- `sortBy`: Sort field (`filename`, `filesize`, `createdAt`, `updatedAt`, `lastActivity`)
+- `sortDescending`: Sort direction (default: `true`)
+- `offset`: Pagination offset (default: `0`)
+- `limit`: Page size (default: `50`)
+
+**Response Headers:**
+- `X-Total`: Total number of tasks matching filters
+
+#### `GET /api/files/upload/progress/{taskId}`
+Gets current progress for a specific task.
+
+#### `GET /api/files/upload/resume/{taskId}`
+Gets task information needed to resume an interrupted upload.
+
+#### `DELETE /api/files/upload/task/{taskId}`
+Cancels an upload task and cleans up resources.
+
+#### `GET /api/files/upload/tasks/{taskId}/details`
+Gets comprehensive details about a specific task including:
+- Full task metadata
+- Pool and bundle information
+- Estimated time remaining
+- Current upload speed
+
+#### `GET /api/files/upload/stats`
+Gets upload statistics for the current user.
+
+**Response:**
+```json
+{
+  "totalTasks": 25,
+  "inProgressTasks": 3,
+  "completedTasks": 20,
+  "failedTasks": 1,
+  "expiredTasks": 1,
+  "totalUploadedBytes": 5368709120,
+  "averageProgress": 67.5,
+  "recentActivity": []
+}
+```
+
+#### `DELETE /api/files/upload/tasks/cleanup`
+Cleans up all failed and expired tasks for the current user.
+
+#### `GET /api/files/upload/tasks/recent?limit=10`
+Gets the most recent upload tasks.
+
+## 🔌 WebSocket Events
+
+The system sends real-time updates via WebSocket using RingService. Connect to the WebSocket endpoint and listen for task-related events.
+
+### Event Types
+
+#### `task.created`
+Sent when a new task is created.
+
+```json
+{
+  "type": "task.created",
+  "data": {
+    "taskId": "task123",
+    "name": "Upload File",
+    "type": "FileUpload",
+    "createdAt": "2025-11-09T02:00:00Z"
+  }
+}
+```
+
+#### `task.progress`
+Sent when task progress changes significantly (every 5% or major milestones).
+
+```json
+{
+  "type": "task.progress",
+  "data": {
+    "taskId": "task123",
+    "name": "Upload File",
+    "type": "FileUpload",
+    "progress": 67.5,
+    "status": "InProgress",
+    "lastActivity": "2025-11-09T02:05:00Z"
+  }
+}
+```
+
+#### `task.completed`
+Sent when a task completes successfully.
+
+```json
+{
+  "type": "task.completed",
+  "data": {
+    "taskId": "task123",
+    "name": "Upload File",
+    "type": "FileUpload",
+    "completedAt": "2025-11-09T02:10:00Z",
+    "results": {
+      "fileId": "file456",
+      "fileName": "document.pdf",
+      "fileSize": 10485760
+    }
+  }
+}
+```
+
+#### `task.failed`
+Sent when a task fails.
+
+```json
+{
+  "type": "task.failed",
+  "data": {
+    "taskId": "task123",
+    "name": "Upload File",
+    "type": "FileUpload",
+    "failedAt": "2025-11-09T02:15:00Z",
+    "errorMessage": "File processing failed: invalid format"
+  }
+}
+```
+
+### Client Integration Example
+
+```javascript
+// WebSocket connection
+const ws = new WebSocket('wss://api.dysonnetwork.com/ws');
+
+// Authentication (implement based on your auth system)
+ws.onopen = () => {
+    ws.send(JSON.stringify({
+        type: 'auth',
+        token: 'your-jwt-token'
+    }));
+};
+
+// Handle task events
+ws.onmessage = (event) => {
+    const packet = JSON.parse(event.data);
+
+    switch (packet.type) {
+        case 'task.progress':
+            updateProgressBar(packet.data);
+            break;
+        case 'task.completed':
+            showSuccessNotification(packet.data);
+            break;
+        case 'task.failed':
+            showErrorNotification(packet.data);
+            break;
+    }
+};
+
+function updateProgressBar(data) {
+    const progressBar = document.getElementById(`progress-${data.taskId}`);
+    if (progressBar) {
+        progressBar.style.width = `${data.progress}%`;
+        progressBar.textContent = `${data.progress.toFixed(1)}%`;
+    }
+}
+```
+
+### Note on Upload-Specific Notifications
+
+The system also includes upload-specific notifications (`upload.progress`, `upload.completed`, `upload.failed`) for backward compatibility. However, for new implementations, it's recommended to use the generic task notifications as they provide the same functionality with less object allocation overhead. Since users are typically in the foreground during upload operations, the generic task notifications provide sufficient progress visibility.
+
+## 🗄️ Database Schema
+
+### `upload_tasks` Table
+
+```sql
+CREATE TABLE upload_tasks (
+    id UUID PRIMARY KEY,
+    task_id VARCHAR NOT NULL UNIQUE,
+    file_name VARCHAR NOT NULL,
+    file_size BIGINT NOT NULL,
+    content_type VARCHAR NOT NULL,
+    chunk_size BIGINT NOT NULL,
+    chunks_count INTEGER NOT NULL,
+    chunks_uploaded INTEGER NOT NULL DEFAULT 0,
+    pool_id UUID NOT NULL,
+    bundle_id UUID,
+    encrypt_password VARCHAR,
+    expired_at TIMESTAMPTZ,
+    hash VARCHAR NOT NULL,
+    account_id UUID NOT NULL,
+    status INTEGER NOT NULL DEFAULT 0,
+    uploaded_chunks JSONB NOT NULL DEFAULT '[]'::jsonb,
+    last_activity TIMESTAMPTZ NOT NULL,
+    created_at TIMESTAMPTZ NOT NULL,
+    updated_at TIMESTAMPTZ NOT NULL,
+    deleted_at TIMESTAMPTZ
+);
+
+-- Indexes for performance
+CREATE INDEX idx_upload_tasks_account_id ON upload_tasks(account_id);
+CREATE INDEX idx_upload_tasks_status ON upload_tasks(status);
+CREATE INDEX idx_upload_tasks_last_activity ON upload_tasks(last_activity);
+CREATE INDEX idx_upload_tasks_hash ON upload_tasks(hash);
+```
+
+### Status Enum Values
+- `0`: InProgress
+- `1`: Completed
+- `2`: Failed
+- `3`: Expired
+
+## ⚙️ Configuration
+
+### Environment Variables
+
+```bash
+# Storage configuration
+STORAGE_UPLOADS_PATH=/tmp/uploads
+STORAGE_PREFERRED_REMOTE=550e8400-e29b-41d4-a716-446655440000
+
+# Chunk size settings
+UPLOAD_DEFAULT_CHUNK_SIZE=5242880  # 5MB
+UPLOAD_MAX_CHUNK_SIZE=16777216    # 16MB
+
+# Cleanup settings
+UPLOAD_STALE_THRESHOLD_HOURS=24
+UPLOAD_CLEANUP_INTERVAL_MINUTES=60
+
+# Cache settings
+UPLOAD_CACHE_DURATION_MINUTES=30
+```
+
+### Dependency Injection
+
+```csharp
+// In Program.cs or Startup.cs
+builder.Services.AddScoped<PersistentTaskService>();
+builder.Services.AddSingleton<RingService.RingServiceClient>(sp => {
+    // Configure gRPC client for RingService
+    var channel = GrpcChannel.ForAddress("https://ring-service:50051");
+    return new RingService.RingServiceClient(channel);
+});
+```
+
+## 💡 Usage Examples
+
+### Basic Upload Flow
+
+```javascript
+class UploadManager {
+    constructor() {
+        this.ws = new WebSocket('wss://api.dysonnetwork.com/ws');
+        this.tasks = new Map();
+    }
+
+    async uploadFile(file, poolId) {
+        // 1. Create upload task
+        const taskResponse = await fetch('/api/files/upload/create', {
+            method: 'POST',
+            headers: { 'Content-Type': 'application/json' },
+            body: JSON.stringify({
+                fileName: file.name,
+                fileSize: file.size,
+                contentType: file.type,
+                poolId: poolId,
+                hash: await this.calculateHash(file)
+            })
+        });
+
+        const task = await taskResponse.json();
+        if (task.fileExists) {
+            return task.file; // File already exists
+        }
+
+        // 2. Upload chunks
+        const chunks = this.splitFileIntoChunks(file, task.chunkSize);
+        for (let i = 0; i < chunks.length; i++) {
+            await this.uploadChunk(task.taskId, i, chunks[i]);
+        }
+
+        // 3. Complete upload
+        const result = await fetch(`/api/files/upload/complete/${task.taskId}`, {
+            method: 'POST'
+        });
+
+        return await result.json();
+    }
+
+    async uploadChunk(taskId, chunkIndex, chunkData) {
+        const formData = new FormData();
+        formData.append('chunk', chunkData);
+
+        const response = await fetch(`/api/files/upload/chunk/${taskId}/${chunkIndex}`, {
+            method: 'POST',
+            body: formData
+        });
+
+        if (response.status === 409) {
+            // Chunk already uploaded, skip
+            return;
+        }
+
+        if (!response.ok) {
+            throw new Error(`Upload failed: ${response.statusText}`);
+        }
+    }
+
+    splitFileIntoChunks(file, chunkSize) {
+        const chunks = [];
+        for (let offset = 0; offset < file.size; offset += chunkSize) {
+            chunks.push(file.slice(offset, offset + chunkSize));
+        }
+        return chunks;
+    }
+
+    async calculateHash(file) {
+        // Implement file hashing (SHA-256 recommended)
+        const buffer = await file.arrayBuffer();
+        const hashBuffer = await crypto.subtle.digest('SHA-256', buffer);
+        return Array.from(new Uint8Array(hashBuffer))
+            .map(b => b.toString(16).padStart(2, '0'))
+            .join('');
+    }
+}
+```
+
+### Resume Interrupted Upload
+
+```javascript
+async resumeUpload(taskId) {
+    // Get task information
+    const resumeResponse = await fetch(`/api/files/upload/resume/${taskId}`);
+    const taskInfo = await resumeResponse.json();
+
+    // Get uploaded chunks
+    const uploadedChunks = new Set(taskInfo.uploadedChunks);
+
+    // Upload missing chunks
+    for (let i = 0; i < taskInfo.chunksCount; i++) {
+        if (!uploadedChunks.has(i)) {
+            await this.uploadChunk(taskId, i, this.getChunkData(i));
+        }
+    }
+
+    // Complete upload
+    await fetch(`/api/files/upload/complete/${taskId}`, {
+        method: 'POST'
+    });
+}
+```
+
+### Monitor Upload Progress
+
+```javascript
+function setupProgressMonitoring(taskId) {
+    // Listen for WebSocket progress events
+    this.ws.addEventListener('message', (event) => {
+        const packet = JSON.parse(event.data);
+        if (packet.type === 'upload.progress' && packet.data.taskId === taskId) {
+            updateProgressUI(packet.data);
+        }
+    });
+}
+
+function updateProgressUI(progressData) {
+    const progressBar = document.getElementById('upload-progress');
+    const progressText = document.getElementById('progress-text');
+    const speedText = document.getElementById('upload-speed');
+
+    progressBar.style.width = `${progressData.progress}%`;
+    progressText.textContent = `${progressData.progress.toFixed(1)}%`;
+
+    // Calculate speed if we have timing data
+    if (this.lastProgress) {
+        const timeDiff = Date.now() - this.lastUpdate;
+        const progressDiff = progressData.progress - this.lastProgress.progress;
+        const speed = (progressDiff / 100) * (progressData.fileSize / 1024 / 1024) / (timeDiff / 1000);
+        speedText.textContent = `${speed.toFixed(1)} MB/s`;
+    }
+
+    this.lastProgress = progressData;
+    this.lastUpdate = Date.now();
+}
+```
+
+## 🚨 Error Handling
+
+### Common Error Codes
+
+- `400 Bad Request`: Invalid request parameters
+- `401 Unauthorized`: Authentication required
+- `403 Forbidden`: Insufficient permissions or quota exceeded
+- `404 Not Found`: Task or resource not found
+- `409 Conflict`: Chunk already uploaded (resumable upload)
+- `413 Payload Too Large`: File exceeds size limits
+- `429 Too Many Requests`: Rate limit exceeded
+
+### Error Response Format
+
+```json
+{
+  "code": "UPLOAD_FAILED",
+  "message": "Failed to complete file upload",
+  "status": 500,
+  "details": {
+    "taskId": "abc123def456",
+    "error": "File processing failed: invalid format"
+  }
+}
+```
+
+### Handling Upload Failures
+
+```javascript
+try {
+    const result = await completeUpload(taskId);
+    showSuccess(result);
+} catch (error) {
+    if (error.status === 500) {
+        // Server error, can retry
+        showRetryButton(taskId);
+    } else if (error.status === 403) {
+        // Permission/quota error
+        showQuotaExceeded();
+    } else {
+        // Other error
+        showGenericError(error.message);
+    }
+}
+```
+
+## ⚡ Performance
+
+### Optimizations
+
+- **Chunked Uploads**: Reduces memory usage for large files
+- **Progress Throttling**: Prevents WebSocket spam during fast uploads
+- **Caching Layer**: Redis-based caching for task metadata
+- **Database Indexing**: Optimized queries for task listing and filtering
+- **Async Processing**: Non-blocking I/O operations throughout
+
+### Benchmarks
+
+- **Small Files (< 10MB)**: ~2-5 seconds total upload time
+- **Large Files (1GB+)**: Maintains consistent throughput
+- **Concurrent Uploads**: Supports 100+ simultaneous uploads per server
+- **WebSocket Updates**: < 10ms latency for progress notifications
+
+### Scaling Considerations
+
+- **Horizontal Scaling**: Stateless design supports multiple instances
+- **Load Balancing**: Session affinity not required for uploads
+- **Storage Backend**: Compatible with S3, local storage, and distributed systems
+- **Database**: PostgreSQL with connection pooling recommended
+
+## 🔒 Security
+
+### Authentication & Authorization
+
+- **JWT Tokens**: All endpoints require valid authentication
+- **Ownership Verification**: Users can only access their own tasks
+- **Permission Checks**: Integration with role-based access control
+- **Rate Limiting**: Built-in protection against abuse
+
+### Data Protection
+
+- **Encryption Support**: Optional client-side encryption
+- **Secure Storage**: Files stored with proper access controls
+- **Hash Verification**: Integrity checking via SHA-256 hashes
+- **Audit Logging**: Comprehensive logging of all operations
+
+### Network Security
+
+- **HTTPS Only**: All communications encrypted in transit
+- **CORS Configuration**: Proper cross-origin resource sharing
+- **Input Validation**: Comprehensive validation of all inputs
+- **SQL Injection Prevention**: Parameterized queries throughout
+
+## 🔧 Troubleshooting
+
+### Common Issues
+
+#### Upload Stuck at 99%
+**Problem**: Final chunk fails to upload or process
+**Solution**: Check server logs, verify file integrity, retry completion
+
+#### WebSocket Not Connecting
+**Problem**: Real-time updates not working
+**Solution**: Check WebSocket server configuration, verify client authentication
+
+#### Progress Not Updating
+**Problem**: UI not reflecting upload progress
+**Solution**: Verify WebSocket connection, check for JavaScript errors
+
+#### Upload Fails with 403
+**Problem**: Permission denied errors
+**Solution**: Check user permissions, quota limits, and pool access
+
+### Debug Mode
+
+Enable detailed logging by setting environment variable:
+```bash
+LOG_LEVEL=DysonNetwork.Drive.Storage:Debug
+```
+
+### Health Checks
+
+Monitor system health via:
+```http
+GET /health/uploads
+```
+
+Returns status of upload service, database connectivity, and queue lengths.
+
+## 📞 Support
+
+For issues and questions:
+
+1. Check the troubleshooting section above
+2. Review server logs for error details
+3. Verify client implementation against examples
+4. Contact the development team with specific error messages
+
+## 📝 Changelog
+
+### Version 1.0.0
+- Initial release with resumable uploads
+- Real-time progress tracking via WebSocket
+- Push notification integration
+- Comprehensive task management APIs
+- Automatic cleanup and quota management
+
+---
+
+## 🎯 Generic Task System (v2.0)
+
+The upload system has been extended with a powerful generic task framework that supports various types of background operations beyond just file uploads.
+
+### Supported Task Types
+
+#### File Operations
+- **FileUpload**: Resumable file uploads (original functionality)
+- **FileMove**: Move files between storage pools or bundles
+- **FileCompress**: Compress multiple files into archives
+- **FileDecompress**: Extract compressed archives
+- **FileEncrypt**: Encrypt files with passwords
+- **FileDecrypt**: Decrypt encrypted files
+
+#### Bulk Operations
+- **BulkOperation**: Custom bulk operations on multiple files
+- **StorageMigration**: Migrate files between storage pools
+- **FileConversion**: Convert files between formats
+
+#### Custom Operations
+- **Custom**: Extensible framework for custom task types
+
+### Task Architecture
+
+#### Core Classes
+
+```csharp
+// Base task class with common functionality
+public class PersistentTask : ModelBase
+{
+    public Guid Id { get; set; }
+    public string TaskId { get; set; } = null!;
+    public string Name { get; set; } = null!;
+    public string? Description { get; set; }
+    public TaskType Type { get; set; }
+    public TaskStatus Status { get; set; }
+    public Guid AccountId { get; set; }
+    public double Progress { get; set; }
+    public Dictionary<string, object?> Parameters { get; set; } = new();
+    public Dictionary<string, object?> Results { get; set; } = new();
+    public string? ErrorMessage { get; set; }
+    public Instant LastActivity { get; set; }
+    public int Priority { get; set; }
+    public long? EstimatedDurationSeconds { get; set; }
+}
+
+// Specialized task implementations
+public class FileMoveTask : PersistentTask
+{
+    public FileMoveTask() { Type = TaskType.FileMove; Name = "Move Files"; }
+    public List<string> FileIds { get; set; } = new();
+    public Guid TargetPoolId { get; set; }
+    public Guid? TargetBundleId { get; set; }
+    public int FilesProcessed { get; set; }
+}
+
+public class FileCompressTask : PersistentTask
+{
+    public FileCompressTask() { Type = TaskType.FileCompress; Name = "Compress Files"; }
+    public List<string> FileIds { get; set; } = new();
+    public string CompressionFormat { get; set; } = "zip";
+    public int CompressionLevel { get; set; } = 6;
+    public string? OutputFileName { get; set; }
+    public int FilesProcessed { get; set; }
+    public string? ResultFileId { get; set; }
+}
+```
+
+#### Task Service
+
+```csharp
+public class PersistentTaskService(
+    AppDatabase db,
+    ICacheService cache,
+    ILogger<PersistentTaskService> logger,
+    RingService.RingServiceClient ringService
+)
+{
+    // Create any type of task
+    public async Task<T> CreateTaskAsync<T>(T task) where T : PersistentTask
+
+    // Update progress with automatic notifications
+    public async Task UpdateTaskProgressAsync(string taskId, double progress, string? statusMessage = null)
+
+    // Mark tasks as completed/failed with results
+    public async Task MarkTaskCompletedAsync(string taskId, Dictionary<string, object?>? results = null)
+    public async Task MarkTaskFailedAsync(string taskId, string? errorMessage = null)
+
+    // Task lifecycle management
+    public async Task PauseTaskAsync(string taskId)
+    public async Task ResumeTaskAsync(string taskId)
+    public async Task CancelTaskAsync(string taskId)
+
+    // Query tasks with filtering and pagination
+    public async Task<(List<PersistentTask> Items, int TotalCount)> GetUserTasksAsync(
+        Guid accountId,
+        TaskType? type = null,
+        TaskStatus? status = null,
+        string? sortBy = "lastActivity",
+        bool sortDescending = true,
+        int offset = 0,
+        int limit = 50
+    )
+}
+```
+
+### Real-Time Task Notifications
+
+All task operations send WebSocket notifications via RingService using the shared `GrpcTypeHelper` for consistent JSON serialization:
+
+#### Task Created
+```json
+{
+  "type": "task.created",
+  "data": {
+    "taskId": "task123",
+    "name": "Compress Files",
+    "type": "FileCompress",
+    "createdAt": "2025-11-09T02:00:00Z"
+  }
+}
+```
+
+#### Task Progress
+```json
+{
+  "type": "task.progress",
+  "data": {
+    "taskId": "task123",
+    "name": "Compress Files",
+    "type": "FileCompress",
+    "progress": 67.5,
+    "status": "InProgress",
+    "lastActivity": "2025-11-09T02:05:00Z"
+  }
+}
+```
+
+#### Task Completed
+```json
+{
+  "type": "task.completed",
+  "data": {
+    "taskId": "task123",
+    "name": "Compress Files",
+    "type": "FileCompress",
+    "completedAt": "2025-11-09T02:10:00Z",
+    "results": {
+      "resultFileId": "file456",
+      "compressedSize": 10485760,
+      "compressionRatio": 0.75
+    }
+  }
+}
+```
+
+### Usage Examples
+
+#### Create a File Compression Task
+
+```csharp
+var compressTask = new FileCompressTask
+{
+    Name = "Compress Project Files",
+    Description = "Compress all project files into a ZIP archive",
+    AccountId = userId,
+    FileIds = new List<string> { "file1", "file2", "file3" },
+    CompressionFormat = "zip",
+    CompressionLevel = 9,
+    OutputFileName = "project-backup.zip"
+};
+
+var createdTask = await taskService.CreateTaskAsync(compressTask);
+// Task ID: createdTask.TaskId
+```
+
+#### Monitor Task Progress
+
+```javascript
+// WebSocket monitoring
+ws.onmessage = (event) => {
+    const packet = JSON.parse(event.data);
+
+    if (packet.type === 'task.progress') {
+        const { taskId, progress, name } = packet.data;
+        updateTaskProgress(taskId, progress, name);
+    } else if (packet.type === 'task.completed') {
+        const { taskId, results } = packet.data;
+        handleTaskCompletion(taskId, results);
+    }
+};
+```
+
+#### Bulk File Operations
+
+```csharp
+var bulkTask = new BulkOperationTask
+{
+    Name = "Bulk Delete Old Files",
+    OperationType = "delete",
+    TargetIds = fileIds,
+    OperationParameters = new Dictionary<string, object?> {
+        { "olderThanDays", 30 },
+        { "confirm", true }
+    }
+};
+
+await taskService.CreateTaskAsync(bulkTask);
+```
+
+### Task Status Management
+
+Tasks support multiple statuses:
+- **Pending**: Queued for execution
+- **InProgress**: Currently executing
+- **Paused**: Temporarily suspended
+- **Completed**: Successfully finished
+- **Failed**: Execution failed
+- **Cancelled**: Manually cancelled
+- **Expired**: Timed out or expired
+
+### Available Service Methods
+
+Based on the `PersistentTaskService` implementation, the following methods are available:
+
+#### Core Task Operations
+- `CreateTaskAsync<T>(T task)`: Creates any type of persistent task
+- `GetTaskAsync<T>(string taskId)`: Retrieves a task by ID with caching
+- `UpdateTaskProgressAsync(string taskId, double progress, string? statusMessage)`: Updates task progress with automatic notifications
+- `MarkTaskCompletedAsync(string taskId, Dictionary<string, object?>? results)`: Marks task as completed with optional results
+- `MarkTaskFailedAsync(string taskId, string? errorMessage)`: Marks task as failed with error message
+- `PauseTaskAsync(string taskId)`: Pauses an in-progress task
+- `ResumeTaskAsync(string taskId)`: Resumes a paused task
+- `CancelTaskAsync(string taskId)`: Cancels a task
+
+#### Task Querying & Statistics
+- `GetUserTasksAsync()`: Gets tasks for a user with filtering and pagination
+- `GetUserTaskStatsAsync(Guid accountId)`: Gets comprehensive task statistics
+- `CleanupOldTasksAsync(Guid accountId, Duration maxAge)`: Cleans up old completed/failed tasks
+
+#### Upload-Specific Operations
+- `CreateUploadTaskAsync()`: Creates a new persistent upload task
+- `GetUploadTaskAsync(string taskId)`: Gets an existing upload task
+- `UpdateChunkProgressAsync(string taskId, int chunkIndex)`: Updates chunk upload progress
+- `IsChunkUploadedAsync(string taskId, int chunkIndex)`: Checks if a chunk has been uploaded
+- `GetUploadProgressAsync(string taskId)`: Gets upload progress as percentage
+- `GetUserUploadTasksAsync()`: Gets user upload tasks with filtering
+- `GetUserUploadStatsAsync(Guid accountId)`: Gets upload statistics for a user
+- `CleanupUserFailedTasksAsync(Guid accountId)`: Cleans up failed upload tasks
+- `GetRecentUserTasksAsync(Guid accountId, int limit)`: Gets recent upload tasks
+
+### Priority System
+
+Tasks can be assigned priorities (0-100, higher = more important) to control execution order in background processing.
+
+### Automatic Cleanup
+
+Old completed/failed tasks are automatically cleaned up after 30 days to prevent database bloat.
+
+### Extensibility
+
+The task system is designed to be easily extensible:
+
+```csharp
+// Create custom task types
+public class CustomProcessingTask : PersistentTask
+{
+    public CustomProcessingTask()
+    {
+        Type = TaskType.Custom;
+        Name = "Custom Processing";
+    }
+
+    public string CustomParameter
+    {
+        get => Parameters.GetValueOrDefault("customParam") as string ?? "";
+        set => Parameters["customParam"] = value;
+    }
+
+    public object? CustomResult
+    {
+        get => Results.GetValueOrDefault("customResult");
+        set => Results["customResult"] = value;
+    }
+}
+```
+
+### Database Schema Extensions
+
+The task system uses JSONB columns for flexible parameter and result storage:
+
+```sql
+-- Extended tasks table
+ALTER TABLE tasks ADD COLUMN priority INTEGER DEFAULT 0;
+ALTER TABLE tasks ADD COLUMN estimated_duration_seconds BIGINT;
+ALTER TABLE tasks ADD COLUMN started_at TIMESTAMPTZ;
+ALTER TABLE tasks ADD COLUMN completed_at TIMESTAMPTZ;
+
+-- Indexes for performance
+CREATE INDEX idx_tasks_type ON tasks(type);
+CREATE INDEX idx_tasks_status ON tasks(status);
+CREATE INDEX idx_tasks_priority ON tasks(priority);
+CREATE INDEX idx_tasks_account_type ON tasks(account_id, type);
+```
+
+### Migration Notes
+
+The system maintains backward compatibility with existing upload tasks while adding the new generic framework. Existing `PersistentUploadTask` entities continue to work unchanged.
+
+---
+
+**Note**: This system is designed for production use and includes comprehensive error handling, security measures, and performance optimizations. Always test thoroughly in your environment before deploying to production.

DysonNetwork.Drive/Storage/TusService.cs

@@ -1,306 +0,0 @@
-using System.Net;
-using System.Text;
-using System.Text.Json;
-using DysonNetwork.Drive.Billing;
-using DysonNetwork.Shared.Auth;
-using DysonNetwork.Shared.Proto;
-using Microsoft.AspNetCore.Mvc;
-using Microsoft.Extensions.Options;
-using NodaTime;
-using tusdotnet.Interfaces;
-using tusdotnet.Models;
-using tusdotnet.Models.Configuration;
-
-namespace DysonNetwork.Drive.Storage;
-
-public abstract class TusService
-{
-    public static DefaultTusConfiguration BuildConfiguration(
-        ITusStore store,
-        IConfiguration configuration
-    ) => new()
-    {
-        Store = store,
-        Events = new Events
-        {
-            OnAuthorizeAsync = async eventContext =>
-            {
-                if (eventContext.Intent == IntentType.DeleteFile)
-                {
-                    eventContext.FailRequest(
-                        HttpStatusCode.BadRequest,
-                        "Deleting files from this endpoint was disabled, please refer to the Dyson Network File API."
-                    );
-                    return;
-                }
-
-                var httpContext = eventContext.HttpContext;
-                if (httpContext.Items["CurrentUser"] is not Account currentUser)
-                {
-                    eventContext.FailRequest(HttpStatusCode.Unauthorized);
-                    return;
-                }
-
-                if (eventContext.Intent != IntentType.CreateFile) return;
-
-                using var scope = httpContext.RequestServices.CreateScope();
-
-                if (!currentUser.IsSuperuser)
-                {
-                    var pm = scope.ServiceProvider.GetRequiredService<PermissionService.PermissionServiceClient>();
-                    var allowed = await pm.HasPermissionAsync(new HasPermissionRequest
-                        { Actor = $"user:{currentUser.Id}", Area = "global", Key = "files.create" });
-                    if (!allowed.HasPermission)
-                        eventContext.FailRequest(HttpStatusCode.Forbidden);
-                }
-
-                var filePool = httpContext.Request.Headers["X-FilePool"].FirstOrDefault();
-                if (string.IsNullOrEmpty(filePool)) filePool = configuration["Storage:PreferredRemote"];
-                if (!Guid.TryParse(filePool, out _))
-                {
-                    eventContext.FailRequest(HttpStatusCode.BadRequest, "Invalid file pool id");
-                    return;
-                }
-
-                var fs = scope.ServiceProvider.GetRequiredService<FileService>();
-                var pool = await fs.GetPoolAsync(Guid.Parse(filePool!));
-                if (pool is null)
-                {
-                    eventContext.FailRequest(HttpStatusCode.BadRequest, "Pool not found");
-                    return;
-                }
-
-                if (pool.PolicyConfig.RequirePrivilege > 0)
-                {
-                    if (currentUser.PerkSubscription is null)
-                    {
-                        eventContext.FailRequest(
-                            HttpStatusCode.Forbidden,
-                            $"You need to have join the Stellar Program to use this pool"
-                        );
-                        return;
-                    }
-
-                    var privilege =
-                        PerkSubscriptionPrivilege.GetPrivilegeFromIdentifier(currentUser.PerkSubscription.Identifier);
-                    if (privilege < pool.PolicyConfig.RequirePrivilege)
-                    {
-                        eventContext.FailRequest(
-                            HttpStatusCode.Forbidden,
-                            $"You need Stellar Program tier {pool.PolicyConfig.RequirePrivilege} to use this pool, you are tier {privilege}"
-                        );
-                    }
-                }
-                
-                var bundleId = eventContext.HttpContext.Request.Headers["X-FileBundle"].FirstOrDefault();
-                if (!string.IsNullOrEmpty(bundleId) && !Guid.TryParse(bundleId, out _))
-                {
-                    eventContext.FailRequest(HttpStatusCode.BadRequest, "Invalid file bundle id");
-                }
-            },
-            OnFileCompleteAsync = async eventContext =>
-            {
-                using var scope = eventContext.HttpContext.RequestServices.CreateScope();
-                var services = scope.ServiceProvider;
-
-                var httpContext = eventContext.HttpContext;
-                if (httpContext.Items["CurrentUser"] is not Account user) return;
-
-                var file = await eventContext.GetFileAsync();
-                var metadata = await file.GetMetadataAsync(eventContext.CancellationToken);
-                var fileName = metadata.TryGetValue("filename", out var fn)
-                    ? fn.GetString(Encoding.UTF8)
-                    : "uploaded_file";
-                var contentType = metadata.TryGetValue("content-type", out var ct) ? ct.GetString(Encoding.UTF8) : null;
-
-                var fileStream = await file.GetContentAsync(eventContext.CancellationToken);
-
-                var filePool = httpContext.Request.Headers["X-FilePool"].FirstOrDefault();
-                var bundleId = eventContext.HttpContext.Request.Headers["X-FileBundle"].FirstOrDefault();
-                var encryptPassword = httpContext.Request.Headers["X-FilePass"].FirstOrDefault();
-
-                if (string.IsNullOrEmpty(filePool))
-                    filePool = configuration["Storage:PreferredRemote"];
-
-                Instant? expiredAt = null;
-                var expiredString = httpContext.Request.Headers["X-FileExpire"].FirstOrDefault();
-                if (!string.IsNullOrEmpty(expiredString) && int.TryParse(expiredString, out var expired))
-                    expiredAt = Instant.FromUnixTimeSeconds(expired);
-                
-                try
-                {
-                    var fileService = services.GetRequiredService<FileService>();
-                    var info = await fileService.ProcessNewFileAsync(
-                        user,
-                        file.Id,
-                        filePool!,
-                        bundleId,
-                        fileStream,
-                        fileName,
-                        contentType,
-                        encryptPassword,
-                        expiredAt
-                    );
-
-                    using var finalScope = eventContext.HttpContext.RequestServices.CreateScope();
-                    var jsonOptions = finalScope.ServiceProvider.GetRequiredService<IOptions<JsonOptions>>().Value
-                        .JsonSerializerOptions;
-                    var infoJson = JsonSerializer.Serialize(info, jsonOptions);
-                    eventContext.HttpContext.Response.Headers.Append("X-FileInfo", infoJson);
-                }
-                catch (Exception ex)
-                {
-                    var logger = services.GetRequiredService<ILogger<TusService>>();
-                    eventContext.HttpContext.Response.StatusCode = StatusCodes.Status400BadRequest;
-                    await eventContext.HttpContext.Response.WriteAsync(ex.Message);
-                    logger.LogError(ex, "Error handling file upload...");
-                }
-                finally
-                {
-                    // Dispose the stream after all processing is complete
-                    await fileStream.DisposeAsync();
-                }
-            },
-            OnBeforeCreateAsync = async eventContext =>
-            {
-                var httpContext = eventContext.HttpContext;
-                if (httpContext.Items["CurrentUser"] is not Account currentUser)
-                {
-                    eventContext.FailRequest(HttpStatusCode.Unauthorized);
-                    return;
-                }
-                var accountId = Guid.Parse(currentUser.Id);
-
-                var poolId = eventContext.HttpContext.Request.Headers["X-FilePool"].FirstOrDefault();
-                if (string.IsNullOrEmpty(poolId)) poolId = configuration["Storage:PreferredRemote"];
-                if (!Guid.TryParse(poolId, out _))
-                {
-                    eventContext.FailRequest(HttpStatusCode.BadRequest, "Invalid file pool id");
-                    return;
-                }
-
-                var bundleId = eventContext.HttpContext.Request.Headers["X-FileBundle"].FirstOrDefault();
-                if (!string.IsNullOrEmpty(bundleId) && !Guid.TryParse(bundleId, out _))
-                {
-                    eventContext.FailRequest(HttpStatusCode.BadRequest, "Invalid file bundle id");
-                    return;
-                }
-
-                var metadata = eventContext.Metadata;
-                var contentType = metadata.TryGetValue("content-type", out var ct) ? ct.GetString(Encoding.UTF8) : null;
-
-                var scope = eventContext.HttpContext.RequestServices.CreateScope();
-
-                var rejected = false;
-
-                var fs = scope.ServiceProvider.GetRequiredService<FileService>();
-                var pool = await fs.GetPoolAsync(Guid.Parse(poolId!));
-                if (pool is null)
-                {
-                    eventContext.FailRequest(HttpStatusCode.BadRequest, "Pool not found");
-                    rejected = true;
-                }
-
-                var logger = scope.ServiceProvider.GetRequiredService<ILogger<TusService>>();
-
-                // Do the policy check
-                var policy = pool!.PolicyConfig;
-                if (!rejected && !pool.PolicyConfig.AllowEncryption)
-                {
-                    var encryptPassword = eventContext.HttpContext.Request.Headers["X-FilePass"].FirstOrDefault();
-                    if (!string.IsNullOrEmpty(encryptPassword))
-                    {
-                        eventContext.FailRequest(
-                            HttpStatusCode.Forbidden,
-                            "File encryption is not allowed in this pool"
-                        );
-                        rejected = true;
-                    }
-                }
-
-                if (!rejected && policy.AcceptTypes is not null)
-                {
-                    if (string.IsNullOrEmpty(contentType))
-                    {
-                        eventContext.FailRequest(
-                            HttpStatusCode.BadRequest,
-                            "Content type is required by the pool's policy"
-                        );
-                        rejected = true;
-                    }
-                    else
-                    {
-                        var foundMatch = false;
-                        foreach (var acceptType in policy.AcceptTypes)
-                        {
-                            if (acceptType.EndsWith("/*", StringComparison.OrdinalIgnoreCase))
-                            {
-                                var type = acceptType[..^2];
-                                if (!contentType.StartsWith($"{type}/", StringComparison.OrdinalIgnoreCase)) continue;
-                                foundMatch = true;
-                                break;
-                            }
-                            else if (acceptType.Equals(contentType, StringComparison.OrdinalIgnoreCase))
-                            {
-                                foundMatch = true;
-                                break;
-                            }
-                        }
-
-                        if (!foundMatch)
-                        {
-                            eventContext.FailRequest(
-                                HttpStatusCode.Forbidden,
-                                $"Content type {contentType} is not allowed by the pool's policy"
-                            );
-                            rejected = true;
-                        }
-                    }
-                }
-
-                if (!rejected && policy.MaxFileSize is not null)
-                {
-                    if (eventContext.UploadLength > policy.MaxFileSize)
-                    {
-                        eventContext.FailRequest(
-                            HttpStatusCode.Forbidden,
-                            $"File size {eventContext.UploadLength} is larger than the pool's maximum file size {policy.MaxFileSize}"
-                        );
-                        rejected = true;
-                    }
-                }
-
-                if (!rejected)
-                {
-                    var quotaService = scope.ServiceProvider.GetRequiredService<QuotaService>();
-                    var (ok, billableUnit, quota) = await quotaService.IsFileAcceptable(
-                        accountId,
-                        pool.BillingConfig.CostMultiplier ?? 1.0,
-                        eventContext.UploadLength
-                    );
-                    if (!ok)
-                    {
-                        eventContext.FailRequest(
-                            HttpStatusCode.Forbidden,
-                            $"File size {billableUnit} MiB is exceeded the user's quota {quota} MiB"
-                        );
-                        rejected = true;
-                    }
-                }
-
-                if (rejected)
-                    logger.LogInformation("File rejected #{FileId}", eventContext.FileId);
-            },
-            OnCreateCompleteAsync = eventContext =>
-            {
-                var directUpload = eventContext.HttpContext.Request.Headers["X-DirectUpload"].FirstOrDefault();
-                if (!string.IsNullOrEmpty(directUpload)) return Task.CompletedTask;
-
-                var gatewayUrl = configuration["GatewayUrl"];
-                if (gatewayUrl is not null)
-                    eventContext.SetUploadUrl(new Uri(gatewayUrl + "/drive/tus/" + eventContext.FileId));
-                return Task.CompletedTask;
-            },
-        }
-    };
-}

DysonNetwork.Drive/VersionController.cs

@@ -1,20 +0,0 @@
-using DysonNetwork.Shared.Data;
-using Microsoft.AspNetCore.Mvc;
-
-namespace DysonNetwork.Drive;
-
-[ApiController]
-[Route("/api/version")]
-public class VersionController : ControllerBase
-{
-    [HttpGet]
-    public IActionResult Get()
-    {
-        return Ok(new AppVersion
-        {
-            Version = ThisAssembly.AssemblyVersion,
-            Commit = ThisAssembly.GitCommitId,
-            UpdateDate = ThisAssembly.GitCommitDate
-        });
-    }
-}

DysonNetwork.Drive/appsettings.json

@@ -1,133 +1,121 @@
 {
-  "Debug": true,
-  "BaseUrl": "http://localhost:5090",
-  "GatewayUrl": "http://localhost:5094",
-  "Logging": {
-    "LogLevel": {
-      "Default": "Information",
-      "Microsoft.AspNetCore": "Warning"
-    }
-  },
-  "AllowedHosts": "*",
-  "ConnectionStrings": {
-    "App": "Host=localhost;Port=5432;Database=dyson_drive;Username=postgres;Password=postgres;Include Error Detail=True;Maximum Pool Size=20;Connection Idle Lifetime=60"
-  },
-  "Authentication": {
-    "Schemes": {
-      "Bearer": {
-        "ValidAudiences": [
-          "http://localhost:5071",
-          "https://localhost:7099"
-        ],
-        "ValidIssuer": "solar-network"
-      }
-    }
-  },
-  "AuthToken": {
-    "PublicKeyPath": "Keys/PublicKey.pem",
-    "PrivateKeyPath": "Keys/PrivateKey.pem"
-  },
-  "OidcProvider": {
-    "IssuerUri": "https://nt.solian.app",
-    "PublicKeyPath": "Keys/PublicKey.pem",
-    "PrivateKeyPath": "Keys/PrivateKey.pem",
-    "AccessTokenLifetime": "01:00:00",
-    "RefreshTokenLifetime": "30.00:00:00",
-    "AuthorizationCodeLifetime": "00:30:00",
-    "RequireHttpsMetadata": true
-  },
-  "Tus": {
-    "StorePath": "Uploads"
-  },
-  "Storage": {
-    "PreferredRemote": "2adceae3-981a-4564-9b8d-5d71a211c873",
-    "Remote": [
-      {
-        "Id": "minio",
-        "Label": "Minio",
-        "Region": "auto",
-        "Bucket": "solar-network-development",
-        "Endpoint": "localhost:9000",
-        "SecretId": "littlesheep",
-        "SecretKey": "password",
-        "EnabledSigned": true,
-        "EnableSsl": false
-      },
-      {
-        "Id": "cloudflare",
-        "Label": "Cloudflare R2",
-        "Region": "auto",
-        "Bucket": "solar-network",
-        "Endpoint": "0a70a6d1b7128888c823359d0008f4e1.r2.cloudflarestorage.com",
-        "SecretId": "8ff5d06c7b1639829d60bc6838a542e6",
-        "SecretKey": "fd58158c5201be16d1872c9209d9cf199421dae3c2f9972f94b2305976580d67",
-        "EnableSigned": true,
-        "EnableSsl": true
-      }
+    "Debug": true,
+    "BaseUrl": "http://localhost:5090",
+    "GatewayUrl": "http://localhost:5094",
+    "Logging": {
+        "LogLevel": {
+            "Default": "Information",
+            "Microsoft.AspNetCore": "Warning"
+        }
+    },
+    "AllowedHosts": "*",
+    "ConnectionStrings": {
+        "App": "Host=localhost;Port=5432;Database=dyson_drive;Username=postgres;Password=postgres;Include Error Detail=True;Maximum Pool Size=20;Connection Idle Lifetime=60"
+    },
+    "Authentication": {
+        "Schemes": {
+            "Bearer": {
+                "ValidAudiences": [
+                    "http://localhost:5071",
+                    "https://localhost:7099"
+                ],
+                "ValidIssuer": "solar-network"
+            }
+        }
+    },
+    "AuthToken": {
+        "PublicKeyPath": "Keys/PublicKey.pem",
+        "PrivateKeyPath": "Keys/PrivateKey.pem"
+    },
+    "Storage": {
+        "Uploads": "Uploads",
+        "PreferredRemote": "c53136a6-9152-4ecb-9f88-43c41438c23e",
+        "Remote": [
+            {
+                "Id": "minio",
+                "Label": "Minio",
+                "Region": "auto",
+                "Bucket": "solar-network-development",
+                "Endpoint": "localhost:9000",
+                "SecretId": "littlesheep",
+                "SecretKey": "password",
+                "EnabledSigned": true,
+                "EnableSsl": false
+            },
+            {
+                "Id": "cloudflare",
+                "Label": "Cloudflare R2",
+                "Region": "auto",
+                "Bucket": "solar-network",
+                "Endpoint": "0a70a6d1b7128888c823359d0008f4e1.r2.cloudflarestorage.com",
+                "SecretId": "8ff5d06c7b1639829d60bc6838a542e6",
+                "SecretKey": "fd58158c5201be16d1872c9209d9cf199421dae3c2f9972f94b2305976580d67",
+                "EnableSigned": true,
+                "EnableSsl": true
+            }
+        ]
+    },
+    "Captcha": {
+        "Provider": "cloudflare",
+        "ApiKey": "0x4AAAAAABCDUdOujj4feOb_",
+        "ApiSecret": "0x4AAAAAABCDUWABiJQweqlB7tYq-IqIm8U"
+    },
+    "Notifications": {
+        "Topic": "dev.solsynth.solian",
+        "Endpoint": "http://localhost:8088"
+    },
+    "Email": {
+        "Server": "smtp4dev.orb.local",
+        "Port": 25,
+        "UseSsl": false,
+        "Username": "no-reply@mail.solsynth.dev",
+        "Password": "password",
+        "FromAddress": "no-reply@mail.solsynth.dev",
+        "FromName": "Alphabot",
+        "SubjectPrefix": "Solar Network"
+    },
+    "RealtimeChat": {
+        "Endpoint": "https://solar-network-im44o8gq.livekit.cloud",
+        "ApiKey": "APIs6TiL8wj3A4j",
+        "ApiSecret": "SffxRneIwTnlHPtEf3zicmmv3LUEl7xXael4PvWZrEhE"
+    },
+    "GeoIp": {
+        "DatabasePath": "./Keys/GeoLite2-City.mmdb"
+    },
+    "Oidc": {
+        "Google": {
+            "ClientId": "961776991058-963m1qin2vtp8fv693b5fdrab5hmpl89.apps.googleusercontent.com",
+            "ClientSecret": ""
+        },
+        "Apple": {
+            "ClientId": "dev.solsynth.solian",
+            "TeamId": "W7HPZ53V6B",
+            "KeyId": "B668YP4KBG",
+            "PrivateKeyPath": "./Keys/Solarpass.p8"
+        },
+        "Microsoft": {
+            "ClientId": "YOUR_MICROSOFT_CLIENT_ID",
+            "ClientSecret": "YOUR_MICROSOFT_CLIENT_SECRET",
+            "DiscoveryEndpoint": "YOUR_MICROSOFT_DISCOVERY_ENDPOINT"
+        }
+    },
+    "Payment": {
+        "Auth": {
+            "Afdian": "<token here>"
+        },
+        "Subscriptions": {
+            "Afdian": {
+                "7d17aae23c9611f0b5705254001e7c00": "solian.stellar.primary",
+                "7dfae4743c9611f0b3a55254001e7c00": "solian.stellar.nova",
+                "141713ee3d6211f085b352540025c377": "solian.stellar.supernova"
+            }
+        }
+    },
+    "Cache": {
+        "Serializer": "MessagePack"
+    },
+    "KnownProxies": [
+        "127.0.0.1",
+        "::1"
     ]
-  },
-  "Captcha": {
-    "Provider": "cloudflare",
-    "ApiKey": "0x4AAAAAABCDUdOujj4feOb_",
-    "ApiSecret": "0x4AAAAAABCDUWABiJQweqlB7tYq-IqIm8U"
-  },
-  "Notifications": {
-    "Topic": "dev.solsynth.solian",
-    "Endpoint": "http://localhost:8088"
-  },
-  "Email": {
-    "Server": "smtp4dev.orb.local",
-    "Port": 25,
-    "UseSsl": false,
-    "Username": "no-reply@mail.solsynth.dev",
-    "Password": "password",
-    "FromAddress": "no-reply@mail.solsynth.dev",
-    "FromName": "Alphabot",
-    "SubjectPrefix": "Solar Network"
-  },
-  "RealtimeChat": {
-    "Endpoint": "https://solar-network-im44o8gq.livekit.cloud",
-    "ApiKey": "APIs6TiL8wj3A4j",
-    "ApiSecret": "SffxRneIwTnlHPtEf3zicmmv3LUEl7xXael4PvWZrEhE"
-  },
-  "GeoIp": {
-    "DatabasePath": "./Keys/GeoLite2-City.mmdb"
-  },
-  "Oidc": {
-    "Google": {
-      "ClientId": "961776991058-963m1qin2vtp8fv693b5fdrab5hmpl89.apps.googleusercontent.com",
-      "ClientSecret": ""
-    },
-    "Apple": {
-      "ClientId": "dev.solsynth.solian",
-      "TeamId": "W7HPZ53V6B",
-      "KeyId": "B668YP4KBG",
-      "PrivateKeyPath": "./Keys/Solarpass.p8"
-    },
-    "Microsoft": {
-      "ClientId": "YOUR_MICROSOFT_CLIENT_ID",
-      "ClientSecret": "YOUR_MICROSOFT_CLIENT_SECRET",
-      "DiscoveryEndpoint": "YOUR_MICROSOFT_DISCOVERY_ENDPOINT"
-    }
-  },
-  "Payment": {
-    "Auth": {
-      "Afdian": "<token here>"
-    },
-    "Subscriptions": {
-      "Afdian": {
-        "7d17aae23c9611f0b5705254001e7c00": "solian.stellar.primary",
-        "7dfae4743c9611f0b3a55254001e7c00": "solian.stellar.nova",
-        "141713ee3d6211f085b352540025c377": "solian.stellar.supernova"
-      }
-    }
-  },
-  "KnownProxies": [
-    "127.0.0.1",
-    "::1"
-  ],
-  "Service": {
-    "Name": "DysonNetwork.Drive",
-    "Url": "https://localhost:7092"
-  }
 }

DysonNetwork.Drive/bun.lock

@@ -1,13 +0,0 @@
-{
-  "lockfileVersion": 1,
-  "workspaces": {
-    "": {
-      "dependencies": {
-        "highlight.js": "^11.11.1",
-      },
-    },
-  },
-  "packages": {
-    "highlight.js": ["highlight.js@11.11.1", "", {}, "sha512-Xwwo44whKBVCYoliBQwaPvtd/2tYFkRQtXDWj1nackaV2JPXx3L0+Jvd8/qCJ2p+ML0/XVkJ2q+Mr+UVdpJK5w=="],
-  }
-}

DysonNetwork.Gateway/Configuration/ConfigurationController.cs

@@ -0,0 +1,12 @@
+using Microsoft.AspNetCore.Mvc;
+
+[ApiController]
+[Route("config")]
+public class ConfigurationController(IConfiguration configuration) : ControllerBase
+{
+    [HttpGet]
+    public IActionResult Get() => Ok(configuration.GetSection("Client").Get<Dictionary<string, object>>());
+
+    [HttpGet("site")]
+    public IActionResult GetSiteUrl() => Ok(configuration["SiteUrl"]);
+}

DysonNetwork.Gateway/Dockerfile

@@ -0,0 +1,23 @@
+FROM mcr.microsoft.com/dotnet/aspnet:10.0 AS base
+USER $APP_UID
+WORKDIR /app
+EXPOSE 8080
+EXPOSE 8081
+
+FROM mcr.microsoft.com/dotnet/sdk:10.0 AS build
+ARG BUILD_CONFIGURATION=Release
+WORKDIR /src
+COPY ["DysonNetwork.Gateway/DysonNetwork.Gateway.csproj", "DysonNetwork.Gateway/"]
+RUN dotnet restore "DysonNetwork.Gateway/DysonNetwork.Gateway.csproj"
+COPY . .
+WORKDIR "/src/DysonNetwork.Gateway"
+RUN dotnet build "./DysonNetwork.Gateway.csproj" -c $BUILD_CONFIGURATION -o /app/build
+
+FROM build AS publish
+ARG BUILD_CONFIGURATION=Release
+RUN dotnet publish "./DysonNetwork.Gateway.csproj" -c $BUILD_CONFIGURATION -o /app/publish /p:UseAppHost=false
+
+FROM base AS final
+WORKDIR /app
+COPY --from=publish /app/publish .
+ENTRYPOINT ["dotnet", "DysonNetwork.Gateway.dll"]

DysonNetwork.Gateway/DysonNetwork.Gateway.csproj

@@ -0,0 +1,22 @@
+<Project Sdk="Microsoft.NET.Sdk.Web">
+
+  <PropertyGroup>
+    <TargetFramework>net10.0</TargetFramework>
+    <Nullable>enable</Nullable>
+    <ImplicitUsings>enable</ImplicitUsings>
+  </PropertyGroup>
+
+  <ItemGroup>
+    <PackageReference Include="Microsoft.Extensions.ServiceDiscovery.Yarp" Version="10.1.0" />
+    <PackageReference Include="Nerdbank.GitVersioning" Version="3.9.50">
+      <PrivateAssets>all</PrivateAssets>
+      <IncludeAssets>runtime; build; native; contentfiles; analyzers; buildtransitive</IncludeAssets>
+    </PackageReference>
+    <PackageReference Include="Yarp.ReverseProxy" Version="2.3.0" />
+  </ItemGroup>
+
+  <ItemGroup>
+      <ProjectReference Include="..\DysonNetwork.Shared\DysonNetwork.Shared.csproj" />
+  </ItemGroup>
+
+</Project>

DysonNetwork.Gateway/Health/GatewayConstant.cs

@@ -0,0 +1,9 @@
+namespace DysonNetwork.Gateway.Health;
+
+public abstract class GatewayConstant
+{
+    public static readonly string[] ServiceNames = ["ring", "pass", "drive", "sphere", "develop", "insight", "zone"];
+    
+    // Core services stands with w/o these services the functional of entire app will broke.
+    public static readonly string[] CoreServiceNames = ["ring", "pass", "drive", "sphere"];
+}

DysonNetwork.Gateway/Health/GatewayHealthAggregator.cs

@@ -0,0 +1,60 @@
+using NodaTime;
+
+namespace DysonNetwork.Gateway.Health;
+
+public class GatewayHealthAggregator(IHttpClientFactory httpClientFactory, GatewayReadinessStore store)
+    : BackgroundService
+{
+    private async Task<ServiceHealthState> CheckService(string serviceName)
+    {
+        var client = httpClientFactory.CreateClient("health");
+        var now = SystemClock.Instance.GetCurrentInstant();
+
+        try
+        {
+            // Use the service discovery to lookup service
+            // The service defaults give every single service a health endpoint that we can use here
+            using var response = await client.GetAsync($"http://{serviceName}/health");
+
+            if (response.IsSuccessStatusCode)
+            {
+                return new ServiceHealthState(
+                    serviceName,
+                    true,
+                    now,
+                    null
+                );
+            }
+
+            return new ServiceHealthState(
+                serviceName,
+                false,
+                now,
+                $"StatusCode: {(int)response.StatusCode}"
+            );
+        }
+        catch (Exception ex)
+        {
+            return new ServiceHealthState(
+                serviceName,
+                false,
+                now,
+                ex.Message
+            );
+        }
+    }
+
+    protected override async Task ExecuteAsync(CancellationToken stoppingToken)
+    {
+        while (!stoppingToken.IsCancellationRequested)
+        {
+            foreach (var service in GatewayConstant.ServiceNames)
+            {
+                var result = await CheckService(service);
+                store.Update(result);
+            }
+
+            await Task.Delay(TimeSpan.FromSeconds(5), stoppingToken);
+        }
+    }
+}

DysonNetwork.Gateway/Health/GatewayReadinessMiddleware.cs

@@ -0,0 +1,35 @@
+namespace DysonNetwork.Gateway.Health;
+
+using Microsoft.AspNetCore.Http;
+
+public sealed class GatewayReadinessMiddleware(RequestDelegate next)
+{
+    public async Task InvokeAsync(HttpContext context, GatewayReadinessStore store)
+    {
+        if (context.Request.Path.StartsWithSegments("/health"))
+        {
+            await next(context);
+            return;
+        }
+
+        var readiness = store.Current;
+
+        // Only core services participate in readiness gating
+        var notReadyCoreServices = readiness.Services
+            .Where(kv => GatewayConstant.CoreServiceNames.Contains(kv.Key))
+            .Where(kv => !kv.Value.IsHealthy)
+            .Select(kv => kv.Key)
+            .ToArray();
+
+        if (notReadyCoreServices.Length > 0)
+        {
+            context.Response.StatusCode = StatusCodes.Status503ServiceUnavailable;
+            var unavailableServices = string.Join(", ", notReadyCoreServices);
+            context.Response.Headers["X-NotReady"] = unavailableServices;
+            await context.Response.WriteAsync("Solar Network is warming up. Try again later please.");
+            return;
+        }
+
+        await next(context);
+    }
+}

DysonNetwork.Gateway/Health/GatewayReadinessStore.cs

@@ -0,0 +1,76 @@
+using NodaTime;
+
+namespace DysonNetwork.Gateway.Health;
+
+public record ServiceHealthState(
+    string ServiceName,
+    bool IsHealthy,
+    Instant LastChecked,
+    string? Error
+);
+
+public record GatewayReadinessState(
+    bool IsReady,
+    IReadOnlyDictionary<string, ServiceHealthState> Services,
+    Instant LastUpdated
+);
+
+public class GatewayReadinessStore
+{
+    private readonly Lock _lock = new();
+
+    private readonly Dictionary<string, ServiceHealthState> _services = new();
+
+    public GatewayReadinessState Current { get; private set; } = new(
+        IsReady: false,
+        Services: new Dictionary<string, ServiceHealthState>(),
+        LastUpdated: SystemClock.Instance.GetCurrentInstant()
+    );
+
+    public IReadOnlyCollection<string> ServiceNames => _services.Keys;
+
+    public GatewayReadinessStore()
+    {
+        InitializeServices(GatewayConstant.ServiceNames);
+    }
+
+    private void InitializeServices(IEnumerable<string> serviceNames)
+    {
+        lock (_lock)
+        {
+            _services.Clear();
+
+            foreach (var name in serviceNames)
+            {
+                _services[name] = new ServiceHealthState(
+                    name,
+                    IsHealthy: false,
+                    LastChecked: SystemClock.Instance.GetCurrentInstant(),
+                    Error: "Not checked yet"
+                );
+            }
+
+            RecalculateLocked();
+        }
+    }
+
+    public void Update(ServiceHealthState state)
+    {
+        lock (_lock)
+        {
+            _services[state.ServiceName] = state;
+            RecalculateLocked();
+        }
+    }
+
+    private void RecalculateLocked()
+    {
+        var isReady = _services.Count > 0 && _services.Values.All(s => s.IsHealthy);
+
+        Current = new GatewayReadinessState(
+            IsReady: isReady,
+            Services: new Dictionary<string, ServiceHealthState>(_services),
+            LastUpdated: SystemClock.Instance.GetCurrentInstant()
+        );
+    }
+}

DysonNetwork.Gateway/Health/GatewayStatusController.cs

@@ -0,0 +1,14 @@
+using Microsoft.AspNetCore.Mvc;
+
+namespace DysonNetwork.Gateway.Health;
+
+[ApiController]
+[Route("/health")]
+public class GatewayStatusController(GatewayReadinessStore readinessStore) : ControllerBase
+{
+    [HttpGet]
+    public ActionResult<GatewayReadinessState> GetHealthStatus()
+    {
+        return Ok(readinessStore.Current);
+    }
+}

DysonNetwork.Gateway/Program.cs

@@ -0,0 +1,189 @@
+using System.Text.Json;
+using System.Text.Json.Serialization;
+using System.Threading.RateLimiting;
+using DysonNetwork.Gateway.Health;
+using DysonNetwork.Shared.Http;
+using Yarp.ReverseProxy.Configuration;
+using Microsoft.AspNetCore.HttpOverrides;
+using NodaTime;
+using NodaTime.Serialization.SystemTextJson;
+
+var builder = WebApplication.CreateBuilder(args);
+
+builder.AddServiceDefaults();
+
+builder.ConfigureAppKestrel(builder.Configuration, maxRequestBodySize: long.MaxValue, enableGrpc: false);
+
+builder.Services.AddSingleton<GatewayReadinessStore>();
+builder.Services.AddHostedService<GatewayHealthAggregator>();
+
+builder.Services.AddCors(options =>
+{
+    options.AddDefaultPolicy(policy =>
+    {
+        policy.SetIsOriginAllowed(origin => true)
+            .AllowAnyMethod()
+            .AllowAnyHeader()
+            .AllowCredentials()
+            .WithExposedHeaders("X-Total", "X-NotReady");
+    });
+});
+
+builder.Services.AddRateLimiter(options =>
+{
+    options.AddPolicy("fixed", context =>
+    {
+        var ip = context.Connection.RemoteIpAddress?.ToString() ?? "unknown";
+
+        return RateLimitPartition.GetFixedWindowLimiter(
+            partitionKey: ip,
+            factory: _ => new FixedWindowRateLimiterOptions
+            {
+                PermitLimit = 120, // 120 requests...
+                Window = TimeSpan.FromMinutes(1), // ...per minute per IP
+                QueueProcessingOrder = QueueProcessingOrder.OldestFirst,
+                QueueLimit = 10 // allow short bursts instead of instant 503s
+            });
+    });
+
+    options.OnRejected = async (context, token) =>
+    {
+        // Log the rejected IP
+        var logger = context.HttpContext.RequestServices
+            .GetRequiredService<ILoggerFactory>()
+            .CreateLogger("RateLimiter");
+
+        var ip = context.HttpContext.Connection.RemoteIpAddress?.ToString() ?? "unknown";
+        logger.LogWarning("Rate limit exceeded for IP: {IP}", ip);
+
+        // Respond to the client
+        context.HttpContext.Response.StatusCode = StatusCodes.Status429TooManyRequests;
+        await context.HttpContext.Response.WriteAsync(
+            "Rate limit exceeded. Try again later.", token);
+    };
+});
+
+
+var specialRoutes = new[]
+{
+    new RouteConfig
+    {
+        RouteId = "ring-ws",
+        ClusterId = "ring",
+        Match = new RouteMatch { Path = "/ws" }
+    },
+    new RouteConfig
+    {
+        RouteId = "pass-openid",
+        ClusterId = "pass",
+        Match = new RouteMatch { Path = "/.well-known/openid-configuration" }
+    },
+    new RouteConfig
+    {
+        RouteId = "pass-jwks",
+        ClusterId = "pass",
+        Match = new RouteMatch { Path = "/.well-known/jwks" }
+    },
+    new RouteConfig
+    {
+        RouteId = "sphere-webfinger",
+        ClusterId = "sphere",
+        Match = new RouteMatch { Path = "/.well-known/webfinger" }
+    },
+    new RouteConfig
+    {
+        RouteId = "sphere-activitypub",
+        ClusterId = "sphere",
+        Match = new RouteMatch { Path = "/activitypub/{**catch-all}" }
+    },
+};
+
+var apiRoutes = GatewayConstant.ServiceNames.Select(serviceName =>
+{
+    var apiPath = serviceName switch
+    {
+        _ => $"/{serviceName}"
+    };
+    return new RouteConfig
+    {
+        RouteId = $"{serviceName}-api",
+        ClusterId = serviceName,
+        Match = new RouteMatch { Path = $"{apiPath}/{{**catch-all}}" },
+        Transforms =
+        [
+            new Dictionary<string, string> { { "PathRemovePrefix", apiPath } },
+            new Dictionary<string, string> { { "PathPrefix", "/api" } }
+        ]
+    };
+});
+
+var swaggerRoutes = GatewayConstant.ServiceNames.Select(serviceName => new RouteConfig
+{
+    RouteId = $"{serviceName}-swagger",
+    ClusterId = serviceName,
+    Match = new RouteMatch { Path = $"/swagger/{serviceName}/{{**catch-all}}" },
+    Transforms =
+    [
+        new Dictionary<string, string> { { "PathRemovePrefix", $"/swagger/{serviceName}" } },
+        new Dictionary<string, string> { { "PathPrefix", "/swagger" } }
+    ]
+});
+
+var routes = specialRoutes.Concat(apiRoutes).Concat(swaggerRoutes).ToArray();
+
+var clusters = GatewayConstant.ServiceNames.Select(serviceName => new ClusterConfig
+{
+    ClusterId = serviceName,
+    HealthCheck = new HealthCheckConfig
+    {
+        Active = new ActiveHealthCheckConfig
+        {
+            Enabled = true,
+            Interval = TimeSpan.FromSeconds(10),
+            Timeout = TimeSpan.FromSeconds(5),
+            Path = "/health"
+        },
+        Passive = new PassiveHealthCheckConfig
+        {
+            Enabled = true
+        }
+    },
+    Destinations = new Dictionary<string, DestinationConfig>
+    {
+        { "destination1", new DestinationConfig { Address = $"http://{serviceName}" } }
+    }
+}).ToArray();
+
+builder.Services
+    .AddReverseProxy()
+    .LoadFromMemory(routes, clusters)
+    .AddServiceDiscoveryDestinationResolver();
+
+builder.Services.AddControllers().AddJsonOptions(options =>
+{
+    options.JsonSerializerOptions.NumberHandling = JsonNumberHandling.AllowNamedFloatingPointLiterals;
+    options.JsonSerializerOptions.PropertyNamingPolicy = JsonNamingPolicy.SnakeCaseLower;
+    options.JsonSerializerOptions.DictionaryKeyPolicy = JsonNamingPolicy.SnakeCaseLower;
+
+    options.JsonSerializerOptions.ConfigureForNodaTime(DateTimeZoneProviders.Tzdb);
+});
+
+var app = builder.Build();
+
+var forwardedHeadersOptions = new ForwardedHeadersOptions
+{
+    ForwardedHeaders = ForwardedHeaders.All
+};
+forwardedHeadersOptions.KnownIPNetworks.Clear();
+forwardedHeadersOptions.KnownProxies.Clear();
+app.UseForwardedHeaders(forwardedHeadersOptions);
+
+app.UseCors();
+
+app.UseMiddleware<GatewayReadinessMiddleware>();
+
+app.MapReverseProxy().RequireRateLimiting("fixed");
+
+app.MapControllers();
+
+app.Run();

DysonNetwork.Gateway/Properties/launchSettings.json

@@ -0,0 +1,21 @@
+{
+  "$schema": "https://json.schemastore.org/launchsettings.json",
+  "profiles": {
+    "http": {
+      "commandName": "Project",
+      "dotnetRunMessages": true,
+      "launchBrowser": true,
+      "environmentVariables": {
+        "ASPNETCORE_ENVIRONMENT": "Development"
+      }
+    },
+    "https": {
+      "commandName": "Project",
+      "dotnetRunMessages": true,
+      "launchBrowser": true,
+      "environmentVariables": {
+        "ASPNETCORE_ENVIRONMENT": "Development"
+      }
+    }
+  }
+}

DysonNetwork.Gateway/VersionController.cs

@@ -1,10 +1,10 @@
-using Microsoft.AspNetCore.Mvc;
 using DysonNetwork.Shared.Data;
+using Microsoft.AspNetCore.Mvc;
 
-namespace DysonNetwork.Pass;
+namespace DysonNetwork.Gateway;
 
 [ApiController]
-[Route("/api/version")]
+[Route("/version")]
 public class VersionController : ControllerBase
 {
     [HttpGet]

DysonNetwork.Gateway/appsettings.json

@@ -0,0 +1,16 @@
+{
+    "Logging": {
+        "LogLevel": {
+            "Default": "Information",
+            "Microsoft.AspNetCore": "Warning"
+        }
+    },
+    "Cache": {
+        "Serializer": "MessagePack"
+    },
+    "AllowedHosts": "*",
+    "SiteUrl": "http://localhost:3000",
+    "Client": {
+        "SomeSetting": "SomeValue"
+    }
+}

DysonNetwork.Insight/AppDatabase.cs

@@ -0,0 +1,57 @@
+using DysonNetwork.Shared.Data;
+using DysonNetwork.Shared.Models;
+using Microsoft.EntityFrameworkCore;
+using Microsoft.EntityFrameworkCore.Design;
+using NodaTime;
+
+namespace DysonNetwork.Insight;
+
+public class AppDatabase(
+    DbContextOptions<AppDatabase> options,
+    IConfiguration configuration
+) : DbContext(options)
+{
+    public DbSet<SnThinkingSequence> ThinkingSequences { get; set; }
+    public DbSet<SnThinkingThought> ThinkingThoughts { get; set; }
+    public DbSet<SnUnpaidAccount> UnpaidAccounts { get; set; }
+    
+    protected override void OnConfiguring(DbContextOptionsBuilder optionsBuilder)
+    {
+        optionsBuilder.UseNpgsql(
+            configuration.GetConnectionString("App"),
+            opt => opt
+                .ConfigureDataSource(optSource => optSource.EnableDynamicJson())
+                .UseQuerySplittingBehavior(QuerySplittingBehavior.SplitQuery)
+                .UseNodaTime()
+        ).UseSnakeCaseNamingConvention();
+
+        base.OnConfiguring(optionsBuilder);
+    }
+    
+    public override async Task<int> SaveChangesAsync(CancellationToken cancellationToken = default)
+    {
+        this.ApplyAuditableAndSoftDelete();
+        return await base.SaveChangesAsync(cancellationToken);
+    }
+
+    protected override void OnModelCreating(ModelBuilder modelBuilder)
+    {
+        base.OnModelCreating(modelBuilder);
+        
+        modelBuilder.ApplySoftDeleteFilters();
+    }
+}
+
+public class AppDatabaseFactory : IDesignTimeDbContextFactory<AppDatabase>
+{
+    public AppDatabase CreateDbContext(string[] args)
+    {
+        var configuration = new ConfigurationBuilder()
+            .SetBasePath(Directory.GetCurrentDirectory())
+            .AddJsonFile("appsettings.json")
+            .Build();
+
+        var optionsBuilder = new DbContextOptionsBuilder<AppDatabase>();
+        return new AppDatabase(optionsBuilder.Options, configuration);
+    }
+}