SolarNetwork/Swarm
2
3
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

✨ File index

Browse Source
This commit is contained in:
LittleSheep
2025-11-12 22:09:13 +08:00
parent ce715cd6b0
commit e2b2bdd262
19 changed files with 2189 additions and 62 deletions
Download Patch File Download Diff File Expand all files Collapse all files

1
DysonNetwork.Drive/AppDatabase.cs
View File

@@ -25,6 +25,7 @@ public class AppDatabase(
public DbSet<SnCloudFile> Files { get; set; } = null!;
public DbSet<CloudFileReference> FileReferences { get; set; } = null!;
public DbSet<SnCloudFileIndex> FileIndexes { get; set; }
public DbSet<PersistentTask> Tasks { get; set; } = null!;
public DbSet<PersistentUploadTask> UploadTasks { get; set; } = null!; // Backward compatibility

411
DysonNetwork.Drive/Index/FileIndexController.cs Normal file
View File

@@ -0,0 +1,411 @@
using System.ComponentModel.DataAnnotations;
using DysonNetwork.Drive.Storage;
using DysonNetwork.Shared.Auth;
using DysonNetwork.Shared.Http;
using DysonNetwork.Shared.Models;
using DysonNetwork.Shared.Proto;
using Microsoft.AspNetCore.Authorization;
using Microsoft.AspNetCore.Mvc;
using Microsoft.EntityFrameworkCore;
namespace DysonNetwork.Drive.Index;
[ApiController]
[Route("/api/index")]
[Authorize]
public class FileIndexController(
FileIndexService fileIndexService,
AppDatabase db,
ILogger<FileIndexController> logger
) : ControllerBase
{
/// <summary>
/// Gets files in a specific path for the current user
/// </summary>
/// <param name="path">The path to browse (defaults to root "/")</param>
/// <returns>List of files in the specified path</returns>
[HttpGet("browse")]
public async Task<IActionResult> BrowseFiles([FromQuery] string path = "/")
{
if (HttpContext.Items["CurrentUser"] is not Account currentUser)
return new ObjectResult(ApiError.Unauthorized()) { StatusCode = 401 };
var accountId = Guid.Parse(currentUser.Id);
try
{
var fileIndexes = await fileIndexService.GetByPathAsync(accountId, path);
return Ok(new
{
Path = path,
Files = fileIndexes,
TotalCount = fileIndexes.Count
});
}
catch (Exception ex)
{
logger.LogError(ex, "Failed to browse files for account {AccountId} at path {Path}", accountId, path);
return new ObjectResult(new ApiError
{
Code = "BROWSE_FAILED",
Message = "Failed to browse files",
Status = 500
}) { StatusCode = 500 };
}
}
/// <summary>
/// Gets all files for the current user (across all paths)
/// </summary>
/// <returns>List of all files for the user</returns>
[HttpGet("all")]
public async Task<IActionResult> GetAllFiles()
{
if (HttpContext.Items["CurrentUser"] is not Account currentUser)
return new ObjectResult(ApiError.Unauthorized()) { StatusCode = 401 };
var accountId = Guid.Parse(currentUser.Id);
try
{
var fileIndexes = await fileIndexService.GetByAccountIdAsync(accountId);
return Ok(new
{
Files = fileIndexes,
TotalCount = fileIndexes.Count()
});
}
catch (Exception ex)
{
logger.LogError(ex, "Failed to get all files for account {AccountId}", accountId);
return new ObjectResult(new ApiError
{
Code = "GET_ALL_FAILED",
Message = "Failed to get files",
Status = 500
}) { StatusCode = 500 };
}
}
/// <summary>
/// Moves a file to a new path
/// </summary>
/// <param name="indexId">The file index ID</param>
/// <param name="newPath">The new path</param>
/// <returns>The updated file index</returns>
[HttpPost("move/{indexId}")]
public async Task<IActionResult> MoveFile(Guid indexId, [FromBody] MoveFileRequest request)
{
if (HttpContext.Items["CurrentUser"] is not Account currentUser)
return new ObjectResult(ApiError.Unauthorized()) { StatusCode = 401 };
var accountId = Guid.Parse(currentUser.Id);
try
{
// Verify ownership
var existingIndex = await db.FileIndexes
.Include(fi => fi.File)
.FirstOrDefaultAsync(fi => fi.Id == indexId && fi.AccountId == accountId);
if (existingIndex == null)
return new ObjectResult(ApiError.NotFound("File index")) { StatusCode = 404 };
var updatedIndex = await fileIndexService.UpdateAsync(indexId, request.NewPath);
if (updatedIndex == null)
return new ObjectResult(ApiError.NotFound("File index")) { StatusCode = 404 };
return Ok(new
{
updatedIndex.FileId,
IndexId = updatedIndex.Id,
OldPath = existingIndex.Path,
NewPath = updatedIndex.Path,
Message = "File moved successfully"
});
}
catch (Exception ex)
{
logger.LogError(ex, "Failed to move file index {IndexId} for account {AccountId}", indexId, accountId);
return new ObjectResult(new ApiError
{
Code = "MOVE_FAILED",
Message = "Failed to move file",
Status = 500
}) { StatusCode = 500 };
}
}
/// <summary>
/// Removes a file index (does not delete the actual file by default)
/// </summary>
/// <param name="indexId">The file index ID</param>
/// <param name="deleteFile">Whether to also delete the actual file data</param>
/// <returns>Success message</returns>
[HttpDelete("remove/{indexId}")]
public async Task<IActionResult> RemoveFileIndex(Guid indexId, [FromQuery] bool deleteFile = false)
{
if (HttpContext.Items["CurrentUser"] is not Account currentUser)
return new ObjectResult(ApiError.Unauthorized()) { StatusCode = 401 };
var accountId = Guid.Parse(currentUser.Id);
try
{
// Verify ownership
var existingIndex = await db.FileIndexes
.Include(fi => fi.File)
.FirstOrDefaultAsync(fi => fi.Id == indexId && fi.AccountId == accountId);
if (existingIndex == null)
return new ObjectResult(ApiError.NotFound("File index")) { StatusCode = 404 };
var fileId = existingIndex.FileId;
var fileName = existingIndex.File.Name;
var filePath = existingIndex.Path;
// Remove the index
var removed = await fileIndexService.RemoveAsync(indexId);
if (!removed)
return new ObjectResult(ApiError.NotFound("File index")) { StatusCode = 404 };
// Optionally delete the actual file
if (!deleteFile)
return Ok(new
{
Message = deleteFile
? "File index and file data removed successfully"
: "File index removed successfully",
FileId = fileId,
FileName = fileName,
Path = filePath,
FileDataDeleted = deleteFile
});
try
{
// Check if there are any other indexes for this file
var remainingIndexes = await fileIndexService.GetByFileIdAsync(fileId);
if (remainingIndexes.Count == 0)
{
// No other indexes exist, safe to delete the file
var file = await db.Files.FirstOrDefaultAsync(f => f.Id == fileId.ToString());
if (file != null)
{
db.Files.Remove(file);
await db.SaveChangesAsync();
logger.LogInformation("Deleted file {FileId} ({FileName}) as requested", fileId, fileName);
}
}
}
catch (Exception ex)
{
logger.LogWarning(ex, "Failed to delete file {FileId} while removing index", fileId);
// Continue even if file deletion fails
}
return Ok(new
{
Message = deleteFile ? "File index and file data removed successfully" : "File index removed successfully",
FileId = fileId,
FileName = fileName,
Path = filePath,
FileDataDeleted = deleteFile
});
}
catch (Exception ex)
{
logger.LogError(ex, "Failed to remove file index {IndexId} for account {AccountId}", indexId, accountId);
return new ObjectResult(new ApiError
{
Code = "REMOVE_FAILED",
Message = "Failed to remove file",
Status = 500
}) { StatusCode = 500 };
}
}
/// <summary>
/// Removes all file indexes in a specific path
/// </summary>
/// <param name="path">The path to clear</param>
/// <param name="deleteFiles">Whether to also delete the actual file data</param>
/// <returns>Success message with count of removed items</returns>
[HttpDelete("clear-path")]
public async Task<IActionResult> ClearPath([FromQuery] string path = "/", [FromQuery] bool deleteFiles = false)
{
if (HttpContext.Items["CurrentUser"] is not Account currentUser)
return new ObjectResult(ApiError.Unauthorized()) { StatusCode = 401 };
var accountId = Guid.Parse(currentUser.Id);
try
{
var removedCount = await fileIndexService.RemoveByPathAsync(accountId, path);
if (!deleteFiles || removedCount <= 0)
return Ok(new
{
Message = deleteFiles
? $"Cleared {removedCount} file indexes from path and deleted orphaned files"
: $"Cleared {removedCount} file indexes from path",
Path = path,
RemovedCount = removedCount,
FilesDeleted = deleteFiles
});
// Get the files that were in this path and check if they have other indexes
var filesInPath = await fileIndexService.GetByPathAsync(accountId, path);
var fileIdsToCheck = filesInPath.Select(fi => fi.FileId).Distinct().ToList();
foreach (var fileId in fileIdsToCheck)
{
var remainingIndexes = await fileIndexService.GetByFileIdAsync(fileId);
if (remainingIndexes.Count != 0) continue;
// No other indexes exist, safe to delete the file
var file = await db.Files.FirstOrDefaultAsync(f => f.Id == fileId.ToString());
if (file == null) continue;
db.Files.Remove(file);
logger.LogInformation("Deleted orphaned file {FileId} after clearing path {Path}", fileId, path);
}
await db.SaveChangesAsync();
return Ok(new
{
Message = deleteFiles ?
$"Cleared {removedCount} file indexes from path and deleted orphaned files" :
$"Cleared {removedCount} file indexes from path",
Path = path,
RemovedCount = removedCount,
FilesDeleted = deleteFiles
});
}
catch (Exception ex)
{
logger.LogError(ex, "Failed to clear path {Path} for account {AccountId}", path, accountId);
return new ObjectResult(new ApiError
{
Code = "CLEAR_PATH_FAILED",
Message = "Failed to clear path",
Status = 500
}) { StatusCode = 500 };
}
}
/// <summary>
/// Creates a new file index (useful for adding existing files to a path)
/// </summary>
/// <param name="request">The create index request</param>
/// <returns>The created file index</returns>
[HttpPost("create")]
public async Task<IActionResult> CreateFileIndex([FromBody] CreateFileIndexRequest request)
{
if (HttpContext.Items["CurrentUser"] is not Account currentUser)
return new ObjectResult(ApiError.Unauthorized()) { StatusCode = 401 };
var accountId = Guid.Parse(currentUser.Id);
try
{
// Verify the file exists and belongs to the user
var file = await db.Files.FirstOrDefaultAsync(f => f.Id == request.FileId);
if (file == null)
return new ObjectResult(ApiError.NotFound("File")) { StatusCode = 404 };
if (file.AccountId != accountId)
return new ObjectResult(ApiError.Unauthorized(forbidden: true)) { StatusCode = 403 };
// Check if index already exists for this file and path
var existingIndex = await db.FileIndexes
.FirstOrDefaultAsync(fi => fi.FileId == request.FileId && fi.Path == request.Path && fi.AccountId == accountId);
if (existingIndex != null)
return new ObjectResult(ApiError.Validation(new Dictionary<string, string[]>
{
{ "fileId", ["File index already exists for this path"] }
})) { StatusCode = 400 };
var fileIndex = await fileIndexService.CreateAsync(request.Path, request.FileId, accountId);
return Ok(new
{
IndexId = fileIndex.Id,
fileIndex.FileId,
fileIndex.Path,
Message = "File index created successfully"
});
}
catch (Exception ex)
{
logger.LogError(ex, "Failed to create file index for file {FileId} at path {Path} for account {AccountId}",
request.FileId, request.Path, accountId);
return new ObjectResult(new ApiError
{
Code = "CREATE_INDEX_FAILED",
Message = "Failed to create file index",
Status = 500
}) { StatusCode = 500 };
}
}
/// <summary>
/// Searches for files by name or metadata
/// </summary>
/// <param name="query">The search query</param>
/// <param name="path">Optional path to limit search to</param>
/// <returns>Matching files</returns>
[HttpGet("search")]
public async Task<IActionResult> SearchFiles([FromQuery] string query, [FromQuery] string? path = null)
{
if (HttpContext.Items["CurrentUser"] is not Account currentUser)
return new ObjectResult(ApiError.Unauthorized()) { StatusCode = 401 };
var accountId = Guid.Parse(currentUser.Id);
try
{
// Build the query with all conditions at once
var searchTerm = query.ToLower();
var fileIndexes = await db.FileIndexes
.Where(fi => fi.AccountId == accountId)
.Include(fi => fi.File)
.Where(fi =>
(string.IsNullOrEmpty(path) || fi.Path == FileIndexService.NormalizePath(path)) &&
(fi.File.Name.ToLower().Contains(searchTerm) ||
(fi.File.Description != null && fi.File.Description.ToLower().Contains(searchTerm)) ||
(fi.File.MimeType != null && fi.File.MimeType.ToLower().Contains(searchTerm))))
.ToListAsync();
return Ok(new
{
Query = query,
Path = path,
Results = fileIndexes,
TotalCount = fileIndexes.Count()
});
}
catch (Exception ex)
{
logger.LogError(ex, "Failed to search files for account {AccountId} with query {Query}", accountId, query);
return new ObjectResult(new ApiError
{
Code = "SEARCH_FAILED",
Message = "Failed to search files",
Status = 500
}) { StatusCode = 500 };
}
}
}
public class MoveFileRequest
{
public string NewPath { get; set; } = null!;
}
public class CreateFileIndexRequest
{
[MaxLength(32)] public string FileId { get; set; } = null!;
public string Path { get; set; } = null!;
}

187
DysonNetwork.Drive/Index/FileIndexService.cs Normal file
View File

@@ -0,0 +1,187 @@
using DysonNetwork.Shared.Models;
using Microsoft.EntityFrameworkCore;
namespace DysonNetwork.Drive.Index;
public class FileIndexService(AppDatabase db)
{
/// <summary>
/// Creates a new file index entry
/// </summary>
/// <param name="path">The parent folder path with trailing slash</param>
/// <param name="fileId">The file ID</param>
/// <param name="accountId">The account ID</param>
/// <returns>The created file index</returns>
public async Task<SnCloudFileIndex> CreateAsync(string path, string fileId, Guid accountId)
{
// Ensure a path has trailing slash and is query-safe
var normalizedPath = NormalizePath(path);
var fileIndex = new SnCloudFileIndex
{
Path = normalizedPath,
FileId = fileId,
AccountId = accountId
};
db.FileIndexes.Add(fileIndex);
await db.SaveChangesAsync();
return fileIndex;
}
/// <summary>
/// Updates an existing file index entry by removing the old one and creating a new one
/// </summary>
/// <param name="id">The file index ID</param>
/// <param name="newPath">The new parent folder path with trailing slash</param>
/// <returns>The updated file index</returns>
public async Task<SnCloudFileIndex?> UpdateAsync(Guid id, string newPath)
{
var fileIndex = await db.FileIndexes.FindAsync(id);
if (fileIndex == null)
return null;
// Since properties are init-only, we need to remove the old index and create a new one
db.FileIndexes.Remove(fileIndex);
var newFileIndex = new SnCloudFileIndex
{
Path = NormalizePath(newPath),
FileId = fileIndex.FileId,
AccountId = fileIndex.AccountId
};
db.FileIndexes.Add(newFileIndex);
await db.SaveChangesAsync();
return newFileIndex;
}
/// <summary>
/// Removes a file index entry by ID
/// </summary>
/// <param name="id">The file index ID</param>
/// <returns>True if the index was found and removed, false otherwise</returns>
public async Task<bool> RemoveAsync(Guid id)
{
var fileIndex = await db.FileIndexes.FindAsync(id);
if (fileIndex == null)
return false;
db.FileIndexes.Remove(fileIndex);
await db.SaveChangesAsync();
return true;
}
/// <summary>
/// Removes file index entries by file ID
/// </summary>
/// <param name="fileId">The file ID</param>
/// <returns>The number of indexes removed</returns>
public async Task<int> RemoveByFileIdAsync(string fileId)
{
var indexes = await db.FileIndexes
.Where(fi => fi.FileId == fileId)
.ToListAsync();
if (indexes.Count == 0)
return 0;
db.FileIndexes.RemoveRange(indexes);
await db.SaveChangesAsync();
return indexes.Count;
}
/// <summary>
/// Removes file index entries by account ID and path
/// </summary>
/// <param name="accountId">The account ID</param>
/// <param name="path">The parent folder path</param>
/// <returns>The number of indexes removed</returns>
public async Task<int> RemoveByPathAsync(Guid accountId, string path)
{
var normalizedPath = NormalizePath(path);
var indexes = await db.FileIndexes
.Where(fi => fi.AccountId == accountId && fi.Path == normalizedPath)
.ToListAsync();
if (!indexes.Any())
return 0;
db.FileIndexes.RemoveRange(indexes);
await db.SaveChangesAsync();
return indexes.Count;
}
/// <summary>
/// Gets file indexes by account ID and path
/// </summary>
/// <param name="accountId">The account ID</param>
/// <param name="path">The parent folder path</param>
/// <returns>List of file indexes</returns>
public async Task<List<SnCloudFileIndex>> GetByPathAsync(Guid accountId, string path)
{
var normalizedPath = NormalizePath(path);
return await db.FileIndexes
.Where(fi => fi.AccountId == accountId && fi.Path == normalizedPath)
.Include(fi => fi.File)
.ToListAsync();
}
/// <summary>
/// Gets file indexes by file ID
/// </summary>
/// <param name="fileId">The file ID</param>
/// <returns>List of file indexes</returns>
public async Task<List<SnCloudFileIndex>> GetByFileIdAsync(string fileId)
{
return await db.FileIndexes
.Where(fi => fi.FileId == fileId)
.Include(fi => fi.File)
.ToListAsync();
}
/// <summary>
/// Gets all file indexes for an account
/// </summary>
/// <param name="accountId">The account ID</param>
/// <returns>List of file indexes</returns>
public async Task<List<SnCloudFileIndex>> GetByAccountIdAsync(Guid accountId)
{
return await db.FileIndexes
.Where(fi => fi.AccountId == accountId)
.Include(fi => fi.File)
.ToListAsync();
}
/// <summary>
/// Normalizes the path to ensure it has a trailing slash and is query-safe
/// </summary>
/// <param name="path">The original path</param>
/// <returns>The normalized path</returns>
public static string NormalizePath(string path)
{
if (string.IsNullOrEmpty(path))
return "/";
// Ensure the path starts with a slash
if (!path.StartsWith('/'))
path = "/" + path;
// Ensure the path ends with a slash (unless it's just the root)
if (path != "/" && !path.EndsWith('/'))
path += "/";
// Make path query-safe by removing problematic characters
// This is a basic implementation - you might want to add more robust validation
path = path.Replace("%", "").Replace("'", "").Replace("\"", "");
return path;
}
}

341
DysonNetwork.Drive/Index/README.md Normal file
View File

@@ -0,0 +1,341 @@
# File Indexing System Documentation
## Overview
The File Indexing System provides a hierarchical file organization layer on top of the existing file storage system in DysonNetwork Drive. It allows users to organize their files in folders and paths while maintaining the underlying file storage capabilities.
When using with the gateway, replace the `/api` with the `/drive` in the path.
And all the arguments will be transformed into snake case via the gateway.
## Architecture
### Core Components
1. **SnCloudFileIndex Model** - Represents the file-to-path mapping
2. **FileIndexService** - Business logic for file index operations
3. **FileIndexController** - REST API endpoints for file management
4. **FileUploadController Integration** - Automatic index creation during upload
### Database Schema
```sql
-- File Indexes table
CREATE TABLE "FileIndexes" (
"Id" uuid NOT NULL DEFAULT gen_random_uuid(),
"Path" character varying(8192) NOT NULL,
"FileId" uuid NOT NULL,
"AccountId" uuid NOT NULL,
"CreatedAt" timestamp with time zone NOT NULL DEFAULT (now() at time zone 'utc'),
"UpdatedAt" timestamp with time zone NOT NULL DEFAULT (now() at time zone 'utc'),
CONSTRAINT "PK_FileIndexes" PRIMARY KEY ("Id"),
CONSTRAINT "FK_FileIndexes_Files_FileId" FOREIGN KEY ("FileId") REFERENCES "Files" ("Id") ON DELETE CASCADE,
INDEX "IX_FileIndexes_Path_AccountId" ("Path", "AccountId")
);
```
## API Endpoints
### Browse Files
**GET** `/api/index/browse?path=/documents/`
Browse files in a specific path.
**Query Parameters:**
- `path` (optional, default: "/") - The path to browse
**Response:**
```json
{
"path": "/documents/",
"files": [
{
"id": "guid",
"path": "/documents/",
"fileId": "guid",
"accountId": "guid",
"createdAt": "2024-01-01T00:00:00Z",
"updatedAt": "2024-01-01T00:00:00Z",
"file": {
"id": "string",
"name": "document.pdf",
"size": 1024,
"mimeType": "application/pdf",
"hash": "sha256-hash",
"uploadedAt": "2024-01-01T00:00:00Z",
"expiredAt": null,
"hasCompression": false,
"hasThumbnail": true,
"isEncrypted": false,
"description": null
}
}
],
"totalCount": 1
}
```
### Get All Files
**GET** `/api/index/all`
Get all files for the current user across all paths.
**Response:**
```json
{
"files": [
// Same structure as browse endpoint
],
"totalCount": 10
}
```
### Move File
**POST** `/api/index/move/{indexId}`
Move a file to a new path.
**Path Parameters:**
- `indexId` - The file index ID
**Request Body:**
```json
{
"newPath": "/archived/"
}
```
**Response:**
```json
{
"fileId": "guid",
"indexId": "guid",
"oldPath": "/documents/",
"newPath": "/archived/",
"message": "File moved successfully"
}
```
### Remove File Index
**DELETE** `/api/index/remove/{indexId}?deleteFile=false`
Remove a file index. Optionally delete the actual file data.
**Path Parameters:**
- `indexId` - The file index ID
**Query Parameters:**
- `deleteFile` (optional, default: false) - Whether to also delete the file data
**Response:**
```json
{
"message": "File index removed successfully",
"fileId": "guid",
"fileName": "document.pdf",
"path": "/documents/",
"fileDataDeleted": false
}
```
### Clear Path
**DELETE** `/api/index/clear-path?path=/temp/&deleteFiles=false`
Remove all file indexes in a specific path.
**Query Parameters:**
- `path` (optional, default: "/") - The path to clear
- `deleteFiles` (optional, default: false) - Whether to also delete orphaned files
**Response:**
```json
{
"message": "Cleared 5 file indexes from path",
"path": "/temp/",
"removedCount": 5,
"filesDeleted": false
}
```
### Create File Index
**POST** `/api/index/create`
Create a new file index for an existing file.
**Request Body:**
```json
{
"fileId": "guid",
"path": "/documents/"
}
```
**Response:**
```json
{
"indexId": "guid",
"fileId": "guid",
"path": "/documents/",
"message": "File index created successfully"
}
```
### Search Files
**GET** `/api/index/search?query=report&path=/documents/`
Search for files by name or metadata.
**Query Parameters:**
- `query` (required) - The search query
- `path` (optional) - Limit search to specific path
**Response:**
```json
{
"query": "report",
"path": "/documents/",
"results": [
// Same structure as browse endpoint
],
"totalCount": 3
}
```
## Path Normalization
The system automatically normalizes paths to ensure consistency:
- **Trailing Slash**: All paths end with `/`
- **Root Path**: User home folder is represented as `/`
- **Query Safety**: Paths are validated to avoid SQL injection
- **Examples**:
- `/documents/` ✅ (correct)
- `/documents` → `/documents/` ✅ (normalized)
- `/documents/reports/` ✅ (correct)
- `/documents/reports` → `/documents/reports/` ✅ (normalized)
## File Upload Integration
When uploading files with the `FileUploadController`, you can specify a path to automatically create file indexes:
**Create Upload Task Request:**
```json
{
"fileName": "document.pdf",
"fileSize": 1024,
"contentType": "application/pdf",
"hash": "sha256-hash",
"path": "/documents/" // New field for file indexing
}
```
The system will automatically create a file index when the upload completes successfully.
## Service Methods
### FileIndexService
```csharp
public class FileIndexService
{
// Create a new file index
Task<SnCloudFileIndex> CreateAsync(string path, Guid fileId, Guid accountId);
// Get files by path
Task<List<SnCloudFileIndex>> GetByPathAsync(Guid accountId, string path);
// Get all files for account
Task<List<SnCloudFileIndex>> GetByAccountIdAsync(Guid accountId);
// Get indexes for specific file
Task<List<SnCloudFileIndex>> GetByFileIdAsync(Guid fileId);
// Move file to new path
Task<SnCloudFileIndex?> UpdateAsync(Guid indexId, string newPath);
// Remove file index
Task<bool> RemoveAsync(Guid indexId);
// Remove all indexes in path
Task<int> RemoveByPathAsync(Guid accountId, string path);
// Normalize path format
public static string NormalizePath(string path);
}
```
## Error Handling
The API returns appropriate HTTP status codes and error messages:
- **400 Bad Request**: Invalid input parameters
- **401 Unauthorized**: User not authenticated
- **403 Forbidden**: User lacks permission
- **404 Not Found**: Resource not found
- **500 Internal Server Error**: Server-side error
**Error Response Format:**
```json
{
"code": "BROWSE_FAILED",
"message": "Failed to browse files",
"status": 500
}
```
## Security Considerations
1. **Ownership Verification**: All operations verify that the user owns the file indexes
2. **Path Validation**: Paths are normalized and validated
3. **Cascade Deletion**: File indexes are automatically removed when files are deleted
4. **Safe File Deletion**: Files are only deleted when no other indexes reference them
## Usage Examples
### Upload File to Specific Path
```bash
# Create upload task with path
curl -X POST /api/files/upload/create \
-H "Authorization: Bearer {token}" \
-H "Content-Type: application/json" \
-d '{
"fileName": "report.pdf",
"fileSize": 2048,
"contentType": "application/pdf",
"path": "/documents/reports/"
}'
```
### Browse Files
```bash
curl -X GET "/api/index/browse?path=/documents/reports/" \
-H "Authorization: Bearer {token}"
```
### Move File
```bash
curl -X POST "/api/index/move/{indexId}" \
-H "Authorization: Bearer {token}" \
-H "Content-Type: application/json" \
-d '{"newPath": "/archived/"}'
```
### Search Files
```bash
curl -X GET "/api/index/search?query=invoice&path=/documents/" \
-H "Authorization: Bearer {token}"
```
## Best Practices
1. **Use Trailing Slashes**: Always include trailing slashes in paths
2. **Organize Hierarchically**: Use meaningful folder structures
3. **Search Efficiently**: Use the search endpoint instead of client-side filtering
4. **Clean Up**: Use the clear-path endpoint for temporary directories
5. **Monitor Usage**: Check total file counts for quota management
## Integration Notes
- The file indexing system works alongside the existing file storage
- Files can exist in multiple paths (hard links)
- File deletion is optional and only removes data when safe
- The system maintains referential integrity between files and indexes

632
DysonNetwork.Drive/Migrations/20251112135535_AddFileIndex.Designer.cs generated Normal file
View File

@@ -0,0 +1,632 @@
// <auto-generated />
using System;
using System.Collections.Generic;
using DysonNetwork.Drive;
using DysonNetwork.Shared.Models;
using Microsoft.EntityFrameworkCore;
using Microsoft.EntityFrameworkCore.Infrastructure;
using Microsoft.EntityFrameworkCore.Migrations;
using Microsoft.EntityFrameworkCore.Storage.ValueConversion;
using NodaTime;
using Npgsql.EntityFrameworkCore.PostgreSQL.Metadata;
#nullable disable
namespace DysonNetwork.Drive.Migrations
{
[DbContext(typeof(AppDatabase))]
[Migration("20251112135535_AddFileIndex")]
partial class AddFileIndex
{
/// <inheritdoc />
protected override void BuildTargetModel(ModelBuilder modelBuilder)
{
#pragma warning disable 612, 618
modelBuilder
.HasAnnotation("ProductVersion", "9.0.10")
.HasAnnotation("Relational:MaxIdentifierLength", 63);
NpgsqlModelBuilderExtensions.UseIdentityByDefaultColumns(modelBuilder);
modelBuilder.Entity("DysonNetwork.Drive.Billing.QuotaRecord", b =>
{
b.Property<Guid>("Id")
.ValueGeneratedOnAdd()
.HasColumnType("uuid")
.HasColumnName("id");
b.Property<Guid>("AccountId")
.HasColumnType("uuid")
.HasColumnName("account_id");
b.Property<Instant>("CreatedAt")
.HasColumnType("timestamp with time zone")
.HasColumnName("created_at");
b.Property<Instant?>("DeletedAt")
.HasColumnType("timestamp with time zone")
.HasColumnName("deleted_at");
b.Property<string>("Description")
.IsRequired()
.HasColumnType("text")
.HasColumnName("description");
b.Property<Instant?>("ExpiredAt")
.HasColumnType("timestamp with time zone")
.HasColumnName("expired_at");
b.Property<string>("Name")
.IsRequired()
.HasColumnType("text")
.HasColumnName("name");
b.Property<long>("Quota")
.HasColumnType("bigint")
.HasColumnName("quota");
b.Property<Instant>("UpdatedAt")
.HasColumnType("timestamp with time zone")
.HasColumnName("updated_at");
b.HasKey("Id")
.HasName("pk_quota_records");
b.ToTable("quota_records", (string)null);
});
modelBuilder.Entity("DysonNetwork.Drive.Storage.Model.PersistentTask", b =>
{
b.Property<Guid>("Id")
.ValueGeneratedOnAdd()
.HasColumnType("uuid")
.HasColumnName("id");
b.Property<Guid>("AccountId")
.HasColumnType("uuid")
.HasColumnName("account_id");
b.Property<Instant?>("CompletedAt")
.HasColumnType("timestamp with time zone")
.HasColumnName("completed_at");
b.Property<Instant>("CreatedAt")
.HasColumnType("timestamp with time zone")
.HasColumnName("created_at");
b.Property<Instant?>("DeletedAt")
.HasColumnType("timestamp with time zone")
.HasColumnName("deleted_at");
b.Property<string>("Description")
.HasMaxLength(1024)
.HasColumnType("character varying(1024)")
.HasColumnName("description");
b.Property<string>("Discriminator")
.IsRequired()
.HasMaxLength(21)
.HasColumnType("character varying(21)")
.HasColumnName("discriminator");
b.Property<string>("ErrorMessage")
.HasMaxLength(1024)
.HasColumnType("character varying(1024)")
.HasColumnName("error_message");
b.Property<long?>("EstimatedDurationSeconds")
.HasColumnType("bigint")
.HasColumnName("estimated_duration_seconds");
b.Property<Instant?>("ExpiredAt")
.HasColumnType("timestamp with time zone")
.HasColumnName("expired_at");
b.Property<Instant>("LastActivity")
.HasColumnType("timestamp with time zone")
.HasColumnName("last_activity");
b.Property<string>("Name")
.IsRequired()
.HasMaxLength(256)
.HasColumnType("character varying(256)")
.HasColumnName("name");
b.Property<Dictionary<string, object>>("Parameters")
.IsRequired()
.HasColumnType("jsonb")
.HasColumnName("parameters");
b.Property<int>("Priority")
.HasColumnType("integer")
.HasColumnName("priority");
b.Property<double>("Progress")
.HasColumnType("double precision")
.HasColumnName("progress");
b.Property<Dictionary<string, object>>("Results")
.IsRequired()
.HasColumnType("jsonb")
.HasColumnName("results");
b.Property<Instant?>("StartedAt")
.HasColumnType("timestamp with time zone")
.HasColumnName("started_at");
b.Property<int>("Status")
.HasColumnType("integer")
.HasColumnName("status");
b.Property<string>("TaskId")
.IsRequired()
.HasMaxLength(64)
.HasColumnType("character varying(64)")
.HasColumnName("task_id");
b.Property<int>("Type")
.HasColumnType("integer")
.HasColumnName("type");
b.Property<Instant>("UpdatedAt")
.HasColumnType("timestamp with time zone")
.HasColumnName("updated_at");
b.HasKey("Id")
.HasName("pk_tasks");
b.ToTable("tasks", (string)null);
b.HasDiscriminator().HasValue("PersistentTask");
b.UseTphMappingStrategy();
});
modelBuilder.Entity("DysonNetwork.Shared.Models.CloudFileReference", b =>
{
b.Property<Guid>("Id")
.ValueGeneratedOnAdd()
.HasColumnType("uuid")
.HasColumnName("id");
b.Property<Instant>("CreatedAt")
.HasColumnType("timestamp with time zone")
.HasColumnName("created_at");
b.Property<Instant?>("DeletedAt")
.HasColumnType("timestamp with time zone")
.HasColumnName("deleted_at");
b.Property<Instant?>("ExpiredAt")
.HasColumnType("timestamp with time zone")
.HasColumnName("expired_at");
b.Property<string>("FileId")
.IsRequired()
.HasMaxLength(32)
.HasColumnType("character varying(32)")
.HasColumnName("file_id");
b.Property<string>("ResourceId")
.IsRequired()
.HasMaxLength(1024)
.HasColumnType("character varying(1024)")
.HasColumnName("resource_id");
b.Property<Instant>("UpdatedAt")
.HasColumnType("timestamp with time zone")
.HasColumnName("updated_at");
b.Property<string>("Usage")
.IsRequired()
.HasMaxLength(1024)
.HasColumnType("character varying(1024)")
.HasColumnName("usage");
b.HasKey("Id")
.HasName("pk_file_references");
b.HasIndex("FileId")
.HasDatabaseName("ix_file_references_file_id");
b.ToTable("file_references", (string)null);
});
modelBuilder.Entity("DysonNetwork.Shared.Models.FilePool", b =>
{
b.Property<Guid>("Id")
.ValueGeneratedOnAdd()
.HasColumnType("uuid")
.HasColumnName("id");
b.Property<Guid?>("AccountId")
.HasColumnType("uuid")
.HasColumnName("account_id");
b.Property<BillingConfig>("BillingConfig")
.IsRequired()
.HasColumnType("jsonb")
.HasColumnName("billing_config");
b.Property<Instant>("CreatedAt")
.HasColumnType("timestamp with time zone")
.HasColumnName("created_at");
b.Property<Instant?>("DeletedAt")
.HasColumnType("timestamp with time zone")
.HasColumnName("deleted_at");
b.Property<string>("Description")
.IsRequired()
.HasMaxLength(8192)
.HasColumnType("character varying(8192)")
.HasColumnName("description");
b.Property<bool>("IsHidden")
.HasColumnType("boolean")
.HasColumnName("is_hidden");
b.Property<string>("Name")
.IsRequired()
.HasMaxLength(1024)
.HasColumnType("character varying(1024)")
.HasColumnName("name");
b.Property<PolicyConfig>("PolicyConfig")
.IsRequired()
.HasColumnType("jsonb")
.HasColumnName("policy_config");
b.Property<RemoteStorageConfig>("StorageConfig")
.IsRequired()
.HasColumnType("jsonb")
.HasColumnName("storage_config");
b.Property<Instant>("UpdatedAt")
.HasColumnType("timestamp with time zone")
.HasColumnName("updated_at");
b.HasKey("Id")
.HasName("pk_pools");
b.ToTable("pools", (string)null);
});
modelBuilder.Entity("DysonNetwork.Shared.Models.SnCloudFile", b =>
{
b.Property<string>("Id")
.HasMaxLength(32)
.HasColumnType("character varying(32)")
.HasColumnName("id");
b.Property<Guid>("AccountId")
.HasColumnType("uuid")
.HasColumnName("account_id");
b.Property<Guid?>("BundleId")
.HasColumnType("uuid")
.HasColumnName("bundle_id");
b.Property<Instant>("CreatedAt")
.HasColumnType("timestamp with time zone")
.HasColumnName("created_at");
b.Property<Instant?>("DeletedAt")
.HasColumnType("timestamp with time zone")
.HasColumnName("deleted_at");
b.Property<string>("Description")
.HasMaxLength(4096)
.HasColumnType("character varying(4096)")
.HasColumnName("description");
b.Property<Instant?>("ExpiredAt")
.HasColumnType("timestamp with time zone")
.HasColumnName("expired_at");
b.Property<Dictionary<string, object>>("FileMeta")
.HasColumnType("jsonb")
.HasColumnName("file_meta");
b.Property<bool>("HasCompression")
.HasColumnType("boolean")
.HasColumnName("has_compression");
b.Property<bool>("HasThumbnail")
.HasColumnType("boolean")
.HasColumnName("has_thumbnail");
b.Property<string>("Hash")
.HasMaxLength(256)
.HasColumnType("character varying(256)")
.HasColumnName("hash");
b.Property<bool>("IsEncrypted")
.HasColumnType("boolean")
.HasColumnName("is_encrypted");
b.Property<bool>("IsMarkedRecycle")
.HasColumnType("boolean")
.HasColumnName("is_marked_recycle");
b.Property<string>("MimeType")
.HasMaxLength(256)
.HasColumnType("character varying(256)")
.HasColumnName("mime_type");
b.Property<string>("Name")
.IsRequired()
.HasMaxLength(1024)
.HasColumnType("character varying(1024)")
.HasColumnName("name");
b.Property<Guid?>("PoolId")
.HasColumnType("uuid")
.HasColumnName("pool_id");
b.Property<List<ContentSensitiveMark>>("SensitiveMarks")
.HasColumnType("jsonb")
.HasColumnName("sensitive_marks");
b.Property<long>("Size")
.HasColumnType("bigint")
.HasColumnName("size");
b.Property<string>("StorageId")
.HasMaxLength(32)
.HasColumnType("character varying(32)")
.HasColumnName("storage_id");
b.Property<string>("StorageUrl")
.HasMaxLength(4096)
.HasColumnType("character varying(4096)")
.HasColumnName("storage_url");
b.Property<Instant>("UpdatedAt")
.HasColumnType("timestamp with time zone")
.HasColumnName("updated_at");
b.Property<Instant?>("UploadedAt")
.HasColumnType("timestamp with time zone")
.HasColumnName("uploaded_at");
b.Property<Dictionary<string, object>>("UserMeta")
.HasColumnType("jsonb")
.HasColumnName("user_meta");
b.HasKey("Id")
.HasName("pk_files");
b.HasIndex("BundleId")
.HasDatabaseName("ix_files_bundle_id");
b.HasIndex("PoolId")
.HasDatabaseName("ix_files_pool_id");
b.ToTable("files", (string)null);
});
modelBuilder.Entity("DysonNetwork.Shared.Models.SnCloudFileIndex", b =>
{
b.Property<Guid>("Id")
.ValueGeneratedOnAdd()
.HasColumnType("uuid")
.HasColumnName("id");
b.Property<Guid>("AccountId")
.HasColumnType("uuid")
.HasColumnName("account_id");
b.Property<Instant>("CreatedAt")
.HasColumnType("timestamp with time zone")
.HasColumnName("created_at");
b.Property<Instant?>("DeletedAt")
.HasColumnType("timestamp with time zone")
.HasColumnName("deleted_at");
b.Property<string>("FileId")
.IsRequired()
.HasMaxLength(32)
.HasColumnType("character varying(32)")
.HasColumnName("file_id");
b.Property<string>("Path")
.IsRequired()
.HasMaxLength(8192)
.HasColumnType("character varying(8192)")
.HasColumnName("path");
b.Property<Instant>("UpdatedAt")
.HasColumnType("timestamp with time zone")
.HasColumnName("updated_at");
b.HasKey("Id")
.HasName("pk_file_indexes");
b.HasIndex("FileId")
.HasDatabaseName("ix_file_indexes_file_id");
b.HasIndex("Path", "AccountId")
.HasDatabaseName("ix_file_indexes_path_account_id");
b.ToTable("file_indexes", (string)null);
});
modelBuilder.Entity("DysonNetwork.Shared.Models.SnFileBundle", b =>
{
b.Property<Guid>("Id")
.ValueGeneratedOnAdd()
.HasColumnType("uuid")
.HasColumnName("id");
b.Property<Guid>("AccountId")
.HasColumnType("uuid")
.HasColumnName("account_id");
b.Property<Instant>("CreatedAt")
.HasColumnType("timestamp with time zone")
.HasColumnName("created_at");
b.Property<Instant?>("DeletedAt")
.HasColumnType("timestamp with time zone")
.HasColumnName("deleted_at");
b.Property<string>("Description")
.HasMaxLength(8192)
.HasColumnType("character varying(8192)")
.HasColumnName("description");
b.Property<Instant?>("ExpiredAt")
.HasColumnType("timestamp with time zone")
.HasColumnName("expired_at");
b.Property<string>("Name")
.IsRequired()
.HasMaxLength(1024)
.HasColumnType("character varying(1024)")
.HasColumnName("name");
b.Property<string>("Passcode")
.HasMaxLength(256)
.HasColumnType("character varying(256)")
.HasColumnName("passcode");
b.Property<string>("Slug")
.IsRequired()
.HasMaxLength(1024)
.HasColumnType("character varying(1024)")
.HasColumnName("slug");
b.Property<Instant>("UpdatedAt")
.HasColumnType("timestamp with time zone")
.HasColumnName("updated_at");
b.HasKey("Id")
.HasName("pk_bundles");
b.HasIndex("Slug")
.IsUnique()
.HasDatabaseName("ix_bundles_slug");
b.ToTable("bundles", (string)null);
});
modelBuilder.Entity("DysonNetwork.Drive.Storage.Model.PersistentUploadTask", b =>
{
b.HasBaseType("DysonNetwork.Drive.Storage.Model.PersistentTask");
b.Property<Guid?>("BundleId")
.HasColumnType("uuid")
.HasColumnName("bundle_id");
b.Property<long>("ChunkSize")
.HasColumnType("bigint")
.HasColumnName("chunk_size");
b.Property<int>("ChunksCount")
.HasColumnType("integer")
.HasColumnName("chunks_count");
b.Property<int>("ChunksUploaded")
.HasColumnType("integer")
.HasColumnName("chunks_uploaded");
b.Property<string>("ContentType")
.IsRequired()
.HasMaxLength(128)
.HasColumnType("character varying(128)")
.HasColumnName("content_type");
b.Property<string>("EncryptPassword")
.HasMaxLength(256)
.HasColumnType("character varying(256)")
.HasColumnName("encrypt_password");
b.Property<string>("FileName")
.IsRequired()
.HasMaxLength(256)
.HasColumnType("character varying(256)")
.HasColumnName("file_name");
b.Property<long>("FileSize")
.HasColumnType("bigint")
.HasColumnName("file_size");
b.Property<string>("Hash")
.IsRequired()
.HasColumnType("text")
.HasColumnName("hash");
b.Property<string>("Path")
.HasColumnType("text")
.HasColumnName("path");
b.Property<Guid>("PoolId")
.HasColumnType("uuid")
.HasColumnName("pool_id");
b.PrimitiveCollection<List<int>>("UploadedChunks")
.IsRequired()
.HasColumnType("integer[]")
.HasColumnName("uploaded_chunks");
b.HasDiscriminator().HasValue("PersistentUploadTask");
});
modelBuilder.Entity("DysonNetwork.Shared.Models.CloudFileReference", b =>
{
b.HasOne("DysonNetwork.Shared.Models.SnCloudFile", "File")
.WithMany("References")
.HasForeignKey("FileId")
.OnDelete(DeleteBehavior.Cascade)
.IsRequired()
.HasConstraintName("fk_file_references_files_file_id");
b.Navigation("File");
});
modelBuilder.Entity("DysonNetwork.Shared.Models.SnCloudFile", b =>
{
b.HasOne("DysonNetwork.Shared.Models.SnFileBundle", "Bundle")
.WithMany("Files")
.HasForeignKey("BundleId")
.HasConstraintName("fk_files_bundles_bundle_id");
b.HasOne("DysonNetwork.Shared.Models.FilePool", "Pool")
.WithMany()
.HasForeignKey("PoolId")
.HasConstraintName("fk_files_pools_pool_id");
b.Navigation("Bundle");
b.Navigation("Pool");
});
modelBuilder.Entity("DysonNetwork.Shared.Models.SnCloudFileIndex", b =>
{
b.HasOne("DysonNetwork.Shared.Models.SnCloudFile", "File")
.WithMany("FileIndexes")
.HasForeignKey("FileId")
.OnDelete(DeleteBehavior.Cascade)
.IsRequired()
.HasConstraintName("fk_file_indexes_files_file_id");
b.Navigation("File");
});
modelBuilder.Entity("DysonNetwork.Shared.Models.SnCloudFile", b =>
{
b.Navigation("FileIndexes");
b.Navigation("References");
});
modelBuilder.Entity("DysonNetwork.Shared.Models.SnFileBundle", b =>
{
b.Navigation("Files");
});
#pragma warning restore 612, 618
}
}
}

66
DysonNetwork.Drive/Migrations/20251112135535_AddFileIndex.cs Normal file
View File

@@ -0,0 +1,66 @@
using System;
using Microsoft.EntityFrameworkCore.Migrations;
using NodaTime;
#nullable disable
namespace DysonNetwork.Drive.Migrations
{
/// <inheritdoc />
public partial class AddFileIndex : Migration
{
/// <inheritdoc />
protected override void Up(MigrationBuilder migrationBuilder)
{
migrationBuilder.AddColumn<string>(
name: "path",
table: "tasks",
type: "text",
nullable: true);
migrationBuilder.CreateTable(
name: "file_indexes",
columns: table => new
{
id = table.Column<Guid>(type: "uuid", nullable: false),
path = table.Column<string>(type: "character varying(8192)", maxLength: 8192, nullable: false),
file_id = table.Column<string>(type: "character varying(32)", maxLength: 32, nullable: false),
account_id = table.Column<Guid>(type: "uuid", nullable: false),
created_at = table.Column<Instant>(type: "timestamp with time zone", nullable: false),
updated_at = table.Column<Instant>(type: "timestamp with time zone", nullable: false),
deleted_at = table.Column<Instant>(type: "timestamp with time zone", nullable: true)
},
constraints: table =>
{
table.PrimaryKey("pk_file_indexes", x => x.id);
table.ForeignKey(
name: "fk_file_indexes_files_file_id",
column: x => x.file_id,
principalTable: "files",
principalColumn: "id",
onDelete: ReferentialAction.Cascade);
});
migrationBuilder.CreateIndex(
name: "ix_file_indexes_file_id",
table: "file_indexes",
column: "file_id");
migrationBuilder.CreateIndex(
name: "ix_file_indexes_path_account_id",
table: "file_indexes",
columns: new[] { "path", "account_id" });
}
/// <inheritdoc />
protected override void Down(MigrationBuilder migrationBuilder)
{
migrationBuilder.DropTable(
name: "file_indexes");
migrationBuilder.DropColumn(
name: "path",
table: "tasks");
}
}
}

65
DysonNetwork.Drive/Migrations/AppDatabaseModelSnapshot.cs
View File

@@ -403,6 +403,53 @@ namespace DysonNetwork.Drive.Migrations
b.ToTable("files", (string)null);
});
modelBuilder.Entity("DysonNetwork.Shared.Models.SnCloudFileIndex", b =>
{
b.Property<Guid>("Id")
.ValueGeneratedOnAdd()
.HasColumnType("uuid")
.HasColumnName("id");
b.Property<Guid>("AccountId")
.HasColumnType("uuid")
.HasColumnName("account_id");
b.Property<Instant>("CreatedAt")
.HasColumnType("timestamp with time zone")
.HasColumnName("created_at");
b.Property<Instant?>("DeletedAt")
.HasColumnType("timestamp with time zone")
.HasColumnName("deleted_at");
b.Property<string>("FileId")
.IsRequired()
.HasMaxLength(32)
.HasColumnType("character varying(32)")
.HasColumnName("file_id");
b.Property<string>("Path")
.IsRequired()
.HasMaxLength(8192)
.HasColumnType("character varying(8192)")
.HasColumnName("path");
b.Property<Instant>("UpdatedAt")
.HasColumnType("timestamp with time zone")
.HasColumnName("updated_at");
b.HasKey("Id")
.HasName("pk_file_indexes");
b.HasIndex("FileId")
.HasDatabaseName("ix_file_indexes_file_id");
b.HasIndex("Path", "AccountId")
.HasDatabaseName("ix_file_indexes_path_account_id");
b.ToTable("file_indexes", (string)null);
});
modelBuilder.Entity("DysonNetwork.Shared.Models.SnFileBundle", b =>
{
b.Property<Guid>("Id")
@@ -508,6 +555,10 @@ namespace DysonNetwork.Drive.Migrations
.HasColumnType("text")
.HasColumnName("hash");
b.Property<string>("Path")
.HasColumnType("text")
.HasColumnName("path");
b.Property<Guid>("PoolId")
.HasColumnType("uuid")
.HasColumnName("pool_id");
@@ -549,8 +600,22 @@ namespace DysonNetwork.Drive.Migrations
b.Navigation("Pool");
});
modelBuilder.Entity("DysonNetwork.Shared.Models.SnCloudFileIndex", b =>
{
b.HasOne("DysonNetwork.Shared.Models.SnCloudFile", "File")
.WithMany("FileIndexes")
.HasForeignKey("FileId")
.OnDelete(DeleteBehavior.Cascade)
.IsRequired()
.HasConstraintName("fk_file_indexes_files_file_id");
b.Navigation("File");
});
modelBuilder.Entity("DysonNetwork.Shared.Models.SnCloudFile", b =>
{
b.Navigation("FileIndexes");
b.Navigation("References");
});

2
DysonNetwork.Drive/Startup/ServiceCollectionExtensions.cs
View File

@@ -1,5 +1,6 @@
using System.Text.Json;
using System.Text.Json.Serialization;
using DysonNetwork.Drive.Index;
using DysonNetwork.Shared.Cache;
using NodaTime;
using NodaTime.Serialization.SystemTextJson;
@@ -56,6 +57,7 @@ public static class ServiceCollectionExtensions
services.AddScoped<Storage.FileService>();
services.AddScoped<Storage.FileReferenceService>();
services.AddScoped<Storage.PersistentTaskService>();
services.AddScoped<FileIndexService>();
services.AddScoped<Billing.UsageService>();
services.AddScoped<Billing.QuotaService>();

3
DysonNetwork.Drive/Storage/CloudFileUnusedRecyclingJob.cs
View File

@@ -14,7 +14,7 @@ public class CloudFileUnusedRecyclingJob(
public async Task Execute(IJobExecutionContext context)
{
logger.LogInformation("Cleaning tus cloud files...");
var storePath = configuration["Tus:StorePath"];
var storePath = configuration["Storage:Uploads"];
if (Directory.Exists(storePath))
{
var oneHourAgo = SystemClock.Instance.GetCurrentInstant() - Duration.FromHours(1);
@@ -39,6 +39,7 @@ public class CloudFileUnusedRecyclingJob(
var processedCount = 0;
var markedCount = 0;
var totalFiles = await db.Files
.Where(f => f.FileIndexes.Count == 0)
.Where(f => f.PoolId.HasValue && recyclablePools.Contains(f.PoolId.Value))
.Where(f => !f.IsMarkedRecycle)
.CountAsync();

2
DysonNetwork.Drive/Storage/FileController.cs
View File

@@ -76,7 +76,7 @@ public class FileController(
}
// Fallback for tus uploads
var tusStorePath = configuration.GetValue<string>("Tus:StorePath");
var tusStorePath = configuration.GetValue<string>("Storage:Uploads");
if (!string.IsNullOrEmpty(tusStorePath))
{
var tusFilePath = Path.Combine(env.ContentRootPath, tusStorePath, file.Id);

22
DysonNetwork.Drive/Storage/FileUploadController.cs
View File

@@ -1,5 +1,6 @@
using System.ComponentModel.DataAnnotations;
using DysonNetwork.Drive.Billing;
using DysonNetwork.Drive.Index;
using DysonNetwork.Drive.Storage.Model;
using DysonNetwork.Shared.Auth;
using DysonNetwork.Shared.Http;
@@ -24,6 +25,7 @@ public class FileUploadController(
PermissionService.PermissionServiceClient permission,
QuotaService quotaService,
PersistentTaskService persistentTaskService,
FileIndexService fileIndexService,
ILogger<FileUploadController> logger
)
: ControllerBase
@@ -260,10 +262,26 @@ public class FileUploadController(
persistentTask.ExpiredAt
);
// Update task status to "processing" - background processing is now happening
// Create the file index if a path is provided
if (!string.IsNullOrEmpty(persistentTask.Path))
{
try
{
var accountId = Guid.Parse(currentUser.Id);
await fileIndexService.CreateAsync(persistentTask.Path, fileId, accountId);
logger.LogInformation("Created file index for file {FileId} at path {Path}", fileId, persistentTask.Path);
}
catch (Exception ex)
{
logger.LogWarning(ex, "Failed to create file index for file {FileId} at path {Path}", fileId, persistentTask.Path);
// Don't fail the upload if index creation fails, just log it
}
}
// Update the task status to "processing" - background processing is now happening
await persistentTaskService.UpdateTaskProgressAsync(taskId, 0.95, "Processing file in background...");
// Send upload completion notification (file is uploaded, but processing continues)
// Send upload completion notification (a file is uploaded, but processing continues)
await persistentTaskService.SendUploadCompletedNotificationAsync(persistentTask, fileId);
return Ok(cloudFile);

15
DysonNetwork.Drive/Storage/Model/FileUploadModels.cs
View File

@@ -22,6 +22,7 @@ public class FileUploadParameters
public string? EncryptPassword { get; set; }
public string Hash { get; set; } = string.Empty;
public List<int> UploadedChunks { get; set; } = [];
public string? Path { get; set; }
}
// File Move Task Parameters
@@ -93,6 +94,7 @@ public class CreateUploadTaskRequest
public string? EncryptPassword { get; set; }
public Instant? ExpiredAt { get; set; }
public long? ChunkSize { get; set; }
public string? Path { get; set; }
}
public class CreateUploadTaskResponse
@@ -301,6 +303,17 @@ public class PersistentUploadTask : PersistentTask
TypedParameters = parameters;
}
}
public string? Path
{
get => TypedParameters.Path;
set
{
var parameters = TypedParameters;
parameters.Path = value;
TypedParameters = parameters;
}
}
}
public enum TaskType
@@ -654,4 +667,4 @@ public enum UploadTaskStatus
Completed = TaskStatus.Completed,
Failed = TaskStatus.Failed,
Expired = TaskStatus.Expired
}
}

12
DysonNetwork.Drive/Storage/PersistentTaskService.cs
View File

@@ -615,7 +615,14 @@ public class PersistentTaskService(
var chunkSize = request.ChunkSize ?? 1024 * 1024 * 5; // 5MB default
var chunksCount = (int)Math.Ceiling((double)request.FileSize / chunkSize);
// Use the default pool if no pool is specified, or find first available pool
// If the second chunk is too small (less than 1MB), merge it with the first chunk
if (chunksCount == 2 && (request.FileSize - chunkSize) < 1024 * 1024)
{
chunksCount = 1;
chunkSize = request.FileSize;
}
// Use the default pool if no pool is specified, or find the first available pool
var poolId = request.PoolId ?? await GetFirstAvailablePoolIdAsync();
var uploadTask = new PersistentUploadTask
@@ -632,6 +639,7 @@ public class PersistentTaskService(
EncryptPassword = request.EncryptPassword,
ExpiredAt = request.ExpiredAt,
Hash = request.Hash,
Path = request.Path,
AccountId = accountId,
Status = TaskStatus.InProgress,
UploadedChunks = [],
@@ -1132,4 +1140,4 @@ public class RecentActivity
public double Progress { get; set; }
}
#endregion
#endregion

3
DysonNetwork.Drive/appsettings.json
View File

@@ -27,9 +27,6 @@
"PublicKeyPath": "Keys/PublicKey.pem",
"PrivateKeyPath": "Keys/PrivateKey.pem"
},
"Tus": {
"StorePath": "Uploads"
},
"Storage": {
"Uploads": "Uploads",
"PreferredRemote": "c53136a6-9152-4ecb-9f88-43c41438c23e",