SolarNetwork/Swarm
2
3
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

🐛 Fix some issues when creating duplicate indexes and instant upload triggered won't create index

Browse Source
This commit is contained in:
LittleSheep
2025-11-13 01:12:13 +08:00
parent e2b2bdd262
commit ffca94f789
2 changed files with 90 additions and 48 deletions
Download Patch File Download Diff File Expand all files Collapse all files

22
DysonNetwork.Drive/Index/FileIndexService.cs
View File

@@ -8,15 +8,25 @@ public class FileIndexService(AppDatabase db)
/// <summary> /// <summary>
/// Creates a new file index entry /// Creates a new file index entry
/// </summary> /// </summary>
/// <param name="path">The parent folder path with trailing slash</param> /// <param name="path">The parent folder path with a trailing slash</param>
/// <param name="fileId">The file ID</param> /// <param name="fileId">The file ID</param>
/// <param name="accountId">The account ID</param> /// <param name="accountId">The account ID</param>
/// <returns>The created file index</returns> /// <returns>The created file index</returns>
public async Task<SnCloudFileIndex> CreateAsync(string path, string fileId, Guid accountId) public async Task<SnCloudFileIndex> CreateAsync(string path, string fileId, Guid accountId)
{ {
// Ensure a path has trailing slash and is query-safe // Ensure a path has a trailing slash and is query-safe
var normalizedPath = NormalizePath(path); var normalizedPath = NormalizePath(path);
// Check if a file with the same name already exists in the same path for this account
var existingFileIndex = await db.FileIndexes
.FirstOrDefaultAsync(fi => fi.AccountId == accountId && fi.Path == normalizedPath && fi.FileId == fileId);
if (existingFileIndex != null)
{
throw new InvalidOperationException(
$"A file with ID '{fileId}' already exists in path '{normalizedPath}' for account '{accountId}'");
}
var fileIndex = new SnCloudFileIndex var fileIndex = new SnCloudFileIndex
{ {
Path = normalizedPath, Path = normalizedPath,
@@ -44,7 +54,7 @@ public class FileIndexService(AppDatabase db)
// Since properties are init-only, we need to remove the old index and create a new one // Since properties are init-only, we need to remove the old index and create a new one
db.FileIndexes.Remove(fileIndex); db.FileIndexes.Remove(fileIndex);
var newFileIndex = new SnCloudFileIndex var newFileIndex = new SnCloudFileIndex
{ {
Path = NormalizePath(newPath), Path = NormalizePath(newPath),
@@ -104,7 +114,7 @@ public class FileIndexService(AppDatabase db)
public async Task<int> RemoveByPathAsync(Guid accountId, string path) public async Task<int> RemoveByPathAsync(Guid accountId, string path)
{ {
var normalizedPath = NormalizePath(path); var normalizedPath = NormalizePath(path);
var indexes = await db.FileIndexes var indexes = await db.FileIndexes
.Where(fi => fi.AccountId == accountId && fi.Path == normalizedPath) .Where(fi => fi.AccountId == accountId && fi.Path == normalizedPath)
.ToListAsync(); .ToListAsync();
@@ -127,7 +137,7 @@ public class FileIndexService(AppDatabase db)
public async Task<List<SnCloudFileIndex>> GetByPathAsync(Guid accountId, string path) public async Task<List<SnCloudFileIndex>> GetByPathAsync(Guid accountId, string path)
{ {
var normalizedPath = NormalizePath(path); var normalizedPath = NormalizePath(path);
return await db.FileIndexes return await db.FileIndexes
.Where(fi => fi.AccountId == accountId && fi.Path == normalizedPath) .Where(fi => fi.AccountId == accountId && fi.Path == normalizedPath)
.Include(fi => fi.File) .Include(fi => fi.File)
@@ -184,4 +194,4 @@ public class FileIndexService(AppDatabase db)
return path; return path;
} }
} }

116
DysonNetwork.Drive/Storage/FileUploadController.cs
View File

@@ -61,10 +61,32 @@ public class FileUploadController(
EnsureTempDirectoryExists(); EnsureTempDirectoryExists();
var accountId = Guid.Parse(currentUser.Id);
// Check if a file with the same hash already exists // Check if a file with the same hash already exists
var existingFile = await db.Files.FirstOrDefaultAsync(f => f.Hash == request.Hash); var existingFile = await db.Files.FirstOrDefaultAsync(f => f.Hash == request.Hash);
if (existingFile != null) if (existingFile != null)
{ {
// Create the file index if a path is provided, even for existing files
if (string.IsNullOrEmpty(request.Path))
return Ok(new CreateUploadTaskResponse
{
FileExists = true,
File = existingFile
});
try
{
await fileIndexService.CreateAsync(request.Path, existingFile.Id, accountId);
logger.LogInformation("Created file index for existing file {FileId} at path {Path}",
existingFile.Id, request.Path);
}
catch (Exception ex)
{
logger.LogWarning(ex, "Failed to create file index for existing file {FileId} at path {Path}",
existingFile.Id, request.Path);
// Don't fail the request if index creation fails, just log it
}
return Ok(new CreateUploadTaskResponse return Ok(new CreateUploadTaskResponse
{ {
FileExists = true, FileExists = true,
@@ -72,7 +94,6 @@ public class FileUploadController(
}); });
} }
var accountId = Guid.Parse(currentUser.Id);
var taskId = await Nanoid.GenerateAsync(); var taskId = await Nanoid.GenerateAsync();
// Create persistent upload task // Create persistent upload task
@@ -92,18 +113,20 @@ public class FileUploadController(
if (currentUser.IsSuperuser) return null; if (currentUser.IsSuperuser) return null;
var allowed = await permission.HasPermissionAsync(new HasPermissionRequest var allowed = await permission.HasPermissionAsync(new HasPermissionRequest
{ Actor = $"user:{currentUser.Id}", Area = "global", Key = "files.create" }); { Actor = $"user:{currentUser.Id}", Area = "global", Key = "files.create" });
return allowed.HasPermission ? null : return allowed.HasPermission
new ObjectResult(ApiError.Unauthorized(forbidden: true)) { StatusCode = 403 }; ? null
: new ObjectResult(ApiError.Unauthorized(forbidden: true)) { StatusCode = 403 };
} }
private Task<IActionResult?> ValidatePoolAccess(Account currentUser, FilePool pool, CreateUploadTaskRequest request) private Task<IActionResult?> ValidatePoolAccess(Account currentUser, FilePool pool, CreateUploadTaskRequest request)
{ {
if (pool.PolicyConfig.RequirePrivilege <= 0) return Task.FromResult<IActionResult?>(null); if (pool.PolicyConfig.RequirePrivilege <= 0) return Task.FromResult<IActionResult?>(null);
var privilege = currentUser.PerkSubscription is null ? 0 : var privilege = currentUser.PerkSubscription is null
PerkSubscriptionPrivilege.GetPrivilegeFromIdentifier(currentUser.PerkSubscription.Identifier); ? 0
: PerkSubscriptionPrivilege.GetPrivilegeFromIdentifier(currentUser.PerkSubscription.Identifier);
if (privilege < pool.PolicyConfig.RequirePrivilege) if (privilege < pool.PolicyConfig.RequirePrivilege)
{ {
@@ -121,7 +144,7 @@ public class FileUploadController(
if (!policy.AllowEncryption && !string.IsNullOrEmpty(request.EncryptPassword)) if (!policy.AllowEncryption && !string.IsNullOrEmpty(request.EncryptPassword))
{ {
return new ObjectResult(ApiError.Unauthorized("File encryption is not allowed in this pool", true)) return new ObjectResult(ApiError.Unauthorized("File encryption is not allowed in this pool", true))
{ StatusCode = 403 }; { StatusCode = 403 };
} }
if (policy.AcceptTypes is { Count: > 0 }) if (policy.AcceptTypes is { Count: > 0 })
@@ -129,10 +152,10 @@ public class FileUploadController(
if (string.IsNullOrEmpty(request.ContentType)) if (string.IsNullOrEmpty(request.ContentType))
{ {
return new ObjectResult(ApiError.Validation(new Dictionary<string, string[]> return new ObjectResult(ApiError.Validation(new Dictionary<string, string[]>
{ {
{ "contentType", new[] { "Content type is required by the pool's policy" } } { "contentType", new[] { "Content type is required by the pool's policy" } }
})) }))
{ StatusCode = 400 }; { StatusCode = 400 };
} }
var foundMatch = policy.AcceptTypes.Any(acceptType => var foundMatch = policy.AcceptTypes.Any(acceptType =>
@@ -141,22 +164,23 @@ public class FileUploadController(
return acceptType.Equals(request.ContentType, StringComparison.OrdinalIgnoreCase); return acceptType.Equals(request.ContentType, StringComparison.OrdinalIgnoreCase);
var type = acceptType[..^2]; var type = acceptType[..^2];
return request.ContentType.StartsWith($"{type}/", StringComparison.OrdinalIgnoreCase); return request.ContentType.StartsWith($"{type}/", StringComparison.OrdinalIgnoreCase);
}); });
if (!foundMatch) if (!foundMatch)
{ {
return new ObjectResult( return new ObjectResult(
ApiError.Unauthorized($"Content type {request.ContentType} is not allowed by the pool's policy", true)) ApiError.Unauthorized($"Content type {request.ContentType} is not allowed by the pool's policy",
{ StatusCode = 403 }; true))
{ StatusCode = 403 };
} }
} }
if (policy.MaxFileSize is not null && request.FileSize > policy.MaxFileSize) if (policy.MaxFileSize is not null && request.FileSize > policy.MaxFileSize)
{ {
return new ObjectResult(ApiError.Unauthorized( return new ObjectResult(ApiError.Unauthorized(
$"File size {request.FileSize} is larger than the pool's maximum file size {policy.MaxFileSize}", true)) $"File size {request.FileSize} is larger than the pool's maximum file size {policy.MaxFileSize}",
{ StatusCode = 403 }; true))
{ StatusCode = 403 };
} }
return null; return null;
@@ -173,8 +197,9 @@ public class FileUploadController(
if (!ok) if (!ok)
{ {
return new ObjectResult( return new ObjectResult(
ApiError.Unauthorized($"File size {billableUnit} MiB is exceeded the user's quota {quota} MiB", true)) ApiError.Unauthorized($"File size {billableUnit} MiB is exceeded the user's quota {quota} MiB",
{ StatusCode = 403 }; true))
{ StatusCode = 403 };
} }
return null; return null;
@@ -190,8 +215,7 @@ public class FileUploadController(
public class UploadChunkRequest public class UploadChunkRequest
{ {
[Required] [Required] public IFormFile Chunk { get; set; } = null!;
public IFormFile Chunk { get; set; } = null!;
} }
[HttpPost("chunk/{taskId}/{chunkIndex:int}")] [HttpPost("chunk/{taskId}/{chunkIndex:int}")]
@@ -269,11 +293,13 @@ public class FileUploadController(
{ {
var accountId = Guid.Parse(currentUser.Id); var accountId = Guid.Parse(currentUser.Id);
await fileIndexService.CreateAsync(persistentTask.Path, fileId, accountId); await fileIndexService.CreateAsync(persistentTask.Path, fileId, accountId);
logger.LogInformation("Created file index for file {FileId} at path {Path}", fileId, persistentTask.Path); logger.LogInformation("Created file index for file {FileId} at path {Path}", fileId,
persistentTask.Path);
} }
catch (Exception ex) catch (Exception ex)
{ {
logger.LogWarning(ex, "Failed to create file index for file {FileId} at path {Path}", fileId, persistentTask.Path); logger.LogWarning(ex, "Failed to create file index for file {FileId} at path {Path}", fileId,
persistentTask.Path);
// Don't fail the upload if index creation fails, just log it // Don't fail the upload if index creation fails, just log it
} }
} }
@@ -289,7 +315,8 @@ public class FileUploadController(
catch (Exception ex) catch (Exception ex)
{ {
// Log the actual exception for debugging // Log the actual exception for debugging
logger.LogError(ex, "Failed to complete upload for task {TaskId}. Error: {ErrorMessage}", taskId, ex.Message); logger.LogError(ex, "Failed to complete upload for task {TaskId}. Error: {ErrorMessage}", taskId,
ex.Message);
// Mark task as failed // Mark task as failed
await persistentTaskService.MarkTaskFailedAsync(taskId); await persistentTaskService.MarkTaskFailedAsync(taskId);
@@ -314,9 +341,9 @@ public class FileUploadController(
} }
private static async Task MergeChunks( private static async Task MergeChunks(
string taskId, string taskId,
string taskPath, string taskPath,
string mergedFilePath, string mergedFilePath,
int chunksCount, int chunksCount,
PersistentTaskService persistentTaskService) PersistentTaskService persistentTaskService)
{ {
@@ -338,8 +365,8 @@ public class FileUploadController(
// Update progress after each chunk is merged // Update progress after each chunk is merged
var currentProgress = baseProgress + progressPerChunk * (i + 1); var currentProgress = baseProgress + progressPerChunk * (i + 1);
await persistentTaskService.UpdateTaskProgressAsync( await persistentTaskService.UpdateTaskProgressAsync(
taskId, taskId,
currentProgress, currentProgress,
"Merging chunks... (" + (i + 1) + "/" + chunksCount + ")" "Merging chunks... (" + (i + 1) + "/" + chunksCount + ")"
); );
} }
@@ -379,7 +406,8 @@ public class FileUploadController(
return new ObjectResult(ApiError.Unauthorized()) { StatusCode = 401 }; return new ObjectResult(ApiError.Unauthorized()) { StatusCode = 401 };
var accountId = Guid.Parse(currentUser.Id); var accountId = Guid.Parse(currentUser.Id);
var tasks = await persistentTaskService.GetUserUploadTasksAsync(accountId, status, sortBy, sortDescending, offset, limit); var tasks = await persistentTaskService.GetUserUploadTasksAsync(accountId, status, sortBy, sortDescending,
offset, limit);
Response.Headers.Append("X-Total", tasks.TotalCount.ToString()); Response.Headers.Append("X-Total", tasks.TotalCount.ToString());
@@ -595,18 +623,22 @@ public class FileUploadController(
task.Hash, task.Hash,
task.UploadedChunks task.UploadedChunks
}, },
Pool = pool != null ? new Pool = pool != null
{ ? new
pool.Id, {
pool.Name, pool.Id,
pool.Description pool.Name,
} : null, pool.Description
Bundle = bundle != null ? new }
{ : null,
bundle.Id, Bundle = bundle != null
bundle.Name, ? new
bundle.Description {
} : null, bundle.Id,
bundle.Name,
bundle.Description
}
: null,
EstimatedTimeRemaining = CalculateEstimatedTime(task), EstimatedTimeRemaining = CalculateEstimatedTime(task),
UploadSpeed = CalculateUploadSpeed(task) UploadSpeed = CalculateUploadSpeed(task)
}); });
@@ -652,4 +684,4 @@ public class FileUploadController(
_ => $"{bytesPerSecond / (1024 * 1024):F1} MB/s" _ => $"{bytesPerSecond / (1024 * 1024):F1} MB/s"
}; };
} }
} }