Solar/Swarm
2
1
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

✨ File pool policy check

Browse Source
This commit is contained in:
LittleSheep
2025-07-26 19:46:38 +08:00
parent eaf0b366d3
commit b0683576b9
23 changed files with 255 additions and 1575 deletions
Download Patch File Download Diff File Expand all files Collapse all files

12
DysonNetwork.Drive/Storage/FileController.cs
View File

@@ -1,4 +1,5 @@
using DysonNetwork.Shared.Proto;
using Grpc.Core;
using Microsoft.AspNetCore.Authorization;
using Microsoft.AspNetCore.Mvc;
using Microsoft.EntityFrameworkCore;
@@ -45,7 +46,14 @@ public class FileController(
return PhysicalFile(filePath, file.MimeType ?? "application/octet-stream", file.Name);
}
var dest = await fs.GetRemoteStorageConfig(file.PoolId.Value);
var pool = await fs.GetPoolAsync(file.PoolId.Value);
if (pool is null) return StatusCode(StatusCodes.Status410Gone, "The pool of the file no longer exists or not accessible.");
var dest = pool.StorageConfig;
if (!pool.PolicyConfig.AllowAnonymous)
if(HttpContext.Items["CurrentUser"] is not Account currentUser) return Unauthorized();
// TODO: Provide ability to add access log
var fileName = string.IsNullOrWhiteSpace(file.StorageId) ? file.Id : file.StorageId;
if (!original && file.HasCompression)
@@ -115,7 +123,7 @@ public class FileController(
[HttpGet("{id}/info")]
public async Task<ActionResult<CloudFile>> GetFileInfo(string id)
{
var file = await db.Files.FindAsync(id);
var file = await fs.GetFileAsync(id);
if (file is null) return NotFound();
return file;

19
DysonNetwork.Drive/Storage/FilePool.cs
View File

@@ -24,19 +24,26 @@ public class BillingConfig
public double CostMultiplier { get; set; } = 1.0;
}
public class FilePool : ModelBase, IIdentifiedResource
public class PolicyConfig
{
public Guid Id { get; set; } = Guid.NewGuid();
[MaxLength(1024)] public string Name { get; set; } = string.Empty;
[Column(TypeName = "jsonb")] public RemoteStorageConfig StorageConfig { get; set; } = new();
[Column(TypeName = "jsonb")] public BillingConfig BillingConfig { get; set; } = new();
public bool PublicIndexable { get; set; } = false;
public bool PublicUsable { get; set; } = false;
public bool NoOptimization { get; set; } = false;
public bool NoMetadata { get; set; } = false;
public bool AllowEncryption { get; set; } = true;
public bool AllowAnonymous { get; set; } = true;
public int RequirePrivilege { get; set; } = 0;
public List<string>? AcceptTypes { get; set; }
public long? MaxFileSize { get; set; }
public int RequirePrivilege { get; set; } = 0;
}
public class FilePool : ModelBase, IIdentifiedResource
{
public Guid Id { get; set; } = Guid.NewGuid();
[MaxLength(1024)] public string Name { get; set; } = string.Empty;
[Column(TypeName = "jsonb")] public RemoteStorageConfig StorageConfig { get; set; } = new();
[Column(TypeName = "jsonb")] public BillingConfig BillingConfig { get; set; } = new();
[Column(TypeName = "jsonb")] public PolicyConfig PolicyConfig { get; set; } = new();
public Guid? AccountId { get; set; }

2
DysonNetwork.Drive/Storage/FilePoolController.cs
View File

@@ -17,7 +17,7 @@ public class FilePoolController(AppDatabase db) : ControllerBase
var accountId = Guid.Parse(currentUser.Id);
var pools = await db.Pools
.Where(p => p.PublicUsable || p.AccountId == accountId)
.Where(p => p.PolicyConfig.PublicUsable || p.AccountId == accountId)
.ToListAsync();
return Ok(pools);

10
DysonNetwork.Drive/Storage/FileService.cs
View File

@@ -45,6 +45,7 @@ public class FileService(
var file = await db.Files
.Where(f => f.Id == fileId)
.Include(f => f.Pool)
.FirstOrDefaultAsync();
if (file != null)
@@ -75,6 +76,7 @@ public class FileService(
{
var dbFiles = await db.Files
.Where(f => uncachedIds.Contains(f.Id))
.Include(f => f.Pool)
.ToListAsync();
// Add to cache
@@ -118,7 +120,7 @@ public class FileService(
if (!string.IsNullOrWhiteSpace(encryptPassword))
{
if (!pool.AllowEncryption) throw new InvalidOperationException("Encryption is not allowed in this pool");
if (!pool.PolicyConfig.AllowEncryption) throw new InvalidOperationException("Encryption is not allowed in this pool");
var encryptedPath = Path.Combine(Path.GetTempPath(), $"{fileId}.encrypted");
FileEncryptor.EncryptFile(ogFilePath, encryptedPath, encryptPassword);
File.Delete(ogFilePath); // Delete original unencrypted
@@ -136,7 +138,7 @@ public class FileService(
Size = fileSize,
Hash = hash,
AccountId = Guid.Parse(account.Id),
IsEncrypted = !string.IsNullOrWhiteSpace(encryptPassword) && pool.AllowEncryption
IsEncrypted = !string.IsNullOrWhiteSpace(encryptPassword) && pool.PolicyConfig.AllowEncryption
};
var existingFile = await db.Files.AsNoTracking().FirstOrDefaultAsync(f => f.Hash == hash);
@@ -160,7 +162,7 @@ public class FileService(
}
// Extract metadata on the current thread for a faster initial response
if (!pool.NoMetadata)
if (!pool.PolicyConfig.NoMetadata)
await ExtractMetadataAsync(file, ogFilePath, stream);
db.Files.Add(file);
@@ -302,7 +304,7 @@ public class FileService(
{
logger.LogInformation("Processing file {FileId} in background...", fileId);
if (!pool.NoOptimization)
if (!pool.PolicyConfig.NoOptimization)
switch (contentType.Split('/')[0])
{
case "image" when !AnimatedImageTypes.Contains(contentType):

160
DysonNetwork.Drive/Storage/TusService.cs
View File

@@ -1,12 +1,14 @@
using System.Net;
using System.Text;
using System.Text.Json;
using DysonNetwork.Shared.Auth;
using DysonNetwork.Shared.Proto;
using Microsoft.AspNetCore.Mvc;
using Microsoft.Extensions.Options;
using tusdotnet.Interfaces;
using tusdotnet.Models;
using tusdotnet.Models.Configuration;
using tusdotnet.Stores;
namespace DysonNetwork.Drive.Storage;
@@ -29,25 +31,63 @@ public abstract class TusService
}
var httpContext = eventContext.HttpContext;
if (httpContext.Items["CurrentUser"] is not Account user)
if (httpContext.Items["CurrentUser"] is not Account currentUser)
{
eventContext.FailRequest(HttpStatusCode.Unauthorized);
return;
}
if (!user.IsSuperuser)
if (eventContext.Intent != IntentType.CreateFile) return;
using var scope = httpContext.RequestServices.CreateScope();
if (!currentUser.IsSuperuser)
{
using var scope = httpContext.RequestServices.CreateScope();
var pm = scope.ServiceProvider.GetRequiredService<PermissionService.PermissionServiceClient>();
var allowed = await pm.HasPermissionAsync(new HasPermissionRequest
{ Actor = $"user:{user.Id}", Area = "global", Key = "files.create" });
{ Actor = $"user:{currentUser.Id}", Area = "global", Key = "files.create" });
if (!allowed.HasPermission)
eventContext.FailRequest(HttpStatusCode.Forbidden);
}
var filePool = httpContext.Request.Headers["X-FilePool"].FirstOrDefault();
if (!string.IsNullOrEmpty(filePool) && !Guid.TryParse(filePool, out _))
if (string.IsNullOrEmpty(filePool)) filePool = configuration["Storage:PreferredRemote"];
if (!Guid.TryParse(filePool, out _))
{
eventContext.FailRequest(HttpStatusCode.BadRequest, "Invalid file pool id");
return;
}
var fs = scope.ServiceProvider.GetRequiredService<FileService>();
var pool = await fs.GetPoolAsync(Guid.Parse(filePool!));
if (pool is null)
{
eventContext.FailRequest(HttpStatusCode.BadRequest, "Pool not found");
return;
}
if (pool.PolicyConfig.RequirePrivilege > 0)
{
if (currentUser.PerkSubscription is null)
{
eventContext.FailRequest(
HttpStatusCode.Forbidden,
$"You need to have join the Stellar Program tier {pool.PolicyConfig.RequirePrivilege} to use this pool"
);
return;
}
var privilege =
PerkSubscriptionPrivilege.GetPrivilegeFromIdentifier(currentUser.PerkSubscription.Identifier);
if (privilege < pool.PolicyConfig.RequirePrivilege)
{
eventContext.FailRequest(
HttpStatusCode.Forbidden,
$"You need to have join the Stellar Program tier {pool.PolicyConfig.RequirePrivilege} to use this pool"
);
return;
}
}
},
OnFileCompleteAsync = async eventContext =>
{
@@ -72,25 +112,101 @@ public abstract class TusService
if (string.IsNullOrEmpty(filePool))
filePool = configuration["Storage:PreferredRemote"];
var fileService = services.GetRequiredService<FileService>();
var info = await fileService.ProcessNewFileAsync(
user,
file.Id,
filePool,
fileStream,
fileName,
contentType,
encryptPassword
);
try
{
var fileService = services.GetRequiredService<FileService>();
var info = await fileService.ProcessNewFileAsync(
user,
file.Id,
filePool!,
fileStream,
fileName,
contentType,
encryptPassword
);
using var finalScope = eventContext.HttpContext.RequestServices.CreateScope();
var jsonOptions = finalScope.ServiceProvider.GetRequiredService<IOptions<JsonOptions>>().Value
.JsonSerializerOptions;
var infoJson = JsonSerializer.Serialize(info, jsonOptions);
eventContext.HttpContext.Response.Headers.Append("X-FileInfo", infoJson);
using var finalScope = eventContext.HttpContext.RequestServices.CreateScope();
var jsonOptions = finalScope.ServiceProvider.GetRequiredService<IOptions<JsonOptions>>().Value
.JsonSerializerOptions;
var infoJson = JsonSerializer.Serialize(info, jsonOptions);
eventContext.HttpContext.Response.Headers.Append("X-FileInfo", infoJson);
}
catch (Exception ex)
{
eventContext.HttpContext.Response.StatusCode = StatusCodes.Status400BadRequest;
await eventContext.HttpContext.Response.WriteAsync(ex.Message);
if (eventContext.Store is TusDiskStore disk)
await disk.DeleteFileAsync(file.Id, eventContext.CancellationToken);
}
finally
{
// Dispose the stream after all processing is complete
await fileStream.DisposeAsync();
}
},
OnBeforeCreateAsync = async eventContext =>
{
var filePool = eventContext.HttpContext.Request.Headers["X-FilePool"].FirstOrDefault();
if (string.IsNullOrEmpty(filePool)) filePool = configuration["Storage:PreferredRemote"];
if (!Guid.TryParse(filePool, out _))
{
eventContext.FailRequest(HttpStatusCode.BadRequest, "Invalid file pool id");
return;
}
// Dispose the stream after all processing is complete
await fileStream.DisposeAsync();
var metadata = eventContext.Metadata;
var contentType = metadata.TryGetValue("content-type", out var ct) ? ct.GetString(Encoding.UTF8) : null;
var scope = eventContext.HttpContext.RequestServices.CreateScope();
var rejected = false;
var fs = scope.ServiceProvider.GetRequiredService<FileService>();
var pool = await fs.GetPoolAsync(Guid.Parse(filePool!));
if (pool is null)
{
eventContext.FailRequest(HttpStatusCode.BadRequest, "Pool not found");
rejected = true;
}
var logger = scope.ServiceProvider.GetRequiredService<ILogger<TusService>>();
// Do the policy check
var policy = pool!.PolicyConfig;
if (!rejected && policy.AcceptTypes is not null)
{
if (contentType is null)
{
eventContext.FailRequest(
HttpStatusCode.BadRequest,
"Content type is required by the pool's policy"
);
rejected = true;
}
else if (!policy.AcceptTypes.Contains(contentType))
{
eventContext.FailRequest(
HttpStatusCode.Forbidden,
$"Content type {contentType} is not allowed by the pool's policy"
);
rejected = true;
}
}
if (!rejected && policy.MaxFileSize is not null)
{
if (eventContext.UploadLength > policy.MaxFileSize)
{
eventContext.FailRequest(
HttpStatusCode.Forbidden,
$"File size {eventContext.UploadLength} is larger than the pool's maximum file size {policy.MaxFileSize}"
);
rejected = true;
}
}
if (rejected)
logger.LogInformation("File rejected #{FileId}", eventContext.FileId);
},
OnCreateCompleteAsync = eventContext =>
{