Solar/Swarm
2
1
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

✨ File upload frontpage and download decryption

Browse Source
This commit is contained in:
LittleSheep
2025-07-26 03:11:42 +08:00
parent 0486c0d0e5
commit f1867e7916
19 changed files with 1051 additions and 229 deletions
Download Patch File Download Diff File Expand all files Collapse all files

4
DysonNetwork.Drive/Storage/CloudFile.cs
View File

@@ -33,8 +33,8 @@ public class CloudFile : ModelBase, ICloudFile, IIdentifiedResource
[MaxLength(1024)] public string Name { get; set; } = string.Empty;
[MaxLength(4096)] public string? Description { get; set; }
[Column(TypeName = "jsonb")] public Dictionary<string, object?> FileMeta { get; set; } = null!;
[Column(TypeName = "jsonb")] public Dictionary<string, object>? UserMeta { get; set; } = null!;
[Column(TypeName = "jsonb")] public Dictionary<string, object?>? FileMeta { get; set; }
[Column(TypeName = "jsonb")] public Dictionary<string, object?>? UserMeta { get; set; }
[Column(TypeName = "jsonb")] public List<ContentSensitiveMark>? SensitiveMarks { get; set; } = [];
[MaxLength(256)] public string? MimeType { get; set; }
[MaxLength(256)] public string? Hash { get; set; }

7
DysonNetwork.Drive/Storage/FilePool.cs
View File

@@ -30,6 +30,11 @@ public class FilePool : ModelBase, IIdentifiedResource
[MaxLength(1024)] public string Name { get; set; } = string.Empty;
[Column(TypeName = "jsonb")] public RemoteStorageConfig StorageConfig { get; set; } = new();
[Column(TypeName = "jsonb")] public BillingConfig BillingConfig { get; set; } = new();
public bool NoOptimization { get; set; } = false;
public bool NoMetadata { get; set; } = false;
public bool AllowEncryption { get; set; } = true;
public bool AllowAnonymous { get; set; } = true;
public int RequirePrivilege { get; set; } = 0;
public string ResourceIdentifier => $"file-pool/{Id}";
}

138
DysonNetwork.Drive/Storage/FileService.cs
View File

@@ -102,24 +102,30 @@ public class FileService(
public async Task<CloudFile> ProcessNewFileAsync(
Account account,
string fileId,
string filePool,
Stream stream,
string fileName,
string? contentType,
string? encryptPassword
)
{
var pool = await GetPoolAsync(Guid.Parse(filePool));
if (pool is null) throw new InvalidOperationException("Pool not found");
var ogFilePath = Path.GetFullPath(Path.Join(configuration.GetValue<string>("Tus:StorePath"), fileId));
var fileSize = stream.Length;
contentType ??= !fileName.Contains('.') ? "application/octet-stream" : MimeTypes.GetMimeType(fileName);
if (!string.IsNullOrWhiteSpace(encryptPassword))
{
if (!pool.AllowEncryption) throw new InvalidOperationException("Encryption is not allowed in this pool");
var encryptedPath = Path.Combine(Path.GetTempPath(), $"{fileId}.encrypted");
FileEncryptor.EncryptFile(ogFilePath, encryptedPath, encryptPassword);
File.Delete(ogFilePath); // Delete original unencrypted
File.Move(encryptedPath, ogFilePath); // Replace the original one with encrypted
contentType = "application/octet-stream";
}
var hash = await HashFileAsync(stream, fileSize: fileSize);
var file = new CloudFile
@@ -154,14 +160,15 @@ public class FileService(
}
// Extract metadata on the current thread for a faster initial response
await ExtractMetadataAsync(file, ogFilePath, stream);
if (!pool.NoMetadata)
await ExtractMetadataAsync(file, ogFilePath, stream);
db.Files.Add(file);
await db.SaveChangesAsync();
// Offload optimization (image conversion, thumbnailing) and uploading to a background task
_ = Task.Run(() =>
ProcessAndUploadInBackgroundAsync(file.Id, file.StorageId, contentType, ogFilePath, stream));
ProcessAndUploadInBackgroundAsync(file.Id, filePool, file.StorageId, contentType, ogFilePath, stream));
return file;
}
@@ -269,9 +276,18 @@ public class FileService(
/// <summary>
/// Handles file optimization (image compression, video thumbnailing) and uploads to remote storage in the background.
/// </summary>
private async Task ProcessAndUploadInBackgroundAsync(string fileId, string storageId, string contentType,
string originalFilePath, Stream stream)
private async Task ProcessAndUploadInBackgroundAsync(
string fileId,
string remoteId,
string storageId,
string contentType,
string originalFilePath,
Stream stream
)
{
var pool = await GetPoolAsync(Guid.Parse(remoteId));
if (pool is null) return;
await using var bgStream = stream; // Ensure stream is disposed at the end of this task
using var scope = scopeFactory.CreateScope();
var nfs = scope.ServiceProvider.GetRequiredService<FileService>();
@@ -286,74 +302,76 @@ public class FileService(
{
logger.LogInformation("Processing file {FileId} in background...", fileId);
switch (contentType.Split('/')[0])
{
case "image" when !AnimatedImageTypes.Contains(contentType):
newMimeType = "image/webp";
using (var vipsImage = Image.NewFromFile(originalFilePath))
{
var imageToWrite = vipsImage;
if (vipsImage.Interpretation is Enums.Interpretation.Scrgb or Enums.Interpretation.Xyz)
if (!pool.NoOptimization)
switch (contentType.Split('/')[0])
{
case "image" when !AnimatedImageTypes.Contains(contentType):
newMimeType = "image/webp";
using (var vipsImage = Image.NewFromFile(originalFilePath))
{
imageToWrite = vipsImage.Colourspace(Enums.Interpretation.Srgb);
var imageToWrite = vipsImage;
if (vipsImage.Interpretation is Enums.Interpretation.Scrgb or Enums.Interpretation.Xyz)
{
imageToWrite = vipsImage.Colourspace(Enums.Interpretation.Srgb);
}
var webpPath = Path.Join(Path.GetTempPath(), $"{TempFilePrefix}#{fileId}.webp");
imageToWrite.Autorot().WriteToFile(webpPath,
new VOption { { "lossless", true }, { "strip", true } });
uploads.Add((webpPath, string.Empty, newMimeType, true));
if (imageToWrite.Width * imageToWrite.Height >= 1024 * 1024)
{
var scale = 1024.0 / Math.Max(imageToWrite.Width, imageToWrite.Height);
var compressedPath =
Path.Join(Path.GetTempPath(), $"{TempFilePrefix}#{fileId}-compressed.webp");
using var compressedImage = imageToWrite.Resize(scale);
compressedImage.Autorot().WriteToFile(compressedPath,
new VOption { { "Q", 80 }, { "strip", true } });
uploads.Add((compressedPath, ".compressed", newMimeType, true));
hasCompression = true;
}
if (!ReferenceEquals(imageToWrite, vipsImage))
{
imageToWrite.Dispose(); // Clean up manually created colourspace-converted image
}
}
var webpPath = Path.Join(Path.GetTempPath(), $"{TempFilePrefix}#{fileId}.webp");
imageToWrite.Autorot().WriteToFile(webpPath,
new VOption { { "lossless", true }, { "strip", true } });
uploads.Add((webpPath, string.Empty, newMimeType, true));
break;
if (imageToWrite.Width * imageToWrite.Height >= 1024 * 1024)
case "video":
uploads.Add((originalFilePath, string.Empty, contentType, false));
var thumbnailPath = Path.Join(Path.GetTempPath(), $"{TempFilePrefix}#{fileId}.thumbnail.webp");
try
{
var scale = 1024.0 / Math.Max(imageToWrite.Width, imageToWrite.Height);
var compressedPath =
Path.Join(Path.GetTempPath(), $"{TempFilePrefix}#{fileId}-compressed.webp");
using var compressedImage = imageToWrite.Resize(scale);
compressedImage.Autorot().WriteToFile(compressedPath,
new VOption { { "Q", 80 }, { "strip", true } });
uploads.Add((compressedPath, ".compressed", newMimeType, true));
hasCompression = true;
var mediaInfo = await FFProbe.AnalyseAsync(originalFilePath);
var snapshotTime = mediaInfo.Duration > TimeSpan.FromSeconds(5)
? TimeSpan.FromSeconds(5)
: TimeSpan.FromSeconds(1);
await FFMpeg.SnapshotAsync(originalFilePath, thumbnailPath, captureTime: snapshotTime);
uploads.Add((thumbnailPath, ".thumbnail.webp", "image/webp", true));
hasThumbnail = true;
}
catch (Exception ex)
{
logger.LogError(ex, "Failed to generate thumbnail for video {FileId}", fileId);
}
if (!ReferenceEquals(imageToWrite, vipsImage))
{
imageToWrite.Dispose(); // Clean up manually created colourspace-converted image
}
}
break;
break;
case "video":
uploads.Add((originalFilePath, string.Empty, contentType, false));
var thumbnailPath = Path.Join(Path.GetTempPath(), $"{TempFilePrefix}#{fileId}.thumbnail.webp");
try
{
var mediaInfo = await FFProbe.AnalyseAsync(originalFilePath);
var snapshotTime = mediaInfo.Duration > TimeSpan.FromSeconds(5)
? TimeSpan.FromSeconds(5)
: TimeSpan.FromSeconds(1);
await FFMpeg.SnapshotAsync(originalFilePath, thumbnailPath, captureTime: snapshotTime);
uploads.Add((thumbnailPath, ".thumbnail.webp", "image/webp", true));
hasThumbnail = true;
}
catch (Exception ex)
{
logger.LogError(ex, "Failed to generate thumbnail for video {FileId}", fileId);
}
break;
default:
uploads.Add((originalFilePath, string.Empty, contentType, false));
break;
}
default:
uploads.Add((originalFilePath, string.Empty, contentType, false));
break;
}
else uploads.Add((originalFilePath, string.Empty, contentType, false));
logger.LogInformation("Optimized file {FileId}, now uploading...", fileId);
if (uploads.Count > 0)
{
var destPool = Guid.Parse(configuration.GetValue<string>("Storage:PreferredRemote")!);
var destPool = Guid.Parse(remoteId!);
var uploadTasks = uploads.Select(item =>
nfs.UploadFileToRemoteAsync(storageId, destPool, item.FilePath, item.Suffix, item.ContentType,
item.SelfDestruct)

11
DysonNetwork.Drive/Storage/TusService.cs
View File

@@ -44,6 +44,10 @@ public abstract class TusService
if (!allowed.HasPermission)
eventContext.FailRequest(HttpStatusCode.Forbidden);
}
var filePool = httpContext.Request.Headers["X-FilePool"].FirstOrDefault();
if (!string.IsNullOrEmpty(filePool) && !Guid.TryParse(filePool, out _))
eventContext.FailRequest(HttpStatusCode.BadRequest, "Invalid file pool id");
},
OnFileCompleteAsync = async eventContext =>
{
@@ -62,12 +66,17 @@ public abstract class TusService
var fileStream = await file.GetContentAsync(eventContext.CancellationToken);
var filePool = httpContext.Request.Headers["X-FilePool"].FirstOrDefault();
var encryptPassword = httpContext.Request.Headers["X-FilePass"].FirstOrDefault();
if (string.IsNullOrEmpty(filePool))
filePool = configuration["Storage:PreferredRemote"];
var fileService = services.GetRequiredService<FileService>();
var info = await fileService.ProcessNewFileAsync(
user,
file.Id,
filePool,
fileStream,
fileName,
contentType,
@@ -89,7 +98,7 @@ public abstract class TusService
if (gatewayUrl is not null)
eventContext.SetUploadUrl(new Uri(gatewayUrl + "/drive/tus/" + eventContext.FileId));
return Task.CompletedTask;
}
},
}
};
}