SolarNetwork/Swarm
2
3
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

♻️ Move file analyze, upload into message queue

Browse Source
This commit is contained in:
LittleSheep
2025-09-21 19:38:40 +08:00
parent b2a0d25ffa
commit f81e3dc9f4
6 changed files with 418 additions and 343 deletions
Download Patch File Download Diff File Expand all files Collapse all files

218
DysonNetwork.Drive/Storage/FileService.cs
View File

@@ -1,27 +1,28 @@
using System.Drawing;
using System.Globalization;
using FFMpegCore;
using System.Security.Cryptography;
using DysonNetwork.Drive.Storage.Model;
using DysonNetwork.Shared.Cache;
using DysonNetwork.Shared.Proto;
using Google.Protobuf.WellKnownTypes;
using Microsoft.EntityFrameworkCore;
using Minio;
using Minio.DataModel.Args;
using NATS.Client.Core;
using NetVips;
using NodaTime;
using tusdotnet.Stores;
using System.Linq.Expressions;
using DysonNetwork.Shared.Data;
using Microsoft.EntityFrameworkCore.Query;
using NATS.Net;
namespace DysonNetwork.Drive.Storage;
public class FileService(
AppDatabase db,
ILogger<FileService> logger,
IServiceScopeFactory scopeFactory,
ICacheService cache
ICacheService cache,
INatsConnection nats
)
{
private const string CacheKeyPrefix = "file:";
@@ -85,14 +86,6 @@ public class FileService(
.ToList();
}
private const string TempFilePrefix = "dyn-cloudfile";
private static readonly string[] AnimatedImageTypes =
["image/gif", "image/apng", "image/avif"];
private static readonly string[] AnimatedImageExtensions =
[".gif", ".apng", ".avif"];
public async Task<CloudFile> ProcessNewFileAsync(
Account account,
string fileId,
@@ -129,13 +122,14 @@ public class FileService(
if (bundle?.ExpiredAt != null)
expiredAt = bundle.ExpiredAt.Value;
var managedTempPath = Path.Combine(Path.GetTempPath(), fileId);
File.Copy(filePath, managedTempPath, true);
var fileInfo = new FileInfo(managedTempPath);
var fileSize = fileInfo.Length;
var finalContentType = contentType ?? (!fileName.Contains('.') ? "application/octet-stream" : MimeTypes.GetMimeType(fileName));
var finalContentType = contentType ??
(!fileName.Contains('.') ? "application/octet-stream" : MimeTypes.GetMimeType(fileName));
var file = new CloudFile
{
@@ -154,20 +148,20 @@ public class FileService(
}
string processingPath = managedTempPath;
bool isTempFile = true;
bool isTempFile = true;
if (!string.IsNullOrWhiteSpace(encryptPassword))
{
if (!pool.PolicyConfig.AllowEncryption)
throw new InvalidOperationException("Encryption is not allowed in this pool");
var encryptedPath = Path.Combine(Path.GetTempPath(), $"{fileId}.encrypted");
FileEncryptor.EncryptFile(managedTempPath, encryptedPath, encryptPassword);
File.Delete(managedTempPath);
processingPath = encryptedPath;
file.IsEncrypted = true;
file.MimeType = "application/octet-stream";
file.Size = new FileInfo(processingPath).Length;
@@ -180,8 +174,18 @@ public class FileService(
file.StorageId ??= file.Id;
_ = Task.Run(() =>
ProcessAndUploadInBackgroundAsync(file.Id, filePool, file.StorageId, file.MimeType, processingPath, isTempFile));
var js = nats.CreateJetStreamContext();
await js.PublishAsync(
FileUploadedEvent.Type,
GrpcTypeHelper.ConvertObjectToByteString(new FileUploadedEventPayload(
file.Id,
pool.Id,
file.StorageId,
file.MimeType,
processingPath,
isTempFile)
).ToByteArray()
);
return file;
}
@@ -296,170 +300,6 @@ public class FileService(
}
}
private async Task ProcessAndUploadInBackgroundAsync(
string fileId,
string remoteId,
string storageId,
string contentType,
string processingFilePath,
bool isTempFile
)
{
var pool = await GetPoolAsync(Guid.Parse(remoteId));
if (pool is null) return;
using var scope = scopeFactory.CreateScope();
var nfs = scope.ServiceProvider.GetRequiredService<FileService>();
var scopedDb = scope.ServiceProvider.GetRequiredService<AppDatabase>();
var uploads = new List<(string FilePath, string Suffix, string ContentType, bool SelfDestruct)>();
var newMimeType = contentType;
var hasCompression = false;
var hasThumbnail = false;
try
{
logger.LogInformation("Processing file {FileId} in background...", fileId);
var fileToUpdate = await scopedDb.Files.AsNoTracking().FirstAsync(f => f.Id == fileId);
if (fileToUpdate.IsEncrypted)
{
uploads.Add((processingFilePath, string.Empty, contentType, false));
}
else if (!pool.PolicyConfig.NoOptimization)
{
var fileExtension = Path.GetExtension(processingFilePath);
switch (contentType.Split('/')[0])
{
case "image":
if (AnimatedImageTypes.Contains(contentType) || AnimatedImageExtensions.Contains(fileExtension))
{
logger.LogInformation("Skip optimize file {FileId} due to it is animated...", fileId);
uploads.Add((processingFilePath, string.Empty, contentType, false));
break;
}
newMimeType = "image/webp";
using (var vipsImage = Image.NewFromFile(processingFilePath))
{
var imageToWrite = vipsImage;
if (vipsImage.Interpretation is Enums.Interpretation.Scrgb or Enums.Interpretation.Xyz)
{
imageToWrite = vipsImage.Colourspace(Enums.Interpretation.Srgb);
}
var webpPath = Path.Join(Path.GetTempPath(), $"{TempFilePrefix}#{fileId}.webp");
imageToWrite.Autorot().WriteToFile(webpPath,
new VOption { { "lossless", true }, { "strip", true } });
uploads.Add((webpPath, string.Empty, newMimeType, true));
if (imageToWrite.Width * imageToWrite.Height >= 1024 * 1024)
{
var scale = 1024.0 / Math.Max(imageToWrite.Width, imageToWrite.Height);
var compressedPath =
Path.Join(Path.GetTempPath(), $"{TempFilePrefix}#{fileId}-compressed.webp");
using var compressedImage = imageToWrite.Resize(scale);
compressedImage.Autorot().WriteToFile(compressedPath,
new VOption { { "Q", 80 }, { "strip", true } });
uploads.Add((compressedPath, ".compressed", newMimeType, true));
hasCompression = true;
}
if (!ReferenceEquals(imageToWrite, vipsImage))
{
imageToWrite.Dispose();
}
}
break;
case "video":
uploads.Add((processingFilePath, string.Empty, contentType, false));
var thumbnailPath = Path.Join(Path.GetTempPath(), $"{TempFilePrefix}#{fileId}.thumbnail.jpg");
try
{
await FFMpegArguments
.FromFileInput(processingFilePath, verifyExists: true)
.OutputToFile(thumbnailPath, overwrite: true, options => options
.Seek(TimeSpan.FromSeconds(0))
.WithFrameOutputCount(1)
.WithCustomArgument("-q:v 2")
)
.NotifyOnOutput(line => logger.LogInformation("[FFmpeg] {Line}", line))
.NotifyOnError(line => logger.LogWarning("[FFmpeg] {Line}", line))
.ProcessAsynchronously();
if (File.Exists(thumbnailPath))
{
uploads.Add((thumbnailPath, ".thumbnail", "image/jpeg", true));
hasThumbnail = true;
}
else
{
logger.LogWarning("FFMpeg did not produce thumbnail for video {FileId}", fileId);
}
}
catch (Exception ex)
{
logger.LogError(ex, "Failed to generate thumbnail for video {FileId}", fileId);
}
break;
default:
uploads.Add((processingFilePath, string.Empty, contentType, false));
break;
}
}
else uploads.Add((processingFilePath, string.Empty, contentType, false));
logger.LogInformation("Optimized file {FileId}, now uploading...", fileId);
if (uploads.Count > 0)
{
var destPool = Guid.Parse(remoteId!);
var uploadTasks = uploads.Select(item =>
nfs.UploadFileToRemoteAsync(
storageId,
destPool,
item.FilePath,
item.Suffix,
item.ContentType,
item.SelfDestruct
)
).ToList();
await Task.WhenAll(uploadTasks);
logger.LogInformation("Uploaded file {FileId} done!", fileId);
var now = SystemClock.Instance.GetCurrentInstant();
await scopedDb.Files.Where(f => f.Id == fileId).ExecuteUpdateAsync(setter => setter
.SetProperty(f => f.UploadedAt, now)
.SetProperty(f => f.PoolId, destPool)
.SetProperty(f => f.MimeType, newMimeType)
.SetProperty(f => f.HasCompression, hasCompression)
.SetProperty(f => f.HasThumbnail, hasThumbnail)
);
}
}
catch (Exception err)
{
logger.LogError(err, "Failed to process and upload {FileId}", fileId);
}
finally
{
if (isTempFile)
{
File.Delete(processingFilePath);
}
await nfs._PurgeCacheAsync(fileId);
}
}
private static async Task<string> HashFileAsync(string filePath, int chunkSize = 1024 * 1024)
{
var fileInfo = new FileInfo(filePath);
@@ -492,7 +332,7 @@ public class FileService(
return Convert.ToHexString(hash).ToLowerInvariant();
}
private async Task UploadFileToRemoteAsync(
public async Task UploadFileToRemoteAsync(
string storageId,
Guid targetRemote,
string filePath,
@@ -506,7 +346,7 @@ public class FileService(
if (selfDestruct) File.Delete(filePath);
}
private async Task UploadFileToRemoteAsync(
public async Task UploadFileToRemoteAsync(
string storageId,
Guid targetRemote,
Stream stream,
@@ -882,7 +722,7 @@ file class UpdatableCloudFile(CloudFile file)
.SetProperty(f => f.Name, Name)
.SetProperty(f => f.Description, Description)
.SetProperty(f => f.FileMeta, FileMeta)
.SetProperty(f => f.UserMeta, userMeta!)
.SetProperty(f => f.UserMeta, userMeta)
.SetProperty(f => f.IsMarkedRecycle, IsMarkedRecycle);
}
}
}