✨ File uploading

2025-04-13 13:50:30 +08:00
parent 31d98199e7
commit d22a15c42d
11 changed files with 918 additions and 3 deletions
--- a/DysonNetwork.Sphere/Storage/FileService.cs
+++ b/DysonNetwork.Sphere/Storage/FileService.cs
@ -0,0 +1,142 @@
+using System.Globalization;
+using FFMpegCore;
+using System.Security.Cryptography;
+using Microsoft.EntityFrameworkCore;
+using Minio;
+using Minio.DataModel.Args;
+using Minio.DataModel.Tags;
+using NodaTime;
+
+namespace DysonNetwork.Sphere.Storage;
+
+public class FileService(AppDatabase db, IConfiguration configuration)
+{
+    public async Task<CloudFile> AnalyzeFileAsync(
+        Account.Account account,
+        string fileId,
+        Stream stream,
+        string fileName,
+        string? contentType
+    )
+    {
+        var fileSize = stream.Length;
+        var hash = await HashFileAsync(stream, fileSize: fileSize);
+        contentType ??= !fileName.Contains('.') ? "application/octet-stream" : MimeTypes.GetMimeType(fileName);
+
+        var existingFile = await db.Files.Where(f => f.Hash == hash).FirstOrDefaultAsync();
+        if (existingFile is not null) return existingFile;
+
+        var file = new CloudFile
+        {
+            Id = fileId,
+            Name = fileName,
+            MimeType = contentType,
+            Size = fileSize,
+            Hash = hash,
+            Account = account,
+        };
+
+        switch (contentType.Split('/')[0])
+        {
+            case "video":
+            case "audio":
+                var mediaInfo = await FFProbe.AnalyseAsync(stream);
+                file.FileMeta = new Dictionary<string, object>
+                {
+                    ["duration"] = mediaInfo.Duration.TotalSeconds,
+                    ["format_name"] = mediaInfo.Format.FormatName,
+                    ["format_long_name"] = mediaInfo.Format.FormatLongName,
+                    ["start_time"] = mediaInfo.Format.StartTime.ToString(),
+                    ["bit_rate"] = mediaInfo.Format.BitRate.ToString(CultureInfo.InvariantCulture),
+                    ["tags"] = mediaInfo.Format.Tags ?? [],
+                    ["chapters"] = mediaInfo.Chapters,
+                };
+                break;
+        }
+
+        db.Files.Add(file);
+        await db.SaveChangesAsync();
+        return file;
+    }
+
+    private static async Task<string> HashFileAsync(Stream stream, int chunkSize = 1024 * 1024, long? fileSize = null)
+    {
+        fileSize ??= stream.Length;
+        if (fileSize > chunkSize * 1024 * 5)
+            return await HashFastApproximateAsync(stream, chunkSize);
+
+        using var md5 = MD5.Create();
+        var hashBytes = await md5.ComputeHashAsync(stream);
+        return Convert.ToHexString(hashBytes).ToLowerInvariant();
+    }
+
+    private static async Task<string> HashFastApproximateAsync(Stream stream, int chunkSize = 1024 * 1024)
+    {
+        // Scale the chunk size to kB level
+        chunkSize *= 1024;
+
+        using var md5 = MD5.Create();
+
+        var buffer = new byte[chunkSize * 2];
+        var fileLength = stream.Length;
+
+        var bytesRead = await stream.ReadAsync(buffer.AsMemory(0, chunkSize));
+
+        if (fileLength > chunkSize)
+        {
+            stream.Seek(-chunkSize, SeekOrigin.End);
+            bytesRead += await stream.ReadAsync(buffer.AsMemory(chunkSize, chunkSize));
+        }
+
+        var hash = md5.ComputeHash(buffer, 0, bytesRead);
+        return Convert.ToHexString(hash).ToLowerInvariant();
+    }
+
+    public async Task<CloudFile> UploadFileToRemoteAsync(CloudFile file, Stream stream, string? targetRemote)
+    {
+        if (file.UploadedAt.HasValue) return file;
+
+        file.UploadedTo = targetRemote ?? configuration.GetValue<string>("Storage:PreferredRemote")!;
+
+        var dest = GetRemoteStorageConfig(file.UploadedTo);
+        var client = CreateMinioClient(dest);
+        if (client is null)
+            throw new InvalidOperationException(
+                $"Failed to configure client for remote destination '{file.UploadedTo}'"
+            );
+
+        var bucket = dest.Bucket;
+        var contentType = file.MimeType ?? "application/octet-stream";
+
+        await client.PutObjectAsync(new PutObjectArgs()
+            .WithBucket(bucket)
+            .WithObject(file.Id)
+            .WithStreamData(stream)
+            .WithObjectSize(stream.Length)
+            .WithContentType(contentType)
+        );
+
+        file.UploadedAt = Instant.FromDateTimeUtc(DateTime.UtcNow);
+        await db.SaveChangesAsync();
+        return file;
+    }
+
+    private RemoteStorageConfig GetRemoteStorageConfig(string destination)
+    {
+        var destinations = configuration.GetSection("Storage:Remote").Get<List<RemoteStorageConfig>>()!;
+        var dest = destinations.FirstOrDefault(d => d.Id == destination);
+        if (dest is null) throw new InvalidOperationException($"Remote destination '{destination}' not found");
+        return dest;
+    }
+
+    private IMinioClient? CreateMinioClient(RemoteStorageConfig dest)
+    {
+        var client = new MinioClient()
+            .WithEndpoint(dest.Endpoint)
+            .WithRegion(dest.Region)
+            .WithCredentials(dest.SecretId, dest.SecretKey);
+        if (dest.EnableSsl) client = client.WithSSL();
+
+        return client.Build();
+    }
+}