✨ Shared auth scheme
This commit is contained in:
131
DysonNetwork.Drive/Storage/CloudFile.cs
Normal file
131
DysonNetwork.Drive/Storage/CloudFile.cs
Normal file
@@ -0,0 +1,131 @@
|
||||
using System.ComponentModel.DataAnnotations;
|
||||
using System.ComponentModel.DataAnnotations.Schema;
|
||||
using System.Text.Json.Serialization;
|
||||
using DysonNetwork.Shared.Data;
|
||||
using DysonNetwork.Shared.Proto;
|
||||
using NodaTime;
|
||||
|
||||
namespace DysonNetwork.Drive.Storage;
|
||||
|
||||
public class RemoteStorageConfig
|
||||
{
|
||||
public string Id { get; set; } = string.Empty;
|
||||
public string Label { get; set; } = string.Empty;
|
||||
public string Region { get; set; } = string.Empty;
|
||||
public string Bucket { get; set; } = string.Empty;
|
||||
public string Endpoint { get; set; } = string.Empty;
|
||||
public string SecretId { get; set; } = string.Empty;
|
||||
public string SecretKey { get; set; } = string.Empty;
|
||||
public bool EnableSigned { get; set; }
|
||||
public bool EnableSsl { get; set; }
|
||||
public string? ImageProxy { get; set; }
|
||||
public string? AccessProxy { get; set; }
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// The class that used in jsonb columns which referenced the cloud file.
|
||||
/// The aim of this class is to store some properties that won't change to a file to reduce the database load.
|
||||
/// </summary>
|
||||
public class CloudFileReferenceObject : ModelBase, ICloudFile
|
||||
{
|
||||
public string Id { get; set; } = null!;
|
||||
public string Name { get; set; } = string.Empty;
|
||||
public Dictionary<string, object>? FileMeta { get; set; } = null!;
|
||||
public Dictionary<string, object>? UserMeta { get; set; } = null!;
|
||||
public string? MimeType { get; set; }
|
||||
public string? Hash { get; set; }
|
||||
public long Size { get; set; }
|
||||
public bool HasCompression { get; set; } = false;
|
||||
}
|
||||
|
||||
public class CloudFile : ModelBase, ICloudFile, IIdentifiedResource
|
||||
{
|
||||
/// The id generated by TuS, basically just UUID remove the dash lines
|
||||
[MaxLength(32)]
|
||||
public string Id { get; set; } = Guid.NewGuid().ToString();
|
||||
|
||||
[MaxLength(1024)] public string Name { get; set; } = string.Empty;
|
||||
[MaxLength(4096)] public string? Description { get; set; }
|
||||
[Column(TypeName = "jsonb")] public Dictionary<string, object>? FileMeta { get; set; } = null!;
|
||||
[Column(TypeName = "jsonb")] public Dictionary<string, object>? UserMeta { get; set; } = null!;
|
||||
[Column(TypeName = "jsonb")] public List<ContentSensitiveMark>? SensitiveMarks { get; set; } = [];
|
||||
[MaxLength(256)] public string? MimeType { get; set; }
|
||||
[MaxLength(256)] public string? Hash { get; set; }
|
||||
public long Size { get; set; }
|
||||
public Instant? UploadedAt { get; set; }
|
||||
[MaxLength(128)] public string? UploadedTo { get; set; }
|
||||
public bool HasCompression { get; set; } = false;
|
||||
|
||||
/// <summary>
|
||||
/// The field is set to true if the recycling job plans to delete the file.
|
||||
/// Due to the unstable of the recycling job, this doesn't really delete the file until a human verifies it.
|
||||
/// </summary>
|
||||
public bool IsMarkedRecycle { get; set; } = false;
|
||||
|
||||
/// The object name which stored remotely,
|
||||
/// multiple cloud file may have same storage id to indicate they are the same file
|
||||
///
|
||||
/// If the storage id was null and the uploaded at is not null, means it is an embedding file,
|
||||
/// The embedding file means the file is store on another site,
|
||||
/// or it is a webpage (based on mimetype)
|
||||
[MaxLength(32)]
|
||||
public string? StorageId { get; set; }
|
||||
|
||||
/// This field should be null when the storage id is filled
|
||||
/// Indicates the off-site accessible url of the file
|
||||
[MaxLength(4096)]
|
||||
public string? StorageUrl { get; set; }
|
||||
|
||||
public Guid AccountId { get; set; }
|
||||
|
||||
public CloudFileReferenceObject ToReferenceObject()
|
||||
{
|
||||
return new CloudFileReferenceObject
|
||||
{
|
||||
CreatedAt = CreatedAt,
|
||||
UpdatedAt = UpdatedAt,
|
||||
DeletedAt = DeletedAt,
|
||||
Id = Id,
|
||||
Name = Name,
|
||||
FileMeta = FileMeta,
|
||||
UserMeta = UserMeta,
|
||||
MimeType = MimeType,
|
||||
Hash = Hash,
|
||||
Size = Size,
|
||||
HasCompression = HasCompression
|
||||
};
|
||||
}
|
||||
|
||||
public string ResourceIdentifier => $"file/{Id}";
|
||||
}
|
||||
|
||||
public enum ContentSensitiveMark
|
||||
{
|
||||
Language,
|
||||
SexualContent,
|
||||
Violence,
|
||||
Profanity,
|
||||
HateSpeech,
|
||||
Racism,
|
||||
AdultContent,
|
||||
DrugAbuse,
|
||||
AlcoholAbuse,
|
||||
Gambling,
|
||||
SelfHarm,
|
||||
ChildAbuse,
|
||||
Other
|
||||
}
|
||||
|
||||
public class CloudFileReference : ModelBase
|
||||
{
|
||||
public Guid Id { get; set; } = Guid.NewGuid();
|
||||
[MaxLength(32)] public string FileId { get; set; } = null!;
|
||||
public CloudFile File { get; set; } = null!;
|
||||
[MaxLength(1024)] public string Usage { get; set; } = null!;
|
||||
[MaxLength(1024)] public string ResourceId { get; set; } = null!;
|
||||
|
||||
/// <summary>
|
||||
/// Optional expiration date for the file reference
|
||||
/// </summary>
|
||||
public Instant? ExpiredAt { get; set; }
|
||||
}
|
||||
93
DysonNetwork.Drive/Storage/CloudFileUnusedRecyclingJob.cs
Normal file
93
DysonNetwork.Drive/Storage/CloudFileUnusedRecyclingJob.cs
Normal file
@@ -0,0 +1,93 @@
|
||||
using Microsoft.EntityFrameworkCore;
|
||||
using NodaTime;
|
||||
using Quartz;
|
||||
|
||||
namespace DysonNetwork.Drive.Storage;
|
||||
|
||||
public class CloudFileUnusedRecyclingJob(
|
||||
AppDatabase db,
|
||||
FileReferenceService fileRefService,
|
||||
ILogger<CloudFileUnusedRecyclingJob> logger
|
||||
)
|
||||
: IJob
|
||||
{
|
||||
public async Task Execute(IJobExecutionContext context)
|
||||
{
|
||||
logger.LogInformation("Marking unused cloud files...");
|
||||
|
||||
var now = SystemClock.Instance.GetCurrentInstant();
|
||||
const int batchSize = 1000; // Process larger batches for efficiency
|
||||
var processedCount = 0;
|
||||
var markedCount = 0;
|
||||
var totalFiles = await db.Files.Where(f => !f.IsMarkedRecycle).CountAsync();
|
||||
|
||||
logger.LogInformation("Found {TotalFiles} files to check for unused status", totalFiles);
|
||||
|
||||
// Define a timestamp to limit the age of files we're processing in this run
|
||||
// This spreads the processing across multiple job runs for very large databases
|
||||
var ageThreshold = now - Duration.FromDays(30); // Process files up to 90 days old in this run
|
||||
|
||||
// Instead of loading all files at once, use pagination
|
||||
var hasMoreFiles = true;
|
||||
string? lastProcessedId = null;
|
||||
|
||||
while (hasMoreFiles)
|
||||
{
|
||||
// Query for the next batch of files using keyset pagination
|
||||
var filesQuery = db.Files
|
||||
.Where(f => !f.IsMarkedRecycle)
|
||||
.Where(f => f.CreatedAt <= ageThreshold); // Only process older files first
|
||||
|
||||
if (lastProcessedId != null)
|
||||
{
|
||||
filesQuery = filesQuery.Where(f => string.Compare(f.Id, lastProcessedId) > 0);
|
||||
}
|
||||
|
||||
var fileBatch = await filesQuery
|
||||
.OrderBy(f => f.Id) // Ensure consistent ordering for pagination
|
||||
.Take(batchSize)
|
||||
.Select(f => f.Id)
|
||||
.ToListAsync();
|
||||
|
||||
if (fileBatch.Count == 0)
|
||||
{
|
||||
hasMoreFiles = false;
|
||||
continue;
|
||||
}
|
||||
|
||||
processedCount += fileBatch.Count;
|
||||
lastProcessedId = fileBatch.Last();
|
||||
|
||||
// Get all relevant file references for this batch
|
||||
var fileReferences = await fileRefService.GetReferencesAsync(fileBatch);
|
||||
|
||||
// Filter to find files that have no references or all expired references
|
||||
var filesToMark = fileBatch.Where(fileId =>
|
||||
!fileReferences.TryGetValue(fileId, out var references) ||
|
||||
references.Count == 0 ||
|
||||
references.All(r => r.ExpiredAt.HasValue && r.ExpiredAt.Value <= now)
|
||||
).ToList();
|
||||
|
||||
if (filesToMark.Count > 0)
|
||||
{
|
||||
// Use a bulk update for better performance - mark all qualifying files at once
|
||||
var updateCount = await db.Files
|
||||
.Where(f => filesToMark.Contains(f.Id))
|
||||
.ExecuteUpdateAsync(setter => setter
|
||||
.SetProperty(f => f.IsMarkedRecycle, true));
|
||||
|
||||
markedCount += updateCount;
|
||||
}
|
||||
|
||||
// Log progress periodically
|
||||
if (processedCount % 10000 == 0 || !hasMoreFiles)
|
||||
{
|
||||
logger.LogInformation(
|
||||
"Progress: processed {ProcessedCount}/{TotalFiles} files, marked {MarkedCount} for recycling",
|
||||
processedCount, totalFiles, markedCount);
|
||||
}
|
||||
}
|
||||
|
||||
logger.LogInformation("Completed marking {MarkedCount} files for recycling", markedCount);
|
||||
}
|
||||
}
|
||||
144
DysonNetwork.Drive/Storage/FileController.cs
Normal file
144
DysonNetwork.Drive/Storage/FileController.cs
Normal file
@@ -0,0 +1,144 @@
|
||||
using DysonNetwork.Shared.Proto;
|
||||
using Microsoft.AspNetCore.Authorization;
|
||||
using Microsoft.AspNetCore.Mvc;
|
||||
using Microsoft.EntityFrameworkCore;
|
||||
using Minio.DataModel.Args;
|
||||
|
||||
namespace DysonNetwork.Drive.Storage;
|
||||
|
||||
[ApiController]
|
||||
[Route("/api/files")]
|
||||
public class FileController(
|
||||
AppDatabase db,
|
||||
FileService fs,
|
||||
IConfiguration configuration,
|
||||
IWebHostEnvironment env
|
||||
) : ControllerBase
|
||||
{
|
||||
[HttpGet("{id}")]
|
||||
public async Task<ActionResult> OpenFile(
|
||||
string id,
|
||||
[FromQuery] bool download = false,
|
||||
[FromQuery] bool original = false,
|
||||
[FromQuery] string? overrideMimeType = null
|
||||
)
|
||||
{
|
||||
// Support the file extension for client side data recognize
|
||||
string? fileExtension = null;
|
||||
if (id.Contains('.'))
|
||||
{
|
||||
var splitId = id.Split('.');
|
||||
id = splitId.First();
|
||||
fileExtension = splitId.Last();
|
||||
}
|
||||
|
||||
var file = await fs.GetFileAsync(id);
|
||||
if (file is null) return NotFound();
|
||||
|
||||
if (!string.IsNullOrWhiteSpace(file.StorageUrl)) return Redirect(file.StorageUrl);
|
||||
|
||||
if (file.UploadedTo is null)
|
||||
{
|
||||
var tusStorePath = configuration.GetValue<string>("Tus:StorePath")!;
|
||||
var filePath = Path.Combine(env.ContentRootPath, tusStorePath, file.Id);
|
||||
if (!System.IO.File.Exists(filePath)) return new NotFoundResult();
|
||||
return PhysicalFile(filePath, file.MimeType ?? "application/octet-stream", file.Name);
|
||||
}
|
||||
|
||||
var dest = fs.GetRemoteStorageConfig(file.UploadedTo);
|
||||
var fileName = string.IsNullOrWhiteSpace(file.StorageId) ? file.Id : file.StorageId;
|
||||
|
||||
if (!original && file.HasCompression)
|
||||
fileName += ".compressed";
|
||||
|
||||
if (dest.ImageProxy is not null && (file.MimeType?.StartsWith("image/") ?? false))
|
||||
{
|
||||
var proxyUrl = dest.ImageProxy;
|
||||
var baseUri = new Uri(proxyUrl.EndsWith('/') ? proxyUrl : $"{proxyUrl}/");
|
||||
var fullUri = new Uri(baseUri, fileName);
|
||||
return Redirect(fullUri.ToString());
|
||||
}
|
||||
|
||||
if (dest.AccessProxy is not null)
|
||||
{
|
||||
var proxyUrl = dest.AccessProxy;
|
||||
var baseUri = new Uri(proxyUrl.EndsWith('/') ? proxyUrl : $"{proxyUrl}/");
|
||||
var fullUri = new Uri(baseUri, fileName);
|
||||
return Redirect(fullUri.ToString());
|
||||
}
|
||||
|
||||
if (dest.EnableSigned)
|
||||
{
|
||||
var client = fs.CreateMinioClient(dest);
|
||||
if (client is null)
|
||||
return BadRequest(
|
||||
"Failed to configure client for remote destination, file got an invalid storage remote.");
|
||||
|
||||
var headers = new Dictionary<string, string>();
|
||||
if (fileExtension is not null)
|
||||
{
|
||||
if (MimeTypes.TryGetMimeType(fileExtension, out var mimeType))
|
||||
headers.Add("Response-Content-Type", mimeType);
|
||||
}
|
||||
else if (overrideMimeType is not null)
|
||||
{
|
||||
headers.Add("Response-Content-Type", overrideMimeType);
|
||||
}
|
||||
else if (file.MimeType is not null && !file.MimeType!.EndsWith("unknown"))
|
||||
{
|
||||
headers.Add("Response-Content-Type", file.MimeType);
|
||||
}
|
||||
|
||||
if (download)
|
||||
{
|
||||
headers.Add("Response-Content-Disposition", $"attachment; filename=\"{file.Name}\"");
|
||||
}
|
||||
|
||||
var bucket = dest.Bucket;
|
||||
var openUrl = await client.PresignedGetObjectAsync(
|
||||
new PresignedGetObjectArgs()
|
||||
.WithBucket(bucket)
|
||||
.WithObject(fileName)
|
||||
.WithExpiry(3600)
|
||||
.WithHeaders(headers)
|
||||
);
|
||||
|
||||
return Redirect(openUrl);
|
||||
}
|
||||
|
||||
// Fallback redirect to the S3 endpoint (public read)
|
||||
var protocol = dest.EnableSsl ? "https" : "http";
|
||||
// Use the path bucket lookup mode
|
||||
return Redirect($"{protocol}://{dest.Endpoint}/{dest.Bucket}/{fileName}");
|
||||
}
|
||||
|
||||
[HttpGet("{id}/info")]
|
||||
public async Task<ActionResult<CloudFile>> GetFileInfo(string id)
|
||||
{
|
||||
var file = await db.Files.FindAsync(id);
|
||||
if (file is null) return NotFound();
|
||||
|
||||
return file;
|
||||
}
|
||||
|
||||
[Authorize]
|
||||
[HttpDelete("{id}")]
|
||||
public async Task<ActionResult> DeleteFile(string id)
|
||||
{
|
||||
if (HttpContext.Items["CurrentUser"] is not Account currentUser) return Unauthorized();
|
||||
var userId = Guid.Parse(currentUser.Id);
|
||||
|
||||
var file = await db.Files
|
||||
.Where(e => e.Id == id)
|
||||
.Where(e => e.AccountId == userId)
|
||||
.FirstOrDefaultAsync();
|
||||
if (file is null) return NotFound();
|
||||
|
||||
await fs.DeleteFileAsync(file);
|
||||
|
||||
db.Files.Remove(file);
|
||||
await db.SaveChangesAsync();
|
||||
|
||||
return NoContent();
|
||||
}
|
||||
}
|
||||
66
DysonNetwork.Drive/Storage/FileExpirationJob.cs
Normal file
66
DysonNetwork.Drive/Storage/FileExpirationJob.cs
Normal file
@@ -0,0 +1,66 @@
|
||||
using Microsoft.EntityFrameworkCore;
|
||||
using NodaTime;
|
||||
using Quartz;
|
||||
|
||||
namespace DysonNetwork.Drive.Storage;
|
||||
|
||||
/// <summary>
|
||||
/// Job responsible for cleaning up expired file references
|
||||
/// </summary>
|
||||
public class FileExpirationJob(AppDatabase db, FileService fileService, ILogger<FileExpirationJob> logger) : IJob
|
||||
{
|
||||
public async Task Execute(IJobExecutionContext context)
|
||||
{
|
||||
var now = SystemClock.Instance.GetCurrentInstant();
|
||||
logger.LogInformation("Running file reference expiration job at {now}", now);
|
||||
|
||||
// Find all expired references
|
||||
var expiredReferences = await db.FileReferences
|
||||
.Where(r => r.ExpiredAt < now && r.ExpiredAt != null)
|
||||
.ToListAsync();
|
||||
|
||||
if (!expiredReferences.Any())
|
||||
{
|
||||
logger.LogInformation("No expired file references found");
|
||||
return;
|
||||
}
|
||||
|
||||
logger.LogInformation("Found {count} expired file references", expiredReferences.Count);
|
||||
|
||||
// Get unique file IDs
|
||||
var fileIds = expiredReferences.Select(r => r.FileId).Distinct().ToList();
|
||||
var filesAndReferenceCount = new Dictionary<string, int>();
|
||||
|
||||
// Delete expired references
|
||||
db.FileReferences.RemoveRange(expiredReferences);
|
||||
await db.SaveChangesAsync();
|
||||
|
||||
// Check remaining references for each file
|
||||
foreach (var fileId in fileIds)
|
||||
{
|
||||
var remainingReferences = await db.FileReferences
|
||||
.Where(r => r.FileId == fileId)
|
||||
.CountAsync();
|
||||
|
||||
filesAndReferenceCount[fileId] = remainingReferences;
|
||||
|
||||
// If no references remain, delete the file
|
||||
if (remainingReferences == 0)
|
||||
{
|
||||
var file = await db.Files.FirstOrDefaultAsync(f => f.Id == fileId);
|
||||
if (file != null)
|
||||
{
|
||||
logger.LogInformation("Deleting file {fileId} as all references have expired", fileId);
|
||||
await fileService.DeleteFileAsync(file);
|
||||
}
|
||||
}
|
||||
else
|
||||
{
|
||||
// Just purge the cache
|
||||
await fileService._PurgeCacheAsync(fileId);
|
||||
}
|
||||
}
|
||||
|
||||
logger.LogInformation("Completed file reference expiration job");
|
||||
}
|
||||
}
|
||||
434
DysonNetwork.Drive/Storage/FileReferenceService.cs
Normal file
434
DysonNetwork.Drive/Storage/FileReferenceService.cs
Normal file
@@ -0,0 +1,434 @@
|
||||
using DysonNetwork.Shared.Cache;
|
||||
using Microsoft.EntityFrameworkCore;
|
||||
using NodaTime;
|
||||
|
||||
namespace DysonNetwork.Drive.Storage;
|
||||
|
||||
public class FileReferenceService(AppDatabase db, FileService fileService, ICacheService cache)
|
||||
{
|
||||
private const string CacheKeyPrefix = "fileref:";
|
||||
private static readonly TimeSpan CacheDuration = TimeSpan.FromMinutes(15);
|
||||
|
||||
/// <summary>
|
||||
/// Creates a new reference to a file for a specific resource
|
||||
/// </summary>
|
||||
/// <param name="fileId">The ID of the file to reference</param>
|
||||
/// <param name="usage">The usage context (e.g., "avatar", "post-attachment")</param>
|
||||
/// <param name="resourceId">The ID of the resource using the file</param>
|
||||
/// <param name="expiredAt">Optional expiration time for the file</param>
|
||||
/// <param name="duration">Optional duration after which the file expires (alternative to expiredAt)</param>
|
||||
/// <returns>The created file reference</returns>
|
||||
public async Task<CloudFileReference> CreateReferenceAsync(
|
||||
string fileId,
|
||||
string usage,
|
||||
string resourceId,
|
||||
Instant? expiredAt = null,
|
||||
Duration? duration = null)
|
||||
{
|
||||
// Calculate expiration time if needed
|
||||
var finalExpiration = expiredAt;
|
||||
if (duration.HasValue)
|
||||
finalExpiration = SystemClock.Instance.GetCurrentInstant() + duration.Value;
|
||||
|
||||
var reference = new CloudFileReference
|
||||
{
|
||||
FileId = fileId,
|
||||
Usage = usage,
|
||||
ResourceId = resourceId,
|
||||
ExpiredAt = finalExpiration
|
||||
};
|
||||
|
||||
db.FileReferences.Add(reference);
|
||||
|
||||
await db.SaveChangesAsync();
|
||||
await fileService._PurgeCacheAsync(fileId);
|
||||
|
||||
return reference;
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Gets all references to a file
|
||||
/// </summary>
|
||||
/// <param name="fileId">The ID of the file</param>
|
||||
/// <returns>A list of all references to the file</returns>
|
||||
public async Task<List<CloudFileReference>> GetReferencesAsync(string fileId)
|
||||
{
|
||||
var cacheKey = $"{CacheKeyPrefix}list:{fileId}";
|
||||
|
||||
var cachedReferences = await cache.GetAsync<List<CloudFileReference>>(cacheKey);
|
||||
if (cachedReferences is not null)
|
||||
return cachedReferences;
|
||||
|
||||
var references = await db.FileReferences
|
||||
.Where(r => r.FileId == fileId)
|
||||
.ToListAsync();
|
||||
|
||||
await cache.SetAsync(cacheKey, references, CacheDuration);
|
||||
|
||||
return references;
|
||||
}
|
||||
|
||||
public async Task<Dictionary<string, List<CloudFileReference>>> GetReferencesAsync(IEnumerable<string> fileId)
|
||||
{
|
||||
var references = await db.FileReferences
|
||||
.Where(r => fileId.Contains(r.FileId))
|
||||
.GroupBy(r => r.FileId)
|
||||
.ToDictionaryAsync(r => r.Key, r => r.ToList());
|
||||
return references;
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Gets the number of references to a file
|
||||
/// </summary>
|
||||
/// <param name="fileId">The ID of the file</param>
|
||||
/// <returns>The number of references to the file</returns>
|
||||
public async Task<int> GetReferenceCountAsync(string fileId)
|
||||
{
|
||||
var cacheKey = $"{CacheKeyPrefix}count:{fileId}";
|
||||
|
||||
var cachedCount = await cache.GetAsync<int?>(cacheKey);
|
||||
if (cachedCount.HasValue)
|
||||
return cachedCount.Value;
|
||||
|
||||
var count = await db.FileReferences
|
||||
.Where(r => r.FileId == fileId)
|
||||
.CountAsync();
|
||||
|
||||
await cache.SetAsync(cacheKey, count, CacheDuration);
|
||||
|
||||
return count;
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Gets all references for a specific resource
|
||||
/// </summary>
|
||||
/// <param name="resourceId">The ID of the resource</param>
|
||||
/// <returns>A list of file references associated with the resource</returns>
|
||||
public async Task<List<CloudFileReference>> GetResourceReferencesAsync(string resourceId)
|
||||
{
|
||||
var cacheKey = $"{CacheKeyPrefix}resource:{resourceId}";
|
||||
|
||||
var cachedReferences = await cache.GetAsync<List<CloudFileReference>>(cacheKey);
|
||||
if (cachedReferences is not null)
|
||||
return cachedReferences;
|
||||
|
||||
var references = await db.FileReferences
|
||||
.Where(r => r.ResourceId == resourceId)
|
||||
.ToListAsync();
|
||||
|
||||
await cache.SetAsync(cacheKey, references, CacheDuration);
|
||||
|
||||
return references;
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Gets all file references for a specific usage context
|
||||
/// </summary>
|
||||
/// <param name="usage">The usage context</param>
|
||||
/// <returns>A list of file references with the specified usage</returns>
|
||||
public async Task<List<CloudFileReference>> GetUsageReferencesAsync(string usage)
|
||||
{
|
||||
return await db.FileReferences
|
||||
.Where(r => r.Usage == usage)
|
||||
.ToListAsync();
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Deletes references for a specific resource
|
||||
/// </summary>
|
||||
/// <param name="resourceId">The ID of the resource</param>
|
||||
/// <returns>The number of deleted references</returns>
|
||||
public async Task<int> DeleteResourceReferencesAsync(string resourceId)
|
||||
{
|
||||
var references = await db.FileReferences
|
||||
.Where(r => r.ResourceId == resourceId)
|
||||
.ToListAsync();
|
||||
|
||||
var fileIds = references.Select(r => r.FileId).Distinct().ToList();
|
||||
|
||||
db.FileReferences.RemoveRange(references);
|
||||
var deletedCount = await db.SaveChangesAsync();
|
||||
|
||||
// Purge caches
|
||||
var tasks = fileIds.Select(fileService._PurgeCacheAsync).ToList();
|
||||
tasks.Add(PurgeCacheForResourceAsync(resourceId));
|
||||
await Task.WhenAll(tasks);
|
||||
|
||||
return deletedCount;
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Deletes references for a specific resource and usage
|
||||
/// </summary>
|
||||
/// <param name="resourceId">The ID of the resource</param>
|
||||
/// <param name="usage">The usage context</param>
|
||||
/// <returns>The number of deleted references</returns>
|
||||
public async Task<int> DeleteResourceReferencesAsync(string resourceId, string usage)
|
||||
{
|
||||
var references = await db.FileReferences
|
||||
.Where(r => r.ResourceId == resourceId && r.Usage == usage)
|
||||
.ToListAsync();
|
||||
|
||||
if (!references.Any())
|
||||
{
|
||||
return 0;
|
||||
}
|
||||
|
||||
var fileIds = references.Select(r => r.FileId).Distinct().ToList();
|
||||
|
||||
db.FileReferences.RemoveRange(references);
|
||||
var deletedCount = await db.SaveChangesAsync();
|
||||
|
||||
// Purge caches
|
||||
var tasks = fileIds.Select(fileService._PurgeCacheAsync).ToList();
|
||||
tasks.Add(PurgeCacheForResourceAsync(resourceId));
|
||||
await Task.WhenAll(tasks);
|
||||
|
||||
return deletedCount;
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Deletes a specific file reference
|
||||
/// </summary>
|
||||
/// <param name="referenceId">The ID of the reference to delete</param>
|
||||
/// <returns>True if the reference was deleted, false otherwise</returns>
|
||||
public async Task<bool> DeleteReferenceAsync(Guid referenceId)
|
||||
{
|
||||
var reference = await db.FileReferences
|
||||
.FirstOrDefaultAsync(r => r.Id == referenceId);
|
||||
|
||||
if (reference == null)
|
||||
return false;
|
||||
|
||||
db.FileReferences.Remove(reference);
|
||||
await db.SaveChangesAsync();
|
||||
|
||||
// Purge caches
|
||||
await fileService._PurgeCacheAsync(reference.FileId);
|
||||
await PurgeCacheForResourceAsync(reference.ResourceId);
|
||||
await PurgeCacheForFileAsync(reference.FileId);
|
||||
|
||||
return true;
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Updates the files referenced by a resource
|
||||
/// </summary>
|
||||
/// <param name="resourceId">The ID of the resource</param>
|
||||
/// <param name="newFileIds">The new list of file IDs</param>
|
||||
/// <param name="usage">The usage context</param>
|
||||
/// <param name="expiredAt">Optional expiration time for newly added files</param>
|
||||
/// <param name="duration">Optional duration after which newly added files expire</param>
|
||||
/// <returns>A list of the updated file references</returns>
|
||||
public async Task<List<CloudFileReference>> UpdateResourceFilesAsync(
|
||||
string resourceId,
|
||||
IEnumerable<string>? newFileIds,
|
||||
string usage,
|
||||
Instant? expiredAt = null,
|
||||
Duration? duration = null)
|
||||
{
|
||||
if (newFileIds == null)
|
||||
return new List<CloudFileReference>();
|
||||
|
||||
var existingReferences = await db.FileReferences
|
||||
.Where(r => r.ResourceId == resourceId && r.Usage == usage)
|
||||
.ToListAsync();
|
||||
|
||||
var existingFileIds = existingReferences.Select(r => r.FileId).ToHashSet();
|
||||
var newFileIdsList = newFileIds.ToList();
|
||||
var newFileIdsSet = newFileIdsList.ToHashSet();
|
||||
|
||||
// Files to remove
|
||||
var toRemove = existingReferences
|
||||
.Where(r => !newFileIdsSet.Contains(r.FileId))
|
||||
.ToList();
|
||||
|
||||
// Files to add
|
||||
var toAdd = newFileIdsList
|
||||
.Where(id => !existingFileIds.Contains(id))
|
||||
.Select(id => new CloudFileReference
|
||||
{
|
||||
FileId = id,
|
||||
Usage = usage,
|
||||
ResourceId = resourceId
|
||||
})
|
||||
.ToList();
|
||||
|
||||
// Apply changes
|
||||
if (toRemove.Any())
|
||||
db.FileReferences.RemoveRange(toRemove);
|
||||
|
||||
if (toAdd.Any())
|
||||
db.FileReferences.AddRange(toAdd);
|
||||
|
||||
await db.SaveChangesAsync();
|
||||
|
||||
// Update expiration for newly added references if specified
|
||||
if ((expiredAt.HasValue || duration.HasValue) && toAdd.Any())
|
||||
{
|
||||
var finalExpiration = expiredAt;
|
||||
if (duration.HasValue)
|
||||
{
|
||||
finalExpiration = SystemClock.Instance.GetCurrentInstant() + duration.Value;
|
||||
}
|
||||
|
||||
// Update newly added references with the expiration time
|
||||
var referenceIds = await db.FileReferences
|
||||
.Where(r => toAdd.Select(a => a.FileId).Contains(r.FileId) &&
|
||||
r.ResourceId == resourceId &&
|
||||
r.Usage == usage)
|
||||
.Select(r => r.Id)
|
||||
.ToListAsync();
|
||||
|
||||
await db.FileReferences
|
||||
.Where(r => referenceIds.Contains(r.Id))
|
||||
.ExecuteUpdateAsync(setter => setter.SetProperty(
|
||||
r => r.ExpiredAt,
|
||||
_ => finalExpiration
|
||||
));
|
||||
}
|
||||
|
||||
// Purge caches
|
||||
var allFileIds = existingFileIds.Union(newFileIdsSet).ToList();
|
||||
var tasks = allFileIds.Select(fileService._PurgeCacheAsync).ToList();
|
||||
tasks.Add(PurgeCacheForResourceAsync(resourceId));
|
||||
await Task.WhenAll(tasks);
|
||||
|
||||
// Return updated references
|
||||
return await db.FileReferences
|
||||
.Where(r => r.ResourceId == resourceId && r.Usage == usage)
|
||||
.ToListAsync();
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Gets all files referenced by a resource
|
||||
/// </summary>
|
||||
/// <param name="resourceId">The ID of the resource</param>
|
||||
/// <param name="usage">Optional filter by usage context</param>
|
||||
/// <returns>A list of files referenced by the resource</returns>
|
||||
public async Task<List<CloudFile>> GetResourceFilesAsync(string resourceId, string? usage = null)
|
||||
{
|
||||
var query = db.FileReferences.Where(r => r.ResourceId == resourceId);
|
||||
|
||||
if (usage != null)
|
||||
query = query.Where(r => r.Usage == usage);
|
||||
|
||||
var references = await query.ToListAsync();
|
||||
var fileIds = references.Select(r => r.FileId).ToList();
|
||||
|
||||
return await db.Files
|
||||
.Where(f => fileIds.Contains(f.Id))
|
||||
.ToListAsync();
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Purges all caches related to a resource
|
||||
/// </summary>
|
||||
private async Task PurgeCacheForResourceAsync(string resourceId)
|
||||
{
|
||||
var cacheKey = $"{CacheKeyPrefix}resource:{resourceId}";
|
||||
await cache.RemoveAsync(cacheKey);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Purges all caches related to a file
|
||||
/// </summary>
|
||||
private async Task PurgeCacheForFileAsync(string fileId)
|
||||
{
|
||||
var cacheKeys = new[]
|
||||
{
|
||||
$"{CacheKeyPrefix}list:{fileId}",
|
||||
$"{CacheKeyPrefix}count:{fileId}"
|
||||
};
|
||||
|
||||
var tasks = cacheKeys.Select(cache.RemoveAsync);
|
||||
await Task.WhenAll(tasks);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Updates the expiration time for a file reference
|
||||
/// </summary>
|
||||
/// <param name="referenceId">The ID of the reference</param>
|
||||
/// <param name="expiredAt">The new expiration time, or null to remove expiration</param>
|
||||
/// <returns>True if the reference was found and updated, false otherwise</returns>
|
||||
public async Task<bool> SetReferenceExpirationAsync(Guid referenceId, Instant? expiredAt)
|
||||
{
|
||||
var reference = await db.FileReferences
|
||||
.FirstOrDefaultAsync(r => r.Id == referenceId);
|
||||
|
||||
if (reference == null)
|
||||
return false;
|
||||
|
||||
reference.ExpiredAt = expiredAt;
|
||||
await db.SaveChangesAsync();
|
||||
|
||||
await PurgeCacheForFileAsync(reference.FileId);
|
||||
await PurgeCacheForResourceAsync(reference.ResourceId);
|
||||
|
||||
return true;
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Updates the expiration time for all references to a file
|
||||
/// </summary>
|
||||
/// <param name="fileId">The ID of the file</param>
|
||||
/// <param name="expiredAt">The new expiration time, or null to remove expiration</param>
|
||||
/// <returns>The number of references updated</returns>
|
||||
public async Task<int> SetFileReferencesExpirationAsync(string fileId, Instant? expiredAt)
|
||||
{
|
||||
var rowsAffected = await db.FileReferences
|
||||
.Where(r => r.FileId == fileId)
|
||||
.ExecuteUpdateAsync(setter => setter.SetProperty(
|
||||
r => r.ExpiredAt,
|
||||
_ => expiredAt
|
||||
));
|
||||
|
||||
if (rowsAffected > 0)
|
||||
{
|
||||
await fileService._PurgeCacheAsync(fileId);
|
||||
await PurgeCacheForFileAsync(fileId);
|
||||
}
|
||||
|
||||
return rowsAffected;
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Get all file references for a specific resource and usage type
|
||||
/// </summary>
|
||||
/// <param name="resourceId">The resource ID</param>
|
||||
/// <param name="usageType">The usage type</param>
|
||||
/// <returns>List of file references</returns>
|
||||
public async Task<List<CloudFileReference>> GetResourceReferencesAsync(string resourceId, string usageType)
|
||||
{
|
||||
return await db.FileReferences
|
||||
.Where(r => r.ResourceId == resourceId && r.Usage == usageType)
|
||||
.ToListAsync();
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Check if a file has any references
|
||||
/// </summary>
|
||||
/// <param name="fileId">The file ID to check</param>
|
||||
/// <returns>True if the file has references, false otherwise</returns>
|
||||
public async Task<bool> HasFileReferencesAsync(string fileId)
|
||||
{
|
||||
return await db.FileReferences.AnyAsync(r => r.FileId == fileId);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Updates the expiration time for a file reference using a duration from now
|
||||
/// </summary>
|
||||
/// <param name="referenceId">The ID of the reference</param>
|
||||
/// <param name="duration">The duration after which the reference expires, or null to remove expiration</param>
|
||||
/// <returns>True if the reference was found and updated, false otherwise</returns>
|
||||
public async Task<bool> SetReferenceExpirationDurationAsync(Guid referenceId, Duration? duration)
|
||||
{
|
||||
Instant? expiredAt = null;
|
||||
if (duration.HasValue)
|
||||
{
|
||||
expiredAt = SystemClock.Instance.GetCurrentInstant() + duration.Value;
|
||||
}
|
||||
|
||||
return await SetReferenceExpirationAsync(referenceId, expiredAt);
|
||||
}
|
||||
}
|
||||
555
DysonNetwork.Drive/Storage/FileService.cs
Normal file
555
DysonNetwork.Drive/Storage/FileService.cs
Normal file
@@ -0,0 +1,555 @@
|
||||
using System.Globalization;
|
||||
using FFMpegCore;
|
||||
using System.Security.Cryptography;
|
||||
using DysonNetwork.Shared.Cache;
|
||||
using DysonNetwork.Shared.Proto;
|
||||
using Microsoft.EntityFrameworkCore;
|
||||
using Minio;
|
||||
using Minio.DataModel.Args;
|
||||
using NetVips;
|
||||
using NodaTime;
|
||||
using tusdotnet.Stores;
|
||||
|
||||
namespace DysonNetwork.Drive.Storage;
|
||||
|
||||
public class FileService(
|
||||
AppDatabase db,
|
||||
IConfiguration configuration,
|
||||
TusDiskStore store,
|
||||
ILogger<FileService> logger,
|
||||
IServiceScopeFactory scopeFactory,
|
||||
ICacheService cache
|
||||
)
|
||||
{
|
||||
private const string CacheKeyPrefix = "file:";
|
||||
private static readonly TimeSpan CacheDuration = TimeSpan.FromMinutes(15);
|
||||
|
||||
/// <summary>
|
||||
/// The api for getting file meta with cache,
|
||||
/// the best use case is for accessing the file data.
|
||||
///
|
||||
/// <b>This function won't load uploader's information, only keep minimal file meta</b>
|
||||
/// </summary>
|
||||
/// <param name="fileId">The id of the cloud file requested</param>
|
||||
/// <returns>The minimal file meta</returns>
|
||||
public async Task<CloudFile?> GetFileAsync(string fileId)
|
||||
{
|
||||
var cacheKey = $"{CacheKeyPrefix}{fileId}";
|
||||
|
||||
var cachedFile = await cache.GetAsync<CloudFile>(cacheKey);
|
||||
if (cachedFile is not null)
|
||||
return cachedFile;
|
||||
|
||||
var file = await db.Files
|
||||
.Where(f => f.Id == fileId)
|
||||
.FirstOrDefaultAsync();
|
||||
|
||||
if (file != null)
|
||||
await cache.SetAsync(cacheKey, file, CacheDuration);
|
||||
|
||||
return file;
|
||||
}
|
||||
|
||||
private static readonly string TempFilePrefix = "dyn-cloudfile";
|
||||
|
||||
private static readonly string[] AnimatedImageTypes =
|
||||
["image/gif", "image/apng", "image/webp", "image/avif"];
|
||||
|
||||
// The analysis file method no longer will remove the GPS EXIF data
|
||||
// It should be handled on the client side, and for some specific cases it should be keep
|
||||
public async Task<CloudFile> ProcessNewFileAsync(
|
||||
Account account,
|
||||
string fileId,
|
||||
Stream stream,
|
||||
string fileName,
|
||||
string? contentType
|
||||
)
|
||||
{
|
||||
var result = new List<(string filePath, string suffix)>();
|
||||
|
||||
var ogFilePath = Path.GetFullPath(Path.Join(configuration.GetValue<string>("Tus:StorePath"), fileId));
|
||||
var fileSize = stream.Length;
|
||||
var hash = await HashFileAsync(stream, fileSize: fileSize);
|
||||
contentType ??= !fileName.Contains('.') ? "application/octet-stream" : MimeTypes.GetMimeType(fileName);
|
||||
|
||||
var file = new CloudFile
|
||||
{
|
||||
Id = fileId,
|
||||
Name = fileName,
|
||||
MimeType = contentType,
|
||||
Size = fileSize,
|
||||
Hash = hash,
|
||||
AccountId = Guid.Parse(account.Id)
|
||||
};
|
||||
|
||||
var existingFile = await db.Files.FirstOrDefaultAsync(f => f.Hash == hash);
|
||||
file.StorageId = existingFile is not null ? existingFile.StorageId : file.Id;
|
||||
|
||||
if (existingFile is not null)
|
||||
{
|
||||
file.FileMeta = existingFile.FileMeta;
|
||||
file.HasCompression = existingFile.HasCompression;
|
||||
file.SensitiveMarks = existingFile.SensitiveMarks;
|
||||
|
||||
db.Files.Add(file);
|
||||
await db.SaveChangesAsync();
|
||||
return file;
|
||||
}
|
||||
|
||||
switch (contentType.Split('/')[0])
|
||||
{
|
||||
case "image":
|
||||
var blurhash =
|
||||
BlurHashSharp.SkiaSharp.BlurHashEncoder.Encode(xComponent: 3, yComponent: 3, filename: ogFilePath);
|
||||
|
||||
// Rewind stream
|
||||
stream.Position = 0;
|
||||
|
||||
// Use NetVips for the rest
|
||||
using (var vipsImage = NetVips.Image.NewFromStream(stream))
|
||||
{
|
||||
var width = vipsImage.Width;
|
||||
var height = vipsImage.Height;
|
||||
var format = vipsImage.Get("vips-loader") ?? "unknown";
|
||||
|
||||
// Try to get orientation from exif data
|
||||
var orientation = 1;
|
||||
var meta = new Dictionary<string, object>
|
||||
{
|
||||
["blur"] = blurhash,
|
||||
["format"] = format,
|
||||
["width"] = width,
|
||||
["height"] = height,
|
||||
["orientation"] = orientation,
|
||||
};
|
||||
Dictionary<string, object> exif = [];
|
||||
|
||||
foreach (var field in vipsImage.GetFields())
|
||||
{
|
||||
var value = vipsImage.Get(field);
|
||||
|
||||
// Skip GPS-related EXIF fields to remove location data
|
||||
if (IsIgnoredField(field))
|
||||
continue;
|
||||
|
||||
if (field.StartsWith("exif-")) exif[field.Replace("exif-", "")] = value;
|
||||
else meta[field] = value;
|
||||
|
||||
if (field == "orientation") orientation = (int)value;
|
||||
}
|
||||
|
||||
if (orientation is 6 or 8)
|
||||
(width, height) = (height, width);
|
||||
|
||||
var aspectRatio = height != 0 ? (double)width / height : 0;
|
||||
|
||||
meta["exif"] = exif;
|
||||
meta["ratio"] = aspectRatio;
|
||||
file.FileMeta = meta;
|
||||
}
|
||||
|
||||
break;
|
||||
case "video":
|
||||
case "audio":
|
||||
try
|
||||
{
|
||||
var mediaInfo = await FFProbe.AnalyseAsync(ogFilePath);
|
||||
file.FileMeta = new Dictionary<string, object>
|
||||
{
|
||||
["duration"] = mediaInfo.Duration.TotalSeconds,
|
||||
["format_name"] = mediaInfo.Format.FormatName,
|
||||
["format_long_name"] = mediaInfo.Format.FormatLongName,
|
||||
["start_time"] = mediaInfo.Format.StartTime.ToString(),
|
||||
["bit_rate"] = mediaInfo.Format.BitRate.ToString(CultureInfo.InvariantCulture),
|
||||
["tags"] = mediaInfo.Format.Tags ?? [],
|
||||
["chapters"] = mediaInfo.Chapters,
|
||||
};
|
||||
if (mediaInfo.PrimaryVideoStream is not null)
|
||||
file.FileMeta["ratio"] =
|
||||
mediaInfo.PrimaryVideoStream.Width / mediaInfo.PrimaryVideoStream.Height;
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
logger.LogError("File analyzed failed, unable collect video / audio information: {Message}",
|
||||
ex.Message);
|
||||
}
|
||||
|
||||
break;
|
||||
}
|
||||
|
||||
db.Files.Add(file);
|
||||
await db.SaveChangesAsync();
|
||||
|
||||
_ = Task.Run(async () =>
|
||||
{
|
||||
using var scope = scopeFactory.CreateScope();
|
||||
var nfs = scope.ServiceProvider.GetRequiredService<FileService>();
|
||||
|
||||
try
|
||||
{
|
||||
logger.LogInformation("Processed file {fileId}, now trying optimizing if possible...", fileId);
|
||||
|
||||
if (contentType.Split('/')[0] == "image")
|
||||
{
|
||||
// Skip compression for animated image types
|
||||
var animatedMimeTypes = AnimatedImageTypes;
|
||||
if (Enumerable.Contains(animatedMimeTypes, contentType))
|
||||
{
|
||||
logger.LogInformation(
|
||||
"File {fileId} is an animated image (MIME: {mime}), skipping WebP conversion.", fileId,
|
||||
contentType
|
||||
);
|
||||
var tempFilePath = Path.Join(Path.GetTempPath(), $"{TempFilePrefix}#{file.Id}");
|
||||
result.Add((tempFilePath, string.Empty));
|
||||
return;
|
||||
}
|
||||
|
||||
file.MimeType = "image/webp";
|
||||
|
||||
using var vipsImage = Image.NewFromFile(ogFilePath);
|
||||
var imagePath = Path.Join(Path.GetTempPath(), $"{TempFilePrefix}#{file.Id}");
|
||||
vipsImage.Autorot().WriteToFile(imagePath + ".webp",
|
||||
new VOption { { "lossless", true }, { "strip", true } });
|
||||
result.Add((imagePath + ".webp", string.Empty));
|
||||
|
||||
if (vipsImage.Width * vipsImage.Height >= 1024 * 1024)
|
||||
{
|
||||
var scale = 1024.0 / Math.Max(vipsImage.Width, vipsImage.Height);
|
||||
var imageCompressedPath =
|
||||
Path.Join(Path.GetTempPath(), $"{TempFilePrefix}#{file.Id}-compressed");
|
||||
|
||||
// Create and save image within the same synchronous block to avoid disposal issues
|
||||
using var compressedImage = vipsImage.Resize(scale);
|
||||
compressedImage.Autorot().WriteToFile(imageCompressedPath + ".webp",
|
||||
new VOption { { "Q", 80 }, { "strip", true } });
|
||||
|
||||
result.Add((imageCompressedPath + ".webp", ".compressed"));
|
||||
file.HasCompression = true;
|
||||
}
|
||||
}
|
||||
else
|
||||
{
|
||||
// No extra process for video, add it to the upload queue.
|
||||
result.Add((ogFilePath, string.Empty));
|
||||
}
|
||||
|
||||
logger.LogInformation("Optimized file {fileId}, now uploading...", fileId);
|
||||
|
||||
if (result.Count > 0)
|
||||
{
|
||||
List<Task<CloudFile>> tasks = [];
|
||||
tasks.AddRange(result.Select(item =>
|
||||
nfs.UploadFileToRemoteAsync(file, item.filePath, null, item.suffix, true))
|
||||
);
|
||||
|
||||
await Task.WhenAll(tasks);
|
||||
file = await tasks.First();
|
||||
}
|
||||
else
|
||||
{
|
||||
file = await nfs.UploadFileToRemoteAsync(file, stream, null);
|
||||
}
|
||||
|
||||
logger.LogInformation("Uploaded file {fileId} done!", fileId);
|
||||
|
||||
var scopedDb = scope.ServiceProvider.GetRequiredService<AppDatabase>();
|
||||
await scopedDb.Files.Where(f => f.Id == file.Id).ExecuteUpdateAsync(setter => setter
|
||||
.SetProperty(f => f.UploadedAt, file.UploadedAt)
|
||||
.SetProperty(f => f.UploadedTo, file.UploadedTo)
|
||||
.SetProperty(f => f.MimeType, file.MimeType)
|
||||
.SetProperty(f => f.HasCompression, file.HasCompression)
|
||||
);
|
||||
}
|
||||
catch (Exception err)
|
||||
{
|
||||
logger.LogError(err, "Failed to process {fileId}", fileId);
|
||||
}
|
||||
|
||||
await stream.DisposeAsync();
|
||||
await store.DeleteFileAsync(file.Id, CancellationToken.None);
|
||||
await nfs._PurgeCacheAsync(file.Id);
|
||||
});
|
||||
|
||||
return file;
|
||||
}
|
||||
|
||||
private static async Task<string> HashFileAsync(Stream stream, int chunkSize = 1024 * 1024, long? fileSize = null)
|
||||
{
|
||||
fileSize ??= stream.Length;
|
||||
if (fileSize > chunkSize * 1024 * 5)
|
||||
return await HashFastApproximateAsync(stream, chunkSize);
|
||||
|
||||
using var md5 = MD5.Create();
|
||||
var hashBytes = await md5.ComputeHashAsync(stream);
|
||||
return Convert.ToHexString(hashBytes).ToLowerInvariant();
|
||||
}
|
||||
|
||||
private static async Task<string> HashFastApproximateAsync(Stream stream, int chunkSize = 1024 * 1024)
|
||||
{
|
||||
// Scale the chunk size to kB level
|
||||
chunkSize *= 1024;
|
||||
|
||||
using var md5 = MD5.Create();
|
||||
|
||||
var buffer = new byte[chunkSize * 2];
|
||||
var fileLength = stream.Length;
|
||||
|
||||
var bytesRead = await stream.ReadAsync(buffer.AsMemory(0, chunkSize));
|
||||
|
||||
if (fileLength > chunkSize)
|
||||
{
|
||||
stream.Seek(-chunkSize, SeekOrigin.End);
|
||||
bytesRead += await stream.ReadAsync(buffer.AsMemory(chunkSize, chunkSize));
|
||||
}
|
||||
|
||||
var hash = md5.ComputeHash(buffer, 0, bytesRead);
|
||||
return Convert.ToHexString(hash).ToLowerInvariant();
|
||||
}
|
||||
|
||||
public async Task<CloudFile> UploadFileToRemoteAsync(CloudFile file, string filePath, string? targetRemote,
|
||||
string? suffix = null, bool selfDestruct = false)
|
||||
{
|
||||
var fileStream = File.OpenRead(filePath);
|
||||
var result = await UploadFileToRemoteAsync(file, fileStream, targetRemote, suffix);
|
||||
if (selfDestruct) File.Delete(filePath);
|
||||
return result;
|
||||
}
|
||||
|
||||
public async Task<CloudFile> UploadFileToRemoteAsync(CloudFile file, Stream stream, string? targetRemote,
|
||||
string? suffix = null)
|
||||
{
|
||||
if (file.UploadedAt.HasValue) return file;
|
||||
|
||||
file.UploadedTo = targetRemote ?? configuration.GetValue<string>("Storage:PreferredRemote")!;
|
||||
|
||||
var dest = GetRemoteStorageConfig(file.UploadedTo);
|
||||
var client = CreateMinioClient(dest);
|
||||
if (client is null)
|
||||
throw new InvalidOperationException(
|
||||
$"Failed to configure client for remote destination '{file.UploadedTo}'"
|
||||
);
|
||||
|
||||
var bucket = dest.Bucket;
|
||||
var contentType = file.MimeType ?? "application/octet-stream";
|
||||
|
||||
await client.PutObjectAsync(new PutObjectArgs()
|
||||
.WithBucket(bucket)
|
||||
.WithObject(string.IsNullOrWhiteSpace(suffix) ? file.Id : file.Id + suffix)
|
||||
.WithStreamData(stream) // Fix this disposed
|
||||
.WithObjectSize(stream.Length)
|
||||
.WithContentType(contentType)
|
||||
);
|
||||
|
||||
file.UploadedAt = Instant.FromDateTimeUtc(DateTime.UtcNow);
|
||||
return file;
|
||||
}
|
||||
|
||||
public async Task DeleteFileAsync(CloudFile file)
|
||||
{
|
||||
await DeleteFileDataAsync(file);
|
||||
|
||||
db.Remove(file);
|
||||
await db.SaveChangesAsync();
|
||||
await _PurgeCacheAsync(file.Id);
|
||||
}
|
||||
|
||||
public async Task DeleteFileDataAsync(CloudFile file)
|
||||
{
|
||||
if (file.StorageId is null) return;
|
||||
if (file.UploadedTo is null) return;
|
||||
|
||||
// Check if any other file with the same storage ID is referenced
|
||||
var otherFilesWithSameStorageId = await db.Files
|
||||
.Where(f => f.StorageId == file.StorageId && f.Id != file.Id)
|
||||
.Select(f => f.Id)
|
||||
.ToListAsync();
|
||||
|
||||
// Check if any of these files are referenced
|
||||
var anyReferenced = false;
|
||||
if (otherFilesWithSameStorageId.Any())
|
||||
{
|
||||
anyReferenced = await db.FileReferences
|
||||
.Where(r => otherFilesWithSameStorageId.Contains(r.FileId))
|
||||
.AnyAsync();
|
||||
}
|
||||
|
||||
// If any other file with the same storage ID is referenced, don't delete the actual file data
|
||||
if (anyReferenced) return;
|
||||
|
||||
var dest = GetRemoteStorageConfig(file.UploadedTo);
|
||||
var client = CreateMinioClient(dest);
|
||||
if (client is null)
|
||||
throw new InvalidOperationException(
|
||||
$"Failed to configure client for remote destination '{file.UploadedTo}'"
|
||||
);
|
||||
|
||||
var bucket = dest.Bucket;
|
||||
var objectId = file.StorageId ?? file.Id; // Use StorageId if available, otherwise fall back to Id
|
||||
|
||||
await client.RemoveObjectAsync(
|
||||
new RemoveObjectArgs().WithBucket(bucket).WithObject(objectId)
|
||||
);
|
||||
|
||||
if (file.HasCompression)
|
||||
{
|
||||
// Also remove the compressed version if it exists
|
||||
try
|
||||
{
|
||||
await client.RemoveObjectAsync(
|
||||
new RemoveObjectArgs().WithBucket(bucket).WithObject(objectId + ".compressed")
|
||||
);
|
||||
}
|
||||
catch
|
||||
{
|
||||
// Ignore errors when deleting compressed version
|
||||
logger.LogWarning("Failed to delete compressed version of file {fileId}", file.Id);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
public RemoteStorageConfig GetRemoteStorageConfig(string destination)
|
||||
{
|
||||
var destinations = configuration.GetSection("Storage:Remote").Get<List<RemoteStorageConfig>>()!;
|
||||
var dest = destinations.FirstOrDefault(d => d.Id == destination);
|
||||
if (dest is null) throw new InvalidOperationException($"Remote destination '{destination}' not found");
|
||||
return dest;
|
||||
}
|
||||
|
||||
public IMinioClient? CreateMinioClient(RemoteStorageConfig dest)
|
||||
{
|
||||
var client = new MinioClient()
|
||||
.WithEndpoint(dest.Endpoint)
|
||||
.WithRegion(dest.Region)
|
||||
.WithCredentials(dest.SecretId, dest.SecretKey);
|
||||
if (dest.EnableSsl) client = client.WithSSL();
|
||||
|
||||
return client.Build();
|
||||
}
|
||||
|
||||
// Helper method to purge the cache for a specific file
|
||||
// Made internal to allow FileReferenceService to use it
|
||||
internal async Task _PurgeCacheAsync(string fileId)
|
||||
{
|
||||
var cacheKey = $"{CacheKeyPrefix}{fileId}";
|
||||
await cache.RemoveAsync(cacheKey);
|
||||
}
|
||||
|
||||
// Helper method to purge cache for multiple files
|
||||
internal async Task _PurgeCacheRangeAsync(IEnumerable<string> fileIds)
|
||||
{
|
||||
var tasks = fileIds.Select(_PurgeCacheAsync);
|
||||
await Task.WhenAll(tasks);
|
||||
}
|
||||
|
||||
public async Task<List<CloudFile?>> LoadFromReference(List<CloudFileReferenceObject> references)
|
||||
{
|
||||
var cachedFiles = new Dictionary<string, CloudFile>();
|
||||
var uncachedIds = new List<string>();
|
||||
|
||||
// Check cache first
|
||||
foreach (var reference in references)
|
||||
{
|
||||
var cacheKey = $"{CacheKeyPrefix}{reference.Id}";
|
||||
var cachedFile = await cache.GetAsync<CloudFile>(cacheKey);
|
||||
|
||||
if (cachedFile != null)
|
||||
{
|
||||
cachedFiles[reference.Id] = cachedFile;
|
||||
}
|
||||
else
|
||||
{
|
||||
uncachedIds.Add(reference.Id);
|
||||
}
|
||||
}
|
||||
|
||||
// Load uncached files from database
|
||||
if (uncachedIds.Count > 0)
|
||||
{
|
||||
var dbFiles = await db.Files
|
||||
.Where(f => uncachedIds.Contains(f.Id))
|
||||
.ToListAsync();
|
||||
|
||||
// Add to cache
|
||||
foreach (var file in dbFiles)
|
||||
{
|
||||
var cacheKey = $"{CacheKeyPrefix}{file.Id}";
|
||||
await cache.SetAsync(cacheKey, file, CacheDuration);
|
||||
cachedFiles[file.Id] = file;
|
||||
}
|
||||
}
|
||||
|
||||
// Preserve original order
|
||||
return references
|
||||
.Select(r => cachedFiles.GetValueOrDefault(r.Id))
|
||||
.Where(f => f != null)
|
||||
.ToList();
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Gets the number of references to a file based on CloudFileReference records
|
||||
/// </summary>
|
||||
/// <param name="fileId">The ID of the file</param>
|
||||
/// <returns>The number of references to the file</returns>
|
||||
public async Task<int> GetReferenceCountAsync(string fileId)
|
||||
{
|
||||
return await db.FileReferences
|
||||
.Where(r => r.FileId == fileId)
|
||||
.CountAsync();
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Checks if a file is referenced by any resource
|
||||
/// </summary>
|
||||
/// <param name="fileId">The ID of the file to check</param>
|
||||
/// <returns>True if the file is referenced, false otherwise</returns>
|
||||
public async Task<bool> IsReferencedAsync(string fileId)
|
||||
{
|
||||
return await db.FileReferences
|
||||
.Where(r => r.FileId == fileId)
|
||||
.AnyAsync();
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Checks if an EXIF field contains GPS location data
|
||||
/// </summary>
|
||||
/// <param name="fieldName">The EXIF field name</param>
|
||||
/// <returns>True if the field contains GPS data, false otherwise</returns>
|
||||
private static bool IsGpsExifField(string fieldName)
|
||||
{
|
||||
// Common GPS EXIF field names
|
||||
var gpsFields = new[]
|
||||
{
|
||||
"gps-latitude",
|
||||
"gps-longitude",
|
||||
"gps-altitude",
|
||||
"gps-latitude-ref",
|
||||
"gps-longitude-ref",
|
||||
"gps-altitude-ref",
|
||||
"gps-timestamp",
|
||||
"gps-datestamp",
|
||||
"gps-speed",
|
||||
"gps-speed-ref",
|
||||
"gps-track",
|
||||
"gps-track-ref",
|
||||
"gps-img-direction",
|
||||
"gps-img-direction-ref",
|
||||
"gps-dest-latitude",
|
||||
"gps-dest-longitude",
|
||||
"gps-dest-latitude-ref",
|
||||
"gps-dest-longitude-ref",
|
||||
"gps-processing-method",
|
||||
"gps-area-information"
|
||||
};
|
||||
|
||||
return gpsFields.Any(gpsField =>
|
||||
fieldName.Equals(gpsField, StringComparison.OrdinalIgnoreCase) ||
|
||||
fieldName.StartsWith("gps", StringComparison.OrdinalIgnoreCase));
|
||||
}
|
||||
|
||||
private static bool IsIgnoredField(string fieldName)
|
||||
{
|
||||
if (IsGpsExifField(fieldName)) return true;
|
||||
if (fieldName.EndsWith("-data")) return true;
|
||||
return false;
|
||||
}
|
||||
}
|
||||
79
DysonNetwork.Drive/Storage/TusService.cs
Normal file
79
DysonNetwork.Drive/Storage/TusService.cs
Normal file
@@ -0,0 +1,79 @@
|
||||
using System.Net;
|
||||
using System.Text;
|
||||
using System.Text.Json;
|
||||
using DysonNetwork.Shared.Proto;
|
||||
using Microsoft.AspNetCore.Mvc;
|
||||
using Microsoft.Extensions.Options;
|
||||
using tusdotnet.Interfaces;
|
||||
using tusdotnet.Models;
|
||||
using tusdotnet.Models.Configuration;
|
||||
|
||||
namespace DysonNetwork.Drive.Storage;
|
||||
|
||||
public abstract class TusService
|
||||
{
|
||||
public static DefaultTusConfiguration BuildConfiguration(ITusStore store) => new()
|
||||
{
|
||||
Store = store,
|
||||
Events = new Events
|
||||
{
|
||||
OnAuthorizeAsync = async eventContext =>
|
||||
{
|
||||
if (eventContext.Intent == IntentType.DeleteFile)
|
||||
{
|
||||
eventContext.FailRequest(
|
||||
HttpStatusCode.BadRequest,
|
||||
"Deleting files from this endpoint was disabled, please refer to the Dyson Network File API."
|
||||
);
|
||||
return;
|
||||
}
|
||||
|
||||
var httpContext = eventContext.HttpContext;
|
||||
if (httpContext.Items["CurrentUser"] is not Account user)
|
||||
{
|
||||
eventContext.FailRequest(HttpStatusCode.Unauthorized);
|
||||
return;
|
||||
}
|
||||
|
||||
if (!user.IsSuperuser)
|
||||
{
|
||||
using var scope = httpContext.RequestServices.CreateScope();
|
||||
var pm = scope.ServiceProvider.GetRequiredService<PermissionService.PermissionServiceClient>();
|
||||
var allowed = await pm.HasPermissionAsync(new HasPermissionRequest
|
||||
{ Actor = $"user:{user.Id}", Area = "global", Key = "files.create" });
|
||||
if (!allowed.HasPermission)
|
||||
eventContext.FailRequest(HttpStatusCode.Forbidden);
|
||||
}
|
||||
},
|
||||
OnFileCompleteAsync = async eventContext =>
|
||||
{
|
||||
using var scope = eventContext.HttpContext.RequestServices.CreateScope();
|
||||
var services = scope.ServiceProvider;
|
||||
|
||||
var httpContext = eventContext.HttpContext;
|
||||
if (httpContext.Items["CurrentUser"] is not Account user) return;
|
||||
|
||||
var file = await eventContext.GetFileAsync();
|
||||
var metadata = await file.GetMetadataAsync(eventContext.CancellationToken);
|
||||
var fileName = metadata.TryGetValue("filename", out var fn)
|
||||
? fn.GetString(Encoding.UTF8)
|
||||
: "uploaded_file";
|
||||
var contentType = metadata.TryGetValue("content-type", out var ct) ? ct.GetString(Encoding.UTF8) : null;
|
||||
|
||||
var fileStream = await file.GetContentAsync(eventContext.CancellationToken);
|
||||
|
||||
var fileService = services.GetRequiredService<FileService>();
|
||||
var info = await fileService.ProcessNewFileAsync(user, file.Id, fileStream, fileName, contentType);
|
||||
|
||||
using var finalScope = eventContext.HttpContext.RequestServices.CreateScope();
|
||||
var jsonOptions = finalScope.ServiceProvider.GetRequiredService<IOptions<JsonOptions>>().Value
|
||||
.JsonSerializerOptions;
|
||||
var infoJson = JsonSerializer.Serialize(info, jsonOptions);
|
||||
eventContext.HttpContext.Response.Headers.Append("X-FileInfo", infoJson);
|
||||
|
||||
// Dispose the stream after all processing is complete
|
||||
await fileStream.DisposeAsync();
|
||||
}
|
||||
}
|
||||
};
|
||||
}
|
||||
Reference in New Issue
Block a user