✨ File expiration

DysonNetwork.Drive/Storage/CloudFile.cs

@@ -38,6 +38,7 @@ public class CloudFile : ModelBase, ICloudFile, IIdentifiedResource
     [Column(TypeName = "jsonb")] public List<ContentSensitiveMark>? SensitiveMarks { get; set; } = [];
     [MaxLength(256)] public string? MimeType { get; set; }
     [MaxLength(256)] public string? Hash { get; set; }
+    public Instant? ExpiredAt { get; set; }
     public long Size { get; set; }
     public Instant? UploadedAt { get; set; }
     public bool HasCompression { get; set; } = false;

DysonNetwork.Drive/Storage/CloudFileUnusedRecyclingJob.cs

@@ -15,11 +15,19 @@ public class CloudFileUnusedRecyclingJob(
     {
         logger.LogInformation("Marking unused cloud files...");
 
+        var recyclablePools = await db.Pools
+            .Where(p => p.PolicyConfig.EnableRecycle)
+            .Select(p => p.Id)
+            .ToListAsync();
+
         var now = SystemClock.Instance.GetCurrentInstant();
         const int batchSize = 1000; // Process larger batches for efficiency
         var processedCount = 0;
         var markedCount = 0;
-        var totalFiles = await db.Files.Where(f => !f.IsMarkedRecycle).CountAsync();
+        var totalFiles = await db.Files
+            .Where(f => f.PoolId.HasValue && recyclablePools.Contains(f.PoolId.Value))
+            .Where(f => !f.IsMarkedRecycle)
+            .CountAsync();
 
         logger.LogInformation("Found {TotalFiles} files to check for unused status", totalFiles);
 
@@ -35,13 +43,12 @@ public class CloudFileUnusedRecyclingJob(
         {
             // Query for the next batch of files using keyset pagination
             var filesQuery = db.Files
+                .Where(f => f.PoolId.HasValue && recyclablePools.Contains(f.PoolId.Value))
                 .Where(f => !f.IsMarkedRecycle)
                 .Where(f => f.CreatedAt <= ageThreshold); // Only process older files first
 
             if (lastProcessedId != null)
-            {
                 filesQuery = filesQuery.Where(f => string.Compare(f.Id, lastProcessedId) > 0);
-            }
 
             var fileBatch = await filesQuery
                 .OrderBy(f => f.Id) // Ensure consistent ordering for pagination
@@ -84,9 +91,17 @@ public class CloudFileUnusedRecyclingJob(
             {
                 logger.LogInformation(
                     "Progress: processed {ProcessedCount}/{TotalFiles} files, marked {MarkedCount} for recycling",
-                    processedCount, totalFiles, markedCount);
+                    processedCount,
+                    totalFiles,
+                    markedCount
+                );
             }
         }
+        
+        var expiredCount = await db.Files
+            .Where(f => f.ExpiredAt.HasValue && f.ExpiredAt.Value <= now)
+            .ExecuteUpdateAsync(s => s.SetProperty(f => f.IsMarkedRecycle, true));
+        markedCount += expiredCount;
 
         logger.LogInformation("Completed marking {MarkedCount} files for recycling", markedCount);
     }

DysonNetwork.Drive/Storage/FilePool.cs

@@ -26,6 +26,7 @@ public class BillingConfig
 
 public class PolicyConfig
 {
+    public bool EnableRecycle { get; set; } = false;
     public bool PublicIndexable { get; set; } = false;
     public bool PublicUsable { get; set; } = false;
     public bool NoOptimization { get; set; } = false;

DysonNetwork.Drive/Storage/FileReferenceService.cs

@@ -32,19 +32,6 @@ public class FileReferenceService(AppDatabase db, FileService fileService, ICach
         if (duration.HasValue)
             finalExpiration = SystemClock.Instance.GetCurrentInstant() + duration.Value;
 
-        var file = await db.Files
-            .Where(f => f.Id == fileId)
-            .Include(f => f.Pool)
-            .FirstOrDefaultAsync();
-        if (file is null) throw new InvalidOperationException("File not found");
-        if (file.Pool?.StorageConfig.Expiration != null)
-        {
-            var now = SystemClock.Instance.GetCurrentInstant();
-            var expectedDuration = finalExpiration - now;
-            if (finalExpiration == null || expectedDuration > file.Pool.StorageConfig.Expiration)
-                finalExpiration = now.Plus(file.Pool.StorageConfig.Expiration.Value);
-        }
-
         var reference = new CloudFileReference
         {
             FileId = fileId,

DysonNetwork.Drive/Storage/FileService.cs

@@ -26,7 +26,7 @@ public class FileService(
 {
     private const string CacheKeyPrefix = "file:";
     private static readonly TimeSpan CacheDuration = TimeSpan.FromMinutes(15);
-    
+
     /// <summary>
     /// The api for getting file meta with cache,
     /// the best use case is for accessing the file data.
@@ -108,19 +108,30 @@ public class FileService(
         Stream stream,
         string fileName,
         string? contentType,
-        string? encryptPassword
+        string? encryptPassword,
+        Instant? expiredAt
     )
     {
         var pool = await GetPoolAsync(Guid.Parse(filePool));
         if (pool is null) throw new InvalidOperationException("Pool not found");
 
+        if (pool.StorageConfig.Expiration is not null && expiredAt.HasValue)
+        {
+            var expectedExpiration = SystemClock.Instance.GetCurrentInstant() - expiredAt.Value;
+            var effectiveExpiration = pool.StorageConfig.Expiration < expectedExpiration
+                ? pool.StorageConfig.Expiration
+                : expectedExpiration;
+            expiredAt = SystemClock.Instance.GetCurrentInstant() + effectiveExpiration;
+        }
+
         var ogFilePath = Path.GetFullPath(Path.Join(configuration.GetValue<string>("Tus:StorePath"), fileId));
         var fileSize = stream.Length;
         contentType ??= !fileName.Contains('.') ? "application/octet-stream" : MimeTypes.GetMimeType(fileName);
 
         if (!string.IsNullOrWhiteSpace(encryptPassword))
         {
-            if (!pool.PolicyConfig.AllowEncryption) throw new InvalidOperationException("Encryption is not allowed in this pool");
+            if (!pool.PolicyConfig.AllowEncryption)
+                throw new InvalidOperationException("Encryption is not allowed in this pool");
             var encryptedPath = Path.Combine(Path.GetTempPath(), $"{fileId}.encrypted");
             FileEncryptor.EncryptFile(ogFilePath, encryptedPath, encryptPassword);
             File.Delete(ogFilePath); // Delete original unencrypted
@@ -137,6 +148,7 @@ public class FileService(
             MimeType = contentType,
             Size = fileSize,
             Hash = hash,
+            ExpiredAt = expiredAt,
             AccountId = Guid.Parse(account.Id),
             IsEncrypted = !string.IsNullOrWhiteSpace(encryptPassword) && pool.PolicyConfig.AllowEncryption
         };
@@ -369,6 +381,7 @@ public class FileService(
                         {
                             logger.LogError(ex, "Failed to generate thumbnail for video {FileId}", fileId);
                         }
+
                         break;
 
                     default:
@@ -431,7 +444,7 @@ public class FileService(
     private static async Task<string> HashFastApproximateAsync(string filePath, int chunkSize = 1024 * 1024)
     {
         await using var stream = File.OpenRead(filePath);
-        
+
         // Scale the chunk size to kB level
         chunkSize *= 1024;
 

DysonNetwork.Drive/Storage/TusService.cs

@@ -5,6 +5,7 @@ using DysonNetwork.Shared.Auth;
 using DysonNetwork.Shared.Proto;
 using Microsoft.AspNetCore.Mvc;
 using Microsoft.Extensions.Options;
+using NodaTime;
 using tusdotnet.Interfaces;
 using tusdotnet.Models;
 using tusdotnet.Models.Configuration;
@@ -112,6 +113,11 @@ public abstract class TusService
                 if (string.IsNullOrEmpty(filePool))
                     filePool = configuration["Storage:PreferredRemote"];
 
+                Instant? expiredAt = null;
+                var expiredString = httpContext.Request.Headers["X-FileExpire"].FirstOrDefault();
+                if (!string.IsNullOrEmpty(expiredString) && int.TryParse(expiredString, out var expired))
+                    expiredAt = Instant.FromUnixTimeSeconds(expired);
+
                 try
                 {
                     var fileService = services.GetRequiredService<FileService>();
@@ -122,7 +128,8 @@ public abstract class TusService
                         fileStream,
                         fileName,
                         contentType,
-                        encryptPassword
+                        encryptPassword,
+                        expiredAt
                     );
 
                     using var finalScope = eventContext.HttpContext.RequestServices.CreateScope();