♻️ Better delivery service for AP

DysonNetwork.Shared/Models/ActivityPubDelivery.cs

@@ -0,0 +1,35 @@
+using System.ComponentModel.DataAnnotations;
+using System.ComponentModel.DataAnnotations.Schema;
+using NodaTime;
+
+namespace DysonNetwork.Shared.Models;
+
+public class SnActivityPubDelivery : ModelBase
+{
+    public Guid Id { get; set; } = Guid.NewGuid();
+
+    [MaxLength(2048)] public string ActivityId { get; set; } = null!;
+    [MaxLength(128)] public string ActivityType { get; set; } = null!;
+    [MaxLength(2048)] public string InboxUri { get; set; } = null!;
+    [MaxLength(2048)] public string ActorUri { get; set; } = null!;
+    
+    public DeliveryStatus Status { get; set; } = DeliveryStatus.Pending;
+    public int RetryCount { get; set; } = 0;
+    
+    [MaxLength(4096)] public string? ErrorMessage { get; set; }
+    
+    public Instant? LastAttemptAt { get; set; }
+    public Instant? NextRetryAt { get; set; }
+    public Instant? SentAt { get; set; }
+    
+    [MaxLength(2048)] public string? ResponseStatusCode { get; set; }
+}
+
+public enum DeliveryStatus
+{
+    Pending,
+    Processing,
+    Sent,
+    Failed,
+    ExhaustedRetries
+}

DysonNetwork.Sphere/ActivityPub/ActivityPubDeliveryRetryJob.cs

@@ -0,0 +1,91 @@
+using DysonNetwork.Shared.Models;
+using Microsoft.EntityFrameworkCore;
+using NodaTime;
+using Quartz;
+
+namespace DysonNetwork.Sphere.ActivityPub;
+
+public class ActivityPubDeliveryRetryJob(AppDatabase db, ActivityPubQueueService queueService, ILogger<ActivityPubDeliveryRetryJob> logger, IClock clock)
+    : IJob
+{
+    public async Task Execute(IJobExecutionContext context)
+    {
+        var now = clock.GetCurrentInstant();
+
+        logger.LogInformation("Starting ActivityPub delivery retry job");
+
+        try
+        {
+            var failedDeliveries = await db.ActivityPubDeliveries
+                .Where(d => d.Status == DeliveryStatus.Failed && d.NextRetryAt != null && d.NextRetryAt <= now)
+                .OrderBy(d => d.NextRetryAt)
+                .Take(100)
+                .ToListAsync();
+
+            logger.LogInformation("Found {Count} failed deliveries ready for retry", failedDeliveries.Count);
+
+            foreach (var delivery in failedDeliveries)
+            {
+                var message = new ActivityPubDeliveryMessage
+                {
+                    DeliveryId = delivery.Id,
+                    ActivityId = delivery.ActivityId,
+                    ActivityType = delivery.ActivityType,
+                    Activity = new Dictionary<string, object>(),
+                    ActorUri = delivery.ActorUri,
+                    InboxUri = delivery.InboxUri,
+                    CurrentRetry = delivery.RetryCount
+                };
+
+                delivery.Status = DeliveryStatus.Pending;
+                await queueService.EnqueueDeliveryAsync(message);
+                logger.LogDebug("Re-enqueued delivery {DeliveryId} for retry {RetryCount}", delivery.Id, delivery.RetryCount);
+            }
+
+            await db.SaveChangesAsync();
+            logger.LogInformation("Successfully re-enqueued {Count} deliveries for retry", failedDeliveries.Count);
+        }
+        catch (Exception ex)
+        {
+            logger.LogError(ex, "Error executing ActivityPub delivery retry job");
+        }
+    }
+}
+
+public class ActivityPubDeliveryCleanupJob(AppDatabase db, ILogger<ActivityPubDeliveryCleanupJob> logger, IClock clock)
+    : IJob
+{
+    public async Task Execute(IJobExecutionContext context)
+    {
+        var now = clock.GetCurrentInstant();
+        var cutoffDate = now - Duration.FromDays(7);
+
+        logger.LogInformation("Starting ActivityPub delivery cleanup job");
+
+        try
+        {
+            var oldDeliveries = await db.ActivityPubDeliveries
+                .Where(d => 
+                    (d.Status == DeliveryStatus.Sent || d.Status == DeliveryStatus.ExhaustedRetries) &&
+                    d.CreatedAt < cutoffDate)
+                .OrderBy(d => d.CreatedAt)
+                .Take(1000)
+                .ToListAsync();
+
+            if (oldDeliveries.Count > 0)
+            {
+                db.ActivityPubDeliveries.RemoveRange(oldDeliveries);
+                await db.SaveChangesAsync();
+                logger.LogInformation("Cleaned up {Count} old ActivityPub deliveries", oldDeliveries.Count);
+            }
+            else
+            {
+                logger.LogInformation("No old deliveries to clean up");
+            }
+        }
+        catch (Exception ex)
+        {
+            logger.LogError(ex, "Error executing ActivityPub delivery cleanup job");
+        }
+    }
+}

DysonNetwork.Sphere/ActivityPub/ActivityPubDeliveryService.cs

@@ -1,33 +1,21 @@
 using DysonNetwork.Shared.Models;
 using Microsoft.EntityFrameworkCore;
-using System.Security.Cryptography;
-using System.Text;
-using System.Text.Json;
+using NodaTime;
 
 namespace DysonNetwork.Sphere.ActivityPub;
 
 public class ActivityPubDeliveryService(
     AppDatabase db,
-    ActivityPubSignatureService signatureService,
     ActivityPubDiscoveryService discoveryService,
-    IHttpClientFactory httpClientFactory,
+    ActivityPubQueueService queueService,
     IConfiguration configuration,
-    ILogger<ActivityPubDeliveryService> logger
+    ILogger<ActivityPubDeliveryService> logger,
+    IClock clock
 )
 {
     private string Domain => configuration["ActivityPub:Domain"] ?? "localhost";
     private string AssetsBaseUrl => configuration["ActivityPub:FileBaseUrl"] ?? $"https://{Domain}/files";
 
-    private HttpClient HttpClient
-    {
-        get
-        {
-            var client = httpClientFactory.CreateClient();
-            client.DefaultRequestHeaders.Clear();
-            return client;
-        }
-    }
-
     public async Task<bool> SendAcceptActivityAsync(
         SnFediverseActor actor,
         string followerActorUri,
@@ -53,7 +41,7 @@ public class ActivityPubDeliveryService(
             ["object"] = followActivityId
         };
 
-        return await SendActivityToInboxAsync(activity, followerActor.InboxUri, actorUrl);
+        return await EnqueueActivityDeliveryAsync("Accept", activity, actorUrl, followerActor.InboxUri);
     }
 
     public async Task<bool> SendFollowActivityAsync(
@@ -74,10 +62,11 @@ public class ActivityPubDeliveryService(
             return false;
         }
 
+        var activityId = $"{actorUrl}/follows/{Guid.NewGuid()}";
         var activity = new Dictionary<string, object>
         {
             ["@context"] = "https://www.w3.org/ns/activitystreams",
-            ["id"] = $"{actorUrl}/follows/{Guid.NewGuid()}",
+            ["id"] = activityId,
             ["type"] = "Follow",
             ["actor"] = actorUrl,
             ["object"] = targetActorUri
@@ -105,7 +94,7 @@ public class ActivityPubDeliveryService(
 
         await db.SaveChangesAsync();
 
-        return await SendActivityToInboxAsync(activity, targetActor.InboxUri, actorUrl);
+        return await EnqueueActivityDeliveryAsync("Follow", activity, actorUrl, targetActor.InboxUri, activityId);
     }
 
     public async Task<bool> SendUnfollowActivityAsync(
@@ -126,10 +115,11 @@ public class ActivityPubDeliveryService(
             return false;
         }
 
+        var activityId = $"{actorUrl}/undo/{Guid.NewGuid()}";
         var activity = new Dictionary<string, object>
         {
             ["@context"] = "https://www.w3.org/ns/activitystreams",
-            ["id"] = $"{actorUrl}/undo/{Guid.NewGuid()}",
+            ["id"] = activityId,
             ["type"] = "Undo",
             ["actor"] = actorUrl,
             ["object"] = new Dictionary<string, object>
@@ -145,8 +135,7 @@ public class ActivityPubDeliveryService(
                 r.TargetActorId == targetActor.Id);
         if (relationship == null) return false;
 
-        var success = await SendActivityToInboxAsync(activity, targetActor.InboxUri, actorUrl);
-        if (!success) return success;
+        var success = await EnqueueActivityDeliveryAsync("Undo", activity, actorUrl, targetActor.InboxUri, activityId);
         
         db.Remove(relationship);
         await db.SaveChangesAsync();
@@ -164,11 +153,12 @@ public class ActivityPubDeliveryService(
 
         var actorUrl = localActor.Uri;
         var postUrl = $"https://{Domain}/posts/{post.Id}";
+        var activityId = $"{postUrl}/activity";
 
         var activity = new Dictionary<string, object>
         {
             ["@context"] = "https://www.w3.org/ns/activitystreams",
-            ["id"] = $"{postUrl}/activity",
+            ["id"] = activityId,
             ["type"] = "Create",
             ["actor"] = actorUrl,
             ["published"] = (post.PublishedAt ?? post.CreatedAt).ToDateTimeOffset(),
@@ -193,20 +183,15 @@ public class ActivityPubDeliveryService(
         };
 
         var followers = await GetRemoteFollowersAsync();
-        var successCount = 0;
+        logger.LogInformation("Enqueuing Create activity for {Count} followers", followers.Count);
 
         foreach (var follower in followers)
         {
             if (follower.InboxUri == null) continue;
-            var success = await SendActivityToInboxAsync(activity, follower.InboxUri, actorUrl);
-            if (success)
-                successCount++;
+            await EnqueueActivityDeliveryAsync("Create", activity, actorUrl, follower.InboxUri, activityId);
         }
 
-        logger.LogInformation("Sent Create activity to {Count}/{Total} followers",
-            successCount, followers.Count);
-
-        return successCount > 0;
+        return followers.Count > 0;
     }
 
     public async Task<bool> SendUpdateActivityAsync(SnPost post)
@@ -219,11 +204,12 @@ public class ActivityPubDeliveryService(
 
         var actorUrl = localActor.Uri;
         var postUrl = $"https://{Domain}/posts/{post.Id}";
+        var activityId = $"{postUrl}/activity/{Guid.NewGuid()}";
 
         var activity = new Dictionary<string, object>
         {
             ["@context"] = "https://www.w3.org/ns/activitystreams",
-            ["id"] = $"{postUrl}/activity/{Guid.NewGuid()}",
+            ["id"] = activityId,
             ["type"] = "Update",
             ["actor"] = actorUrl,
             ["published"] = (post.PublishedAt ?? post.CreatedAt).ToDateTimeOffset(),
@@ -249,20 +235,15 @@ public class ActivityPubDeliveryService(
         };
 
         var followers = await GetRemoteFollowersAsync();
-        var successCount = 0;
+        logger.LogInformation("Enqueuing Update activity for {Count} followers", followers.Count);
 
         foreach (var follower in followers)
         {
             if (follower.InboxUri == null) continue;
-            var success = await SendActivityToInboxAsync(activity, follower.InboxUri, actorUrl);
-            if (success)
-                successCount++;
+            await EnqueueActivityDeliveryAsync("Update", activity, actorUrl, follower.InboxUri, activityId);
         }
 
-        logger.LogInformation("Sent Update activity to {Count}/{Total} followers",
-            successCount, followers.Count);
-
-        return successCount > 0;
+        return followers.Count > 0;
     }
 
     public async Task<bool> SendDeleteActivityAsync(SnPost post)
@@ -275,11 +256,12 @@ public class ActivityPubDeliveryService(
 
         var actorUrl = localActor.Uri;
         var postUrl = $"https://{Domain}/posts/{post.Id}";
+        var activityId = $"{postUrl}/delete/{Guid.NewGuid()}";
 
         var activity = new Dictionary<string, object>
         {
             ["@context"] = "https://www.w3.org/ns/activitystreams",
-            ["id"] = $"{postUrl}/delete/{Guid.NewGuid()}",
+            ["id"] = activityId,
             ["type"] = "Delete",
             ["actor"] = actorUrl,
             ["to"] = new[] { "https://www.w3.org/ns/activitystreams#Public" },
@@ -292,20 +274,15 @@ public class ActivityPubDeliveryService(
         };
 
         var followers = await GetRemoteFollowersAsync();
-        var successCount = 0;
+        logger.LogInformation("Enqueuing Delete activity for {Count} followers", followers.Count);
 
         foreach (var follower in followers)
         {
             if (follower.InboxUri == null) continue;
-            var success = await SendActivityToInboxAsync(activity, follower.InboxUri, actorUrl);
-            if (success)
-                successCount++;
+            await EnqueueActivityDeliveryAsync("Delete", activity, actorUrl, follower.InboxUri, activityId);
         }
 
-        logger.LogInformation("Sent Delete activity to {Count}/{Total} followers",
-            successCount, followers.Count);
-
-        return successCount > 0;
+        return followers.Count > 0;
     }
 
     public async Task<bool> SendUpdateActorActivityAsync(SnFediverseActor actor)
@@ -359,6 +336,7 @@ public class ActivityPubDeliveryService(
             };
         }
 
+        var activityId = $"{actorUrl}#update-{Guid.NewGuid()}";
         var activity = new Dictionary<string, object>
         {
             ["@context"] = new List<object>
@@ -366,7 +344,7 @@ public class ActivityPubDeliveryService(
                 "https://www.w3.org/ns/activitystreams",
                 "https://w3id.org/security/v1"
             },
-            ["id"] = $"{actorUrl}#update-{Guid.NewGuid()}",
+            ["id"] = activityId,
             ["type"] = "Update",
             ["actor"] = actorUrl,
             ["published"] = DateTimeOffset.UtcNow,
@@ -376,21 +354,15 @@ public class ActivityPubDeliveryService(
         };
 
         var followers = await GetRemoteFollowersAsync(actor.Id);
-
-        var successCount = 0;
+        logger.LogInformation("Enqueuing Update actor activity for {Count} followers", followers.Count);
 
         foreach (var follower in followers)
         {
             if (follower.InboxUri == null) continue;
-            var success = await SendActivityToInboxAsync(activity, follower.InboxUri, actorUrl);
-            if (success)
-                successCount++;
+            await EnqueueActivityDeliveryAsync("Update", activity, actorUrl, follower.InboxUri, activityId);
         }
 
-        logger.LogInformation("Sent Update actor activity to {Count}/{Total} followers",
-            successCount, followers.Count);
-
-        return successCount > 0;
+        return followers.Count > 0;
     }
 
     public async Task<bool> SendLikeActivityToLocalPostAsync(
@@ -418,11 +390,12 @@ public class ActivityPubDeliveryService(
 
         var actorUrl = publisherActor.Uri;
         var postUrl = $"https://{Domain}/posts/{postId}";
+        var activityId = $"{actorUrl}/likes/{Guid.NewGuid()}";
 
         var activity = new Dictionary<string, object>
         {
             ["@context"] = "https://www.w3.org/ns/activitystreams",
-            ["id"] = $"{actorUrl}/likes/{Guid.NewGuid()}",
+            ["id"] = activityId,
             ["type"] = "Like",
             ["actor"] = actor.Uri,
             ["object"] = postUrl,
@@ -432,20 +405,13 @@ public class ActivityPubDeliveryService(
 
         var followers = await GetRemoteFollowersAsync(actor.Id);
 
-        var successCount = 0;
-
         foreach (var follower in followers)
         {
             if (follower.InboxUri == null) continue;
-            var success = await SendActivityToInboxAsync(activity, follower.InboxUri, actorUrl);
-            if (success)
-                successCount++;
+            await EnqueueActivityDeliveryAsync("Like", activity, actorUrl, follower.InboxUri, activityId);
         }
 
-        logger.LogInformation("Sent Like activity for post {PostId} to {Count}/{Total} followers",
-            postId, successCount, followers.Count);
-
-        return successCount > 0;
+        return followers.Count > 0;
     }
 
     public async Task<bool> SendUndoLikeActivityAsync(
@@ -467,11 +433,12 @@ public class ActivityPubDeliveryService(
 
         var actorUrl = localActor.Uri;
         var postUrl = $"https://{Domain}/posts/{postId}";
+        var activityId = $"{actorUrl}/undo/{Guid.NewGuid()}";
 
         var activity = new Dictionary<string, object>
         {
             ["@context"] = "https://www.w3.org/ns/activitystreams",
-            ["id"] = $"{actorUrl}/undo/{Guid.NewGuid()}",
+            ["id"] = activityId,
             ["type"] = "Undo",
             ["actor"] = actorUrl,
             ["object"] = new Dictionary<string, object>
@@ -485,20 +452,13 @@ public class ActivityPubDeliveryService(
 
         var followers = await GetRemoteFollowersAsync(localActor.Id);
 
-        var successCount = 0;
-
         foreach (var follower in followers)
         {
             if (follower.InboxUri == null) continue;
-            var success = await SendActivityToInboxAsync(activity, follower.InboxUri, actorUrl);
-            if (success)
-                successCount++;
+            await EnqueueActivityDeliveryAsync("Undo", activity, actorUrl, follower.InboxUri, activityId);
         }
 
-        logger.LogInformation("Sent Undo Like activity for post {PostId} to {Count}/{Total} followers",
-            postId, successCount, followers.Count);
-
-        return successCount > 0;
+        return followers.Count > 0;
     }
 
     public async Task<bool> SendLikeActivityAsync(
@@ -522,16 +482,17 @@ public class ActivityPubDeliveryService(
         if (targetActor?.InboxUri == null)
             return false;
 
+        var activityId = $"{actorUrl}/likes/{Guid.NewGuid()}";
         var activity = new Dictionary<string, object>
         {
             ["@context"] = "https://www.w3.org/ns/activitystreams",
-            ["id"] = $"{actorUrl}/likes/{Guid.NewGuid()}",
+            ["id"] = activityId,
             ["type"] = "Like",
             ["actor"] = actorUrl,
             ["object"] = postUrl
         };
 
-        return await SendActivityToInboxAsync(activity, targetActor.InboxUri, actorUrl);
+        return await EnqueueActivityDeliveryAsync("Like", activity, actorUrl, targetActor.InboxUri, activityId);
     }
 
     public async Task<bool> SendUndoActivityAsync(
@@ -547,10 +508,11 @@ public class ActivityPubDeliveryService(
         var actorUrl = $"https://{Domain}/activitypub/actors/{publisher.Name}";
         var followers = await GetRemoteFollowersAsync();
 
+        var activityId = $"{actorUrl}/undo/{Guid.NewGuid()}";
         var activity = new Dictionary<string, object>
         {
             ["@context"] = "https://www.w3.org/ns/activitystreams",
-            ["id"] = $"{actorUrl}/undo/{Guid.NewGuid()}",
+            ["id"] = activityId,
             ["type"] = "Undo",
             ["actor"] = actorUrl,
             ["object"] = new Dictionary<string, object>
@@ -560,82 +522,92 @@ public class ActivityPubDeliveryService(
             }
         };
 
-        var successCount = 0;
         foreach (var follower in followers)
         {
             if (follower.InboxUri == null) continue;
-            var success = await SendActivityToInboxAsync(activity, follower.InboxUri, actorUrl);
-            if (success)
-                successCount++;
+            await EnqueueActivityDeliveryAsync("Undo", activity, actorUrl, follower.InboxUri, activityId);
         }
 
-        return successCount > 0;
+        return followers.Count > 0;
     }
 
-    private async Task<bool> SendActivityToInboxAsync(
+    public async Task<List<SnActivityPubDelivery>> GetDeliveriesByActivityIdAsync(string activityId)
+    {
+        return await db.ActivityPubDeliveries
+            .Where(d => d.ActivityId == activityId)
+            .OrderBy(d => d.CreatedAt)
+            .ToListAsync();
+    }
+
+    public async Task<DeliveryStats> GetDeliveryStatsAsync(DateTimeOffset from, DateTimeOffset to)
+    {
+        var fromInstant = Instant.FromDateTimeOffset(from);
+        var toInstant = Instant.FromDateTimeOffset(to);
+
+        var stats = new DeliveryStats
+        {
+            From = from,
+            To = to
+        };
+
+        var deliveries = await db.ActivityPubDeliveries
+            .Where(d => d.CreatedAt >= fromInstant && d.CreatedAt <= toInstant)
+            .ToListAsync();
+
+        stats.TotalDeliveries = deliveries.Count;
+        stats.SentDeliveries = deliveries.Count(d => d.Status == DeliveryStatus.Sent);
+        stats.FailedDeliveries = deliveries.Count(d => d.Status == DeliveryStatus.Failed || d.Status == DeliveryStatus.ExhaustedRetries);
+        stats.PendingDeliveries = deliveries.Count(d => d.Status == DeliveryStatus.Pending || d.Status == DeliveryStatus.Processing);
+
+        return stats;
+    }
+
+    private async Task<bool> EnqueueActivityDeliveryAsync(
+        string activityType,
         Dictionary<string, object> activity,
-        string inboxUrl,
-        string actorUri
+        string actorUri,
+        string inboxUri,
+        string? activityId = null
     )
     {
         try
         {
-            var json = JsonSerializer.Serialize(activity);
-            var request = new HttpRequestMessage(HttpMethod.Post, inboxUrl);
+            activityId ??= activity.ContainsKey("id") ? activity["id"].ToString() : Guid.NewGuid().ToString();
 
-            request.Content = new StringContent(json, Encoding.UTF8, "application/activity+json");
-            request.Headers.Date = DateTimeOffset.UtcNow;
-
-            var bodyBytes = Encoding.UTF8.GetBytes(json);
-            var hash = SHA256.HashData(bodyBytes);
-            var digest = $"SHA-256={Convert.ToBase64String(hash)}";
-            request.Headers.Add("Digest", digest);
-            request.Headers.Host = new Uri(inboxUrl).Host;
-
-            logger.LogInformation("Preparing request to {Inbox}", inboxUrl);
-            logger.LogInformation("Request body (truncated): {Body}", json[..Math.Min(200, json.Length)] + "...");
-            logger.LogInformation("Request headers before signing: Date={Date}, Digest={Digest}, Host={Host}",
-                request.Headers.Date, digest, request.Headers.Host);
-
-            var signatureHeaders = await signatureService.SignOutgoingRequest(request, actorUri);
-
-            var signatureString = $"keyId=\"{signatureHeaders["keyId"]}\"," +
-                                  $"algorithm=\"{signatureHeaders["algorithm"]}\"," +
-                                  $"headers=\"{signatureHeaders["headers"]}\"," +
-                                  $"signature=\"{signatureHeaders["signature"]}\"";
-
-            request.Headers.Add("Signature", signatureString);
-
-            logger.LogInformation("Full signature header: {Signature}", signatureString);
-            logger.LogInformation("Request headers after signing:");
-            foreach (var header in request.Headers)
+            var delivery = new SnActivityPubDelivery
             {
-                var value = header.Value.Any() ? header.Value.First() : string.Empty;
-                if (header.Key == "signature")
-                    value = value[..Math.Min(100, value.Length)] + "...";
-                logger.LogInformation("  {Key}: {Value}", header.Key, value);
-            }
+                ActivityId = activityId,
+                ActivityType = activityType,
+                InboxUri = inboxUri,
+                ActorUri = actorUri,
+                Status = DeliveryStatus.Pending,
+                RetryCount = 0
+            };
 
-            var response = await HttpClient.SendAsync(request);
-            var responseContent = await response.Content.ReadAsStringAsync();
+            db.ActivityPubDeliveries.Add(delivery);
+            await db.SaveChangesAsync();
 
-            logger.LogInformation("Response from {Inbox}. Status: {Status}", inboxUrl, response.StatusCode);
-
-            if (!response.IsSuccessStatusCode)
+            var message = new ActivityPubDeliveryMessage
             {
-                logger.LogError("Failed to send activity to {Inbox}. Status: {Status}, Response: {Response}",
-                    inboxUrl, response.StatusCode, responseContent);
-                logger.LogError("Full request details: Method={Method}, Uri={Uri}, ContentType={ContentType}",
-                    request.Method, request.RequestUri, request.Content?.Headers.ContentType);
-                return false;
-            }
+                DeliveryId = delivery.Id,
+                ActivityId = activityId,
+                ActivityType = activityType,
+                Activity = activity,
+                ActorUri = actorUri,
+                InboxUri = inboxUri,
+                CurrentRetry = 0
+            };
+
+            await queueService.EnqueueDeliveryAsync(message);
+
+            logger.LogDebug("Enqueued delivery {DeliveryId} of type {ActivityType} to {Inbox}",
+                delivery.Id, activityType, inboxUri);
 
-            logger.LogInformation("Successfully sent activity to {Inbox}", inboxUrl);
             return true;
         }
         catch (Exception ex)
         {
-            logger.LogError(ex, "Error sending activity to {Inbox}. Exception: {Message}", inboxUrl, ex.Message);
+            logger.LogError(ex, "Failed to enqueue delivery to {Inbox}", inboxUri);
             return false;
         }
     }
@@ -749,7 +721,7 @@ public class ActivityPubDeliveryService(
                 Uri = actorUri,
                 Username = ExtractUsername(actorUri),
                 InstanceId = instance.Id,
-                LastFetchedAt = NodaTime.SystemClock.Instance.GetCurrentInstant()
+                LastFetchedAt = clock.GetCurrentInstant()
             };
 
             db.FediverseActors.Add(actor);
@@ -772,4 +744,14 @@ public class ActivityPubDeliveryService(
     {
         return actorUri.Split('/').Last();
     }
-}
+}
+
+public class DeliveryStats
+{
+    public DateTimeOffset From { get; set; }
+    public DateTimeOffset To { get; set; }
+    public int TotalDeliveries { get; set; }
+    public int SentDeliveries { get; set; }
+    public int FailedDeliveries { get; set; }
+    public int PendingDeliveries { get; set; }
+}

DysonNetwork.Sphere/ActivityPub/ActivityPubDeliveryWorker.cs

@@ -0,0 +1,215 @@
+using System.Net;
+using System.Security.Cryptography;
+using System.Text;
+using System.Text.Json;
+using DysonNetwork.Shared.Models;
+using DysonNetwork.Shared.Proto;
+using Google.Protobuf;
+using Microsoft.Extensions.Options;
+using NATS.Client.Core;
+using NodaTime;
+
+namespace DysonNetwork.Sphere.ActivityPub;
+
+public class ActivityPubDeliveryWorker(
+    INatsConnection nats,
+    IServiceProvider serviceProvider,
+    IHttpClientFactory httpClientFactory,
+    IOptions<ActivityPubDeliveryOptions> options,
+    ILogger<ActivityPubDeliveryWorker> logger,
+    IClock clock
+) : BackgroundService
+{
+    public const string QueueName = "activitypub_delivery_queue";
+    private const string QueueGroup = "activitypub_delivery_workers";
+    private readonly List<Task> _consumerTasks = [];
+
+    protected override async Task ExecuteAsync(CancellationToken stoppingToken)
+    {
+        logger.LogInformation("Starting {ConsumerCount} ActivityPub delivery consumers", options.Value.ConsumerCount);
+
+        for (var i = 0; i < options.Value.ConsumerCount; i++)
+            _consumerTasks.Add(Task.Run(() => RunConsumerAsync(stoppingToken), stoppingToken));
+
+        await Task.WhenAll(_consumerTasks);
+    }
+
+    private async Task RunConsumerAsync(CancellationToken stoppingToken)
+    {
+        logger.LogInformation("ActivityPub delivery consumer started");
+
+        await foreach (var msg in nats.SubscribeAsync<byte[]>(QueueName, queueGroup: QueueGroup, cancellationToken: stoppingToken))
+        {
+            try
+            {
+                var message = GrpcTypeHelper.ConvertByteStringToObject<ActivityPubDeliveryMessage>(ByteString.CopyFrom(msg.Data));
+                if (message is not null)
+                {
+                    await ProcessDeliveryAsync(message, stoppingToken);
+                }
+                else
+                {
+                    logger.LogWarning("Invalid message format for ActivityPub delivery");
+                }
+            }
+            catch (OperationCanceledException) when (stoppingToken.IsCancellationRequested)
+            {
+                break;
+            }
+            catch (Exception ex)
+            {
+                logger.LogError(ex, "Error in ActivityPub delivery consumer");
+            }
+        }
+    }
+
+    private async Task ProcessDeliveryAsync(ActivityPubDeliveryMessage message, CancellationToken cancellationToken)
+    {
+        using var scope = serviceProvider.CreateScope();
+        var db = scope.ServiceProvider.GetRequiredService<AppDatabase>();
+        var signatureService = scope.ServiceProvider.GetRequiredService<ActivityPubSignatureService>();
+
+        logger.LogDebug("Processing ActivityPub delivery {DeliveryId} to {Inbox}", message.DeliveryId, message.InboxUri);
+
+        var delivery = await db.ActivityPubDeliveries.FindAsync([message.DeliveryId], cancellationToken);
+        if (delivery == null)
+        {
+            logger.LogWarning("Delivery record not found: {DeliveryId}", message.DeliveryId);
+            return;
+        }
+
+        delivery.Status = DeliveryStatus.Processing;
+        delivery.LastAttemptAt = clock.GetCurrentInstant();
+        await db.SaveChangesAsync(cancellationToken);
+
+        try
+        {
+            var success = await SendActivityToInboxAsync(
+                message.Activity,
+                message.InboxUri,
+                message.ActorUri,
+                signatureService,
+                httpClientFactory,
+                logger,
+                cancellationToken
+            );
+
+            if (success.IsSuccessStatusCode)
+            {
+                delivery.Status = DeliveryStatus.Sent;
+                delivery.SentAt = clock.GetCurrentInstant();
+                delivery.ResponseStatusCode = success.StatusCode.ToString();
+                logger.LogInformation("Successfully delivered activity {ActivityId} to {Inbox}",
+                    message.ActivityId, message.InboxUri);
+            }
+            else
+            {
+                var shouldRetry = ShouldRetry(success.StatusCode);
+                delivery.ResponseStatusCode = success.StatusCode.ToString();
+                delivery.ErrorMessage = await success.Content.ReadAsStringAsync(cancellationToken);
+
+                if (shouldRetry && delivery.RetryCount < options.Value.MaxRetries)
+                {
+                    delivery.Status = DeliveryStatus.Failed;
+                    delivery.RetryCount++;
+                    delivery.NextRetryAt = CalculateNextRetryAt(delivery.RetryCount, clock);
+                    logger.LogWarning("Failed to deliver activity {ActivityId} to {Inbox}. Status: {Status}. Retry {RetryCount}/{MaxRetries} at {NextRetry}",
+                        message.ActivityId, message.InboxUri, success.StatusCode, delivery.RetryCount, options.Value.MaxRetries, delivery.NextRetryAt);
+                }
+                else
+                {
+                    delivery.Status = DeliveryStatus.ExhaustedRetries;
+                    logger.LogError("Exhausted retries for activity {ActivityId} to {Inbox}. Status: {Status}",
+                        message.ActivityId, message.InboxUri, success.StatusCode);
+                }
+            }
+        }
+        catch (Exception ex)
+        {
+            delivery.Status = DeliveryStatus.Failed;
+            delivery.ErrorMessage = ex.Message;
+
+            if (delivery.RetryCount < options.Value.MaxRetries)
+            {
+                delivery.RetryCount++;
+                delivery.NextRetryAt = CalculateNextRetryAt(delivery.RetryCount, clock);
+                logger.LogError(ex, "Error delivering activity {ActivityId} to {Inbox}. Retry {RetryCount}/{MaxRetries} at {NextRetry}",
+                    message.ActivityId, message.InboxUri, delivery.RetryCount, options.Value.MaxRetries, delivery.NextRetryAt);
+            }
+            else
+            {
+                delivery.Status = DeliveryStatus.ExhaustedRetries;
+                logger.LogError(ex, "Exhausted retries for activity {ActivityId} to {Inbox}",
+                    message.ActivityId, message.InboxUri);
+            }
+        }
+
+        await db.SaveChangesAsync(cancellationToken);
+    }
+
+    private static async Task<HttpResponseMessage> SendActivityToInboxAsync(
+        Dictionary<string, object> activity,
+        string inboxUrl,
+        string actorUri,
+        ActivityPubSignatureService signatureService,
+        IHttpClientFactory httpClientFactory,
+        ILogger logger,
+        CancellationToken cancellationToken)
+    {
+        var client = httpClientFactory.CreateClient();
+        var json = JsonSerializer.Serialize(activity);
+        var request = new HttpRequestMessage(HttpMethod.Post, inboxUrl)
+        {
+            Content = new StringContent(json, Encoding.UTF8, "application/activity+json")
+        };
+
+        request.Headers.Date = DateTimeOffset.UtcNow;
+
+        var bodyBytes = Encoding.UTF8.GetBytes(json);
+        var hash = SHA256.HashData(bodyBytes);
+        var digest = $"SHA-256={Convert.ToBase64String(hash)}";
+        request.Headers.Add("Digest", digest);
+        request.Headers.Host = new Uri(inboxUrl).Host;
+
+        logger.LogDebug("Sending request to {Inbox}", inboxUrl);
+
+        var signatureHeaders = await signatureService.SignOutgoingRequest(request, actorUri);
+
+        var signatureString = $"keyId=\"{signatureHeaders["keyId"]}\"," +
+                              $"algorithm=\"{signatureHeaders["algorithm"]}\"," +
+                              $"headers=\"{signatureHeaders["headers"]}\"," +
+                              $"signature=\"{signatureHeaders["signature"]}\"";
+
+        request.Headers.Add("Signature", signatureString);
+
+        var response = await client.SendAsync(request, cancellationToken);
+        logger.LogDebug("Response from {Inbox}. Status: {Status}", inboxUrl, response.StatusCode);
+
+        return response;
+    }
+
+    private static bool ShouldRetry(HttpStatusCode statusCode)
+    {
+        return statusCode == HttpStatusCode.InternalServerError ||
+               statusCode == HttpStatusCode.BadGateway ||
+               statusCode == HttpStatusCode.ServiceUnavailable ||
+               statusCode == HttpStatusCode.GatewayTimeout ||
+               statusCode == HttpStatusCode.RequestTimeout ||
+               statusCode == (HttpStatusCode)429; // Too Many Requests
+    }
+
+    private static Instant CalculateNextRetryAt(int retryCount, IClock clock)
+    {
+        var baseDelaySeconds = 1;
+        var maxDelaySeconds = 300;
+        var delaySeconds = Math.Min(maxDelaySeconds, baseDelaySeconds * (int)Math.Pow(2, retryCount - 1));
+        return clock.GetCurrentInstant() + Duration.FromSeconds(delaySeconds);
+    }
+}
+
+public class ActivityPubDeliveryOptions
+{
+    public const string SectionName = "ActivityPubDelivery";
+    public int MaxRetries { get; set; } = 5;
+    public int ConsumerCount { get; set; } = 4;
+}

DysonNetwork.Sphere/ActivityPub/ActivityPubQueueService.cs

@@ -0,0 +1,25 @@
+using System.Text.Json;
+using DysonNetwork.Shared.Proto;
+using NATS.Client.Core;
+
+namespace DysonNetwork.Sphere.ActivityPub;
+
+public class ActivityPubQueueService(INatsConnection nats)
+{
+    public async Task EnqueueDeliveryAsync(ActivityPubDeliveryMessage message)
+    {
+        var rawMessage = GrpcTypeHelper.ConvertObjectToByteString(message).ToByteArray();
+        await nats.PublishAsync(ActivityPubDeliveryWorker.QueueName, rawMessage);
+    }
+}
+
+public class ActivityPubDeliveryMessage
+{
+    public Guid DeliveryId { get; set; }
+    public string ActivityId { get; set; } = string.Empty;
+    public string ActivityType { get; set; } = string.Empty;
+    public Dictionary<string, object> Activity { get; set; } = [];
+    public string ActorUri { get; set; } = string.Empty;
+    public string InboxUri { get; set; } = string.Empty;
+    public int CurrentRetry { get; set; } = 0;
+}

DysonNetwork.Sphere/ActivityPub/ActivityPubSignatureService.cs

@@ -197,7 +197,7 @@ public class ActivityPubSignatureService(
         return (privateKeyPem, publicKeyPem);
     }
 
-    private string? GetPublisherKey(SnPublisher publisher, string keyName)
+    private static string? GetPublisherKey(SnPublisher publisher, string keyName)
     {
         return keyName switch
         {

DysonNetwork.Sphere/AppDatabase.cs

@@ -51,6 +51,7 @@ public class AppDatabase(
     public DbSet<SnFediverseInstance> FediverseInstances { get; set; } = null!;
     public DbSet<SnFediverseActor> FediverseActors { get; set; } = null!;
     public DbSet<SnFediverseRelationship> FediverseRelationships { get; set; } = null!;
+    public DbSet<SnActivityPubDelivery> ActivityPubDeliveries { get; set; } = null!;
 
     public DbSet<WebArticle> WebArticles { get; set; } = null!;
     public DbSet<WebFeed> WebFeeds { get; set; } = null!;

DysonNetwork.Sphere/Migrations/20251231163256_AddActivityPubDelivery.cs

@@ -0,0 +1,48 @@
+using System;
+using Microsoft.EntityFrameworkCore.Migrations;
+using NodaTime;
+
+#nullable disable
+
+namespace DysonNetwork.Sphere.Migrations
+{
+    /// <inheritdoc />
+    public partial class AddActivityPubDelivery : Migration
+    {
+        /// <inheritdoc />
+        protected override void Up(MigrationBuilder migrationBuilder)
+        {
+            migrationBuilder.CreateTable(
+                name: "activity_pub_deliveries",
+                columns: table => new
+                {
+                    id = table.Column<Guid>(type: "uuid", nullable: false),
+                    activity_id = table.Column<string>(type: "character varying(2048)", maxLength: 2048, nullable: false),
+                    activity_type = table.Column<string>(type: "character varying(128)", maxLength: 128, nullable: false),
+                    inbox_uri = table.Column<string>(type: "character varying(2048)", maxLength: 2048, nullable: false),
+                    actor_uri = table.Column<string>(type: "character varying(2048)", maxLength: 2048, nullable: false),
+                    status = table.Column<int>(type: "integer", nullable: false),
+                    retry_count = table.Column<int>(type: "integer", nullable: false),
+                    error_message = table.Column<string>(type: "character varying(4096)", maxLength: 4096, nullable: true),
+                    last_attempt_at = table.Column<Instant>(type: "timestamp with time zone", nullable: true),
+                    next_retry_at = table.Column<Instant>(type: "timestamp with time zone", nullable: true),
+                    sent_at = table.Column<Instant>(type: "timestamp with time zone", nullable: true),
+                    response_status_code = table.Column<string>(type: "character varying(2048)", maxLength: 2048, nullable: true),
+                    created_at = table.Column<Instant>(type: "timestamp with time zone", nullable: false),
+                    updated_at = table.Column<Instant>(type: "timestamp with time zone", nullable: false),
+                    deleted_at = table.Column<Instant>(type: "timestamp with time zone", nullable: true)
+                },
+                constraints: table =>
+                {
+                    table.PrimaryKey("pk_activity_pub_deliveries", x => x.id);
+                });
+        }
+
+        /// <inheritdoc />
+        protected override void Down(MigrationBuilder migrationBuilder)
+        {
+            migrationBuilder.DropTable(
+                name: "activity_pub_deliveries");
+        }
+    }
+}

DysonNetwork.Sphere/Migrations/AppDatabaseModelSnapshot.cs

@@ -27,6 +27,85 @@ namespace DysonNetwork.Sphere.Migrations
 
             NpgsqlModelBuilderExtensions.UseIdentityByDefaultColumns(modelBuilder);
 
+            modelBuilder.Entity("DysonNetwork.Shared.Models.SnActivityPubDelivery", b =>
+                {
+                    b.Property<Guid>("Id")
+                        .ValueGeneratedOnAdd()
+                        .HasColumnType("uuid")
+                        .HasColumnName("id");
+
+                    b.Property<string>("ActivityId")
+                        .IsRequired()
+                        .HasMaxLength(2048)
+                        .HasColumnType("character varying(2048)")
+                        .HasColumnName("activity_id");
+
+                    b.Property<string>("ActivityType")
+                        .IsRequired()
+                        .HasMaxLength(128)
+                        .HasColumnType("character varying(128)")
+                        .HasColumnName("activity_type");
+
+                    b.Property<string>("ActorUri")
+                        .IsRequired()
+                        .HasMaxLength(2048)
+                        .HasColumnType("character varying(2048)")
+                        .HasColumnName("actor_uri");
+
+                    b.Property<Instant>("CreatedAt")
+                        .HasColumnType("timestamp with time zone")
+                        .HasColumnName("created_at");
+
+                    b.Property<Instant?>("DeletedAt")
+                        .HasColumnType("timestamp with time zone")
+                        .HasColumnName("deleted_at");
+
+                    b.Property<string>("ErrorMessage")
+                        .HasMaxLength(4096)
+                        .HasColumnType("character varying(4096)")
+                        .HasColumnName("error_message");
+
+                    b.Property<string>("InboxUri")
+                        .IsRequired()
+                        .HasMaxLength(2048)
+                        .HasColumnType("character varying(2048)")
+                        .HasColumnName("inbox_uri");
+
+                    b.Property<Instant?>("LastAttemptAt")
+                        .HasColumnType("timestamp with time zone")
+                        .HasColumnName("last_attempt_at");
+
+                    b.Property<Instant?>("NextRetryAt")
+                        .HasColumnType("timestamp with time zone")
+                        .HasColumnName("next_retry_at");
+
+                    b.Property<string>("ResponseStatusCode")
+                        .HasMaxLength(2048)
+                        .HasColumnType("character varying(2048)")
+                        .HasColumnName("response_status_code");
+
+                    b.Property<int>("RetryCount")
+                        .HasColumnType("integer")
+                        .HasColumnName("retry_count");
+
+                    b.Property<Instant?>("SentAt")
+                        .HasColumnType("timestamp with time zone")
+                        .HasColumnName("sent_at");
+
+                    b.Property<int>("Status")
+                        .HasColumnType("integer")
+                        .HasColumnName("status");
+
+                    b.Property<Instant>("UpdatedAt")
+                        .HasColumnType("timestamp with time zone")
+                        .HasColumnName("updated_at");
+
+                    b.HasKey("Id")
+                        .HasName("pk_activity_pub_deliveries");
+
+                    b.ToTable("activity_pub_deliveries", (string)null);
+                });
+
             modelBuilder.Entity("DysonNetwork.Shared.Models.SnChatMember", b =>
                 {
                     b.Property<Guid>("Id")

DysonNetwork.Sphere/Startup/ScheduledJobsConfiguration.cs

@@ -1,3 +1,4 @@
+using DysonNetwork.Sphere.ActivityPub;
 using DysonNetwork.Sphere.Post;
 using DysonNetwork.Sphere.Publisher;
 using DysonNetwork.Sphere.WebReader;
@@ -39,6 +40,22 @@ public static class ScheduledJobsConfiguration
                 .WithIdentity("PublisherSettlementTrigger")
                 .WithCronSchedule("0 0 0 * * ?")
             );
+
+            q.AddJob<ActivityPubDeliveryRetryJob>(opts => opts.WithIdentity("ActivityPubDeliveryRetry"));
+            q.AddTrigger(opts => opts
+                .ForJob("ActivityPubDeliveryRetry")
+                .WithIdentity("ActivityPubDeliveryRetryTrigger")
+                .WithSimpleSchedule(o => o
+                    .WithIntervalInMinutes(1)
+                    .RepeatForever())
+            );
+
+            q.AddJob<ActivityPubDeliveryCleanupJob>(opts => opts.WithIdentity("ActivityPubDeliveryCleanup"));
+            q.AddTrigger(opts => opts
+                .ForJob("ActivityPubDeliveryCleanup")
+                .WithIdentity("ActivityPubDeliveryCleanupTrigger")
+                .WithCronSchedule("0 0 3 * * ?")
+            );
         });
         services.AddQuartzHostedService(q => q.WaitForJobsToComplete = true);
 

DysonNetwork.Sphere/Startup/ServiceCollectionExtensions.cs

@@ -65,6 +65,7 @@ public static class ServiceCollectionExtensions
         });
 
         services.AddHostedService<BroadcastEventHandler>();
+        services.AddHostedService<ActivityPubDeliveryWorker>();
 
         return services;
     }
@@ -89,6 +90,7 @@ public static class ServiceCollectionExtensions
     )
     {
         services.Configure<GeoOptions>(configuration.GetSection("GeoIP"));
+        services.Configure<ActivityPubDeliveryOptions>(configuration.GetSection("ActivityPubDelivery"));
         services.AddScoped<GeoService>();
         services.AddScoped<PublisherService>();
         services.AddScoped<PublisherSubscriptionService>();
@@ -108,6 +110,7 @@ public static class ServiceCollectionExtensions
         services.AddScoped<ActivityPubActivityHandler>();
         services.AddScoped<ActivityPubDeliveryService>();
         services.AddScoped<ActivityPubDiscoveryService>();
+        services.AddSingleton<ActivityPubQueueService>();
         services.AddScoped<ActivityPubFollowController>();
         services.AddScoped<ActivityPubController>();
 

DysonNetwork.Sphere/appsettings.json

@@ -40,5 +40,9 @@
     "Service": {
         "Name": "DysonNetwork.Sphere",
         "Url": "https://localhost:7099"
+    },
+    "ActivityPubDelivery": {
+        "MaxRetries": 5,
+        "ConsumerCount": 4
     }
 }