✨ Web articles

DysonNetwork.Sphere/Activity/ActivityService.cs

@@ -1,4 +1,5 @@
 using DysonNetwork.Sphere.Account;
+using DysonNetwork.Sphere.Connection.WebReader;
 using DysonNetwork.Sphere.Discovery;
 using DysonNetwork.Sphere.Post;
 using DysonNetwork.Sphere.Publisher;
@@ -39,6 +40,31 @@ public class ActivityService(
             }
         }
 
+        if (debugInclude.Contains("articles") || Random.Shared.NextDouble() < 0.2)
+        {
+            var recentArticlesQuery = db.WebArticles
+                .Take(20); // Get a larger pool for randomization
+
+            // Apply random ordering 50% of the time
+            if (Random.Shared.NextDouble() < 0.5)
+            {
+                recentArticlesQuery = recentArticlesQuery.OrderBy(_ => EF.Functions.Random());
+            }
+            else
+            {
+                recentArticlesQuery = recentArticlesQuery.OrderByDescending(a => a.PublishedAt);
+            }
+
+            var recentArticles = await recentArticlesQuery.Take(5).ToListAsync();
+
+            if (recentArticles.Count > 0)
+            {
+                activities.Add(new DiscoveryActivity(
+                    recentArticles.Select(x => new DiscoveryItem("article", x)).ToList()
+                ).ToActivity());
+            }
+        }
+
         // Fetch a larger batch of recent posts to rank
         var postsQuery = db.Posts
             .Include(e => e.RepliedPost)
@@ -115,6 +141,27 @@ public class ActivityService(
                     ).ToActivity());
                 }
             }
+            
+            if (debugInclude.Contains("articles") || Random.Shared.NextDouble() < 0.2)
+            {
+                var recentArticlesQuery = db.WebArticles
+                    .Take(20); // Get a larger pool for randomization
+
+                // Apply random ordering 50% of the time
+                if (Random.Shared.NextDouble() < 0.5)
+                    recentArticlesQuery = recentArticlesQuery.OrderBy(_ => EF.Functions.Random());
+                else
+                    recentArticlesQuery = recentArticlesQuery.OrderByDescending(a => a.PublishedAt);
+
+                var recentArticles = await recentArticlesQuery.Take(5).ToListAsync();
+
+                if (recentArticles.Count > 0)
+                {
+                    activities.Add(new DiscoveryActivity(
+                        recentArticles.Select(x => new DiscoveryItem("article", x)).ToList()
+                    ).ToActivity());
+                }
+            }
         }
 
         // Get publishers based on filter

DysonNetwork.Sphere/Connection/WebReader/WebArticleController.cs

@@ -0,0 +1,82 @@
+using Microsoft.AspNetCore.Mvc;
+using Microsoft.EntityFrameworkCore;
+
+namespace DysonNetwork.Sphere.Connection.WebReader;
+
+[ApiController]
+[Route("/feeds/articles")]
+public class WebArticleController(AppDatabase db) : ControllerBase
+{
+    /// <summary>
+    /// Get a list of recent web articles
+    /// </summary>
+    /// <param name="limit">Maximum number of articles to return</param>
+    /// <param name="offset">Number of articles to skip</param>
+    /// <param name="feedId">Optional feed ID to filter by</param>
+    /// <param name="publisherId">Optional publisher ID to filter by</param>
+    /// <returns>List of web articles</returns>
+    [HttpGet]
+    public async Task<IActionResult> GetArticles(
+        [FromQuery] int limit = 20,
+        [FromQuery] int offset = 0,
+        [FromQuery] Guid? feedId = null,
+        [FromQuery] Guid? publisherId = null
+    )
+    {
+        var query = db.WebArticles
+            .OrderByDescending(a => a.PublishedAt)
+            .Include(a => a.Feed)
+            .AsQueryable();
+
+        if (feedId.HasValue)
+            query = query.Where(a => a.FeedId == feedId.Value);
+        if (publisherId.HasValue)
+            query = query.Where(a => a.Feed.PublisherId == publisherId.Value);
+
+        var totalCount = await query.CountAsync();
+        var articles = await query
+            .Skip(offset)
+            .Take(limit)
+            .ToListAsync();
+        
+        Response.Headers["X-Total"] = totalCount.ToString();
+
+        return Ok(articles);
+    }
+
+    /// <summary>
+    /// Get a specific web article by ID
+    /// </summary>
+    /// <param name="id">The article ID</param>
+    /// <returns>The web article</returns>
+    [HttpGet("{id:guid}")]
+    [ProducesResponseType(404)]
+    public async Task<IActionResult> GetArticle(Guid id)
+    {
+        var article = await db.WebArticles
+            .Include(a => a.Feed)
+            .FirstOrDefaultAsync(a => a.Id == id);
+
+        if (article == null)
+            return NotFound();
+        
+        return Ok(article);
+    }
+
+    /// <summary>
+    /// Get random web articles
+    /// </summary>
+    /// <param name="limit">Maximum number of articles to return</param>
+    /// <returns>List of random web articles</returns>
+    [HttpGet("random")]
+    public async Task<IActionResult> GetRandomArticles([FromQuery] int limit = 5)
+    {
+        var articles = await db.WebArticles
+            .OrderBy(_ => EF.Functions.Random())
+            .Include(a => a.Feed)
+            .Take(limit)
+            .ToListAsync();
+
+        return Ok(articles);
+    }
+}

DysonNetwork.Sphere/Connection/WebReader/WebFeedService.cs

@@ -112,6 +112,7 @@ public class WebFeedService(
             {
                 var scrapedArticle = await webReaderService.ScrapeArticleAsync(itemUrl, cancellationToken);
                 preview = scrapedArticle.LinkEmbed;
+                if (scrapedArticle.Content is not null)
                     content = scrapedArticle.Content;
             }
             else