✨ Web articles and feed

DysonNetwork.Sphere/AppDatabase.cs

@@ -95,6 +95,8 @@ public class AppDatabase(
 
     public DbSet<Subscription> WalletSubscriptions { get; set; }
     public DbSet<Coupon> WalletCoupons { get; set; }
+    public DbSet<Connection.WebReader.WebArticle> WebArticles { get; set; }
+    public DbSet<Connection.WebReader.WebFeed> WebFeeds { get; set; }
 
     protected override void OnConfiguring(DbContextOptionsBuilder optionsBuilder)
     {
@@ -137,6 +139,8 @@ public class AppDatabase(
             }
         });
 
+        optionsBuilder.UseSeeding((context, _) => {});
+
         base.OnConfiguring(optionsBuilder);
     }
 
@@ -261,6 +265,14 @@ public class AppDatabase(
             .HasForeignKey(m => m.SenderId)
             .OnDelete(DeleteBehavior.Cascade);
 
+        modelBuilder.Entity<Connection.WebReader.WebFeed>()
+            .HasIndex(f => f.Url)
+            .IsUnique();
+
+        modelBuilder.Entity<Connection.WebReader.WebArticle>()
+            .HasIndex(a => a.Url)
+            .IsUnique();
+
         // Automatically apply soft-delete filter to all entities inheriting BaseModel
         foreach (var entityType in modelBuilder.Model.GetEntityTypes())
         {

DysonNetwork.Sphere/Connection/WebReader/WebArticle.cs

@@ -0,0 +1,39 @@
+using System.ComponentModel.DataAnnotations;
+using System.ComponentModel.DataAnnotations.Schema;
+
+namespace DysonNetwork.Sphere.Connection.WebReader;
+
+public class WebArticle : ModelBase
+{
+    public Guid Id { get; set; } = Guid.NewGuid();
+
+    [MaxLength(4096)] public string Title { get; set; }
+    [MaxLength(8192)] public string Url { get; set; }
+    [MaxLength(4096)] public string? Author { get; set; }
+    
+    [Column(TypeName = "jsonb")] public Dictionary<string, object>? Meta { get; set; }
+    [Column(TypeName = "jsonb")] public LinkEmbed? Preview { get; set; }
+
+    // ReSharper disable once EntityFramework.ModelValidation.UnlimitedStringLength
+    public string? Content { get; set; }
+
+    public DateTime? PublishedAt { get; set; }
+
+    public Guid FeedId { get; set; }
+    public WebFeed Feed { get; set; } = null!;
+}
+
+public class WebFeed : ModelBase
+{
+    public Guid Id { get; set; } = Guid.NewGuid();
+    [MaxLength(8192)] public string Url { get; set; }
+    [MaxLength(4096)] public string Title { get; set; }
+    [MaxLength(8192)] public string? Description { get; set; }
+    
+    [Column(TypeName = "jsonb")] public LinkEmbed? Preview { get; set; }
+
+    public Guid PublisherId { get; set; }
+    public Publisher.Publisher Publisher { get; set; } = null!;
+
+    public ICollection<WebArticle> Articles { get; set; } = new List<WebArticle>();
+}

DysonNetwork.Sphere/Connection/WebReader/WebFeedController.cs

@@ -0,0 +1,33 @@
+using System.ComponentModel.DataAnnotations;
+using Microsoft.AspNetCore.Authorization;
+using Microsoft.AspNetCore.Mvc;
+
+namespace DysonNetwork.Sphere.Connection.WebReader;
+
+[Authorize]
+[ApiController]
+[Route("feeds")]
+public class WebFeedController(WebFeedService webFeedService) : ControllerBase
+{
+    public class CreateWebFeedRequest
+    {
+        [Required]
+        [MaxLength(8192)]
+        public required string Url { get; set; }
+
+        [Required]
+        [MaxLength(4096)]
+        public required string Title { get; set; }
+
+        [MaxLength(8192)]
+        public string? Description { get; set; }
+    }
+
+    
+    [HttpPost]
+    public async Task<IActionResult> CreateWebFeed([FromBody] CreateWebFeedRequest request)
+    {
+        var feed = await webFeedService.CreateWebFeedAsync(request, User);
+        return Ok(feed);
+    }
+}

DysonNetwork.Sphere/Connection/WebReader/WebFeedScraperJob.cs

@@ -0,0 +1,35 @@
+using Microsoft.EntityFrameworkCore;
+using Microsoft.Extensions.Logging;
+using Quartz;
+
+namespace DysonNetwork.Sphere.Connection.WebReader;
+
+[DisallowConcurrentExecution]
+public class WebFeedScraperJob(
+    AppDatabase database,
+    WebFeedService webFeedService,
+    ILogger<WebFeedScraperJob> logger
+)
+    : IJob
+{
+    public async Task Execute(IJobExecutionContext context)
+    {
+        logger.LogInformation("Starting web feed scraper job.");
+
+        var feeds = await database.Set<WebFeed>().ToListAsync(context.CancellationToken);
+
+        foreach (var feed in feeds)
+        {
+            try
+            {
+                await webFeedService.ScrapeFeedAsync(feed, context.CancellationToken);
+            }
+            catch (Exception ex)
+            {
+                logger.LogError(ex, "Failed to scrape web feed {FeedId}", feed.Id);
+            }
+        }
+
+        logger.LogInformation("Web feed scraper job finished.");
+    }
+}

DysonNetwork.Sphere/Connection/WebReader/WebFeedService.cs

@@ -0,0 +1,91 @@
+using System.Security.Claims;
+using System.ServiceModel.Syndication;
+using System.Xml;
+using DysonNetwork.Sphere.Account;
+using Microsoft.EntityFrameworkCore;
+using Microsoft.Extensions.Logging;
+
+namespace DysonNetwork.Sphere.Connection.WebReader;
+
+public class WebFeedService(
+    AppDatabase database,
+    IHttpClientFactory httpClientFactory,
+    ILogger<WebFeedService> logger,
+    AccountService accountService
+)
+{
+    public async Task<WebFeed> CreateWebFeedAsync(WebFeedController.CreateWebFeedRequest dto, ClaimsPrincipal claims)
+    {
+        if (claims.Identity?.Name == null)
+        {
+            throw new UnauthorizedAccessException();
+        }
+
+        var account = await accountService.LookupAccount(claims.Identity.Name);
+        if (account == null)
+        {
+            throw new UnauthorizedAccessException();
+        }
+
+        var feed = new WebFeed
+        {
+            Url = dto.Url,
+            Title = dto.Title,
+            Description = dto.Description,
+            PublisherId = account.Id,
+        };
+
+        database.Set<WebFeed>().Add(feed);
+        await database.SaveChangesAsync();
+
+        return feed;
+    }
+
+    public async Task ScrapeFeedAsync(WebFeed feed, CancellationToken cancellationToken = default)
+    {
+        var httpClient = httpClientFactory.CreateClient();
+        var response = await httpClient.GetAsync(feed.Url, cancellationToken);
+        response.EnsureSuccessStatusCode();
+
+        await using var stream = await response.Content.ReadAsStreamAsync(cancellationToken);
+        using var reader = XmlReader.Create(stream);
+        var syndicationFeed = SyndicationFeed.Load(reader);
+
+        if (syndicationFeed == null)
+        {
+            logger.LogWarning("Could not parse syndication feed for {FeedUrl}", feed.Url);
+            return;
+        }
+
+        foreach (var item in syndicationFeed.Items)
+        {
+            var itemUrl = item.Links.FirstOrDefault()?.Uri.ToString();
+            if (string.IsNullOrEmpty(itemUrl))
+            {
+                continue;
+            }
+
+            var articleExists = await database.Set<WebArticle>()
+                .AnyAsync(a => a.FeedId == feed.Id && a.Url == itemUrl, cancellationToken);
+
+            if (articleExists)
+            {
+                continue;
+            }
+
+            var newArticle = new WebArticle
+            {
+                FeedId = feed.Id,
+                Title = item.Title.Text,
+                Url = itemUrl,
+                Author = item.Authors.FirstOrDefault()?.Name,
+                Content = (item.Content as TextSyndicationContent)?.Text ?? item.Summary.Text,
+                PublishedAt = item.PublishDate.UtcDateTime,
+            };
+
+            database.Set<WebArticle>().Add(newArticle);
+        }
+
+        await database.SaveChangesAsync(cancellationToken);
+    }
+}

DysonNetwork.Sphere/DysonNetwork.Sphere.csproj

@@ -71,6 +71,7 @@
         <PackageReference Include="StackExchange.Redis.Extensions.AspNetCore" Version="11.0.0" />
         <PackageReference Include="Swashbuckle.AspNetCore" Version="8.1.0" />
         <PackageReference Include="Swashbuckle.AspNetCore.SwaggerUI" Version="8.1.0" />
+        <PackageReference Include="System.ServiceModel.Syndication" Version="9.0.6" />
         <PackageReference Include="tusdotnet" Version="2.8.1" />
     </ItemGroup>
 

DysonNetwork.Sphere/Migrations/20250626093051_AddWebArticles.cs

@@ -0,0 +1,103 @@
+using System;
+using System.Collections.Generic;
+using DysonNetwork.Sphere.Connection.WebReader;
+using Microsoft.EntityFrameworkCore.Migrations;
+using NodaTime;
+
+#nullable disable
+
+namespace DysonNetwork.Sphere.Migrations
+{
+    /// <inheritdoc />
+    public partial class AddWebArticles : Migration
+    {
+        /// <inheritdoc />
+        protected override void Up(MigrationBuilder migrationBuilder)
+        {
+            migrationBuilder.CreateTable(
+                name: "web_feeds",
+                columns: table => new
+                {
+                    id = table.Column<Guid>(type: "uuid", nullable: false),
+                    url = table.Column<string>(type: "character varying(8192)", maxLength: 8192, nullable: false),
+                    title = table.Column<string>(type: "character varying(4096)", maxLength: 4096, nullable: false),
+                    description = table.Column<string>(type: "character varying(8192)", maxLength: 8192, nullable: true),
+                    preview = table.Column<LinkEmbed>(type: "jsonb", nullable: true),
+                    publisher_id = table.Column<Guid>(type: "uuid", nullable: false),
+                    created_at = table.Column<Instant>(type: "timestamp with time zone", nullable: false),
+                    updated_at = table.Column<Instant>(type: "timestamp with time zone", nullable: false),
+                    deleted_at = table.Column<Instant>(type: "timestamp with time zone", nullable: true)
+                },
+                constraints: table =>
+                {
+                    table.PrimaryKey("pk_web_feeds", x => x.id);
+                    table.ForeignKey(
+                        name: "fk_web_feeds_publishers_publisher_id",
+                        column: x => x.publisher_id,
+                        principalTable: "publishers",
+                        principalColumn: "id",
+                        onDelete: ReferentialAction.Cascade);
+                });
+
+            migrationBuilder.CreateTable(
+                name: "web_articles",
+                columns: table => new
+                {
+                    id = table.Column<Guid>(type: "uuid", nullable: false),
+                    title = table.Column<string>(type: "character varying(4096)", maxLength: 4096, nullable: false),
+                    url = table.Column<string>(type: "character varying(8192)", maxLength: 8192, nullable: false),
+                    author = table.Column<string>(type: "character varying(4096)", maxLength: 4096, nullable: true),
+                    meta = table.Column<Dictionary<string, object>>(type: "jsonb", nullable: true),
+                    preview = table.Column<LinkEmbed>(type: "jsonb", nullable: true),
+                    content = table.Column<string>(type: "text", nullable: true),
+                    published_at = table.Column<DateTime>(type: "timestamp with time zone", nullable: true),
+                    feed_id = table.Column<Guid>(type: "uuid", nullable: false),
+                    created_at = table.Column<Instant>(type: "timestamp with time zone", nullable: false),
+                    updated_at = table.Column<Instant>(type: "timestamp with time zone", nullable: false),
+                    deleted_at = table.Column<Instant>(type: "timestamp with time zone", nullable: true)
+                },
+                constraints: table =>
+                {
+                    table.PrimaryKey("pk_web_articles", x => x.id);
+                    table.ForeignKey(
+                        name: "fk_web_articles_web_feeds_feed_id",
+                        column: x => x.feed_id,
+                        principalTable: "web_feeds",
+                        principalColumn: "id",
+                        onDelete: ReferentialAction.Cascade);
+                });
+
+            migrationBuilder.CreateIndex(
+                name: "ix_web_articles_feed_id",
+                table: "web_articles",
+                column: "feed_id");
+
+            migrationBuilder.CreateIndex(
+                name: "ix_web_articles_url",
+                table: "web_articles",
+                column: "url",
+                unique: true);
+
+            migrationBuilder.CreateIndex(
+                name: "ix_web_feeds_publisher_id",
+                table: "web_feeds",
+                column: "publisher_id");
+
+            migrationBuilder.CreateIndex(
+                name: "ix_web_feeds_url",
+                table: "web_feeds",
+                column: "url",
+                unique: true);
+        }
+
+        /// <inheritdoc />
+        protected override void Down(MigrationBuilder migrationBuilder)
+        {
+            migrationBuilder.DropTable(
+                name: "web_articles");
+
+            migrationBuilder.DropTable(
+                name: "web_feeds");
+        }
+    }
+}

DysonNetwork.Sphere/Migrations/AppDatabaseModelSnapshot.cs

@@ -5,6 +5,7 @@ using System.Text.Json;
 using DysonNetwork.Sphere;
 using DysonNetwork.Sphere.Account;
 using DysonNetwork.Sphere.Chat;
+using DysonNetwork.Sphere.Connection.WebReader;
 using DysonNetwork.Sphere.Storage;
 using DysonNetwork.Sphere.Wallet;
 using Microsoft.EntityFrameworkCore;
@@ -1361,6 +1362,132 @@ namespace DysonNetwork.Sphere.Migrations
                     b.ToTable("chat_realtime_call", (string)null);
                 });
 
+            modelBuilder.Entity("DysonNetwork.Sphere.Connection.WebReader.WebArticle", b =>
+                {
+                    b.Property<Guid>("Id")
+                        .ValueGeneratedOnAdd()
+                        .HasColumnType("uuid")
+                        .HasColumnName("id");
+
+                    b.Property<string>("Author")
+                        .HasMaxLength(4096)
+                        .HasColumnType("character varying(4096)")
+                        .HasColumnName("author");
+
+                    b.Property<string>("Content")
+                        .HasColumnType("text")
+                        .HasColumnName("content");
+
+                    b.Property<Instant>("CreatedAt")
+                        .HasColumnType("timestamp with time zone")
+                        .HasColumnName("created_at");
+
+                    b.Property<Instant?>("DeletedAt")
+                        .HasColumnType("timestamp with time zone")
+                        .HasColumnName("deleted_at");
+
+                    b.Property<Guid>("FeedId")
+                        .HasColumnType("uuid")
+                        .HasColumnName("feed_id");
+
+                    b.Property<Dictionary<string, object>>("Meta")
+                        .HasColumnType("jsonb")
+                        .HasColumnName("meta");
+
+                    b.Property<LinkEmbed>("Preview")
+                        .HasColumnType("jsonb")
+                        .HasColumnName("preview");
+
+                    b.Property<DateTime?>("PublishedAt")
+                        .HasColumnType("timestamp with time zone")
+                        .HasColumnName("published_at");
+
+                    b.Property<string>("Title")
+                        .IsRequired()
+                        .HasMaxLength(4096)
+                        .HasColumnType("character varying(4096)")
+                        .HasColumnName("title");
+
+                    b.Property<Instant>("UpdatedAt")
+                        .HasColumnType("timestamp with time zone")
+                        .HasColumnName("updated_at");
+
+                    b.Property<string>("Url")
+                        .IsRequired()
+                        .HasMaxLength(8192)
+                        .HasColumnType("character varying(8192)")
+                        .HasColumnName("url");
+
+                    b.HasKey("Id")
+                        .HasName("pk_web_articles");
+
+                    b.HasIndex("FeedId")
+                        .HasDatabaseName("ix_web_articles_feed_id");
+
+                    b.HasIndex("Url")
+                        .IsUnique()
+                        .HasDatabaseName("ix_web_articles_url");
+
+                    b.ToTable("web_articles", (string)null);
+                });
+
+            modelBuilder.Entity("DysonNetwork.Sphere.Connection.WebReader.WebFeed", b =>
+                {
+                    b.Property<Guid>("Id")
+                        .ValueGeneratedOnAdd()
+                        .HasColumnType("uuid")
+                        .HasColumnName("id");
+
+                    b.Property<Instant>("CreatedAt")
+                        .HasColumnType("timestamp with time zone")
+                        .HasColumnName("created_at");
+
+                    b.Property<Instant?>("DeletedAt")
+                        .HasColumnType("timestamp with time zone")
+                        .HasColumnName("deleted_at");
+
+                    b.Property<string>("Description")
+                        .HasMaxLength(8192)
+                        .HasColumnType("character varying(8192)")
+                        .HasColumnName("description");
+
+                    b.Property<LinkEmbed>("Preview")
+                        .HasColumnType("jsonb")
+                        .HasColumnName("preview");
+
+                    b.Property<Guid>("PublisherId")
+                        .HasColumnType("uuid")
+                        .HasColumnName("publisher_id");
+
+                    b.Property<string>("Title")
+                        .IsRequired()
+                        .HasMaxLength(4096)
+                        .HasColumnType("character varying(4096)")
+                        .HasColumnName("title");
+
+                    b.Property<Instant>("UpdatedAt")
+                        .HasColumnType("timestamp with time zone")
+                        .HasColumnName("updated_at");
+
+                    b.Property<string>("Url")
+                        .IsRequired()
+                        .HasMaxLength(8192)
+                        .HasColumnType("character varying(8192)")
+                        .HasColumnName("url");
+
+                    b.HasKey("Id")
+                        .HasName("pk_web_feeds");
+
+                    b.HasIndex("PublisherId")
+                        .HasDatabaseName("ix_web_feeds_publisher_id");
+
+                    b.HasIndex("Url")
+                        .IsUnique()
+                        .HasDatabaseName("ix_web_feeds_url");
+
+                    b.ToTable("web_feeds", (string)null);
+                });
+
             modelBuilder.Entity("DysonNetwork.Sphere.Developer.CustomApp", b =>
                 {
                     b.Property<Guid>("Id")
@@ -3211,6 +3338,30 @@ namespace DysonNetwork.Sphere.Migrations
                     b.Navigation("Sender");
                 });
 
+            modelBuilder.Entity("DysonNetwork.Sphere.Connection.WebReader.WebArticle", b =>
+                {
+                    b.HasOne("DysonNetwork.Sphere.Connection.WebReader.WebFeed", "Feed")
+                        .WithMany("Articles")
+                        .HasForeignKey("FeedId")
+                        .OnDelete(DeleteBehavior.Cascade)
+                        .IsRequired()
+                        .HasConstraintName("fk_web_articles_web_feeds_feed_id");
+
+                    b.Navigation("Feed");
+                });
+
+            modelBuilder.Entity("DysonNetwork.Sphere.Connection.WebReader.WebFeed", b =>
+                {
+                    b.HasOne("DysonNetwork.Sphere.Publisher.Publisher", "Publisher")
+                        .WithMany()
+                        .HasForeignKey("PublisherId")
+                        .OnDelete(DeleteBehavior.Cascade)
+                        .IsRequired()
+                        .HasConstraintName("fk_web_feeds_publishers_publisher_id");
+
+                    b.Navigation("Publisher");
+                });
+
             modelBuilder.Entity("DysonNetwork.Sphere.Developer.CustomApp", b =>
                 {
                     b.HasOne("DysonNetwork.Sphere.Publisher.Publisher", "Developer")
@@ -3651,6 +3802,11 @@ namespace DysonNetwork.Sphere.Migrations
                     b.Navigation("Reactions");
                 });
 
+            modelBuilder.Entity("DysonNetwork.Sphere.Connection.WebReader.WebFeed", b =>
+                {
+                    b.Navigation("Articles");
+                });
+
             modelBuilder.Entity("DysonNetwork.Sphere.Permission.PermissionGroup", b =>
                 {
                     b.Navigation("Members");

DysonNetwork.Sphere/Startup/ScheduledJobsConfiguration.cs

@@ -1,4 +1,5 @@
 using DysonNetwork.Sphere.Storage;
+using DysonNetwork.Sphere.Connection.WebReader;
 using DysonNetwork.Sphere.Storage.Handlers;
 using DysonNetwork.Sphere.Wallet;
 using Quartz;
@@ -75,6 +76,14 @@ public static class ScheduledJobsConfiguration
                     .WithIntervalInMinutes(30)
                     .RepeatForever())
             );
+
+            var webFeedScraperJob = new JobKey("WebFeedScraper");
+            q.AddJob<WebFeedScraperJob>(opts => opts.WithIdentity(webFeedScraperJob));
+            q.AddTrigger(opts => opts
+                .ForJob(webFeedScraperJob)
+                .WithIdentity("WebFeedScraperTrigger")
+                .WithSimpleSchedule(o => o.WithIntervalInHours(24).RepeatForever())
+            );
         });
         services.AddQuartzHostedService(q => q.WaitForJobsToComplete = true);
 

DysonNetwork.Sphere/Startup/ServiceCollectionExtensions.cs

@@ -224,6 +224,7 @@ public static class ServiceCollectionExtensions
         services.AddScoped<PaymentService>();
         services.AddScoped<IRealtimeService, LivekitRealtimeService>();
         services.AddScoped<WebReaderService>();
+        services.AddScoped<WebFeedService>();
         services.AddScoped<AfdianPaymentHandler>();
         services.AddScoped<SafetyService>();
 

DysonNetwork.sln.DotSettings.user

@@ -85,6 +85,9 @@
 	<s:String x:Key="/Default/CodeInspection/ExcludedFiles/FilesAndFoldersToSkip2/=7020124F_002D9FFC_002D4AC3_002D8F3D_002DAAB8E0240759_002Ff_003ATusDiskStore_002Ecs_002Fl_003A_002E_002E_003F_002E_002E_003F_002E_002E_003FLibrary_003FApplication_0020Support_003FJetBrains_003FRider2025_002E1_003Fresharper_002Dhost_003FDecompilerCache_003Fdecompiler_003F8bb08a178b5b43c5bac20a5a54159a5b2a800_003F1c_003F21999acd_003FTusDiskStore_002Ecs/@EntryIndexedValue">ForceIncluded</s:String>
 	<s:String x:Key="/Default/CodeInspection/ExcludedFiles/FilesAndFoldersToSkip2/=7020124F_002D9FFC_002D4AC3_002D8F3D_002DAAB8E0240759_002Ff_003AUri_002Ecs_002Fl_003A_002E_002E_003F_002E_002E_003F_002E_002E_003FLibrary_003FApplication_0020Support_003FJetBrains_003FRider2024_002E3_003Fresharper_002Dhost_003FDecompilerCache_003Fdecompiler_003F5d2c480da9be415dab9be535bb6d08713cc00_003Fd0_003Fffc36a51_003FUri_002Ecs/@EntryIndexedValue">ForceIncluded</s:String>
 	<s:String x:Key="/Default/CodeInspection/ExcludedFiles/FilesAndFoldersToSkip2/=7020124F_002D9FFC_002D4AC3_002D8F3D_002DAAB8E0240759_002Ff_003AValidationContext_00601_002Ecs_002Fl_003A_002E_002E_003F_002E_002E_003F_002E_002E_003FLibrary_003FApplication_0020Support_003FJetBrains_003FRider2024_002E3_003Fresharper_002Dhost_003FDecompilerCache_003Fdecompiler_003F8bb08a178b5b43c5bac20a5a54159a5b2a800_003F6b_003F741ceebe_003FValidationContext_00601_002Ecs/@EntryIndexedValue">ForceIncluded</s:String>
+	<s:String x:Key="/Default/Environment/AssemblyExplorer/XmlDocument/@EntryValue">&lt;AssemblyExplorer&gt;
+  &lt;Assembly Path="/opt/homebrew/Cellar/dotnet/9.0.6/libexec/packs/Microsoft.AspNetCore.App.Ref/9.0.6/ref/net9.0/Microsoft.AspNetCore.RateLimiting.dll" /&gt;
+&lt;/AssemblyExplorer&gt;</s:String>
 	<s:Boolean x:Key="/Default/ResxEditorPersonal/CheckedGroups/=DysonNetwork_002ESphere_002FLocalization_002FResources_002FAccountEventResource/@EntryIndexedValue">False</s:Boolean>
 	<s:Boolean x:Key="/Default/ResxEditorPersonal/CheckedGroups/=DysonNetwork_002ESphere_002FLocalization_002FResources_002FAccountEventResource/@EntryIndexRemoved">True</s:Boolean>
 	<s:Boolean x:Key="/Default/ResxEditorPersonal/CheckedGroups/=DysonNetwork_002ESphere_002FLocalization_002FResources_002FSharedResource/@EntryIndexedValue">False</s:Boolean>