From ede49333f83b765ef476dab7052a1d55a703ba08 Mon Sep 17 00:00:00 2001 From: LittleSheep Date: Fri, 2 Jan 2026 00:15:56 +0800 Subject: [PATCH] :recycle: Move the web reader from Sphere to Insight (w.i.p) --- DysonNetwork.Insight/AppDatabase.cs | 4 +++ .../DysonNetwork.Insight.csproj | 2 ++ .../Reader}/ScrapedArticle.cs | 4 ++- .../Reader}/WebArticleController.cs | 2 +- .../Reader}/WebFeedController.cs | 6 +++-- .../Reader}/WebFeedPublicController.cs | 17 ++++++------ .../Reader}/WebFeedScraperJob.cs | 5 ++-- .../Reader}/WebFeedService.cs | 27 ++++++++++--------- .../Reader}/WebReaderController.cs | 3 ++- .../Reader}/WebReaderException.cs | 2 +- .../Reader}/WebReaderService.cs | 3 ++- .../DysonNetwork.Messager.csproj | 2 -- .../Models/Embed}/EmbeddableBase.cs | 2 +- .../Models/Embed}/LinkEmbed.cs | 2 +- .../Models}/WebArticle.cs | 20 ++++++++------ DysonNetwork.Sphere/AppDatabase.cs | 18 ------------- .../DysonNetwork.Sphere.csproj | 3 --- DysonNetwork.Zone/DysonNetwork.Zone.csproj | 1 - 18 files changed, 59 insertions(+), 64 deletions(-) rename {DysonNetwork.Sphere/WebReader => DysonNetwork.Insight/Reader}/ScrapedArticle.cs (61%) rename {DysonNetwork.Sphere/WebReader => DysonNetwork.Insight/Reader}/WebArticleController.cs (98%) rename {DysonNetwork.Sphere/WebReader => DysonNetwork.Insight/Reader}/WebFeedController.cs (95%) rename {DysonNetwork.Sphere/WebReader => DysonNetwork.Insight/Reader}/WebFeedPublicController.cs (93%) rename {DysonNetwork.Sphere/WebReader => DysonNetwork.Insight/Reader}/WebFeedScraperJob.cs (82%) rename {DysonNetwork.Sphere/WebReader => DysonNetwork.Insight/Reader}/WebFeedService.cs (78%) rename {DysonNetwork.Sphere/WebReader => DysonNetwork.Insight/Reader}/WebReaderController.cs (97%) rename {DysonNetwork.Sphere/WebReader => DysonNetwork.Insight/Reader}/WebReaderException.cs (89%) rename {DysonNetwork.Sphere/WebReader => DysonNetwork.Insight/Reader}/WebReaderService.cs (99%) rename {DysonNetwork.Sphere/WebReader => DysonNetwork.Shared/Models/Embed}/EmbeddableBase.cs (95%) rename {DysonNetwork.Sphere/WebReader => DysonNetwork.Shared/Models/Embed}/LinkEmbed.cs (97%) rename {DysonNetwork.Sphere/WebReader => DysonNetwork.Shared/Models}/WebArticle.cs (74%) diff --git a/DysonNetwork.Insight/AppDatabase.cs b/DysonNetwork.Insight/AppDatabase.cs index 79063f2..7eaa493 100644 --- a/DysonNetwork.Insight/AppDatabase.cs +++ b/DysonNetwork.Insight/AppDatabase.cs @@ -15,6 +15,10 @@ public class AppDatabase( public DbSet ThinkingThoughts { get; set; } public DbSet UnpaidAccounts { get; set; } + public DbSet WebArticles { get; set; } + public DbSet WebFeeds { get; set; } + public DbSet WebFeedSubscriptions { get; set; } + protected override void OnConfiguring(DbContextOptionsBuilder optionsBuilder) { optionsBuilder.UseNpgsql( diff --git a/DysonNetwork.Insight/DysonNetwork.Insight.csproj b/DysonNetwork.Insight/DysonNetwork.Insight.csproj index 5a90763..a685c46 100644 --- a/DysonNetwork.Insight/DysonNetwork.Insight.csproj +++ b/DysonNetwork.Insight/DysonNetwork.Insight.csproj @@ -7,6 +7,7 @@ + all @@ -18,6 +19,7 @@ + diff --git a/DysonNetwork.Sphere/WebReader/ScrapedArticle.cs b/DysonNetwork.Insight/Reader/ScrapedArticle.cs similarity index 61% rename from DysonNetwork.Sphere/WebReader/ScrapedArticle.cs rename to DysonNetwork.Insight/Reader/ScrapedArticle.cs index 2c724e3..2c48aa9 100644 --- a/DysonNetwork.Sphere/WebReader/ScrapedArticle.cs +++ b/DysonNetwork.Insight/Reader/ScrapedArticle.cs @@ -1,4 +1,6 @@ -namespace DysonNetwork.Sphere.WebReader; +using DysonNetwork.Shared.Models.Embed; + +namespace DysonNetwork.Insight.Reader; public class ScrapedArticle { diff --git a/DysonNetwork.Sphere/WebReader/WebArticleController.cs b/DysonNetwork.Insight/Reader/WebArticleController.cs similarity index 98% rename from DysonNetwork.Sphere/WebReader/WebArticleController.cs rename to DysonNetwork.Insight/Reader/WebArticleController.cs index 05f4d50..7d4bcd5 100644 --- a/DysonNetwork.Sphere/WebReader/WebArticleController.cs +++ b/DysonNetwork.Insight/Reader/WebArticleController.cs @@ -1,7 +1,7 @@ using Microsoft.AspNetCore.Mvc; using Microsoft.EntityFrameworkCore; -namespace DysonNetwork.Sphere.WebReader; +namespace DysonNetwork.Insight.Reader; [ApiController] [Route("/api/feeds/articles")] diff --git a/DysonNetwork.Sphere/WebReader/WebFeedController.cs b/DysonNetwork.Insight/Reader/WebFeedController.cs similarity index 95% rename from DysonNetwork.Sphere/WebReader/WebFeedController.cs rename to DysonNetwork.Insight/Reader/WebFeedController.cs index 91c080a..1b56afc 100644 --- a/DysonNetwork.Sphere/WebReader/WebFeedController.cs +++ b/DysonNetwork.Insight/Reader/WebFeedController.cs @@ -1,14 +1,16 @@ using System.ComponentModel.DataAnnotations; +using DysonNetwork.Shared.Models; using DysonNetwork.Shared.Proto; +using DysonNetwork.Shared.Registry; using Microsoft.AspNetCore.Authorization; using Microsoft.AspNetCore.Mvc; -namespace DysonNetwork.Sphere.WebReader; +namespace DysonNetwork.Insight.Reader; [Authorize] [ApiController] [Route("/api/publishers/{pubName}/feeds")] -public class WebFeedController(WebFeedService webFeed, Publisher.PublisherService ps) : ControllerBase +public class WebFeedController(WebFeedService webFeed, RemotePublisherService ps) : ControllerBase { public record WebFeedRequest( [MaxLength(8192)] string? Url, diff --git a/DysonNetwork.Sphere/WebReader/WebFeedPublicController.cs b/DysonNetwork.Insight/Reader/WebFeedPublicController.cs similarity index 93% rename from DysonNetwork.Sphere/WebReader/WebFeedPublicController.cs rename to DysonNetwork.Insight/Reader/WebFeedPublicController.cs index 0503ea7..b91ec0b 100644 --- a/DysonNetwork.Sphere/WebReader/WebFeedPublicController.cs +++ b/DysonNetwork.Insight/Reader/WebFeedPublicController.cs @@ -1,9 +1,10 @@ +using DysonNetwork.Shared.Models; using DysonNetwork.Shared.Proto; using Microsoft.AspNetCore.Authorization; using Microsoft.AspNetCore.Mvc; using Microsoft.EntityFrameworkCore; -namespace DysonNetwork.Sphere.WebReader; +namespace DysonNetwork.Insight.Reader; [ApiController] [Route("/api/feeds")] @@ -39,7 +40,7 @@ public class WebFeedPublicController( return Ok(existingSubscription); // Create new subscription - var subscription = new WebFeedSubscription + var subscription = new SnWebFeedSubscription { FeedId = feedId, AccountId = accountId @@ -83,7 +84,7 @@ public class WebFeedPublicController( /// Subscription status [HttpGet("{feedId:guid}/subscription")] [Authorize] - public async Task> GetSubscriptionStatus(Guid feedId) + public async Task> GetSubscriptionStatus(Guid feedId) { if (HttpContext.Items["CurrentUser"] is not Account currentUser) return Unauthorized(); @@ -105,7 +106,7 @@ public class WebFeedPublicController( /// List of subscribed feeds [HttpGet("subscribed")] [Authorize] - public async Task> GetSubscribedFeeds( + public async Task> GetSubscribedFeeds( [FromQuery] int offset = 0, [FromQuery] int take = 20 ) @@ -137,7 +138,7 @@ public class WebFeedPublicController( /// [HttpGet] [Authorize] - public async Task> GetWebFeedArticles( + public async Task> GetWebFeedArticles( [FromQuery] int offset = 0, [FromQuery] int take = 20 ) @@ -174,7 +175,7 @@ public class WebFeedPublicController( /// Feed metadata [AllowAnonymous] [HttpGet("{feedId:guid}")] - public async Task> GetFeedById(Guid feedId) + public async Task> GetFeedById(Guid feedId) { var feed = await webFeed.GetFeedAsync(feedId); if (feed == null) @@ -192,7 +193,7 @@ public class WebFeedPublicController( /// List of articles from the feed [AllowAnonymous] [HttpGet("{feedId:guid}/articles")] - public async Task> GetFeedArticles( + public async Task> GetFeedArticles( [FromRoute] Guid feedId, [FromQuery] int offset = 0, [FromQuery] int take = 20 @@ -224,7 +225,7 @@ public class WebFeedPublicController( /// [HttpGet("explore")] [Authorize] - public async Task> ExploreFeeds( + public async Task> ExploreFeeds( [FromQuery] int offset = 0, [FromQuery] int take = 20, [FromQuery] string? query = null diff --git a/DysonNetwork.Sphere/WebReader/WebFeedScraperJob.cs b/DysonNetwork.Insight/Reader/WebFeedScraperJob.cs similarity index 82% rename from DysonNetwork.Sphere/WebReader/WebFeedScraperJob.cs rename to DysonNetwork.Insight/Reader/WebFeedScraperJob.cs index 80ded46..06c97a9 100644 --- a/DysonNetwork.Sphere/WebReader/WebFeedScraperJob.cs +++ b/DysonNetwork.Insight/Reader/WebFeedScraperJob.cs @@ -1,7 +1,8 @@ +using DysonNetwork.Shared.Models; using Microsoft.EntityFrameworkCore; using Quartz; -namespace DysonNetwork.Sphere.WebReader; +namespace DysonNetwork.Insight.Reader; [DisallowConcurrentExecution] public class WebFeedScraperJob( @@ -15,7 +16,7 @@ public class WebFeedScraperJob( { logger.LogInformation("Starting web feed scraper job."); - var feeds = await database.Set().ToListAsync(context.CancellationToken); + var feeds = await database.Set().ToListAsync(context.CancellationToken); foreach (var feed in feeds) { diff --git a/DysonNetwork.Sphere/WebReader/WebFeedService.cs b/DysonNetwork.Insight/Reader/WebFeedService.cs similarity index 78% rename from DysonNetwork.Sphere/WebReader/WebFeedService.cs rename to DysonNetwork.Insight/Reader/WebFeedService.cs index a1558ad..ebdb1f3 100644 --- a/DysonNetwork.Sphere/WebReader/WebFeedService.cs +++ b/DysonNetwork.Insight/Reader/WebFeedService.cs @@ -1,20 +1,21 @@ using System.ServiceModel.Syndication; using System.Xml; +using DysonNetwork.Shared.Models; +using DysonNetwork.Shared.Models.Embed; using Microsoft.EntityFrameworkCore; -namespace DysonNetwork.Sphere.WebReader; +namespace DysonNetwork.Insight.Reader; public class WebFeedService( AppDatabase database, IHttpClientFactory httpClientFactory, ILogger logger, - WebReaderService webReaderService + WebReaderService readerService ) { - public async Task CreateWebFeedAsync(Shared.Models.SnPublisher publisher, - WebFeedController.WebFeedRequest request) + public async Task CreateWebFeedAsync(SnPublisher publisher, WebFeedController.WebFeedRequest request) { - var feed = new WebFeed + var feed = new SnWebFeed { Url = request.Url!, Title = request.Title!, @@ -29,7 +30,7 @@ public class WebFeedService( return feed; } - public async Task GetFeedAsync(Guid id, Guid? publisherId = null) + public async Task GetFeedAsync(Guid id, Guid? publisherId = null) { var query = database.WebFeeds .Include(a => a.Publisher) @@ -40,12 +41,12 @@ public class WebFeedService( return await query.FirstOrDefaultAsync(); } - public async Task> GetFeedsByPublisherAsync(Guid publisherId) + public async Task> GetFeedsByPublisherAsync(Guid publisherId) { return await database.WebFeeds.Where(a => a.PublisherId == publisherId).ToListAsync(); } - public async Task UpdateFeedAsync(WebFeed feed, WebFeedController.WebFeedRequest request) + public async Task UpdateFeedAsync(SnWebFeed feed, WebFeedController.WebFeedRequest request) { if (request.Url is not null) feed.Url = request.Url; @@ -76,7 +77,7 @@ public class WebFeedService( return true; } - public async Task ScrapeFeedAsync(WebFeed feed, CancellationToken cancellationToken = default) + public async Task ScrapeFeedAsync(SnWebFeed feed, CancellationToken cancellationToken = default) { var httpClient = httpClientFactory.CreateClient(); var response = await httpClient.GetAsync(feed.Url, cancellationToken); @@ -98,7 +99,7 @@ public class WebFeedService( if (string.IsNullOrEmpty(itemUrl)) continue; - var articleExists = await database.Set() + var articleExists = await database.Set() .AnyAsync(a => a.FeedId == feed.Id && a.Url == itemUrl, cancellationToken); if (articleExists) @@ -109,17 +110,17 @@ public class WebFeedService( if (feed.Config.ScrapPage) { - var scrapedArticle = await webReaderService.ScrapeArticleAsync(itemUrl, cancellationToken); + var scrapedArticle = await readerService.ScrapeArticleAsync(itemUrl, cancellationToken); preview = scrapedArticle.LinkEmbed; if (scrapedArticle.Content is not null) content = scrapedArticle.Content; } else { - preview = await webReaderService.GetLinkPreviewAsync(itemUrl, cancellationToken); + preview = await readerService.GetLinkPreviewAsync(itemUrl, cancellationToken); } - var newArticle = new WebArticle + var newArticle = new SnWebArticle { FeedId = feed.Id, Title = item.Title.Text, diff --git a/DysonNetwork.Sphere/WebReader/WebReaderController.cs b/DysonNetwork.Insight/Reader/WebReaderController.cs similarity index 97% rename from DysonNetwork.Sphere/WebReader/WebReaderController.cs rename to DysonNetwork.Insight/Reader/WebReaderController.cs index c9843a5..5f5606d 100644 --- a/DysonNetwork.Sphere/WebReader/WebReaderController.cs +++ b/DysonNetwork.Insight/Reader/WebReaderController.cs @@ -1,9 +1,10 @@ using DysonNetwork.Shared.Auth; +using DysonNetwork.Shared.Models.Embed; using Microsoft.AspNetCore.Authorization; using Microsoft.AspNetCore.Mvc; using Microsoft.AspNetCore.RateLimiting; -namespace DysonNetwork.Sphere.WebReader; +namespace DysonNetwork.Insight.Reader; /// /// Controller for web scraping and link preview services diff --git a/DysonNetwork.Sphere/WebReader/WebReaderException.cs b/DysonNetwork.Insight/Reader/WebReaderException.cs similarity index 89% rename from DysonNetwork.Sphere/WebReader/WebReaderException.cs rename to DysonNetwork.Insight/Reader/WebReaderException.cs index 559af6d..5c05e17 100644 --- a/DysonNetwork.Sphere/WebReader/WebReaderException.cs +++ b/DysonNetwork.Insight/Reader/WebReaderException.cs @@ -1,4 +1,4 @@ -namespace DysonNetwork.Sphere.WebReader; +namespace DysonNetwork.Insight.Reader; /// /// Exception thrown when an error occurs during web reading operations diff --git a/DysonNetwork.Sphere/WebReader/WebReaderService.cs b/DysonNetwork.Insight/Reader/WebReaderService.cs similarity index 99% rename from DysonNetwork.Sphere/WebReader/WebReaderService.cs rename to DysonNetwork.Insight/Reader/WebReaderService.cs index 3992be7..932d260 100644 --- a/DysonNetwork.Sphere/WebReader/WebReaderService.cs +++ b/DysonNetwork.Insight/Reader/WebReaderService.cs @@ -2,9 +2,10 @@ using System.Globalization; using AngleSharp; using AngleSharp.Dom; using DysonNetwork.Shared.Cache; +using DysonNetwork.Shared.Models.Embed; using HtmlAgilityPack; -namespace DysonNetwork.Sphere.WebReader; +namespace DysonNetwork.Insight.Reader; /// /// The service is amin to providing scrapping service to the Solar Network. diff --git a/DysonNetwork.Messager/DysonNetwork.Messager.csproj b/DysonNetwork.Messager/DysonNetwork.Messager.csproj index 818459d..5776d4a 100644 --- a/DysonNetwork.Messager/DysonNetwork.Messager.csproj +++ b/DysonNetwork.Messager/DysonNetwork.Messager.csproj @@ -10,9 +10,7 @@ - - diff --git a/DysonNetwork.Sphere/WebReader/EmbeddableBase.cs b/DysonNetwork.Shared/Models/Embed/EmbeddableBase.cs similarity index 95% rename from DysonNetwork.Sphere/WebReader/EmbeddableBase.cs rename to DysonNetwork.Shared/Models/Embed/EmbeddableBase.cs index e0839bf..535af71 100644 --- a/DysonNetwork.Sphere/WebReader/EmbeddableBase.cs +++ b/DysonNetwork.Shared/Models/Embed/EmbeddableBase.cs @@ -1,7 +1,7 @@ using System.Text.Json; using DysonNetwork.Shared.Proto; -namespace DysonNetwork.Sphere.WebReader; +namespace DysonNetwork.Shared.Models.Embed; /// /// The embeddable can be used in the post or messages' meta's embeds fields diff --git a/DysonNetwork.Sphere/WebReader/LinkEmbed.cs b/DysonNetwork.Shared/Models/Embed/LinkEmbed.cs similarity index 97% rename from DysonNetwork.Sphere/WebReader/LinkEmbed.cs rename to DysonNetwork.Shared/Models/Embed/LinkEmbed.cs index b83cdcc..7c64c20 100644 --- a/DysonNetwork.Sphere/WebReader/LinkEmbed.cs +++ b/DysonNetwork.Shared/Models/Embed/LinkEmbed.cs @@ -1,4 +1,4 @@ -namespace DysonNetwork.Sphere.WebReader; +namespace DysonNetwork.Shared.Models.Embed; /// /// The link embed is a part of the embeddable implementations diff --git a/DysonNetwork.Sphere/WebReader/WebArticle.cs b/DysonNetwork.Shared/Models/WebArticle.cs similarity index 74% rename from DysonNetwork.Sphere/WebReader/WebArticle.cs rename to DysonNetwork.Shared/Models/WebArticle.cs index ff19b4a..d498634 100644 --- a/DysonNetwork.Sphere/WebReader/WebArticle.cs +++ b/DysonNetwork.Shared/Models/WebArticle.cs @@ -1,11 +1,12 @@ using System.ComponentModel.DataAnnotations; using System.ComponentModel.DataAnnotations.Schema; using System.Text.Json.Serialization; -using DysonNetwork.Shared.Models; +using DysonNetwork.Shared.Models.Embed; +using NodaTime; -namespace DysonNetwork.Sphere.WebReader; +namespace DysonNetwork.Shared.Models; -public class WebArticle : ModelBase +public class SnWebArticle : ModelBase { public Guid Id { get; set; } = Guid.NewGuid(); @@ -22,7 +23,7 @@ public class WebArticle : ModelBase public DateTime? PublishedAt { get; set; } public Guid FeedId { get; set; } - public WebFeed Feed { get; set; } = null!; + public SnWebFeed Feed { get; set; } = null!; } public class WebFeedConfig @@ -30,28 +31,31 @@ public class WebFeedConfig public bool ScrapPage { get; set; } } -public class WebFeed : ModelBase +public class SnWebFeed : ModelBase { public Guid Id { get; set; } = Guid.NewGuid(); [MaxLength(8192)] public string Url { get; set; } = null!; [MaxLength(4096)] public string Title { get; set; } = null!; [MaxLength(8192)] public string? Description { get; set; } + public Instant? VerifiedAt { get; set; } + [JsonIgnore] [MaxLength(8192)] public string? VerificationKey { get; set; } + [Column(TypeName = "jsonb")] public LinkEmbed? Preview { get; set; } [Column(TypeName = "jsonb")] public WebFeedConfig Config { get; set; } = new(); public Guid PublisherId { get; set; } public SnPublisher Publisher { get; set; } = null!; - [JsonIgnore] public List Articles { get; set; } = new List(); + [JsonIgnore] public List Articles { get; set; } = new(); } -public class WebFeedSubscription : ModelBase +public class SnWebFeedSubscription : ModelBase { public Guid Id { get; set; } = Guid.NewGuid(); public Guid FeedId { get; set; } - public WebFeed Feed { get; set; } = null!; + public SnWebFeed Feed { get; set; } = null!; public Guid AccountId { get; set; } [NotMapped] public SnAccount Account { get; set; } = null!; } \ No newline at end of file diff --git a/DysonNetwork.Sphere/AppDatabase.cs b/DysonNetwork.Sphere/AppDatabase.cs index 2d8b54a..9aad114 100644 --- a/DysonNetwork.Sphere/AppDatabase.cs +++ b/DysonNetwork.Sphere/AppDatabase.cs @@ -1,20 +1,13 @@ using System.Linq.Expressions; using DysonNetwork.Shared.Data; using DysonNetwork.Shared.Models; -using DysonNetwork.Sphere.WebReader; using Microsoft.EntityFrameworkCore; using Microsoft.EntityFrameworkCore.Design; -using Microsoft.EntityFrameworkCore.Query; using NodaTime; using Quartz; namespace DysonNetwork.Sphere; -public interface IIdentifiedResource -{ - public string ResourceIdentifier { get; } -} - public class AppDatabase( DbContextOptions options, IConfiguration configuration @@ -53,10 +46,6 @@ public class AppDatabase( public DbSet FediverseRelationships { get; set; } = null!; public DbSet ActivityPubDeliveries { get; set; } = null!; - public DbSet WebArticles { get; set; } = null!; - public DbSet WebFeeds { get; set; } = null!; - public DbSet WebFeedSubscriptions { get; set; } = null!; - protected override void OnConfiguring(DbContextOptionsBuilder optionsBuilder) { optionsBuilder.UseNpgsql( @@ -140,13 +129,6 @@ public class AppDatabase( .HasForeignKey(m => m.SenderId) .OnDelete(DeleteBehavior.Cascade); - modelBuilder.Entity() - .HasIndex(f => f.Url) - .IsUnique(); - modelBuilder.Entity() - .HasIndex(a => a.Url) - .IsUnique(); - modelBuilder.Entity() .HasOne(a => a.Instance) .WithMany(i => i.Actors) diff --git a/DysonNetwork.Sphere/DysonNetwork.Sphere.csproj b/DysonNetwork.Sphere/DysonNetwork.Sphere.csproj index 004d835..ebf7d12 100644 --- a/DysonNetwork.Sphere/DysonNetwork.Sphere.csproj +++ b/DysonNetwork.Sphere/DysonNetwork.Sphere.csproj @@ -11,9 +11,7 @@ - - @@ -38,7 +36,6 @@ - diff --git a/DysonNetwork.Zone/DysonNetwork.Zone.csproj b/DysonNetwork.Zone/DysonNetwork.Zone.csproj index 1887542..a0a5ba8 100644 --- a/DysonNetwork.Zone/DysonNetwork.Zone.csproj +++ b/DysonNetwork.Zone/DysonNetwork.Zone.csproj @@ -30,7 +30,6 @@ -