♻️ Move the web reader to insight completely

DysonNetwork.Insight/Reader/ScrapedArticle.cs

@@ -1,9 +1,33 @@
 using DysonNetwork.Shared.Models.Embed;
+using DysonNetwork.Shared.Proto;
+using EmbedLinkEmbed = DysonNetwork.Shared.Models.Embed.LinkEmbed;
 
 namespace DysonNetwork.Insight.Reader;
 
 public class ScrapedArticle
 {
-    public LinkEmbed LinkEmbed { get; set; } = null!;
+    public EmbedLinkEmbed LinkEmbed { get; set; } = null!;
     public string? Content { get; set; }
+
+    public Shared.Proto.ScrapedArticle ToProtoValue()
+    {
+        var proto = new Shared.Proto.ScrapedArticle
+        {
+            LinkEmbed = LinkEmbed.ToProtoValue()
+        };
+
+        if (!string.IsNullOrEmpty(Content))
+            proto.Content = Content;
+
+        return proto;
+    }
+
+    public static ScrapedArticle FromProtoValue(Shared.Proto.ScrapedArticle proto)
+    {
+        return new ScrapedArticle
+        {
+            LinkEmbed = EmbedLinkEmbed.FromProtoValue(proto.LinkEmbed),
+            Content = proto.Content == "" ? null : proto.Content
+        };
+    }
 }

DysonNetwork.Insight/Reader/WebArticleGrpcService.cs

@@ -0,0 +1,90 @@
+using DysonNetwork.Shared.Models;
+using DysonNetwork.Shared.Proto;
+using Grpc.Core;
+using Microsoft.EntityFrameworkCore;
+
+namespace DysonNetwork.Insight.Reader;
+
+public class WebArticleGrpcService(AppDatabase db) : WebArticleService.WebArticleServiceBase
+{
+    public override async Task<GetWebArticleResponse> GetWebArticle(
+        GetWebArticleRequest request,
+        ServerCallContext context
+    )
+    {
+        if (!Guid.TryParse(request.Id, out var id))
+            throw new RpcException(new Status(StatusCode.InvalidArgument, "invalid id"));
+
+        var article = await db.WebArticles
+            .Include(a => a.Feed)
+            .FirstOrDefaultAsync(a => a.Id == id);
+
+        return article == null
+            ? throw new RpcException(new Status(StatusCode.NotFound, "article not found"))
+            : new GetWebArticleResponse { Article = article.ToProtoValue() };
+    }
+
+    public override async Task<GetWebArticleBatchResponse> GetWebArticleBatch(
+        GetWebArticleBatchRequest request,
+        ServerCallContext context
+    )
+    {
+        var ids = request.Ids
+            .Where(s => !string.IsNullOrWhiteSpace(s) && Guid.TryParse(s, out _))
+            .Select(Guid.Parse)
+            .ToList();
+
+        if (ids.Count == 0)
+            return new GetWebArticleBatchResponse();
+
+        var articles = await db.WebArticles
+            .Include(a => a.Feed)
+            .Where(a => ids.Contains(a.Id))
+            .ToListAsync();
+
+        var response = new GetWebArticleBatchResponse();
+        response.Articles.AddRange(articles.Select(a => a.ToProtoValue()));
+        return response;
+    }
+
+    public override async Task<ListWebArticlesResponse> ListWebArticles(
+        ListWebArticlesRequest request,
+        ServerCallContext context
+    )
+    {
+        if (!Guid.TryParse(request.FeedId, out var feedId))
+            throw new RpcException(new Status(StatusCode.InvalidArgument, "invalid feed_id"));
+
+        var query = db.WebArticles
+            .Include(a => a.Feed)
+            .Where(a => a.FeedId == feedId);
+
+        var articles = await query.ToListAsync();
+
+        var response = new ListWebArticlesResponse
+        {
+            TotalSize = articles.Count
+        };
+        response.Articles.AddRange(articles.Select(a => a.ToProtoValue()));
+        return response;
+    }
+
+    public override async Task<GetRecentArticlesResponse> GetRecentArticles(
+        GetRecentArticlesRequest request,
+        ServerCallContext context
+    )
+    {
+        var limit = request.Limit > 0 ? request.Limit : 20;
+
+        var articles = await db.WebArticles
+            .Include(a => a.Feed)
+            .OrderByDescending(a => a.PublishedAt ?? DateTime.MinValue)
+            .ThenByDescending(a => a.CreatedAt)
+            .Take(limit)
+            .ToListAsync();
+
+        var response = new GetRecentArticlesResponse();
+        response.Articles.AddRange(articles.Select(a => a.ToProtoValue()));
+        return response;
+    }
+}

DysonNetwork.Insight/Reader/WebFeedController.cs

@@ -2,6 +2,7 @@ using System.ComponentModel.DataAnnotations;
 using DysonNetwork.Shared.Models;
 using DysonNetwork.Shared.Proto;
 using DysonNetwork.Shared.Registry;
+using WebFeedConfig = DysonNetwork.Shared.Models.WebFeedConfig;
 using Microsoft.AspNetCore.Authorization;
 using Microsoft.AspNetCore.Mvc;
 

DysonNetwork.Insight/Reader/WebFeedGrpcService.cs

@@ -0,0 +1,55 @@
+using DysonNetwork.Shared.Models;
+using DysonNetwork.Shared.Proto;
+using Grpc.Core;
+using Microsoft.EntityFrameworkCore;
+
+namespace DysonNetwork.Insight.Reader;
+
+public class WebFeedGrpcService(WebFeedService service, AppDatabase db)
+    : Shared.Proto.WebFeedService.WebFeedServiceBase
+{
+    public override async Task<GetWebFeedResponse> GetWebFeed(
+        GetWebFeedRequest request,
+        ServerCallContext context
+    )
+    {
+        SnWebFeed? feed = null;
+
+        switch (request.IdentifierCase)
+        {
+            case GetWebFeedRequest.IdentifierOneofCase.Id:
+                if (!string.IsNullOrWhiteSpace(request.Id) && Guid.TryParse(request.Id, out var id))
+                    feed = await service.GetFeedAsync(id);
+                break;
+            case GetWebFeedRequest.IdentifierOneofCase.Url:
+                feed = await db.WebFeeds.FirstOrDefaultAsync(f => f.Url == request.Url);
+                break;
+            case GetWebFeedRequest.IdentifierOneofCase.None:
+                break;
+            default:
+                throw new ArgumentOutOfRangeException();
+        }
+
+        return feed == null
+            ? throw new RpcException(new Status(StatusCode.NotFound, "feed not found"))
+            : new GetWebFeedResponse { Feed = feed.ToProtoValue() };
+    }
+
+    public override async Task<ListWebFeedsResponse> ListWebFeeds(
+        ListWebFeedsRequest request,
+        ServerCallContext context
+    )
+    {
+        if (!Guid.TryParse(request.PublisherId, out var publisherId))
+            throw new RpcException(new Status(StatusCode.InvalidArgument, "invalid publisher_id"));
+
+        var feeds = await service.GetFeedsByPublisherAsync(publisherId);
+
+        var response = new ListWebFeedsResponse
+        {
+            TotalSize = feeds.Count
+        };
+        response.Feeds.AddRange(feeds.Select(f => f.ToProtoValue()));
+        return response;
+    }
+}

DysonNetwork.Insight/Reader/WebReaderGrpcService.cs

@@ -0,0 +1,49 @@
+using DysonNetwork.Shared.Proto;
+using Grpc.Core;
+
+namespace DysonNetwork.Insight.Reader;
+
+public class WebReaderGrpcService(WebReaderService service) : Shared.Proto.WebReaderService.WebReaderServiceBase
+{
+    public override async Task<ScrapeArticleResponse> ScrapeArticle(
+        ScrapeArticleRequest request,
+        ServerCallContext context
+    )
+    {
+        if (string.IsNullOrWhiteSpace(request.Url))
+            throw new RpcException(new Status(StatusCode.InvalidArgument, "url is required"));
+
+        var scrapedArticle = await service.ScrapeArticleAsync(request.Url, context.CancellationToken);
+        return new ScrapeArticleResponse { Article = scrapedArticle.ToProtoValue() };
+    }
+
+    public override async Task<GetLinkPreviewResponse> GetLinkPreview(
+        GetLinkPreviewRequest request,
+        ServerCallContext context
+    )
+    {
+        if (string.IsNullOrWhiteSpace(request.Url))
+            throw new RpcException(new Status(StatusCode.InvalidArgument, "url is required"));
+
+        var linkEmbed = await service.GetLinkPreviewAsync(
+            request.Url,
+            context.CancellationToken,
+            bypassCache: request.BypassCache
+        );
+
+        return new GetLinkPreviewResponse { Preview = linkEmbed.ToProtoValue() };
+    }
+
+    public override async Task<InvalidateLinkPreviewCacheResponse> InvalidateLinkPreviewCache(
+        InvalidateLinkPreviewCacheRequest request,
+        ServerCallContext context
+    )
+    {
+        if (string.IsNullOrWhiteSpace(request.Url))
+            throw new RpcException(new Status(StatusCode.InvalidArgument, "url is required"));
+
+        await service.InvalidateCacheForUrlAsync(request.Url);
+
+        return new InvalidateLinkPreviewCacheResponse { Success = true };
+    }
+}