SolarNetwork/Swarm
2
3
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

♻️ Move the web reader to insight completely

Browse Source
This commit is contained in:
LittleSheep
2026-01-02 01:23:45 +08:00
parent ede49333f8
commit 07b8c99682
65 changed files with 806 additions and 864 deletions
Download Patch File Download Diff File Expand all files Collapse all files

12
DysonNetwork.Insight/DysonNetwork.Insight.csproj
View File

@@ -8,6 +8,14 @@
<ItemGroup>
<PackageReference Include="AngleSharp" Version="1.4.0" />
<PackageReference Include="Google.Protobuf" Version="3.33.2" />
<PackageReference Include="Grpc.AspNetCore.Server.ClientFactory" Version="2.76.0" />
<PackageReference Include="Grpc.AspNetCore.Server.Reflection" Version="2.76.0" />
<PackageReference Include="Grpc.Net.Client" Version="2.76.0" />
<PackageReference Include="Grpc.Tools" Version="2.76.0">
<PrivateAssets>all</PrivateAssets>
<IncludeAssets>runtime; build; native; contentfiles; analyzers; buildtransitive</IncludeAssets>
</PackageReference>
<PackageReference Include="Microsoft.AspNetCore.OpenApi" Version="10.0.1" />
<PackageReference Include="Microsoft.EntityFrameworkCore.Design" Version="10.0.1">
<PrivateAssets>all</PrivateAssets>
@@ -30,4 +38,8 @@
<Folder Include="Controllers\" />
</ItemGroup>
<ItemGroup>
<Protobuf Remove="..\DysonNetwork.Shared\Proto\**" />
</ItemGroup>
</Project>

3
DysonNetwork.Insight/Program.cs
View File

@@ -11,6 +11,9 @@ builder.AddServiceDefaults();
builder.ConfigureAppKestrel(builder.Configuration);
builder.Services.AddGrpc();
builder.Services.AddGrpcReflection();
builder.Services.AddControllers();
builder.Services.AddAppServices();
builder.Services.AddAppAuthentication();

26
DysonNetwork.Insight/Reader/ScrapedArticle.cs
View File

@@ -1,9 +1,33 @@
using DysonNetwork.Shared.Models.Embed;
using DysonNetwork.Shared.Proto;
using EmbedLinkEmbed = DysonNetwork.Shared.Models.Embed.LinkEmbed;
namespace DysonNetwork.Insight.Reader;
public class ScrapedArticle
{
public LinkEmbed LinkEmbed { get; set; } = null!;
public EmbedLinkEmbed LinkEmbed { get; set; } = null!;
public string? Content { get; set; }
public Shared.Proto.ScrapedArticle ToProtoValue()
{
var proto = new Shared.Proto.ScrapedArticle
{
LinkEmbed = LinkEmbed.ToProtoValue()
};
if (!string.IsNullOrEmpty(Content))
proto.Content = Content;
return proto;
}
public static ScrapedArticle FromProtoValue(Shared.Proto.ScrapedArticle proto)
{
return new ScrapedArticle
{
LinkEmbed = EmbedLinkEmbed.FromProtoValue(proto.LinkEmbed),
Content = proto.Content == "" ? null : proto.Content
};
}
}

90
DysonNetwork.Insight/Reader/WebArticleGrpcService.cs Normal file
View File

@@ -0,0 +1,90 @@
using DysonNetwork.Shared.Models;
using DysonNetwork.Shared.Proto;
using Grpc.Core;
using Microsoft.EntityFrameworkCore;
namespace DysonNetwork.Insight.Reader;
public class WebArticleGrpcService(AppDatabase db) : WebArticleService.WebArticleServiceBase
{
public override async Task<GetWebArticleResponse> GetWebArticle(
GetWebArticleRequest request,
ServerCallContext context
)
{
if (!Guid.TryParse(request.Id, out var id))
throw new RpcException(new Status(StatusCode.InvalidArgument, "invalid id"));
var article = await db.WebArticles
.Include(a => a.Feed)
.FirstOrDefaultAsync(a => a.Id == id);
return article == null
? throw new RpcException(new Status(StatusCode.NotFound, "article not found"))
: new GetWebArticleResponse { Article = article.ToProtoValue() };
}
public override async Task<GetWebArticleBatchResponse> GetWebArticleBatch(
GetWebArticleBatchRequest request,
ServerCallContext context
)
{
var ids = request.Ids
.Where(s => !string.IsNullOrWhiteSpace(s) && Guid.TryParse(s, out _))
.Select(Guid.Parse)
.ToList();
if (ids.Count == 0)
return new GetWebArticleBatchResponse();
var articles = await db.WebArticles
.Include(a => a.Feed)
.Where(a => ids.Contains(a.Id))
.ToListAsync();
var response = new GetWebArticleBatchResponse();
response.Articles.AddRange(articles.Select(a => a.ToProtoValue()));
return response;
}
public override async Task<ListWebArticlesResponse> ListWebArticles(
ListWebArticlesRequest request,
ServerCallContext context
)
{
if (!Guid.TryParse(request.FeedId, out var feedId))
throw new RpcException(new Status(StatusCode.InvalidArgument, "invalid feed_id"));
var query = db.WebArticles
.Include(a => a.Feed)
.Where(a => a.FeedId == feedId);
var articles = await query.ToListAsync();
var response = new ListWebArticlesResponse
{
TotalSize = articles.Count
};
response.Articles.AddRange(articles.Select(a => a.ToProtoValue()));
return response;
}
public override async Task<GetRecentArticlesResponse> GetRecentArticles(
GetRecentArticlesRequest request,
ServerCallContext context
)
{
var limit = request.Limit > 0 ? request.Limit : 20;
var articles = await db.WebArticles
.Include(a => a.Feed)
.OrderByDescending(a => a.PublishedAt ?? DateTime.MinValue)
.ThenByDescending(a => a.CreatedAt)
.Take(limit)
.ToListAsync();
var response = new GetRecentArticlesResponse();
response.Articles.AddRange(articles.Select(a => a.ToProtoValue()));
return response;
}
}

1
DysonNetwork.Insight/Reader/WebFeedController.cs
View File

@@ -2,6 +2,7 @@ using System.ComponentModel.DataAnnotations;
using DysonNetwork.Shared.Models;
using DysonNetwork.Shared.Proto;
using DysonNetwork.Shared.Registry;
using WebFeedConfig = DysonNetwork.Shared.Models.WebFeedConfig;
using Microsoft.AspNetCore.Authorization;
using Microsoft.AspNetCore.Mvc;

55
DysonNetwork.Insight/Reader/WebFeedGrpcService.cs Normal file
View File

@@ -0,0 +1,55 @@
using DysonNetwork.Shared.Models;
using DysonNetwork.Shared.Proto;
using Grpc.Core;
using Microsoft.EntityFrameworkCore;
namespace DysonNetwork.Insight.Reader;
public class WebFeedGrpcService(WebFeedService service, AppDatabase db)
: Shared.Proto.WebFeedService.WebFeedServiceBase
{
public override async Task<GetWebFeedResponse> GetWebFeed(
GetWebFeedRequest request,
ServerCallContext context
)
{
SnWebFeed? feed = null;
switch (request.IdentifierCase)
{
case GetWebFeedRequest.IdentifierOneofCase.Id:
if (!string.IsNullOrWhiteSpace(request.Id) && Guid.TryParse(request.Id, out var id))
feed = await service.GetFeedAsync(id);
break;
case GetWebFeedRequest.IdentifierOneofCase.Url:
feed = await db.WebFeeds.FirstOrDefaultAsync(f => f.Url == request.Url);
break;
case GetWebFeedRequest.IdentifierOneofCase.None:
break;
default:
throw new ArgumentOutOfRangeException();
}
return feed == null
? throw new RpcException(new Status(StatusCode.NotFound, "feed not found"))
: new GetWebFeedResponse { Feed = feed.ToProtoValue() };
}
public override async Task<ListWebFeedsResponse> ListWebFeeds(
ListWebFeedsRequest request,
ServerCallContext context
)
{
if (!Guid.TryParse(request.PublisherId, out var publisherId))
throw new RpcException(new Status(StatusCode.InvalidArgument, "invalid publisher_id"));
var feeds = await service.GetFeedsByPublisherAsync(publisherId);
var response = new ListWebFeedsResponse
{
TotalSize = feeds.Count
};
response.Feeds.AddRange(feeds.Select(f => f.ToProtoValue()));
return response;
}
}

49
DysonNetwork.Insight/Reader/WebReaderGrpcService.cs Normal file
View File

@@ -0,0 +1,49 @@
using DysonNetwork.Shared.Proto;
using Grpc.Core;
namespace DysonNetwork.Insight.Reader;
public class WebReaderGrpcService(WebReaderService service) : Shared.Proto.WebReaderService.WebReaderServiceBase
{
public override async Task<ScrapeArticleResponse> ScrapeArticle(
ScrapeArticleRequest request,
ServerCallContext context
)
{
if (string.IsNullOrWhiteSpace(request.Url))
throw new RpcException(new Status(StatusCode.InvalidArgument, "url is required"));
var scrapedArticle = await service.ScrapeArticleAsync(request.Url, context.CancellationToken);
return new ScrapeArticleResponse { Article = scrapedArticle.ToProtoValue() };
}
public override async Task<GetLinkPreviewResponse> GetLinkPreview(
GetLinkPreviewRequest request,
ServerCallContext context
)
{
if (string.IsNullOrWhiteSpace(request.Url))
throw new RpcException(new Status(StatusCode.InvalidArgument, "url is required"));
var linkEmbed = await service.GetLinkPreviewAsync(
request.Url,
context.CancellationToken,
bypassCache: request.BypassCache
);
return new GetLinkPreviewResponse { Preview = linkEmbed.ToProtoValue() };
}
public override async Task<InvalidateLinkPreviewCacheResponse> InvalidateLinkPreviewCache(
InvalidateLinkPreviewCacheRequest request,
ServerCallContext context
)
{
if (string.IsNullOrWhiteSpace(request.Url))
throw new RpcException(new Status(StatusCode.InvalidArgument, "url is required"));
await service.InvalidateCacheForUrlAsync(request.Url);
return new InvalidateLinkPreviewCacheResponse { Success = true };
}
}

6
DysonNetwork.Insight/Startup/ApplicationConfiguration.cs
View File

@@ -1,3 +1,4 @@
using DysonNetwork.Insight.Reader;
using DysonNetwork.Shared.Http;
namespace DysonNetwork.Insight.Startup;
@@ -17,6 +18,11 @@ public static class ApplicationConfiguration
app.MapControllers();
app.MapGrpcService<WebReaderGrpcService>();
app.MapGrpcService<WebArticleGrpcService>();
app.MapGrpcService<WebFeedGrpcService>();
app.MapGrpcReflectionService();
return app;
}
}