SolarNetwork/DysonNetwork
2
3
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

♻️ Move the web reader to insight completely

Browse Source
This commit is contained in:
LittleSheep
2026-01-02 01:23:45 +08:00
parent ede49333f8
commit 07b8c99682
65 changed files with 806 additions and 864 deletions
Download Patch File Download Diff File Expand all files Collapse all files

53
DysonNetwork.Shared/Models/Embed/LinkEmbed.cs
View File

@@ -1,3 +1,6 @@
using DysonNetwork.Shared.Proto;
using Google.Protobuf.WellKnownTypes;
namespace DysonNetwork.Shared.Models.Embed;
/// <summary>
@@ -52,4 +55,54 @@ public class LinkEmbed : EmbeddableBase
/// Published date of the content if available
/// </summary>
public DateTime? PublishedDate { get; set; }
public Proto.LinkEmbed ToProtoValue()
{
var proto = new Proto.LinkEmbed
{
Url = Url
};
if (!string.IsNullOrEmpty(Title))
proto.Title = Title;
if (!string.IsNullOrEmpty(Description))
proto.Description = Description;
if (!string.IsNullOrEmpty(ImageUrl))
proto.ImageUrl = ImageUrl;
if (!string.IsNullOrEmpty(FaviconUrl))
proto.FaviconUrl = FaviconUrl;
if (!string.IsNullOrEmpty(SiteName))
proto.SiteName = SiteName;
if (!string.IsNullOrEmpty(ContentType))
proto.ContentType = ContentType;
if (!string.IsNullOrEmpty(Author))
proto.Author = Author;
if (PublishedDate.HasValue)
proto.PublishedDate = Timestamp.FromDateTime(PublishedDate.Value.ToUniversalTime());
return proto;
}
public static LinkEmbed FromProtoValue(Proto.LinkEmbed proto)
{
return new LinkEmbed
{
Url = proto.Url,
Title = proto.Title == "" ? null : proto.Title,
Description = proto.Description == "" ? null : proto.Description,
ImageUrl = proto.ImageUrl == "" ? null : proto.ImageUrl,
FaviconUrl = proto.FaviconUrl == "" ? null : proto.FaviconUrl,
SiteName = proto.SiteName == "" ? null : proto.SiteName,
ContentType = proto.ContentType == "" ? null : proto.ContentType,
Author = proto.Author == "" ? null : proto.Author,
PublishedDate = proto.PublishedDate != null ? proto.PublishedDate.ToDateTime() : null
};
}
}

163
DysonNetwork.Shared/Models/WebArticle.cs
View File

@@ -2,7 +2,10 @@ using System.ComponentModel.DataAnnotations;
using System.ComponentModel.DataAnnotations.Schema;
using System.Text.Json.Serialization;
using DysonNetwork.Shared.Models.Embed;
using DysonNetwork.Shared.Proto;
using Google.Protobuf.WellKnownTypes;
using NodaTime;
using EmbedLinkEmbed = DysonNetwork.Shared.Models.Embed.LinkEmbed;
namespace DysonNetwork.Shared.Models;
@@ -13,9 +16,9 @@ public class SnWebArticle : ModelBase
[MaxLength(4096)] public string Title { get; set; } = null!;
[MaxLength(8192)] public string Url { get; set; } = null!;
[MaxLength(4096)] public string? Author { get; set; }
[Column(TypeName = "jsonb")] public Dictionary<string, object>? Meta { get; set; }
[Column(TypeName = "jsonb")] public LinkEmbed? Preview { get; set; }
[Column(TypeName = "jsonb")] public EmbedLinkEmbed? Preview { get; set; }
// ReSharper disable once EntityFramework.ModelValidation.UnlimitedStringLength
public string? Content { get; set; }
@@ -24,11 +27,79 @@ public class SnWebArticle : ModelBase
public Guid FeedId { get; set; }
public SnWebFeed Feed { get; set; } = null!;
public WebArticle ToProtoValue()
{
var proto = new WebArticle
{
Id = Id.ToString(),
Title = Title,
Url = Url,
FeedId = FeedId.ToString(),
CreatedAt = Timestamp.FromDateTimeOffset(CreatedAt.ToDateTimeOffset()),
UpdatedAt = Timestamp.FromDateTimeOffset(UpdatedAt.ToDateTimeOffset())
};
if (!string.IsNullOrEmpty(Author))
proto.Author = Author;
if (Meta != null)
proto.Meta = GrpcTypeHelper.ConvertObjectToByteString(Meta);
if (Preview != null)
proto.Preview = Preview.ToProtoValue();
if (!string.IsNullOrEmpty(Content))
proto.Content = Content;
if (PublishedAt.HasValue)
proto.PublishedAt = Timestamp.FromDateTime(PublishedAt.Value.ToUniversalTime());
if (DeletedAt.HasValue)
proto.DeletedAt = Timestamp.FromDateTimeOffset(DeletedAt.Value.ToDateTimeOffset());
return proto;
}
public static SnWebArticle FromProtoValue(WebArticle proto)
{
return new SnWebArticle
{
Id = Guid.Parse(proto.Id),
Title = proto.Title,
Url = proto.Url,
FeedId = Guid.Parse(proto.FeedId),
Author = proto.Author == "" ? null : proto.Author,
Meta = proto.Meta != null ? GrpcTypeHelper.ConvertByteStringToObject<Dictionary<string, object>>(proto.Meta) : null,
Preview = proto.Preview != null ? EmbedLinkEmbed.FromProtoValue(proto.Preview) : null,
Content = proto.Content == "" ? null : proto.Content,
PublishedAt = proto.PublishedAt != null ? proto.PublishedAt.ToDateTime() : null,
CreatedAt = Instant.FromDateTimeOffset(proto.CreatedAt.ToDateTimeOffset()),
UpdatedAt = Instant.FromDateTimeOffset(proto.UpdatedAt.ToDateTimeOffset()),
DeletedAt = proto.DeletedAt != null ? Instant.FromDateTimeOffset(proto.DeletedAt.ToDateTimeOffset()) : null
};
}
}
public class WebFeedConfig
{
public bool ScrapPage { get; set; }
public Proto.WebFeedConfig ToProtoValue()
{
return new Proto.WebFeedConfig
{
ScrapPage = ScrapPage
};
}
public static WebFeedConfig FromProtoValue(Proto.WebFeedConfig proto)
{
return new WebFeedConfig
{
ScrapPage = proto.ScrapPage
};
}
}
public class SnWebFeed : ModelBase
@@ -37,25 +108,105 @@ public class SnWebFeed : ModelBase
[MaxLength(8192)] public string Url { get; set; } = null!;
[MaxLength(4096)] public string Title { get; set; } = null!;
[MaxLength(8192)] public string? Description { get; set; }
public Instant? VerifiedAt { get; set; }
[JsonIgnore] [MaxLength(8192)] public string? VerificationKey { get; set; }
[Column(TypeName = "jsonb")] public LinkEmbed? Preview { get; set; }
[Column(TypeName = "jsonb")] public EmbedLinkEmbed? Preview { get; set; }
[Column(TypeName = "jsonb")] public WebFeedConfig Config { get; set; } = new();
public Guid PublisherId { get; set; }
public SnPublisher Publisher { get; set; } = null!;
[JsonIgnore] public List<SnWebArticle> Articles { get; set; } = new();
public WebFeed ToProtoValue()
{
var proto = new WebFeed
{
Id = Id.ToString(),
Url = Url,
Title = Title,
Config = Config.ToProtoValue(),
PublisherId = PublisherId.ToString(),
CreatedAt = Timestamp.FromDateTimeOffset(CreatedAt.ToDateTimeOffset()),
UpdatedAt = Timestamp.FromDateTimeOffset(UpdatedAt.ToDateTimeOffset())
};
if (!string.IsNullOrEmpty(Description))
proto.Description = Description;
if (VerifiedAt.HasValue)
proto.VerifiedAt = Timestamp.FromDateTimeOffset(VerifiedAt.Value.ToDateTimeOffset());
if (Preview != null)
proto.Preview = Preview.ToProtoValue();
if (Publisher != null)
proto.Publisher = Publisher.ToProtoValue();
if (DeletedAt.HasValue)
proto.DeletedAt = Timestamp.FromDateTimeOffset(DeletedAt.Value.ToDateTimeOffset());
return proto;
}
public static SnWebFeed FromProtoValue(WebFeed proto)
{
return new SnWebFeed
{
Id = Guid.Parse(proto.Id),
Url = proto.Url,
Title = proto.Title,
Description = proto.Description == "" ? null : proto.Description,
VerifiedAt = proto.VerifiedAt != null ? Instant.FromDateTimeOffset(proto.VerifiedAt.ToDateTimeOffset()) : null,
Preview = proto.Preview != null ? EmbedLinkEmbed.FromProtoValue(proto.Preview) : null,
Config = WebFeedConfig.FromProtoValue(proto.Config),
PublisherId = Guid.Parse(proto.PublisherId),
Publisher = proto.Publisher != null ? SnPublisher.FromProtoValue(proto.Publisher) : null,
CreatedAt = Instant.FromDateTimeOffset(proto.CreatedAt.ToDateTimeOffset()),
UpdatedAt = Instant.FromDateTimeOffset(proto.UpdatedAt.ToDateTimeOffset()),
DeletedAt = proto.DeletedAt != null ? Instant.FromDateTimeOffset(proto.DeletedAt.ToDateTimeOffset()) : null
};
}
}
public class SnWebFeedSubscription : ModelBase
{
public Guid Id { get; set; } = Guid.NewGuid();
public Guid FeedId { get; set; }
public SnWebFeed Feed { get; set; } = null!;
public Guid AccountId { get; set; }
[NotMapped] public SnAccount Account { get; set; } = null!;
public WebFeedSubscription ToProtoValue()
{
var proto = new WebFeedSubscription
{
Id = Id.ToString(),
FeedId = FeedId.ToString(),
AccountId = AccountId.ToString(),
CreatedAt = Timestamp.FromDateTimeOffset(CreatedAt.ToDateTimeOffset()),
UpdatedAt = Timestamp.FromDateTimeOffset(UpdatedAt.ToDateTimeOffset())
};
if (Feed != null)
proto.Feed = Feed.ToProtoValue();
return proto;
}
public static SnWebFeedSubscription FromProtoValue(WebFeedSubscription proto)
{
return new SnWebFeedSubscription
{
Id = Guid.Parse(proto.Id),
FeedId = Guid.Parse(proto.FeedId),
Feed = proto.Feed != null ? SnWebFeed.FromProtoValue(proto.Feed) : null,
AccountId = Guid.Parse(proto.AccountId),
CreatedAt = Instant.FromDateTimeOffset(proto.CreatedAt.ToDateTimeOffset()),
UpdatedAt = Instant.FromDateTimeOffset(proto.UpdatedAt.ToDateTimeOffset())
};
}
}

19
DysonNetwork.Shared/Proto/embed.proto Normal file
View File

@@ -0,0 +1,19 @@
syntax = "proto3";
package proto;
option csharp_namespace = "DysonNetwork.Shared.Proto";
import "google/protobuf/timestamp.proto";
message LinkEmbed {
string url = 1;
optional string title = 2;
optional string description = 3;
optional string image_url = 4;
optional string favicon_url = 5;
optional string site_name = 6;
optional string content_type = 7;
optional string author = 8;
optional google.protobuf.Timestamp published_date = 9;
}

160
DysonNetwork.Shared/Proto/web_article.proto Normal file
View File

@@ -0,0 +1,160 @@
syntax = "proto3";
package proto;
option csharp_namespace = "DysonNetwork.Shared.Proto";
import "google/protobuf/timestamp.proto";
import "embed.proto";
import "publisher.proto";
message WebFeedConfig {
bool scrap_page = 1;
}
message WebFeed {
string id = 1;
string url = 2;
string title = 3;
optional string description = 4;
optional google.protobuf.Timestamp verified_at = 5;
optional LinkEmbed preview = 6;
WebFeedConfig config = 7;
string publisher_id = 8;
optional Publisher publisher = 9;
google.protobuf.Timestamp created_at = 10;
google.protobuf.Timestamp updated_at = 11;
optional google.protobuf.Timestamp deleted_at = 12;
}
message WebArticle {
string id = 1;
string title = 2;
string url = 3;
optional string author = 4;
optional bytes meta = 5;
optional LinkEmbed preview = 6;
optional string content = 7;
optional google.protobuf.Timestamp published_at = 8;
string feed_id = 9;
optional WebFeed feed = 10;
google.protobuf.Timestamp created_at = 11;
google.protobuf.Timestamp updated_at = 12;
optional google.protobuf.Timestamp deleted_at = 13;
}
message WebFeedSubscription {
string id = 1;
string feed_id = 2;
optional WebFeed feed = 3;
string account_id = 4;
google.protobuf.Timestamp created_at = 5;
google.protobuf.Timestamp updated_at = 6;
}
message ScrapedArticle {
LinkEmbed link_embed = 1;
optional string content = 2;
}
message GetWebArticleRequest {
string id = 1;
}
message GetWebArticleResponse {
WebArticle article = 1;
}
message GetWebArticleBatchRequest {
repeated string ids = 1;
}
message GetWebArticleBatchResponse {
repeated WebArticle articles = 1;
}
message ListWebArticlesRequest {
string feed_id = 1;
int32 page_size = 2;
string page_token = 3;
}
message ListWebArticlesResponse {
repeated WebArticle articles = 1;
string next_page_token = 2;
int32 total_size = 3;
}
message GetRecentArticlesRequest {
int32 limit = 1;
}
message GetRecentArticlesResponse {
repeated WebArticle articles = 1;
}
message GetWebFeedRequest {
oneof identifier {
string id = 1;
string url = 2;
}
}
message GetWebFeedResponse {
WebFeed feed = 1;
}
message ListWebFeedsRequest {
string publisher_id = 1;
int32 page_size = 2;
string page_token = 3;
}
message ListWebFeedsResponse {
repeated WebFeed feeds = 1;
string next_page_token = 2;
int32 total_size = 3;
}
message ScrapeArticleRequest {
string url = 1;
}
message ScrapeArticleResponse {
ScrapedArticle article = 1;
}
message GetLinkPreviewRequest {
string url = 1;
bool bypass_cache = 2;
}
message GetLinkPreviewResponse {
LinkEmbed preview = 1;
}
message InvalidateLinkPreviewCacheRequest {
string url = 1;
}
message InvalidateLinkPreviewCacheResponse {
bool success = 1;
}
service WebArticleService {
rpc GetWebArticle(GetWebArticleRequest) returns (GetWebArticleResponse);
rpc GetWebArticleBatch(GetWebArticleBatchRequest) returns (GetWebArticleBatchResponse);
rpc ListWebArticles(ListWebArticlesRequest) returns (ListWebArticlesResponse);
rpc GetRecentArticles(GetRecentArticlesRequest) returns (GetRecentArticlesResponse);
}
service WebFeedService {
rpc GetWebFeed(GetWebFeedRequest) returns (GetWebFeedResponse);
rpc ListWebFeeds(ListWebFeedsRequest) returns (ListWebFeedsResponse);
}
service WebReaderService {
rpc ScrapeArticle(ScrapeArticleRequest) returns (ScrapeArticleResponse);
rpc GetLinkPreview(GetLinkPreviewRequest) returns (GetLinkPreviewResponse);
rpc InvalidateLinkPreviewCache(InvalidateLinkPreviewCacheRequest) returns (InvalidateLinkPreviewCacheResponse);
}

36
DysonNetwork.Shared/Registry/RemoteWebArticleService.cs Normal file
View File

@@ -0,0 +1,36 @@
using DysonNetwork.Shared.Models;
using DysonNetwork.Shared.Proto;
namespace DysonNetwork.Shared.Registry;
public class RemoteWebArticleService(WebArticleService.WebArticleServiceClient webArticles)
{
public async Task<SnWebArticle> GetWebArticle(Guid id)
{
var request = new GetWebArticleRequest { Id = id.ToString() };
var response = await webArticles.GetWebArticleAsync(request);
return response.Article != null ? SnWebArticle.FromProtoValue(response.Article) : null!;
}
public async Task<List<SnWebArticle>> GetWebArticleBatch(List<Guid> ids)
{
var request = new GetWebArticleBatchRequest();
request.Ids.AddRange(ids.Select(id => id.ToString()));
var response = await webArticles.GetWebArticleBatchAsync(request);
return response.Articles.Select(SnWebArticle.FromProtoValue).ToList();
}
public async Task<List<SnWebArticle>> ListWebArticles(Guid feedId)
{
var request = new ListWebArticlesRequest { FeedId = feedId.ToString() };
var response = await webArticles.ListWebArticlesAsync(request);
return response.Articles.Select(SnWebArticle.FromProtoValue).ToList();
}
public async Task<List<SnWebArticle>> GetRecentArticles(int limit = 20)
{
var request = new GetRecentArticlesRequest { Limit = limit };
var response = await webArticles.GetRecentArticlesAsync(request);
return response.Articles.Select(SnWebArticle.FromProtoValue).ToList();
}
}

28
DysonNetwork.Shared/Registry/RemoteWebFeedService.cs Normal file
View File

@@ -0,0 +1,28 @@
using DysonNetwork.Shared.Models;
using DysonNetwork.Shared.Proto;
namespace DysonNetwork.Shared.Registry;
public class RemoteWebFeedService(WebFeedService.WebFeedServiceClient webFeeds)
{
public async Task<SnWebFeed> GetWebFeed(Guid id)
{
var request = new GetWebFeedRequest { Id = id.ToString() };
var response = await webFeeds.GetWebFeedAsync(request);
return response.Feed != null ? SnWebFeed.FromProtoValue(response.Feed) : null!;
}
public async Task<SnWebFeed> GetWebFeedByUrl(string url)
{
var request = new GetWebFeedRequest { Url = url };
var response = await webFeeds.GetWebFeedAsync(request);
return response.Feed != null ? SnWebFeed.FromProtoValue(response.Feed) : null!;
}
public async Task<List<SnWebFeed>> ListWebFeeds(Guid publisherId)
{
var request = new ListWebFeedsRequest { PublisherId = publisherId.ToString() };
var response = await webFeeds.ListWebFeedsAsync(request);
return response.Feeds.Select(SnWebFeed.FromProtoValue).ToList();
}
}

34
DysonNetwork.Shared/Registry/RemoteWebReaderService.cs Normal file
View File

@@ -0,0 +1,34 @@
using DysonNetwork.Shared.Models;
using DysonNetwork.Shared.Models.Embed;
using DysonNetwork.Shared.Proto;
using ProtoLinkEmbed = DysonNetwork.Shared.Proto.LinkEmbed;
using ModelsLinkEmbed = DysonNetwork.Shared.Models.Embed.LinkEmbed;
namespace DysonNetwork.Shared.Registry;
public class RemoteWebReaderService(WebReaderService.WebReaderServiceClient webReader)
{
public async Task<(ModelsLinkEmbed LinkEmbed, string? Content)> ScrapeArticle(string url)
{
var request = new ScrapeArticleRequest { Url = url };
var response = await webReader.ScrapeArticleAsync(request);
return (
LinkEmbed: response.Article?.LinkEmbed != null ? ModelsLinkEmbed.FromProtoValue(response.Article.LinkEmbed) : null!,
Content: response.Article?.Content == "" ? null : response.Article?.Content
);
}
public async Task<ModelsLinkEmbed> GetLinkPreview(string url, bool bypassCache = false)
{
var request = new GetLinkPreviewRequest { Url = url, BypassCache = bypassCache };
var response = await webReader.GetLinkPreviewAsync(request);
return response.Preview != null ? ModelsLinkEmbed.FromProtoValue(response.Preview) : null!;
}
public async Task<bool> InvalidateLinkPreviewCache(string url)
{
var request = new InvalidateLinkPreviewCacheRequest { Url = url };
var response = await webReader.InvalidateLinkPreviewCacheAsync(request);
return response.Success;
}
}

24
DysonNetwork.Shared/Registry/ServiceInjectionHelper.cs
View File

@@ -21,7 +21,6 @@ public static class ServiceInjectionHelper
services.AddGrpcClientWithSharedChannel<AuthService.AuthServiceClient>(
"https://_grpc.pass",
"AuthService");
services.AddGrpcClientWithSharedChannel<PermissionService.PermissionServiceClient>(
"https://_grpc.pass",
"PermissionService");
@@ -39,19 +38,15 @@ public static class ServiceInjectionHelper
services.AddGrpcClientWithSharedChannel<BotAccountReceiverService.BotAccountReceiverServiceClient>(
"https://_grpc.pass",
"BotAccountReceiverService");
services.AddGrpcClientWithSharedChannel<ActionLogService.ActionLogServiceClient>(
"https://_grpc.pass",
"ActionLogService");
services.AddGrpcClientWithSharedChannel<PaymentService.PaymentServiceClient>(
"https://_grpc.pass",
"PaymentService");
services.AddGrpcClientWithSharedChannel<WalletService.WalletServiceClient>(
"https://_grpc.pass",
"WalletService");
services.AddGrpcClientWithSharedChannel<RealmService.RealmServiceClient>(
"https://_grpc.pass",
"RealmService");
@@ -107,5 +102,24 @@ public static class ServiceInjectionHelper
return services;
}
public IServiceCollection AddInsightService()
{
services.AddGrpcClientWithSharedChannel<WebFeedService.WebFeedServiceClient>(
"https://_grpc.insight",
"WebFeedServiceClient");
services.AddGrpcClientWithSharedChannel<WebArticleService.WebArticleServiceClient>(
"https://_grpc.insight",
"WebArticleService");
services.AddGrpcClientWithSharedChannel<WebReaderService.WebReaderServiceClient>(
"https://_grpc.insight",
"WebReaderServiceClient");
services.AddSingleton<RemoteWebFeedService>();
services.AddSingleton<RemoteWebReaderService>();
services.AddSingleton<RemoteWebArticleService>();
return services;
}
}
}