diff --git a/DysonNetwork.Insight/DysonNetwork.Insight.csproj b/DysonNetwork.Insight/DysonNetwork.Insight.csproj
index a685c46..3a020b8 100644
--- a/DysonNetwork.Insight/DysonNetwork.Insight.csproj
+++ b/DysonNetwork.Insight/DysonNetwork.Insight.csproj
@@ -8,6 +8,14 @@
+
+
+
+
+
+ all
+ runtime; build; native; contentfiles; analyzers; buildtransitive
+
all
@@ -30,4 +38,8 @@
+
+
+
+
diff --git a/DysonNetwork.Insight/Program.cs b/DysonNetwork.Insight/Program.cs
index 9682800..44a2278 100644
--- a/DysonNetwork.Insight/Program.cs
+++ b/DysonNetwork.Insight/Program.cs
@@ -11,6 +11,9 @@ builder.AddServiceDefaults();
builder.ConfigureAppKestrel(builder.Configuration);
+builder.Services.AddGrpc();
+builder.Services.AddGrpcReflection();
+
builder.Services.AddControllers();
builder.Services.AddAppServices();
builder.Services.AddAppAuthentication();
diff --git a/DysonNetwork.Insight/Reader/ScrapedArticle.cs b/DysonNetwork.Insight/Reader/ScrapedArticle.cs
index 2c48aa9..d53c92d 100644
--- a/DysonNetwork.Insight/Reader/ScrapedArticle.cs
+++ b/DysonNetwork.Insight/Reader/ScrapedArticle.cs
@@ -1,9 +1,33 @@
using DysonNetwork.Shared.Models.Embed;
+using DysonNetwork.Shared.Proto;
+using EmbedLinkEmbed = DysonNetwork.Shared.Models.Embed.LinkEmbed;
namespace DysonNetwork.Insight.Reader;
public class ScrapedArticle
{
- public LinkEmbed LinkEmbed { get; set; } = null!;
+ public EmbedLinkEmbed LinkEmbed { get; set; } = null!;
public string? Content { get; set; }
+
+ public Shared.Proto.ScrapedArticle ToProtoValue()
+ {
+ var proto = new Shared.Proto.ScrapedArticle
+ {
+ LinkEmbed = LinkEmbed.ToProtoValue()
+ };
+
+ if (!string.IsNullOrEmpty(Content))
+ proto.Content = Content;
+
+ return proto;
+ }
+
+ public static ScrapedArticle FromProtoValue(Shared.Proto.ScrapedArticle proto)
+ {
+ return new ScrapedArticle
+ {
+ LinkEmbed = EmbedLinkEmbed.FromProtoValue(proto.LinkEmbed),
+ Content = proto.Content == "" ? null : proto.Content
+ };
+ }
}
\ No newline at end of file
diff --git a/DysonNetwork.Insight/Reader/WebArticleGrpcService.cs b/DysonNetwork.Insight/Reader/WebArticleGrpcService.cs
new file mode 100644
index 0000000..c701a5c
--- /dev/null
+++ b/DysonNetwork.Insight/Reader/WebArticleGrpcService.cs
@@ -0,0 +1,90 @@
+using DysonNetwork.Shared.Models;
+using DysonNetwork.Shared.Proto;
+using Grpc.Core;
+using Microsoft.EntityFrameworkCore;
+
+namespace DysonNetwork.Insight.Reader;
+
+public class WebArticleGrpcService(AppDatabase db) : WebArticleService.WebArticleServiceBase
+{
+ public override async Task GetWebArticle(
+ GetWebArticleRequest request,
+ ServerCallContext context
+ )
+ {
+ if (!Guid.TryParse(request.Id, out var id))
+ throw new RpcException(new Status(StatusCode.InvalidArgument, "invalid id"));
+
+ var article = await db.WebArticles
+ .Include(a => a.Feed)
+ .FirstOrDefaultAsync(a => a.Id == id);
+
+ return article == null
+ ? throw new RpcException(new Status(StatusCode.NotFound, "article not found"))
+ : new GetWebArticleResponse { Article = article.ToProtoValue() };
+ }
+
+ public override async Task GetWebArticleBatch(
+ GetWebArticleBatchRequest request,
+ ServerCallContext context
+ )
+ {
+ var ids = request.Ids
+ .Where(s => !string.IsNullOrWhiteSpace(s) && Guid.TryParse(s, out _))
+ .Select(Guid.Parse)
+ .ToList();
+
+ if (ids.Count == 0)
+ return new GetWebArticleBatchResponse();
+
+ var articles = await db.WebArticles
+ .Include(a => a.Feed)
+ .Where(a => ids.Contains(a.Id))
+ .ToListAsync();
+
+ var response = new GetWebArticleBatchResponse();
+ response.Articles.AddRange(articles.Select(a => a.ToProtoValue()));
+ return response;
+ }
+
+ public override async Task ListWebArticles(
+ ListWebArticlesRequest request,
+ ServerCallContext context
+ )
+ {
+ if (!Guid.TryParse(request.FeedId, out var feedId))
+ throw new RpcException(new Status(StatusCode.InvalidArgument, "invalid feed_id"));
+
+ var query = db.WebArticles
+ .Include(a => a.Feed)
+ .Where(a => a.FeedId == feedId);
+
+ var articles = await query.ToListAsync();
+
+ var response = new ListWebArticlesResponse
+ {
+ TotalSize = articles.Count
+ };
+ response.Articles.AddRange(articles.Select(a => a.ToProtoValue()));
+ return response;
+ }
+
+ public override async Task GetRecentArticles(
+ GetRecentArticlesRequest request,
+ ServerCallContext context
+ )
+ {
+ var limit = request.Limit > 0 ? request.Limit : 20;
+
+ var articles = await db.WebArticles
+ .Include(a => a.Feed)
+ .OrderByDescending(a => a.PublishedAt ?? DateTime.MinValue)
+ .ThenByDescending(a => a.CreatedAt)
+ .Take(limit)
+ .ToListAsync();
+
+ var response = new GetRecentArticlesResponse();
+ response.Articles.AddRange(articles.Select(a => a.ToProtoValue()));
+ return response;
+ }
+}
\ No newline at end of file
diff --git a/DysonNetwork.Insight/Reader/WebFeedController.cs b/DysonNetwork.Insight/Reader/WebFeedController.cs
index 1b56afc..66f0d85 100644
--- a/DysonNetwork.Insight/Reader/WebFeedController.cs
+++ b/DysonNetwork.Insight/Reader/WebFeedController.cs
@@ -2,6 +2,7 @@ using System.ComponentModel.DataAnnotations;
using DysonNetwork.Shared.Models;
using DysonNetwork.Shared.Proto;
using DysonNetwork.Shared.Registry;
+using WebFeedConfig = DysonNetwork.Shared.Models.WebFeedConfig;
using Microsoft.AspNetCore.Authorization;
using Microsoft.AspNetCore.Mvc;
diff --git a/DysonNetwork.Insight/Reader/WebFeedGrpcService.cs b/DysonNetwork.Insight/Reader/WebFeedGrpcService.cs
new file mode 100644
index 0000000..c8ec847
--- /dev/null
+++ b/DysonNetwork.Insight/Reader/WebFeedGrpcService.cs
@@ -0,0 +1,55 @@
+using DysonNetwork.Shared.Models;
+using DysonNetwork.Shared.Proto;
+using Grpc.Core;
+using Microsoft.EntityFrameworkCore;
+
+namespace DysonNetwork.Insight.Reader;
+
+public class WebFeedGrpcService(WebFeedService service, AppDatabase db)
+ : Shared.Proto.WebFeedService.WebFeedServiceBase
+{
+ public override async Task GetWebFeed(
+ GetWebFeedRequest request,
+ ServerCallContext context
+ )
+ {
+ SnWebFeed? feed = null;
+
+ switch (request.IdentifierCase)
+ {
+ case GetWebFeedRequest.IdentifierOneofCase.Id:
+ if (!string.IsNullOrWhiteSpace(request.Id) && Guid.TryParse(request.Id, out var id))
+ feed = await service.GetFeedAsync(id);
+ break;
+ case GetWebFeedRequest.IdentifierOneofCase.Url:
+ feed = await db.WebFeeds.FirstOrDefaultAsync(f => f.Url == request.Url);
+ break;
+ case GetWebFeedRequest.IdentifierOneofCase.None:
+ break;
+ default:
+ throw new ArgumentOutOfRangeException();
+ }
+
+ return feed == null
+ ? throw new RpcException(new Status(StatusCode.NotFound, "feed not found"))
+ : new GetWebFeedResponse { Feed = feed.ToProtoValue() };
+ }
+
+ public override async Task ListWebFeeds(
+ ListWebFeedsRequest request,
+ ServerCallContext context
+ )
+ {
+ if (!Guid.TryParse(request.PublisherId, out var publisherId))
+ throw new RpcException(new Status(StatusCode.InvalidArgument, "invalid publisher_id"));
+
+ var feeds = await service.GetFeedsByPublisherAsync(publisherId);
+
+ var response = new ListWebFeedsResponse
+ {
+ TotalSize = feeds.Count
+ };
+ response.Feeds.AddRange(feeds.Select(f => f.ToProtoValue()));
+ return response;
+ }
+}
\ No newline at end of file
diff --git a/DysonNetwork.Insight/Reader/WebReaderGrpcService.cs b/DysonNetwork.Insight/Reader/WebReaderGrpcService.cs
new file mode 100644
index 0000000..8513159
--- /dev/null
+++ b/DysonNetwork.Insight/Reader/WebReaderGrpcService.cs
@@ -0,0 +1,49 @@
+using DysonNetwork.Shared.Proto;
+using Grpc.Core;
+
+namespace DysonNetwork.Insight.Reader;
+
+public class WebReaderGrpcService(WebReaderService service) : Shared.Proto.WebReaderService.WebReaderServiceBase
+{
+ public override async Task ScrapeArticle(
+ ScrapeArticleRequest request,
+ ServerCallContext context
+ )
+ {
+ if (string.IsNullOrWhiteSpace(request.Url))
+ throw new RpcException(new Status(StatusCode.InvalidArgument, "url is required"));
+
+ var scrapedArticle = await service.ScrapeArticleAsync(request.Url, context.CancellationToken);
+ return new ScrapeArticleResponse { Article = scrapedArticle.ToProtoValue() };
+ }
+
+ public override async Task GetLinkPreview(
+ GetLinkPreviewRequest request,
+ ServerCallContext context
+ )
+ {
+ if (string.IsNullOrWhiteSpace(request.Url))
+ throw new RpcException(new Status(StatusCode.InvalidArgument, "url is required"));
+
+ var linkEmbed = await service.GetLinkPreviewAsync(
+ request.Url,
+ context.CancellationToken,
+ bypassCache: request.BypassCache
+ );
+
+ return new GetLinkPreviewResponse { Preview = linkEmbed.ToProtoValue() };
+ }
+
+ public override async Task InvalidateLinkPreviewCache(
+ InvalidateLinkPreviewCacheRequest request,
+ ServerCallContext context
+ )
+ {
+ if (string.IsNullOrWhiteSpace(request.Url))
+ throw new RpcException(new Status(StatusCode.InvalidArgument, "url is required"));
+
+ await service.InvalidateCacheForUrlAsync(request.Url);
+
+ return new InvalidateLinkPreviewCacheResponse { Success = true };
+ }
+}
diff --git a/DysonNetwork.Insight/Startup/ApplicationConfiguration.cs b/DysonNetwork.Insight/Startup/ApplicationConfiguration.cs
index 8a30ed6..f012859 100644
--- a/DysonNetwork.Insight/Startup/ApplicationConfiguration.cs
+++ b/DysonNetwork.Insight/Startup/ApplicationConfiguration.cs
@@ -1,3 +1,4 @@
+using DysonNetwork.Insight.Reader;
using DysonNetwork.Shared.Http;
namespace DysonNetwork.Insight.Startup;
@@ -17,6 +18,11 @@ public static class ApplicationConfiguration
app.MapControllers();
+ app.MapGrpcService();
+ app.MapGrpcService();
+ app.MapGrpcService();
+ app.MapGrpcReflectionService();
+
return app;
}
}
diff --git a/DysonNetwork.Messager/AppDatabase.cs b/DysonNetwork.Messager/AppDatabase.cs
index bf7d15d..fb835b1 100644
--- a/DysonNetwork.Messager/AppDatabase.cs
+++ b/DysonNetwork.Messager/AppDatabase.cs
@@ -70,12 +70,6 @@ public class AppDatabase(
modelBuilder.ApplySoftDeleteFilters();
}
- private static void SetSoftDeleteFilter(ModelBuilder modelBuilder)
- where TEntity : ModelBase
- {
- modelBuilder.Entity().HasQueryFilter(e => e.DeletedAt == null);
- }
-
public override async Task SaveChangesAsync(CancellationToken cancellationToken = default)
{
this.ApplyAuditableAndSoftDelete();
diff --git a/DysonNetwork.Messager/Chat/ChatController.cs b/DysonNetwork.Messager/Chat/ChatController.cs
index 7817782..e6761e3 100644
--- a/DysonNetwork.Messager/Chat/ChatController.cs
+++ b/DysonNetwork.Messager/Chat/ChatController.cs
@@ -6,7 +6,7 @@ using DysonNetwork.Shared.Models;
using DysonNetwork.Shared.Proto;
using DysonNetwork.Messager.Poll;
using DysonNetwork.Messager.Wallet;
-using DysonNetwork.Messager.WebReader;
+using DysonNetwork.Shared.Models.Embed;
using Grpc.Core;
using Microsoft.AspNetCore.Authorization;
using Microsoft.AspNetCore.Mvc;
diff --git a/DysonNetwork.Messager/Chat/ChatService.cs b/DysonNetwork.Messager/Chat/ChatService.cs
index 485d2b4..99f5145 100644
--- a/DysonNetwork.Messager/Chat/ChatService.cs
+++ b/DysonNetwork.Messager/Chat/ChatService.cs
@@ -2,8 +2,9 @@ using System.Text.RegularExpressions;
using DysonNetwork.Shared.Models;
using DysonNetwork.Shared.Proto;
using DysonNetwork.Messager.Chat.Realtime;
+using DysonNetwork.Shared.Models.Embed;
+using DysonNetwork.Shared.Registry;
using Microsoft.EntityFrameworkCore;
-using DysonNetwork.Messager.WebReader;
using NodaTime;
using WebSocketPacket = DysonNetwork.Shared.Proto.WebSocketPacket;
@@ -16,7 +17,8 @@ public partial class ChatService(
FileReferenceService.FileReferenceServiceClient fileRefs,
IServiceScopeFactory scopeFactory,
IRealtimeService realtime,
- ILogger logger
+ ILogger logger,
+ RemoteWebReaderService webReader
)
{
private const string ChatFileUsageIdentifier = "chat";
@@ -36,10 +38,9 @@ public partial class ChatService(
// Create a new scope for database operations
using var scope = scopeFactory.CreateScope();
var dbContext = scope.ServiceProvider.GetRequiredService();
- var webReader = scope.ServiceProvider.GetRequiredService();
// Preview the links in the message
- var updatedMessage = await CreateLinkPreviewAsync(message, webReader);
+ var updatedMessage = await CreateLinkPreviewAsync(message);
// If embeds were added, update the message in the database
if (updatedMessage.Meta != null &&
@@ -111,7 +112,7 @@ public partial class ChatService(
/// The message to process
/// The web reader service
/// The message with link previews added to its meta data
- public async Task CreateLinkPreviewAsync(SnChatMessage message, WebReaderService? webReader = null)
+ public async Task CreateLinkPreviewAsync(SnChatMessage message)
{
if (string.IsNullOrEmpty(message.Content))
return message;
@@ -133,7 +134,6 @@ public partial class ChatService(
}
var embeds = (List>)message.Meta["embeds"];
- webReader ??= scopeFactory.CreateScope().ServiceProvider.GetRequiredService();
// Process up to 3 links to avoid excessive processing
var processedLinks = 0;
@@ -153,7 +153,7 @@ public partial class ChatService(
continue;
// Preview the link
- var linkEmbed = await webReader.GetLinkPreviewAsync(url);
+ var linkEmbed = await webReader.GetLinkPreview(url);
embeds.Add(EmbeddableBase.ToDictionary(linkEmbed));
processedLinks++;
}
diff --git a/DysonNetwork.Messager/Program.cs b/DysonNetwork.Messager/Program.cs
index 7ef4001..0d31c15 100644
--- a/DysonNetwork.Messager/Program.cs
+++ b/DysonNetwork.Messager/Program.cs
@@ -18,6 +18,7 @@ builder.Services.AddAccountService();
builder.Services.AddRingService();
builder.Services.AddDriveService();
builder.Services.AddSphereService();
+builder.Services.AddInsightService();
builder.Services.AddAppBusinessServices(builder.Configuration);
builder.Services.AddAppScheduledJobs();
diff --git a/DysonNetwork.Messager/WebReader/EmbeddableBase.cs b/DysonNetwork.Messager/WebReader/EmbeddableBase.cs
deleted file mode 100644
index 5a0a890..0000000
--- a/DysonNetwork.Messager/WebReader/EmbeddableBase.cs
+++ /dev/null
@@ -1,41 +0,0 @@
-using System.Text.Json;
-using DysonNetwork.Shared.Proto;
-
-namespace DysonNetwork.Messager.WebReader;
-
-///
-/// The embeddable can be used in the post or messages' meta's embeds fields
-/// To render a richer type of content.
-///
-/// A simple example of using link preview embed:
-///
-/// {
-/// // ... post content
-/// "meta": {
-/// "embeds": [
-/// {
-/// "type": "link",
-/// "title: "...",
-/// /// ...
-/// }
-/// ]
-/// }
-/// }
-///
-///
-public abstract class EmbeddableBase
-{
- public abstract string Type { get; }
-
- public static Dictionary ToDictionary(dynamic input)
- {
- var jsonRaw = JsonSerializer.Serialize(
- input,
- GrpcTypeHelper.SerializerOptionsWithoutIgnore
- );
- return JsonSerializer.Deserialize>(
- jsonRaw,
- GrpcTypeHelper.SerializerOptionsWithoutIgnore
- );
- }
-}
\ No newline at end of file
diff --git a/DysonNetwork.Messager/WebReader/LinkEmbed.cs b/DysonNetwork.Messager/WebReader/LinkEmbed.cs
deleted file mode 100644
index a7b907a..0000000
--- a/DysonNetwork.Messager/WebReader/LinkEmbed.cs
+++ /dev/null
@@ -1,55 +0,0 @@
-namespace DysonNetwork.Messager.WebReader;
-
-///
-/// The link embed is a part of the embeddable implementations
-/// It can be used in the post or messages' meta's embeds fields
-///
-public class LinkEmbed : EmbeddableBase
-{
- public override string Type => "link";
-
- ///
- /// The original URL that was processed
- ///
- public required string Url { get; set; }
-
- ///
- /// Title of the linked content (from OpenGraph og:title, meta title, or page title)
- ///
- public string? Title { get; set; }
-
- ///
- /// Description of the linked content (from OpenGraph og:description or meta description)
- ///
- public string? Description { get; set; }
-
- ///
- /// URL to the thumbnail image (from OpenGraph og:image or other meta tags)
- ///
- public string? ImageUrl { get; set; }
-
- ///
- /// The favicon URL of the site
- ///
- public string? FaviconUrl { get; set; }
-
- ///
- /// The site name (from OpenGraph og:site_name)
- ///
- public string? SiteName { get; set; }
-
- ///
- /// Type of the content (from OpenGraph og:type)
- ///
- public string? ContentType { get; set; }
-
- ///
- /// Author of the content if available
- ///
- public string? Author { get; set; }
-
- ///
- /// Published date of the content if available
- ///
- public DateTime? PublishedDate { get; set; }
-}
\ No newline at end of file
diff --git a/DysonNetwork.Messager/WebReader/ScrapedArticle.cs b/DysonNetwork.Messager/WebReader/ScrapedArticle.cs
deleted file mode 100644
index dad40cd..0000000
--- a/DysonNetwork.Messager/WebReader/ScrapedArticle.cs
+++ /dev/null
@@ -1,7 +0,0 @@
-namespace DysonNetwork.Messager.WebReader;
-
-public class ScrapedArticle
-{
- public LinkEmbed LinkEmbed { get; set; } = null!;
- public string? Content { get; set; }
-}
\ No newline at end of file
diff --git a/DysonNetwork.Messager/WebReader/WebReaderController.cs b/DysonNetwork.Messager/WebReader/WebReaderController.cs
deleted file mode 100644
index 6677391..0000000
--- a/DysonNetwork.Messager/WebReader/WebReaderController.cs
+++ /dev/null
@@ -1,110 +0,0 @@
-using DysonNetwork.Shared.Auth;
-using Microsoft.AspNetCore.Authorization;
-using Microsoft.AspNetCore.Mvc;
-using Microsoft.AspNetCore.RateLimiting;
-
-namespace DysonNetwork.Messager.WebReader;
-
-///
-/// Controller for web scraping and link preview services
-///
-[ApiController]
-[Route("/api/scrap")]
-[EnableRateLimiting("fixed")]
-public class WebReaderController(WebReaderService reader, ILogger logger)
- : ControllerBase
-{
- ///
- /// Retrieves a preview for the provided URL
- ///
- /// URL-encoded link to generate preview for
- /// Link preview data including title, description, and image
- [HttpGet("link")]
- public async Task> ScrapLink([FromQuery] string url)
- {
- if (string.IsNullOrEmpty(url))
- {
- return BadRequest(new { error = "URL parameter is required" });
- }
-
- try
- {
- // Ensure URL is properly decoded
- var decodedUrl = UrlDecoder.Decode(url);
-
- // Validate URL format
- if (!Uri.TryCreate(decodedUrl, UriKind.Absolute, out _))
- {
- return BadRequest(new { error = "Invalid URL format" });
- }
-
- var linkEmbed = await reader.GetLinkPreviewAsync(decodedUrl);
- return Ok(linkEmbed);
- }
- catch (WebReaderException ex)
- {
- logger.LogWarning(ex, "Error scraping link: {Url}", url);
- return BadRequest(new { error = ex.Message });
- }
- catch (Exception ex)
- {
- logger.LogError(ex, "Unexpected error scraping link: {Url}", url);
- return StatusCode(StatusCodes.Status500InternalServerError,
- new { error = "An unexpected error occurred while processing the link" });
- }
- }
-
- ///
- /// Force invalidates the cache for a specific URL
- ///
- [HttpDelete("link/cache")]
- [Authorize]
- [AskPermission("cache.scrap")]
- public async Task InvalidateCache([FromQuery] string url)
- {
- if (string.IsNullOrEmpty(url))
- {
- return BadRequest(new { error = "URL parameter is required" });
- }
-
- await reader.InvalidateCacheForUrlAsync(url);
- return Ok(new { message = "Cache invalidated for URL" });
- }
-
- ///
- /// Force invalidates all cached link previews
- ///
- [HttpDelete("cache/all")]
- [Authorize]
- [AskPermission("cache.scrap")]
- public async Task InvalidateAllCache()
- {
- await reader.InvalidateAllCachedPreviewsAsync();
- return Ok(new { message = "All link preview caches invalidated" });
- }
-}
-
-///
-/// Helper class for URL decoding
-///
-public static class UrlDecoder
-{
- public static string Decode(string url)
- {
- // First check if URL is already decoded
- if (!url.Contains('%') && !url.Contains('+'))
- {
- return url;
- }
-
- try
- {
- return System.Net.WebUtility.UrlDecode(url);
- }
- catch
- {
- // If decoding fails, return the original string
- return url;
- }
- }
-}
\ No newline at end of file
diff --git a/DysonNetwork.Messager/WebReader/WebReaderException.cs b/DysonNetwork.Messager/WebReader/WebReaderException.cs
deleted file mode 100644
index 92e4071..0000000
--- a/DysonNetwork.Messager/WebReader/WebReaderException.cs
+++ /dev/null
@@ -1,15 +0,0 @@
-namespace DysonNetwork.Messager.WebReader;
-
-///
-/// Exception thrown when an error occurs during web reading operations
-///
-public class WebReaderException : Exception
-{
- public WebReaderException(string message) : base(message)
- {
- }
-
- public WebReaderException(string message, Exception innerException) : base(message, innerException)
- {
- }
-}
diff --git a/DysonNetwork.Messager/WebReader/WebReaderService.cs b/DysonNetwork.Messager/WebReader/WebReaderService.cs
deleted file mode 100644
index 13de1ed..0000000
--- a/DysonNetwork.Messager/WebReader/WebReaderService.cs
+++ /dev/null
@@ -1,367 +0,0 @@
-using System.Globalization;
-using AngleSharp;
-using AngleSharp.Dom;
-using DysonNetwork.Shared.Cache;
-using HtmlAgilityPack;
-
-namespace DysonNetwork.Messager.WebReader;
-
-///
-/// The service is amin to providing scrapping service to the Solar Network.
-/// Such as news feed, external articles and link preview.
-///
-public class WebReaderService(
- IHttpClientFactory httpClientFactory,
- ILogger logger,
- ICacheService cache
-)
-{
- private const string LinkPreviewCachePrefix = "scrap:preview:";
- private const string LinkPreviewCacheGroup = "scrap:preview";
-
- public async Task ScrapeArticleAsync(string url, CancellationToken cancellationToken = default)
- {
- var linkEmbed = await GetLinkPreviewAsync(url, cancellationToken);
- var content = await GetArticleContentAsync(url, cancellationToken);
- return new ScrapedArticle
- {
- LinkEmbed = linkEmbed,
- Content = content
- };
- }
-
- private async Task GetArticleContentAsync(string url, CancellationToken cancellationToken)
- {
- var httpClient = httpClientFactory.CreateClient("WebReader");
- var response = await httpClient.GetAsync(url, cancellationToken);
- if (!response.IsSuccessStatusCode)
- {
- logger.LogWarning("Failed to scrap article content for URL: {Url}", url);
- return null;
- }
-
- var html = await response.Content.ReadAsStringAsync(cancellationToken);
- var doc = new HtmlDocument();
- doc.LoadHtml(html);
- var articleNode = doc.DocumentNode.SelectSingleNode("//article");
- return articleNode?.InnerHtml;
- }
-
-
- ///
- /// Generate a link preview embed from a URL
- ///
- /// The URL to generate the preview for
- /// Cancellation token
- /// If true, bypass cache and fetch fresh data
- /// Custom cache expiration time
- /// A LinkEmbed object containing the preview data
- public async Task GetLinkPreviewAsync(
- string url,
- CancellationToken cancellationToken = default,
- TimeSpan? cacheExpiry = null,
- bool bypassCache = false
- )
- {
- // Ensure URL is valid
- if (!Uri.TryCreate(url, UriKind.Absolute, out var uri))
- {
- throw new ArgumentException(@"Invalid URL format", nameof(url));
- }
-
- // Try to get from cache if not bypassing
- if (!bypassCache)
- {
- var cachedPreview = await GetCachedLinkPreview(url);
- if (cachedPreview is not null)
- return cachedPreview;
- }
-
- // Cache miss or bypass, fetch fresh data
- logger.LogDebug("Fetching fresh link preview for URL: {Url}", url);
- var httpClient = httpClientFactory.CreateClient("WebReader");
- httpClient.MaxResponseContentBufferSize =
- 10 * 1024 * 1024; // 10MB, prevent scrap some directly accessible files
- httpClient.Timeout = TimeSpan.FromSeconds(3);
- // Setting UA to facebook's bot to get the opengraph.
- httpClient.DefaultRequestHeaders.Add("User-Agent", "facebookexternalhit/1.1");
-
- try
- {
- var response = await httpClient.GetAsync(url, cancellationToken);
- response.EnsureSuccessStatusCode();
-
- var contentType = response.Content.Headers.ContentType?.MediaType;
- if (contentType == null || !contentType.StartsWith("text/html"))
- {
- logger.LogWarning("URL is not an HTML page: {Url}, ContentType: {ContentType}", url, contentType);
- var nonHtmlEmbed = new LinkEmbed
- {
- Url = url,
- Title = uri.Host,
- ContentType = contentType
- };
-
- // Cache non-HTML responses too
- await CacheLinkPreview(nonHtmlEmbed, url, cacheExpiry);
- return nonHtmlEmbed;
- }
-
- var html = await response.Content.ReadAsStringAsync(cancellationToken);
- var linkEmbed = await ExtractLinkData(url, html, uri);
-
- // Cache the result
- await CacheLinkPreview(linkEmbed, url, cacheExpiry);
-
- return linkEmbed;
- }
- catch (HttpRequestException ex)
- {
- logger.LogError(ex, "Failed to fetch URL: {Url}", url);
- throw new WebReaderException($"Failed to fetch URL: {url}", ex);
- }
- }
-
- private async Task ExtractLinkData(string url, string html, Uri uri)
- {
- var embed = new LinkEmbed
- {
- Url = url
- };
-
- // Configure AngleSharp context
- var config = Configuration.Default;
- var context = BrowsingContext.New(config);
- var document = await context.OpenAsync(req => req.Content(html));
-
- // Extract OpenGraph tags
- var ogTitle = GetMetaTagContent(document, "og:title");
- var ogDescription = GetMetaTagContent(document, "og:description");
- var ogImage = GetMetaTagContent(document, "og:image");
- var ogSiteName = GetMetaTagContent(document, "og:site_name");
- var ogType = GetMetaTagContent(document, "og:type");
-
- // Extract Twitter card tags as fallback
- var twitterTitle = GetMetaTagContent(document, "twitter:title");
- var twitterDescription = GetMetaTagContent(document, "twitter:description");
- var twitterImage = GetMetaTagContent(document, "twitter:image");
-
- // Extract standard meta tags as final fallback
- var metaTitle = GetMetaTagContent(document, "title") ??
- GetMetaContent(document, "title");
- var metaDescription = GetMetaTagContent(document, "description");
-
- // Extract page title
- var pageTitle = document.Title?.Trim();
-
- // Extract publish date
- var publishedTime = GetMetaTagContent(document, "article:published_time") ??
- GetMetaTagContent(document, "datePublished") ??
- GetMetaTagContent(document, "pubdate");
-
- // Extract author
- var author = GetMetaTagContent(document, "author") ??
- GetMetaTagContent(document, "article:author");
-
- // Extract favicon
- var faviconUrl = GetFaviconUrl(document, uri);
-
- // Populate the embed with the data, prioritizing OpenGraph
- embed.Title = ogTitle ?? twitterTitle ?? metaTitle ?? pageTitle ?? uri.Host;
- embed.Description = ogDescription ?? twitterDescription ?? metaDescription;
- embed.ImageUrl = ResolveRelativeUrl(ogImage ?? twitterImage, uri);
- embed.SiteName = ogSiteName ?? uri.Host;
- embed.ContentType = ogType;
- embed.FaviconUrl = faviconUrl;
- embed.Author = author;
-
- // Parse and set published date
- if (!string.IsNullOrEmpty(publishedTime) &&
- DateTime.TryParse(publishedTime, CultureInfo.InvariantCulture, DateTimeStyles.AdjustToUniversal,
- out DateTime parsedDate))
- {
- embed.PublishedDate = parsedDate;
- }
-
- return embed;
- }
-
- private static string? GetMetaTagContent(IDocument doc, string property)
- {
- // Check for OpenGraph/Twitter style meta tags
- var node = doc.QuerySelector($"meta[property='{property}'][content]")
- ?? doc.QuerySelector($"meta[name='{property}'][content]");
-
- return node?.GetAttribute("content")?.Trim();
- }
-
- private static string? GetMetaContent(IDocument doc, string name)
- {
- var node = doc.QuerySelector($"meta[name='{name}'][content]");
- return node?.GetAttribute("content")?.Trim();
- }
-
- private static string? GetFaviconUrl(IDocument doc, Uri baseUri)
- {
- // Look for apple-touch-icon first as it's typically higher quality
- var appleIconNode = doc.QuerySelector("link[rel='apple-touch-icon'][href]");
- if (appleIconNode != null)
- {
- return ResolveRelativeUrl(appleIconNode.GetAttribute("href"), baseUri);
- }
-
- // Then check for standard favicon
- var faviconNode = doc.QuerySelector("link[rel='icon'][href]") ??
- doc.QuerySelector("link[rel='shortcut icon'][href]");
-
- return faviconNode != null
- ? ResolveRelativeUrl(faviconNode.GetAttribute("href"), baseUri)
- : new Uri(baseUri, "/favicon.ico").ToString();
- }
-
- private static string? ResolveRelativeUrl(string? url, Uri baseUri)
- {
- if (string.IsNullOrEmpty(url))
- {
- return null;
- }
-
- if (Uri.TryCreate(url, UriKind.Absolute, out _))
- {
- return url; // Already absolute
- }
-
- return Uri.TryCreate(baseUri, url, out var absoluteUri) ? absoluteUri.ToString() : null;
- }
-
- ///
- /// Generate a hash-based cache key for a URL
- ///
- private string GenerateUrlCacheKey(string url)
- {
- // Normalize the URL first
- var normalizedUrl = NormalizeUrl(url);
-
- // Create SHA256 hash of the normalized URL
- using var sha256 = System.Security.Cryptography.SHA256.Create();
- var urlBytes = System.Text.Encoding.UTF8.GetBytes(normalizedUrl);
- var hashBytes = sha256.ComputeHash(urlBytes);
-
- // Convert to hex string
- var hashString = BitConverter.ToString(hashBytes).Replace("-", "").ToLowerInvariant();
-
- // Return prefixed key
- return $"{LinkPreviewCachePrefix}{hashString}";
- }
-
- ///
- /// Normalize URL by trimming trailing slashes but preserving query parameters
- ///
- private string NormalizeUrl(string url)
- {
- if (string.IsNullOrEmpty(url))
- return string.Empty;
-
- // First ensure we have a valid URI
- if (!Uri.TryCreate(url, UriKind.Absolute, out var uri))
- return url.TrimEnd('/');
-
- // Rebuild the URL without trailing slashes but with query parameters
- var scheme = uri.Scheme;
- var host = uri.Host;
- var port = uri.IsDefaultPort ? string.Empty : $":{uri.Port}";
- var path = uri.AbsolutePath.TrimEnd('/');
- var query = uri.Query;
-
- return $"{scheme}://{host}{port}{path}{query}".ToLowerInvariant();
- }
-
- ///
- /// Cache a link preview
- ///
- private async Task CacheLinkPreview(LinkEmbed? linkEmbed, string url, TimeSpan? expiry = null)
- {
- if (linkEmbed == null || string.IsNullOrEmpty(url))
- return;
-
- try
- {
- var cacheKey = GenerateUrlCacheKey(url);
- var expiryTime = expiry ?? TimeSpan.FromHours(24);
-
- await cache.SetWithGroupsAsync(
- cacheKey,
- linkEmbed,
- [LinkPreviewCacheGroup],
- expiryTime);
-
- logger.LogDebug("Cached link preview for URL: {Url} with key: {CacheKey}", url, cacheKey);
- }
- catch (Exception ex)
- {
- // Log but don't throw - caching failures shouldn't break the main functionality
- logger.LogWarning(ex, "Failed to cache link preview for URL: {Url}", url);
- }
- }
-
- ///
- /// Try to get a cached link preview
- ///
- private async Task GetCachedLinkPreview(string url)
- {
- if (string.IsNullOrEmpty(url))
- return null;
-
- try
- {
- var cacheKey = GenerateUrlCacheKey(url);
- var cachedPreview = await cache.GetAsync(cacheKey);
-
- if (cachedPreview is not null)
- logger.LogDebug("Retrieved cached link preview for URL: {Url}", url);
-
- return cachedPreview;
- }
- catch (Exception ex)
- {
- logger.LogWarning(ex, "Failed to retrieve cached link preview for URL: {Url}", url);
- return null;
- }
- }
-
- ///
- /// Invalidate cache for a specific URL
- ///
- public async Task InvalidateCacheForUrlAsync(string url)
- {
- if (string.IsNullOrEmpty(url))
- return;
-
- try
- {
- var cacheKey = GenerateUrlCacheKey(url);
- await cache.RemoveAsync(cacheKey);
- logger.LogDebug("Invalidated cache for URL: {Url} with key: {CacheKey}", url, cacheKey);
- }
- catch (Exception ex)
- {
- logger.LogWarning(ex, "Failed to invalidate cache for URL: {Url}", url);
- }
- }
-
- ///
- /// Invalidate all cached link previews
- ///
- public async Task InvalidateAllCachedPreviewsAsync()
- {
- try
- {
- await cache.RemoveGroupAsync(LinkPreviewCacheGroup);
- logger.LogInformation("Invalidated all cached link previews");
- }
- catch (Exception ex)
- {
- logger.LogWarning(ex, "Failed to invalidate all cached link previews");
- }
- }
-}
\ No newline at end of file
diff --git a/DysonNetwork.Shared/Models/Embed/LinkEmbed.cs b/DysonNetwork.Shared/Models/Embed/LinkEmbed.cs
index 7c64c20..b2153cc 100644
--- a/DysonNetwork.Shared/Models/Embed/LinkEmbed.cs
+++ b/DysonNetwork.Shared/Models/Embed/LinkEmbed.cs
@@ -1,3 +1,6 @@
+using DysonNetwork.Shared.Proto;
+using Google.Protobuf.WellKnownTypes;
+
namespace DysonNetwork.Shared.Models.Embed;
///
@@ -52,4 +55,54 @@ public class LinkEmbed : EmbeddableBase
/// Published date of the content if available
///
public DateTime? PublishedDate { get; set; }
+
+ public Proto.LinkEmbed ToProtoValue()
+ {
+ var proto = new Proto.LinkEmbed
+ {
+ Url = Url
+ };
+
+ if (!string.IsNullOrEmpty(Title))
+ proto.Title = Title;
+
+ if (!string.IsNullOrEmpty(Description))
+ proto.Description = Description;
+
+ if (!string.IsNullOrEmpty(ImageUrl))
+ proto.ImageUrl = ImageUrl;
+
+ if (!string.IsNullOrEmpty(FaviconUrl))
+ proto.FaviconUrl = FaviconUrl;
+
+ if (!string.IsNullOrEmpty(SiteName))
+ proto.SiteName = SiteName;
+
+ if (!string.IsNullOrEmpty(ContentType))
+ proto.ContentType = ContentType;
+
+ if (!string.IsNullOrEmpty(Author))
+ proto.Author = Author;
+
+ if (PublishedDate.HasValue)
+ proto.PublishedDate = Timestamp.FromDateTime(PublishedDate.Value.ToUniversalTime());
+
+ return proto;
+ }
+
+ public static LinkEmbed FromProtoValue(Proto.LinkEmbed proto)
+ {
+ return new LinkEmbed
+ {
+ Url = proto.Url,
+ Title = proto.Title == "" ? null : proto.Title,
+ Description = proto.Description == "" ? null : proto.Description,
+ ImageUrl = proto.ImageUrl == "" ? null : proto.ImageUrl,
+ FaviconUrl = proto.FaviconUrl == "" ? null : proto.FaviconUrl,
+ SiteName = proto.SiteName == "" ? null : proto.SiteName,
+ ContentType = proto.ContentType == "" ? null : proto.ContentType,
+ Author = proto.Author == "" ? null : proto.Author,
+ PublishedDate = proto.PublishedDate != null ? proto.PublishedDate.ToDateTime() : null
+ };
+ }
}
\ No newline at end of file
diff --git a/DysonNetwork.Shared/Models/WebArticle.cs b/DysonNetwork.Shared/Models/WebArticle.cs
index d498634..5f0232f 100644
--- a/DysonNetwork.Shared/Models/WebArticle.cs
+++ b/DysonNetwork.Shared/Models/WebArticle.cs
@@ -2,7 +2,10 @@ using System.ComponentModel.DataAnnotations;
using System.ComponentModel.DataAnnotations.Schema;
using System.Text.Json.Serialization;
using DysonNetwork.Shared.Models.Embed;
+using DysonNetwork.Shared.Proto;
+using Google.Protobuf.WellKnownTypes;
using NodaTime;
+using EmbedLinkEmbed = DysonNetwork.Shared.Models.Embed.LinkEmbed;
namespace DysonNetwork.Shared.Models;
@@ -13,9 +16,9 @@ public class SnWebArticle : ModelBase
[MaxLength(4096)] public string Title { get; set; } = null!;
[MaxLength(8192)] public string Url { get; set; } = null!;
[MaxLength(4096)] public string? Author { get; set; }
-
+
[Column(TypeName = "jsonb")] public Dictionary? Meta { get; set; }
- [Column(TypeName = "jsonb")] public LinkEmbed? Preview { get; set; }
+ [Column(TypeName = "jsonb")] public EmbedLinkEmbed? Preview { get; set; }
// ReSharper disable once EntityFramework.ModelValidation.UnlimitedStringLength
public string? Content { get; set; }
@@ -24,11 +27,79 @@ public class SnWebArticle : ModelBase
public Guid FeedId { get; set; }
public SnWebFeed Feed { get; set; } = null!;
+
+ public WebArticle ToProtoValue()
+ {
+ var proto = new WebArticle
+ {
+ Id = Id.ToString(),
+ Title = Title,
+ Url = Url,
+ FeedId = FeedId.ToString(),
+ CreatedAt = Timestamp.FromDateTimeOffset(CreatedAt.ToDateTimeOffset()),
+ UpdatedAt = Timestamp.FromDateTimeOffset(UpdatedAt.ToDateTimeOffset())
+ };
+
+ if (!string.IsNullOrEmpty(Author))
+ proto.Author = Author;
+
+ if (Meta != null)
+ proto.Meta = GrpcTypeHelper.ConvertObjectToByteString(Meta);
+
+ if (Preview != null)
+ proto.Preview = Preview.ToProtoValue();
+
+ if (!string.IsNullOrEmpty(Content))
+ proto.Content = Content;
+
+ if (PublishedAt.HasValue)
+ proto.PublishedAt = Timestamp.FromDateTime(PublishedAt.Value.ToUniversalTime());
+
+ if (DeletedAt.HasValue)
+ proto.DeletedAt = Timestamp.FromDateTimeOffset(DeletedAt.Value.ToDateTimeOffset());
+
+ return proto;
+ }
+
+ public static SnWebArticle FromProtoValue(WebArticle proto)
+ {
+ return new SnWebArticle
+ {
+ Id = Guid.Parse(proto.Id),
+ Title = proto.Title,
+ Url = proto.Url,
+ FeedId = Guid.Parse(proto.FeedId),
+ Author = proto.Author == "" ? null : proto.Author,
+ Meta = proto.Meta != null ? GrpcTypeHelper.ConvertByteStringToObject>(proto.Meta) : null,
+ Preview = proto.Preview != null ? EmbedLinkEmbed.FromProtoValue(proto.Preview) : null,
+ Content = proto.Content == "" ? null : proto.Content,
+ PublishedAt = proto.PublishedAt != null ? proto.PublishedAt.ToDateTime() : null,
+ CreatedAt = Instant.FromDateTimeOffset(proto.CreatedAt.ToDateTimeOffset()),
+ UpdatedAt = Instant.FromDateTimeOffset(proto.UpdatedAt.ToDateTimeOffset()),
+ DeletedAt = proto.DeletedAt != null ? Instant.FromDateTimeOffset(proto.DeletedAt.ToDateTimeOffset()) : null
+ };
+ }
}
public class WebFeedConfig
{
public bool ScrapPage { get; set; }
+
+ public Proto.WebFeedConfig ToProtoValue()
+ {
+ return new Proto.WebFeedConfig
+ {
+ ScrapPage = ScrapPage
+ };
+ }
+
+ public static WebFeedConfig FromProtoValue(Proto.WebFeedConfig proto)
+ {
+ return new WebFeedConfig
+ {
+ ScrapPage = proto.ScrapPage
+ };
+ }
}
public class SnWebFeed : ModelBase
@@ -37,25 +108,105 @@ public class SnWebFeed : ModelBase
[MaxLength(8192)] public string Url { get; set; } = null!;
[MaxLength(4096)] public string Title { get; set; } = null!;
[MaxLength(8192)] public string? Description { get; set; }
-
+
public Instant? VerifiedAt { get; set; }
[JsonIgnore] [MaxLength(8192)] public string? VerificationKey { get; set; }
-
- [Column(TypeName = "jsonb")] public LinkEmbed? Preview { get; set; }
+
+ [Column(TypeName = "jsonb")] public EmbedLinkEmbed? Preview { get; set; }
[Column(TypeName = "jsonb")] public WebFeedConfig Config { get; set; } = new();
public Guid PublisherId { get; set; }
public SnPublisher Publisher { get; set; } = null!;
[JsonIgnore] public List Articles { get; set; } = new();
+
+ public WebFeed ToProtoValue()
+ {
+ var proto = new WebFeed
+ {
+ Id = Id.ToString(),
+ Url = Url,
+ Title = Title,
+ Config = Config.ToProtoValue(),
+ PublisherId = PublisherId.ToString(),
+ CreatedAt = Timestamp.FromDateTimeOffset(CreatedAt.ToDateTimeOffset()),
+ UpdatedAt = Timestamp.FromDateTimeOffset(UpdatedAt.ToDateTimeOffset())
+ };
+
+ if (!string.IsNullOrEmpty(Description))
+ proto.Description = Description;
+
+ if (VerifiedAt.HasValue)
+ proto.VerifiedAt = Timestamp.FromDateTimeOffset(VerifiedAt.Value.ToDateTimeOffset());
+
+ if (Preview != null)
+ proto.Preview = Preview.ToProtoValue();
+
+ if (Publisher != null)
+ proto.Publisher = Publisher.ToProtoValue();
+
+ if (DeletedAt.HasValue)
+ proto.DeletedAt = Timestamp.FromDateTimeOffset(DeletedAt.Value.ToDateTimeOffset());
+
+ return proto;
+ }
+
+ public static SnWebFeed FromProtoValue(WebFeed proto)
+ {
+ return new SnWebFeed
+ {
+ Id = Guid.Parse(proto.Id),
+ Url = proto.Url,
+ Title = proto.Title,
+ Description = proto.Description == "" ? null : proto.Description,
+ VerifiedAt = proto.VerifiedAt != null ? Instant.FromDateTimeOffset(proto.VerifiedAt.ToDateTimeOffset()) : null,
+ Preview = proto.Preview != null ? EmbedLinkEmbed.FromProtoValue(proto.Preview) : null,
+ Config = WebFeedConfig.FromProtoValue(proto.Config),
+ PublisherId = Guid.Parse(proto.PublisherId),
+ Publisher = proto.Publisher != null ? SnPublisher.FromProtoValue(proto.Publisher) : null,
+ CreatedAt = Instant.FromDateTimeOffset(proto.CreatedAt.ToDateTimeOffset()),
+ UpdatedAt = Instant.FromDateTimeOffset(proto.UpdatedAt.ToDateTimeOffset()),
+ DeletedAt = proto.DeletedAt != null ? Instant.FromDateTimeOffset(proto.DeletedAt.ToDateTimeOffset()) : null
+ };
+ }
}
public class SnWebFeedSubscription : ModelBase
{
public Guid Id { get; set; } = Guid.NewGuid();
-
+
public Guid FeedId { get; set; }
public SnWebFeed Feed { get; set; } = null!;
public Guid AccountId { get; set; }
[NotMapped] public SnAccount Account { get; set; } = null!;
+
+ public WebFeedSubscription ToProtoValue()
+ {
+ var proto = new WebFeedSubscription
+ {
+ Id = Id.ToString(),
+ FeedId = FeedId.ToString(),
+ AccountId = AccountId.ToString(),
+ CreatedAt = Timestamp.FromDateTimeOffset(CreatedAt.ToDateTimeOffset()),
+ UpdatedAt = Timestamp.FromDateTimeOffset(UpdatedAt.ToDateTimeOffset())
+ };
+
+ if (Feed != null)
+ proto.Feed = Feed.ToProtoValue();
+
+ return proto;
+ }
+
+ public static SnWebFeedSubscription FromProtoValue(WebFeedSubscription proto)
+ {
+ return new SnWebFeedSubscription
+ {
+ Id = Guid.Parse(proto.Id),
+ FeedId = Guid.Parse(proto.FeedId),
+ Feed = proto.Feed != null ? SnWebFeed.FromProtoValue(proto.Feed) : null,
+ AccountId = Guid.Parse(proto.AccountId),
+ CreatedAt = Instant.FromDateTimeOffset(proto.CreatedAt.ToDateTimeOffset()),
+ UpdatedAt = Instant.FromDateTimeOffset(proto.UpdatedAt.ToDateTimeOffset())
+ };
+ }
}
\ No newline at end of file
diff --git a/DysonNetwork.Shared/Proto/embed.proto b/DysonNetwork.Shared/Proto/embed.proto
new file mode 100644
index 0000000..5a59a28
--- /dev/null
+++ b/DysonNetwork.Shared/Proto/embed.proto
@@ -0,0 +1,19 @@
+syntax = "proto3";
+
+package proto;
+
+option csharp_namespace = "DysonNetwork.Shared.Proto";
+
+import "google/protobuf/timestamp.proto";
+
+message LinkEmbed {
+ string url = 1;
+ optional string title = 2;
+ optional string description = 3;
+ optional string image_url = 4;
+ optional string favicon_url = 5;
+ optional string site_name = 6;
+ optional string content_type = 7;
+ optional string author = 8;
+ optional google.protobuf.Timestamp published_date = 9;
+}
diff --git a/DysonNetwork.Shared/Proto/web_article.proto b/DysonNetwork.Shared/Proto/web_article.proto
new file mode 100644
index 0000000..9687fdb
--- /dev/null
+++ b/DysonNetwork.Shared/Proto/web_article.proto
@@ -0,0 +1,160 @@
+syntax = "proto3";
+
+package proto;
+
+option csharp_namespace = "DysonNetwork.Shared.Proto";
+
+import "google/protobuf/timestamp.proto";
+import "embed.proto";
+import "publisher.proto";
+
+message WebFeedConfig {
+ bool scrap_page = 1;
+}
+
+message WebFeed {
+ string id = 1;
+ string url = 2;
+ string title = 3;
+ optional string description = 4;
+ optional google.protobuf.Timestamp verified_at = 5;
+ optional LinkEmbed preview = 6;
+ WebFeedConfig config = 7;
+ string publisher_id = 8;
+ optional Publisher publisher = 9;
+ google.protobuf.Timestamp created_at = 10;
+ google.protobuf.Timestamp updated_at = 11;
+ optional google.protobuf.Timestamp deleted_at = 12;
+}
+
+message WebArticle {
+ string id = 1;
+ string title = 2;
+ string url = 3;
+ optional string author = 4;
+ optional bytes meta = 5;
+ optional LinkEmbed preview = 6;
+ optional string content = 7;
+ optional google.protobuf.Timestamp published_at = 8;
+ string feed_id = 9;
+ optional WebFeed feed = 10;
+ google.protobuf.Timestamp created_at = 11;
+ google.protobuf.Timestamp updated_at = 12;
+ optional google.protobuf.Timestamp deleted_at = 13;
+}
+
+message WebFeedSubscription {
+ string id = 1;
+ string feed_id = 2;
+ optional WebFeed feed = 3;
+ string account_id = 4;
+ google.protobuf.Timestamp created_at = 5;
+ google.protobuf.Timestamp updated_at = 6;
+}
+
+message ScrapedArticle {
+ LinkEmbed link_embed = 1;
+ optional string content = 2;
+}
+
+message GetWebArticleRequest {
+ string id = 1;
+}
+
+message GetWebArticleResponse {
+ WebArticle article = 1;
+}
+
+message GetWebArticleBatchRequest {
+ repeated string ids = 1;
+}
+
+message GetWebArticleBatchResponse {
+ repeated WebArticle articles = 1;
+}
+
+message ListWebArticlesRequest {
+ string feed_id = 1;
+ int32 page_size = 2;
+ string page_token = 3;
+}
+
+ message ListWebArticlesResponse {
+ repeated WebArticle articles = 1;
+ string next_page_token = 2;
+ int32 total_size = 3;
+}
+
+message GetRecentArticlesRequest {
+ int32 limit = 1;
+}
+
+message GetRecentArticlesResponse {
+ repeated WebArticle articles = 1;
+}
+
+message GetWebFeedRequest {
+ oneof identifier {
+ string id = 1;
+ string url = 2;
+ }
+}
+
+message GetWebFeedResponse {
+ WebFeed feed = 1;
+}
+
+message ListWebFeedsRequest {
+ string publisher_id = 1;
+ int32 page_size = 2;
+ string page_token = 3;
+}
+
+message ListWebFeedsResponse {
+ repeated WebFeed feeds = 1;
+ string next_page_token = 2;
+ int32 total_size = 3;
+}
+
+message ScrapeArticleRequest {
+ string url = 1;
+}
+
+message ScrapeArticleResponse {
+ ScrapedArticle article = 1;
+}
+
+message GetLinkPreviewRequest {
+ string url = 1;
+ bool bypass_cache = 2;
+}
+
+message GetLinkPreviewResponse {
+ LinkEmbed preview = 1;
+}
+
+message InvalidateLinkPreviewCacheRequest {
+ string url = 1;
+}
+
+message InvalidateLinkPreviewCacheResponse {
+ bool success = 1;
+}
+
+ service WebArticleService {
+ rpc GetWebArticle(GetWebArticleRequest) returns (GetWebArticleResponse);
+ rpc GetWebArticleBatch(GetWebArticleBatchRequest) returns (GetWebArticleBatchResponse);
+ rpc ListWebArticles(ListWebArticlesRequest) returns (ListWebArticlesResponse);
+ rpc GetRecentArticles(GetRecentArticlesRequest) returns (GetRecentArticlesResponse);
+}
+
+service WebFeedService {
+ rpc GetWebFeed(GetWebFeedRequest) returns (GetWebFeedResponse);
+ rpc ListWebFeeds(ListWebFeedsRequest) returns (ListWebFeedsResponse);
+}
+
+service WebReaderService {
+ rpc ScrapeArticle(ScrapeArticleRequest) returns (ScrapeArticleResponse);
+ rpc GetLinkPreview(GetLinkPreviewRequest) returns (GetLinkPreviewResponse);
+ rpc InvalidateLinkPreviewCache(InvalidateLinkPreviewCacheRequest) returns (InvalidateLinkPreviewCacheResponse);
+}
diff --git a/DysonNetwork.Shared/Registry/RemoteWebArticleService.cs b/DysonNetwork.Shared/Registry/RemoteWebArticleService.cs
new file mode 100644
index 0000000..3066af8
--- /dev/null
+++ b/DysonNetwork.Shared/Registry/RemoteWebArticleService.cs
@@ -0,0 +1,36 @@
+using DysonNetwork.Shared.Models;
+using DysonNetwork.Shared.Proto;
+
+namespace DysonNetwork.Shared.Registry;
+
+public class RemoteWebArticleService(WebArticleService.WebArticleServiceClient webArticles)
+{
+ public async Task GetWebArticle(Guid id)
+ {
+ var request = new GetWebArticleRequest { Id = id.ToString() };
+ var response = await webArticles.GetWebArticleAsync(request);
+ return response.Article != null ? SnWebArticle.FromProtoValue(response.Article) : null!;
+ }
+
+ public async Task> GetWebArticleBatch(List ids)
+ {
+ var request = new GetWebArticleBatchRequest();
+ request.Ids.AddRange(ids.Select(id => id.ToString()));
+ var response = await webArticles.GetWebArticleBatchAsync(request);
+ return response.Articles.Select(SnWebArticle.FromProtoValue).ToList();
+ }
+
+ public async Task> ListWebArticles(Guid feedId)
+ {
+ var request = new ListWebArticlesRequest { FeedId = feedId.ToString() };
+ var response = await webArticles.ListWebArticlesAsync(request);
+ return response.Articles.Select(SnWebArticle.FromProtoValue).ToList();
+ }
+
+ public async Task> GetRecentArticles(int limit = 20)
+ {
+ var request = new GetRecentArticlesRequest { Limit = limit };
+ var response = await webArticles.GetRecentArticlesAsync(request);
+ return response.Articles.Select(SnWebArticle.FromProtoValue).ToList();
+ }
+}
diff --git a/DysonNetwork.Shared/Registry/RemoteWebFeedService.cs b/DysonNetwork.Shared/Registry/RemoteWebFeedService.cs
new file mode 100644
index 0000000..b48c240
--- /dev/null
+++ b/DysonNetwork.Shared/Registry/RemoteWebFeedService.cs
@@ -0,0 +1,28 @@
+using DysonNetwork.Shared.Models;
+using DysonNetwork.Shared.Proto;
+
+namespace DysonNetwork.Shared.Registry;
+
+public class RemoteWebFeedService(WebFeedService.WebFeedServiceClient webFeeds)
+{
+ public async Task GetWebFeed(Guid id)
+ {
+ var request = new GetWebFeedRequest { Id = id.ToString() };
+ var response = await webFeeds.GetWebFeedAsync(request);
+ return response.Feed != null ? SnWebFeed.FromProtoValue(response.Feed) : null!;
+ }
+
+ public async Task GetWebFeedByUrl(string url)
+ {
+ var request = new GetWebFeedRequest { Url = url };
+ var response = await webFeeds.GetWebFeedAsync(request);
+ return response.Feed != null ? SnWebFeed.FromProtoValue(response.Feed) : null!;
+ }
+
+ public async Task> ListWebFeeds(Guid publisherId)
+ {
+ var request = new ListWebFeedsRequest { PublisherId = publisherId.ToString() };
+ var response = await webFeeds.ListWebFeedsAsync(request);
+ return response.Feeds.Select(SnWebFeed.FromProtoValue).ToList();
+ }
+}
diff --git a/DysonNetwork.Shared/Registry/RemoteWebReaderService.cs b/DysonNetwork.Shared/Registry/RemoteWebReaderService.cs
new file mode 100644
index 0000000..45b16a6
--- /dev/null
+++ b/DysonNetwork.Shared/Registry/RemoteWebReaderService.cs
@@ -0,0 +1,34 @@
+using DysonNetwork.Shared.Models;
+using DysonNetwork.Shared.Models.Embed;
+using DysonNetwork.Shared.Proto;
+using ProtoLinkEmbed = DysonNetwork.Shared.Proto.LinkEmbed;
+using ModelsLinkEmbed = DysonNetwork.Shared.Models.Embed.LinkEmbed;
+
+namespace DysonNetwork.Shared.Registry;
+
+public class RemoteWebReaderService(WebReaderService.WebReaderServiceClient webReader)
+{
+ public async Task<(ModelsLinkEmbed LinkEmbed, string? Content)> ScrapeArticle(string url)
+ {
+ var request = new ScrapeArticleRequest { Url = url };
+ var response = await webReader.ScrapeArticleAsync(request);
+ return (
+ LinkEmbed: response.Article?.LinkEmbed != null ? ModelsLinkEmbed.FromProtoValue(response.Article.LinkEmbed) : null!,
+ Content: response.Article?.Content == "" ? null : response.Article?.Content
+ );
+ }
+
+ public async Task GetLinkPreview(string url, bool bypassCache = false)
+ {
+ var request = new GetLinkPreviewRequest { Url = url, BypassCache = bypassCache };
+ var response = await webReader.GetLinkPreviewAsync(request);
+ return response.Preview != null ? ModelsLinkEmbed.FromProtoValue(response.Preview) : null!;
+ }
+
+ public async Task InvalidateLinkPreviewCache(string url)
+ {
+ var request = new InvalidateLinkPreviewCacheRequest { Url = url };
+ var response = await webReader.InvalidateLinkPreviewCacheAsync(request);
+ return response.Success;
+ }
+}
diff --git a/DysonNetwork.Shared/Registry/ServiceInjectionHelper.cs b/DysonNetwork.Shared/Registry/ServiceInjectionHelper.cs
index 463a943..329245a 100644
--- a/DysonNetwork.Shared/Registry/ServiceInjectionHelper.cs
+++ b/DysonNetwork.Shared/Registry/ServiceInjectionHelper.cs
@@ -21,7 +21,6 @@ public static class ServiceInjectionHelper
services.AddGrpcClientWithSharedChannel(
"https://_grpc.pass",
"AuthService");
-
services.AddGrpcClientWithSharedChannel(
"https://_grpc.pass",
"PermissionService");
@@ -39,19 +38,15 @@ public static class ServiceInjectionHelper
services.AddGrpcClientWithSharedChannel(
"https://_grpc.pass",
"BotAccountReceiverService");
-
services.AddGrpcClientWithSharedChannel(
"https://_grpc.pass",
"ActionLogService");
-
services.AddGrpcClientWithSharedChannel(
"https://_grpc.pass",
"PaymentService");
-
services.AddGrpcClientWithSharedChannel(
"https://_grpc.pass",
"WalletService");
-
services.AddGrpcClientWithSharedChannel(
"https://_grpc.pass",
"RealmService");
@@ -107,5 +102,24 @@ public static class ServiceInjectionHelper
return services;
}
+
+ public IServiceCollection AddInsightService()
+ {
+ services.AddGrpcClientWithSharedChannel(
+ "https://_grpc.insight",
+ "WebFeedServiceClient");
+ services.AddGrpcClientWithSharedChannel(
+ "https://_grpc.insight",
+ "WebArticleService");
+ services.AddGrpcClientWithSharedChannel(
+ "https://_grpc.insight",
+ "WebReaderServiceClient");
+
+ services.AddSingleton();
+ services.AddSingleton();
+ services.AddSingleton();
+
+ return services;
+ }
}
}
diff --git a/DysonNetwork.Sphere/AppDatabase.cs b/DysonNetwork.Sphere/AppDatabase.cs
index 9aad114..5a9aba3 100644
--- a/DysonNetwork.Sphere/AppDatabase.cs
+++ b/DysonNetwork.Sphere/AppDatabase.cs
@@ -149,12 +149,6 @@ public class AppDatabase(
modelBuilder.ApplySoftDeleteFilters();
}
- private static void SetSoftDeleteFilter(ModelBuilder modelBuilder)
- where TEntity : ModelBase
- {
- modelBuilder.Entity().HasQueryFilter(e => e.DeletedAt == null);
- }
-
public override async Task SaveChangesAsync(CancellationToken cancellationToken = default)
{
this.ApplyAuditableAndSoftDelete();
diff --git a/DysonNetwork.Sphere/DysonNetwork.Sphere.csproj b/DysonNetwork.Sphere/DysonNetwork.Sphere.csproj
index ebf7d12..f64e82d 100644
--- a/DysonNetwork.Sphere/DysonNetwork.Sphere.csproj
+++ b/DysonNetwork.Sphere/DysonNetwork.Sphere.csproj
@@ -98,6 +98,7 @@
+
diff --git a/DysonNetwork.Sphere/Migrations/20250717135738_InitialMigration.Designer.cs b/DysonNetwork.Sphere/Migrations/20250717135738_InitialMigration.Designer.cs
index a5cacfb..c1ec95d 100644
--- a/DysonNetwork.Sphere/Migrations/20250717135738_InitialMigration.Designer.cs
+++ b/DysonNetwork.Sphere/Migrations/20250717135738_InitialMigration.Designer.cs
@@ -3,7 +3,6 @@ using System;
using System.Collections.Generic;
using DysonNetwork.Shared.Models;
using DysonNetwork.Sphere;
-using DysonNetwork.Sphere.WebReader;
using Microsoft.EntityFrameworkCore;
using Microsoft.EntityFrameworkCore.Infrastructure;
using Microsoft.EntityFrameworkCore.Migrations;
@@ -1304,9 +1303,9 @@ namespace DysonNetwork.Sphere.Migrations
.HasColumnType("jsonb")
.HasColumnName("meta");
- b.Property("Preview")
- .HasColumnType("jsonb")
- .HasColumnName("preview");
+ // b.Property("Preview")
+ // .HasColumnType("jsonb")
+ // .HasColumnName("preview");
b.Property("PublishedAt")
.HasColumnType("timestamp with time zone")
@@ -1366,9 +1365,9 @@ namespace DysonNetwork.Sphere.Migrations
.HasColumnType("character varying(8192)")
.HasColumnName("description");
- b.Property("Preview")
- .HasColumnType("jsonb")
- .HasColumnName("preview");
+ // b.Property("Preview")
+ // .HasColumnType("jsonb")
+ // .HasColumnName("preview");
b.Property("PublisherId")
.HasColumnType("uuid")
diff --git a/DysonNetwork.Sphere/Migrations/20250717135738_InitialMigration.cs b/DysonNetwork.Sphere/Migrations/20250717135738_InitialMigration.cs
index dbafa21..1644f26 100644
--- a/DysonNetwork.Sphere/Migrations/20250717135738_InitialMigration.cs
+++ b/DysonNetwork.Sphere/Migrations/20250717135738_InitialMigration.cs
@@ -1,5 +1,5 @@
using DysonNetwork.Shared.Models;
-using DysonNetwork.Sphere.WebReader;
+
using Microsoft.EntityFrameworkCore.Migrations;
using NodaTime;
using NpgsqlTypes;
@@ -439,7 +439,6 @@ namespace DysonNetwork.Sphere.Migrations
url = table.Column(type: "character varying(8192)", maxLength: 8192, nullable: false),
title = table.Column(type: "character varying(4096)", maxLength: 4096, nullable: false),
description = table.Column(type: "character varying(8192)", maxLength: 8192, nullable: true),
- preview = table.Column(type: "jsonb", nullable: true),
config = table.Column(type: "jsonb", nullable: false),
publisher_id = table.Column(type: "uuid", nullable: false),
created_at = table.Column(type: "timestamp with time zone", nullable: false),
@@ -692,7 +691,6 @@ namespace DysonNetwork.Sphere.Migrations
url = table.Column(type: "character varying(8192)", maxLength: 8192, nullable: false),
author = table.Column(type: "character varying(4096)", maxLength: 4096, nullable: true),
meta = table.Column>(type: "jsonb", nullable: true),
- preview = table.Column(type: "jsonb", nullable: true),
content = table.Column(type: "text", nullable: true),
published_at = table.Column(type: "timestamp with time zone", nullable: true),
feed_id = table.Column(type: "uuid", nullable: false),
diff --git a/DysonNetwork.Sphere/Migrations/20250731065135_AddCheckInBackdated.Designer.cs b/DysonNetwork.Sphere/Migrations/20250731065135_AddCheckInBackdated.Designer.cs
index 872332f..9f1c82c 100644
--- a/DysonNetwork.Sphere/Migrations/20250731065135_AddCheckInBackdated.Designer.cs
+++ b/DysonNetwork.Sphere/Migrations/20250731065135_AddCheckInBackdated.Designer.cs
@@ -3,7 +3,6 @@ using System;
using System.Collections.Generic;
using DysonNetwork.Shared.Models;
using DysonNetwork.Sphere;
-using DysonNetwork.Sphere.WebReader;
using Microsoft.EntityFrameworkCore;
using Microsoft.EntityFrameworkCore.Infrastructure;
using Microsoft.EntityFrameworkCore.Migrations;
@@ -1304,9 +1303,6 @@ namespace DysonNetwork.Sphere.Migrations
.HasColumnType("jsonb")
.HasColumnName("meta");
- b.Property("Preview")
- .HasColumnType("jsonb")
- .HasColumnName("preview");
b.Property("PublishedAt")
.HasColumnType("timestamp with time zone")
@@ -1366,9 +1362,6 @@ namespace DysonNetwork.Sphere.Migrations
.HasColumnType("character varying(8192)")
.HasColumnName("description");
- b.Property("Preview")
- .HasColumnType("jsonb")
- .HasColumnName("preview");
b.Property("PublisherId")
.HasColumnType("uuid")
diff --git a/DysonNetwork.Sphere/Migrations/20250802095248_AddPoll.Designer.cs b/DysonNetwork.Sphere/Migrations/20250802095248_AddPoll.Designer.cs
index 6259820..fdbff7d 100644
--- a/DysonNetwork.Sphere/Migrations/20250802095248_AddPoll.Designer.cs
+++ b/DysonNetwork.Sphere/Migrations/20250802095248_AddPoll.Designer.cs
@@ -4,7 +4,6 @@ using System.Collections.Generic;
using System.Text.Json;
using DysonNetwork.Shared.Models;
using DysonNetwork.Sphere;
-using DysonNetwork.Sphere.WebReader;
using Microsoft.EntityFrameworkCore;
using Microsoft.EntityFrameworkCore.Infrastructure;
using Microsoft.EntityFrameworkCore.Migrations;
@@ -1451,9 +1450,6 @@ namespace DysonNetwork.Sphere.Migrations
.HasColumnType("jsonb")
.HasColumnName("meta");
- b.Property("Preview")
- .HasColumnType("jsonb")
- .HasColumnName("preview");
b.Property("PublishedAt")
.HasColumnType("timestamp with time zone")
@@ -1513,9 +1509,6 @@ namespace DysonNetwork.Sphere.Migrations
.HasColumnType("character varying(8192)")
.HasColumnName("description");
- b.Property("Preview")
- .HasColumnType("jsonb")
- .HasColumnName("preview");
b.Property("PublisherId")
.HasColumnType("uuid")
diff --git a/DysonNetwork.Sphere/Migrations/20250805183629_AddStickerOwnerships.Designer.cs b/DysonNetwork.Sphere/Migrations/20250805183629_AddStickerOwnerships.Designer.cs
index da15722..6269f59 100644
--- a/DysonNetwork.Sphere/Migrations/20250805183629_AddStickerOwnerships.Designer.cs
+++ b/DysonNetwork.Sphere/Migrations/20250805183629_AddStickerOwnerships.Designer.cs
@@ -4,7 +4,6 @@ using System.Collections.Generic;
using System.Text.Json;
using DysonNetwork.Shared.Models;
using DysonNetwork.Sphere;
-using DysonNetwork.Sphere.WebReader;
using Microsoft.EntityFrameworkCore;
using Microsoft.EntityFrameworkCore.Infrastructure;
using Microsoft.EntityFrameworkCore.Migrations;
@@ -1487,9 +1486,6 @@ namespace DysonNetwork.Sphere.Migrations
.HasColumnType("jsonb")
.HasColumnName("meta");
- b.Property("Preview")
- .HasColumnType("jsonb")
- .HasColumnName("preview");
b.Property("PublishedAt")
.HasColumnType("timestamp with time zone")
@@ -1549,9 +1545,6 @@ namespace DysonNetwork.Sphere.Migrations
.HasColumnType("character varying(8192)")
.HasColumnName("description");
- b.Property("Preview")
- .HasColumnType("jsonb")
- .HasColumnName("preview");
b.Property("PublisherId")
.HasColumnType("uuid")
diff --git a/DysonNetwork.Sphere/Migrations/20250807162646_RemoveDevelopers.Designer.cs b/DysonNetwork.Sphere/Migrations/20250807162646_RemoveDevelopers.Designer.cs
index ce70ac6..3cb532b 100644
--- a/DysonNetwork.Sphere/Migrations/20250807162646_RemoveDevelopers.Designer.cs
+++ b/DysonNetwork.Sphere/Migrations/20250807162646_RemoveDevelopers.Designer.cs
@@ -4,7 +4,7 @@ using System.Collections.Generic;
using System.Text.Json;
using DysonNetwork.Shared.Models;
using DysonNetwork.Sphere;
-using DysonNetwork.Sphere.WebReader;
+
using Microsoft.EntityFrameworkCore;
using Microsoft.EntityFrameworkCore.Infrastructure;
using Microsoft.EntityFrameworkCore.Migrations;
@@ -1305,9 +1305,6 @@ namespace DysonNetwork.Sphere.Migrations
.HasColumnType("jsonb")
.HasColumnName("meta");
- b.Property("Preview")
- .HasColumnType("jsonb")
- .HasColumnName("preview");
b.Property("PublishedAt")
.HasColumnType("timestamp with time zone")
@@ -1367,9 +1364,6 @@ namespace DysonNetwork.Sphere.Migrations
.HasColumnType("character varying(8192)")
.HasColumnName("description");
- b.Property("Preview")
- .HasColumnType("jsonb")
- .HasColumnName("preview");
b.Property("PublisherId")
.HasColumnType("uuid")
diff --git a/DysonNetwork.Sphere/Migrations/20250812041519_AddPostFeaturedRecord.Designer.cs b/DysonNetwork.Sphere/Migrations/20250812041519_AddPostFeaturedRecord.Designer.cs
index 47ab6cf..cd4130a 100644
--- a/DysonNetwork.Sphere/Migrations/20250812041519_AddPostFeaturedRecord.Designer.cs
+++ b/DysonNetwork.Sphere/Migrations/20250812041519_AddPostFeaturedRecord.Designer.cs
@@ -4,7 +4,7 @@ using System.Collections.Generic;
using System.Text.Json;
using DysonNetwork.Shared.Models;
using DysonNetwork.Sphere;
-using DysonNetwork.Sphere.WebReader;
+
using Microsoft.EntityFrameworkCore;
using Microsoft.EntityFrameworkCore.Infrastructure;
using Microsoft.EntityFrameworkCore.Migrations;
@@ -1345,9 +1345,6 @@ namespace DysonNetwork.Sphere.Migrations
.HasColumnType("jsonb")
.HasColumnName("meta");
- b.Property("Preview")
- .HasColumnType("jsonb")
- .HasColumnName("preview");
b.Property("PublishedAt")
.HasColumnType("timestamp with time zone")
@@ -1407,9 +1404,6 @@ namespace DysonNetwork.Sphere.Migrations
.HasColumnType("character varying(8192)")
.HasColumnName("description");
- b.Property("Preview")
- .HasColumnType("jsonb")
- .HasColumnName("preview");
b.Property("PublisherId")
.HasColumnType("uuid")
diff --git a/DysonNetwork.Sphere/Migrations/20250814183405_AddRealmPost.Designer.cs b/DysonNetwork.Sphere/Migrations/20250814183405_AddRealmPost.Designer.cs
index b22d059..4104e75 100644
--- a/DysonNetwork.Sphere/Migrations/20250814183405_AddRealmPost.Designer.cs
+++ b/DysonNetwork.Sphere/Migrations/20250814183405_AddRealmPost.Designer.cs
@@ -4,7 +4,7 @@ using System.Collections.Generic;
using System.Text.Json;
using DysonNetwork.Shared.Models;
using DysonNetwork.Sphere;
-using DysonNetwork.Sphere.WebReader;
+
using Microsoft.EntityFrameworkCore;
using Microsoft.EntityFrameworkCore.Infrastructure;
using Microsoft.EntityFrameworkCore.Migrations;
@@ -1352,9 +1352,6 @@ namespace DysonNetwork.Sphere.Migrations
.HasColumnType("jsonb")
.HasColumnName("meta");
- b.Property("Preview")
- .HasColumnType("jsonb")
- .HasColumnName("preview");
b.Property("PublishedAt")
.HasColumnType("timestamp with time zone")
@@ -1414,9 +1411,6 @@ namespace DysonNetwork.Sphere.Migrations
.HasColumnType("character varying(8192)")
.HasColumnName("description");
- b.Property("Preview")
- .HasColumnType("jsonb")
- .HasColumnName("preview");
b.Property("PublisherId")
.HasColumnType("uuid")
diff --git a/DysonNetwork.Sphere/Migrations/20250815041220_AddPostSlug.Designer.cs b/DysonNetwork.Sphere/Migrations/20250815041220_AddPostSlug.Designer.cs
index 44f364e..b7feeea 100644
--- a/DysonNetwork.Sphere/Migrations/20250815041220_AddPostSlug.Designer.cs
+++ b/DysonNetwork.Sphere/Migrations/20250815041220_AddPostSlug.Designer.cs
@@ -4,7 +4,7 @@ using System.Collections.Generic;
using System.Text.Json;
using DysonNetwork.Shared.Models;
using DysonNetwork.Sphere;
-using DysonNetwork.Sphere.WebReader;
+
using Microsoft.EntityFrameworkCore;
using Microsoft.EntityFrameworkCore.Infrastructure;
using Microsoft.EntityFrameworkCore.Migrations;
@@ -1352,9 +1352,6 @@ namespace DysonNetwork.Sphere.Migrations
.HasColumnType("jsonb")
.HasColumnName("meta");
- b.Property("Preview")
- .HasColumnType("jsonb")
- .HasColumnName("preview");
b.Property("PublishedAt")
.HasColumnType("timestamp with time zone")
@@ -1414,9 +1411,6 @@ namespace DysonNetwork.Sphere.Migrations
.HasColumnType("character varying(8192)")
.HasColumnName("description");
- b.Property("Preview")
- .HasColumnType("jsonb")
- .HasColumnName("preview");
b.Property("PublisherId")
.HasColumnType("uuid")
diff --git a/DysonNetwork.Sphere/Migrations/20250819175925_AddPostParentGone.Designer.cs b/DysonNetwork.Sphere/Migrations/20250819175925_AddPostParentGone.Designer.cs
index 24ade42..8e1e1a9 100644
--- a/DysonNetwork.Sphere/Migrations/20250819175925_AddPostParentGone.Designer.cs
+++ b/DysonNetwork.Sphere/Migrations/20250819175925_AddPostParentGone.Designer.cs
@@ -4,7 +4,7 @@ using System.Collections.Generic;
using System.Text.Json;
using DysonNetwork.Shared.Models;
using DysonNetwork.Sphere;
-using DysonNetwork.Sphere.WebReader;
+
using Microsoft.EntityFrameworkCore;
using Microsoft.EntityFrameworkCore.Infrastructure;
using Microsoft.EntityFrameworkCore.Migrations;
@@ -1360,9 +1360,6 @@ namespace DysonNetwork.Sphere.Migrations
.HasColumnType("jsonb")
.HasColumnName("meta");
- b.Property("Preview")
- .HasColumnType("jsonb")
- .HasColumnName("preview");
b.Property("PublishedAt")
.HasColumnType("timestamp with time zone")
@@ -1422,9 +1419,6 @@ namespace DysonNetwork.Sphere.Migrations
.HasColumnType("character varying(8192)")
.HasColumnName("description");
- b.Property("Preview")
- .HasColumnType("jsonb")
- .HasColumnName("preview");
b.Property("PublisherId")
.HasColumnType("uuid")
diff --git a/DysonNetwork.Sphere/Migrations/20250820060654_AddWebFeedSubscription.Designer.cs b/DysonNetwork.Sphere/Migrations/20250820060654_AddWebFeedSubscription.Designer.cs
index 0f6ede2..5631e4b 100644
--- a/DysonNetwork.Sphere/Migrations/20250820060654_AddWebFeedSubscription.Designer.cs
+++ b/DysonNetwork.Sphere/Migrations/20250820060654_AddWebFeedSubscription.Designer.cs
@@ -4,7 +4,7 @@ using System.Collections.Generic;
using System.Text.Json;
using DysonNetwork.Shared.Models;
using DysonNetwork.Sphere;
-using DysonNetwork.Sphere.WebReader;
+
using Microsoft.EntityFrameworkCore;
using Microsoft.EntityFrameworkCore.Infrastructure;
using Microsoft.EntityFrameworkCore.Migrations;
@@ -1360,9 +1360,6 @@ namespace DysonNetwork.Sphere.Migrations
.HasColumnType("jsonb")
.HasColumnName("meta");
- b.Property("Preview")
- .HasColumnType("jsonb")
- .HasColumnName("preview");
b.Property("PublishedAt")
.HasColumnType("timestamp with time zone")
@@ -1422,9 +1419,6 @@ namespace DysonNetwork.Sphere.Migrations
.HasColumnType("character varying(8192)")
.HasColumnName("description");
- b.Property("Preview")
- .HasColumnType("jsonb")
- .HasColumnName("preview");
b.Property("PublisherId")
.HasColumnType("uuid")
diff --git a/DysonNetwork.Sphere/Migrations/20250825045548_AddPostPin.Designer.cs b/DysonNetwork.Sphere/Migrations/20250825045548_AddPostPin.Designer.cs
index ba3e9e0..5e8b206 100644
--- a/DysonNetwork.Sphere/Migrations/20250825045548_AddPostPin.Designer.cs
+++ b/DysonNetwork.Sphere/Migrations/20250825045548_AddPostPin.Designer.cs
@@ -4,7 +4,7 @@ using System.Collections.Generic;
using System.Text.Json;
using DysonNetwork.Shared.Models;
using DysonNetwork.Sphere;
-using DysonNetwork.Sphere.WebReader;
+
using Microsoft.EntityFrameworkCore;
using Microsoft.EntityFrameworkCore.Infrastructure;
using Microsoft.EntityFrameworkCore.Migrations;
@@ -1364,9 +1364,6 @@ namespace DysonNetwork.Sphere.Migrations
.HasColumnType("jsonb")
.HasColumnName("meta");
- b.Property("Preview")
- .HasColumnType("jsonb")
- .HasColumnName("preview");
b.Property("PublishedAt")
.HasColumnType("timestamp with time zone")
@@ -1426,9 +1423,6 @@ namespace DysonNetwork.Sphere.Migrations
.HasColumnType("character varying(8192)")
.HasColumnName("description");
- b.Property("Preview")
- .HasColumnType("jsonb")
- .HasColumnName("preview");
b.Property("PublisherId")
.HasColumnType("uuid")
diff --git a/DysonNetwork.Sphere/Migrations/20250825054901_AddPostCategoryTagSubscription.Designer.cs b/DysonNetwork.Sphere/Migrations/20250825054901_AddPostCategoryTagSubscription.Designer.cs
index 3b59048..5ab2ec3 100644
--- a/DysonNetwork.Sphere/Migrations/20250825054901_AddPostCategoryTagSubscription.Designer.cs
+++ b/DysonNetwork.Sphere/Migrations/20250825054901_AddPostCategoryTagSubscription.Designer.cs
@@ -4,7 +4,7 @@ using System.Collections.Generic;
using System.Text.Json;
using DysonNetwork.Shared.Models;
using DysonNetwork.Sphere;
-using DysonNetwork.Sphere.WebReader;
+
using Microsoft.EntityFrameworkCore;
using Microsoft.EntityFrameworkCore.Infrastructure;
using Microsoft.EntityFrameworkCore.Migrations;
@@ -1407,9 +1407,6 @@ namespace DysonNetwork.Sphere.Migrations
.HasColumnType("jsonb")
.HasColumnName("meta");
- b.Property("Preview")
- .HasColumnType("jsonb")
- .HasColumnName("preview");
b.Property("PublishedAt")
.HasColumnType("timestamp with time zone")
@@ -1469,9 +1466,6 @@ namespace DysonNetwork.Sphere.Migrations
.HasColumnType("character varying(8192)")
.HasColumnName("description");
- b.Property("Preview")
- .HasColumnType("jsonb")
- .HasColumnName("preview");
b.Property("PublisherId")
.HasColumnType("uuid")
diff --git a/DysonNetwork.Sphere/Migrations/20250904162157_AddPostAwardScore.Designer.cs b/DysonNetwork.Sphere/Migrations/20250904162157_AddPostAwardScore.Designer.cs
index d82928e..b66d4ee 100644
--- a/DysonNetwork.Sphere/Migrations/20250904162157_AddPostAwardScore.Designer.cs
+++ b/DysonNetwork.Sphere/Migrations/20250904162157_AddPostAwardScore.Designer.cs
@@ -4,7 +4,7 @@ using System.Collections.Generic;
using System.Text.Json;
using DysonNetwork.Shared.Models;
using DysonNetwork.Sphere;
-using DysonNetwork.Sphere.WebReader;
+
using Microsoft.EntityFrameworkCore;
using Microsoft.EntityFrameworkCore.Infrastructure;
using Microsoft.EntityFrameworkCore.Migrations;
@@ -1460,9 +1460,6 @@ namespace DysonNetwork.Sphere.Migrations
.HasColumnType("jsonb")
.HasColumnName("meta");
- b.Property("Preview")
- .HasColumnType("jsonb")
- .HasColumnName("preview");
b.Property("PublishedAt")
.HasColumnType("timestamp with time zone")
@@ -1522,9 +1519,6 @@ namespace DysonNetwork.Sphere.Migrations
.HasColumnType("character varying(8192)")
.HasColumnName("description");
- b.Property("Preview")
- .HasColumnType("jsonb")
- .HasColumnName("preview");
b.Property("PublisherId")
.HasColumnType("uuid")
diff --git a/DysonNetwork.Sphere/Migrations/20250907070000_RemoveNetTopo.Designer.cs b/DysonNetwork.Sphere/Migrations/20250907070000_RemoveNetTopo.Designer.cs
index 8cfa241..7ef3c40 100644
--- a/DysonNetwork.Sphere/Migrations/20250907070000_RemoveNetTopo.Designer.cs
+++ b/DysonNetwork.Sphere/Migrations/20250907070000_RemoveNetTopo.Designer.cs
@@ -4,7 +4,7 @@ using System.Collections.Generic;
using System.Text.Json;
using DysonNetwork.Shared.Models;
using DysonNetwork.Sphere;
-using DysonNetwork.Sphere.WebReader;
+
using Microsoft.EntityFrameworkCore;
using Microsoft.EntityFrameworkCore.Infrastructure;
using Microsoft.EntityFrameworkCore.Migrations;
@@ -1459,9 +1459,6 @@ namespace DysonNetwork.Sphere.Migrations
.HasColumnType("jsonb")
.HasColumnName("meta");
- b.Property("Preview")
- .HasColumnType("jsonb")
- .HasColumnName("preview");
b.Property("PublishedAt")
.HasColumnType("timestamp with time zone")
@@ -1521,9 +1518,6 @@ namespace DysonNetwork.Sphere.Migrations
.HasColumnType("character varying(8192)")
.HasColumnName("description");
- b.Property("Preview")
- .HasColumnType("jsonb")
- .HasColumnName("preview");
b.Property("PublisherId")
.HasColumnType("uuid")
diff --git a/DysonNetwork.Sphere/Migrations/20250907142032_AddPostEmbedView.Designer.cs b/DysonNetwork.Sphere/Migrations/20250907142032_AddPostEmbedView.Designer.cs
index c7c8591..a65b8e8 100644
--- a/DysonNetwork.Sphere/Migrations/20250907142032_AddPostEmbedView.Designer.cs
+++ b/DysonNetwork.Sphere/Migrations/20250907142032_AddPostEmbedView.Designer.cs
@@ -4,7 +4,7 @@ using System.Collections.Generic;
using System.Text.Json;
using DysonNetwork.Shared.Models;
using DysonNetwork.Sphere;
-using DysonNetwork.Sphere.WebReader;
+
using Microsoft.EntityFrameworkCore;
using Microsoft.EntityFrameworkCore.Infrastructure;
using Microsoft.EntityFrameworkCore.Migrations;
@@ -1463,9 +1463,6 @@ namespace DysonNetwork.Sphere.Migrations
.HasColumnType("jsonb")
.HasColumnName("meta");
- b.Property("Preview")
- .HasColumnType("jsonb")
- .HasColumnName("preview");
b.Property("PublishedAt")
.HasColumnType("timestamp with time zone")
@@ -1525,9 +1522,6 @@ namespace DysonNetwork.Sphere.Migrations
.HasColumnType("character varying(8192)")
.HasColumnName("description");
- b.Property("Preview")
- .HasColumnType("jsonb")
- .HasColumnName("preview");
b.Property("PublisherId")
.HasColumnType("uuid")
diff --git a/DysonNetwork.Sphere/Migrations/20250907150520_AddPollAnonymousOrNot.Designer.cs b/DysonNetwork.Sphere/Migrations/20250907150520_AddPollAnonymousOrNot.Designer.cs
index e19df8b..af5c3c8 100644
--- a/DysonNetwork.Sphere/Migrations/20250907150520_AddPollAnonymousOrNot.Designer.cs
+++ b/DysonNetwork.Sphere/Migrations/20250907150520_AddPollAnonymousOrNot.Designer.cs
@@ -4,7 +4,7 @@ using System.Collections.Generic;
using System.Text.Json;
using DysonNetwork.Shared.Models;
using DysonNetwork.Sphere;
-using DysonNetwork.Sphere.WebReader;
+
using Microsoft.EntityFrameworkCore;
using Microsoft.EntityFrameworkCore.Infrastructure;
using Microsoft.EntityFrameworkCore.Migrations;
@@ -1467,9 +1467,6 @@ namespace DysonNetwork.Sphere.Migrations
.HasColumnType("jsonb")
.HasColumnName("meta");
- b.Property("Preview")
- .HasColumnType("jsonb")
- .HasColumnName("preview");
b.Property("PublishedAt")
.HasColumnType("timestamp with time zone")
@@ -1529,9 +1526,6 @@ namespace DysonNetwork.Sphere.Migrations
.HasColumnType("character varying(8192)")
.HasColumnName("description");
- b.Property("Preview")
- .HasColumnType("jsonb")
- .HasColumnName("preview");
b.Property("PublisherId")
.HasColumnType("uuid")
diff --git a/DysonNetwork.Sphere/Migrations/20251021154500_ChangeRealmReferenceMode.Designer.cs b/DysonNetwork.Sphere/Migrations/20251021154500_ChangeRealmReferenceMode.Designer.cs
index 6af23d1..241af2b 100644
--- a/DysonNetwork.Sphere/Migrations/20251021154500_ChangeRealmReferenceMode.Designer.cs
+++ b/DysonNetwork.Sphere/Migrations/20251021154500_ChangeRealmReferenceMode.Designer.cs
@@ -4,7 +4,7 @@ using System.Collections.Generic;
using System.Text.Json;
using DysonNetwork.Shared.Models;
using DysonNetwork.Sphere;
-using DysonNetwork.Sphere.WebReader;
+
using Microsoft.EntityFrameworkCore;
using Microsoft.EntityFrameworkCore.Infrastructure;
using Microsoft.EntityFrameworkCore.Migrations;
@@ -1468,9 +1468,6 @@ namespace DysonNetwork.Sphere.Migrations
.HasColumnType("jsonb")
.HasColumnName("meta");
- b.Property("Preview")
- .HasColumnType("jsonb")
- .HasColumnName("preview");
b.Property("PublishedAt")
.HasColumnType("timestamp with time zone")
@@ -1530,9 +1527,6 @@ namespace DysonNetwork.Sphere.Migrations
.HasColumnType("character varying(8192)")
.HasColumnName("description");
- b.Property("Preview")
- .HasColumnType("jsonb")
- .HasColumnName("preview");
b.Property("PublisherId")
.HasColumnType("uuid")
diff --git a/DysonNetwork.Sphere/Migrations/20251022170100_RemoveRealms.Designer.cs b/DysonNetwork.Sphere/Migrations/20251022170100_RemoveRealms.Designer.cs
index 2e1b529..dc051ad 100644
--- a/DysonNetwork.Sphere/Migrations/20251022170100_RemoveRealms.Designer.cs
+++ b/DysonNetwork.Sphere/Migrations/20251022170100_RemoveRealms.Designer.cs
@@ -4,7 +4,7 @@ using System.Collections.Generic;
using System.Text.Json;
using DysonNetwork.Shared.Models;
using DysonNetwork.Sphere;
-using DysonNetwork.Sphere.WebReader;
+
using Microsoft.EntityFrameworkCore;
using Microsoft.EntityFrameworkCore.Infrastructure;
using Microsoft.EntityFrameworkCore.Migrations;
@@ -1337,9 +1337,6 @@ namespace DysonNetwork.Sphere.Migrations
.HasColumnType("jsonb")
.HasColumnName("meta");
- b.Property("Preview")
- .HasColumnType("jsonb")
- .HasColumnName("preview");
b.Property("PublishedAt")
.HasColumnType("timestamp with time zone")
@@ -1399,9 +1396,6 @@ namespace DysonNetwork.Sphere.Migrations
.HasColumnType("character varying(8192)")
.HasColumnName("description");
- b.Property("Preview")
- .HasColumnType("jsonb")
- .HasColumnName("preview");
b.Property("PublisherId")
.HasColumnType("uuid")
diff --git a/DysonNetwork.Sphere/Migrations/20251025183901_RemoveOutdatedFileIds.Designer.cs b/DysonNetwork.Sphere/Migrations/20251025183901_RemoveOutdatedFileIds.Designer.cs
index 4f16b6e..4698544 100644
--- a/DysonNetwork.Sphere/Migrations/20251025183901_RemoveOutdatedFileIds.Designer.cs
+++ b/DysonNetwork.Sphere/Migrations/20251025183901_RemoveOutdatedFileIds.Designer.cs
@@ -4,7 +4,7 @@ using System.Collections.Generic;
using System.Text.Json;
using DysonNetwork.Shared.Models;
using DysonNetwork.Sphere;
-using DysonNetwork.Sphere.WebReader;
+
using Microsoft.EntityFrameworkCore;
using Microsoft.EntityFrameworkCore.Infrastructure;
using Microsoft.EntityFrameworkCore.Migrations;
@@ -1317,9 +1317,6 @@ namespace DysonNetwork.Sphere.Migrations
.HasColumnType("jsonb")
.HasColumnName("meta");
- b.Property("Preview")
- .HasColumnType("jsonb")
- .HasColumnName("preview");
b.Property("PublishedAt")
.HasColumnType("timestamp with time zone")
@@ -1379,9 +1376,6 @@ namespace DysonNetwork.Sphere.Migrations
.HasColumnType("character varying(8192)")
.HasColumnName("description");
- b.Property("Preview")
- .HasColumnType("jsonb")
- .HasColumnName("preview");
b.Property("PublisherId")
.HasColumnType("uuid")
diff --git a/DysonNetwork.Sphere/Migrations/20251025184407_RemovePostSearchVector.Designer.cs b/DysonNetwork.Sphere/Migrations/20251025184407_RemovePostSearchVector.Designer.cs
index 6aad72e..6107701 100644
--- a/DysonNetwork.Sphere/Migrations/20251025184407_RemovePostSearchVector.Designer.cs
+++ b/DysonNetwork.Sphere/Migrations/20251025184407_RemovePostSearchVector.Designer.cs
@@ -4,7 +4,7 @@ using System.Collections.Generic;
using System.Text.Json;
using DysonNetwork.Shared.Models;
using DysonNetwork.Sphere;
-using DysonNetwork.Sphere.WebReader;
+
using Microsoft.EntityFrameworkCore;
using Microsoft.EntityFrameworkCore.Infrastructure;
using Microsoft.EntityFrameworkCore.Migrations;
@@ -1303,9 +1303,6 @@ namespace DysonNetwork.Sphere.Migrations
.HasColumnType("jsonb")
.HasColumnName("meta");
- b.Property("Preview")
- .HasColumnType("jsonb")
- .HasColumnName("preview");
b.Property("PublishedAt")
.HasColumnType("timestamp with time zone")
@@ -1365,9 +1362,6 @@ namespace DysonNetwork.Sphere.Migrations
.HasColumnType("character varying(8192)")
.HasColumnName("description");
- b.Property("Preview")
- .HasColumnType("jsonb")
- .HasColumnName("preview");
b.Property("PublisherId")
.HasColumnType("uuid")
diff --git a/DysonNetwork.Sphere/Migrations/20251118153823_AddPublicationSites.Designer.cs b/DysonNetwork.Sphere/Migrations/20251118153823_AddPublicationSites.Designer.cs
index ac3ef80..f074d80 100644
--- a/DysonNetwork.Sphere/Migrations/20251118153823_AddPublicationSites.Designer.cs
+++ b/DysonNetwork.Sphere/Migrations/20251118153823_AddPublicationSites.Designer.cs
@@ -4,7 +4,7 @@ using System.Collections.Generic;
using System.Text.Json;
using DysonNetwork.Shared.Models;
using DysonNetwork.Sphere;
-using DysonNetwork.Sphere.WebReader;
+
using Microsoft.EntityFrameworkCore;
using Microsoft.EntityFrameworkCore.Infrastructure;
using Microsoft.EntityFrameworkCore.Migrations;
@@ -1405,9 +1405,6 @@ namespace DysonNetwork.Sphere.Migrations
.HasColumnType("jsonb")
.HasColumnName("meta");
- b.Property("Preview")
- .HasColumnType("jsonb")
- .HasColumnName("preview");
b.Property("PublishedAt")
.HasColumnType("timestamp with time zone")
@@ -1467,9 +1464,6 @@ namespace DysonNetwork.Sphere.Migrations
.HasColumnType("character varying(8192)")
.HasColumnName("description");
- b.Property("Preview")
- .HasColumnType("jsonb")
- .HasColumnName("preview");
b.Property("PublisherId")
.HasColumnType("uuid")
diff --git a/DysonNetwork.Sphere/Migrations/20251119143354_RemovePublicationSites.Designer.cs b/DysonNetwork.Sphere/Migrations/20251119143354_RemovePublicationSites.Designer.cs
index d6c57d4..7904fcb 100644
--- a/DysonNetwork.Sphere/Migrations/20251119143354_RemovePublicationSites.Designer.cs
+++ b/DysonNetwork.Sphere/Migrations/20251119143354_RemovePublicationSites.Designer.cs
@@ -4,7 +4,7 @@ using System.Collections.Generic;
using System.Text.Json;
using DysonNetwork.Shared.Models;
using DysonNetwork.Sphere;
-using DysonNetwork.Sphere.WebReader;
+
using Microsoft.EntityFrameworkCore;
using Microsoft.EntityFrameworkCore.Infrastructure;
using Microsoft.EntityFrameworkCore.Migrations;
@@ -1303,9 +1303,6 @@ namespace DysonNetwork.Sphere.Migrations
.HasColumnType("jsonb")
.HasColumnName("meta");
- b.Property("Preview")
- .HasColumnType("jsonb")
- .HasColumnName("preview");
b.Property("PublishedAt")
.HasColumnType("timestamp with time zone")
@@ -1365,9 +1362,6 @@ namespace DysonNetwork.Sphere.Migrations
.HasColumnType("character varying(8192)")
.HasColumnName("description");
- b.Property("Preview")
- .HasColumnType("jsonb")
- .HasColumnName("preview");
b.Property("PublisherId")
.HasColumnType("uuid")
diff --git a/DysonNetwork.Sphere/Migrations/20251130115519_AddAccountIdBackToChatRoom.Designer.cs b/DysonNetwork.Sphere/Migrations/20251130115519_AddAccountIdBackToChatRoom.Designer.cs
index 4b4aa8c..5318059 100644
--- a/DysonNetwork.Sphere/Migrations/20251130115519_AddAccountIdBackToChatRoom.Designer.cs
+++ b/DysonNetwork.Sphere/Migrations/20251130115519_AddAccountIdBackToChatRoom.Designer.cs
@@ -4,7 +4,7 @@ using System.Collections.Generic;
using System.Text.Json;
using DysonNetwork.Shared.Models;
using DysonNetwork.Sphere;
-using DysonNetwork.Sphere.WebReader;
+
using Microsoft.EntityFrameworkCore;
using Microsoft.EntityFrameworkCore.Infrastructure;
using Microsoft.EntityFrameworkCore.Migrations;
@@ -1307,9 +1307,6 @@ namespace DysonNetwork.Sphere.Migrations
.HasColumnType("jsonb")
.HasColumnName("meta");
- b.Property("Preview")
- .HasColumnType("jsonb")
- .HasColumnName("preview");
b.Property("PublishedAt")
.HasColumnType("timestamp with time zone")
@@ -1369,9 +1366,6 @@ namespace DysonNetwork.Sphere.Migrations
.HasColumnType("character varying(8192)")
.HasColumnName("description");
- b.Property("Preview")
- .HasColumnType("jsonb")
- .HasColumnName("preview");
b.Property("PublisherId")
.HasColumnType("uuid")
diff --git a/DysonNetwork.Sphere/Migrations/20251130125717_SimplerChatRoom.Designer.cs b/DysonNetwork.Sphere/Migrations/20251130125717_SimplerChatRoom.Designer.cs
index a30dd7b..0be6e71 100644
--- a/DysonNetwork.Sphere/Migrations/20251130125717_SimplerChatRoom.Designer.cs
+++ b/DysonNetwork.Sphere/Migrations/20251130125717_SimplerChatRoom.Designer.cs
@@ -4,7 +4,7 @@ using System.Collections.Generic;
using System.Text.Json;
using DysonNetwork.Shared.Models;
using DysonNetwork.Sphere;
-using DysonNetwork.Sphere.WebReader;
+
using Microsoft.EntityFrameworkCore;
using Microsoft.EntityFrameworkCore.Infrastructure;
using Microsoft.EntityFrameworkCore.Migrations;
@@ -1306,9 +1306,6 @@ namespace DysonNetwork.Sphere.Migrations
.HasColumnType("jsonb")
.HasColumnName("meta");
- b.Property("Preview")
- .HasColumnType("jsonb")
- .HasColumnName("preview");
b.Property("PublishedAt")
.HasColumnType("timestamp with time zone")
@@ -1368,9 +1365,6 @@ namespace DysonNetwork.Sphere.Migrations
.HasColumnType("character varying(8192)")
.HasColumnName("description");
- b.Property("Preview")
- .HasColumnType("jsonb")
- .HasColumnName("preview");
b.Property("PublisherId")
.HasColumnType("uuid")
diff --git a/DysonNetwork.Sphere/Migrations/20251203161208_AddStickerPackIcon.Designer.cs b/DysonNetwork.Sphere/Migrations/20251203161208_AddStickerPackIcon.Designer.cs
index 1a5165c..a92eeae 100644
--- a/DysonNetwork.Sphere/Migrations/20251203161208_AddStickerPackIcon.Designer.cs
+++ b/DysonNetwork.Sphere/Migrations/20251203161208_AddStickerPackIcon.Designer.cs
@@ -4,7 +4,7 @@ using System.Collections.Generic;
using System.Text.Json;
using DysonNetwork.Shared.Models;
using DysonNetwork.Sphere;
-using DysonNetwork.Sphere.WebReader;
+
using Microsoft.EntityFrameworkCore;
using Microsoft.EntityFrameworkCore.Infrastructure;
using Microsoft.EntityFrameworkCore.Migrations;
@@ -1306,9 +1306,6 @@ namespace DysonNetwork.Sphere.Migrations
.HasColumnType("jsonb")
.HasColumnName("meta");
- b.Property("Preview")
- .HasColumnType("jsonb")
- .HasColumnName("preview");
b.Property("PublishedAt")
.HasColumnType("timestamp with time zone")
@@ -1368,9 +1365,6 @@ namespace DysonNetwork.Sphere.Migrations
.HasColumnType("character varying(8192)")
.HasColumnName("description");
- b.Property("Preview")
- .HasColumnType("jsonb")
- .HasColumnName("preview");
b.Property("PublisherId")
.HasColumnType("uuid")
diff --git a/DysonNetwork.Sphere/Migrations/20260101135529_AddActivityPub.Designer.cs b/DysonNetwork.Sphere/Migrations/20260101135529_AddActivityPub.Designer.cs
index 3b99942..494efd2 100644
--- a/DysonNetwork.Sphere/Migrations/20260101135529_AddActivityPub.Designer.cs
+++ b/DysonNetwork.Sphere/Migrations/20260101135529_AddActivityPub.Designer.cs
@@ -4,7 +4,7 @@ using System.Collections.Generic;
using System.Text.Json;
using DysonNetwork.Shared.Models;
using DysonNetwork.Sphere;
-using DysonNetwork.Sphere.WebReader;
+
using Microsoft.EntityFrameworkCore;
using Microsoft.EntityFrameworkCore.Infrastructure;
using Microsoft.EntityFrameworkCore.Migrations;
@@ -1761,9 +1761,6 @@ namespace DysonNetwork.Sphere.Migrations
.HasColumnType("jsonb")
.HasColumnName("meta");
- b.Property("Preview")
- .HasColumnType("jsonb")
- .HasColumnName("preview");
b.Property("PublishedAt")
.HasColumnType("timestamp with time zone")
@@ -1823,9 +1820,6 @@ namespace DysonNetwork.Sphere.Migrations
.HasColumnType("character varying(8192)")
.HasColumnName("description");
- b.Property("Preview")
- .HasColumnType("jsonb")
- .HasColumnName("preview");
b.Property("PublisherId")
.HasColumnType("uuid")
diff --git a/DysonNetwork.Sphere/Migrations/AppDatabaseModelSnapshot.cs b/DysonNetwork.Sphere/Migrations/AppDatabaseModelSnapshot.cs
index aff4ef9..f137592 100644
--- a/DysonNetwork.Sphere/Migrations/AppDatabaseModelSnapshot.cs
+++ b/DysonNetwork.Sphere/Migrations/AppDatabaseModelSnapshot.cs
@@ -4,7 +4,7 @@ using System.Collections.Generic;
using System.Text.Json;
using DysonNetwork.Shared.Models;
using DysonNetwork.Sphere;
-using DysonNetwork.Sphere.WebReader;
+
using Microsoft.EntityFrameworkCore;
using Microsoft.EntityFrameworkCore.Infrastructure;
using Microsoft.EntityFrameworkCore.Storage.ValueConversion;
@@ -1758,9 +1758,6 @@ namespace DysonNetwork.Sphere.Migrations
.HasColumnType("jsonb")
.HasColumnName("meta");
- b.Property("Preview")
- .HasColumnType("jsonb")
- .HasColumnName("preview");
b.Property("PublishedAt")
.HasColumnType("timestamp with time zone")
@@ -1820,9 +1817,6 @@ namespace DysonNetwork.Sphere.Migrations
.HasColumnType("character varying(8192)")
.HasColumnName("description");
- b.Property("Preview")
- .HasColumnType("jsonb")
- .HasColumnName("preview");
b.Property("PublisherId")
.HasColumnType("uuid")
diff --git a/DysonNetwork.Sphere/Poll/PollEmbed.cs b/DysonNetwork.Sphere/Poll/PollEmbed.cs
index 512dae0..1299a21 100644
--- a/DysonNetwork.Sphere/Poll/PollEmbed.cs
+++ b/DysonNetwork.Sphere/Poll/PollEmbed.cs
@@ -1,5 +1,6 @@
using DysonNetwork.Shared.Models;
-using DysonNetwork.Sphere.WebReader;
+using DysonNetwork.Shared.Models.Embed;
+
namespace DysonNetwork.Sphere.Poll;
diff --git a/DysonNetwork.Sphere/Post/PostActionController.cs b/DysonNetwork.Sphere/Post/PostActionController.cs
index a7e51ed..84d9b6f 100644
--- a/DysonNetwork.Sphere/Post/PostActionController.cs
+++ b/DysonNetwork.Sphere/Post/PostActionController.cs
@@ -3,11 +3,12 @@ using System.Globalization;
using DysonNetwork.Shared.Auth;
using DysonNetwork.Shared.Data;
using DysonNetwork.Shared.Models;
+using DysonNetwork.Shared.Models.Embed;
using DysonNetwork.Shared.Proto;
using DysonNetwork.Shared.Registry;
using DysonNetwork.Sphere.Poll;
using DysonNetwork.Sphere.Wallet;
-using DysonNetwork.Sphere.WebReader;
+
using Grpc.Core;
using Microsoft.AspNetCore.Authorization;
using Microsoft.AspNetCore.Mvc;
diff --git a/DysonNetwork.Sphere/Post/PostService.cs b/DysonNetwork.Sphere/Post/PostService.cs
index bdc61cb..315923d 100644
--- a/DysonNetwork.Sphere/Post/PostService.cs
+++ b/DysonNetwork.Sphere/Post/PostService.cs
@@ -3,7 +3,7 @@ using DysonNetwork.Shared;
using DysonNetwork.Shared.Cache;
using DysonNetwork.Shared.Proto;
using DysonNetwork.Shared.Registry;
-using DysonNetwork.Sphere.WebReader;
+
using DysonNetwork.Sphere.Localization;
using DysonNetwork.Sphere.Publisher;
using DysonNetwork.Sphere.ActivityPub;
@@ -12,6 +12,7 @@ using Microsoft.Extensions.Localization;
using NodaTime;
using Markdig;
using DysonNetwork.Shared.Models;
+using DysonNetwork.Shared.Models.Embed;
namespace DysonNetwork.Sphere.Post;
@@ -25,7 +26,7 @@ public partial class PostService(
FileService.FileServiceClient files,
FileReferenceService.FileReferenceServiceClient fileRefs,
Publisher.PublisherService ps,
- WebReaderService reader,
+ RemoteWebReaderService reader,
AccountService.AccountServiceClient accounts,
ActivityPubObjectFactory objFactory
)
@@ -365,7 +366,7 @@ public partial class PostService(
continue;
// Preview the link
- var linkEmbed = await reader.GetLinkPreviewAsync(url);
+ var linkEmbed = await reader.GetLinkPreview(url);
embeds.Add(EmbeddableBase.ToDictionary(linkEmbed));
processedLinks++;
}
diff --git a/DysonNetwork.Sphere/Program.cs b/DysonNetwork.Sphere/Program.cs
index f820430..cc803a0 100644
--- a/DysonNetwork.Sphere/Program.cs
+++ b/DysonNetwork.Sphere/Program.cs
@@ -20,6 +20,7 @@ builder.Services.AddDysonAuth();
builder.Services.AddAccountService();
builder.Services.AddRingService();
builder.Services.AddDriveService();
+builder.Services.AddInsightService();
builder.Services.AddAppFlushHandlers();
builder.Services.AddAppBusinessServices(builder.Configuration);
diff --git a/DysonNetwork.Sphere/Startup/ScheduledJobsConfiguration.cs b/DysonNetwork.Sphere/Startup/ScheduledJobsConfiguration.cs
index e17f4f5..344f649 100644
--- a/DysonNetwork.Sphere/Startup/ScheduledJobsConfiguration.cs
+++ b/DysonNetwork.Sphere/Startup/ScheduledJobsConfiguration.cs
@@ -1,7 +1,8 @@
+using DysonNetwork.Insight.Reader;
using DysonNetwork.Sphere.ActivityPub;
using DysonNetwork.Sphere.Post;
using DysonNetwork.Sphere.Publisher;
-using DysonNetwork.Sphere.WebReader;
+
using Quartz;
namespace DysonNetwork.Sphere.Startup;
diff --git a/DysonNetwork.Sphere/Startup/ServiceCollectionExtensions.cs b/DysonNetwork.Sphere/Startup/ServiceCollectionExtensions.cs
index a632e2a..fd494d0 100644
--- a/DysonNetwork.Sphere/Startup/ServiceCollectionExtensions.cs
+++ b/DysonNetwork.Sphere/Startup/ServiceCollectionExtensions.cs
@@ -12,7 +12,7 @@ using DysonNetwork.Sphere.Post;
using DysonNetwork.Sphere.Publisher;
using DysonNetwork.Sphere.Timeline;
using DysonNetwork.Sphere.Translation;
-using DysonNetwork.Sphere.WebReader;
+
using NodaTime;
using NodaTime.Serialization.SystemTextJson;
@@ -93,8 +93,6 @@ public static class ServiceCollectionExtensions
services.AddScoped();
services.AddScoped();
services.AddScoped();
- services.AddScoped();
- services.AddScoped();
services.AddScoped();
services.AddScoped();
services.AddScoped();
diff --git a/DysonNetwork.Sphere/Timeline/TimelineService.cs b/DysonNetwork.Sphere/Timeline/TimelineService.cs
index f5f4323..141207a 100644
--- a/DysonNetwork.Sphere/Timeline/TimelineService.cs
+++ b/DysonNetwork.Sphere/Timeline/TimelineService.cs
@@ -3,7 +3,6 @@ using DysonNetwork.Shared.Proto;
using DysonNetwork.Shared.Registry;
using DysonNetwork.Sphere.Discovery;
using DysonNetwork.Sphere.Post;
-using DysonNetwork.Sphere.WebReader;
using Microsoft.EntityFrameworkCore;
using NodaTime;
@@ -15,7 +14,8 @@ public class TimelineService(
Post.PostService ps,
RemoteRealmService rs,
DiscoveryService ds,
- AccountService.AccountServiceClient accounts
+ AccountService.AccountServiceClient accounts,
+ RemoteWebArticleService webArticles
)
{
private static double CalculateHotRank(SnPost post, Instant now)
@@ -235,40 +235,10 @@ public class TimelineService(
).ToActivity();
}
- private async Task GetArticleDiscoveryActivity(
- int count = 5,
- int feedSampleSize = 10
- )
+ private async Task GetArticleDiscoveryActivity(int count = 5)
{
var now = SystemClock.Instance.GetCurrentInstant();
- var today = now.InZone(DateTimeZone.Utc).Date;
- var todayBegin = today.AtStartOfDayInZone(DateTimeZone.Utc).ToInstant();
- var todayEnd = today.PlusDays(1).AtStartOfDayInZone(DateTimeZone.Utc).ToInstant();
- var recentFeedIds = await db
- .WebArticles.Where(a => a.CreatedAt >= todayBegin && a.CreatedAt < todayEnd)
- .GroupBy(a => a.FeedId)
- .OrderByDescending(g => g.Max(a => a.PublishedAt))
- .Take(feedSampleSize)
- .Select(g => g.Key)
- .ToListAsync();
-
- var recentArticles = new List();
- var random = new Random();
-
- foreach (var feedId in recentFeedIds.OrderBy(_ => random.Next()))
- {
- var article = await db
- .WebArticles.Include(a => a.Feed)
- .Where(a => a.FeedId == feedId)
- .OrderBy(_ => EF.Functions.Random())
- .FirstOrDefaultAsync();
-
- if (article == null)
- continue;
- recentArticles.Add(article);
- if (recentArticles.Count >= count)
- break;
- }
+ var recentArticles = await webArticles.GetRecentArticles(count);
return recentArticles.Count > 0
? new TimelineDiscoveryEvent(
@@ -379,4 +349,4 @@ public class TimelineService(
var postCount = posts.Count;
return score + postCount;
}
-}
+}
\ No newline at end of file
diff --git a/DysonNetwork.Sphere/Wallet/FundEmbed.cs b/DysonNetwork.Sphere/Wallet/FundEmbed.cs
index d8e7d46..02482c1 100644
--- a/DysonNetwork.Sphere/Wallet/FundEmbed.cs
+++ b/DysonNetwork.Sphere/Wallet/FundEmbed.cs
@@ -1,4 +1,4 @@
-using DysonNetwork.Sphere.WebReader;
+using DysonNetwork.Shared.Models.Embed;
namespace DysonNetwork.Sphere.Wallet;
diff --git a/DysonNetwork.Zone/DysonNetwork.Zone.csproj b/DysonNetwork.Zone/DysonNetwork.Zone.csproj
index a0a5ba8..1887542 100644
--- a/DysonNetwork.Zone/DysonNetwork.Zone.csproj
+++ b/DysonNetwork.Zone/DysonNetwork.Zone.csproj
@@ -30,6 +30,7 @@
+