DysonNetwork/Swarm
2
1
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

✨ Link scrapping for preview

Browse Source
This commit is contained in:
LittleSheep 2025-06-21 13:46:30 +08:00
parent 95010e4188
commit 5f30b56ef8
8 changed files with 546 additions and 2 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

26
DysonNetwork.Sphere/Connection/WebReader/IEmbeddable.cs Normal file
View File

@ -0,0 +1,26 @@
namespace DysonNetwork.Sphere.Connection.WebReader;
/// <summary>
/// The embeddable can be used in the post or messages' meta's embeds fields
/// To render richer type of content.
///
/// A simple example of using link preview embed:
/// <code>
/// {
/// // ... post content
/// "meta": {
/// "embeds": [
/// {
/// "type": "link",
/// "title: "...",
/// /// ...
/// }
/// ]
/// }
/// }
/// </code>
/// </summary>
public interface IEmbeddable
{
public string Type { get; }
}

55
DysonNetwork.Sphere/Connection/WebReader/LinkEmbed.cs Normal file
View File

@ -0,0 +1,55 @@
namespace DysonNetwork.Sphere.Connection.WebReader;
/// <summary>
/// The link embed is a part of the embeddable implementations
/// It can be used in the post or messages' meta's embeds fields
/// </summary>
public class LinkEmbed : IEmbeddable
{
public string Type => "link";
/// <summary>
/// The original URL that was processed
/// </summary>
public required string Url { get; set; }
/// <summary>
/// Title of the linked content (from OpenGraph og:title, meta title, or page title)
/// </summary>
public string? Title { get; set; }
/// <summary>
/// Description of the linked content (from OpenGraph og:description or meta description)
/// </summary>
public string? Description { get; set; }
/// <summary>
/// URL to the thumbnail image (from OpenGraph og:image or other meta tags)
/// </summary>
public string? ImageUrl { get; set; }
/// <summary>
/// The favicon URL of the site
/// </summary>
public string? FaviconUrl { get; set; }
/// <summary>
/// The site name (from OpenGraph og:site_name)
/// </summary>
public string? SiteName { get; set; }
/// <summary>
/// Type of the content (from OpenGraph og:type)
/// </summary>
public string? ContentType { get; set; }
/// <summary>
/// Author of the content if available
/// </summary>
public string? Author { get; set; }
/// <summary>
/// Published date of the content if available
/// </summary>
public DateTime? PublishedDate { get; set; }
}

110
DysonNetwork.Sphere/Connection/WebReader/WebReaderController.cs Normal file
View File

@ -0,0 +1,110 @@
using DysonNetwork.Sphere.Permission;
using Microsoft.AspNetCore.Authorization;
using Microsoft.AspNetCore.Mvc;
using Microsoft.AspNetCore.RateLimiting;
namespace DysonNetwork.Sphere.Connection.WebReader;
/// <summary>
/// Controller for web scraping and link preview services
/// </summary>
[ApiController]
[Route("/scrap")]
[EnableRateLimiting("fixed")]
public class WebReaderController(WebReaderService reader, ILogger<WebReaderController> logger)
: ControllerBase
{
/// <summary>
/// Retrieves a preview for the provided URL
/// </summary>
/// <param name="url">URL-encoded link to generate preview for</param>
/// <returns>Link preview data including title, description, and image</returns>
[HttpGet("link")]
public async Task<ActionResult<LinkEmbed>> ScrapLink([FromQuery] string url)
{
if (string.IsNullOrEmpty(url))
{
return BadRequest(new { error = "URL parameter is required" });
}
try
{
// Ensure URL is properly decoded
var decodedUrl = UrlDecoder.Decode(url);
// Validate URL format
if (!Uri.TryCreate(decodedUrl, UriKind.Absolute, out _))
{
return BadRequest(new { error = "Invalid URL format" });
}
var linkEmbed = await reader.GetLinkPreviewAsync(decodedUrl);
return Ok(linkEmbed);
}
catch (WebReaderException ex)
{
logger.LogWarning(ex, "Error scraping link: {Url}", url);
return BadRequest(new { error = ex.Message });
}
catch (Exception ex)
{
logger.LogError(ex, "Unexpected error scraping link: {Url}", url);
return StatusCode(StatusCodes.Status500InternalServerError,
new { error = "An unexpected error occurred while processing the link" });
}
}
/// <summary>
/// Force invalidates the cache for a specific URL
/// </summary>
[HttpDelete("link/cache")]
[Authorize]
[RequiredPermission("maintenance", "cache.scrap")]
public async Task<IActionResult> InvalidateCache([FromQuery] string url)
{
if (string.IsNullOrEmpty(url))
{
return BadRequest(new { error = "URL parameter is required" });
}
await reader.InvalidateCacheForUrlAsync(url);
return Ok(new { message = "Cache invalidated for URL" });
}
/// <summary>
/// Force invalidates all cached link previews
/// </summary>
[HttpDelete("cache/all")]
[Authorize]
[RequiredPermission("maintenance", "cache.scrap")]
public async Task<IActionResult> InvalidateAllCache()
{
await reader.InvalidateAllCachedPreviewsAsync();
return Ok(new { message = "All link preview caches invalidated" });
}
}
/// <summary>
/// Helper class for URL decoding
/// </summary>
public static class UrlDecoder
{
public static string Decode(string url)
{
// First check if URL is already decoded
if (!url.Contains('%') && !url.Contains('+'))
{
return url;
}
try
{
return System.Net.WebUtility.UrlDecode(url);
}
catch
{
// If decoding fails, return the original string
return url;
}
}
}

17
DysonNetwork.Sphere/Connection/WebReader/WebReaderException.cs Normal file
View File

@ -0,0 +1,17 @@
using System;
namespace DysonNetwork.Sphere.Connection.WebReader;
/// <summary>
/// Exception thrown when an error occurs during web reading operations
/// </summary>
public class WebReaderException : Exception
{
public WebReaderException(string message) : base(message)
{
}
public WebReaderException(string message, Exception innerException) : base(message, innerException)
{
}
}

334
DysonNetwork.Sphere/Connection/WebReader/WebReaderService.cs Normal file
View File

@ -0,0 +1,334 @@
using System.Globalization;
using AngleSharp;
using AngleSharp.Dom;
using DysonNetwork.Sphere.Storage;
namespace DysonNetwork.Sphere.Connection.WebReader;
/// <summary>
/// The service is amin to providing scrapping service to the Solar Network.
/// Such as news feed, external articles and link preview.
/// </summary>
public class WebReaderService(
IHttpClientFactory httpClientFactory,
ILogger<WebReaderService> logger,
ICacheService cache)
{
private const string LinkPreviewCachePrefix = "scrap:preview:";
private const string LinkPreviewCacheGroup = "scrap:preview";
/// <summary>
/// Generate a link preview embed from a URL
/// </summary>
/// <param name="url">The URL to generate the preview for</param>
/// <param name="cancellationToken">Cancellation token</param>
/// <param name="bypassCache">If true, bypass cache and fetch fresh data</param>
/// <param name="cacheExpiry">Custom cache expiration time</param>
/// <returns>A LinkEmbed object containing the preview data</returns>
public async Task<LinkEmbed> GetLinkPreviewAsync(
string url,
CancellationToken cancellationToken = default,
TimeSpan? cacheExpiry = null,
bool bypassCache = false
)
{
// Ensure URL is valid
if (!Uri.TryCreate(url, UriKind.Absolute, out var uri))
{
throw new ArgumentException(@"Invalid URL format", nameof(url));
}
// Try to get from cache if not bypassing
if (!bypassCache)
{
var cachedPreview = await GetCachedLinkPreview(url);
if (cachedPreview is not null)
return cachedPreview;
}
// Cache miss or bypass, fetch fresh data
logger.LogDebug("Fetching fresh link preview for URL: {Url}", url);
var httpClient = httpClientFactory.CreateClient("WebReader");
httpClient.MaxResponseContentBufferSize = 10 * 1024 * 1024; // 10MB, prevent scrap some directly accessible files
httpClient.Timeout = TimeSpan.FromSeconds(3);
httpClient.DefaultRequestHeaders.Add("User-Agent", "DysonNetwork/1.0 LinkPreview Bot");
try
{
var response = await httpClient.GetAsync(url, cancellationToken);
response.EnsureSuccessStatusCode();
var contentType = response.Content.Headers.ContentType?.MediaType;
if (contentType == null || !contentType.StartsWith("text/html"))
{
logger.LogWarning("URL is not an HTML page: {Url}, ContentType: {ContentType}", url, contentType);
var nonHtmlEmbed = new LinkEmbed
{
Url = url,
Title = uri.Host,
ContentType = contentType
};
// Cache non-HTML responses too
await CacheLinkPreview(nonHtmlEmbed, url, cacheExpiry);
return nonHtmlEmbed;
}
var html = await response.Content.ReadAsStringAsync(cancellationToken);
var linkEmbed = await ExtractLinkData(url, html, uri);
// Cache the result
await CacheLinkPreview(linkEmbed, url, cacheExpiry);
return linkEmbed;
}
catch (HttpRequestException ex)
{
logger.LogError(ex, "Failed to fetch URL: {Url}", url);
throw new WebReaderException($"Failed to fetch URL: {url}", ex);
}
}
private async Task<LinkEmbed> ExtractLinkData(string url, string html, Uri uri)
{
var embed = new LinkEmbed
{
Url = url
};
// Configure AngleSharp context
var config = Configuration.Default;
var context = BrowsingContext.New(config);
var document = await context.OpenAsync(req => req.Content(html));
// Extract OpenGraph tags
var ogTitle = GetMetaTagContent(document, "og:title");
var ogDescription = GetMetaTagContent(document, "og:description");
var ogImage = GetMetaTagContent(document, "og:image");
var ogSiteName = GetMetaTagContent(document, "og:site_name");
var ogType = GetMetaTagContent(document, "og:type");
// Extract Twitter card tags as fallback
var twitterTitle = GetMetaTagContent(document, "twitter:title");
var twitterDescription = GetMetaTagContent(document, "twitter:description");
var twitterImage = GetMetaTagContent(document, "twitter:image");
// Extract standard meta tags as final fallback
var metaTitle = GetMetaTagContent(document, "title") ??
GetMetaContent(document, "title");
var metaDescription = GetMetaTagContent(document, "description");
// Extract page title
var pageTitle = document.Title?.Trim();
// Extract publish date
var publishedTime = GetMetaTagContent(document, "article:published_time") ??
GetMetaTagContent(document, "datePublished") ??
GetMetaTagContent(document, "pubdate");
// Extract author
var author = GetMetaTagContent(document, "author") ??
GetMetaTagContent(document, "article:author");
// Extract favicon
var faviconUrl = GetFaviconUrl(document, uri);
// Populate the embed with the data, prioritizing OpenGraph
embed.Title = ogTitle ?? twitterTitle ?? metaTitle ?? pageTitle ?? uri.Host;
embed.Description = ogDescription ?? twitterDescription ?? metaDescription;
embed.ImageUrl = ResolveRelativeUrl(ogImage ?? twitterImage, uri);
embed.SiteName = ogSiteName ?? uri.Host;
embed.ContentType = ogType;
embed.FaviconUrl = faviconUrl;
embed.Author = author;
// Parse and set published date
if (!string.IsNullOrEmpty(publishedTime) &&
DateTime.TryParse(publishedTime, CultureInfo.InvariantCulture, DateTimeStyles.AdjustToUniversal,
out DateTime parsedDate))
{
embed.PublishedDate = parsedDate;
}
return embed;
}
private static string? GetMetaTagContent(IDocument doc, string property)
{
// Check for OpenGraph/Twitter style meta tags
var node = doc.QuerySelector($"meta[property='{property}'][content]")
?? doc.QuerySelector($"meta[name='{property}'][content]");
return node?.GetAttribute("content")?.Trim();
}
private static string? GetMetaContent(IDocument doc, string name)
{
var node = doc.QuerySelector($"meta[name='{name}'][content]");
return node?.GetAttribute("content")?.Trim();
}
private static string? GetFaviconUrl(IDocument doc, Uri baseUri)
{
// Look for apple-touch-icon first as it's typically higher quality
var appleIconNode = doc.QuerySelector("link[rel='apple-touch-icon'][href]");
if (appleIconNode != null)
{
return ResolveRelativeUrl(appleIconNode.GetAttribute("href"), baseUri);
}
// Then check for standard favicon
var faviconNode = doc.QuerySelector("link[rel='icon'][href]") ??
doc.QuerySelector("link[rel='shortcut icon'][href]");
return faviconNode != null
? ResolveRelativeUrl(faviconNode.GetAttribute("href"), baseUri)
: new Uri(baseUri, "/favicon.ico").ToString();
}
private static string? ResolveRelativeUrl(string? url, Uri baseUri)
{
if (string.IsNullOrEmpty(url))
{
return null;
}
if (Uri.TryCreate(url, UriKind.Absolute, out _))
{
return url; // Already absolute
}
return Uri.TryCreate(baseUri, url, out var absoluteUri) ? absoluteUri.ToString() : null;
}
/// <summary>
/// Generate a hash-based cache key for a URL
/// </summary>
private string GenerateUrlCacheKey(string url)
{
// Normalize the URL first
var normalizedUrl = NormalizeUrl(url);
// Create SHA256 hash of the normalized URL
using var sha256 = System.Security.Cryptography.SHA256.Create();
var urlBytes = System.Text.Encoding.UTF8.GetBytes(normalizedUrl);
var hashBytes = sha256.ComputeHash(urlBytes);
// Convert to hex string
var hashString = BitConverter.ToString(hashBytes).Replace("-", "").ToLowerInvariant();
// Return prefixed key
return $"{LinkPreviewCachePrefix}{hashString}";
}
/// <summary>
/// Normalize URL by trimming trailing slashes but preserving query parameters
/// </summary>
private string NormalizeUrl(string url)
{
if (string.IsNullOrEmpty(url))
return string.Empty;
// First ensure we have a valid URI
if (!Uri.TryCreate(url, UriKind.Absolute, out var uri))
return url.TrimEnd('/');
// Rebuild the URL without trailing slashes but with query parameters
var scheme = uri.Scheme;
var host = uri.Host;
var port = uri.IsDefaultPort ? string.Empty : $":{uri.Port}";
var path = uri.AbsolutePath.TrimEnd('/');
var query = uri.Query;
return $"{scheme}://{host}{port}{path}{query}".ToLowerInvariant();
}
/// <summary>
/// Cache a link preview
/// </summary>
private async Task CacheLinkPreview(LinkEmbed? linkEmbed, string url, TimeSpan? expiry = null)
{
if (linkEmbed == null || string.IsNullOrEmpty(url))
return;
try
{
var cacheKey = GenerateUrlCacheKey(url);
var expiryTime = expiry ?? TimeSpan.FromHours(24);
await cache.SetWithGroupsAsync(
cacheKey,
linkEmbed,
[LinkPreviewCacheGroup],
expiryTime);
logger.LogDebug("Cached link preview for URL: {Url} with key: {CacheKey}", url, cacheKey);
}
catch (Exception ex)
{
// Log but don't throw - caching failures shouldn't break the main functionality
logger.LogWarning(ex, "Failed to cache link preview for URL: {Url}", url);
}
}
/// <summary>
/// Try to get a cached link preview
/// </summary>
private async Task<LinkEmbed?> GetCachedLinkPreview(string url)
{
if (string.IsNullOrEmpty(url))
return null;
try
{
var cacheKey = GenerateUrlCacheKey(url);
var cachedPreview = await cache.GetAsync<LinkEmbed>(cacheKey);
if (cachedPreview is not null)
logger.LogDebug("Retrieved cached link preview for URL: {Url}", url);
return cachedPreview;
}
catch (Exception ex)
{
logger.LogWarning(ex, "Failed to retrieve cached link preview for URL: {Url}", url);
return null;
}
}
/// <summary>
/// Invalidate cache for a specific URL
/// </summary>
public async Task InvalidateCacheForUrlAsync(string url)
{
if (string.IsNullOrEmpty(url))
return;
try
{
var cacheKey = GenerateUrlCacheKey(url);
await cache.RemoveAsync(cacheKey);
logger.LogDebug("Invalidated cache for URL: {Url} with key: {CacheKey}", url, cacheKey);
}
catch (Exception ex)
{
logger.LogWarning(ex, "Failed to invalidate cache for URL: {Url}", url);
}
}
/// <summary>
/// Invalidate all cached link previews
/// </summary>
public async Task InvalidateAllCachedPreviewsAsync()
{
try
{
await cache.RemoveGroupAsync(LinkPreviewCacheGroup);
logger.LogInformation("Invalidated all cached link previews");
}
catch (Exception ex)
{
logger.LogWarning(ex, "Failed to invalidate all cached link previews");
}
}
}

1
DysonNetwork.Sphere/DysonNetwork.Sphere.csproj
View File

@ -16,6 +16,7 @@
</PropertyGroup>
<ItemGroup>
<PackageReference Include="AngleSharp" Version="1.3.0" />
<PackageReference Include="BCrypt.Net-Next" Version="4.0.3" />
<PackageReference Include="BlurHashSharp.SkiaSharp" Version="1.3.4" />
<PackageReference Include="EFCore.BulkExtensions" Version="9.0.1" />

4
DysonNetwork.Sphere/Startup/ServiceCollectionExtensions.cs
View File

@ -18,14 +18,13 @@ using DysonNetwork.Sphere.Storage;
using DysonNetwork.Sphere.Storage.Handlers;
using DysonNetwork.Sphere.Wallet;
using Microsoft.AspNetCore.RateLimiting;
using Microsoft.EntityFrameworkCore;
using Microsoft.OpenApi.Models;
using NodaTime;
using NodaTime.Serialization.SystemTextJson;
using Quartz;
using StackExchange.Redis;
using System.Text.Json;
using System.Threading.RateLimiting;
using DysonNetwork.Sphere.Connection.WebReader;
using tusdotnet.Stores;
namespace DysonNetwork.Sphere.Startup;
@ -221,6 +220,7 @@ public static class ServiceCollectionExtensions
services.AddScoped<WalletService>();
services.AddScoped<PaymentService>();
services.AddScoped<IRealtimeService, LivekitRealtimeService>();
services.AddScoped<WebReaderService>();
return services;
}

1
DysonNetwork.sln.DotSettings.user
View File

@ -74,6 +74,7 @@
<s:String x:Key="/Default/CodeInspection/ExcludedFiles/FilesAndFoldersToSkip2/=7020124F_002D9FFC_002D4AC3_002D8F3D_002DAAB8E0240759_002Ff_003AServiceProvider_002Ecs_002Fl_003A_002E_002E_003F_002E_002E_003F_002E_002E_003FLibrary_003FApplication_0020Support_003FJetBrains_003FRider2025_002E1_003Fresharper_002Dhost_003FSourcesCache_003Fce37be1a06b16c6faa02038d2cc477dd3bca5b217ceeb41c5f2ad45c1bf9_003FServiceProvider_002Ecs/@EntryIndexedValue">ForceIncluded</s:String>
<s:String x:Key="/Default/CodeInspection/ExcludedFiles/FilesAndFoldersToSkip2/=7020124F_002D9FFC_002D4AC3_002D8F3D_002DAAB8E0240759_002Ff_003ASetPropertyCalls_002Ecs_002Fl_003A_002E_002E_003F_002E_002E_003F_002E_002E_003FLibrary_003FApplication_0020Support_003FJetBrains_003FRider2024_002E3_003Fresharper_002Dhost_003FSourcesCache_003F458b5f22476b4599b87176214d5e4026c2327b148f4d3f885ee92362b4dac3_003FSetPropertyCalls_002Ecs/@EntryIndexedValue">ForceIncluded</s:String>
<s:String x:Key="/Default/CodeInspection/ExcludedFiles/FilesAndFoldersToSkip2/=7020124F_002D9FFC_002D4AC3_002D8F3D_002DAAB8E0240759_002Ff_003ASourceCustom_002Ecs_002Fl_003A_002E_002E_003F_002E_002E_003F_002E_002E_003FLibrary_003FApplication_0020Support_003FJetBrains_003FRider2025_002E1_003Fresharper_002Dhost_003FDecompilerCache_003Fdecompiler_003Fdaa8d9c408cd4b4286bbef7e35f1a42e31c00_003F45_003F5839ca6c_003FSourceCustom_002Ecs/@EntryIndexedValue">ForceIncluded</s:String>
<s:String x:Key="/Default/CodeInspection/ExcludedFiles/FilesAndFoldersToSkip2/=7020124F_002D9FFC_002D4AC3_002D8F3D_002DAAB8E0240759_002Ff_003AStackFrameIterator_002Ecs_002Fl_003A_002E_002E_003F_002E_002E_003F_002E_002E_003FLibrary_003FApplication_0020Support_003FJetBrains_003FRider2025_002E1_003Fresharper_002Dhost_003FDecompilerCache_003Fdecompiler_003F3bef61b8a21d4c8e96872ecdd7782fa0e55000_003F7a_003F870020d0_003FStackFrameIterator_002Ecs/@EntryIndexedValue">ForceIncluded</s:String>
<s:String x:Key="/Default/CodeInspection/ExcludedFiles/FilesAndFoldersToSkip2/=7020124F_002D9FFC_002D4AC3_002D8F3D_002DAAB8E0240759_002Ff_003AStackFrameIterator_002Ecs_002Fl_003A_002E_002E_003F_002E_002E_003F_002E_002E_003FLibrary_003FApplication_0020Support_003FJetBrains_003FRider2025_002E1_003Fresharper_002Dhost_003FDecompilerCache_003Fdecompiler_003Fb6f0571a6bc744b0b551fd4578292582e54c00_003Fdf_003F3fcdc4d2_003FStackFrameIterator_002Ecs/@EntryIndexedValue">ForceIncluded</s:String>
<s:String x:Key="/Default/CodeInspection/ExcludedFiles/FilesAndFoldersToSkip2/=7020124F_002D9FFC_002D4AC3_002D8F3D_002DAAB8E0240759_002Ff_003AStatusCodeResult_002Ecs_002Fl_003A_002E_002E_003F_002E_002E_003F_002E_002E_003FLibrary_003FApplication_0020Support_003FJetBrains_003FRider2024_002E3_003Fresharper_002Dhost_003FDecompilerCache_003Fdecompiler_003F0b5acdd962e549369896cece0026e556214600_003F7c_003F8b7572ae_003FStatusCodeResult_002Ecs/@EntryIndexedValue">ForceIncluded</s:String>
<s:String x:Key="/Default/CodeInspection/ExcludedFiles/FilesAndFoldersToSkip2/=7020124F_002D9FFC_002D4AC3_002D8F3D_002DAAB8E0240759_002Ff_003ATagging_002Ecs_002Fl_003A_002E_002E_003F_002E_002E_003F_002E_002E_003FLibrary_003FApplication_0020Support_003FJetBrains_003FRider2024_002E3_003Fresharper_002Dhost_003FSourcesCache_003F36f4c2e6baa65ba603de42eedad12ea36845aa35a910a6a82d82baf688e3e1_003FTagging_002Ecs/@EntryIndexedValue">ForceIncluded</s:String>