Compare commits
	
		
			2 Commits
		
	
	
		
			3824fba8e5
			...
			5f30b56ef8
		
	
	| Author | SHA1 | Date | |
|---|---|---|---|
| 5f30b56ef8 | |||
| 95010e4188 | 
							
								
								
									
										26
									
								
								DysonNetwork.Sphere/Connection/WebReader/IEmbeddable.cs
									
									
									
									
									
										Normal file
									
								
							
							
						
						
									
										26
									
								
								DysonNetwork.Sphere/Connection/WebReader/IEmbeddable.cs
									
									
									
									
									
										Normal file
									
								
							| @@ -0,0 +1,26 @@ | |||||||
|  | namespace DysonNetwork.Sphere.Connection.WebReader; | ||||||
|  |  | ||||||
|  | /// <summary> | ||||||
|  | /// The embeddable can be used in the post or messages' meta's embeds fields | ||||||
|  | /// To render richer type of content. | ||||||
|  | /// | ||||||
|  | /// A simple example of using link preview embed: | ||||||
|  | /// <code> | ||||||
|  | /// { | ||||||
|  | ///     // ... post content | ||||||
|  | ///     "meta": { | ||||||
|  | ///         "embeds": [ | ||||||
|  | ///             { | ||||||
|  | ///                 "type": "link", | ||||||
|  | ///                 "title: "...", | ||||||
|  | ///                 /// ... | ||||||
|  | ///             } | ||||||
|  | ///         ] | ||||||
|  | ///     } | ||||||
|  | /// } | ||||||
|  | /// </code> | ||||||
|  | /// </summary> | ||||||
|  | public interface IEmbeddable | ||||||
|  | { | ||||||
|  |     public string Type { get; } | ||||||
|  | } | ||||||
							
								
								
									
										55
									
								
								DysonNetwork.Sphere/Connection/WebReader/LinkEmbed.cs
									
									
									
									
									
										Normal file
									
								
							
							
						
						
									
										55
									
								
								DysonNetwork.Sphere/Connection/WebReader/LinkEmbed.cs
									
									
									
									
									
										Normal file
									
								
							| @@ -0,0 +1,55 @@ | |||||||
|  | namespace DysonNetwork.Sphere.Connection.WebReader; | ||||||
|  |  | ||||||
|  | /// <summary> | ||||||
|  | /// The link embed is a part of the embeddable implementations | ||||||
|  | /// It can be used in the post or messages' meta's embeds fields | ||||||
|  | /// </summary> | ||||||
|  | public class LinkEmbed : IEmbeddable | ||||||
|  | { | ||||||
|  |     public string Type => "link"; | ||||||
|  |  | ||||||
|  |     /// <summary> | ||||||
|  |     /// The original URL that was processed | ||||||
|  |     /// </summary> | ||||||
|  |     public required string Url { get; set; } | ||||||
|  |  | ||||||
|  |     /// <summary> | ||||||
|  |     /// Title of the linked content (from OpenGraph og:title, meta title, or page title) | ||||||
|  |     /// </summary> | ||||||
|  |     public string? Title { get; set; } | ||||||
|  |  | ||||||
|  |     /// <summary> | ||||||
|  |     /// Description of the linked content (from OpenGraph og:description or meta description) | ||||||
|  |     /// </summary> | ||||||
|  |     public string? Description { get; set; } | ||||||
|  |  | ||||||
|  |     /// <summary> | ||||||
|  |     /// URL to the thumbnail image (from OpenGraph og:image or other meta tags) | ||||||
|  |     /// </summary> | ||||||
|  |     public string? ImageUrl { get; set; } | ||||||
|  |  | ||||||
|  |     /// <summary> | ||||||
|  |     /// The favicon URL of the site | ||||||
|  |     /// </summary> | ||||||
|  |     public string? FaviconUrl { get; set; } | ||||||
|  |  | ||||||
|  |     /// <summary> | ||||||
|  |     /// The site name (from OpenGraph og:site_name) | ||||||
|  |     /// </summary> | ||||||
|  |     public string? SiteName { get; set; } | ||||||
|  |  | ||||||
|  |     /// <summary> | ||||||
|  |     /// Type of the content (from OpenGraph og:type) | ||||||
|  |     /// </summary> | ||||||
|  |     public string? ContentType { get; set; } | ||||||
|  |  | ||||||
|  |     /// <summary> | ||||||
|  |     /// Author of the content if available | ||||||
|  |     /// </summary> | ||||||
|  |     public string? Author { get; set; } | ||||||
|  |  | ||||||
|  |     /// <summary> | ||||||
|  |     /// Published date of the content if available | ||||||
|  |     /// </summary> | ||||||
|  |     public DateTime? PublishedDate { get; set; } | ||||||
|  | } | ||||||
							
								
								
									
										110
									
								
								DysonNetwork.Sphere/Connection/WebReader/WebReaderController.cs
									
									
									
									
									
										Normal file
									
								
							
							
						
						
									
										110
									
								
								DysonNetwork.Sphere/Connection/WebReader/WebReaderController.cs
									
									
									
									
									
										Normal file
									
								
							| @@ -0,0 +1,110 @@ | |||||||
|  | using DysonNetwork.Sphere.Permission; | ||||||
|  | using Microsoft.AspNetCore.Authorization; | ||||||
|  | using Microsoft.AspNetCore.Mvc; | ||||||
|  | using Microsoft.AspNetCore.RateLimiting; | ||||||
|  |  | ||||||
|  | namespace DysonNetwork.Sphere.Connection.WebReader; | ||||||
|  |  | ||||||
|  | /// <summary> | ||||||
|  | /// Controller for web scraping and link preview services | ||||||
|  | /// </summary> | ||||||
|  | [ApiController] | ||||||
|  | [Route("/scrap")] | ||||||
|  | [EnableRateLimiting("fixed")] | ||||||
|  | public class WebReaderController(WebReaderService reader, ILogger<WebReaderController> logger) | ||||||
|  |     : ControllerBase | ||||||
|  | { | ||||||
|  |     /// <summary> | ||||||
|  |     /// Retrieves a preview for the provided URL | ||||||
|  |     /// </summary> | ||||||
|  |     /// <param name="url">URL-encoded link to generate preview for</param> | ||||||
|  |     /// <returns>Link preview data including title, description, and image</returns> | ||||||
|  |     [HttpGet("link")] | ||||||
|  |     public async Task<ActionResult<LinkEmbed>> ScrapLink([FromQuery] string url) | ||||||
|  |     { | ||||||
|  |         if (string.IsNullOrEmpty(url)) | ||||||
|  |         { | ||||||
|  |             return BadRequest(new { error = "URL parameter is required" }); | ||||||
|  |         } | ||||||
|  |  | ||||||
|  |         try | ||||||
|  |         { | ||||||
|  |             // Ensure URL is properly decoded | ||||||
|  |             var decodedUrl = UrlDecoder.Decode(url); | ||||||
|  |  | ||||||
|  |             // Validate URL format | ||||||
|  |             if (!Uri.TryCreate(decodedUrl, UriKind.Absolute, out _)) | ||||||
|  |             { | ||||||
|  |                 return BadRequest(new { error = "Invalid URL format" }); | ||||||
|  |             } | ||||||
|  |  | ||||||
|  |             var linkEmbed = await reader.GetLinkPreviewAsync(decodedUrl); | ||||||
|  |             return Ok(linkEmbed); | ||||||
|  |         } | ||||||
|  |         catch (WebReaderException ex) | ||||||
|  |         { | ||||||
|  |             logger.LogWarning(ex, "Error scraping link: {Url}", url); | ||||||
|  |             return BadRequest(new { error = ex.Message }); | ||||||
|  |         } | ||||||
|  |         catch (Exception ex) | ||||||
|  |         { | ||||||
|  |             logger.LogError(ex, "Unexpected error scraping link: {Url}", url); | ||||||
|  |             return StatusCode(StatusCodes.Status500InternalServerError,  | ||||||
|  |                 new { error = "An unexpected error occurred while processing the link" }); | ||||||
|  |         } | ||||||
|  |     } | ||||||
|  |  | ||||||
|  |     /// <summary> | ||||||
|  |     /// Force invalidates the cache for a specific URL | ||||||
|  |     /// </summary> | ||||||
|  |     [HttpDelete("link/cache")] | ||||||
|  |     [Authorize] | ||||||
|  |     [RequiredPermission("maintenance", "cache.scrap")] | ||||||
|  |     public async Task<IActionResult> InvalidateCache([FromQuery] string url) | ||||||
|  |     { | ||||||
|  |         if (string.IsNullOrEmpty(url)) | ||||||
|  |         { | ||||||
|  |             return BadRequest(new { error = "URL parameter is required" }); | ||||||
|  |         } | ||||||
|  |  | ||||||
|  |         await reader.InvalidateCacheForUrlAsync(url); | ||||||
|  |         return Ok(new { message = "Cache invalidated for URL" }); | ||||||
|  |     } | ||||||
|  |  | ||||||
|  |     /// <summary> | ||||||
|  |     /// Force invalidates all cached link previews | ||||||
|  |     /// </summary> | ||||||
|  |     [HttpDelete("cache/all")] | ||||||
|  |     [Authorize] | ||||||
|  |     [RequiredPermission("maintenance", "cache.scrap")] | ||||||
|  |     public async Task<IActionResult> InvalidateAllCache() | ||||||
|  |     { | ||||||
|  |         await reader.InvalidateAllCachedPreviewsAsync(); | ||||||
|  |         return Ok(new { message = "All link preview caches invalidated" }); | ||||||
|  |     } | ||||||
|  | } | ||||||
|  |  | ||||||
|  | /// <summary> | ||||||
|  | /// Helper class for URL decoding | ||||||
|  | /// </summary> | ||||||
|  | public static class UrlDecoder | ||||||
|  | { | ||||||
|  |     public static string Decode(string url) | ||||||
|  |     { | ||||||
|  |         // First check if URL is already decoded | ||||||
|  |         if (!url.Contains('%') && !url.Contains('+')) | ||||||
|  |         {    | ||||||
|  |             return url; | ||||||
|  |         } | ||||||
|  |  | ||||||
|  |         try | ||||||
|  |         { | ||||||
|  |             return System.Net.WebUtility.UrlDecode(url); | ||||||
|  |         } | ||||||
|  |         catch | ||||||
|  |         { | ||||||
|  |             // If decoding fails, return the original string | ||||||
|  |             return url; | ||||||
|  |         } | ||||||
|  |     } | ||||||
|  | } | ||||||
| @@ -0,0 +1,17 @@ | |||||||
|  | using System; | ||||||
|  |  | ||||||
|  | namespace DysonNetwork.Sphere.Connection.WebReader; | ||||||
|  |  | ||||||
|  | /// <summary> | ||||||
|  | /// Exception thrown when an error occurs during web reading operations | ||||||
|  | /// </summary> | ||||||
|  | public class WebReaderException : Exception | ||||||
|  | { | ||||||
|  |     public WebReaderException(string message) : base(message) | ||||||
|  |     { | ||||||
|  |     } | ||||||
|  |  | ||||||
|  |     public WebReaderException(string message, Exception innerException) : base(message, innerException) | ||||||
|  |     { | ||||||
|  |     } | ||||||
|  | } | ||||||
							
								
								
									
										334
									
								
								DysonNetwork.Sphere/Connection/WebReader/WebReaderService.cs
									
									
									
									
									
										Normal file
									
								
							
							
						
						
									
										334
									
								
								DysonNetwork.Sphere/Connection/WebReader/WebReaderService.cs
									
									
									
									
									
										Normal file
									
								
							| @@ -0,0 +1,334 @@ | |||||||
|  | using System.Globalization; | ||||||
|  | using AngleSharp; | ||||||
|  | using AngleSharp.Dom; | ||||||
|  | using DysonNetwork.Sphere.Storage; | ||||||
|  |  | ||||||
|  | namespace DysonNetwork.Sphere.Connection.WebReader; | ||||||
|  |  | ||||||
|  | /// <summary> | ||||||
|  | /// The service is amin to providing scrapping service to the Solar Network. | ||||||
|  | /// Such as news feed, external articles and link preview. | ||||||
|  | /// </summary> | ||||||
|  | public class WebReaderService( | ||||||
|  |     IHttpClientFactory httpClientFactory, | ||||||
|  |     ILogger<WebReaderService> logger, | ||||||
|  |     ICacheService cache) | ||||||
|  | { | ||||||
|  |     private const string LinkPreviewCachePrefix = "scrap:preview:"; | ||||||
|  |     private const string LinkPreviewCacheGroup = "scrap:preview"; | ||||||
|  |  | ||||||
|  |     /// <summary> | ||||||
|  |     /// Generate a link preview embed from a URL | ||||||
|  |     /// </summary> | ||||||
|  |     /// <param name="url">The URL to generate the preview for</param> | ||||||
|  |     /// <param name="cancellationToken">Cancellation token</param> | ||||||
|  |     /// <param name="bypassCache">If true, bypass cache and fetch fresh data</param> | ||||||
|  |     /// <param name="cacheExpiry">Custom cache expiration time</param> | ||||||
|  |     /// <returns>A LinkEmbed object containing the preview data</returns> | ||||||
|  |     public async Task<LinkEmbed> GetLinkPreviewAsync( | ||||||
|  |         string url, | ||||||
|  |         CancellationToken cancellationToken = default, | ||||||
|  |         TimeSpan? cacheExpiry = null, | ||||||
|  |         bool bypassCache = false | ||||||
|  |     ) | ||||||
|  |     { | ||||||
|  |         // Ensure URL is valid | ||||||
|  |         if (!Uri.TryCreate(url, UriKind.Absolute, out var uri)) | ||||||
|  |         { | ||||||
|  |             throw new ArgumentException(@"Invalid URL format", nameof(url)); | ||||||
|  |         } | ||||||
|  |  | ||||||
|  |         // Try to get from cache if not bypassing | ||||||
|  |         if (!bypassCache) | ||||||
|  |         { | ||||||
|  |             var cachedPreview = await GetCachedLinkPreview(url); | ||||||
|  |             if (cachedPreview is not null) | ||||||
|  |                 return cachedPreview; | ||||||
|  |         } | ||||||
|  |  | ||||||
|  |         // Cache miss or bypass, fetch fresh data | ||||||
|  |         logger.LogDebug("Fetching fresh link preview for URL: {Url}", url); | ||||||
|  |         var httpClient = httpClientFactory.CreateClient("WebReader"); | ||||||
|  |         httpClient.MaxResponseContentBufferSize = 10 * 1024 * 1024; // 10MB, prevent scrap some directly accessible files | ||||||
|  |         httpClient.Timeout = TimeSpan.FromSeconds(3); | ||||||
|  |         httpClient.DefaultRequestHeaders.Add("User-Agent", "DysonNetwork/1.0 LinkPreview Bot"); | ||||||
|  |  | ||||||
|  |         try | ||||||
|  |         { | ||||||
|  |             var response = await httpClient.GetAsync(url, cancellationToken); | ||||||
|  |             response.EnsureSuccessStatusCode(); | ||||||
|  |  | ||||||
|  |             var contentType = response.Content.Headers.ContentType?.MediaType; | ||||||
|  |             if (contentType == null || !contentType.StartsWith("text/html")) | ||||||
|  |             { | ||||||
|  |                 logger.LogWarning("URL is not an HTML page: {Url}, ContentType: {ContentType}", url, contentType); | ||||||
|  |                 var nonHtmlEmbed = new LinkEmbed | ||||||
|  |                 { | ||||||
|  |                     Url = url, | ||||||
|  |                     Title = uri.Host, | ||||||
|  |                     ContentType = contentType | ||||||
|  |                 }; | ||||||
|  |  | ||||||
|  |                 // Cache non-HTML responses too | ||||||
|  |                 await CacheLinkPreview(nonHtmlEmbed, url, cacheExpiry); | ||||||
|  |                 return nonHtmlEmbed; | ||||||
|  |             } | ||||||
|  |  | ||||||
|  |             var html = await response.Content.ReadAsStringAsync(cancellationToken); | ||||||
|  |             var linkEmbed = await ExtractLinkData(url, html, uri); | ||||||
|  |  | ||||||
|  |             // Cache the result | ||||||
|  |             await CacheLinkPreview(linkEmbed, url, cacheExpiry); | ||||||
|  |  | ||||||
|  |             return linkEmbed; | ||||||
|  |         } | ||||||
|  |         catch (HttpRequestException ex) | ||||||
|  |         { | ||||||
|  |             logger.LogError(ex, "Failed to fetch URL: {Url}", url); | ||||||
|  |             throw new WebReaderException($"Failed to fetch URL: {url}", ex); | ||||||
|  |         } | ||||||
|  |     } | ||||||
|  |  | ||||||
|  |     private async Task<LinkEmbed> ExtractLinkData(string url, string html, Uri uri) | ||||||
|  |     { | ||||||
|  |         var embed = new LinkEmbed | ||||||
|  |         { | ||||||
|  |             Url = url | ||||||
|  |         }; | ||||||
|  |  | ||||||
|  |         // Configure AngleSharp context | ||||||
|  |         var config = Configuration.Default; | ||||||
|  |         var context = BrowsingContext.New(config); | ||||||
|  |         var document = await context.OpenAsync(req => req.Content(html)); | ||||||
|  |  | ||||||
|  |         // Extract OpenGraph tags | ||||||
|  |         var ogTitle = GetMetaTagContent(document, "og:title"); | ||||||
|  |         var ogDescription = GetMetaTagContent(document, "og:description"); | ||||||
|  |         var ogImage = GetMetaTagContent(document, "og:image"); | ||||||
|  |         var ogSiteName = GetMetaTagContent(document, "og:site_name"); | ||||||
|  |         var ogType = GetMetaTagContent(document, "og:type"); | ||||||
|  |  | ||||||
|  |         // Extract Twitter card tags as fallback | ||||||
|  |         var twitterTitle = GetMetaTagContent(document, "twitter:title"); | ||||||
|  |         var twitterDescription = GetMetaTagContent(document, "twitter:description"); | ||||||
|  |         var twitterImage = GetMetaTagContent(document, "twitter:image"); | ||||||
|  |  | ||||||
|  |         // Extract standard meta tags as final fallback | ||||||
|  |         var metaTitle = GetMetaTagContent(document, "title") ?? | ||||||
|  |                         GetMetaContent(document, "title"); | ||||||
|  |         var metaDescription = GetMetaTagContent(document, "description"); | ||||||
|  |  | ||||||
|  |         // Extract page title | ||||||
|  |         var pageTitle = document.Title?.Trim(); | ||||||
|  |  | ||||||
|  |         // Extract publish date | ||||||
|  |         var publishedTime = GetMetaTagContent(document, "article:published_time") ?? | ||||||
|  |                             GetMetaTagContent(document, "datePublished") ?? | ||||||
|  |                             GetMetaTagContent(document, "pubdate"); | ||||||
|  |  | ||||||
|  |         // Extract author | ||||||
|  |         var author = GetMetaTagContent(document, "author") ?? | ||||||
|  |                      GetMetaTagContent(document, "article:author"); | ||||||
|  |  | ||||||
|  |         // Extract favicon | ||||||
|  |         var faviconUrl = GetFaviconUrl(document, uri); | ||||||
|  |  | ||||||
|  |         // Populate the embed with the data, prioritizing OpenGraph | ||||||
|  |         embed.Title = ogTitle ?? twitterTitle ?? metaTitle ?? pageTitle ?? uri.Host; | ||||||
|  |         embed.Description = ogDescription ?? twitterDescription ?? metaDescription; | ||||||
|  |         embed.ImageUrl = ResolveRelativeUrl(ogImage ?? twitterImage, uri); | ||||||
|  |         embed.SiteName = ogSiteName ?? uri.Host; | ||||||
|  |         embed.ContentType = ogType; | ||||||
|  |         embed.FaviconUrl = faviconUrl; | ||||||
|  |         embed.Author = author; | ||||||
|  |  | ||||||
|  |         // Parse and set published date | ||||||
|  |         if (!string.IsNullOrEmpty(publishedTime) && | ||||||
|  |             DateTime.TryParse(publishedTime, CultureInfo.InvariantCulture, DateTimeStyles.AdjustToUniversal, | ||||||
|  |                 out DateTime parsedDate)) | ||||||
|  |         { | ||||||
|  |             embed.PublishedDate = parsedDate; | ||||||
|  |         } | ||||||
|  |  | ||||||
|  |         return embed; | ||||||
|  |     } | ||||||
|  |  | ||||||
|  |     private static string? GetMetaTagContent(IDocument doc, string property) | ||||||
|  |     { | ||||||
|  |         // Check for OpenGraph/Twitter style meta tags | ||||||
|  |         var node = doc.QuerySelector($"meta[property='{property}'][content]") | ||||||
|  |                    ?? doc.QuerySelector($"meta[name='{property}'][content]"); | ||||||
|  |  | ||||||
|  |         return node?.GetAttribute("content")?.Trim(); | ||||||
|  |     } | ||||||
|  |  | ||||||
|  |     private static string? GetMetaContent(IDocument doc, string name) | ||||||
|  |     { | ||||||
|  |         var node = doc.QuerySelector($"meta[name='{name}'][content]"); | ||||||
|  |         return node?.GetAttribute("content")?.Trim(); | ||||||
|  |     } | ||||||
|  |  | ||||||
|  |     private static string? GetFaviconUrl(IDocument doc, Uri baseUri) | ||||||
|  |     { | ||||||
|  |         // Look for apple-touch-icon first as it's typically higher quality | ||||||
|  |         var appleIconNode = doc.QuerySelector("link[rel='apple-touch-icon'][href]"); | ||||||
|  |         if (appleIconNode != null) | ||||||
|  |         { | ||||||
|  |             return ResolveRelativeUrl(appleIconNode.GetAttribute("href"), baseUri); | ||||||
|  |         } | ||||||
|  |  | ||||||
|  |         // Then check for standard favicon | ||||||
|  |         var faviconNode = doc.QuerySelector("link[rel='icon'][href]") ?? | ||||||
|  |                           doc.QuerySelector("link[rel='shortcut icon'][href]"); | ||||||
|  |  | ||||||
|  |         return faviconNode != null | ||||||
|  |             ? ResolveRelativeUrl(faviconNode.GetAttribute("href"), baseUri) | ||||||
|  |             : new Uri(baseUri, "/favicon.ico").ToString(); | ||||||
|  |     } | ||||||
|  |  | ||||||
|  |     private static string? ResolveRelativeUrl(string? url, Uri baseUri) | ||||||
|  |     { | ||||||
|  |         if (string.IsNullOrEmpty(url)) | ||||||
|  |         { | ||||||
|  |             return null; | ||||||
|  |         } | ||||||
|  |  | ||||||
|  |         if (Uri.TryCreate(url, UriKind.Absolute, out _)) | ||||||
|  |         { | ||||||
|  |             return url; // Already absolute | ||||||
|  |         } | ||||||
|  |  | ||||||
|  |         return Uri.TryCreate(baseUri, url, out var absoluteUri) ? absoluteUri.ToString() : null; | ||||||
|  |     } | ||||||
|  |  | ||||||
|  |     /// <summary> | ||||||
|  |     /// Generate a hash-based cache key for a URL | ||||||
|  |     /// </summary> | ||||||
|  |     private string GenerateUrlCacheKey(string url) | ||||||
|  |     { | ||||||
|  |         // Normalize the URL first | ||||||
|  |         var normalizedUrl = NormalizeUrl(url); | ||||||
|  |  | ||||||
|  |         // Create SHA256 hash of the normalized URL | ||||||
|  |         using var sha256 = System.Security.Cryptography.SHA256.Create(); | ||||||
|  |         var urlBytes = System.Text.Encoding.UTF8.GetBytes(normalizedUrl); | ||||||
|  |         var hashBytes = sha256.ComputeHash(urlBytes); | ||||||
|  |  | ||||||
|  |         // Convert to hex string | ||||||
|  |         var hashString = BitConverter.ToString(hashBytes).Replace("-", "").ToLowerInvariant(); | ||||||
|  |  | ||||||
|  |         // Return prefixed key | ||||||
|  |         return $"{LinkPreviewCachePrefix}{hashString}"; | ||||||
|  |     } | ||||||
|  |  | ||||||
|  |     /// <summary> | ||||||
|  |     /// Normalize URL by trimming trailing slashes but preserving query parameters | ||||||
|  |     /// </summary> | ||||||
|  |     private string NormalizeUrl(string url) | ||||||
|  |     { | ||||||
|  |         if (string.IsNullOrEmpty(url)) | ||||||
|  |             return string.Empty; | ||||||
|  |  | ||||||
|  |         // First ensure we have a valid URI | ||||||
|  |         if (!Uri.TryCreate(url, UriKind.Absolute, out var uri)) | ||||||
|  |             return url.TrimEnd('/'); | ||||||
|  |  | ||||||
|  |         // Rebuild the URL without trailing slashes but with query parameters | ||||||
|  |         var scheme = uri.Scheme; | ||||||
|  |         var host = uri.Host; | ||||||
|  |         var port = uri.IsDefaultPort ? string.Empty : $":{uri.Port}"; | ||||||
|  |         var path = uri.AbsolutePath.TrimEnd('/'); | ||||||
|  |         var query = uri.Query; | ||||||
|  |  | ||||||
|  |         return $"{scheme}://{host}{port}{path}{query}".ToLowerInvariant(); | ||||||
|  |     } | ||||||
|  |  | ||||||
|  |     /// <summary> | ||||||
|  |     /// Cache a link preview | ||||||
|  |     /// </summary> | ||||||
|  |     private async Task CacheLinkPreview(LinkEmbed? linkEmbed, string url, TimeSpan? expiry = null) | ||||||
|  |     { | ||||||
|  |         if (linkEmbed == null || string.IsNullOrEmpty(url)) | ||||||
|  |             return; | ||||||
|  |  | ||||||
|  |         try | ||||||
|  |         { | ||||||
|  |             var cacheKey = GenerateUrlCacheKey(url); | ||||||
|  |             var expiryTime = expiry ?? TimeSpan.FromHours(24); | ||||||
|  |  | ||||||
|  |             await cache.SetWithGroupsAsync( | ||||||
|  |                 cacheKey, | ||||||
|  |                 linkEmbed, | ||||||
|  |                 [LinkPreviewCacheGroup], | ||||||
|  |                 expiryTime); | ||||||
|  |  | ||||||
|  |             logger.LogDebug("Cached link preview for URL: {Url} with key: {CacheKey}", url, cacheKey); | ||||||
|  |         } | ||||||
|  |         catch (Exception ex) | ||||||
|  |         { | ||||||
|  |             // Log but don't throw - caching failures shouldn't break the main functionality | ||||||
|  |             logger.LogWarning(ex, "Failed to cache link preview for URL: {Url}", url); | ||||||
|  |         } | ||||||
|  |     } | ||||||
|  |  | ||||||
|  |     /// <summary> | ||||||
|  |     /// Try to get a cached link preview | ||||||
|  |     /// </summary> | ||||||
|  |     private async Task<LinkEmbed?> GetCachedLinkPreview(string url) | ||||||
|  |     { | ||||||
|  |         if (string.IsNullOrEmpty(url)) | ||||||
|  |             return null; | ||||||
|  |  | ||||||
|  |         try | ||||||
|  |         { | ||||||
|  |             var cacheKey = GenerateUrlCacheKey(url); | ||||||
|  |             var cachedPreview = await cache.GetAsync<LinkEmbed>(cacheKey); | ||||||
|  |  | ||||||
|  |             if (cachedPreview is not null) | ||||||
|  |                 logger.LogDebug("Retrieved cached link preview for URL: {Url}", url); | ||||||
|  |  | ||||||
|  |             return cachedPreview; | ||||||
|  |         } | ||||||
|  |         catch (Exception ex) | ||||||
|  |         { | ||||||
|  |             logger.LogWarning(ex, "Failed to retrieve cached link preview for URL: {Url}", url); | ||||||
|  |             return null; | ||||||
|  |         } | ||||||
|  |     } | ||||||
|  |  | ||||||
|  |     /// <summary> | ||||||
|  |     /// Invalidate cache for a specific URL | ||||||
|  |     /// </summary> | ||||||
|  |     public async Task InvalidateCacheForUrlAsync(string url) | ||||||
|  |     { | ||||||
|  |         if (string.IsNullOrEmpty(url)) | ||||||
|  |             return; | ||||||
|  |  | ||||||
|  |         try | ||||||
|  |         { | ||||||
|  |             var cacheKey = GenerateUrlCacheKey(url); | ||||||
|  |             await cache.RemoveAsync(cacheKey); | ||||||
|  |             logger.LogDebug("Invalidated cache for URL: {Url} with key: {CacheKey}", url, cacheKey); | ||||||
|  |         } | ||||||
|  |         catch (Exception ex) | ||||||
|  |         { | ||||||
|  |             logger.LogWarning(ex, "Failed to invalidate cache for URL: {Url}", url); | ||||||
|  |         } | ||||||
|  |     } | ||||||
|  |  | ||||||
|  |     /// <summary> | ||||||
|  |     /// Invalidate all cached link previews | ||||||
|  |     /// </summary> | ||||||
|  |     public async Task InvalidateAllCachedPreviewsAsync() | ||||||
|  |     { | ||||||
|  |         try | ||||||
|  |         { | ||||||
|  |             await cache.RemoveGroupAsync(LinkPreviewCacheGroup); | ||||||
|  |             logger.LogInformation("Invalidated all cached link previews"); | ||||||
|  |         } | ||||||
|  |         catch (Exception ex) | ||||||
|  |         { | ||||||
|  |             logger.LogWarning(ex, "Failed to invalidate all cached link previews"); | ||||||
|  |         } | ||||||
|  |     } | ||||||
|  | } | ||||||
| @@ -16,6 +16,7 @@ | |||||||
|     </PropertyGroup> |     </PropertyGroup> | ||||||
|  |  | ||||||
|     <ItemGroup> |     <ItemGroup> | ||||||
|  |         <PackageReference Include="AngleSharp" Version="1.3.0" /> | ||||||
|         <PackageReference Include="BCrypt.Net-Next" Version="4.0.3" /> |         <PackageReference Include="BCrypt.Net-Next" Version="4.0.3" /> | ||||||
|         <PackageReference Include="BlurHashSharp.SkiaSharp" Version="1.3.4" /> |         <PackageReference Include="BlurHashSharp.SkiaSharp" Version="1.3.4" /> | ||||||
|         <PackageReference Include="EFCore.BulkExtensions" Version="9.0.1" /> |         <PackageReference Include="EFCore.BulkExtensions" Version="9.0.1" /> | ||||||
|   | |||||||
| @@ -18,14 +18,13 @@ using DysonNetwork.Sphere.Storage; | |||||||
| using DysonNetwork.Sphere.Storage.Handlers; | using DysonNetwork.Sphere.Storage.Handlers; | ||||||
| using DysonNetwork.Sphere.Wallet; | using DysonNetwork.Sphere.Wallet; | ||||||
| using Microsoft.AspNetCore.RateLimiting; | using Microsoft.AspNetCore.RateLimiting; | ||||||
| using Microsoft.EntityFrameworkCore; |  | ||||||
| using Microsoft.OpenApi.Models; | using Microsoft.OpenApi.Models; | ||||||
| using NodaTime; | using NodaTime; | ||||||
| using NodaTime.Serialization.SystemTextJson; | using NodaTime.Serialization.SystemTextJson; | ||||||
| using Quartz; |  | ||||||
| using StackExchange.Redis; | using StackExchange.Redis; | ||||||
| using System.Text.Json; | using System.Text.Json; | ||||||
| using System.Threading.RateLimiting; | using System.Threading.RateLimiting; | ||||||
|  | using DysonNetwork.Sphere.Connection.WebReader; | ||||||
| using tusdotnet.Stores; | using tusdotnet.Stores; | ||||||
|  |  | ||||||
| namespace DysonNetwork.Sphere.Startup; | namespace DysonNetwork.Sphere.Startup; | ||||||
| @@ -221,6 +220,7 @@ public static class ServiceCollectionExtensions | |||||||
|         services.AddScoped<WalletService>(); |         services.AddScoped<WalletService>(); | ||||||
|         services.AddScoped<PaymentService>(); |         services.AddScoped<PaymentService>(); | ||||||
|         services.AddScoped<IRealtimeService, LivekitRealtimeService>(); |         services.AddScoped<IRealtimeService, LivekitRealtimeService>(); | ||||||
|  |         services.AddScoped<WebReaderService>(); | ||||||
|  |  | ||||||
|         return services; |         return services; | ||||||
|     } |     } | ||||||
|   | |||||||
| @@ -324,7 +324,6 @@ public class CacheServiceRedis : ICacheService | |||||||
|     public async Task<bool> SetWithGroupsAsync<T>(string key, T value, IEnumerable<string>? groups = null, |     public async Task<bool> SetWithGroupsAsync<T>(string key, T value, IEnumerable<string>? groups = null, | ||||||
|         TimeSpan? expiry = null) |         TimeSpan? expiry = null) | ||||||
|     { |     { | ||||||
|         key = $"{GlobalKeyPrefix}{key}"; |  | ||||||
|         // First, set the value in the cache |         // First, set the value in the cache | ||||||
|         var setResult = await SetAsync(key, value, expiry); |         var setResult = await SetAsync(key, value, expiry); | ||||||
|  |  | ||||||
|   | |||||||
| @@ -74,6 +74,7 @@ | |||||||
| 	<s:String x:Key="/Default/CodeInspection/ExcludedFiles/FilesAndFoldersToSkip2/=7020124F_002D9FFC_002D4AC3_002D8F3D_002DAAB8E0240759_002Ff_003AServiceProvider_002Ecs_002Fl_003A_002E_002E_003F_002E_002E_003F_002E_002E_003FLibrary_003FApplication_0020Support_003FJetBrains_003FRider2025_002E1_003Fresharper_002Dhost_003FSourcesCache_003Fce37be1a06b16c6faa02038d2cc477dd3bca5b217ceeb41c5f2ad45c1bf9_003FServiceProvider_002Ecs/@EntryIndexedValue">ForceIncluded</s:String> | 	<s:String x:Key="/Default/CodeInspection/ExcludedFiles/FilesAndFoldersToSkip2/=7020124F_002D9FFC_002D4AC3_002D8F3D_002DAAB8E0240759_002Ff_003AServiceProvider_002Ecs_002Fl_003A_002E_002E_003F_002E_002E_003F_002E_002E_003FLibrary_003FApplication_0020Support_003FJetBrains_003FRider2025_002E1_003Fresharper_002Dhost_003FSourcesCache_003Fce37be1a06b16c6faa02038d2cc477dd3bca5b217ceeb41c5f2ad45c1bf9_003FServiceProvider_002Ecs/@EntryIndexedValue">ForceIncluded</s:String> | ||||||
| 	<s:String x:Key="/Default/CodeInspection/ExcludedFiles/FilesAndFoldersToSkip2/=7020124F_002D9FFC_002D4AC3_002D8F3D_002DAAB8E0240759_002Ff_003ASetPropertyCalls_002Ecs_002Fl_003A_002E_002E_003F_002E_002E_003F_002E_002E_003FLibrary_003FApplication_0020Support_003FJetBrains_003FRider2024_002E3_003Fresharper_002Dhost_003FSourcesCache_003F458b5f22476b4599b87176214d5e4026c2327b148f4d3f885ee92362b4dac3_003FSetPropertyCalls_002Ecs/@EntryIndexedValue">ForceIncluded</s:String> | 	<s:String x:Key="/Default/CodeInspection/ExcludedFiles/FilesAndFoldersToSkip2/=7020124F_002D9FFC_002D4AC3_002D8F3D_002DAAB8E0240759_002Ff_003ASetPropertyCalls_002Ecs_002Fl_003A_002E_002E_003F_002E_002E_003F_002E_002E_003FLibrary_003FApplication_0020Support_003FJetBrains_003FRider2024_002E3_003Fresharper_002Dhost_003FSourcesCache_003F458b5f22476b4599b87176214d5e4026c2327b148f4d3f885ee92362b4dac3_003FSetPropertyCalls_002Ecs/@EntryIndexedValue">ForceIncluded</s:String> | ||||||
| 	<s:String x:Key="/Default/CodeInspection/ExcludedFiles/FilesAndFoldersToSkip2/=7020124F_002D9FFC_002D4AC3_002D8F3D_002DAAB8E0240759_002Ff_003ASourceCustom_002Ecs_002Fl_003A_002E_002E_003F_002E_002E_003F_002E_002E_003FLibrary_003FApplication_0020Support_003FJetBrains_003FRider2025_002E1_003Fresharper_002Dhost_003FDecompilerCache_003Fdecompiler_003Fdaa8d9c408cd4b4286bbef7e35f1a42e31c00_003F45_003F5839ca6c_003FSourceCustom_002Ecs/@EntryIndexedValue">ForceIncluded</s:String> | 	<s:String x:Key="/Default/CodeInspection/ExcludedFiles/FilesAndFoldersToSkip2/=7020124F_002D9FFC_002D4AC3_002D8F3D_002DAAB8E0240759_002Ff_003ASourceCustom_002Ecs_002Fl_003A_002E_002E_003F_002E_002E_003F_002E_002E_003FLibrary_003FApplication_0020Support_003FJetBrains_003FRider2025_002E1_003Fresharper_002Dhost_003FDecompilerCache_003Fdecompiler_003Fdaa8d9c408cd4b4286bbef7e35f1a42e31c00_003F45_003F5839ca6c_003FSourceCustom_002Ecs/@EntryIndexedValue">ForceIncluded</s:String> | ||||||
|  | 	<s:String x:Key="/Default/CodeInspection/ExcludedFiles/FilesAndFoldersToSkip2/=7020124F_002D9FFC_002D4AC3_002D8F3D_002DAAB8E0240759_002Ff_003AStackFrameIterator_002Ecs_002Fl_003A_002E_002E_003F_002E_002E_003F_002E_002E_003FLibrary_003FApplication_0020Support_003FJetBrains_003FRider2025_002E1_003Fresharper_002Dhost_003FDecompilerCache_003Fdecompiler_003F3bef61b8a21d4c8e96872ecdd7782fa0e55000_003F7a_003F870020d0_003FStackFrameIterator_002Ecs/@EntryIndexedValue">ForceIncluded</s:String> | ||||||
| 	<s:String x:Key="/Default/CodeInspection/ExcludedFiles/FilesAndFoldersToSkip2/=7020124F_002D9FFC_002D4AC3_002D8F3D_002DAAB8E0240759_002Ff_003AStackFrameIterator_002Ecs_002Fl_003A_002E_002E_003F_002E_002E_003F_002E_002E_003FLibrary_003FApplication_0020Support_003FJetBrains_003FRider2025_002E1_003Fresharper_002Dhost_003FDecompilerCache_003Fdecompiler_003Fb6f0571a6bc744b0b551fd4578292582e54c00_003Fdf_003F3fcdc4d2_003FStackFrameIterator_002Ecs/@EntryIndexedValue">ForceIncluded</s:String> | 	<s:String x:Key="/Default/CodeInspection/ExcludedFiles/FilesAndFoldersToSkip2/=7020124F_002D9FFC_002D4AC3_002D8F3D_002DAAB8E0240759_002Ff_003AStackFrameIterator_002Ecs_002Fl_003A_002E_002E_003F_002E_002E_003F_002E_002E_003FLibrary_003FApplication_0020Support_003FJetBrains_003FRider2025_002E1_003Fresharper_002Dhost_003FDecompilerCache_003Fdecompiler_003Fb6f0571a6bc744b0b551fd4578292582e54c00_003Fdf_003F3fcdc4d2_003FStackFrameIterator_002Ecs/@EntryIndexedValue">ForceIncluded</s:String> | ||||||
| 	<s:String x:Key="/Default/CodeInspection/ExcludedFiles/FilesAndFoldersToSkip2/=7020124F_002D9FFC_002D4AC3_002D8F3D_002DAAB8E0240759_002Ff_003AStatusCodeResult_002Ecs_002Fl_003A_002E_002E_003F_002E_002E_003F_002E_002E_003FLibrary_003FApplication_0020Support_003FJetBrains_003FRider2024_002E3_003Fresharper_002Dhost_003FDecompilerCache_003Fdecompiler_003F0b5acdd962e549369896cece0026e556214600_003F7c_003F8b7572ae_003FStatusCodeResult_002Ecs/@EntryIndexedValue">ForceIncluded</s:String> | 	<s:String x:Key="/Default/CodeInspection/ExcludedFiles/FilesAndFoldersToSkip2/=7020124F_002D9FFC_002D4AC3_002D8F3D_002DAAB8E0240759_002Ff_003AStatusCodeResult_002Ecs_002Fl_003A_002E_002E_003F_002E_002E_003F_002E_002E_003FLibrary_003FApplication_0020Support_003FJetBrains_003FRider2024_002E3_003Fresharper_002Dhost_003FDecompilerCache_003Fdecompiler_003F0b5acdd962e549369896cece0026e556214600_003F7c_003F8b7572ae_003FStatusCodeResult_002Ecs/@EntryIndexedValue">ForceIncluded</s:String> | ||||||
| 	<s:String x:Key="/Default/CodeInspection/ExcludedFiles/FilesAndFoldersToSkip2/=7020124F_002D9FFC_002D4AC3_002D8F3D_002DAAB8E0240759_002Ff_003ATagging_002Ecs_002Fl_003A_002E_002E_003F_002E_002E_003F_002E_002E_003FLibrary_003FApplication_0020Support_003FJetBrains_003FRider2024_002E3_003Fresharper_002Dhost_003FSourcesCache_003F36f4c2e6baa65ba603de42eedad12ea36845aa35a910a6a82d82baf688e3e1_003FTagging_002Ecs/@EntryIndexedValue">ForceIncluded</s:String> | 	<s:String x:Key="/Default/CodeInspection/ExcludedFiles/FilesAndFoldersToSkip2/=7020124F_002D9FFC_002D4AC3_002D8F3D_002DAAB8E0240759_002Ff_003ATagging_002Ecs_002Fl_003A_002E_002E_003F_002E_002E_003F_002E_002E_003FLibrary_003FApplication_0020Support_003FJetBrains_003FRider2024_002E3_003Fresharper_002Dhost_003FSourcesCache_003F36f4c2e6baa65ba603de42eedad12ea36845aa35a910a6a82d82baf688e3e1_003FTagging_002Ecs/@EntryIndexedValue">ForceIncluded</s:String> | ||||||
|   | |||||||
		Reference in New Issue
	
	Block a user