Compare commits
	
		
			2 Commits
		
	
	
		
			3824fba8e5
			...
			5f30b56ef8
		
	
	| Author | SHA1 | Date | |
|---|---|---|---|
| 5f30b56ef8 | |||
| 95010e4188 | 
							
								
								
									
										26
									
								
								DysonNetwork.Sphere/Connection/WebReader/IEmbeddable.cs
									
									
									
									
									
										Normal file
									
								
							
							
						
						
									
										26
									
								
								DysonNetwork.Sphere/Connection/WebReader/IEmbeddable.cs
									
									
									
									
									
										Normal file
									
								
							| @@ -0,0 +1,26 @@ | ||||
| namespace DysonNetwork.Sphere.Connection.WebReader; | ||||
|  | ||||
| /// <summary> | ||||
| /// The embeddable can be used in the post or messages' meta's embeds fields | ||||
| /// To render richer type of content. | ||||
| /// | ||||
| /// A simple example of using link preview embed: | ||||
| /// <code> | ||||
| /// { | ||||
| ///     // ... post content | ||||
| ///     "meta": { | ||||
| ///         "embeds": [ | ||||
| ///             { | ||||
| ///                 "type": "link", | ||||
| ///                 "title: "...", | ||||
| ///                 /// ... | ||||
| ///             } | ||||
| ///         ] | ||||
| ///     } | ||||
| /// } | ||||
| /// </code> | ||||
| /// </summary> | ||||
| public interface IEmbeddable | ||||
| { | ||||
|     public string Type { get; } | ||||
| } | ||||
							
								
								
									
										55
									
								
								DysonNetwork.Sphere/Connection/WebReader/LinkEmbed.cs
									
									
									
									
									
										Normal file
									
								
							
							
						
						
									
										55
									
								
								DysonNetwork.Sphere/Connection/WebReader/LinkEmbed.cs
									
									
									
									
									
										Normal file
									
								
							| @@ -0,0 +1,55 @@ | ||||
| namespace DysonNetwork.Sphere.Connection.WebReader; | ||||
|  | ||||
| /// <summary> | ||||
| /// The link embed is a part of the embeddable implementations | ||||
| /// It can be used in the post or messages' meta's embeds fields | ||||
| /// </summary> | ||||
| public class LinkEmbed : IEmbeddable | ||||
| { | ||||
|     public string Type => "link"; | ||||
|  | ||||
|     /// <summary> | ||||
|     /// The original URL that was processed | ||||
|     /// </summary> | ||||
|     public required string Url { get; set; } | ||||
|  | ||||
|     /// <summary> | ||||
|     /// Title of the linked content (from OpenGraph og:title, meta title, or page title) | ||||
|     /// </summary> | ||||
|     public string? Title { get; set; } | ||||
|  | ||||
|     /// <summary> | ||||
|     /// Description of the linked content (from OpenGraph og:description or meta description) | ||||
|     /// </summary> | ||||
|     public string? Description { get; set; } | ||||
|  | ||||
|     /// <summary> | ||||
|     /// URL to the thumbnail image (from OpenGraph og:image or other meta tags) | ||||
|     /// </summary> | ||||
|     public string? ImageUrl { get; set; } | ||||
|  | ||||
|     /// <summary> | ||||
|     /// The favicon URL of the site | ||||
|     /// </summary> | ||||
|     public string? FaviconUrl { get; set; } | ||||
|  | ||||
|     /// <summary> | ||||
|     /// The site name (from OpenGraph og:site_name) | ||||
|     /// </summary> | ||||
|     public string? SiteName { get; set; } | ||||
|  | ||||
|     /// <summary> | ||||
|     /// Type of the content (from OpenGraph og:type) | ||||
|     /// </summary> | ||||
|     public string? ContentType { get; set; } | ||||
|  | ||||
|     /// <summary> | ||||
|     /// Author of the content if available | ||||
|     /// </summary> | ||||
|     public string? Author { get; set; } | ||||
|  | ||||
|     /// <summary> | ||||
|     /// Published date of the content if available | ||||
|     /// </summary> | ||||
|     public DateTime? PublishedDate { get; set; } | ||||
| } | ||||
							
								
								
									
										110
									
								
								DysonNetwork.Sphere/Connection/WebReader/WebReaderController.cs
									
									
									
									
									
										Normal file
									
								
							
							
						
						
									
										110
									
								
								DysonNetwork.Sphere/Connection/WebReader/WebReaderController.cs
									
									
									
									
									
										Normal file
									
								
							| @@ -0,0 +1,110 @@ | ||||
| using DysonNetwork.Sphere.Permission; | ||||
| using Microsoft.AspNetCore.Authorization; | ||||
| using Microsoft.AspNetCore.Mvc; | ||||
| using Microsoft.AspNetCore.RateLimiting; | ||||
|  | ||||
| namespace DysonNetwork.Sphere.Connection.WebReader; | ||||
|  | ||||
| /// <summary> | ||||
| /// Controller for web scraping and link preview services | ||||
| /// </summary> | ||||
| [ApiController] | ||||
| [Route("/scrap")] | ||||
| [EnableRateLimiting("fixed")] | ||||
| public class WebReaderController(WebReaderService reader, ILogger<WebReaderController> logger) | ||||
|     : ControllerBase | ||||
| { | ||||
|     /// <summary> | ||||
|     /// Retrieves a preview for the provided URL | ||||
|     /// </summary> | ||||
|     /// <param name="url">URL-encoded link to generate preview for</param> | ||||
|     /// <returns>Link preview data including title, description, and image</returns> | ||||
|     [HttpGet("link")] | ||||
|     public async Task<ActionResult<LinkEmbed>> ScrapLink([FromQuery] string url) | ||||
|     { | ||||
|         if (string.IsNullOrEmpty(url)) | ||||
|         { | ||||
|             return BadRequest(new { error = "URL parameter is required" }); | ||||
|         } | ||||
|  | ||||
|         try | ||||
|         { | ||||
|             // Ensure URL is properly decoded | ||||
|             var decodedUrl = UrlDecoder.Decode(url); | ||||
|  | ||||
|             // Validate URL format | ||||
|             if (!Uri.TryCreate(decodedUrl, UriKind.Absolute, out _)) | ||||
|             { | ||||
|                 return BadRequest(new { error = "Invalid URL format" }); | ||||
|             } | ||||
|  | ||||
|             var linkEmbed = await reader.GetLinkPreviewAsync(decodedUrl); | ||||
|             return Ok(linkEmbed); | ||||
|         } | ||||
|         catch (WebReaderException ex) | ||||
|         { | ||||
|             logger.LogWarning(ex, "Error scraping link: {Url}", url); | ||||
|             return BadRequest(new { error = ex.Message }); | ||||
|         } | ||||
|         catch (Exception ex) | ||||
|         { | ||||
|             logger.LogError(ex, "Unexpected error scraping link: {Url}", url); | ||||
|             return StatusCode(StatusCodes.Status500InternalServerError,  | ||||
|                 new { error = "An unexpected error occurred while processing the link" }); | ||||
|         } | ||||
|     } | ||||
|  | ||||
|     /// <summary> | ||||
|     /// Force invalidates the cache for a specific URL | ||||
|     /// </summary> | ||||
|     [HttpDelete("link/cache")] | ||||
|     [Authorize] | ||||
|     [RequiredPermission("maintenance", "cache.scrap")] | ||||
|     public async Task<IActionResult> InvalidateCache([FromQuery] string url) | ||||
|     { | ||||
|         if (string.IsNullOrEmpty(url)) | ||||
|         { | ||||
|             return BadRequest(new { error = "URL parameter is required" }); | ||||
|         } | ||||
|  | ||||
|         await reader.InvalidateCacheForUrlAsync(url); | ||||
|         return Ok(new { message = "Cache invalidated for URL" }); | ||||
|     } | ||||
|  | ||||
|     /// <summary> | ||||
|     /// Force invalidates all cached link previews | ||||
|     /// </summary> | ||||
|     [HttpDelete("cache/all")] | ||||
|     [Authorize] | ||||
|     [RequiredPermission("maintenance", "cache.scrap")] | ||||
|     public async Task<IActionResult> InvalidateAllCache() | ||||
|     { | ||||
|         await reader.InvalidateAllCachedPreviewsAsync(); | ||||
|         return Ok(new { message = "All link preview caches invalidated" }); | ||||
|     } | ||||
| } | ||||
|  | ||||
| /// <summary> | ||||
| /// Helper class for URL decoding | ||||
| /// </summary> | ||||
| public static class UrlDecoder | ||||
| { | ||||
|     public static string Decode(string url) | ||||
|     { | ||||
|         // First check if URL is already decoded | ||||
|         if (!url.Contains('%') && !url.Contains('+')) | ||||
|         {    | ||||
|             return url; | ||||
|         } | ||||
|  | ||||
|         try | ||||
|         { | ||||
|             return System.Net.WebUtility.UrlDecode(url); | ||||
|         } | ||||
|         catch | ||||
|         { | ||||
|             // If decoding fails, return the original string | ||||
|             return url; | ||||
|         } | ||||
|     } | ||||
| } | ||||
| @@ -0,0 +1,17 @@ | ||||
| using System; | ||||
|  | ||||
| namespace DysonNetwork.Sphere.Connection.WebReader; | ||||
|  | ||||
| /// <summary> | ||||
| /// Exception thrown when an error occurs during web reading operations | ||||
| /// </summary> | ||||
| public class WebReaderException : Exception | ||||
| { | ||||
|     public WebReaderException(string message) : base(message) | ||||
|     { | ||||
|     } | ||||
|  | ||||
|     public WebReaderException(string message, Exception innerException) : base(message, innerException) | ||||
|     { | ||||
|     } | ||||
| } | ||||
							
								
								
									
										334
									
								
								DysonNetwork.Sphere/Connection/WebReader/WebReaderService.cs
									
									
									
									
									
										Normal file
									
								
							
							
						
						
									
										334
									
								
								DysonNetwork.Sphere/Connection/WebReader/WebReaderService.cs
									
									
									
									
									
										Normal file
									
								
							| @@ -0,0 +1,334 @@ | ||||
| using System.Globalization; | ||||
| using AngleSharp; | ||||
| using AngleSharp.Dom; | ||||
| using DysonNetwork.Sphere.Storage; | ||||
|  | ||||
| namespace DysonNetwork.Sphere.Connection.WebReader; | ||||
|  | ||||
| /// <summary> | ||||
| /// The service is amin to providing scrapping service to the Solar Network. | ||||
| /// Such as news feed, external articles and link preview. | ||||
| /// </summary> | ||||
| public class WebReaderService( | ||||
|     IHttpClientFactory httpClientFactory, | ||||
|     ILogger<WebReaderService> logger, | ||||
|     ICacheService cache) | ||||
| { | ||||
|     private const string LinkPreviewCachePrefix = "scrap:preview:"; | ||||
|     private const string LinkPreviewCacheGroup = "scrap:preview"; | ||||
|  | ||||
|     /// <summary> | ||||
|     /// Generate a link preview embed from a URL | ||||
|     /// </summary> | ||||
|     /// <param name="url">The URL to generate the preview for</param> | ||||
|     /// <param name="cancellationToken">Cancellation token</param> | ||||
|     /// <param name="bypassCache">If true, bypass cache and fetch fresh data</param> | ||||
|     /// <param name="cacheExpiry">Custom cache expiration time</param> | ||||
|     /// <returns>A LinkEmbed object containing the preview data</returns> | ||||
|     public async Task<LinkEmbed> GetLinkPreviewAsync( | ||||
|         string url, | ||||
|         CancellationToken cancellationToken = default, | ||||
|         TimeSpan? cacheExpiry = null, | ||||
|         bool bypassCache = false | ||||
|     ) | ||||
|     { | ||||
|         // Ensure URL is valid | ||||
|         if (!Uri.TryCreate(url, UriKind.Absolute, out var uri)) | ||||
|         { | ||||
|             throw new ArgumentException(@"Invalid URL format", nameof(url)); | ||||
|         } | ||||
|  | ||||
|         // Try to get from cache if not bypassing | ||||
|         if (!bypassCache) | ||||
|         { | ||||
|             var cachedPreview = await GetCachedLinkPreview(url); | ||||
|             if (cachedPreview is not null) | ||||
|                 return cachedPreview; | ||||
|         } | ||||
|  | ||||
|         // Cache miss or bypass, fetch fresh data | ||||
|         logger.LogDebug("Fetching fresh link preview for URL: {Url}", url); | ||||
|         var httpClient = httpClientFactory.CreateClient("WebReader"); | ||||
|         httpClient.MaxResponseContentBufferSize = 10 * 1024 * 1024; // 10MB, prevent scrap some directly accessible files | ||||
|         httpClient.Timeout = TimeSpan.FromSeconds(3); | ||||
|         httpClient.DefaultRequestHeaders.Add("User-Agent", "DysonNetwork/1.0 LinkPreview Bot"); | ||||
|  | ||||
|         try | ||||
|         { | ||||
|             var response = await httpClient.GetAsync(url, cancellationToken); | ||||
|             response.EnsureSuccessStatusCode(); | ||||
|  | ||||
|             var contentType = response.Content.Headers.ContentType?.MediaType; | ||||
|             if (contentType == null || !contentType.StartsWith("text/html")) | ||||
|             { | ||||
|                 logger.LogWarning("URL is not an HTML page: {Url}, ContentType: {ContentType}", url, contentType); | ||||
|                 var nonHtmlEmbed = new LinkEmbed | ||||
|                 { | ||||
|                     Url = url, | ||||
|                     Title = uri.Host, | ||||
|                     ContentType = contentType | ||||
|                 }; | ||||
|  | ||||
|                 // Cache non-HTML responses too | ||||
|                 await CacheLinkPreview(nonHtmlEmbed, url, cacheExpiry); | ||||
|                 return nonHtmlEmbed; | ||||
|             } | ||||
|  | ||||
|             var html = await response.Content.ReadAsStringAsync(cancellationToken); | ||||
|             var linkEmbed = await ExtractLinkData(url, html, uri); | ||||
|  | ||||
|             // Cache the result | ||||
|             await CacheLinkPreview(linkEmbed, url, cacheExpiry); | ||||
|  | ||||
|             return linkEmbed; | ||||
|         } | ||||
|         catch (HttpRequestException ex) | ||||
|         { | ||||
|             logger.LogError(ex, "Failed to fetch URL: {Url}", url); | ||||
|             throw new WebReaderException($"Failed to fetch URL: {url}", ex); | ||||
|         } | ||||
|     } | ||||
|  | ||||
|     private async Task<LinkEmbed> ExtractLinkData(string url, string html, Uri uri) | ||||
|     { | ||||
|         var embed = new LinkEmbed | ||||
|         { | ||||
|             Url = url | ||||
|         }; | ||||
|  | ||||
|         // Configure AngleSharp context | ||||
|         var config = Configuration.Default; | ||||
|         var context = BrowsingContext.New(config); | ||||
|         var document = await context.OpenAsync(req => req.Content(html)); | ||||
|  | ||||
|         // Extract OpenGraph tags | ||||
|         var ogTitle = GetMetaTagContent(document, "og:title"); | ||||
|         var ogDescription = GetMetaTagContent(document, "og:description"); | ||||
|         var ogImage = GetMetaTagContent(document, "og:image"); | ||||
|         var ogSiteName = GetMetaTagContent(document, "og:site_name"); | ||||
|         var ogType = GetMetaTagContent(document, "og:type"); | ||||
|  | ||||
|         // Extract Twitter card tags as fallback | ||||
|         var twitterTitle = GetMetaTagContent(document, "twitter:title"); | ||||
|         var twitterDescription = GetMetaTagContent(document, "twitter:description"); | ||||
|         var twitterImage = GetMetaTagContent(document, "twitter:image"); | ||||
|  | ||||
|         // Extract standard meta tags as final fallback | ||||
|         var metaTitle = GetMetaTagContent(document, "title") ?? | ||||
|                         GetMetaContent(document, "title"); | ||||
|         var metaDescription = GetMetaTagContent(document, "description"); | ||||
|  | ||||
|         // Extract page title | ||||
|         var pageTitle = document.Title?.Trim(); | ||||
|  | ||||
|         // Extract publish date | ||||
|         var publishedTime = GetMetaTagContent(document, "article:published_time") ?? | ||||
|                             GetMetaTagContent(document, "datePublished") ?? | ||||
|                             GetMetaTagContent(document, "pubdate"); | ||||
|  | ||||
|         // Extract author | ||||
|         var author = GetMetaTagContent(document, "author") ?? | ||||
|                      GetMetaTagContent(document, "article:author"); | ||||
|  | ||||
|         // Extract favicon | ||||
|         var faviconUrl = GetFaviconUrl(document, uri); | ||||
|  | ||||
|         // Populate the embed with the data, prioritizing OpenGraph | ||||
|         embed.Title = ogTitle ?? twitterTitle ?? metaTitle ?? pageTitle ?? uri.Host; | ||||
|         embed.Description = ogDescription ?? twitterDescription ?? metaDescription; | ||||
|         embed.ImageUrl = ResolveRelativeUrl(ogImage ?? twitterImage, uri); | ||||
|         embed.SiteName = ogSiteName ?? uri.Host; | ||||
|         embed.ContentType = ogType; | ||||
|         embed.FaviconUrl = faviconUrl; | ||||
|         embed.Author = author; | ||||
|  | ||||
|         // Parse and set published date | ||||
|         if (!string.IsNullOrEmpty(publishedTime) && | ||||
|             DateTime.TryParse(publishedTime, CultureInfo.InvariantCulture, DateTimeStyles.AdjustToUniversal, | ||||
|                 out DateTime parsedDate)) | ||||
|         { | ||||
|             embed.PublishedDate = parsedDate; | ||||
|         } | ||||
|  | ||||
|         return embed; | ||||
|     } | ||||
|  | ||||
|     private static string? GetMetaTagContent(IDocument doc, string property) | ||||
|     { | ||||
|         // Check for OpenGraph/Twitter style meta tags | ||||
|         var node = doc.QuerySelector($"meta[property='{property}'][content]") | ||||
|                    ?? doc.QuerySelector($"meta[name='{property}'][content]"); | ||||
|  | ||||
|         return node?.GetAttribute("content")?.Trim(); | ||||
|     } | ||||
|  | ||||
|     private static string? GetMetaContent(IDocument doc, string name) | ||||
|     { | ||||
|         var node = doc.QuerySelector($"meta[name='{name}'][content]"); | ||||
|         return node?.GetAttribute("content")?.Trim(); | ||||
|     } | ||||
|  | ||||
|     private static string? GetFaviconUrl(IDocument doc, Uri baseUri) | ||||
|     { | ||||
|         // Look for apple-touch-icon first as it's typically higher quality | ||||
|         var appleIconNode = doc.QuerySelector("link[rel='apple-touch-icon'][href]"); | ||||
|         if (appleIconNode != null) | ||||
|         { | ||||
|             return ResolveRelativeUrl(appleIconNode.GetAttribute("href"), baseUri); | ||||
|         } | ||||
|  | ||||
|         // Then check for standard favicon | ||||
|         var faviconNode = doc.QuerySelector("link[rel='icon'][href]") ?? | ||||
|                           doc.QuerySelector("link[rel='shortcut icon'][href]"); | ||||
|  | ||||
|         return faviconNode != null | ||||
|             ? ResolveRelativeUrl(faviconNode.GetAttribute("href"), baseUri) | ||||
|             : new Uri(baseUri, "/favicon.ico").ToString(); | ||||
|     } | ||||
|  | ||||
|     private static string? ResolveRelativeUrl(string? url, Uri baseUri) | ||||
|     { | ||||
|         if (string.IsNullOrEmpty(url)) | ||||
|         { | ||||
|             return null; | ||||
|         } | ||||
|  | ||||
|         if (Uri.TryCreate(url, UriKind.Absolute, out _)) | ||||
|         { | ||||
|             return url; // Already absolute | ||||
|         } | ||||
|  | ||||
|         return Uri.TryCreate(baseUri, url, out var absoluteUri) ? absoluteUri.ToString() : null; | ||||
|     } | ||||
|  | ||||
|     /// <summary> | ||||
|     /// Generate a hash-based cache key for a URL | ||||
|     /// </summary> | ||||
|     private string GenerateUrlCacheKey(string url) | ||||
|     { | ||||
|         // Normalize the URL first | ||||
|         var normalizedUrl = NormalizeUrl(url); | ||||
|  | ||||
|         // Create SHA256 hash of the normalized URL | ||||
|         using var sha256 = System.Security.Cryptography.SHA256.Create(); | ||||
|         var urlBytes = System.Text.Encoding.UTF8.GetBytes(normalizedUrl); | ||||
|         var hashBytes = sha256.ComputeHash(urlBytes); | ||||
|  | ||||
|         // Convert to hex string | ||||
|         var hashString = BitConverter.ToString(hashBytes).Replace("-", "").ToLowerInvariant(); | ||||
|  | ||||
|         // Return prefixed key | ||||
|         return $"{LinkPreviewCachePrefix}{hashString}"; | ||||
|     } | ||||
|  | ||||
|     /// <summary> | ||||
|     /// Normalize URL by trimming trailing slashes but preserving query parameters | ||||
|     /// </summary> | ||||
|     private string NormalizeUrl(string url) | ||||
|     { | ||||
|         if (string.IsNullOrEmpty(url)) | ||||
|             return string.Empty; | ||||
|  | ||||
|         // First ensure we have a valid URI | ||||
|         if (!Uri.TryCreate(url, UriKind.Absolute, out var uri)) | ||||
|             return url.TrimEnd('/'); | ||||
|  | ||||
|         // Rebuild the URL without trailing slashes but with query parameters | ||||
|         var scheme = uri.Scheme; | ||||
|         var host = uri.Host; | ||||
|         var port = uri.IsDefaultPort ? string.Empty : $":{uri.Port}"; | ||||
|         var path = uri.AbsolutePath.TrimEnd('/'); | ||||
|         var query = uri.Query; | ||||
|  | ||||
|         return $"{scheme}://{host}{port}{path}{query}".ToLowerInvariant(); | ||||
|     } | ||||
|  | ||||
|     /// <summary> | ||||
|     /// Cache a link preview | ||||
|     /// </summary> | ||||
|     private async Task CacheLinkPreview(LinkEmbed? linkEmbed, string url, TimeSpan? expiry = null) | ||||
|     { | ||||
|         if (linkEmbed == null || string.IsNullOrEmpty(url)) | ||||
|             return; | ||||
|  | ||||
|         try | ||||
|         { | ||||
|             var cacheKey = GenerateUrlCacheKey(url); | ||||
|             var expiryTime = expiry ?? TimeSpan.FromHours(24); | ||||
|  | ||||
|             await cache.SetWithGroupsAsync( | ||||
|                 cacheKey, | ||||
|                 linkEmbed, | ||||
|                 [LinkPreviewCacheGroup], | ||||
|                 expiryTime); | ||||
|  | ||||
|             logger.LogDebug("Cached link preview for URL: {Url} with key: {CacheKey}", url, cacheKey); | ||||
|         } | ||||
|         catch (Exception ex) | ||||
|         { | ||||
|             // Log but don't throw - caching failures shouldn't break the main functionality | ||||
|             logger.LogWarning(ex, "Failed to cache link preview for URL: {Url}", url); | ||||
|         } | ||||
|     } | ||||
|  | ||||
|     /// <summary> | ||||
|     /// Try to get a cached link preview | ||||
|     /// </summary> | ||||
|     private async Task<LinkEmbed?> GetCachedLinkPreview(string url) | ||||
|     { | ||||
|         if (string.IsNullOrEmpty(url)) | ||||
|             return null; | ||||
|  | ||||
|         try | ||||
|         { | ||||
|             var cacheKey = GenerateUrlCacheKey(url); | ||||
|             var cachedPreview = await cache.GetAsync<LinkEmbed>(cacheKey); | ||||
|  | ||||
|             if (cachedPreview is not null) | ||||
|                 logger.LogDebug("Retrieved cached link preview for URL: {Url}", url); | ||||
|  | ||||
|             return cachedPreview; | ||||
|         } | ||||
|         catch (Exception ex) | ||||
|         { | ||||
|             logger.LogWarning(ex, "Failed to retrieve cached link preview for URL: {Url}", url); | ||||
|             return null; | ||||
|         } | ||||
|     } | ||||
|  | ||||
|     /// <summary> | ||||
|     /// Invalidate cache for a specific URL | ||||
|     /// </summary> | ||||
|     public async Task InvalidateCacheForUrlAsync(string url) | ||||
|     { | ||||
|         if (string.IsNullOrEmpty(url)) | ||||
|             return; | ||||
|  | ||||
|         try | ||||
|         { | ||||
|             var cacheKey = GenerateUrlCacheKey(url); | ||||
|             await cache.RemoveAsync(cacheKey); | ||||
|             logger.LogDebug("Invalidated cache for URL: {Url} with key: {CacheKey}", url, cacheKey); | ||||
|         } | ||||
|         catch (Exception ex) | ||||
|         { | ||||
|             logger.LogWarning(ex, "Failed to invalidate cache for URL: {Url}", url); | ||||
|         } | ||||
|     } | ||||
|  | ||||
|     /// <summary> | ||||
|     /// Invalidate all cached link previews | ||||
|     /// </summary> | ||||
|     public async Task InvalidateAllCachedPreviewsAsync() | ||||
|     { | ||||
|         try | ||||
|         { | ||||
|             await cache.RemoveGroupAsync(LinkPreviewCacheGroup); | ||||
|             logger.LogInformation("Invalidated all cached link previews"); | ||||
|         } | ||||
|         catch (Exception ex) | ||||
|         { | ||||
|             logger.LogWarning(ex, "Failed to invalidate all cached link previews"); | ||||
|         } | ||||
|     } | ||||
| } | ||||
| @@ -16,6 +16,7 @@ | ||||
|     </PropertyGroup> | ||||
|  | ||||
|     <ItemGroup> | ||||
|         <PackageReference Include="AngleSharp" Version="1.3.0" /> | ||||
|         <PackageReference Include="BCrypt.Net-Next" Version="4.0.3" /> | ||||
|         <PackageReference Include="BlurHashSharp.SkiaSharp" Version="1.3.4" /> | ||||
|         <PackageReference Include="EFCore.BulkExtensions" Version="9.0.1" /> | ||||
|   | ||||
| @@ -18,14 +18,13 @@ using DysonNetwork.Sphere.Storage; | ||||
| using DysonNetwork.Sphere.Storage.Handlers; | ||||
| using DysonNetwork.Sphere.Wallet; | ||||
| using Microsoft.AspNetCore.RateLimiting; | ||||
| using Microsoft.EntityFrameworkCore; | ||||
| using Microsoft.OpenApi.Models; | ||||
| using NodaTime; | ||||
| using NodaTime.Serialization.SystemTextJson; | ||||
| using Quartz; | ||||
| using StackExchange.Redis; | ||||
| using System.Text.Json; | ||||
| using System.Threading.RateLimiting; | ||||
| using DysonNetwork.Sphere.Connection.WebReader; | ||||
| using tusdotnet.Stores; | ||||
|  | ||||
| namespace DysonNetwork.Sphere.Startup; | ||||
| @@ -221,6 +220,7 @@ public static class ServiceCollectionExtensions | ||||
|         services.AddScoped<WalletService>(); | ||||
|         services.AddScoped<PaymentService>(); | ||||
|         services.AddScoped<IRealtimeService, LivekitRealtimeService>(); | ||||
|         services.AddScoped<WebReaderService>(); | ||||
|  | ||||
|         return services; | ||||
|     } | ||||
|   | ||||
| @@ -324,7 +324,6 @@ public class CacheServiceRedis : ICacheService | ||||
|     public async Task<bool> SetWithGroupsAsync<T>(string key, T value, IEnumerable<string>? groups = null, | ||||
|         TimeSpan? expiry = null) | ||||
|     { | ||||
|         key = $"{GlobalKeyPrefix}{key}"; | ||||
|         // First, set the value in the cache | ||||
|         var setResult = await SetAsync(key, value, expiry); | ||||
|  | ||||
|   | ||||
| @@ -74,6 +74,7 @@ | ||||
| 	<s:String x:Key="/Default/CodeInspection/ExcludedFiles/FilesAndFoldersToSkip2/=7020124F_002D9FFC_002D4AC3_002D8F3D_002DAAB8E0240759_002Ff_003AServiceProvider_002Ecs_002Fl_003A_002E_002E_003F_002E_002E_003F_002E_002E_003FLibrary_003FApplication_0020Support_003FJetBrains_003FRider2025_002E1_003Fresharper_002Dhost_003FSourcesCache_003Fce37be1a06b16c6faa02038d2cc477dd3bca5b217ceeb41c5f2ad45c1bf9_003FServiceProvider_002Ecs/@EntryIndexedValue">ForceIncluded</s:String> | ||||
| 	<s:String x:Key="/Default/CodeInspection/ExcludedFiles/FilesAndFoldersToSkip2/=7020124F_002D9FFC_002D4AC3_002D8F3D_002DAAB8E0240759_002Ff_003ASetPropertyCalls_002Ecs_002Fl_003A_002E_002E_003F_002E_002E_003F_002E_002E_003FLibrary_003FApplication_0020Support_003FJetBrains_003FRider2024_002E3_003Fresharper_002Dhost_003FSourcesCache_003F458b5f22476b4599b87176214d5e4026c2327b148f4d3f885ee92362b4dac3_003FSetPropertyCalls_002Ecs/@EntryIndexedValue">ForceIncluded</s:String> | ||||
| 	<s:String x:Key="/Default/CodeInspection/ExcludedFiles/FilesAndFoldersToSkip2/=7020124F_002D9FFC_002D4AC3_002D8F3D_002DAAB8E0240759_002Ff_003ASourceCustom_002Ecs_002Fl_003A_002E_002E_003F_002E_002E_003F_002E_002E_003FLibrary_003FApplication_0020Support_003FJetBrains_003FRider2025_002E1_003Fresharper_002Dhost_003FDecompilerCache_003Fdecompiler_003Fdaa8d9c408cd4b4286bbef7e35f1a42e31c00_003F45_003F5839ca6c_003FSourceCustom_002Ecs/@EntryIndexedValue">ForceIncluded</s:String> | ||||
| 	<s:String x:Key="/Default/CodeInspection/ExcludedFiles/FilesAndFoldersToSkip2/=7020124F_002D9FFC_002D4AC3_002D8F3D_002DAAB8E0240759_002Ff_003AStackFrameIterator_002Ecs_002Fl_003A_002E_002E_003F_002E_002E_003F_002E_002E_003FLibrary_003FApplication_0020Support_003FJetBrains_003FRider2025_002E1_003Fresharper_002Dhost_003FDecompilerCache_003Fdecompiler_003F3bef61b8a21d4c8e96872ecdd7782fa0e55000_003F7a_003F870020d0_003FStackFrameIterator_002Ecs/@EntryIndexedValue">ForceIncluded</s:String> | ||||
| 	<s:String x:Key="/Default/CodeInspection/ExcludedFiles/FilesAndFoldersToSkip2/=7020124F_002D9FFC_002D4AC3_002D8F3D_002DAAB8E0240759_002Ff_003AStackFrameIterator_002Ecs_002Fl_003A_002E_002E_003F_002E_002E_003F_002E_002E_003FLibrary_003FApplication_0020Support_003FJetBrains_003FRider2025_002E1_003Fresharper_002Dhost_003FDecompilerCache_003Fdecompiler_003Fb6f0571a6bc744b0b551fd4578292582e54c00_003Fdf_003F3fcdc4d2_003FStackFrameIterator_002Ecs/@EntryIndexedValue">ForceIncluded</s:String> | ||||
| 	<s:String x:Key="/Default/CodeInspection/ExcludedFiles/FilesAndFoldersToSkip2/=7020124F_002D9FFC_002D4AC3_002D8F3D_002DAAB8E0240759_002Ff_003AStatusCodeResult_002Ecs_002Fl_003A_002E_002E_003F_002E_002E_003F_002E_002E_003FLibrary_003FApplication_0020Support_003FJetBrains_003FRider2024_002E3_003Fresharper_002Dhost_003FDecompilerCache_003Fdecompiler_003F0b5acdd962e549369896cece0026e556214600_003F7c_003F8b7572ae_003FStatusCodeResult_002Ecs/@EntryIndexedValue">ForceIncluded</s:String> | ||||
| 	<s:String x:Key="/Default/CodeInspection/ExcludedFiles/FilesAndFoldersToSkip2/=7020124F_002D9FFC_002D4AC3_002D8F3D_002DAAB8E0240759_002Ff_003ATagging_002Ecs_002Fl_003A_002E_002E_003F_002E_002E_003F_002E_002E_003FLibrary_003FApplication_0020Support_003FJetBrains_003FRider2024_002E3_003Fresharper_002Dhost_003FSourcesCache_003F36f4c2e6baa65ba603de42eedad12ea36845aa35a910a6a82d82baf688e3e1_003FTagging_002Ecs/@EntryIndexedValue">ForceIncluded</s:String> | ||||
|   | ||||
		Reference in New Issue
	
	Block a user