DysonNetwork/Swarm
2
1
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

✨ Web articles and feed

Browse Source
This commit is contained in:
LittleSheep 2025-06-26 17:36:45 +08:00
parent 21cf212d8f
commit 1a137fbb6a
12 changed files with 4335 additions and 0 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

12
DysonNetwork.Sphere/AppDatabase.cs
View File

@ -95,6 +95,8 @@ public class AppDatabase(
public DbSet<Subscription> WalletSubscriptions { get; set; }
public DbSet<Coupon> WalletCoupons { get; set; }
public DbSet<Connection.WebReader.WebArticle> WebArticles { get; set; }
public DbSet<Connection.WebReader.WebFeed> WebFeeds { get; set; }
protected override void OnConfiguring(DbContextOptionsBuilder optionsBuilder)
{
@ -137,6 +139,8 @@ public class AppDatabase(
}
});
optionsBuilder.UseSeeding((context, _) => {});
base.OnConfiguring(optionsBuilder);
}
@ -261,6 +265,14 @@ public class AppDatabase(
.HasForeignKey(m => m.SenderId)
.OnDelete(DeleteBehavior.Cascade);
modelBuilder.Entity<Connection.WebReader.WebFeed>()
.HasIndex(f => f.Url)
.IsUnique();
modelBuilder.Entity<Connection.WebReader.WebArticle>()
.HasIndex(a => a.Url)
.IsUnique();
// Automatically apply soft-delete filter to all entities inheriting BaseModel
foreach (var entityType in modelBuilder.Model.GetEntityTypes())
{

39
DysonNetwork.Sphere/Connection/WebReader/WebArticle.cs Normal file
View File

@ -0,0 +1,39 @@
using System.ComponentModel.DataAnnotations;
using System.ComponentModel.DataAnnotations.Schema;
namespace DysonNetwork.Sphere.Connection.WebReader;
public class WebArticle : ModelBase
{
public Guid Id { get; set; } = Guid.NewGuid();
[MaxLength(4096)] public string Title { get; set; }
[MaxLength(8192)] public string Url { get; set; }
[MaxLength(4096)] public string? Author { get; set; }
[Column(TypeName = "jsonb")] public Dictionary<string, object>? Meta { get; set; }
[Column(TypeName = "jsonb")] public LinkEmbed? Preview { get; set; }
// ReSharper disable once EntityFramework.ModelValidation.UnlimitedStringLength
public string? Content { get; set; }
public DateTime? PublishedAt { get; set; }
public Guid FeedId { get; set; }
public WebFeed Feed { get; set; } = null!;
}
public class WebFeed : ModelBase
{
public Guid Id { get; set; } = Guid.NewGuid();
[MaxLength(8192)] public string Url { get; set; }
[MaxLength(4096)] public string Title { get; set; }
[MaxLength(8192)] public string? Description { get; set; }
[Column(TypeName = "jsonb")] public LinkEmbed? Preview { get; set; }
public Guid PublisherId { get; set; }
public Publisher.Publisher Publisher { get; set; } = null!;
public ICollection<WebArticle> Articles { get; set; } = new List<WebArticle>();
}

33
DysonNetwork.Sphere/Connection/WebReader/WebFeedController.cs Normal file
View File

@ -0,0 +1,33 @@
using System.ComponentModel.DataAnnotations;
using Microsoft.AspNetCore.Authorization;
using Microsoft.AspNetCore.Mvc;
namespace DysonNetwork.Sphere.Connection.WebReader;
[Authorize]
[ApiController]
[Route("feeds")]
public class WebFeedController(WebFeedService webFeedService) : ControllerBase
{
public class CreateWebFeedRequest
{
[Required]
[MaxLength(8192)]
public required string Url { get; set; }
[Required]
[MaxLength(4096)]
public required string Title { get; set; }
[MaxLength(8192)]
public string? Description { get; set; }
}
[HttpPost]
public async Task<IActionResult> CreateWebFeed([FromBody] CreateWebFeedRequest request)
{
var feed = await webFeedService.CreateWebFeedAsync(request, User);
return Ok(feed);
}
}

35
DysonNetwork.Sphere/Connection/WebReader/WebFeedScraperJob.cs Normal file
View File

@ -0,0 +1,35 @@
using Microsoft.EntityFrameworkCore;
using Microsoft.Extensions.Logging;
using Quartz;
namespace DysonNetwork.Sphere.Connection.WebReader;
[DisallowConcurrentExecution]
public class WebFeedScraperJob(
AppDatabase database,
WebFeedService webFeedService,
ILogger<WebFeedScraperJob> logger
)
: IJob
{
public async Task Execute(IJobExecutionContext context)
{
logger.LogInformation("Starting web feed scraper job.");
var feeds = await database.Set<WebFeed>().ToListAsync(context.CancellationToken);
foreach (var feed in feeds)
{
try
{
await webFeedService.ScrapeFeedAsync(feed, context.CancellationToken);
}
catch (Exception ex)
{
logger.LogError(ex, "Failed to scrape web feed {FeedId}", feed.Id);
}
}
logger.LogInformation("Web feed scraper job finished.");
}
}

91
DysonNetwork.Sphere/Connection/WebReader/WebFeedService.cs Normal file
View File

@ -0,0 +1,91 @@
using System.Security.Claims;
using System.ServiceModel.Syndication;
using System.Xml;
using DysonNetwork.Sphere.Account;
using Microsoft.EntityFrameworkCore;
using Microsoft.Extensions.Logging;
namespace DysonNetwork.Sphere.Connection.WebReader;
public class WebFeedService(
AppDatabase database,
IHttpClientFactory httpClientFactory,
ILogger<WebFeedService> logger,
AccountService accountService
)
{
public async Task<WebFeed> CreateWebFeedAsync(WebFeedController.CreateWebFeedRequest dto, ClaimsPrincipal claims)
{
if (claims.Identity?.Name == null)
{
throw new UnauthorizedAccessException();
}
var account = await accountService.LookupAccount(claims.Identity.Name);
if (account == null)
{
throw new UnauthorizedAccessException();
}
var feed = new WebFeed
{
Url = dto.Url,
Title = dto.Title,
Description = dto.Description,
PublisherId = account.Id,
};
database.Set<WebFeed>().Add(feed);
await database.SaveChangesAsync();
return feed;
}
public async Task ScrapeFeedAsync(WebFeed feed, CancellationToken cancellationToken = default)
{
var httpClient = httpClientFactory.CreateClient();
var response = await httpClient.GetAsync(feed.Url, cancellationToken);
response.EnsureSuccessStatusCode();
await using var stream = await response.Content.ReadAsStreamAsync(cancellationToken);
using var reader = XmlReader.Create(stream);
var syndicationFeed = SyndicationFeed.Load(reader);
if (syndicationFeed == null)
{
logger.LogWarning("Could not parse syndication feed for {FeedUrl}", feed.Url);
return;
}
foreach (var item in syndicationFeed.Items)
{
var itemUrl = item.Links.FirstOrDefault()?.Uri.ToString();
if (string.IsNullOrEmpty(itemUrl))
{
continue;
}
var articleExists = await database.Set<WebArticle>()
.AnyAsync(a => a.FeedId == feed.Id && a.Url == itemUrl, cancellationToken);
if (articleExists)
{
continue;
}
var newArticle = new WebArticle
{
FeedId = feed.Id,
Title = item.Title.Text,
Url = itemUrl,
Author = item.Authors.FirstOrDefault()?.Name,
Content = (item.Content as TextSyndicationContent)?.Text ?? item.Summary.Text,
PublishedAt = item.PublishDate.UtcDateTime,
};
database.Set<WebArticle>().Add(newArticle);
}
await database.SaveChangesAsync(cancellationToken);
}
}

1
DysonNetwork.Sphere/DysonNetwork.Sphere.csproj
View File

@ -71,6 +71,7 @@
<PackageReference Include="StackExchange.Redis.Extensions.AspNetCore" Version="11.0.0" />
<PackageReference Include="Swashbuckle.AspNetCore" Version="8.1.0" />
<PackageReference Include="Swashbuckle.AspNetCore.SwaggerUI" Version="8.1.0" />
<PackageReference Include="System.ServiceModel.Syndication" Version="9.0.6" />
<PackageReference Include="tusdotnet" Version="2.8.1" />
</ItemGroup>

3852
DysonNetwork.Sphere/Migrations/20250626093051_AddWebArticles.Designer.cs generated Normal file
View File

File diff suppressed because it is too large Load Diff

103
DysonNetwork.Sphere/Migrations/20250626093051_AddWebArticles.cs Normal file
View File

@ -0,0 +1,103 @@
using System;
using System.Collections.Generic;
using DysonNetwork.Sphere.Connection.WebReader;
using Microsoft.EntityFrameworkCore.Migrations;
using NodaTime;
#nullable disable
namespace DysonNetwork.Sphere.Migrations
{
/// <inheritdoc />
public partial class AddWebArticles : Migration
{
/// <inheritdoc />
protected override void Up(MigrationBuilder migrationBuilder)
{
migrationBuilder.CreateTable(
name: "web_feeds",
columns: table => new
{
id = table.Column<Guid>(type: "uuid", nullable: false),
url = table.Column<string>(type: "character varying(8192)", maxLength: 8192, nullable: false),
title = table.Column<string>(type: "character varying(4096)", maxLength: 4096, nullable: false),
description = table.Column<string>(type: "character varying(8192)", maxLength: 8192, nullable: true),
preview = table.Column<LinkEmbed>(type: "jsonb", nullable: true),
publisher_id = table.Column<Guid>(type: "uuid", nullable: false),
created_at = table.Column<Instant>(type: "timestamp with time zone", nullable: false),
updated_at = table.Column<Instant>(type: "timestamp with time zone", nullable: false),
deleted_at = table.Column<Instant>(type: "timestamp with time zone", nullable: true)
},
constraints: table =>
{
table.PrimaryKey("pk_web_feeds", x => x.id);
table.ForeignKey(
name: "fk_web_feeds_publishers_publisher_id",
column: x => x.publisher_id,
principalTable: "publishers",
principalColumn: "id",
onDelete: ReferentialAction.Cascade);
});
migrationBuilder.CreateTable(
name: "web_articles",
columns: table => new
{
id = table.Column<Guid>(type: "uuid", nullable: false),
title = table.Column<string>(type: "character varying(4096)", maxLength: 4096, nullable: false),
url = table.Column<string>(type: "character varying(8192)", maxLength: 8192, nullable: false),
author = table.Column<string>(type: "character varying(4096)", maxLength: 4096, nullable: true),
meta = table.Column<Dictionary<string, object>>(type: "jsonb", nullable: true),
preview = table.Column<LinkEmbed>(type: "jsonb", nullable: true),
content = table.Column<string>(type: "text", nullable: true),
published_at = table.Column<DateTime>(type: "timestamp with time zone", nullable: true),
feed_id = table.Column<Guid>(type: "uuid", nullable: false),
created_at = table.Column<Instant>(type: "timestamp with time zone", nullable: false),
updated_at = table.Column<Instant>(type: "timestamp with time zone", nullable: false),
deleted_at = table.Column<Instant>(type: "timestamp with time zone", nullable: true)
},
constraints: table =>
{
table.PrimaryKey("pk_web_articles", x => x.id);
table.ForeignKey(
name: "fk_web_articles_web_feeds_feed_id",
column: x => x.feed_id,
principalTable: "web_feeds",
principalColumn: "id",
onDelete: ReferentialAction.Cascade);
});
migrationBuilder.CreateIndex(
name: "ix_web_articles_feed_id",
table: "web_articles",
column: "feed_id");
migrationBuilder.CreateIndex(
name: "ix_web_articles_url",
table: "web_articles",
column: "url",
unique: true);
migrationBuilder.CreateIndex(
name: "ix_web_feeds_publisher_id",
table: "web_feeds",
column: "publisher_id");
migrationBuilder.CreateIndex(
name: "ix_web_feeds_url",
table: "web_feeds",
column: "url",
unique: true);
}
/// <inheritdoc />
protected override void Down(MigrationBuilder migrationBuilder)
{
migrationBuilder.DropTable(
name: "web_articles");
migrationBuilder.DropTable(
name: "web_feeds");
}
}
}

156
DysonNetwork.Sphere/Migrations/AppDatabaseModelSnapshot.cs
View File

@ -5,6 +5,7 @@ using System.Text.Json;
using DysonNetwork.Sphere;
using DysonNetwork.Sphere.Account;
using DysonNetwork.Sphere.Chat;
using DysonNetwork.Sphere.Connection.WebReader;
using DysonNetwork.Sphere.Storage;
using DysonNetwork.Sphere.Wallet;
using Microsoft.EntityFrameworkCore;
@ -1361,6 +1362,132 @@ namespace DysonNetwork.Sphere.Migrations
b.ToTable("chat_realtime_call", (string)null);
});
modelBuilder.Entity("DysonNetwork.Sphere.Connection.WebReader.WebArticle", b =>
{
b.Property<Guid>("Id")
.ValueGeneratedOnAdd()
.HasColumnType("uuid")
.HasColumnName("id");
b.Property<string>("Author")
.HasMaxLength(4096)
.HasColumnType("character varying(4096)")
.HasColumnName("author");
b.Property<string>("Content")
.HasColumnType("text")
.HasColumnName("content");
b.Property<Instant>("CreatedAt")
.HasColumnType("timestamp with time zone")
.HasColumnName("created_at");
b.Property<Instant?>("DeletedAt")
.HasColumnType("timestamp with time zone")
.HasColumnName("deleted_at");
b.Property<Guid>("FeedId")
.HasColumnType("uuid")
.HasColumnName("feed_id");
b.Property<Dictionary<string, object>>("Meta")
.HasColumnType("jsonb")
.HasColumnName("meta");
b.Property<LinkEmbed>("Preview")
.HasColumnType("jsonb")
.HasColumnName("preview");
b.Property<DateTime?>("PublishedAt")
.HasColumnType("timestamp with time zone")
.HasColumnName("published_at");
b.Property<string>("Title")
.IsRequired()
.HasMaxLength(4096)
.HasColumnType("character varying(4096)")
.HasColumnName("title");
b.Property<Instant>("UpdatedAt")
.HasColumnType("timestamp with time zone")
.HasColumnName("updated_at");
b.Property<string>("Url")
.IsRequired()
.HasMaxLength(8192)
.HasColumnType("character varying(8192)")
.HasColumnName("url");
b.HasKey("Id")
.HasName("pk_web_articles");
b.HasIndex("FeedId")
.HasDatabaseName("ix_web_articles_feed_id");
b.HasIndex("Url")
.IsUnique()
.HasDatabaseName("ix_web_articles_url");
b.ToTable("web_articles", (string)null);
});
modelBuilder.Entity("DysonNetwork.Sphere.Connection.WebReader.WebFeed", b =>
{
b.Property<Guid>("Id")
.ValueGeneratedOnAdd()
.HasColumnType("uuid")
.HasColumnName("id");
b.Property<Instant>("CreatedAt")
.HasColumnType("timestamp with time zone")
.HasColumnName("created_at");
b.Property<Instant?>("DeletedAt")
.HasColumnType("timestamp with time zone")
.HasColumnName("deleted_at");
b.Property<string>("Description")
.HasMaxLength(8192)
.HasColumnType("character varying(8192)")
.HasColumnName("description");
b.Property<LinkEmbed>("Preview")
.HasColumnType("jsonb")
.HasColumnName("preview");
b.Property<Guid>("PublisherId")
.HasColumnType("uuid")
.HasColumnName("publisher_id");
b.Property<string>("Title")
.IsRequired()
.HasMaxLength(4096)
.HasColumnType("character varying(4096)")
.HasColumnName("title");
b.Property<Instant>("UpdatedAt")
.HasColumnType("timestamp with time zone")
.HasColumnName("updated_at");
b.Property<string>("Url")
.IsRequired()
.HasMaxLength(8192)
.HasColumnType("character varying(8192)")
.HasColumnName("url");
b.HasKey("Id")
.HasName("pk_web_feeds");
b.HasIndex("PublisherId")
.HasDatabaseName("ix_web_feeds_publisher_id");
b.HasIndex("Url")
.IsUnique()
.HasDatabaseName("ix_web_feeds_url");
b.ToTable("web_feeds", (string)null);
});
modelBuilder.Entity("DysonNetwork.Sphere.Developer.CustomApp", b =>
{
b.Property<Guid>("Id")
@ -3211,6 +3338,30 @@ namespace DysonNetwork.Sphere.Migrations
b.Navigation("Sender");
});
modelBuilder.Entity("DysonNetwork.Sphere.Connection.WebReader.WebArticle", b =>
{
b.HasOne("DysonNetwork.Sphere.Connection.WebReader.WebFeed", "Feed")
.WithMany("Articles")
.HasForeignKey("FeedId")
.OnDelete(DeleteBehavior.Cascade)
.IsRequired()
.HasConstraintName("fk_web_articles_web_feeds_feed_id");
b.Navigation("Feed");
});
modelBuilder.Entity("DysonNetwork.Sphere.Connection.WebReader.WebFeed", b =>
{
b.HasOne("DysonNetwork.Sphere.Publisher.Publisher", "Publisher")
.WithMany()
.HasForeignKey("PublisherId")
.OnDelete(DeleteBehavior.Cascade)
.IsRequired()
.HasConstraintName("fk_web_feeds_publishers_publisher_id");
b.Navigation("Publisher");
});
modelBuilder.Entity("DysonNetwork.Sphere.Developer.CustomApp", b =>
{
b.HasOne("DysonNetwork.Sphere.Publisher.Publisher", "Developer")
@ -3651,6 +3802,11 @@ namespace DysonNetwork.Sphere.Migrations
b.Navigation("Reactions");
});
modelBuilder.Entity("DysonNetwork.Sphere.Connection.WebReader.WebFeed", b =>
{
b.Navigation("Articles");
});
modelBuilder.Entity("DysonNetwork.Sphere.Permission.PermissionGroup", b =>
{
b.Navigation("Members");

9
DysonNetwork.Sphere/Startup/ScheduledJobsConfiguration.cs
View File

@ -1,4 +1,5 @@
using DysonNetwork.Sphere.Storage;
using DysonNetwork.Sphere.Connection.WebReader;
using DysonNetwork.Sphere.Storage.Handlers;
using DysonNetwork.Sphere.Wallet;
using Quartz;
@ -75,6 +76,14 @@ public static class ScheduledJobsConfiguration
.WithIntervalInMinutes(30)
.RepeatForever())
);
var webFeedScraperJob = new JobKey("WebFeedScraper");
q.AddJob<WebFeedScraperJob>(opts => opts.WithIdentity(webFeedScraperJob));
q.AddTrigger(opts => opts
.ForJob(webFeedScraperJob)
.WithIdentity("WebFeedScraperTrigger")
.WithSimpleSchedule(o => o.WithIntervalInHours(24).RepeatForever())
);
});
services.AddQuartzHostedService(q => q.WaitForJobsToComplete = true);

1
DysonNetwork.Sphere/Startup/ServiceCollectionExtensions.cs
View File

@ -224,6 +224,7 @@ public static class ServiceCollectionExtensions
services.AddScoped<PaymentService>();
services.AddScoped<IRealtimeService, LivekitRealtimeService>();
services.AddScoped<WebReaderService>();
services.AddScoped<WebFeedService>();
services.AddScoped<AfdianPaymentHandler>();
services.AddScoped<SafetyService>();

3
DysonNetwork.sln.DotSettings.user
View File

@ -85,6 +85,9 @@
<s:String x:Key="/Default/CodeInspection/ExcludedFiles/FilesAndFoldersToSkip2/=7020124F_002D9FFC_002D4AC3_002D8F3D_002DAAB8E0240759_002Ff_003ATusDiskStore_002Ecs_002Fl_003A_002E_002E_003F_002E_002E_003F_002E_002E_003FLibrary_003FApplication_0020Support_003FJetBrains_003FRider2025_002E1_003Fresharper_002Dhost_003FDecompilerCache_003Fdecompiler_003F8bb08a178b5b43c5bac20a5a54159a5b2a800_003F1c_003F21999acd_003FTusDiskStore_002Ecs/@EntryIndexedValue">ForceIncluded</s:String>
<s:String x:Key="/Default/CodeInspection/ExcludedFiles/FilesAndFoldersToSkip2/=7020124F_002D9FFC_002D4AC3_002D8F3D_002DAAB8E0240759_002Ff_003AUri_002Ecs_002Fl_003A_002E_002E_003F_002E_002E_003F_002E_002E_003FLibrary_003FApplication_0020Support_003FJetBrains_003FRider2024_002E3_003Fresharper_002Dhost_003FDecompilerCache_003Fdecompiler_003F5d2c480da9be415dab9be535bb6d08713cc00_003Fd0_003Fffc36a51_003FUri_002Ecs/@EntryIndexedValue">ForceIncluded</s:String>
<s:String x:Key="/Default/CodeInspection/ExcludedFiles/FilesAndFoldersToSkip2/=7020124F_002D9FFC_002D4AC3_002D8F3D_002DAAB8E0240759_002Ff_003AValidationContext_00601_002Ecs_002Fl_003A_002E_002E_003F_002E_002E_003F_002E_002E_003FLibrary_003FApplication_0020Support_003FJetBrains_003FRider2024_002E3_003Fresharper_002Dhost_003FDecompilerCache_003Fdecompiler_003F8bb08a178b5b43c5bac20a5a54159a5b2a800_003F6b_003F741ceebe_003FValidationContext_00601_002Ecs/@EntryIndexedValue">ForceIncluded</s:String>
<s:String x:Key="/Default/Environment/AssemblyExplorer/XmlDocument/@EntryValue">&lt;AssemblyExplorer&gt;
&lt;Assembly Path="/opt/homebrew/Cellar/dotnet/9.0.6/libexec/packs/Microsoft.AspNetCore.App.Ref/9.0.6/ref/net9.0/Microsoft.AspNetCore.RateLimiting.dll" /&gt;
&lt;/AssemblyExplorer&gt;</s:String>
<s:Boolean x:Key="/Default/ResxEditorPersonal/CheckedGroups/=DysonNetwork_002ESphere_002FLocalization_002FResources_002FAccountEventResource/@EntryIndexedValue">False</s:Boolean>
<s:Boolean x:Key="/Default/ResxEditorPersonal/CheckedGroups/=DysonNetwork_002ESphere_002FLocalization_002FResources_002FAccountEventResource/@EntryIndexRemoved">True</s:Boolean>
<s:Boolean x:Key="/Default/ResxEditorPersonal/CheckedGroups/=DysonNetwork_002ESphere_002FLocalization_002FResources_002FSharedResource/@EntryIndexedValue">False</s:Boolean>