HyperNet/Reader
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

♻️ Save news article to db seprately

Browse Source
This commit is contained in:
LittleSheep 2025-01-26 01:00:59 +08:00
parent fbbe72bd54
commit 2a58d185eb
1 changed files with 11 additions and 10 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

21
pkg/internal/services/reader.go
View File

@ -31,7 +31,7 @@ func ScanNewsSourcesNoEager() {
} }
func ScanNewsSources(eager ...bool) { func ScanNewsSources(eager ...bool) {
var results []models.NewsArticle count := 0
for _, src := range NewsSources { for _, src := range NewsSources {
if !src.Enabled { if !src.Enabled {
continue continue
@ -42,18 +42,19 @@ func ScanNewsSources(eager ...bool) {
if err != nil { if err != nil {
log.Warn().Err(err).Str("source", src.ID).Msg("Failed to scan a news source.") log.Warn().Err(err).Str("source", src.ID).Msg("Failed to scan a news source.")
} }
results = append(results, result...)
result = lo.UniqBy(result, func(item models.NewsArticle) string {
return item.Hash
})
database.C.Clauses(clause.OnConflict{
UpdateAll: true,
}).Create(&result)
log.Info().Str("source", src.ID).Int("count", len(result)).Msg("Scanned a news sources.") log.Info().Str("source", src.ID).Int("count", len(result)).Msg("Scanned a news sources.")
count += len(result)
} }
log.Info().Int("count", len(results)).Msg("Scanned all news sources.")
results = lo.UniqBy(results, func(item models.NewsArticle) string { log.Info().Int("count", count).Msg("Scanned all news sources.")
return item.Hash
})
database.C.Clauses(clause.OnConflict{
UpdateAll: true,
}).Create(&results)
} }
func NewsSourceRead(src models.NewsSource, eager ...bool) ([]models.NewsArticle, error) { func NewsSourceRead(src models.NewsSource, eager ...bool) ([]models.NewsArticle, error) {