diff --git a/pkg/internal/server/api/admin_api.go b/pkg/internal/server/api/admin_api.go index 38d706b..7acd34a 100644 --- a/pkg/internal/server/api/admin_api.go +++ b/pkg/internal/server/api/admin_api.go @@ -2,9 +2,14 @@ package api import ( "git.solsynth.dev/hypernet/nexus/pkg/nex/sec" + "git.solsynth.dev/hypernet/reader/pkg/internal/database" + "git.solsynth.dev/hypernet/reader/pkg/internal/models" "git.solsynth.dev/hypernet/reader/pkg/internal/server/exts" "git.solsynth.dev/hypernet/reader/pkg/internal/services" "github.com/gofiber/fiber/v2" + "github.com/rs/zerolog/log" + "github.com/samber/lo" + "gorm.io/gorm/clause" ) func adminTriggerScanTask(c *fiber.Ctx) error { @@ -13,13 +18,43 @@ func adminTriggerScanTask(c *fiber.Ctx) error { } var data struct { - Eager bool `json:"eager"` + Eager bool `json:"eager"` + Sources []string `json:"sources"` } if err := exts.BindAndValidate(c, &data); err != nil { return err } - go services.ScanNewsSources(data.Eager) + go func() { + count := 0 + for _, src := range services.NewsSources { + if !src.Enabled { + continue + } + if len(data.Sources) > 0 && !lo.Contains(data.Sources, src.ID) { + continue + } + + log.Debug().Str("source", src.ID).Msg("Scanning news source...") + result, err := services.NewsSourceRead(src, data.Eager) + if err != nil { + log.Warn().Err(err).Str("source", src.ID).Msg("Failed to scan a news source.") + } + + result = lo.UniqBy(result, func(item models.NewsArticle) string { + return item.Hash + }) + database.C.Clauses(clause.OnConflict{ + UpdateAll: true, + }).Create(&result) + + log.Info().Str("source", src.ID).Int("count", len(result)).Msg("Scanned a news sources.") + count += len(result) + } + + log.Info().Int("count", count).Msg("Scanned all news sources.") + }() + return c.SendStatus(fiber.StatusOK) } diff --git a/pkg/internal/services/reader.go b/pkg/internal/services/reader.go index e952a10..5315075 100644 --- a/pkg/internal/services/reader.go +++ b/pkg/internal/services/reader.go @@ -62,9 +62,9 @@ func NewsSourceRead(src models.NewsSource, eager ...bool) ([]models.NewsArticle, case "wordpress": return newsSourceReadWordpress(src, eager...) case "scrap": - return newsSourceReadScrap(src) + return newsSourceReadScrap(src, eager...) case "feed": - return newsSourceReadFeed(src) + return newsSourceReadFeed(src, eager...) default: return nil, fmt.Errorf("unsupported news source type: %s", src.Type) }