✨ Admin trigger api now can decide to fetch which source only

pkg/internal/server/api/admin_api.go

@@ -2,9 +2,14 @@ package api
 
 import (
 	"git.solsynth.dev/hypernet/nexus/pkg/nex/sec"
+	"git.solsynth.dev/hypernet/reader/pkg/internal/database"
+	"git.solsynth.dev/hypernet/reader/pkg/internal/models"
 	"git.solsynth.dev/hypernet/reader/pkg/internal/server/exts"
 	"git.solsynth.dev/hypernet/reader/pkg/internal/services"
 	"github.com/gofiber/fiber/v2"
+	"github.com/rs/zerolog/log"
+	"github.com/samber/lo"
+	"gorm.io/gorm/clause"
 )
 
 func adminTriggerScanTask(c *fiber.Ctx) error {
@@ -13,13 +18,43 @@ func adminTriggerScanTask(c *fiber.Ctx) error {
 	}
 
 	var data struct {
-		Eager bool `json:"eager"`
+		Eager   bool     `json:"eager"`
+		Sources []string `json:"sources"`
 	}
 
 	if err := exts.BindAndValidate(c, &data); err != nil {
 		return err
 	}
 
-	go services.ScanNewsSources(data.Eager)
+	go func() {
+		count := 0
+		for _, src := range services.NewsSources {
+			if !src.Enabled {
+				continue
+			}
+			if len(data.Sources) > 0 && !lo.Contains(data.Sources, src.ID) {
+				continue
+			}
+
+			log.Debug().Str("source", src.ID).Msg("Scanning news source...")
+			result, err := services.NewsSourceRead(src, data.Eager)
+			if err != nil {
+				log.Warn().Err(err).Str("source", src.ID).Msg("Failed to scan a news source.")
+			}
+
+			result = lo.UniqBy(result, func(item models.NewsArticle) string {
+				return item.Hash
+			})
+			database.C.Clauses(clause.OnConflict{
+				UpdateAll: true,
+			}).Create(&result)
+
+			log.Info().Str("source", src.ID).Int("count", len(result)).Msg("Scanned a news sources.")
+			count += len(result)
+		}
+
+		log.Info().Int("count", count).Msg("Scanned all news sources.")
+	}()
+
 	return c.SendStatus(fiber.StatusOK)
 }

pkg/internal/services/reader.go

@@ -62,9 +62,9 @@ func NewsSourceRead(src models.NewsSource, eager ...bool) ([]models.NewsArticle,
 	case "wordpress":
 		return newsSourceReadWordpress(src, eager...)
 	case "scrap":
-		return newsSourceReadScrap(src)
+		return newsSourceReadScrap(src, eager...)
 	case "feed":
-		return newsSourceReadFeed(src)
+		return newsSourceReadFeed(src, eager...)
 	default:
 		return nil, fmt.Errorf("unsupported news source type: %s", src.Type)
 	}