✨ Scraper custom user agent

2024-08-19 18:27:54 +08:00
parent 8bea691f2e
commit 93171bf48c
2 changed files with 3 additions and 2 deletions
--- a/pkg/internal/services/link_expander.go
+++ b/pkg/internal/services/link_expander.go
@ -8,6 +8,7 @@ import (
 	"github.com/gocolly/colly"
 	"github.com/rs/zerolog/log"
 	"github.com/samber/lo"
+	"github.com/spf13/viper"
 	"net"
 	"net/http"
 	"time"
@ -37,7 +38,7 @@ func LinkExpand(target string) (*models.LinkMeta, error) {
 	}

 	c := colly.NewCollector(
-		colly.UserAgent("SolarBot/1.0"),
+		colly.UserAgent(viper.GetString("scraper.user-agent")),
 		colly.MaxDepth(3),
 	)