From 93171bf48c4ca447d4d128b14f600d277606c01b Mon Sep 17 00:00:00 2001 From: LittleSheep Date: Mon, 19 Aug 2024 18:27:54 +0800 Subject: [PATCH] :sparkles: Scraper custom user agent --- pkg/internal/services/link_expander.go | 3 ++- settings.toml | 2 +- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/pkg/internal/services/link_expander.go b/pkg/internal/services/link_expander.go index e31bb80..39ec9be 100644 --- a/pkg/internal/services/link_expander.go +++ b/pkg/internal/services/link_expander.go @@ -8,6 +8,7 @@ import ( "github.com/gocolly/colly" "github.com/rs/zerolog/log" "github.com/samber/lo" + "github.com/spf13/viper" "net" "net/http" "time" @@ -37,7 +38,7 @@ func LinkExpand(target string) (*models.LinkMeta, error) { } c := colly.NewCollector( - colly.UserAgent("SolarBot/1.0"), + colly.UserAgent(viper.GetString("scraper.user-agent")), colly.MaxDepth(3), ) diff --git a/settings.toml b/settings.toml index 35d5d95..8ee9fed 100644 --- a/settings.toml +++ b/settings.toml @@ -44,4 +44,4 @@ prefix = "dealer_" aliases = { id = "auth", uc = "files", co = "interactive", im = "messaging" } [scraper] -cache = "uploads/cache.db" \ No newline at end of file +user-agent = "SolarBot/1.0" \ No newline at end of file