Scraper custom user agent

This commit is contained in:
2024-08-19 18:27:54 +08:00
parent 8bea691f2e
commit 93171bf48c
2 changed files with 3 additions and 2 deletions

View File

@ -8,6 +8,7 @@ import (
"github.com/gocolly/colly"
"github.com/rs/zerolog/log"
"github.com/samber/lo"
"github.com/spf13/viper"
"net"
"net/http"
"time"
@ -37,7 +38,7 @@ func LinkExpand(target string) (*models.LinkMeta, error) {
}
c := colly.NewCollector(
colly.UserAgent("SolarBot/1.0"),
colly.UserAgent(viper.GetString("scraper.user-agent")),
colly.MaxDepth(3),
)