Hydrogen/Dealer
2
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

✨ Scraper custom user agent

Browse Source
This commit is contained in:
LittleSheep 2024-08-19 18:27:54 +08:00
parent 8bea691f2e
commit 93171bf48c
2 changed files with 3 additions and 2 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
pkg/internal/services/link_expander.go
View File

@ -8,6 +8,7 @@ import (
"github.com/gocolly/colly" "github.com/gocolly/colly"
"github.com/rs/zerolog/log" "github.com/rs/zerolog/log"
"github.com/samber/lo" "github.com/samber/lo"
"github.com/spf13/viper"
"net" "net"
"net/http" "net/http"
"time" "time"
@ -37,7 +38,7 @@ func LinkExpand(target string) (*models.LinkMeta, error) {
} }
c := colly.NewCollector( c := colly.NewCollector(
colly.UserAgent("SolarBot/1.0"), colly.UserAgent(viper.GetString("scraper.user-agent")),
colly.MaxDepth(3), colly.MaxDepth(3),
) )

2
settings.toml
View File

@ -44,4 +44,4 @@ prefix = "dealer_"
aliases = { id = "auth", uc = "files", co = "interactive", im = "messaging" } aliases = { id = "auth", uc = "files", co = "interactive", im = "messaging" }
[scraper] [scraper]
cache = "uploads/cache.db" user-agent = "SolarBot/1.0"