✨ Cache link meta with database
This commit is contained in:
20
pkg/internal/database/migrator.go
Normal file
20
pkg/internal/database/migrator.go
Normal file
@ -0,0 +1,20 @@
|
||||
package database
|
||||
|
||||
import (
|
||||
"git.solsynth.dev/hydrogen/dealer/pkg/internal/models"
|
||||
"gorm.io/gorm"
|
||||
)
|
||||
|
||||
var AutoMaintainRange = []any{
|
||||
&models.LinkMeta{},
|
||||
}
|
||||
|
||||
func RunMigration(source *gorm.DB) error {
|
||||
if err := source.AutoMigrate(
|
||||
AutoMaintainRange...,
|
||||
); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
28
pkg/internal/database/source.go
Normal file
28
pkg/internal/database/source.go
Normal file
@ -0,0 +1,28 @@
|
||||
package database
|
||||
|
||||
import (
|
||||
"github.com/rs/zerolog/log"
|
||||
"github.com/samber/lo"
|
||||
"github.com/spf13/viper"
|
||||
"gorm.io/driver/postgres"
|
||||
"gorm.io/gorm"
|
||||
"gorm.io/gorm/logger"
|
||||
"gorm.io/gorm/schema"
|
||||
)
|
||||
|
||||
var C *gorm.DB
|
||||
|
||||
func NewSource() error {
|
||||
var err error
|
||||
|
||||
dialector := postgres.Open(viper.GetString("database.dsn"))
|
||||
C, err = gorm.Open(dialector, &gorm.Config{NamingStrategy: schema.NamingStrategy{
|
||||
TablePrefix: viper.GetString("database.prefix"),
|
||||
}, Logger: logger.New(&log.Logger, logger.Config{
|
||||
Colorful: true,
|
||||
IgnoreRecordNotFoundError: true,
|
||||
LogLevel: lo.Ternary(viper.GetBool("debug.database"), logger.Info, logger.Silent),
|
||||
})})
|
||||
|
||||
return err
|
||||
}
|
@ -3,6 +3,7 @@ package models
|
||||
type LinkMeta struct {
|
||||
BaseModel
|
||||
|
||||
Entry string `json:"entry_id" gorm:"uniqueIndex"`
|
||||
Icon string `json:"icon"`
|
||||
URL string `json:"url"`
|
||||
Image *string `json:"image"`
|
||||
|
@ -1,6 +1,9 @@
|
||||
package services
|
||||
|
||||
import (
|
||||
"crypto/md5"
|
||||
"encoding/hex"
|
||||
"git.solsynth.dev/hydrogen/dealer/pkg/internal/database"
|
||||
"git.solsynth.dev/hydrogen/dealer/pkg/internal/models"
|
||||
"github.com/gocolly/colly"
|
||||
"github.com/rs/zerolog/log"
|
||||
@ -10,7 +13,29 @@ import (
|
||||
"time"
|
||||
)
|
||||
|
||||
func RetrieveLinkMetaFromCache(target string) (models.LinkMeta, error) {
|
||||
hash := md5.Sum([]byte(target))
|
||||
entry := hex.EncodeToString(hash[:])
|
||||
var meta models.LinkMeta
|
||||
if err := database.C.Where("entry = ?", entry).First(&meta).Error; err != nil {
|
||||
return meta, err
|
||||
}
|
||||
return meta, nil
|
||||
}
|
||||
|
||||
func SaveLinkMetaToCache(target string, meta models.LinkMeta) error {
|
||||
hash := md5.Sum([]byte(target))
|
||||
entry := hex.EncodeToString(hash[:])
|
||||
meta.Entry = entry
|
||||
return database.C.Save(&meta).Error
|
||||
}
|
||||
|
||||
func LinkExpand(target string) (*models.LinkMeta, error) {
|
||||
if cache, err := RetrieveLinkMetaFromCache(target); err == nil {
|
||||
log.Debug().Str("url", target).Msg("Expanding link... hit cache")
|
||||
return &cache, nil
|
||||
}
|
||||
|
||||
c := colly.NewCollector(
|
||||
colly.UserAgent("SolarBot/1.0"),
|
||||
colly.MaxDepth(3),
|
||||
@ -75,6 +100,7 @@ func LinkExpand(target string) (*models.LinkMeta, error) {
|
||||
})
|
||||
|
||||
c.OnScraped(func(r *colly.Response) {
|
||||
_ = SaveLinkMetaToCache(target, *meta)
|
||||
log.Debug().Str("url", target).Msg("Expanding link... finished")
|
||||
})
|
||||
|
||||
|
Reference in New Issue
Block a user