Cache link meta with database

This commit is contained in:
2024-08-19 14:52:33 +08:00
parent 039e174595
commit a590bca9a4
11 changed files with 107 additions and 3 deletions

BIN
pkg/.DS_Store vendored Normal file

Binary file not shown.

View File

@ -0,0 +1,20 @@
package database
import (
"git.solsynth.dev/hydrogen/dealer/pkg/internal/models"
"gorm.io/gorm"
)
var AutoMaintainRange = []any{
&models.LinkMeta{},
}
func RunMigration(source *gorm.DB) error {
if err := source.AutoMigrate(
AutoMaintainRange...,
); err != nil {
return err
}
return nil
}

View File

@ -0,0 +1,28 @@
package database
import (
"github.com/rs/zerolog/log"
"github.com/samber/lo"
"github.com/spf13/viper"
"gorm.io/driver/postgres"
"gorm.io/gorm"
"gorm.io/gorm/logger"
"gorm.io/gorm/schema"
)
var C *gorm.DB
func NewSource() error {
var err error
dialector := postgres.Open(viper.GetString("database.dsn"))
C, err = gorm.Open(dialector, &gorm.Config{NamingStrategy: schema.NamingStrategy{
TablePrefix: viper.GetString("database.prefix"),
}, Logger: logger.New(&log.Logger, logger.Config{
Colorful: true,
IgnoreRecordNotFoundError: true,
LogLevel: lo.Ternary(viper.GetBool("debug.database"), logger.Info, logger.Silent),
})})
return err
}

View File

@ -3,6 +3,7 @@ package models
type LinkMeta struct {
BaseModel
Entry string `json:"entry_id" gorm:"uniqueIndex"`
Icon string `json:"icon"`
URL string `json:"url"`
Image *string `json:"image"`

View File

@ -1,6 +1,9 @@
package services
import (
"crypto/md5"
"encoding/hex"
"git.solsynth.dev/hydrogen/dealer/pkg/internal/database"
"git.solsynth.dev/hydrogen/dealer/pkg/internal/models"
"github.com/gocolly/colly"
"github.com/rs/zerolog/log"
@ -10,7 +13,29 @@ import (
"time"
)
func RetrieveLinkMetaFromCache(target string) (models.LinkMeta, error) {
hash := md5.Sum([]byte(target))
entry := hex.EncodeToString(hash[:])
var meta models.LinkMeta
if err := database.C.Where("entry = ?", entry).First(&meta).Error; err != nil {
return meta, err
}
return meta, nil
}
func SaveLinkMetaToCache(target string, meta models.LinkMeta) error {
hash := md5.Sum([]byte(target))
entry := hex.EncodeToString(hash[:])
meta.Entry = entry
return database.C.Save(&meta).Error
}
func LinkExpand(target string) (*models.LinkMeta, error) {
if cache, err := RetrieveLinkMetaFromCache(target); err == nil {
log.Debug().Str("url", target).Msg("Expanding link... hit cache")
return &cache, nil
}
c := colly.NewCollector(
colly.UserAgent("SolarBot/1.0"),
colly.MaxDepth(3),
@ -75,6 +100,7 @@ func LinkExpand(target string) (*models.LinkMeta, error) {
})
c.OnScraped(func(r *colly.Response) {
_ = SaveLinkMetaToCache(target, *meta)
log.Debug().Str("url", target).Msg("Expanding link... finished")
})

View File

@ -1,6 +1,7 @@
package main
import (
"git.solsynth.dev/hydrogen/dealer/pkg/internal/database"
"git.solsynth.dev/hydrogen/dealer/pkg/internal/services"
"os"
"os/signal"
@ -33,6 +34,13 @@ func main() {
log.Panic().Err(err).Msg("An error occurred when loading settings.")
}
// Connect to database
if err := database.NewSource(); err != nil {
log.Fatal().Err(err).Msg("An error occurred when connect to database.")
} else if err := database.RunMigration(database.C); err != nil {
log.Fatal().Err(err).Msg("An error occurred when running database auto migration.")
}
// Set up external services
if err := services.SetupFirebase(); err != nil {
log.Warn().Err(err).Msg("An error occurred when setup firebase, firebase notification push is unavailable...")