✨ Cache link meta with database

.idea/dataSources.xml

@@ -0,0 +1,17 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<project version="4">
+  <component name="DataSourceManagerImpl" format="xml" multifile-model="true">
+    <data-source source="LOCAL" name="hy_dealer@localhost" uuid="1636968d-2acc-47c1-845d-b743c12a6f76">
+      <driver-ref>postgresql</driver-ref>
+      <synchronize>true</synchronize>
+      <jdbc-driver>org.postgresql.Driver</jdbc-driver>
+      <jdbc-url>jdbc:postgresql://localhost:5432/hy_dealer</jdbc-url>
+      <jdbc-additional-properties>
+        <property name="com.intellij.clouds.kubernetes.db.host.port" />
+        <property name="com.intellij.clouds.kubernetes.db.enabled" value="false" />
+        <property name="com.intellij.clouds.kubernetes.db.container.port" />
+      </jdbc-additional-properties>
+      <working-dir>$ProjectFileDir$</working-dir>
+    </data-source>
+  </component>
+</project>

go.mod

@@ -22,6 +22,7 @@ require (
 	google.golang.org/grpc v1.65.0
 	google.golang.org/protobuf v1.34.2
 	gorm.io/datatypes v1.2.0
+	gorm.io/driver/postgres v1.5.4
 	gorm.io/gorm v1.25.6
 )
 
@@ -65,8 +66,10 @@ require (
 	github.com/hashicorp/golang-lru v1.0.2 // indirect
 	github.com/hashicorp/hcl v1.0.0 // indirect
 	github.com/hashicorp/serf v0.10.1 // indirect
+	github.com/jackc/pgpassfile v1.0.0 // indirect
 	github.com/jackc/pgservicefile v0.0.0-20231201235250-de7065d80cb9 // indirect
 	github.com/jackc/pgx/v5 v5.5.1 // indirect
+	github.com/jackc/puddle/v2 v2.2.1 // indirect
 	github.com/jinzhu/inflection v1.0.0 // indirect
 	github.com/jinzhu/now v1.1.5 // indirect
 	github.com/jpillora/backoff v1.0.0 // indirect
@@ -115,11 +118,9 @@ require (
 	google.golang.org/genproto v0.0.0-20231106174013-bbf56f31fb17 // indirect
 	google.golang.org/genproto/googleapis/api v0.0.0-20240528184218-531527333157 // indirect
 	google.golang.org/genproto/googleapis/rpc v0.0.0-20240711142825-46eb208f015d // indirect
-	gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c // indirect
 	gopkg.in/ini.v1 v1.67.0 // indirect
 	gopkg.in/yaml.v3 v3.0.1 // indirect
 	gorm.io/driver/mysql v1.5.2 // indirect
-	gorm.io/driver/postgres v1.5.4 // indirect
 )
 
 replace git.solsynth.dev/hydrogen/bus => ../Bus

go.sum

@@ -223,7 +223,6 @@ github.com/klauspost/compress v1.17.8/go.mod h1:Di0epgTjJY877eYKx5yC51cX2A2Vl2ib
 github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ=
 github.com/kr/logfmt v0.0.0-20140226030751-b84e30acd515/go.mod h1:+0opPa2QZZtGFBFZlji/RkVcI2GknAs/DXo4wKdlNEc=
 github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo=
-github.com/kr/pretty v0.2.1/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI=
 github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE=
 github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk=
 github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ=

pkg/internal/database/migrator.go

@@ -0,0 +1,20 @@
+package database
+
+import (
+	"git.solsynth.dev/hydrogen/dealer/pkg/internal/models"
+	"gorm.io/gorm"
+)
+
+var AutoMaintainRange = []any{
+	&models.LinkMeta{},
+}
+
+func RunMigration(source *gorm.DB) error {
+	if err := source.AutoMigrate(
+		AutoMaintainRange...,
+	); err != nil {
+		return err
+	}
+
+	return nil
+}

pkg/internal/database/source.go

@@ -0,0 +1,28 @@
+package database
+
+import (
+	"github.com/rs/zerolog/log"
+	"github.com/samber/lo"
+	"github.com/spf13/viper"
+	"gorm.io/driver/postgres"
+	"gorm.io/gorm"
+	"gorm.io/gorm/logger"
+	"gorm.io/gorm/schema"
+)
+
+var C *gorm.DB
+
+func NewSource() error {
+	var err error
+
+	dialector := postgres.Open(viper.GetString("database.dsn"))
+	C, err = gorm.Open(dialector, &gorm.Config{NamingStrategy: schema.NamingStrategy{
+		TablePrefix: viper.GetString("database.prefix"),
+	}, Logger: logger.New(&log.Logger, logger.Config{
+		Colorful:                  true,
+		IgnoreRecordNotFoundError: true,
+		LogLevel:                  lo.Ternary(viper.GetBool("debug.database"), logger.Info, logger.Silent),
+	})})
+
+	return err
+}

pkg/internal/models/link.go

@@ -3,6 +3,7 @@ package models
 type LinkMeta struct {
 	BaseModel
 
+	Entry       string  `json:"entry_id" gorm:"uniqueIndex"`
 	Icon        string  `json:"icon"`
 	URL         string  `json:"url"`
 	Image       *string `json:"image"`

pkg/internal/services/link_expander.go

@@ -1,6 +1,9 @@
 package services
 
 import (
+	"crypto/md5"
+	"encoding/hex"
+	"git.solsynth.dev/hydrogen/dealer/pkg/internal/database"
 	"git.solsynth.dev/hydrogen/dealer/pkg/internal/models"
 	"github.com/gocolly/colly"
 	"github.com/rs/zerolog/log"
@@ -10,7 +13,29 @@ import (
 	"time"
 )
 
+func RetrieveLinkMetaFromCache(target string) (models.LinkMeta, error) {
+	hash := md5.Sum([]byte(target))
+	entry := hex.EncodeToString(hash[:])
+	var meta models.LinkMeta
+	if err := database.C.Where("entry = ?", entry).First(&meta).Error; err != nil {
+		return meta, err
+	}
+	return meta, nil
+}
+
+func SaveLinkMetaToCache(target string, meta models.LinkMeta) error {
+	hash := md5.Sum([]byte(target))
+	entry := hex.EncodeToString(hash[:])
+	meta.Entry = entry
+	return database.C.Save(&meta).Error
+}
+
 func LinkExpand(target string) (*models.LinkMeta, error) {
+	if cache, err := RetrieveLinkMetaFromCache(target); err == nil {
+		log.Debug().Str("url", target).Msg("Expanding link... hit cache")
+		return &cache, nil
+	}
+
 	c := colly.NewCollector(
 		colly.UserAgent("SolarBot/1.0"),
 		colly.MaxDepth(3),
@@ -75,6 +100,7 @@ func LinkExpand(target string) (*models.LinkMeta, error) {
 	})
 
 	c.OnScraped(func(r *colly.Response) {
+		_ = SaveLinkMetaToCache(target, *meta)
 		log.Debug().Str("url", target).Msg("Expanding link... finished")
 	})
 

pkg/main.go

@@ -1,6 +1,7 @@
 package main
 
 import (
+	"git.solsynth.dev/hydrogen/dealer/pkg/internal/database"
 	"git.solsynth.dev/hydrogen/dealer/pkg/internal/services"
 	"os"
 	"os/signal"
@@ -33,6 +34,13 @@ func main() {
 		log.Panic().Err(err).Msg("An error occurred when loading settings.")
 	}
 
+	// Connect to database
+	if err := database.NewSource(); err != nil {
+		log.Fatal().Err(err).Msg("An error occurred when connect to database.")
+	} else if err := database.RunMigration(database.C); err != nil {
+		log.Fatal().Err(err).Msg("An error occurred when running database auto migration.")
+	}
+
 	// Set up external services
 	if err := services.SetupFirebase(); err != nil {
 		log.Warn().Err(err).Msg("An error occurred when setup firebase, firebase notification push is unavailable...")

settings.toml

@@ -36,6 +36,10 @@ cookie_samesite = "Lax"
 access_token_duration = 300
 refresh_token_duration = 2592000
 
+[database]
+dsn = "host=localhost user=postgres password=password dbname=hy_dealer port=5432 sslmode=disable"
+prefix = "dealer_"
+
 [services]
 aliases = { id = "auth", uc = "files", co = "interactive", im = "messaging" }