From a590bca9a422bb44792f6ded9c69f4b2dcbf7b25 Mon Sep 17 00:00:00 2001
From: LittleSheep <littlesheep.code@hotmail.com>
Date: Mon, 19 Aug 2024 14:52:33 +0800
Subject: [PATCH] :sparkles: Cache link meta with database

---
 .DS_Store                              | Bin 0 -> 6148 bytes
 .idea/dataSources.xml                  |  17 +++++++++++++++
 go.mod                                 |   5 +++--
 go.sum                                 |   1 -
 pkg/.DS_Store                          | Bin 0 -> 6148 bytes
 pkg/internal/database/migrator.go      |  20 ++++++++++++++++++
 pkg/internal/database/source.go        |  28 +++++++++++++++++++++++++
 pkg/internal/models/link.go            |   1 +
 pkg/internal/services/link_expander.go |  26 +++++++++++++++++++++++
 pkg/main.go                            |   8 +++++++
 settings.toml                          |   4 ++++
 11 files changed, 107 insertions(+), 3 deletions(-)
 create mode 100644 .DS_Store
 create mode 100644 .idea/dataSources.xml
 create mode 100644 pkg/.DS_Store
 create mode 100644 pkg/internal/database/migrator.go
 create mode 100644 pkg/internal/database/source.go

diff --git a/.DS_Store b/.DS_Store
new file mode 100644
index 0000000000000000000000000000000000000000..b26819d65a040798e6702b1faaedd1401ac1b51d
GIT binary patch
literal 6148
zcmeHK-AcnS6i&A3GKSC#MK1&14xCQK;Z3RY1+3_W%53SFWwjaWW-rE|xB5c9iO=IX
zNeYhbTEv|L$#;H}=7Z*kF~+@Fbj+B;7!%MCIVx)e-D^V?n~cbDjBFexDg^5@9NXAm
z2mJOb8?zA`hWXF$PvUIaZh!GgwYIfg7j@ATPyUO_{b?|rWnM6OK<ieiBrNqHyo(2U
z-`TxV=`@Jb!9*9t(Ew5&@8dL5xu<4nl<8XE1VlqL`p*7*e%5KX<VAPUlJo8fsHdIt
z#iAkh4v#Kx`lIAERquvP4rFWDu~@<fD0VS#{v=IQIs{LdU1kvy1H=F^Kn!d+1NLAL
z%?;;)mQM^21FH<+{ve?tx)uwAdh37=ug@5-A)<heZwW+c(Y06@ga`;XrGTcC+b0G$
z<=~e#&$U<>H06xznPD6|bNP7TdUo(jozA#xka}W(7+7YYYD@>u|7-YV);{u=OUNPy
zh=G5`0I&7@o(GGvXY0!H@T`^4o}r;&UV#b-=u4LXFmN9^&_NTIs6(D>u`r0EpkJ2*
P(nUZMLLD*i3k-Y%onA>r

literal 0
HcmV?d00001

diff --git a/.idea/dataSources.xml b/.idea/dataSources.xml
new file mode 100644
index 0000000..f252da5
--- /dev/null
+++ b/.idea/dataSources.xml
@@ -0,0 +1,17 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<project version="4">
+  <component name="DataSourceManagerImpl" format="xml" multifile-model="true">
+    <data-source source="LOCAL" name="hy_dealer@localhost" uuid="1636968d-2acc-47c1-845d-b743c12a6f76">
+      <driver-ref>postgresql</driver-ref>
+      <synchronize>true</synchronize>
+      <jdbc-driver>org.postgresql.Driver</jdbc-driver>
+      <jdbc-url>jdbc:postgresql://localhost:5432/hy_dealer</jdbc-url>
+      <jdbc-additional-properties>
+        <property name="com.intellij.clouds.kubernetes.db.host.port" />
+        <property name="com.intellij.clouds.kubernetes.db.enabled" value="false" />
+        <property name="com.intellij.clouds.kubernetes.db.container.port" />
+      </jdbc-additional-properties>
+      <working-dir>$ProjectFileDir$</working-dir>
+    </data-source>
+  </component>
+</project>
\ No newline at end of file
diff --git a/go.mod b/go.mod
index 440e83a..5fb1a64 100644
--- a/go.mod
+++ b/go.mod
@@ -22,6 +22,7 @@ require (
 	google.golang.org/grpc v1.65.0
 	google.golang.org/protobuf v1.34.2
 	gorm.io/datatypes v1.2.0
+	gorm.io/driver/postgres v1.5.4
 	gorm.io/gorm v1.25.6
 )
 
@@ -65,8 +66,10 @@ require (
 	github.com/hashicorp/golang-lru v1.0.2 // indirect
 	github.com/hashicorp/hcl v1.0.0 // indirect
 	github.com/hashicorp/serf v0.10.1 // indirect
+	github.com/jackc/pgpassfile v1.0.0 // indirect
 	github.com/jackc/pgservicefile v0.0.0-20231201235250-de7065d80cb9 // indirect
 	github.com/jackc/pgx/v5 v5.5.1 // indirect
+	github.com/jackc/puddle/v2 v2.2.1 // indirect
 	github.com/jinzhu/inflection v1.0.0 // indirect
 	github.com/jinzhu/now v1.1.5 // indirect
 	github.com/jpillora/backoff v1.0.0 // indirect
@@ -115,11 +118,9 @@ require (
 	google.golang.org/genproto v0.0.0-20231106174013-bbf56f31fb17 // indirect
 	google.golang.org/genproto/googleapis/api v0.0.0-20240528184218-531527333157 // indirect
 	google.golang.org/genproto/googleapis/rpc v0.0.0-20240711142825-46eb208f015d // indirect
-	gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c // indirect
 	gopkg.in/ini.v1 v1.67.0 // indirect
 	gopkg.in/yaml.v3 v3.0.1 // indirect
 	gorm.io/driver/mysql v1.5.2 // indirect
-	gorm.io/driver/postgres v1.5.4 // indirect
 )
 
 replace git.solsynth.dev/hydrogen/bus => ../Bus
diff --git a/go.sum b/go.sum
index f40c382..1fbb36a 100644
--- a/go.sum
+++ b/go.sum
@@ -223,7 +223,6 @@ github.com/klauspost/compress v1.17.8/go.mod h1:Di0epgTjJY877eYKx5yC51cX2A2Vl2ib
 github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ=
 github.com/kr/logfmt v0.0.0-20140226030751-b84e30acd515/go.mod h1:+0opPa2QZZtGFBFZlji/RkVcI2GknAs/DXo4wKdlNEc=
 github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo=
-github.com/kr/pretty v0.2.1/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI=
 github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE=
 github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk=
 github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ=
diff --git a/pkg/.DS_Store b/pkg/.DS_Store
new file mode 100644
index 0000000000000000000000000000000000000000..d8c1b8672fd8fe5097e7b6669f1646131865ba42
GIT binary patch
literal 6148
zcmeHK%}T>S5Z<-bZYe?ziXH=A3s$RQ@e*Qv0V8@)sR;=hjM>tp=1>Z`>I?ZMK94iI
z8?ZD7Pa<{(Hs9?0?CyM!{b7u8cN!irW--PDG(?U{ji9;KwPu15xf)}_Qs%QHD6(<Q
zK!4GMYZokrb>}Q%-_{=_G63I4Fizqu@AW?UTBEtX(Go4u5%=DcEWF&$r<v<dZn1YI
zWfD}nA6&;#F|xMKWt#hO8ckF}97YgwcN3?fEL=HF!%XFR+9BGaJ+gLYv*UiRYo8A0
zU3)e-1ofzYGM~4__TK*4<>)ziN#&d7k^}8ZHVszr4$4wluihk0WcmocDzAzsBnF59
zVt^P}AqLDLAUZ2lGnGmV5CcDB0QUzQ8lq#c)Tp)&_~7*!{Y^v^(D5ySXd844mKwnW
z!gVU3PUYr_!F4*=Z4>7hEH&zM#?{O)j+wc7yl^!;*liWgxTBH2!~ij{$UsAPEj<4(
z;Fqa=<S(XBj~E~Z{uu+jIrN4u6lKoVrSkBswV^#gL&3Nb6%f!DE&<TNePmZTwckb^
Z;v9peMjQq0Djkq60*Vm&5Cgx!z!%wlOyd9m

literal 0
HcmV?d00001

diff --git a/pkg/internal/database/migrator.go b/pkg/internal/database/migrator.go
new file mode 100644
index 0000000..7176d8b
--- /dev/null
+++ b/pkg/internal/database/migrator.go
@@ -0,0 +1,20 @@
+package database
+
+import (
+	"git.solsynth.dev/hydrogen/dealer/pkg/internal/models"
+	"gorm.io/gorm"
+)
+
+var AutoMaintainRange = []any{
+	&models.LinkMeta{},
+}
+
+func RunMigration(source *gorm.DB) error {
+	if err := source.AutoMigrate(
+		AutoMaintainRange...,
+	); err != nil {
+		return err
+	}
+
+	return nil
+}
diff --git a/pkg/internal/database/source.go b/pkg/internal/database/source.go
new file mode 100644
index 0000000..7602278
--- /dev/null
+++ b/pkg/internal/database/source.go
@@ -0,0 +1,28 @@
+package database
+
+import (
+	"github.com/rs/zerolog/log"
+	"github.com/samber/lo"
+	"github.com/spf13/viper"
+	"gorm.io/driver/postgres"
+	"gorm.io/gorm"
+	"gorm.io/gorm/logger"
+	"gorm.io/gorm/schema"
+)
+
+var C *gorm.DB
+
+func NewSource() error {
+	var err error
+
+	dialector := postgres.Open(viper.GetString("database.dsn"))
+	C, err = gorm.Open(dialector, &gorm.Config{NamingStrategy: schema.NamingStrategy{
+		TablePrefix: viper.GetString("database.prefix"),
+	}, Logger: logger.New(&log.Logger, logger.Config{
+		Colorful:                  true,
+		IgnoreRecordNotFoundError: true,
+		LogLevel:                  lo.Ternary(viper.GetBool("debug.database"), logger.Info, logger.Silent),
+	})})
+
+	return err
+}
diff --git a/pkg/internal/models/link.go b/pkg/internal/models/link.go
index b19ecfe..70de75d 100644
--- a/pkg/internal/models/link.go
+++ b/pkg/internal/models/link.go
@@ -3,6 +3,7 @@ package models
 type LinkMeta struct {
 	BaseModel
 
+	Entry       string  `json:"entry_id" gorm:"uniqueIndex"`
 	Icon        string  `json:"icon"`
 	URL         string  `json:"url"`
 	Image       *string `json:"image"`
diff --git a/pkg/internal/services/link_expander.go b/pkg/internal/services/link_expander.go
index 997c946..3c3f4f3 100644
--- a/pkg/internal/services/link_expander.go
+++ b/pkg/internal/services/link_expander.go
@@ -1,6 +1,9 @@
 package services
 
 import (
+	"crypto/md5"
+	"encoding/hex"
+	"git.solsynth.dev/hydrogen/dealer/pkg/internal/database"
 	"git.solsynth.dev/hydrogen/dealer/pkg/internal/models"
 	"github.com/gocolly/colly"
 	"github.com/rs/zerolog/log"
@@ -10,7 +13,29 @@ import (
 	"time"
 )
 
+func RetrieveLinkMetaFromCache(target string) (models.LinkMeta, error) {
+	hash := md5.Sum([]byte(target))
+	entry := hex.EncodeToString(hash[:])
+	var meta models.LinkMeta
+	if err := database.C.Where("entry = ?", entry).First(&meta).Error; err != nil {
+		return meta, err
+	}
+	return meta, nil
+}
+
+func SaveLinkMetaToCache(target string, meta models.LinkMeta) error {
+	hash := md5.Sum([]byte(target))
+	entry := hex.EncodeToString(hash[:])
+	meta.Entry = entry
+	return database.C.Save(&meta).Error
+}
+
 func LinkExpand(target string) (*models.LinkMeta, error) {
+	if cache, err := RetrieveLinkMetaFromCache(target); err == nil {
+		log.Debug().Str("url", target).Msg("Expanding link... hit cache")
+		return &cache, nil
+	}
+
 	c := colly.NewCollector(
 		colly.UserAgent("SolarBot/1.0"),
 		colly.MaxDepth(3),
@@ -75,6 +100,7 @@ func LinkExpand(target string) (*models.LinkMeta, error) {
 	})
 
 	c.OnScraped(func(r *colly.Response) {
+		_ = SaveLinkMetaToCache(target, *meta)
 		log.Debug().Str("url", target).Msg("Expanding link... finished")
 	})
 
diff --git a/pkg/main.go b/pkg/main.go
index fe219cc..315787a 100644
--- a/pkg/main.go
+++ b/pkg/main.go
@@ -1,6 +1,7 @@
 package main
 
 import (
+	"git.solsynth.dev/hydrogen/dealer/pkg/internal/database"
 	"git.solsynth.dev/hydrogen/dealer/pkg/internal/services"
 	"os"
 	"os/signal"
@@ -33,6 +34,13 @@ func main() {
 		log.Panic().Err(err).Msg("An error occurred when loading settings.")
 	}
 
+	// Connect to database
+	if err := database.NewSource(); err != nil {
+		log.Fatal().Err(err).Msg("An error occurred when connect to database.")
+	} else if err := database.RunMigration(database.C); err != nil {
+		log.Fatal().Err(err).Msg("An error occurred when running database auto migration.")
+	}
+
 	// Set up external services
 	if err := services.SetupFirebase(); err != nil {
 		log.Warn().Err(err).Msg("An error occurred when setup firebase, firebase notification push is unavailable...")
diff --git a/settings.toml b/settings.toml
index 67d0ac3..35d5d95 100644
--- a/settings.toml
+++ b/settings.toml
@@ -36,6 +36,10 @@ cookie_samesite = "Lax"
 access_token_duration = 300
 refresh_token_duration = 2592000
 
+[database]
+dsn = "host=localhost user=postgres password=password dbname=hy_dealer port=5432 sslmode=disable"
+prefix = "dealer_"
+
 [services]
 aliases = { id = "auth", uc = "files", co = "interactive", im = "messaging" }