diff --git a/.DS_Store b/.DS_Store new file mode 100644 index 0000000..b26819d Binary files /dev/null and b/.DS_Store differ diff --git a/.idea/dataSources.xml b/.idea/dataSources.xml new file mode 100644 index 0000000..f252da5 --- /dev/null +++ b/.idea/dataSources.xml @@ -0,0 +1,17 @@ + + + + + postgresql + true + org.postgresql.Driver + jdbc:postgresql://localhost:5432/hy_dealer + + + + + + $ProjectFileDir$ + + + \ No newline at end of file diff --git a/go.mod b/go.mod index 440e83a..5fb1a64 100644 --- a/go.mod +++ b/go.mod @@ -22,6 +22,7 @@ require ( google.golang.org/grpc v1.65.0 google.golang.org/protobuf v1.34.2 gorm.io/datatypes v1.2.0 + gorm.io/driver/postgres v1.5.4 gorm.io/gorm v1.25.6 ) @@ -65,8 +66,10 @@ require ( github.com/hashicorp/golang-lru v1.0.2 // indirect github.com/hashicorp/hcl v1.0.0 // indirect github.com/hashicorp/serf v0.10.1 // indirect + github.com/jackc/pgpassfile v1.0.0 // indirect github.com/jackc/pgservicefile v0.0.0-20231201235250-de7065d80cb9 // indirect github.com/jackc/pgx/v5 v5.5.1 // indirect + github.com/jackc/puddle/v2 v2.2.1 // indirect github.com/jinzhu/inflection v1.0.0 // indirect github.com/jinzhu/now v1.1.5 // indirect github.com/jpillora/backoff v1.0.0 // indirect @@ -115,11 +118,9 @@ require ( google.golang.org/genproto v0.0.0-20231106174013-bbf56f31fb17 // indirect google.golang.org/genproto/googleapis/api v0.0.0-20240528184218-531527333157 // indirect google.golang.org/genproto/googleapis/rpc v0.0.0-20240711142825-46eb208f015d // indirect - gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c // indirect gopkg.in/ini.v1 v1.67.0 // indirect gopkg.in/yaml.v3 v3.0.1 // indirect gorm.io/driver/mysql v1.5.2 // indirect - gorm.io/driver/postgres v1.5.4 // indirect ) replace git.solsynth.dev/hydrogen/bus => ../Bus diff --git a/go.sum b/go.sum index f40c382..1fbb36a 100644 --- a/go.sum +++ b/go.sum @@ -223,7 +223,6 @@ github.com/klauspost/compress v1.17.8/go.mod h1:Di0epgTjJY877eYKx5yC51cX2A2Vl2ib github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= github.com/kr/logfmt v0.0.0-20140226030751-b84e30acd515/go.mod h1:+0opPa2QZZtGFBFZlji/RkVcI2GknAs/DXo4wKdlNEc= github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= -github.com/kr/pretty v0.2.1/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI= github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE= github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk= github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= diff --git a/pkg/.DS_Store b/pkg/.DS_Store new file mode 100644 index 0000000..d8c1b86 Binary files /dev/null and b/pkg/.DS_Store differ diff --git a/pkg/internal/database/migrator.go b/pkg/internal/database/migrator.go new file mode 100644 index 0000000..7176d8b --- /dev/null +++ b/pkg/internal/database/migrator.go @@ -0,0 +1,20 @@ +package database + +import ( + "git.solsynth.dev/hydrogen/dealer/pkg/internal/models" + "gorm.io/gorm" +) + +var AutoMaintainRange = []any{ + &models.LinkMeta{}, +} + +func RunMigration(source *gorm.DB) error { + if err := source.AutoMigrate( + AutoMaintainRange..., + ); err != nil { + return err + } + + return nil +} diff --git a/pkg/internal/database/source.go b/pkg/internal/database/source.go new file mode 100644 index 0000000..7602278 --- /dev/null +++ b/pkg/internal/database/source.go @@ -0,0 +1,28 @@ +package database + +import ( + "github.com/rs/zerolog/log" + "github.com/samber/lo" + "github.com/spf13/viper" + "gorm.io/driver/postgres" + "gorm.io/gorm" + "gorm.io/gorm/logger" + "gorm.io/gorm/schema" +) + +var C *gorm.DB + +func NewSource() error { + var err error + + dialector := postgres.Open(viper.GetString("database.dsn")) + C, err = gorm.Open(dialector, &gorm.Config{NamingStrategy: schema.NamingStrategy{ + TablePrefix: viper.GetString("database.prefix"), + }, Logger: logger.New(&log.Logger, logger.Config{ + Colorful: true, + IgnoreRecordNotFoundError: true, + LogLevel: lo.Ternary(viper.GetBool("debug.database"), logger.Info, logger.Silent), + })}) + + return err +} diff --git a/pkg/internal/models/link.go b/pkg/internal/models/link.go index b19ecfe..70de75d 100644 --- a/pkg/internal/models/link.go +++ b/pkg/internal/models/link.go @@ -3,6 +3,7 @@ package models type LinkMeta struct { BaseModel + Entry string `json:"entry_id" gorm:"uniqueIndex"` Icon string `json:"icon"` URL string `json:"url"` Image *string `json:"image"` diff --git a/pkg/internal/services/link_expander.go b/pkg/internal/services/link_expander.go index 997c946..3c3f4f3 100644 --- a/pkg/internal/services/link_expander.go +++ b/pkg/internal/services/link_expander.go @@ -1,6 +1,9 @@ package services import ( + "crypto/md5" + "encoding/hex" + "git.solsynth.dev/hydrogen/dealer/pkg/internal/database" "git.solsynth.dev/hydrogen/dealer/pkg/internal/models" "github.com/gocolly/colly" "github.com/rs/zerolog/log" @@ -10,7 +13,29 @@ import ( "time" ) +func RetrieveLinkMetaFromCache(target string) (models.LinkMeta, error) { + hash := md5.Sum([]byte(target)) + entry := hex.EncodeToString(hash[:]) + var meta models.LinkMeta + if err := database.C.Where("entry = ?", entry).First(&meta).Error; err != nil { + return meta, err + } + return meta, nil +} + +func SaveLinkMetaToCache(target string, meta models.LinkMeta) error { + hash := md5.Sum([]byte(target)) + entry := hex.EncodeToString(hash[:]) + meta.Entry = entry + return database.C.Save(&meta).Error +} + func LinkExpand(target string) (*models.LinkMeta, error) { + if cache, err := RetrieveLinkMetaFromCache(target); err == nil { + log.Debug().Str("url", target).Msg("Expanding link... hit cache") + return &cache, nil + } + c := colly.NewCollector( colly.UserAgent("SolarBot/1.0"), colly.MaxDepth(3), @@ -75,6 +100,7 @@ func LinkExpand(target string) (*models.LinkMeta, error) { }) c.OnScraped(func(r *colly.Response) { + _ = SaveLinkMetaToCache(target, *meta) log.Debug().Str("url", target).Msg("Expanding link... finished") }) diff --git a/pkg/main.go b/pkg/main.go index fe219cc..315787a 100644 --- a/pkg/main.go +++ b/pkg/main.go @@ -1,6 +1,7 @@ package main import ( + "git.solsynth.dev/hydrogen/dealer/pkg/internal/database" "git.solsynth.dev/hydrogen/dealer/pkg/internal/services" "os" "os/signal" @@ -33,6 +34,13 @@ func main() { log.Panic().Err(err).Msg("An error occurred when loading settings.") } + // Connect to database + if err := database.NewSource(); err != nil { + log.Fatal().Err(err).Msg("An error occurred when connect to database.") + } else if err := database.RunMigration(database.C); err != nil { + log.Fatal().Err(err).Msg("An error occurred when running database auto migration.") + } + // Set up external services if err := services.SetupFirebase(); err != nil { log.Warn().Err(err).Msg("An error occurred when setup firebase, firebase notification push is unavailable...") diff --git a/settings.toml b/settings.toml index 67d0ac3..35d5d95 100644 --- a/settings.toml +++ b/settings.toml @@ -36,6 +36,10 @@ cookie_samesite = "Lax" access_token_duration = 300 refresh_token_duration = 2592000 +[database] +dsn = "host=localhost user=postgres password=password dbname=hy_dealer port=5432 sslmode=disable" +prefix = "dealer_" + [services] aliases = { id = "auth", uc = "files", co = "interactive", im = "messaging" }