✨ WatchTower for database

Dockerfile

@@ -10,6 +10,8 @@ RUN CGO_ENABLED=0 GOOS=linux GOARCH=amd64 go build -buildvcs -o /dist ./pkg/main
 # Runtime
 FROM golang:alpine
 
+RUN apk add postgresql-client
+
 COPY --from=nexus-server /dist /nexus/server
 
 EXPOSE 8444

README.md

@@ -41,6 +41,8 @@ the establishment of the connection.
 At the same time, the allocated database will be added into the watchtower for auto maintenance
 (auto remove the soft-deleted records, backup and more).
 
+Currently, the database is only support postgres, there is no plan for supporting other databases.
+
 ### Authorization
 
 All the request forwarded by the Nexus will handle the authorization automatically.

ROADMAP.md

@@ -1,16 +0,0 @@
-# Roadmap
-
-The development progress and plan for Hypernet.Nexus
-
-- [x] Service discovery
-- [x] Command system
-- [x] High availability
-- [x] Microservice gateway
-- [ ] Authenticate (W.I.P)
-- [ ] FastLSF (fast lua based serverless function)
-
-The goal of project Hypernet is going to replace the Hydrogen as Solar Network server-side software.
-And the goal of this project is going to replace Hydrogen.Dealer as the core component of Solar Network.
-
-Other Hydrogen project will be refactored and upgraded to support Nexus as soon as the first stable version is released.
-Some features will moved to command based api, such as daily sign in Passport which isn't in Nexus Standard and will be not in it.

go.mod

@@ -2,8 +2,6 @@ module git.solsynth.dev/hypernet/nexus
 
 go 1.22.0
 
-toolchain go1.23.2
-
 require (
 	github.com/fatih/color v1.18.0
 	github.com/go-playground/validator/v10 v10.22.1
@@ -18,6 +16,7 @@ require (
 	github.com/rs/zerolog v1.33.0
 	github.com/samber/lo v1.47.0
 	github.com/spf13/viper v1.19.0
+	github.com/valyala/fasthttp v1.57.0
 	go.etcd.io/etcd/client/v3 v3.5.16
 	google.golang.org/grpc v1.67.1
 	google.golang.org/protobuf v1.35.1
@@ -71,7 +70,6 @@ require (
 	github.com/subosito/gotenv v1.6.0 // indirect
 	github.com/tinylib/msgp v1.2.4 // indirect
 	github.com/valyala/bytebufferpool v1.0.0 // indirect
-	github.com/valyala/fasthttp v1.57.0 // indirect
 	github.com/valyala/tcplisten v1.0.0 // indirect
 	go.etcd.io/etcd/api/v3 v3.5.16 // indirect
 	go.etcd.io/etcd/client/pkg/v3 v3.5.16 // indirect
@@ -89,5 +87,3 @@ require (
 	gopkg.in/yaml.v3 v3.0.1 // indirect
 	gorm.io/driver/mysql v1.5.7 // indirect
 )
-
-replace git.solsynth.dev/hydrogen/bus => ../Bus

pkg/internal/database/allocator.go

@@ -2,9 +2,11 @@ package database
 
 import (
 	"fmt"
+	"strings"
+
+	"git.solsynth.dev/hypernet/nexus/pkg/internal/watchtower"
 	"github.com/samber/lo"
 	"github.com/spf13/viper"
-	"strings"
 )
 
 func AllocDatabase(name string) (string, error) {
@@ -34,6 +36,9 @@ func AllocDatabase(name string) (string, error) {
 	}
 
 	connString = append(connString, "dbname="+name)
+	dsn := strings.Join(connString, " ")
 
-	return strings.Join(connString, " "), nil
+	watchtower.AddWatchDb(dsn)
+
+	return dsn, nil
 }

pkg/internal/watchtower/database.go

@@ -0,0 +1,109 @@
+package watchtower
+
+import (
+	"fmt"
+	"os"
+	"os/exec"
+	"path/filepath"
+	"strings"
+	"time"
+
+	"github.com/rs/zerolog/log"
+	"github.com/spf13/viper"
+	"gorm.io/driver/postgres"
+	"gorm.io/gorm"
+)
+
+var dbWatchlist []string
+
+func AddWatchDb(dsn string) {
+	dbWatchlist = append(dbWatchlist, dsn)
+}
+
+func BackupDb() error {
+	backupPath := viper.GetString("watchtower.database_backups")
+	if err := os.MkdirAll(backupPath, 0775); err != nil {
+		return fmt.Errorf("failed to create backup path: %v", err)
+	}
+
+	outFile := filepath.Join(
+		backupPath,
+		fmt.Sprintf("watchtower_db_backup_%s", time.Now().Format("2006-01-02 15:04:05")),
+	)
+
+	var password string
+	var user string
+	var host string
+	var port string
+
+	dsnParts := strings.Split(viper.GetString("database.dsn"), " ")
+	for _, part := range dsnParts {
+		if strings.HasPrefix(part, "password=") {
+			password = strings.Replace(part, "password=", "", 1)
+		} else if strings.HasPrefix(part, "user=") {
+			password = strings.Replace(part, "user=", "", 1)
+		} else if strings.HasPrefix(part, "host=") {
+			host = strings.Replace(part, "host=", "", 1)
+		} else if strings.HasPrefix(part, "port=") {
+			port = strings.Replace(part, "port=", "", 1)
+		}
+	}
+
+	cmd := exec.Command("pg_dumpall",
+		"-h", host,
+		"-p", port,
+		"-U", user,
+		"-f", outFile,
+	)
+	cmd.Env = append(os.Environ(), []string{
+		"PGPASSWORD=" + password,
+	}...)
+
+	start := time.Now()
+	log.Info().Msg("Starting backup database...")
+	if err := cmd.Run(); err != nil {
+		log.Error().Err(err).Msg("Failed to backup the database...")
+		return err
+	}
+	took := time.Since(start)
+	log.Info().Str("out", outFile).Dur("took", took).Msg("Backed up database successfully!")
+
+	return nil
+}
+
+func CleanDb(dsn string) error {
+	conn, err := gorm.Open(postgres.Open(dsn))
+	if err != nil {
+		return fmt.Errorf("failed to open database: %v", err)
+	}
+	var tables []string
+	if err := conn.Raw("SELECT table_name FROM information_schema.tables WHERE table_schema = 'public'").Scan(&tables).Error; err != nil {
+		return fmt.Errorf("failed to scan tables: %v", err)
+	}
+
+	deadline := time.Now().Add(-30 * 24 * time.Hour) // 30 days before
+	for _, table := range tables {
+		sql := fmt.Sprintf("DELETE FROM %s WHERE deleted_at < ?", table)
+		if err := conn.Raw(sql, deadline).Error; err != nil {
+			log.Warn().Err(err).Str("table", table).Str("dsn", dsn).Msg("Unable to clean soft deleted records in this table...")
+		}
+	}
+
+	return nil
+}
+
+func CleanAllDb() {
+	for _, database := range dbWatchlist {
+		if err := CleanDb(database); err != nil {
+			log.Error().Err(err).Msg("Failed to clean up a database...")
+		}
+	}
+}
+
+func RunDbMaintenance() {
+	if err := BackupDb(); err != nil {
+		return
+	}
+	CleanAllDb()
+}
+

pkg/main.go

@@ -2,18 +2,20 @@ package main
 
 import (
 	"fmt"
-	"git.solsynth.dev/hypernet/nexus/pkg/internal/auth"
-	"git.solsynth.dev/hypernet/nexus/pkg/internal/database"
-	"git.solsynth.dev/hypernet/nexus/pkg/internal/directory"
-	"git.solsynth.dev/hypernet/nexus/pkg/internal/http"
-	"git.solsynth.dev/hypernet/nexus/pkg/internal/kv"
-	"git.solsynth.dev/hypernet/nexus/pkg/internal/mq"
-	"git.solsynth.dev/hypernet/nexus/pkg/nex/sec"
-	"github.com/fatih/color"
 	"os"
 	"os/signal"
 	"syscall"
 
+	"git.solsynth.dev/hypernet/nexus/pkg/internal/auth"
+	"git.solsynth.dev/hypernet/nexus/pkg/internal/database"
+	"git.solsynth.dev/hypernet/nexus/pkg/internal/directory"
+	server "git.solsynth.dev/hypernet/nexus/pkg/internal/http"
+	"git.solsynth.dev/hypernet/nexus/pkg/internal/kv"
+	"git.solsynth.dev/hypernet/nexus/pkg/internal/mq"
+	"git.solsynth.dev/hypernet/nexus/pkg/internal/watchtower"
+	"git.solsynth.dev/hypernet/nexus/pkg/nex/sec"
+	"github.com/fatih/color"
+
 	pkg "git.solsynth.dev/hypernet/nexus/pkg/internal"
 	"git.solsynth.dev/hypernet/nexus/pkg/internal/grpc"
 	"github.com/robfig/cron/v3"
@@ -114,6 +116,7 @@ func main() {
 
 	// Configure timed tasks
 	quartz := cron.New(cron.WithLogger(cron.VerbosePrintfLogger(&log.Logger)))
+	quartz.AddFunc("@midnight", watchtower.RunDbMaintenance)
 	quartz.Start()
 
 	// Messages

settings.toml

@@ -20,3 +20,6 @@ endpoints = ["localhost:2379"]
 public_key = "keys/public_key.pem"
 internal_public_key = "keys/internal_public_key.pem"
 internal_private_key = "keys/internal_private_key.pem"
+
+[watchtower]
+database_backups = "./backups"