diff --git a/Dockerfile b/Dockerfile index 0c11bb8..a0b77da 100644 --- a/Dockerfile +++ b/Dockerfile @@ -10,6 +10,8 @@ RUN CGO_ENABLED=0 GOOS=linux GOARCH=amd64 go build -buildvcs -o /dist ./pkg/main # Runtime FROM golang:alpine +RUN apk add postgresql-client + COPY --from=nexus-server /dist /nexus/server EXPOSE 8444 diff --git a/README.md b/README.md index c3703ee..243c3eb 100644 --- a/README.md +++ b/README.md @@ -41,6 +41,8 @@ the establishment of the connection. At the same time, the allocated database will be added into the watchtower for auto maintenance (auto remove the soft-deleted records, backup and more). +Currently, the database is only support postgres, there is no plan for supporting other databases. + ### Authorization All the request forwarded by the Nexus will handle the authorization automatically. diff --git a/ROADMAP.md b/ROADMAP.md deleted file mode 100644 index d071330..0000000 --- a/ROADMAP.md +++ /dev/null @@ -1,16 +0,0 @@ -# Roadmap - -The development progress and plan for Hypernet.Nexus - -- [x] Service discovery -- [x] Command system -- [x] High availability -- [x] Microservice gateway -- [ ] Authenticate (W.I.P) -- [ ] FastLSF (fast lua based serverless function) - -The goal of project Hypernet is going to replace the Hydrogen as Solar Network server-side software. -And the goal of this project is going to replace Hydrogen.Dealer as the core component of Solar Network. - -Other Hydrogen project will be refactored and upgraded to support Nexus as soon as the first stable version is released. -Some features will moved to command based api, such as daily sign in Passport which isn't in Nexus Standard and will be not in it. \ No newline at end of file diff --git a/go.mod b/go.mod index b9ed39d..a538dcd 100644 --- a/go.mod +++ b/go.mod @@ -2,8 +2,6 @@ module git.solsynth.dev/hypernet/nexus go 1.22.0 -toolchain go1.23.2 - require ( github.com/fatih/color v1.18.0 github.com/go-playground/validator/v10 v10.22.1 @@ -18,6 +16,7 @@ require ( github.com/rs/zerolog v1.33.0 github.com/samber/lo v1.47.0 github.com/spf13/viper v1.19.0 + github.com/valyala/fasthttp v1.57.0 go.etcd.io/etcd/client/v3 v3.5.16 google.golang.org/grpc v1.67.1 google.golang.org/protobuf v1.35.1 @@ -71,7 +70,6 @@ require ( github.com/subosito/gotenv v1.6.0 // indirect github.com/tinylib/msgp v1.2.4 // indirect github.com/valyala/bytebufferpool v1.0.0 // indirect - github.com/valyala/fasthttp v1.57.0 // indirect github.com/valyala/tcplisten v1.0.0 // indirect go.etcd.io/etcd/api/v3 v3.5.16 // indirect go.etcd.io/etcd/client/pkg/v3 v3.5.16 // indirect @@ -89,5 +87,3 @@ require ( gopkg.in/yaml.v3 v3.0.1 // indirect gorm.io/driver/mysql v1.5.7 // indirect ) - -replace git.solsynth.dev/hydrogen/bus => ../Bus diff --git a/pkg/internal/database/allocator.go b/pkg/internal/database/allocator.go index b403ba7..8c3a259 100644 --- a/pkg/internal/database/allocator.go +++ b/pkg/internal/database/allocator.go @@ -2,9 +2,11 @@ package database import ( "fmt" + "strings" + + "git.solsynth.dev/hypernet/nexus/pkg/internal/watchtower" "github.com/samber/lo" "github.com/spf13/viper" - "strings" ) func AllocDatabase(name string) (string, error) { @@ -34,6 +36,9 @@ func AllocDatabase(name string) (string, error) { } connString = append(connString, "dbname="+name) + dsn := strings.Join(connString, " ") - return strings.Join(connString, " "), nil + watchtower.AddWatchDb(dsn) + + return dsn, nil } diff --git a/pkg/internal/watchtower/database.go b/pkg/internal/watchtower/database.go new file mode 100644 index 0000000..4dd3e64 --- /dev/null +++ b/pkg/internal/watchtower/database.go @@ -0,0 +1,109 @@ +package watchtower + +import ( + "fmt" + "os" + "os/exec" + "path/filepath" + "strings" + "time" + + "github.com/rs/zerolog/log" + "github.com/spf13/viper" + "gorm.io/driver/postgres" + "gorm.io/gorm" +) + +var dbWatchlist []string + +func AddWatchDb(dsn string) { + dbWatchlist = append(dbWatchlist, dsn) +} + +func BackupDb() error { + backupPath := viper.GetString("watchtower.database_backups") + if err := os.MkdirAll(backupPath, 0775); err != nil { + return fmt.Errorf("failed to create backup path: %v", err) + } + + outFile := filepath.Join( + backupPath, + fmt.Sprintf("watchtower_db_backup_%s", time.Now().Format("2006-01-02 15:04:05")), + ) + + var password string + var user string + var host string + var port string + + dsnParts := strings.Split(viper.GetString("database.dsn"), " ") + for _, part := range dsnParts { + if strings.HasPrefix(part, "password=") { + password = strings.Replace(part, "password=", "", 1) + } else if strings.HasPrefix(part, "user=") { + password = strings.Replace(part, "user=", "", 1) + } else if strings.HasPrefix(part, "host=") { + host = strings.Replace(part, "host=", "", 1) + } else if strings.HasPrefix(part, "port=") { + port = strings.Replace(part, "port=", "", 1) + } + } + + cmd := exec.Command("pg_dumpall", + "-h", host, + "-p", port, + "-U", user, + "-f", outFile, + ) + cmd.Env = append(os.Environ(), []string{ + "PGPASSWORD=" + password, + }...) + + start := time.Now() + log.Info().Msg("Starting backup database...") + if err := cmd.Run(); err != nil { + log.Error().Err(err).Msg("Failed to backup the database...") + return err + } + took := time.Since(start) + log.Info().Str("out", outFile).Dur("took", took).Msg("Backed up database successfully!") + + return nil +} + +func CleanDb(dsn string) error { + conn, err := gorm.Open(postgres.Open(dsn)) + if err != nil { + return fmt.Errorf("failed to open database: %v", err) + } + var tables []string + if err := conn.Raw("SELECT table_name FROM information_schema.tables WHERE table_schema = 'public'").Scan(&tables).Error; err != nil { + return fmt.Errorf("failed to scan tables: %v", err) + } + + deadline := time.Now().Add(-30 * 24 * time.Hour) // 30 days before + for _, table := range tables { + sql := fmt.Sprintf("DELETE FROM %s WHERE deleted_at < ?", table) + if err := conn.Raw(sql, deadline).Error; err != nil { + log.Warn().Err(err).Str("table", table).Str("dsn", dsn).Msg("Unable to clean soft deleted records in this table...") + } + } + + return nil +} + +func CleanAllDb() { + for _, database := range dbWatchlist { + if err := CleanDb(database); err != nil { + log.Error().Err(err).Msg("Failed to clean up a database...") + } + } +} + +func RunDbMaintenance() { + if err := BackupDb(); err != nil { + return + } + CleanAllDb() +} + diff --git a/pkg/main.go b/pkg/main.go index 37d8222..d2597d5 100644 --- a/pkg/main.go +++ b/pkg/main.go @@ -2,18 +2,20 @@ package main import ( "fmt" - "git.solsynth.dev/hypernet/nexus/pkg/internal/auth" - "git.solsynth.dev/hypernet/nexus/pkg/internal/database" - "git.solsynth.dev/hypernet/nexus/pkg/internal/directory" - "git.solsynth.dev/hypernet/nexus/pkg/internal/http" - "git.solsynth.dev/hypernet/nexus/pkg/internal/kv" - "git.solsynth.dev/hypernet/nexus/pkg/internal/mq" - "git.solsynth.dev/hypernet/nexus/pkg/nex/sec" - "github.com/fatih/color" "os" "os/signal" "syscall" + "git.solsynth.dev/hypernet/nexus/pkg/internal/auth" + "git.solsynth.dev/hypernet/nexus/pkg/internal/database" + "git.solsynth.dev/hypernet/nexus/pkg/internal/directory" + server "git.solsynth.dev/hypernet/nexus/pkg/internal/http" + "git.solsynth.dev/hypernet/nexus/pkg/internal/kv" + "git.solsynth.dev/hypernet/nexus/pkg/internal/mq" + "git.solsynth.dev/hypernet/nexus/pkg/internal/watchtower" + "git.solsynth.dev/hypernet/nexus/pkg/nex/sec" + "github.com/fatih/color" + pkg "git.solsynth.dev/hypernet/nexus/pkg/internal" "git.solsynth.dev/hypernet/nexus/pkg/internal/grpc" "github.com/robfig/cron/v3" @@ -114,6 +116,7 @@ func main() { // Configure timed tasks quartz := cron.New(cron.WithLogger(cron.VerbosePrintfLogger(&log.Logger))) + quartz.AddFunc("@midnight", watchtower.RunDbMaintenance) quartz.Start() // Messages diff --git a/settings.toml b/settings.toml index 54c557e..2ebb62a 100644 --- a/settings.toml +++ b/settings.toml @@ -20,3 +20,6 @@ endpoints = ["localhost:2379"] public_key = "keys/public_key.pem" internal_public_key = "keys/internal_public_key.pem" internal_private_key = "keys/internal_private_key.pem" + +[watchtower] +database_backups = "./backups"