🔀 Merge pull request '🔥 重构 YAML 配置文件' (#2) from refactor/new-configuration into master

Reviewed-on: https://code.smartsheep.studio/Goatworks/RoadSign/pulls/2

Dockerfile

@@ -3,7 +3,7 @@ FROM golang:alpine as roadsign-server
 
 WORKDIR /source
 COPY . .
-RUN CGO_ENABLED=0 GOOS=linux GOARCH=amd64 go build -o /dist ./pkg/cmd/main.go
+RUN CGO_ENABLED=0 GOOS=linux GOARCH=amd64 go build -o /dist ./pkg/cmd/server/main.go
 
 # Runtime
 FROM golang:alpine

README.md

@@ -23,7 +23,7 @@ Here's the result:
 |:---------------------:|:--------------:|:--------------------:|:--------------------:|:-----------:|:------------:|:------------:|:------------:|
 |        _Nginx_        |     515749     |        4299.58       |        2.05MB        | 13.954846ms |      0s (Cached)      |  410.6972ms  |       0      |
 |       _RoadSign_      |     8905230    |       76626.70       | 30.98MB       |  783.016µs  |      28.542µs      |   46.773083ms  |       0      |
-| _RoadSign w/ Prefork_ | 4784308    |       40170.41       |        16.24MB        | 1.493636ms |      34.291µs      |  8.727666ms  |       0      |
+| _RoadSign w/ Prefork_ |     4784308    |       40170.41       |        16.24MB        | 1.493636ms |      34.291µs      |  8.727666ms  |       0      |
 
 As result, roadsign undoubtedly is the fastest one.
 

go.mod

@@ -40,5 +40,6 @@ require (
 	golang.org/x/sys v0.14.0 // indirect
 	golang.org/x/text v0.13.0 // indirect
 	gopkg.in/ini.v1 v1.67.0 // indirect
+	gopkg.in/yaml.v2 v2.4.0
 	gopkg.in/yaml.v3 v3.0.1 // indirect
 )

go.sum

@@ -589,6 +589,8 @@ gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI=
 gopkg.in/ini.v1 v1.67.0 h1:Dgnx+6+nfE+IfzjUEISNeydPJh9AXNNsWbGP9KzCsOA=
 gopkg.in/ini.v1 v1.67.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k=
 gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
+gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY=
+gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ=
 gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
 gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=
 gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=

pkg/administration/publish.go

@@ -1,12 +1,13 @@
 package administration
 
 import (
-	"code.smartsheep.studio/goatworks/roadsign/pkg/fs"
+	"os"
+	"path/filepath"
+
+	"code.smartsheep.studio/goatworks/roadsign/pkg/filesystem"
 	"code.smartsheep.studio/goatworks/roadsign/pkg/sign"
 	"github.com/gofiber/fiber/v2"
 	"github.com/google/uuid"
-	"os"
-	"path/filepath"
 )
 
 func doPublish(ctx *fiber.Ctx) error {
@@ -15,7 +16,7 @@ func doPublish(ctx *fiber.Ctx) error {
 		if item.ID == ctx.Params("site") {
 			for _, stream := range item.Upstreams {
 				if stream.ID == ctx.Params("upstream") {
-					upstream = &stream
+					upstream = stream
 					break
 				}
 			}
@@ -48,7 +49,7 @@ func doPublish(ctx *fiber.Ctx) error {
 				if err := ctx.SaveFile(file, dst); err != nil {
 					return err
 				} else {
-					_ = fs.Unzip(dst, workdir)
+					_ = filesystem.Unzip(dst, workdir)
 				}
 			default:
 				dst := filepath.Join(workdir, file.Filename)

pkg/cmd/server/main.go

@@ -1,6 +1,11 @@
 package main
 
 import (
+	"os"
+	"os/signal"
+	"strings"
+	"syscall"
+
 	roadsign "code.smartsheep.studio/goatworks/roadsign/pkg"
 	"code.smartsheep.studio/goatworks/roadsign/pkg/administration"
 	"code.smartsheep.studio/goatworks/roadsign/pkg/hypertext"
@@ -9,10 +14,6 @@ import (
 	"github.com/rs/zerolog"
 	"github.com/rs/zerolog/log"
 	"github.com/spf13/viper"
-	"os"
-	"os/signal"
-	"strings"
-	"syscall"
 )
 
 func init() {
@@ -46,7 +47,7 @@ func main() {
 	if err := sign.ReadInConfig(viper.GetString("paths.configs")); err != nil {
 		log.Panic().Err(err).Msg("An error occurred when loading configurations.")
 	} else {
-		log.Debug().Any("sites", sign.App).Msg("All configuration has been loaded.")
+		log.Info().Int("count", len(sign.App.Sites)).Msg("All configuration has been loaded.")
 	}
 
 	// Init hypertext server
@@ -57,7 +58,7 @@ func main() {
 		viper.GetString("hypertext.certificate.pem"),
 		viper.GetString("hypertext.certificate.key"),
 	)
-	
+
 	// Init administration server
 	hypertext.RunServer(
 		administration.InitAdministration(),

pkg/filesystem/zip.go

@@ -1,4 +1,4 @@
-package fs
+package filesystem
 
 import (
 	"archive/zip"

pkg/hypertext/proxies.go

@@ -1,10 +1,11 @@
 package hypertext
 
 import (
+	"regexp"
+
 	"code.smartsheep.studio/goatworks/roadsign/pkg/sign"
 	"github.com/gofiber/fiber/v2"
 	"github.com/samber/lo"
-	"regexp"
 )
 
 func UseProxies(app *fiber.App) {
@@ -88,7 +89,7 @@ func UseProxies(app *fiber.App) {
 	})
 }
 
-func makeResponse(ctx *fiber.Ctx, site sign.SiteConfig) error {
+func makeResponse(ctx *fiber.Ctx, site *sign.SiteConfig) error {
 	// Modify request
 	for _, transformer := range site.Transformers {
 		transformer.TransformRequest(ctx)

pkg/sign/configurator.go

@@ -1,18 +1,19 @@
 package sign
 
 import (
-	"encoding/json"
 	"io"
 	"os"
 	"path/filepath"
 	"strings"
+
+	"gopkg.in/yaml.v2"
 )
 
 var App *AppConfig
 
 func ReadInConfig(root string) error {
 	cfg := &AppConfig{
-		Sites: []SiteConfig{},
+		Sites: []*SiteConfig{},
 	}
 
 	if err := filepath.Walk(root, func(fp string, info os.FileInfo, err error) error {
@@ -23,13 +24,13 @@ func ReadInConfig(root string) error {
 			return err
 		} else if data, err := io.ReadAll(file); err != nil {
 			return err
-		} else if err := json.Unmarshal(data, &site); err != nil {
+		} else if err := yaml.Unmarshal(data, &site); err != nil {
 			return err
 		} else {
 			// Extract file name as site id
 			site.ID = strings.SplitN(filepath.Base(fp), ".", 2)[0]
 
-			cfg.Sites = append(cfg.Sites, site)
+			cfg.Sites = append(cfg.Sites, &site)
 		}
 
 		return nil
@@ -44,7 +45,7 @@ func ReadInConfig(root string) error {
 
 func SaveInConfig(root string, cfg *AppConfig) error {
 	for _, site := range cfg.Sites {
-		data, _ := json.Marshal(site)
+		data, _ := yaml.Marshal(site)
 
 		fp := filepath.Join(root, site.ID)
 		if file, err := os.OpenFile(fp, os.O_RDWR|os.O_CREATE|os.O_TRUNC, 0755); err != nil {

pkg/sign/pm.go

@@ -0,0 +1,72 @@
+package sign
+
+import (
+	"fmt"
+	"os"
+	"os/exec"
+	"path/filepath"
+)
+
+type ProcessConfig struct {
+	ID       string     `json:"id" yaml:"id"`
+	Workdir  string     `json:"workdir" yaml:"workdir"`
+	Command  []string   `json:"command" yaml:"command"`
+	Prepares [][]string `json:"prepares" yaml:"prepares"`
+
+	Cmd *exec.Cmd `json:"-"`
+}
+
+func (v *ProcessConfig) BootProcess() error {
+	if v.Cmd != nil {
+		return nil
+	}
+
+	if err := v.PreapreProcess(); err != nil {
+		return err
+	}
+	if v.Cmd == nil {
+		return v.StartProcess()
+	}
+	if v.Cmd.Process == nil || v.Cmd.ProcessState == nil {
+		return v.StartProcess()
+	}
+	if v.Cmd.ProcessState.Exited() {
+		return v.StartProcess()
+	} else if v.Cmd.ProcessState.Exited() {
+		return fmt.Errorf("process already dead")
+	}
+	if v.Cmd.ProcessState.Exited() {
+		return fmt.Errorf("cannot start process")
+	} else {
+		return nil
+	}
+}
+
+func (v *ProcessConfig) PreapreProcess() error {
+	for _, script := range v.Prepares {
+		if len(script) <= 0 {
+			continue
+		}
+		cmd := exec.Command(script[0], script[1:]...)
+		cmd.Dir = filepath.Join(v.Workdir)
+		if err := cmd.Run(); err != nil {
+			return err
+		}
+	}
+	return nil
+}
+
+func (v *ProcessConfig) StartProcess() error {
+	if len(v.Command) <= 0 {
+		return fmt.Errorf("you need set the command for %s to enable process manager", v.ID)
+	}
+
+	v.Cmd = exec.Command(v.Command[0], v.Command[1:]...)
+	v.Cmd.Dir = filepath.Join(v.Workdir)
+
+	return v.Cmd.Start()
+}
+
+func (v *ProcessConfig) StopProcess() error {
+	return v.Cmd.Process.Signal(os.Interrupt)
+}

pkg/sign/router.go

@@ -5,19 +5,29 @@ import (
 	"math/rand"
 
 	"github.com/gofiber/fiber/v2"
+	"github.com/rs/zerolog/log"
 )
 
 type AppConfig struct {
-	Sites []SiteConfig `json:"sites"`
+	Sites []*SiteConfig `json:"sites"`
 }
 
-func (v *AppConfig) Forward(ctx *fiber.Ctx, site SiteConfig) error {
+func (v *AppConfig) Forward(ctx *fiber.Ctx, site *SiteConfig) error {
 	if len(site.Upstreams) == 0 {
 		return errors.New("invalid configuration")
 	}
 
+	// Boot processes
+	for _, process := range site.Processes {
+		if err := process.BootProcess(); err != nil {
+			log.Warn().Err(err).Msgf("An error occurred when booting process (%s) for %s", process.ID, site.ID)
+			return fiber.ErrBadGateway
+		}
+	}
+
+	// Do forward
 	idx := rand.Intn(len(site.Upstreams))
-	upstream := &site.Upstreams[idx]
+	upstream := site.Upstreams[idx]
 
 	switch upstream.GetType() {
 	case UpstreamTypeHypertext:
@@ -30,15 +40,16 @@ func (v *AppConfig) Forward(ctx *fiber.Ctx, site SiteConfig) error {
 }
 
 type SiteConfig struct {
-	ID           string                     `json:"id"`
-	Rules        []RouterRuleConfig         `json:"rules"`
-	Transformers []RequestTransformerConfig `json:"transformers"`
-	Upstreams    []UpstreamConfig           `json:"upstreams"`
+	ID           string                      `json:"id"`
+	Rules        []*RouterRuleConfig         `json:"rules" yaml:"rules"`
+	Transformers []*RequestTransformerConfig `json:"transformers" yaml:"transformers"`
+	Upstreams    []*UpstreamConfig           `json:"upstreams" yaml:"upstreams"`
+	Processes    []*ProcessConfig            `json:"processes" yaml:"processes"`
 }
 
 type RouterRuleConfig struct {
-	Host    []string            `json:"host"`
-	Path    []string            `json:"path"`
-	Queries map[string]string   `json:"query"`
-	Headers map[string][]string `json:"headers"`
+	Host    []string            `json:"host" yaml:"host"`
+	Path    []string            `json:"path" yaml:"path"`
+	Queries map[string]string   `json:"queries" yaml:"queries"`
+	Headers map[string][]string `json:"headers" yaml:"headers"`
 }

pkg/sign/transformer.go

@@ -1,9 +1,10 @@
 package sign
 
 import (
-	"github.com/gofiber/fiber/v2"
 	"regexp"
 	"strings"
+
+	"github.com/gofiber/fiber/v2"
 )
 
 type RequestTransformer struct {
@@ -12,8 +13,8 @@ type RequestTransformer struct {
 }
 
 type RequestTransformerConfig struct {
-	Type    string `json:"type"`
-	Options any    `json:"options"`
+	Type    string `json:"type" yaml:"type"`
+	Options any    `json:"options" yaml:"options"`
 }
 
 func (v *RequestTransformerConfig) TransformRequest(ctx *fiber.Ctx) {

pkg/sign/upstream.go

@@ -2,10 +2,11 @@ package sign
 
 import (
 	"fmt"
-	"github.com/gofiber/fiber/v2"
-	"github.com/samber/lo"
 	"net/url"
 	"strings"
+
+	"github.com/gofiber/fiber/v2"
+	"github.com/samber/lo"
 )
 
 const (
@@ -15,18 +16,16 @@ const (
 )
 
 type UpstreamConfig struct {
-	ID  string `json:"id"`
-	URI string `json:"uri"`
+	ID  string `json:"id" yaml:"id"`
+	URI string `json:"uri" yaml:"uri"`
 }
 
 func (v *UpstreamConfig) GetType() string {
 	protocol := strings.SplitN(v.URI, "://", 2)[0]
 	switch protocol {
-	case "file":
-	case "files":
+	case "file", "files":
 		return UpstreamTypeFile
-	case "http":
-	case "https":
+	case "http", "https":
 		return UpstreamTypeHypertext
 	}
 

test/benchmark/data/.gitignore

@@ -0,0 +1 @@
+/.output

test/benchmark/roadsign-ssr/config/example.yaml

@@ -0,0 +1,12 @@
+name: Example Site
+rules:
+  - host: ["localhost:8000"]
+    path: ["/"]
+upstreams:
+  - id: example
+    name: Benchmarking Data
+    uri: http://localhost:3000
+processes:
+  - id: nuxt-ssr
+    workdir: ../data
+    command: ["node", ".output/server/index.mjs"]

test/benchmark/roadsign-ssr/settings.yml

@@ -0,0 +1,26 @@
+debug:
+    print_routes: false
+hypertext:
+    administration_ports: []
+    administration_secured_ports: []
+    certificate:
+        administration_key: ./cert.key
+        administration_pem: ./cert.pem
+        key: ./cert.key
+        pem: ./cert.pem
+    limitation:
+        max_body_size: -1
+        max_qps: -1
+    ports:
+        - :8000
+    secured_ports: []
+paths:
+    configs: ./config
+performance:
+    request_logging: false
+    network_timeout: 3000
+    prefork: false
+security:
+    administration_trusted_proxies:
+        - localhost
+    credential: e81f43f32d934271af6322e5376f5f59

test/benchmark/roadsign-with-prefork/config/example.json

@@ -1,20 +0,0 @@
-{
-  "name": "Example Site",
-  "rules": [
-    {
-      "host": [
-        "localhost:8000"
-      ],
-      "path": [
-        "/"
-      ]
-    }
-  ],
-  "upstreams": [
-    {
-      "id": "example",
-      "name": "Benchmarking Data",
-      "uri": "files://../data"
-    }
-  ]
-}

test/benchmark/roadsign-with-prefork/config/example.yaml

@@ -0,0 +1,8 @@
+name: Example Site
+rules:
+  - host: ["localhost:8000"]
+    path: ["/"]
+upstreams:
+  - id: example
+    name: Benchmarking Data
+    uri: files://../data

test/benchmark/roadsign/config/example.json

@@ -1,20 +0,0 @@
-{
-  "name": "Example Site",
-  "rules": [
-    {
-      "host": [
-        "localhost:8000"
-      ],
-      "path": [
-        "/"
-      ]
-    }
-  ],
-  "upstreams": [
-    {
-      "id": "example",
-      "name": "Benchmarking Data",
-      "uri": "files://../data"
-    }
-  ]
-}

test/benchmark/roadsign/config/example.yaml

@@ -0,0 +1,8 @@
+name: Example Site
+rules:
+  - host: ["localhost:8000"]
+    path: ["/"]
+upstreams:
+  - id: example
+    name: Benchmarking Data
+    uri: files://../data