From ee8b7e5660af9f48c5f71ab37e16e37b13640e56 Mon Sep 17 00:00:00 2001 From: LittleSheep Date: Tue, 14 Jan 2025 17:22:23 +0800 Subject: [PATCH] :recycle: Refactored tracing system to cost less memory --- .gitignore | 3 ++- Dockerfile | 4 +-- config/example.toml | 7 +++--- go.mod | 1 + go.sum | 2 ++ pkg/cmd/{server => }/main.go | 3 +++ pkg/navi/config.go | 2 -- pkg/navi/logging.go | 20 +++++++++++++++ pkg/navi/metrics.go | 49 ++++++++++++++++++++---------------- pkg/sideload/metrics.go | 2 +- pkg/sideload/statistics.go | 6 ++--- settings.toml | 6 +++-- 12 files changed, 70 insertions(+), 35 deletions(-) rename pkg/cmd/{server => }/main.go (97%) create mode 100644 pkg/navi/logging.go diff --git a/.gitignore b/.gitignore index bd0ac76..6387d77 100644 --- a/.gitignore +++ b/.gitignore @@ -1,5 +1,6 @@ /letsencrypt /certs /dist +/logs -.DS_Store \ No newline at end of file +.DS_Store diff --git a/Dockerfile b/Dockerfile index ad20bfa..2b028cb 100644 --- a/Dockerfile +++ b/Dockerfile @@ -3,7 +3,7 @@ FROM golang:alpine as roadsign-server WORKDIR /source COPY . . -RUN CGO_ENABLED=0 GOOS=linux GOARCH=amd64 go build -buildvcs -o /dist ./pkg/cmd/server/main.go +RUN CGO_ENABLED=0 GOOS=linux GOARCH=amd64 go build -buildvcs -o /dist ./pkg/cmd/main.go # Runtime FROM golang:alpine @@ -14,4 +14,4 @@ COPY --from=roadsign-server /dist /roadsign/server EXPOSE 81 -CMD ["/roadsign/server"] \ No newline at end of file +CMD ["/roadsign/server"] diff --git a/config/example.toml b/config/example.toml index 9117a1c..c262346 100644 --- a/config/example.toml +++ b/config/example.toml @@ -24,9 +24,10 @@ id = "example-region" [[locations]] id = "example" -host = ["localhost:8000"] -path = ["/"] +hosts = ["localhost:8000"] +paths = ["/"] [[locations.destinations]] id = "example-destination" uri = "https://example.com" -helmet = { x_frame_options = "SAMEORIGIN" } \ No newline at end of file +helmet = { x_frame_options = "SAMEORIGIN" } + diff --git a/go.mod b/go.mod index 1281482..c7ec081 100644 --- a/go.mod +++ b/go.mod @@ -28,6 +28,7 @@ require ( github.com/xrash/smetrics v0.0.0-20201216005158-039620a65673 // indirect golang.org/x/net v0.29.0 // indirect golang.org/x/sync v0.8.0 // indirect + gopkg.in/natefinch/lumberjack.v2 v2.2.1 // indirect ) require ( diff --git a/go.sum b/go.sum index 3e46750..416f3f1 100644 --- a/go.sum +++ b/go.sum @@ -169,6 +169,8 @@ gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15 h1:YR8cESwS4TdDjEe65xsg0ogR gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/ini.v1 v1.67.0 h1:Dgnx+6+nfE+IfzjUEISNeydPJh9AXNNsWbGP9KzCsOA= gopkg.in/ini.v1 v1.67.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k= +gopkg.in/natefinch/lumberjack.v2 v2.2.1 h1:bBRl1b0OH9s/DuPhuXpNl+VtCaJXFZ5/uEFST95x9zc= +gopkg.in/natefinch/lumberjack.v2 v2.2.1/go.mod h1:YD8tP3GAjkrDg1eZH7EGmyESg/lsYskCTPBJVb9jqSc= gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= diff --git a/pkg/cmd/server/main.go b/pkg/cmd/main.go similarity index 97% rename from pkg/cmd/server/main.go rename to pkg/cmd/main.go index 9238e01..5649f2a 100644 --- a/pkg/cmd/server/main.go +++ b/pkg/cmd/main.go @@ -44,6 +44,9 @@ func main() { log.Warn().Msgf("RoadSign auto generated api credential is %s", credential) } + // Initialize access logging + navi.InitializeLogging() + // Load & init navigator if err := navi.ReadInConfig(viper.GetString("paths.configs")); err != nil { log.Panic().Err(err).Msg("An error occurred when loading configurations.") diff --git a/pkg/navi/config.go b/pkg/navi/config.go index 53de8ce..25ed872 100644 --- a/pkg/navi/config.go +++ b/pkg/navi/config.go @@ -16,9 +16,7 @@ func ReadInConfig(root string) error { instance := &RoadApp{ Regions: make([]*Region, 0), Metrics: &RoadMetrics{ - Traces: make([]RoadTrace, 0), Traffic: make(map[string]int64), - TrafficFrom: make(map[string]int64), TotalTraffic: 0, StartupAt: time.Now(), }, diff --git a/pkg/navi/logging.go b/pkg/navi/logging.go new file mode 100644 index 0000000..b5e27c7 --- /dev/null +++ b/pkg/navi/logging.go @@ -0,0 +1,20 @@ +package navi + +import ( + "log" + + "github.com/spf13/viper" + "gopkg.in/natefinch/lumberjack.v2" +) + +var accessLogger *log.Logger + +func InitializeLogging() { + accessLogger = log.New(&lumberjack.Logger{ + Filename: viper.GetString("logging.access"), + MaxSize: 10, + MaxBackups: 3, + MaxAge: 30, + Compress: true, + }, "", 0) +} diff --git a/pkg/navi/metrics.go b/pkg/navi/metrics.go index c7c7857..f80e723 100644 --- a/pkg/navi/metrics.go +++ b/pkg/navi/metrics.go @@ -1,15 +1,16 @@ package navi import ( - "github.com/spf13/viper" + "bufio" + "os" "time" + + jsoniter "github.com/json-iterator/go" + "github.com/spf13/viper" ) type RoadMetrics struct { - Traces []RoadTrace `json:"-"` - Traffic map[string]int64 `json:"traffic"` - TrafficFrom map[string]int64 `json:"traffic_from"` TotalTraffic int64 `json:"total_traffic"` StartupAt time.Time `json:"startup_at"` } @@ -42,22 +43,28 @@ func (v *RoadMetrics) AddTrace(trace RoadTrace) { } else { v.Traffic[trace.Region]++ } - if _, ok := v.TrafficFrom[trace.IpAddress]; !ok { - v.TrafficFrom[trace.IpAddress] = 0 - } else { - v.TrafficFrom[trace.IpAddress]++ - } - v.Traces = append(v.Traces, trace) - - // Garbage recycle - if len(v.Traffic) > viper.GetInt("performance.traces_limit") { - clear(v.Traffic) - } - if len(v.TrafficFrom) > viper.GetInt("performance.traces_limit") { - clear(v.TrafficFrom) - } - if len(v.Traces) > viper.GetInt("performance.traces_limit") { - clear(v.Traces) - } + raw, _ := jsoniter.Marshal(trace) + accessLogger.Println(string(raw)) +} + +func (v *RoadMetrics) ReadTrace() []RoadTrace { + fp := viper.GetString("logging.access") + file, err := os.Open(fp) + if err != nil { + return nil + } + defer file.Close() + + var out []RoadTrace + scanner := bufio.NewScanner(file) + for scanner.Scan() { + line := scanner.Text() + var entry RoadTrace + if err := jsoniter.Unmarshal([]byte(line), &entry); err == nil { + out = append(out, entry) + } + } + + return out } diff --git a/pkg/sideload/metrics.go b/pkg/sideload/metrics.go index 9b46bc7..5374fda 100644 --- a/pkg/sideload/metrics.go +++ b/pkg/sideload/metrics.go @@ -10,5 +10,5 @@ func getTraffic(c *fiber.Ctx) error { } func getTraces(c *fiber.Ctx) error { - return c.JSON(navi.R.Metrics.Traces) + return c.JSON(navi.R.Metrics.ReadTrace()) } diff --git a/pkg/sideload/statistics.go b/pkg/sideload/statistics.go index 89c99d7..fac6418 100644 --- a/pkg/sideload/statistics.go +++ b/pkg/sideload/statistics.go @@ -1,11 +1,12 @@ package sideload import ( + "time" + "git.solsynth.dev/goatworks/roadsign/pkg/navi" "git.solsynth.dev/goatworks/roadsign/pkg/warden" "github.com/gofiber/fiber/v2" "github.com/samber/lo" - "time" ) func getStats(c *fiber.Ctx) error { @@ -26,8 +27,7 @@ func getStats(c *fiber.Ctx) error { "applications": len(applications), "uptime": time.Since(navi.R.Metrics.StartupAt).Milliseconds(), "traffic": fiber.Map{ - "total": navi.R.Metrics.TotalTraffic, - "unique_client": len(navi.R.Metrics.TrafficFrom), + "total": navi.R.Metrics.TotalTraffic, }, }) } diff --git a/settings.toml b/settings.toml index 680a771..794ac6e 100644 --- a/settings.toml +++ b/settings.toml @@ -21,6 +21,9 @@ force_https = false max_body_size = 549_755_813_888 # 512 GiB max_qps = -1 +[logging] +access = "./logs/access.log" + [paths] configs = "./config" @@ -29,8 +32,7 @@ request_logging = true capture_traces = true [performance] -low_memory = true -traces_limit = 256 +low_memory = false prefork = false [security]