♻️ Refactored tracing system to cost less memory

pkg/cmd/main.go

@@ -44,6 +44,9 @@ func main() {
 		log.Warn().Msgf("RoadSign auto generated api credential is %s", credential)
 	}
 
+	// Initialize access logging
+	navi.InitializeLogging()
+
 	// Load & init navigator
 	if err := navi.ReadInConfig(viper.GetString("paths.configs")); err != nil {
 		log.Panic().Err(err).Msg("An error occurred when loading configurations.")

pkg/navi/config.go

@@ -16,9 +16,7 @@ func ReadInConfig(root string) error {
 	instance := &RoadApp{
 		Regions: make([]*Region, 0),
 		Metrics: &RoadMetrics{
-			Traces:       make([]RoadTrace, 0),
 			Traffic:      make(map[string]int64),
-			TrafficFrom:  make(map[string]int64),
 			TotalTraffic: 0,
 			StartupAt:    time.Now(),
 		},

pkg/navi/logging.go

@@ -0,0 +1,20 @@
+package navi
+
+import (
+	"log"
+
+	"github.com/spf13/viper"
+	"gopkg.in/natefinch/lumberjack.v2"
+)
+
+var accessLogger *log.Logger
+
+func InitializeLogging() {
+	accessLogger = log.New(&lumberjack.Logger{
+		Filename:   viper.GetString("logging.access"),
+		MaxSize:    10,
+		MaxBackups: 3,
+		MaxAge:     30,
+		Compress:   true,
+	}, "", 0)
+}

pkg/navi/metrics.go

@@ -1,15 +1,16 @@
 package navi
 
 import (
-	"github.com/spf13/viper"
+	"bufio"
+	"os"
 	"time"
+
+	jsoniter "github.com/json-iterator/go"
+	"github.com/spf13/viper"
 )
 
 type RoadMetrics struct {
-	Traces []RoadTrace `json:"-"`
-
 	Traffic      map[string]int64 `json:"traffic"`
-	TrafficFrom  map[string]int64 `json:"traffic_from"`
 	TotalTraffic int64            `json:"total_traffic"`
 	StartupAt    time.Time        `json:"startup_at"`
 }
@@ -42,22 +43,28 @@ func (v *RoadMetrics) AddTrace(trace RoadTrace) {
 	} else {
 		v.Traffic[trace.Region]++
 	}
-	if _, ok := v.TrafficFrom[trace.IpAddress]; !ok {
-		v.TrafficFrom[trace.IpAddress] = 0
-	} else {
-		v.TrafficFrom[trace.IpAddress]++
-	}
 
-	v.Traces = append(v.Traces, trace)
-
-	// Garbage recycle
-	if len(v.Traffic) > viper.GetInt("performance.traces_limit") {
-		clear(v.Traffic)
-	}
-	if len(v.TrafficFrom) > viper.GetInt("performance.traces_limit") {
-		clear(v.TrafficFrom)
-	}
-	if len(v.Traces) > viper.GetInt("performance.traces_limit") {
-		clear(v.Traces)
-	}
+	raw, _ := jsoniter.Marshal(trace)
+	accessLogger.Println(string(raw))
+}
+
+func (v *RoadMetrics) ReadTrace() []RoadTrace {
+	fp := viper.GetString("logging.access")
+	file, err := os.Open(fp)
+	if err != nil {
+		return nil
+	}
+	defer file.Close()
+
+	var out []RoadTrace
+	scanner := bufio.NewScanner(file)
+	for scanner.Scan() {
+		line := scanner.Text()
+		var entry RoadTrace
+		if err := jsoniter.Unmarshal([]byte(line), &entry); err == nil {
+			out = append(out, entry)
+		}
+	}
+
+	return out
 }

pkg/sideload/metrics.go

@@ -10,5 +10,5 @@ func getTraffic(c *fiber.Ctx) error {
 }
 
 func getTraces(c *fiber.Ctx) error {
-	return c.JSON(navi.R.Metrics.Traces)
+	return c.JSON(navi.R.Metrics.ReadTrace())
 }

pkg/sideload/statistics.go

@@ -1,11 +1,12 @@
 package sideload
 
 import (
+	"time"
+
 	"git.solsynth.dev/goatworks/roadsign/pkg/navi"
 	"git.solsynth.dev/goatworks/roadsign/pkg/warden"
 	"github.com/gofiber/fiber/v2"
 	"github.com/samber/lo"
-	"time"
 )
 
 func getStats(c *fiber.Ctx) error {
@@ -26,8 +27,7 @@ func getStats(c *fiber.Ctx) error {
 		"applications": len(applications),
 		"uptime":       time.Since(navi.R.Metrics.StartupAt).Milliseconds(),
 		"traffic": fiber.Map{
-			"total":         navi.R.Metrics.TotalTraffic,
-			"unique_client": len(navi.R.Metrics.TrafficFrom),
+			"total": navi.R.Metrics.TotalTraffic,
 		},
 	})
 }