Compare commits

...

12 Commits

27 changed files with 217 additions and 93 deletions

3
.gitignore vendored
View File

@ -1,5 +1,6 @@
/letsencrypt
/certs
/dist
/logs
.DS_Store
.DS_Store

View File

@ -3,15 +3,16 @@ FROM golang:alpine as roadsign-server
WORKDIR /source
COPY . .
RUN CGO_ENABLED=0 GOOS=linux GOARCH=amd64 go build -buildvcs -o /dist ./pkg/cmd/server/main.go
RUN CGO_ENABLED=0 GOOS=linux GOARCH=amd64 go build -buildvcs -o /dist ./pkg/cmd/main.go
# Runtime
FROM golang:alpine
RUN apk add zip
RUN apk add nodejs npm
COPY --from=roadsign-server /dist /roadsign/server
EXPOSE 81
CMD ["/roadsign/server"]
CMD ["/roadsign/server"]

View File

@ -1,7 +1,7 @@
{
"name": "roadsign-cli",
"module": "index.ts",
"version": "1.0.0",
"version": "1.0.2",
"repository": "https://github.com/solsynth/roadsign",
"type": "module",
"scripts": {
@ -33,4 +33,4 @@
"figlet": "^1.7.0",
"ora": "^8.1.0"
}
}
}

View File

@ -6,7 +6,8 @@ import * as fs from "node:fs"
import * as child_process from "node:child_process"
import * as path from "node:path"
import { createAuthHeader } from "../utils/auth.ts"
import { RsLocalConfig } from "../utils/config-local.ts"
import { RsLocalConfig, type RsLocalConfigDeploymentPostActionData } from "../utils/config-local.ts"
import * as os from "node:os"
export class DeployCommand extends Command {
static paths = [[`deploy`]]
@ -25,7 +26,7 @@ export class DeployCommand extends Command {
site = Option.String({ required: false })
input = Option.String({ required: false })
async deploy(serverLabel: string, region: string, site: string, input: string) {
async deploy(serverLabel: string, region: string, site: string, input: string, postDeploy: RsLocalConfigDeploymentPostActionData | null = null) {
const cfg = await RsConfig.getInstance()
const server = cfg.config.servers.find(item => item.label === serverLabel)
if (server == null) {
@ -40,12 +41,10 @@ export class DeployCommand extends Command {
let isDirectory = false
if (fs.statSync(input).isDirectory()) {
input = path.join(input, "*")
const compressPrefStart = performance.now()
const compressSpinner = ora(`Compressing ${chalk.bold(input)}...`).start()
const destName = `${Date.now()}-roadsign-archive.zip`
child_process.execSync(`zip -rj ${destName} ${input}`)
const destName = path.join(os.tmpdir(), `${Date.now()}-roadsign-archive.zip`)
child_process.execSync(`cd ${input} && zip -r ${destName} .`)
const compressPrefTook = performance.now() - compressPrefStart
compressSpinner.succeed(`Compressing completed in ${(compressPrefTook / 1000).toFixed(2)}s 🎉`)
input = destName
@ -60,6 +59,18 @@ export class DeployCommand extends Command {
try {
const payload = new FormData()
payload.set("attachments", await fs.openAsBlob(input), isDirectory ? "dist.zip" : path.basename(input))
if(postDeploy) {
if(postDeploy.command) {
payload.set("post-deploy-script", postDeploy.command)
} else if(postDeploy.scriptPath) {
payload.set("post-deploy-script", fs.readFileSync(postDeploy.scriptPath, "utf8"))
} else {
this.context.stdout.write(chalk.yellow(`Configured post deploy action but no script provided, skip performing post deploy action...\n`))
}
payload.set("post-deploy-environment", postDeploy.environment?.join("\n") ?? "")
}
const res = await fetch(`${server.url}/webhooks/publish/${region}/${site}?mimetype=application/zip`, {
method: "PUT",
body: payload,
@ -102,7 +113,7 @@ export class DeployCommand extends Command {
let idx = 0
for (const deployment of localCfg.config.deployments ?? []) {
this.context.stdout.write(chalk.cyan(`Deploying ${idx + 1} out of ${localCfg.config.deployments.length} deployments...\n`))
await this.deploy(this.server, deployment.region, deployment.site, deployment.path)
await this.deploy(this.server, deployment.region, deployment.site, deployment.path, deployment.postDeploy)
}
this.context.stdout.write(chalk.green(`All deployments has been deployed!\n`))

View File

@ -15,6 +15,7 @@ interface RsLocalConfigDeploymentData {
path: string
region: string
site: string
postDeploy?: RsLocalConfigDeploymentPostActionData
autoBuild?: RsLocalConfigDeploymentAutoBuildData
}
@ -23,6 +24,12 @@ interface RsLocalConfigDeploymentAutoBuildData {
environment?: string[]
}
interface RsLocalConfigDeploymentPostActionData {
command?: string
scriptPath?: string
environment?: string[]
}
class RsLocalConfig {
private static instance: RsLocalConfig
@ -57,4 +64,4 @@ class RsLocalConfig {
}
}
export { RsLocalConfig, type RsLocalConfigData }
export { RsLocalConfig, type RsLocalConfigData, type RsLocalConfigDeploymentPostActionData }

View File

@ -24,9 +24,10 @@ id = "example-region"
[[locations]]
id = "example"
host = ["localhost:8000"]
path = ["/"]
hosts = ["localhost:8000"]
paths = ["/"]
[[locations.destinations]]
id = "example-destination"
uri = "https://example.com"
helmet = { x_frame_options = "SAMEORIGIN" }
helmet = { x_frame_options = "SAMEORIGIN" }

1
go.mod
View File

@ -28,6 +28,7 @@ require (
github.com/xrash/smetrics v0.0.0-20201216005158-039620a65673 // indirect
golang.org/x/net v0.29.0 // indirect
golang.org/x/sync v0.8.0 // indirect
gopkg.in/natefinch/lumberjack.v2 v2.2.1 // indirect
)
require (

2
go.sum
View File

@ -169,6 +169,8 @@ gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15 h1:YR8cESwS4TdDjEe65xsg0ogR
gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
gopkg.in/ini.v1 v1.67.0 h1:Dgnx+6+nfE+IfzjUEISNeydPJh9AXNNsWbGP9KzCsOA=
gopkg.in/ini.v1 v1.67.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k=
gopkg.in/natefinch/lumberjack.v2 v2.2.1 h1:bBRl1b0OH9s/DuPhuXpNl+VtCaJXFZ5/uEFST95x9zc=
gopkg.in/natefinch/lumberjack.v2 v2.2.1/go.mod h1:YD8tP3GAjkrDg1eZH7EGmyESg/lsYskCTPBJVb9jqSc=
gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=
gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=

View File

@ -44,6 +44,9 @@ func main() {
log.Warn().Msgf("RoadSign auto generated api credential is %s", credential)
}
// Initialize access logging
navi.InitializeLogging()
// Load & init navigator
if err := navi.ReadInConfig(viper.GetString("paths.configs")); err != nil {
log.Panic().Err(err).Msg("An error occurred when loading configurations.")

View File

@ -16,9 +16,7 @@ func ReadInConfig(root string) error {
instance := &RoadApp{
Regions: make([]*Region, 0),
Metrics: &RoadMetrics{
Traces: make([]RoadTrace, 0),
Traffic: make(map[string]int64),
TrafficFrom: make(map[string]int64),
TotalTraffic: 0,
StartupAt: time.Now(),
},

20
pkg/navi/logging.go Normal file
View File

@ -0,0 +1,20 @@
package navi
import (
"log"
"github.com/spf13/viper"
"gopkg.in/natefinch/lumberjack.v2"
)
var accessLogger *log.Logger
func InitializeLogging() {
accessLogger = log.New(&lumberjack.Logger{
Filename: viper.GetString("logging.access"),
MaxSize: 10,
MaxBackups: 3,
MaxAge: 30,
Compress: true,
}, "", 0)
}

View File

@ -1,15 +1,16 @@
package navi
import (
"github.com/spf13/viper"
"bufio"
"os"
"time"
jsoniter "github.com/json-iterator/go"
"github.com/spf13/viper"
)
type RoadMetrics struct {
Traces []RoadTrace `json:"-"`
Traffic map[string]int64 `json:"traffic"`
TrafficFrom map[string]int64 `json:"traffic_from"`
TotalTraffic int64 `json:"total_traffic"`
StartupAt time.Time `json:"startup_at"`
}
@ -31,6 +32,10 @@ type RoadTraceError struct {
}
func (v *RoadMetrics) AddTrace(trace RoadTrace) {
if viper.GetBool("performance.low_memory") {
return
}
v.TotalTraffic++
trace.Timestamp = time.Now()
if _, ok := v.Traffic[trace.Region]; !ok {
@ -38,22 +43,28 @@ func (v *RoadMetrics) AddTrace(trace RoadTrace) {
} else {
v.Traffic[trace.Region]++
}
if _, ok := v.TrafficFrom[trace.IpAddress]; !ok {
v.TrafficFrom[trace.IpAddress] = 0
} else {
v.TrafficFrom[trace.IpAddress]++
}
v.Traces = append(v.Traces, trace)
// Garbage recycle
if len(v.Traffic) > viper.GetInt("performance.traces_limit") {
v.Traffic = make(map[string]int64)
}
if len(v.TrafficFrom) > viper.GetInt("performance.traces_limit") {
v.TrafficFrom = make(map[string]int64)
}
if len(v.Traces) > viper.GetInt("performance.traces_limit") {
v.Traces = v.Traces[1:]
}
raw, _ := jsoniter.Marshal(trace)
accessLogger.Println(string(raw))
}
func (v *RoadMetrics) ReadTrace() []RoadTrace {
fp := viper.GetString("logging.access")
file, err := os.Open(fp)
if err != nil {
return nil
}
defer file.Close()
var out []RoadTrace
scanner := bufio.NewScanner(file)
for scanner.Scan() {
line := scanner.Text()
var entry RoadTrace
if err := jsoniter.Unmarshal([]byte(line), &entry); err == nil {
out = append(out, entry)
}
}
return out
}

View File

@ -148,10 +148,10 @@ func makeFileResponse(c *fiber.Ctx, dest *Destination) error {
return fmt.Errorf("failed to stat: %w", err)
}
// Serve index if path is directory
// Serve index if the path is a directory
if stat.IsDir() {
indexFile := lo.Ternary(len(queries.Get("index")) > 0, queries.Get("index"), "index.html")
indexPath := utils.TrimRight(path, '/') + indexFile
indexPath := filepath.Join(path, indexFile)
index, err := root.Open(indexPath)
if err == nil {
indexStat, err := index.Stat()

View File

@ -60,7 +60,7 @@ func (v *Destination) GetType() DestinationType {
func (v *Destination) GetRawUri() (string, url.Values) {
uri := strings.SplitN(v.Uri, "://", 2)[1]
data := strings.SplitN(uri, "?", 2)
data = append(data, " ") // Make the data array least have two elements
data = append(data, " ") // Make the data array at least have two elements
qs, _ := url.ParseQuery(data[1])
return data[0], qs

View File

@ -1,10 +1,15 @@
package navi
import "git.solsynth.dev/goatworks/roadsign/pkg/warden"
import (
"git.solsynth.dev/goatworks/roadsign/pkg/warden"
"github.com/rs/zerolog/log"
)
func InitializeWarden(regions []*Region) {
pool := make([]*warden.AppInstance, 0)
log.Info().Msg("Starting Warden applications...")
for _, region := range regions {
for _, application := range region.Applications {
pool = append(pool, &warden.AppInstance{
@ -15,5 +20,7 @@ func InitializeWarden(regions []*Region) {
// Hot swap
warden.InstancePool = pool
warden.StartPool()
errs := warden.StartPool()
log.Info().Any("errs", errs).Msg("Warden applications has been started.")
}

View File

@ -10,5 +10,5 @@ func getTraffic(c *fiber.Ctx) error {
}
func getTraces(c *fiber.Ctx) error {
return c.JSON(navi.R.Metrics.Traces)
return c.JSON(navi.R.Metrics.ReadTrace())
}

View File

@ -2,9 +2,13 @@ package sideload
import (
"context"
"fmt"
"git.solsynth.dev/goatworks/roadsign/pkg/warden"
"github.com/rs/zerolog/log"
"os"
"os/exec"
"path/filepath"
"strings"
"git.solsynth.dev/goatworks/roadsign/pkg/navi"
"github.com/gofiber/fiber/v2"
@ -42,7 +46,9 @@ func doPublish(c *fiber.Ctx) error {
var instance *warden.AppInstance
if application != nil {
if instance = warden.GetFromPool(application.ID); instance != nil {
_ = instance.Stop()
if err := instance.Stop(); err != nil {
log.Warn().Err(err).Str("id", application.ID).Msg("Failed to stop application when publishing...")
}
}
} else if destination != nil && destination.GetType() != navi.DestinationStaticFile {
return fiber.ErrUnprocessableEntity
@ -50,7 +56,7 @@ func doPublish(c *fiber.Ctx) error {
return fiber.ErrNotFound
}
if c.Query("overwrite", "yes") == "yes" {
if c.QueryBool("overwrite", true) {
files, _ := filepath.Glob(filepath.Join(workdir, "*"))
for _, file := range files {
_ = os.Remove(file)
@ -74,6 +80,7 @@ func doPublish(c *fiber.Ctx) error {
return err
}
}
_ = os.Remove(dst)
default:
dst := filepath.Join(workdir, file.Filename)
if err := c.SaveFile(file, dst); err != nil {
@ -83,6 +90,15 @@ func doPublish(c *fiber.Ctx) error {
}
}
if postScript := c.FormValue("post-deploy-script", ""); len(postScript) > 0 {
cmd := exec.Command("sh", "-c", postScript)
cmd.Dir = filepath.Join(workdir)
cmd.Env = append(cmd.Env, strings.Split(c.FormValue("post-deploy-environment", ""), "\n")...)
if err := cmd.Run(); err != nil {
return fiber.NewError(fiber.StatusInternalServerError, fmt.Sprintf("post deploy script runs failed: %v", err))
}
}
if instance != nil {
_ = instance.Wake()
}

View File

@ -77,7 +77,7 @@ func doSync(c *fiber.Ctx) error {
_ = instance.Stop()
}
for _, instance := range startQueue {
_ = instance.Start()
_ = instance.Wake()
}
return c.SendStatus(fiber.StatusOK)

View File

@ -1,11 +1,12 @@
package sideload
import (
"time"
"git.solsynth.dev/goatworks/roadsign/pkg/navi"
"git.solsynth.dev/goatworks/roadsign/pkg/warden"
"github.com/gofiber/fiber/v2"
"github.com/samber/lo"
"time"
)
func getStats(c *fiber.Ctx) error {
@ -26,8 +27,7 @@ func getStats(c *fiber.Ctx) error {
"applications": len(applications),
"uptime": time.Since(navi.R.Metrics.StartupAt).Milliseconds(),
"traffic": fiber.Map{
"total": navi.R.Metrics.TotalTraffic,
"unique_client": len(navi.R.Metrics.TrafficFrom),
"total": navi.R.Metrics.TotalTraffic,
},
})
}

View File

@ -2,13 +2,17 @@ package warden
import (
"fmt"
"github.com/rs/zerolog/log"
"io"
"os"
"os/exec"
"path/filepath"
"strings"
"syscall"
"time"
"github.com/rs/zerolog/log"
"github.com/spf13/viper"
"gopkg.in/natefinch/lumberjack.v2"
"github.com/samber/lo"
)
@ -43,11 +47,12 @@ const (
type AppInstance struct {
Manifest Application `json:"manifest"`
Status AppStatus `json:"status"`
Cmd *exec.Cmd `json:"-"`
Logger strings.Builder `json:"-"`
Cmd *exec.Cmd `json:"-"`
Status AppStatus `json:"status"`
LogPath string `json:"-"`
Logger *lumberjack.Logger `json:"-"`
}
func (v *AppInstance) Wake() error {
@ -62,14 +67,8 @@ func (v *AppInstance) Wake() error {
}
if v.Cmd.ProcessState.Exited() {
return v.Start()
} else if v.Cmd.ProcessState.Exited() {
return fmt.Errorf("process already dead")
}
if v.Cmd.ProcessState.Exited() {
return fmt.Errorf("cannot start process")
} else {
return nil
}
return nil
}
func (v *AppInstance) Start() error {
@ -82,8 +81,21 @@ func (v *AppInstance) Start() error {
v.Cmd = exec.Command(manifest.Command[0], manifest.Command[1:]...)
v.Cmd.Dir = filepath.Join(manifest.Workdir)
v.Cmd.Env = append(v.Cmd.Env, manifest.Environment...)
v.Cmd.Stdout = &v.Logger
v.Cmd.Stderr = &v.Logger
logBasePath := viper.GetString("logging.warden_apps")
logPath := filepath.Join(logBasePath, fmt.Sprintf("%s.log", manifest.ID))
v.LogPath = logPath
v.Logger = &lumberjack.Logger{
Filename: v.LogPath,
MaxSize: 10,
MaxBackups: 3,
MaxAge: 30,
Compress: true,
}
v.Cmd.Stdout = v.Logger
v.Cmd.Stderr = v.Logger
// Monitor
go func() {
@ -93,7 +105,7 @@ func (v *AppInstance) Start() error {
} else if v.Cmd != nil && v.Cmd.ProcessState == nil {
v.Status = AppStarted
} else {
v.Status = lo.Ternary(v.Cmd == nil, AppExited, AppFailure)
v.Status = AppFailure
v.Cmd = nil
return
}
@ -110,18 +122,28 @@ func (v *AppInstance) Stop() error {
log.Warn().Int("pid", v.Cmd.Process.Pid).Err(err).Msgf("Failed to send SIGTERM to process...")
if err = v.Cmd.Process.Kill(); err != nil {
log.Error().Int("pid", v.Cmd.Process.Pid).Err(err).Msgf("Failed to kill process...")
} else {
v.Cmd = nil
return err
}
return err
} else {
v.Cmd = nil
}
}
// We need to wait for the process to exit
// The wait syscall will read the exit status of the process
// So that we don't produce defunct processes
// Refer to https://stackoverflow.com/questions/46293435/golang-exec-command-cause-a-lot-of-defunct-processes
_ = v.Cmd.Wait()
v.Cmd = nil
v.Status = AppExited
v.Logger.Close()
return nil
}
func (v *AppInstance) Logs() string {
return v.Logger.String()
file, err := os.Open(v.LogPath)
if err != nil {
return ""
}
raw, _ := io.ReadAll(file)
return string(raw)
}

View File

@ -21,6 +21,10 @@ force_https = false
max_body_size = 549_755_813_888 # 512 GiB
max_qps = -1
[logging]
access = "./logs/access.log"
warden_apps = "./logs/warden"
[paths]
configs = "./config"
@ -29,7 +33,7 @@ request_logging = true
capture_traces = true
[performance]
traces_limit = 256
low_memory = false
prefork = false
[security]

View File

@ -1,2 +1,3 @@
/spa
/capital
/static-files

View File

@ -1,9 +1,9 @@
id = "example"
id = "static-files"
[[locations]]
id = "example-location"
host = ["localhost:8000"]
path = ["/"]
id = "static-files-loc"
hosts = ["localhost:8000"]
paths = ["/"]
[[locations.destinations]]
id = "example-destination"
uri = "files://../data/spa?fallback=index.html"
id = "static-files-des"
uri = "files://../data/spa?fallback=index.html"

View File

@ -1,18 +1,21 @@
id = "central-dc"
[debug]
print_routes = true
print_routes = false
[sideload]
ports = [":81"]
secured_ports = []
trusted_proxies = ["localhost"]
[hypertext]
sideload_ports = [":81"]
sideload_secured_ports = []
ports = [":8000"]
secured_ports = []
force_https = false
[hypertext.certificate]
redirect = false
sideload_key = "./cert.key"
sideload_pem = "./cert.pem"
key = "./cert.key"
pem = "./cert.pem"
# [[hypertext.certificate]]
# key = "./certs/privkey.pem"
# pem = "./certs/fullchain.pem"
[hypertext.limitation]
max_body_size = 549_755_813_888 # 512 GiB
@ -21,11 +24,13 @@ max_qps = -1
[paths]
configs = "./config"
[performance]
[telemetry]
request_logging = true
network_timeout = 3_000
capture_traces = true
[performance]
traces_limit = 256
prefork = false
[security]
sideload_trusted_proxies = ["localhost"]
credential = "e81f43f32d934271af6322e5376f5f59"

View File

@ -11,5 +11,5 @@ uri = "http://localhost:3000"
[[applications]]
id = "capital-app"
workdir = "../data/capital"
command = ["node", "server/index.mjs"]
environment = []
command = ["node", "standalone/server.js"]
environment = []

View File

@ -21,6 +21,10 @@ force_https = false
max_body_size = 549_755_813_888 # 512 GiB
max_qps = -1
[logging]
access = "../../logs/access.log"
warden_apps = "../../logs/warden"
[paths]
configs = "./config"

View File

@ -0,0 +1,9 @@
id = "static-files-num2"
[[locations]]
id = "static-files-loc-num2"
hosts = ["127.0.0.1:8000"]
paths = ["/"]
[[locations.destinations]]
id = "static-files-des-num2"
uri = "files://../data/static-files"