From 04967a8a16bb8154647923f4104cf1c3b99449ec Mon Sep 17 00:00:00 2001 From: Greg Date: Mon, 26 Oct 2026 00:00:50 -0400 Subject: [PATCH] feat(builder): every boot through the node-agent, every lost file counted, Steam cache What a builder boot sees is now what a node does: - First boots (untouched, or -as-deployed with exactly a new server's settings) are deployed by the node-agent's BootTest in keep mode; the builder only watches the console and inspects what's left. The Go copy of the deploy (its own docker run, mounts and startup prefix) is gone. It had drifted: it gave the image's VOLUME folders their own volumes, which nodes don't, so saves there looked safe while every start loses them. - Files written outside /server and /data are counted whatever their type (it only counted config-like extensions, so .sav or .db saves went unseen), including the image's VOLUME folders; logs, pid and lock files and system folders are left out. - Boot tests ask the server to stay up holdSec (60 s, test.yaml can set it) after its ready line and to listen on every published container port (needs the node-agent's matching BootTest). - Steam apps are downloaded once per build into a gbr.cache volume and copied into each boot, so later boots only check the files (STEAM_CACHE=off). - RUNNER.md no longer suggests docker system prune, which deletes the gbr-boottest network when nothing is using it. Co-Authored-By: Claude Opus 5.5 --- AUTHORING.md | 2 + BUILDER.md | 14 +- RUNNER.md | 14 +- cmd/gbrgames/agentboot.go | 95 ++++++++++++ cmd/gbrgames/boottest.go | 80 ++++++---- cmd/gbrgames/boottest_test.go | 27 ++++ cmd/gbrgames/build.go | 49 ++---- cmd/gbrgames/firstboot.go | 271 ++++++++++++++++++++------------- cmd/gbrgames/firstboot_test.go | 37 +++++ schema/test.schema.json | 6 + 10 files changed, 424 insertions(+), 171 deletions(-) create mode 100644 cmd/gbrgames/agentboot.go diff --git a/AUTHORING.md b/AUTHORING.md index 05d8324..ea34765 100644 --- a/AUTHORING.md +++ b/AUTHORING.md @@ -90,6 +90,8 @@ Know this before writing anything: - `failPatterns`: lines that mean the server will never come up (e.g. an unaccepted EULA). - No reliable ready line: leave `readyPattern` out, and the boot test checks the server stays up (`settleSec` in `test.yaml`, default 120 s). +- After its ready line the server must stay up for `holdSec` (default 60 s), and then listen on every port in + `defaultPorts`. A server that binds other ports, or only `127.0.0.1`, fails: players can't reach it. ### `test.yaml` - `config`: only what the server can't start without in CI (accepting the EULA, for example). diff --git a/BUILDER.md b/BUILDER.md index 357ea9f..66295e0 100644 --- a/BUILDER.md +++ b/BUILDER.md @@ -19,8 +19,18 @@ progress / log / variants ◄─post── build: first boot → draft 2. The admin picks a variant in the panel. 3. **Build** (`gbrgames build -name N -variant @v.json`): first boot of the image untouched (`first-boot`), a draft by the model (`draft`, rules in `AUTHORING.md`), then the draft is deployed with the node-agent's - `BootTest`, the same check CI runs. Failures (crash, error lines, saves written outside `/server`) go - back to the model, three boots at most. + `BootTest`, the same check CI runs. A failed boot is investigated before the model changes anything + (see below). + + **Every** server the builder starts, first boots included, is deployed by the node-agent's own code, so + it runs exactly as on a node. A boot passes when the server reaches its ready line, stays up for + `holdSec` (60 s) after it, and listens on every port the template publishes. Files the server writes + anywhere but `/server` and `/data` (the image's own `VOLUME` folders included) are reported as lost: + nodes recreate the container on every start. + + Steam games are downloaded once per build into a cache volume (`gbr-steamcache-`, labelled + `gbr.cache`) and copied into each boot, so later boots only check the files. `STEAM_CACHE=off` turns + it off. 4. The worker pushes `games//` to a branch and opens a pull request as the bot account. A draft that validates but never booted still gets a PR, titled `WIP:`, with what's left listed. Nothing reaches players until a person merges it and enables the game. diff --git a/RUNNER.md b/RUNNER.md index 919a1b9..c7f19ac 100644 --- a/RUNNER.md +++ b/RUNNER.md @@ -51,5 +51,15 @@ the org or repo, and the workflow passes it through (see the `env:` of the boot- ## Housekeeping -Game images and SteamCMD downloads add up. A weekly `docker system prune -af --filter "until=168h"` -(cron or a systemd timer) keeps the disk in check; each test removes its own container and volume. +Game images and SteamCMD downloads add up; each test removes its own container and volume. Clean up +weekly (cron or a systemd timer) with: + +```bash +docker container prune -f +docker image prune -af --filter "until=168h" +docker volume prune -af --filter "label!=gbr.cache" +``` + +Not `docker system prune`: it also deletes the `gbr-boottest` network whenever no container is using it, +and boot tests then refuse to run until `systemctl restart gbr-boottest-network`. Volumes labelled +`gbr.cache` are the AI builder's Steam download cache. diff --git a/cmd/gbrgames/agentboot.go b/cmd/gbrgames/agentboot.go new file mode 100644 index 0000000..3e4dbff --- /dev/null +++ b/cmd/gbrgames/agentboot.go @@ -0,0 +1,95 @@ +package main + +import ( + "bytes" + "encoding/json" + "errors" + "fmt" + "os" + "os/exec" + "regexp" +) + +// Every server the builder starts, first boots included, is deployed by the node-agent's own code +// (BootTest), the same code that runs servers on nodes and in CI, so what the builder sees is what a +// node does. This file runs it and keeps the Steam download cache that makes repeated boots cheap. + +func agentImageFromEnv() string { + if img := os.Getenv("AGENT_IMAGE"); img != "" { + return img + } + return defaultAgentImage +} + +// runAgentBootTest deploys a payload with BootTest and returns its output; passed is its verdict. +func runAgentBootTest(payload map[string]any, agentImage string) (output string, passed bool, err error) { + container := fmt.Sprint(payload["containerName"]) + // A run that was killed (worker restarted mid-build) leaves its container and volume behind, and the + // next deploy under the same name would trip over them + runCmd("docker", "rm", "-f", "-v", container) + runCmd("docker", "volume", "rm", "-f", container+"-data") + if payload["installMethod"] == "STEAMCMD" { + if ic, ok := payload["installConfig"].(map[string]any); ok && ic["appId"] != nil { + seedSteamApp(fmt.Sprint(ic["appId"]), container+"-data") + } + } + raw, err := json.Marshal(payload) + if err != nil { + return "", false, err + } + cmd := exec.Command("docker", "run", "--rm", "-i", "-v", "/var/run/docker.sock:/var/run/docker.sock", + "--entrypoint", "java", agentImage, "-Dloader.main=com.gbreed.gbr.nodeagent.BootTest", "-cp", "app.jar", + "org.springframework.boot.loader.launch.PropertiesLauncher") + cmd.Stdin = bytes.NewReader(raw) + var out bytes.Buffer + cmd.Stdout = &teeWriter{&out} + cmd.Stderr = cmd.Stdout + if runErr := cmd.Run(); runErr != nil { + var exitErr *exec.ExitError + if !errors.As(runErr, &exitErr) { + return "", false, runErr + } + return out.String(), false, nil + } + return out.String(), true, nil +} + +// Steam apps whose cached install is up to date in this run (one build or first boot) +var steamCacheFresh = map[string]bool{} + +var numericAppID = regexp.MustCompile(`^\d+$`) + +// seedSteamApp fills a server volume from a cached SteamCMD install of the app, so a boot doesn't +// download the whole game again: the node-agent's own install then only checks the files. The cache is +// updated once per run. Without it (STEAM_CACHE=off, or the update failing) the agent downloads as usual. +func seedSteamApp(appID, volume string) { + if os.Getenv("STEAM_CACHE") == "off" || !numericAppID.MatchString(appID) { + return + } + cache := "gbr-steamcache-" + appID + if !steamCacheFresh[appID] { + fmt.Printf("[steam-cache] updating the cached install of Steam app %s\n", appID) + if out, err := runCmd("docker", "volume", "create", "--label", "gbr.cache=steam", cache); err != nil { + fmt.Printf("[steam-cache] %v: %s\n", err, out) + return + } + runCmd("docker", "run", "--rm", "-v", cache+":/server", "alpine", "chown", "-R", nodeUser, "/server") + // SteamCMD's first install on a fresh client sometimes fails ("Missing configuration") until it has + // logged in once; retried a few times + script := "s=/home/steam/steamcmd/steamcmd.sh; $s +login anonymous +quit >/dev/null; for i in 1 2 3; do " + + "$s +force_install_dir /server +login anonymous +app_update " + appID + " +quit && exit 0; sleep 15; done; exit 1" + cmd := exec.Command("docker", "run", "--rm", "--network", bootTestNetwork, "-v", cache+":/server", "cm2network/steamcmd", "bash", "-c", script) + cmd.Stdout, cmd.Stderr = os.Stdout, os.Stderr + if err := cmd.Run(); err != nil { + fmt.Printf("[steam-cache] update failed (%v); the boot downloads the app itself\n", err) + return + } + steamCacheFresh[appID] = true + } + runCmd("docker", "volume", "create", volume) + if out, err := runCmd("docker", "run", "--rm", "-v", cache+":/from:ro", "-v", volume+":/to", "alpine", "cp", "-a", "/from/.", "/to/"); err != nil { + fmt.Printf("[steam-cache] copying into %s failed (%v: %s); the boot downloads the app itself\n", volume, err, out) + return + } + fmt.Printf("[steam-cache] %s seeded from the cached install of app %s\n", volume, appID) +} diff --git a/cmd/gbrgames/boottest.go b/cmd/gbrgames/boottest.go index c53d658..bdd5526 100644 --- a/cmd/gbrgames/boottest.go +++ b/cmd/gbrgames/boottest.go @@ -35,21 +35,53 @@ func runBootTestPayload(args []string) error { fillBlank := fs.Bool("fill-blank-passwords", false, "also give blank password settings a throwaway value. Never for a "+ "boot test: a real server doesn't get one. Only the AI builder uses it, to find out whether an empty setting is "+ "why a draft fails") + keep := fs.Bool("keep", false, "leave the container and its volume after the test, for inspection") fs.Parse(args) if fs.NArg() != 1 { return errors.New("boottest-payload needs a game id") } - id := fs.Arg(0) - dir := filepath.Join(*gamesDir, id) - - g, err := loadMap(filepath.Join(dir, "game.yaml")) + payload, err := bootTestPayload(*gamesDir, fs.Arg(0), *fillBlank) + var missing *missingSecretError + if errors.As(err, &missing) { + fmt.Fprintln(os.Stderr, "skip: "+missing.Error()) + os.Exit(exitMissingSecret) + } if err != nil { return err } + if *keep { + payload["keepContainer"] = true + } + b, err := json.MarshalIndent(payload, "", " ") + if err != nil { + return err + } + if *out == "" { + _, err = os.Stdout.Write(b) + return err + } + return os.WriteFile(*out, b, 0o644) +} + +type missingSecretError struct{ game, secret, key string } + +func (e *missingSecretError) Error() string { + return fmt.Sprintf("%s needs secret %s for %s", e.game, e.secret, e.key) +} + +// bootTestPayload builds the DEPLOY_SERVER message the panel would send for a game, plus the boot-test +// settings BootTest reads: ready and fail patterns, timeouts, how long the server must stay up after it is +// ready, and the container ports it must then listen on. +func bootTestPayload(gamesDir, id string, fillBlank bool) (map[string]any, error) { + dir := filepath.Join(gamesDir, id) + g, err := loadMap(filepath.Join(dir, "game.yaml")) + if err != nil { + return nil, err + } test := map[string]any{} if _, err := os.Stat(filepath.Join(dir, "test.yaml")); err == nil { if test, err = loadMap(filepath.Join(dir, "test.yaml")); err != nil { - return err + return nil, err } } @@ -68,8 +100,7 @@ func runBootTestPayload(args []string) error { for key, name := range s { value, ok := os.LookupEnv(fmt.Sprint(name)) if !ok || value == "" { - fmt.Fprintf(os.Stderr, "skip: %s needs secret %s for %s\n", id, name, key) - os.Exit(exitMissingSecret) + return nil, &missingSecretError{game: id, secret: fmt.Sprint(name), key: key} } config[key] = value } @@ -79,7 +110,7 @@ func runBootTestPayload(args []string) error { // one: they passed CI and failed on the first real deploy. for key, p := range schemaProperties(g) { prop, _ := p.(map[string]any) - if prop["x-generate"] == nil && (!*fillBlank || !passwordKey.MatchString(key)) { + if prop["x-generate"] == nil && (!fillBlank || !passwordKey.MatchString(key)) { continue } // Only missing or empty text: ports, booleans and other values are never touched @@ -93,22 +124,26 @@ func runBootTestPayload(args []string) error { } } - timeout, settle := 900, 120 + timeout, settle, hold := 900, 120, 60 if g["installMethod"] == "STEAMCMD" { timeout = 2400 } - if v, ok := test["timeoutSec"].(json.Number); ok { - if n, err := v.Int64(); err == nil { - timeout = int(n) + for key, dst := range map[string]*int{"timeoutSec": &timeout, "settleSec": &settle, "holdSec": &hold} { + if v, ok := test[key].(json.Number); ok { + if n, err := v.Int64(); err == nil { + *dst = int(n) + } } } - if v, ok := test["settleSec"].(json.Number); ok { - if n, err := v.Int64(); err == nil { - settle = int(n) + // Once up, the server must listen on what nodes publish: the container side of every default port + expectPorts := []any{} + if ports, ok := g["defaultPorts"].([]any); ok { + for _, p := range ports { + expectPorts = append(expectPorts, fmt.Sprint(p)) } } - payload := map[string]any{ + return map[string]any{ "command": "DEPLOY_SERVER", "serverId": "boottest-" + id, "containerName": "boottest-" + id, @@ -130,16 +165,9 @@ func runBootTestPayload(args []string) error { "failPatterns": g["failPatterns"], "timeoutSec": timeout, "settleSec": settle, - } - b, err := json.MarshalIndent(payload, "", " ") - if err != nil { - return err - } - if *out == "" { - _, err = os.Stdout.Write(b) - return err - } - return os.WriteFile(*out, b, 0o644) + "holdSec": hold, + "expectPorts": expectPorts, + }, nil } // resolvePlaceholders stands in for what the panel fills per server; saved values have no owner here. diff --git a/cmd/gbrgames/boottest_test.go b/cmd/gbrgames/boottest_test.go index 18b938b..34693a3 100644 --- a/cmd/gbrgames/boottest_test.go +++ b/cmd/gbrgames/boottest_test.go @@ -2,6 +2,7 @@ package main import ( "encoding/json" + "errors" "os" "path/filepath" "strings" @@ -79,3 +80,29 @@ configSchema: t.Errorf("-fill-blank-passwords: ServerPassword not filled: %v", payload.GameConfig["ServerPassword"]) } } + +func TestBootTestPayloadAsksForPublishedPortsAndHold(t *testing.T) { + dir := t.TempDir() + game := filepath.Join(dir, "example") + os.MkdirAll(game, 0o755) + os.WriteFile(filepath.Join(game, "game.yaml"), []byte("id: example\nname: Example\ndefaultDockerImage: x\ndefaultPorts:\n - 30000/udp\n - 30001:30001/tcp\n"), 0o644) + os.WriteFile(filepath.Join(game, "test.yaml"), []byte("holdSec: 90\n"), 0o644) + p, err := bootTestPayload(dir, "example", false) + if err != nil { + t.Fatal(err) + } + ports, _ := p["expectPorts"].([]any) + if len(ports) != 2 || ports[0] != "30000/udp" || ports[1] != "30001:30001/tcp" { + t.Errorf("expectPorts %v", p["expectPorts"]) + } + if p["holdSec"] != 90 || p["settleSec"] != 120 { + t.Errorf("hold %v settle %v", p["holdSec"], p["settleSec"]) + } + + os.WriteFile(filepath.Join(game, "test.yaml"), []byte("secrets:\n OwnerId: EXAMPLE_OWNER\n"), 0o644) + _, err = bootTestPayload(dir, "example", false) + var missing *missingSecretError + if !errors.As(err, &missing) || missing.secret != "EXAMPLE_OWNER" { + t.Errorf("missing secret: %v", err) + } +} diff --git a/cmd/gbrgames/build.go b/cmd/gbrgames/build.go index b6267f6..e4a07d6 100644 --- a/cmd/gbrgames/build.go +++ b/cmd/gbrgames/build.go @@ -7,7 +7,6 @@ import ( "flag" "fmt" "os" - "os/exec" "path/filepath" "sort" "strings" @@ -298,46 +297,18 @@ func bootCheck(gamesDir, id, agentImage string, wantOutside bool) (problems, ver // CI. fillBlank also fills blank password-like settings, only to find out whether they are why a draft // fails. problems is set when the template can't be boot-tested at all. func agentBoot(gamesDir, id, agentImage string, fillBlank bool) (output string, passed bool, problems []string, err error) { - self, err := os.Executable() + payload, err := bootTestPayload(gamesDir, id, fillBlank) + var missing *missingSecretError + if errors.As(err, &missing) { + // The builder has no CI secrets: that's the template's problem to fix, not a reason to stop + return "", false, []string{"the boot test can't run: " + missing.Error() + ". Remove 'secrets' from " + + "test.yaml; boot tests run with exactly what a new server gets"}, nil + } if err != nil { return "", false, nil, err } - // A boot test that was killed (worker restarted mid-build) leaves its container and volume behind, - // and the next deploy under the same name would trip over them - runCmd("docker", "rm", "-f", "boottest-"+id) - runCmd("docker", "volume", "rm", "-f", "boottest-"+id+"-data") - args := []string{"boottest-payload", "-games", gamesDir, id} - if fillBlank { - args = []string{"boottest-payload", "-fill-blank-passwords", "-games", gamesDir, id} - } - payloadCmd := exec.Command(self, args...) - var stderr bytes.Buffer - payloadCmd.Stderr = &stderr - payload, err := payloadCmd.Output() - if err != nil { - var exitErr *exec.ExitError - if errors.As(err, &exitErr) && exitErr.ExitCode() == exitMissingSecret { - // The builder has no CI secrets: that's the template's problem to fix, not a reason to stop - return "", false, []string{"the boot test can't run: " + strings.TrimSpace(stderr.String()) + ". Remove 'secrets' from " + - "test.yaml; boot tests run with exactly what a new server gets"}, nil - } - return "", false, nil, fmt.Errorf("boottest-payload: %v: %s", err, strings.TrimSpace(stderr.String())) - } - cmd := exec.Command("docker", "run", "--rm", "-i", "-v", "/var/run/docker.sock:/var/run/docker.sock", - "--entrypoint", "java", agentImage, "-Dloader.main=com.gbreed.gbr.nodeagent.BootTest", "-cp", "app.jar", - "org.springframework.boot.loader.launch.PropertiesLauncher") - cmd.Stdin = bytes.NewReader(payload) - var out bytes.Buffer - cmd.Stdout = &teeWriter{&out} - cmd.Stderr = cmd.Stdout - if runErr := cmd.Run(); runErr != nil { - var exitErr *exec.ExitError - if !errors.As(runErr, &exitErr) { - return "", false, nil, runErr - } - return out.String(), false, nil, nil - } - return out.String(), true, nil, nil + output, passed, err = runAgentBootTest(payload, agentImage) + return output, passed, nil, err } // blankRequiredLooking lists the settings a new server gets empty that look like a server might need a @@ -372,7 +343,7 @@ func emptySettingsProblem(keys []string, verdict string) string { func firstBootGame(gamesDir, id string) (*bootReport, error) { outDir := filepath.Join("firstboot-out", id+"-draft") - if err := runFirstBoot([]string{"-games", gamesDir, "-game", id, "-out", outDir}); err != nil { + if err := runFirstBoot([]string{"-games", gamesDir, "-game", id, "-as-deployed", "-out", outDir}); err != nil { return nil, err } return readBootReport(filepath.Join(outDir, "report.json")) diff --git a/cmd/gbrgames/firstboot.go b/cmd/gbrgames/firstboot.go index fd37229..1b33b04 100644 --- a/cmd/gbrgames/firstboot.go +++ b/cmd/gbrgames/firstboot.go @@ -20,15 +20,11 @@ import ( "time" ) -// first-boot starts a game server once with none of a template's settings applied, and records what it -// does on its own: its output (to find the "ready" and error lines) and the files it creates (to learn -// the real config file, format and defaults). It mirrors how the node-agent runs servers: same volumes, -// user, SteamCMD install and startup prefix, but without a restart policy, so a crash shows as a crash. - -// Same prefix the node-agent puts in front of every startup command (DockerService) -const agentStartupPrefix = "mkdir -p ~/.steam/sdk64 /server/.steam/sdk64 2>/dev/null; " + - "[ -f /home/steam/steamcmd/linux64/steamclient.so ] && cp -f /home/steam/steamcmd/linux64/steamclient.so ~/.steam/sdk64/ 2>/dev/null; " + - "[ -f /server/linux64/steamclient.so ] && cp -f /server/linux64/steamclient.so ~/.steam/sdk64/ 2>/dev/null; cd /server && " +// first-boot starts a game server once with none of a template's settings applied (or, -as-deployed, +// with exactly the settings a new server gets), and records what it does on its own: its output (to find +// the "ready" and error lines) and the files it creates (to learn the real config file, format and +// defaults, and what would be lost on a restart). The node-agent's own deploy code starts it, so it runs +// exactly as on a node: volumes, user, limits, SteamCMD install and startup command. var ( ansi = regexp.MustCompile(`\x1b\[[0-9;?]*[A-Za-z]|\x1b\][^\a]*\a`) @@ -104,6 +100,8 @@ func runFirstBoot(args []string) error { quiet := fs.Duration("quiet", 2*time.Minute, "stop once the server has printed nothing for this long") outDir := fs.String("out", "", "where to write the report (default firstboot-out/)") keep := fs.Bool("keep", false, "keep the container and volumes for a closer look") + asDeployed := fs.Bool("as-deployed", false, "with -game: boot with the template's settings exactly as a new server gets "+ + "them (default: untouched, so the server writes its own defaults)") var envFlags multiFlag fs.Var(&envFlags, "env", "KEY=VALUE for the container (repeatable), e.g. EULA=TRUE") fs.Parse(args) @@ -138,7 +136,19 @@ func runFirstBoot(args []string) error { return err } - b := &booter{spec: spec, facts: facts, container: "firstboot-" + spec.Name, outDir: *outDir} + b := &booter{spec: spec, facts: facts, container: "firstboot-" + spec.Name, outDir: *outDir, agentImage: agentImageFromEnv()} + b.payload = untouchedPayload(spec) + if *asDeployed { + if *gameID == "" { + return errors.New("-as-deployed needs -game") + } + if b.payload, err = bootTestPayload(*gamesDir, *gameID, false); err != nil { + return err + } + } + if out, err := runCmd("docker", "pull", "-q", b.agentImage); err != nil { + return fmt.Errorf("docker pull %s: %v: %s", b.agentImage, err, out) + } if !*keep { defer b.cleanup() } @@ -221,11 +231,13 @@ func buildBootSpec(gamesDir, gameID, image, startup, steamApp string, memory int } type booter struct { - spec *bootSpec - facts *imageFacts - container string - outDir string - mounts map[string]string // mount path -> volume name + spec *bootSpec + facts *imageFacts + container string + outDir string + agentImage string + payload map[string]any // what the node-agent deploys + mounts map[string]string // mount path -> volume name mu sync.Mutex lines []logLine @@ -233,64 +245,56 @@ type booter struct { started time.Time } -func (b *booter) run(maxRun, quiet time.Duration) (*bootReport, error) { - exec.Command("docker", "rm", "-f", b.container).Run() // leftovers from an interrupted run +// untouchedPayload deploys an image as a node would, but with none of a template's settings: only the +// install, the startup command and the -env values, so the server writes its own defaults. +func untouchedPayload(spec *bootSpec) map[string]any { + config := map[string]any{} + for k, v := range spec.Env { + config[k] = v + } + p := map[string]any{ + "command": "DEPLOY_SERVER", "dockerImage": spec.Image, "gameConfig": config, "ports": []any{}, + "installMethod": "NONE", "memoryLimitMb": spec.MemoryMb, "containerUser": spec.User, + "dockerNetwork": bootTestNetwork, "timeoutSec": 900, + } + if spec.Startup != "" { + p["startupCommand"] = spec.Startup + } + if spec.SteamAppID != "" { + p["installMethod"], p["installConfig"], p["timeoutSec"] = "STEAMCMD", map[string]any{"appId": spec.SteamAppID}, 2400 + } + return p +} - // Same layout as the node: one volume at /server and /data, plus a fresh one per declared VOLUME +func (b *booter) run(maxRun, quiet time.Duration) (*bootReport, error) { + // Exactly what nodes mount: one volume at /server and /data. Anything else the server writes, the + // image's own VOLUME folders included, is gone when the node recreates the container on the next start data := b.container + "-data" b.mounts = map[string]string{"/server": data, "/data": data} - for i, v := range b.facts.Volumes { - if _, ok := b.mounts[v]; !ok { - b.mounts[v] = fmt.Sprintf("%s-v%d", b.container, i) - } - } - for _, vol := range uniqueValues(b.mounts) { - if err := docker("volume", "create", vol); err != nil { - return nil, err - } - if err := docker("run", "--rm", "-v", vol+":/v", "alpine", "chown", "-R", b.spec.User, "/v"); err != nil { - return nil, err - } - } - if b.spec.SteamAppID != "" { - fmt.Printf("[first-boot] installing Steam app %s\n", b.spec.SteamAppID) - // Like the node-agent: steamcmd's image runs as uid 1000, then the volume goes to the game's user - steam := "/home/steam/steamcmd/steamcmd.sh +login anonymous +quit && /home/steam/steamcmd/steamcmd.sh +force_install_dir /server +login anonymous +app_update " + - b.spec.SteamAppID + " validate +quit" - if err := docker("run", "--rm", "-v", data+":/server", "alpine", "chown", "-R", nodeUser, "/server"); err != nil { - return nil, err - } - cmd := exec.Command("docker", "run", "--rm", "--network", bootTestNetwork, "-v", data+":/server", "cm2network/steamcmd", "bash", "-c", steam) - cmd.Stdout, cmd.Stderr = os.Stdout, os.Stderr - if err := cmd.Run(); err != nil { - return nil, fmt.Errorf("steamcmd install of %s failed: %w", b.spec.SteamAppID, err) - } - if err := docker("run", "--rm", "-v", data+":/server", "alpine", "chown", "-R", b.spec.User, "/server"); err != nil { - return nil, err - } + // The node-agent deploys it (install, config, startup, user, limits) and leaves the container to us. + // It only has to get it running: watching it until it goes quiet is done here + p := map[string]any{} + for k, v := range b.payload { + p[k] = v } + p["containerName"], p["serverId"], p["keepContainer"] = b.container, b.container, true + p["settleSec"], p["holdSec"], p["expectPorts"] = 15, 0, []any{} + delete(p, "readyPattern") + delete(p, "failPatterns") + fmt.Printf("[first-boot] deploying %s as %v with the node-agent (memory %v MB)\n", b.spec.Image, displayUser(p["containerUser"]), p["memoryLimitMb"]) + output, _, err := runAgentBootTest(p, b.agentImage) + if err != nil { + return nil, err + } + created, err := time.Parse(time.RFC3339Nano, inspectField(b.container, "{{.Created}}")) + if err != nil { + return nil, fmt.Errorf("the node-agent could not deploy %s: %s", b.spec.Image, bootVerdict(output)) + } + b.started, b.lastAt = created, created - runArgs := []string{"run", "-d", "-t", "--init", "--name", b.container, "--user", b.spec.User, - "--security-opt", "no-new-privileges:true", "--pids-limit", "1024", "--network", bootTestNetwork, - "-m", fmt.Sprintf("%dm", b.spec.MemoryMb), "--memory-swap", fmt.Sprintf("%dm", b.spec.MemoryMb)} - for mount, vol := range b.mounts { - runArgs = append(runArgs, "-v", vol+":"+mount) - } - for _, k := range sortedKeysString(b.spec.Env) { - runArgs = append(runArgs, "-e", k+"="+b.spec.Env[k]) - } - runArgs = append(runArgs, b.spec.Image) - if b.spec.Startup != "" { - runArgs = append(runArgs, "bash", "-c", agentStartupPrefix+b.spec.Startup) - } - fmt.Printf("[first-boot] starting %s as %s (memory %d MB)\n", b.spec.Image, b.spec.User, b.spec.MemoryMb) - if out, err := exec.Command("docker", runArgs...).CombinedOutput(); err != nil { - return nil, fmt.Errorf("docker run: %v: %s", err, strings.TrimSpace(string(out))) - } - b.started, b.lastAt = time.Now(), time.Now() - - logs := exec.Command("docker", "logs", "-f", b.container) + // The whole console from the start, with Docker's timestamps + logs := exec.Command("docker", "logs", "-t", "-f", b.container) pr, pw := io.Pipe() logs.Stdout, logs.Stderr = pw, pw if err := logs.Start(); err != nil { @@ -301,27 +305,31 @@ func (b *booter) run(maxRun, quiet time.Duration) (*bootReport, error) { sc := bufio.NewScanner(pr) sc.Buffer(make([]byte, 1024*1024), 1024*1024) for sc.Scan() { - b.addLine("console", sc.Text()) + b.addTimestamped(sc.Text()) } close(done) }() - report := &bootReport{Image: b.spec.Image, User: b.spec.User} + report := &bootReport{Image: b.spec.Image, User: displayUser(p["containerUser"])} minRun := time.Minute for { time.Sleep(2 * time.Second) elapsed := time.Since(b.started) running := inspectField(b.container, "{{.State.Running}}") == "true" + restarts, _ := strconv.Atoi(inspectField(b.container, "{{.RestartCount}}")) b.mu.Lock() silent, count := time.Since(b.lastAt), len(b.lines) b.mu.Unlock() switch { - case !running: + case !running || restarts > 0: code, _ := strconv.Atoi(inspectField(b.container, "{{.State.ExitCode}}")) report.ExitCode = &code report.Outcome = fmt.Sprintf("exited by itself with code %d after %.0fs", code, elapsed.Seconds()) + if restarts > 0 { + report.Outcome = fmt.Sprintf("crashed (exit code %d) and was restarted %d time(s) within %.0fs", code, restarts, elapsed.Seconds()) + } if inspectField(b.container, "{{.State.OOMKilled}}") == "true" { - report.Outcome += fmt.Sprintf(" (killed: over the %d MB memory limit)", b.spec.MemoryMb) + report.Outcome += fmt.Sprintf(" (killed: over the %v MB memory limit)", p["memoryLimitMb"]) } case elapsed >= minRun && count > 0 && silent >= quiet: report.Outcome = fmt.Sprintf("running and quiet for %.0fs after %.0fs; stopped gracefully", silent.Seconds(), elapsed.Seconds()) @@ -335,6 +343,7 @@ func (b *booter) run(maxRun, quiet time.Duration) (*bootReport, error) { report.RanSeconds = time.Since(b.started).Seconds() // Many servers only write their settings when they shut down exec.Command("docker", "stop", "-t", "60", b.container).Run() + logs.Process.Kill() pw.Close() <-done @@ -354,17 +363,33 @@ func (b *booter) run(maxRun, quiet time.Duration) (*bootReport, error) { return report, nil } -func (b *booter) addLine(source, raw string) { +// addTimestamped records one line of `docker logs -t`: "2026-10-26T01:11:52.123456789Z text". +func (b *booter) addTimestamped(raw string) { + at := time.Now() + if stamp, rest, ok := strings.Cut(raw, " "); ok { + if t, err := time.Parse(time.RFC3339Nano, stamp); err == nil { + at, raw = t, rest + } + } text := strings.TrimRight(ansi.ReplaceAllString(raw, ""), "\r \t") if text == "" { return } b.mu.Lock() - b.lines = append(b.lines, logLine{At: time.Since(b.started).Round(100 * time.Millisecond).Seconds(), Source: source, Text: text}) - b.lastAt = time.Now() + b.lines = append(b.lines, logLine{At: at.Sub(b.started).Round(100 * time.Millisecond).Seconds(), Source: "console", Text: text}) + if at.After(b.lastAt) { + b.lastAt = at + } b.mu.Unlock() } +func displayUser(u any) string { + if s, ok := u.(string); ok && s != "" { + return s + } + return nodeUser +} + func (b *booter) console() []logLine { b.mu.Lock() defer b.mu.Unlock() @@ -487,39 +512,88 @@ func (b *booter) fileLines(report *bootReport) []logLine { return out } -// containerChanges lists files the server wrote outside its volumes: settings there are lost when the -// node recreates the container, which a template needs to know about. +// System folders and files that aren't the game's state: what's written there doesn't matter when lost +var ( + systemPrefixes = []string{"/tmp", "/proc", "/sys", "/dev", "/run", "/etc", "/var/log", "/var/cache", "/var/tmp", "/var/lib/apt"} + transientExt = map[string]bool{".log": true, ".pid": true, ".lock": true, ".tmp": true, ".swp": true} +) + +// containerChanges lists the files the server wrote anywhere but /server and /data. All of them are +// lost when the node recreates the container (every start): in the container's own filesystem, and in +// the image's VOLUME folders, which Docker gives fresh anonymous volumes each time. Logs, lock and pid +// files are left out; saves of any kind are not. func (b *booter) containerChanges() []string { - out, err := exec.Command("docker", "diff", b.container).Output() - if err != nil { - return nil - } var changes []string - for _, line := range strings.Split(strings.TrimSpace(string(out)), "\n") { - kind, p, ok := strings.Cut(line, " ") - if !ok || kind == "D" { + if out, err := exec.Command("docker", "diff", b.container).Output(); err == nil { + changes = append(changes, outsideFiles(string(out), b.mounts)...) + } + // The image's VOLUME folders: Docker mounted anonymous volumes there, which docker diff doesn't show + mounts, _ := exec.Command("docker", "inspect", "-f", `{{range .Mounts}}{{if eq .Type "volume"}}{{.Name}}|{{.Destination}}{{"\n"}}{{end}}{{end}}`, b.container).Output() + for _, line := range strings.Split(strings.TrimSpace(string(mounts)), "\n") { + vol, dest, ok := strings.Cut(line, "|") + if !ok || b.mounts[dest] != "" { continue } - skip := false - for _, prefix := range []string{"/tmp", "/proc", "/sys", "/dev", "/run", "/var/log", "/var/cache", "/var/tmp"} { - if p == prefix || strings.HasPrefix(p, prefix+"/") { - skip = true - } + files, err := exec.Command("docker", "run", "--rm", "-v", vol+":/v:ro", "alpine", "find", "/v", "-type", "f").Output() + if err != nil { + continue } - for mount := range b.mounts { - if p == mount || strings.HasPrefix(p, mount+"/") { - skip = true + n := 0 + for _, f := range strings.Split(strings.TrimSpace(string(files)), "\n") { + rel := strings.TrimPrefix(f, "/v/") + if f == "" || transientExt[strings.ToLower(path.Ext(rel))] { + continue } - } - if !skip && configExt[strings.ToLower(path.Ext(p))] { - changes = append(changes, kind+" "+p) + if n++; n > 200 { + break + } + changes = append(changes, "A "+path.Join(dest, rel)) } } return changes } +// outsideFiles picks the files out of `docker diff` output: added or changed leaves (docker diff lists +// folders too) outside the mounted volumes and system folders, without transient files. +func outsideFiles(diff string, mounts map[string]string) []string { + var paths []string + kinds := map[string]string{} + for _, line := range strings.Split(strings.TrimSpace(diff), "\n") { + kind, p, ok := strings.Cut(strings.TrimSpace(line), " ") + if !ok || kind == "D" || p == "" { + continue + } + skip := transientExt[strings.ToLower(path.Ext(p))] + for _, prefix := range systemPrefixes { + if p == prefix || strings.HasPrefix(p, prefix+"/") { + skip = true + } + } + for mount := range mounts { + if p == mount || strings.HasPrefix(p, mount+"/") { + skip = true + } + } + if !skip { + paths = append(paths, p) + kinds[p] = kind + } + } + sort.Strings(paths) + var leaves []string + for i, p := range paths { + // A folder is followed (sorted) by the entries inside it + if i+1 < len(paths) && strings.HasPrefix(paths[i+1], p+"/") { + continue + } + leaves = append(leaves, kinds[p]+" "+p) + } + return leaves +} + func (b *booter) cleanup() { - exec.Command("docker", "rm", "-f", b.container).Run() + // -v: the anonymous volumes of the image's VOLUME folders go with it + exec.Command("docker", "rm", "-f", "-v", b.container).Run() for _, vol := range uniqueValues(b.mounts) { exec.Command("docker", "volume", "rm", vol).Run() } @@ -730,13 +804,6 @@ func describeLine(l logLine) string { return fmt.Sprintf("[%s] %s", where, text) } -func docker(args ...string) error { - if out, err := exec.Command("docker", args...).CombinedOutput(); err != nil { - return fmt.Errorf("docker %s: %v: %s", args[0], err, strings.TrimSpace(string(out))) - } - return nil -} - func inspectField(container, format string) string { out, _ := exec.Command("docker", "inspect", "-f", format, container).Output() return strings.TrimSpace(string(out)) diff --git a/cmd/gbrgames/firstboot_test.go b/cmd/gbrgames/firstboot_test.go index 17c58a1..4a6530e 100644 --- a/cmd/gbrgames/firstboot_test.go +++ b/cmd/gbrgames/firstboot_test.go @@ -2,6 +2,7 @@ package main import ( "regexp" + "strings" "testing" ) @@ -103,3 +104,39 @@ func TestConfigCandidates(t *testing.T) { } } } + +func TestOutsideFilesCountsEverySaveButNotSystemOrTransientFiles(t *testing.T) { + diff := `C /home +C /home/steam +A /home/steam/.config +A /home/steam/.config/example +A /home/steam/.config/example/worlds +A /home/steam/.config/example/worlds/world1.db +A /home/steam/.config/example/worlds/world1.fwl +A /home/steam/.config/example/Player.log +C /etc +C /etc/hostname +A /tmp/x.sock +A /server/should-not-count.sav +D /home/steam/old.txt +A /home/steam/server.pid` + got := outsideFiles(diff, map[string]string{"/server": "v", "/data": "v"}) + want := []string{"A /home/steam/.config/example/worlds/world1.db", "A /home/steam/.config/example/worlds/world1.fwl"} + if strings.Join(got, "|") != strings.Join(want, "|") { + t.Errorf("got %q\nwant %q", got, want) + } +} + +func TestUntouchedPayloadDeploysWithoutTemplateSettings(t *testing.T) { + p := untouchedPayload(&bootSpec{Image: "example/image", Startup: "./server", SteamAppID: "1000000", MemoryMb: 4096, + User: "1000:1000", Env: map[string]string{"EULA": "TRUE"}}) + if p["installMethod"] != "STEAMCMD" || p["installConfig"].(map[string]any)["appId"] != "1000000" || p["timeoutSec"] != 2400 { + t.Errorf("steam install: %v", p) + } + if p["startupCommand"] != "./server" || p["dockerNetwork"] != bootTestNetwork || p["gameConfig"].(map[string]any)["EULA"] != "TRUE" { + t.Errorf("payload: %v", p) + } + if _, ok := untouchedPayload(&bootSpec{Image: "x"})["startupCommand"]; ok { + t.Error("no startup command means the image's own entrypoint") + } +} diff --git a/schema/test.schema.json b/schema/test.schema.json index 90bb70b..fcf3124 100644 --- a/schema/test.schema.json +++ b/schema/test.schema.json @@ -32,6 +32,12 @@ "type": "integer", "minimum": 10, "maximum": 3600 + }, + "holdSec": { + "description": "How long the server must stay up after its ready line to pass (default 60). Its published ports are checked after it.", + "type": "integer", + "minimum": 0, + "maximum": 3600 } } }