feat(builder): every boot through the node-agent, every lost file counted, Steam cache

What a builder boot sees is now what a node does:
- First boots (untouched, or -as-deployed with exactly a new server's
  settings) are deployed by the node-agent's BootTest in keep mode; the
  builder only watches the console and inspects what's left. The Go copy of
  the deploy (its own docker run, mounts and startup prefix) is gone. It had
  drifted: it gave the image's VOLUME folders their own volumes, which nodes
  don't, so saves there looked safe while every start loses them.
- Files written outside /server and /data are counted whatever their type
  (it only counted config-like extensions, so .sav or .db saves went
  unseen), including the image's VOLUME folders; logs, pid and lock files
  and system folders are left out.
- Boot tests ask the server to stay up holdSec (60 s, test.yaml can set it)
  after its ready line and to listen on every published container port
  (needs the node-agent's matching BootTest).
- Steam apps are downloaded once per build into a gbr.cache volume and copied
  into each boot, so later boots only check the files (STEAM_CACHE=off).
- RUNNER.md no longer suggests docker system prune, which deletes the
  gbr-boottest network when nothing is using it.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

AUTHORING.md

@@ -90,6 +90,8 @@ Know this before writing anything:
 - `failPatterns`: lines that mean the server will never come up (e.g. an unaccepted EULA).
 - No reliable ready line: leave `readyPattern` out, and the boot test checks the server stays up
   (`settleSec` in `test.yaml`, default 120 s).
+- After its ready line the server must stay up for `holdSec` (default 60 s), and then listen on every port in
+  `defaultPorts`. A server that binds other ports, or only `127.0.0.1`, fails: players can't reach it.
 
 ### `test.yaml`
 - `config`: only what the server can't start without in CI (accepting the EULA, for example).

BUILDER.md

@@ -19,8 +19,18 @@ progress / log / variants        ◄─post── build: first boot → draft
 2. The admin picks a variant in the panel.
 3. **Build** (`gbrgames build -name N -variant @v.json`): first boot of the image untouched (`first-boot`), a
    draft by the model (`draft`, rules in `AUTHORING.md`), then the draft is deployed with the node-agent's
-   `BootTest`, the same check CI runs. Failures (crash, error lines, saves written outside `/server`) go
+   `BootTest`, the same check CI runs. A failed boot is investigated before the model changes anything
-   back to the model, three boots at most.
+   (see below).
+
+   **Every** server the builder starts, first boots included, is deployed by the node-agent's own code, so
+   it runs exactly as on a node. A boot passes when the server reaches its ready line, stays up for
+   `holdSec` (60 s) after it, and listens on every port the template publishes. Files the server writes
+   anywhere but `/server` and `/data` (the image's own `VOLUME` folders included) are reported as lost:
+   nodes recreate the container on every start.
+
+   Steam games are downloaded once per build into a cache volume (`gbr-steamcache-<app>`, labelled
+   `gbr.cache`) and copied into each boot, so later boots only check the files. `STEAM_CACHE=off` turns
+   it off.
 4. The worker pushes `games/<id>/` to a branch and opens a pull request as the bot account. A draft that
    validates but never booted still gets a PR, titled `WIP:`, with what's left listed. Nothing reaches
    players until a person merges it and enables the game.

RUNNER.md

@@ -51,5 +51,15 @@ the org or repo, and the workflow passes it through (see the `env:` of the boot-
 
 ## Housekeeping
 
-Game images and SteamCMD downloads add up. A weekly `docker system prune -af --filter "until=168h"`
+Game images and SteamCMD downloads add up; each test removes its own container and volume. Clean up
-(cron or a systemd timer) keeps the disk in check; each test removes its own container and volume.
+weekly (cron or a systemd timer) with:
+
+```bash
+docker container prune -f
+docker image prune -af --filter "until=168h"
+docker volume prune -af --filter "label!=gbr.cache"
+```
+
+Not `docker system prune`: it also deletes the `gbr-boottest` network whenever no container is using it,
+and boot tests then refuse to run until `systemctl restart gbr-boottest-network`. Volumes labelled
+`gbr.cache` are the AI builder's Steam download cache.

cmd/gbrgames/agentboot.go

@@ -0,0 +1,95 @@
+package main
+
+import (
+	"bytes"
+	"encoding/json"
+	"errors"
+	"fmt"
+	"os"
+	"os/exec"
+	"regexp"
+)
+
+// Every server the builder starts, first boots included, is deployed by the node-agent's own code
+// (BootTest), the same code that runs servers on nodes and in CI, so what the builder sees is what a
+// node does. This file runs it and keeps the Steam download cache that makes repeated boots cheap.
+
+func agentImageFromEnv() string {
+	if img := os.Getenv("AGENT_IMAGE"); img != "" {
+		return img
+	}
+	return defaultAgentImage
+}
+
+// runAgentBootTest deploys a payload with BootTest and returns its output; passed is its verdict.
+func runAgentBootTest(payload map[string]any, agentImage string) (output string, passed bool, err error) {
+	container := fmt.Sprint(payload["containerName"])
+	// A run that was killed (worker restarted mid-build) leaves its container and volume behind, and the
+	// next deploy under the same name would trip over them
+	runCmd("docker", "rm", "-f", "-v", container)
+	runCmd("docker", "volume", "rm", "-f", container+"-data")
+	if payload["installMethod"] == "STEAMCMD" {
+		if ic, ok := payload["installConfig"].(map[string]any); ok && ic["appId"] != nil {
+			seedSteamApp(fmt.Sprint(ic["appId"]), container+"-data")
+		}
+	}
+	raw, err := json.Marshal(payload)
+	if err != nil {
+		return "", false, err
+	}
+	cmd := exec.Command("docker", "run", "--rm", "-i", "-v", "/var/run/docker.sock:/var/run/docker.sock",
+		"--entrypoint", "java", agentImage, "-Dloader.main=com.gbreed.gbr.nodeagent.BootTest", "-cp", "app.jar",
+		"org.springframework.boot.loader.launch.PropertiesLauncher")
+	cmd.Stdin = bytes.NewReader(raw)
+	var out bytes.Buffer
+	cmd.Stdout = &teeWriter{&out}
+	cmd.Stderr = cmd.Stdout
+	if runErr := cmd.Run(); runErr != nil {
+		var exitErr *exec.ExitError
+		if !errors.As(runErr, &exitErr) {
+			return "", false, runErr
+		}
+		return out.String(), false, nil
+	}
+	return out.String(), true, nil
+}
+
+// Steam apps whose cached install is up to date in this run (one build or first boot)
+var steamCacheFresh = map[string]bool{}
+
+var numericAppID = regexp.MustCompile(`^\d+$`)
+
+// seedSteamApp fills a server volume from a cached SteamCMD install of the app, so a boot doesn't
+// download the whole game again: the node-agent's own install then only checks the files. The cache is
+// updated once per run. Without it (STEAM_CACHE=off, or the update failing) the agent downloads as usual.
+func seedSteamApp(appID, volume string) {
+	if os.Getenv("STEAM_CACHE") == "off" || !numericAppID.MatchString(appID) {
+		return
+	}
+	cache := "gbr-steamcache-" + appID
+	if !steamCacheFresh[appID] {
+		fmt.Printf("[steam-cache] updating the cached install of Steam app %s\n", appID)
+		if out, err := runCmd("docker", "volume", "create", "--label", "gbr.cache=steam", cache); err != nil {
+			fmt.Printf("[steam-cache] %v: %s\n", err, out)
+			return
+		}
+		runCmd("docker", "run", "--rm", "-v", cache+":/server", "alpine", "chown", "-R", nodeUser, "/server")
+		// SteamCMD's first install on a fresh client sometimes fails ("Missing configuration") until it has
+		// logged in once; retried a few times
+		script := "s=/home/steam/steamcmd/steamcmd.sh; $s +login anonymous +quit >/dev/null; for i in 1 2 3; do " +
+			"$s +force_install_dir /server +login anonymous +app_update " + appID + " +quit && exit 0; sleep 15; done; exit 1"
+		cmd := exec.Command("docker", "run", "--rm", "--network", bootTestNetwork, "-v", cache+":/server", "cm2network/steamcmd", "bash", "-c", script)
+		cmd.Stdout, cmd.Stderr = os.Stdout, os.Stderr
+		if err := cmd.Run(); err != nil {
+			fmt.Printf("[steam-cache] update failed (%v); the boot downloads the app itself\n", err)
+			return
+		}
+		steamCacheFresh[appID] = true
+	}
+	runCmd("docker", "volume", "create", volume)
+	if out, err := runCmd("docker", "run", "--rm", "-v", cache+":/from:ro", "-v", volume+":/to", "alpine", "cp", "-a", "/from/.", "/to/"); err != nil {
+		fmt.Printf("[steam-cache] copying into %s failed (%v: %s); the boot downloads the app itself\n", volume, err, out)
+		return
+	}
+	fmt.Printf("[steam-cache] %s seeded from the cached install of app %s\n", volume, appID)
+}

cmd/gbrgames/boottest.go

@@ -35,21 +35,53 @@ func runBootTestPayload(args []string) error {
 	fillBlank := fs.Bool("fill-blank-passwords", false, "also give blank password settings a throwaway value. Never for a "+
 		"boot test: a real server doesn't get one. Only the AI builder uses it, to find out whether an empty setting is "+
 		"why a draft fails")
+	keep := fs.Bool("keep", false, "leave the container and its volume after the test, for inspection")
 	fs.Parse(args)
 	if fs.NArg() != 1 {
 		return errors.New("boottest-payload needs a game id")
 	}
-	id := fs.Arg(0)
+	payload, err := bootTestPayload(*gamesDir, fs.Arg(0), *fillBlank)
-	dir := filepath.Join(*gamesDir, id)
+	var missing *missingSecretError
-
+	if errors.As(err, &missing) {
-	g, err := loadMap(filepath.Join(dir, "game.yaml"))
+		fmt.Fprintln(os.Stderr, "skip: "+missing.Error())
+		os.Exit(exitMissingSecret)
+	}
 	if err != nil {
 		return err
 	}
+	if *keep {
+		payload["keepContainer"] = true
+	}
+	b, err := json.MarshalIndent(payload, "", "  ")
+	if err != nil {
+		return err
+	}
+	if *out == "" {
+		_, err = os.Stdout.Write(b)
+		return err
+	}
+	return os.WriteFile(*out, b, 0o644)
+}
+
+type missingSecretError struct{ game, secret, key string }
+
+func (e *missingSecretError) Error() string {
+	return fmt.Sprintf("%s needs secret %s for %s", e.game, e.secret, e.key)
+}
+
+// bootTestPayload builds the DEPLOY_SERVER message the panel would send for a game, plus the boot-test
+// settings BootTest reads: ready and fail patterns, timeouts, how long the server must stay up after it is
+// ready, and the container ports it must then listen on.
+func bootTestPayload(gamesDir, id string, fillBlank bool) (map[string]any, error) {
+	dir := filepath.Join(gamesDir, id)
+	g, err := loadMap(filepath.Join(dir, "game.yaml"))
+	if err != nil {
+		return nil, err
+	}
 	test := map[string]any{}
 	if _, err := os.Stat(filepath.Join(dir, "test.yaml")); err == nil {
 		if test, err = loadMap(filepath.Join(dir, "test.yaml")); err != nil {
-			return err
+			return nil, err
 		}
 	}
 
@@ -68,8 +100,7 @@ func runBootTestPayload(args []string) error {
 		for key, name := range s {
 			value, ok := os.LookupEnv(fmt.Sprint(name))
 			if !ok || value == "" {
-				fmt.Fprintf(os.Stderr, "skip: %s needs secret %s for %s\n", id, name, key)
+				return nil, &missingSecretError{game: id, secret: fmt.Sprint(name), key: key}
-				os.Exit(exitMissingSecret)
 			}
 			config[key] = value
 		}
@@ -79,7 +110,7 @@ func runBootTestPayload(args []string) error {
 	// one: they passed CI and failed on the first real deploy.
 	for key, p := range schemaProperties(g) {
 		prop, _ := p.(map[string]any)
-		if prop["x-generate"] == nil && (!*fillBlank || !passwordKey.MatchString(key)) {
+		if prop["x-generate"] == nil && (!fillBlank || !passwordKey.MatchString(key)) {
 			continue
 		}
 		// Only missing or empty text: ports, booleans and other values are never touched
@@ -93,22 +124,26 @@ func runBootTestPayload(args []string) error {
 		}
 	}
 
-	timeout, settle := 900, 120
+	timeout, settle, hold := 900, 120, 60
 	if g["installMethod"] == "STEAMCMD" {
 		timeout = 2400
 	}
-	if v, ok := test["timeoutSec"].(json.Number); ok {
+	for key, dst := range map[string]*int{"timeoutSec": &timeout, "settleSec": &settle, "holdSec": &hold} {
-		if n, err := v.Int64(); err == nil {
+		if v, ok := test[key].(json.Number); ok {
-			timeout = int(n)
+			if n, err := v.Int64(); err == nil {
+				*dst = int(n)
+			}
 		}
 	}
-	if v, ok := test["settleSec"].(json.Number); ok {
+	// Once up, the server must listen on what nodes publish: the container side of every default port
-		if n, err := v.Int64(); err == nil {
+	expectPorts := []any{}
-			settle = int(n)
+	if ports, ok := g["defaultPorts"].([]any); ok {
+		for _, p := range ports {
+			expectPorts = append(expectPorts, fmt.Sprint(p))
 		}
 	}
 
-	payload := map[string]any{
+	return map[string]any{
 		"command":       "DEPLOY_SERVER",
 		"serverId":      "boottest-" + id,
 		"containerName": "boottest-" + id,
@@ -130,16 +165,9 @@ func runBootTestPayload(args []string) error {
 		"failPatterns":        g["failPatterns"],
 		"timeoutSec":          timeout,
 		"settleSec":           settle,
-	}
+		"holdSec":             hold,
-	b, err := json.MarshalIndent(payload, "", "  ")
+		"expectPorts":         expectPorts,
-	if err != nil {
+	}, nil
-		return err
-	}
-	if *out == "" {
-		_, err = os.Stdout.Write(b)
-		return err
-	}
-	return os.WriteFile(*out, b, 0o644)
 }
 
 // resolvePlaceholders stands in for what the panel fills per server; saved values have no owner here.

cmd/gbrgames/boottest_test.go

@@ -2,6 +2,7 @@ package main
 
 import (
 	"encoding/json"
+	"errors"
 	"os"
 	"path/filepath"
 	"strings"
@@ -79,3 +80,29 @@ configSchema:
 		t.Errorf("-fill-blank-passwords: ServerPassword not filled: %v", payload.GameConfig["ServerPassword"])
 	}
 }
+
+func TestBootTestPayloadAsksForPublishedPortsAndHold(t *testing.T) {
+	dir := t.TempDir()
+	game := filepath.Join(dir, "example")
+	os.MkdirAll(game, 0o755)
+	os.WriteFile(filepath.Join(game, "game.yaml"), []byte("id: example\nname: Example\ndefaultDockerImage: x\ndefaultPorts:\n  - 30000/udp\n  - 30001:30001/tcp\n"), 0o644)
+	os.WriteFile(filepath.Join(game, "test.yaml"), []byte("holdSec: 90\n"), 0o644)
+	p, err := bootTestPayload(dir, "example", false)
+	if err != nil {
+		t.Fatal(err)
+	}
+	ports, _ := p["expectPorts"].([]any)
+	if len(ports) != 2 || ports[0] != "30000/udp" || ports[1] != "30001:30001/tcp" {
+		t.Errorf("expectPorts %v", p["expectPorts"])
+	}
+	if p["holdSec"] != 90 || p["settleSec"] != 120 {
+		t.Errorf("hold %v settle %v", p["holdSec"], p["settleSec"])
+	}
+
+	os.WriteFile(filepath.Join(game, "test.yaml"), []byte("secrets:\n  OwnerId: EXAMPLE_OWNER\n"), 0o644)
+	_, err = bootTestPayload(dir, "example", false)
+	var missing *missingSecretError
+	if !errors.As(err, &missing) || missing.secret != "EXAMPLE_OWNER" {
+		t.Errorf("missing secret: %v", err)
+	}
+}

cmd/gbrgames/build.go

@@ -7,7 +7,6 @@ import (
 	"flag"
 	"fmt"
 	"os"
-	"os/exec"
 	"path/filepath"
 	"sort"
 	"strings"
@@ -298,46 +297,18 @@ func bootCheck(gamesDir, id, agentImage string, wantOutside bool) (problems, ver
 // CI. fillBlank also fills blank password-like settings, only to find out whether they are why a draft
 // fails. problems is set when the template can't be boot-tested at all.
 func agentBoot(gamesDir, id, agentImage string, fillBlank bool) (output string, passed bool, problems []string, err error) {
-	self, err := os.Executable()
+	payload, err := bootTestPayload(gamesDir, id, fillBlank)
+	var missing *missingSecretError
+	if errors.As(err, &missing) {
+		// The builder has no CI secrets: that's the template's problem to fix, not a reason to stop
+		return "", false, []string{"the boot test can't run: " + missing.Error() + ". Remove 'secrets' from " +
+			"test.yaml; boot tests run with exactly what a new server gets"}, nil
+	}
 	if err != nil {
 		return "", false, nil, err
 	}
-	// A boot test that was killed (worker restarted mid-build) leaves its container and volume behind,
+	output, passed, err = runAgentBootTest(payload, agentImage)
-	// and the next deploy under the same name would trip over them
+	return output, passed, nil, err
-	runCmd("docker", "rm", "-f", "boottest-"+id)
-	runCmd("docker", "volume", "rm", "-f", "boottest-"+id+"-data")
-	args := []string{"boottest-payload", "-games", gamesDir, id}
-	if fillBlank {
-		args = []string{"boottest-payload", "-fill-blank-passwords", "-games", gamesDir, id}
-	}
-	payloadCmd := exec.Command(self, args...)
-	var stderr bytes.Buffer
-	payloadCmd.Stderr = &stderr
-	payload, err := payloadCmd.Output()
-	if err != nil {
-		var exitErr *exec.ExitError
-		if errors.As(err, &exitErr) && exitErr.ExitCode() == exitMissingSecret {
-			// The builder has no CI secrets: that's the template's problem to fix, not a reason to stop
-			return "", false, []string{"the boot test can't run: " + strings.TrimSpace(stderr.String()) + ". Remove 'secrets' from " +
-				"test.yaml; boot tests run with exactly what a new server gets"}, nil
-		}
-		return "", false, nil, fmt.Errorf("boottest-payload: %v: %s", err, strings.TrimSpace(stderr.String()))
-	}
-	cmd := exec.Command("docker", "run", "--rm", "-i", "-v", "/var/run/docker.sock:/var/run/docker.sock",
-		"--entrypoint", "java", agentImage, "-Dloader.main=com.gbreed.gbr.nodeagent.BootTest", "-cp", "app.jar",
-		"org.springframework.boot.loader.launch.PropertiesLauncher")
-	cmd.Stdin = bytes.NewReader(payload)
-	var out bytes.Buffer
-	cmd.Stdout = &teeWriter{&out}
-	cmd.Stderr = cmd.Stdout
-	if runErr := cmd.Run(); runErr != nil {
-		var exitErr *exec.ExitError
-		if !errors.As(runErr, &exitErr) {
-			return "", false, nil, runErr
-		}
-		return out.String(), false, nil, nil
-	}
-	return out.String(), true, nil, nil
 }
 
 // blankRequiredLooking lists the settings a new server gets empty that look like a server might need a
@@ -372,7 +343,7 @@ func emptySettingsProblem(keys []string, verdict string) string {
 
 func firstBootGame(gamesDir, id string) (*bootReport, error) {
 	outDir := filepath.Join("firstboot-out", id+"-draft")
-	if err := runFirstBoot([]string{"-games", gamesDir, "-game", id, "-out", outDir}); err != nil {
+	if err := runFirstBoot([]string{"-games", gamesDir, "-game", id, "-as-deployed", "-out", outDir}); err != nil {
 		return nil, err
 	}
 	return readBootReport(filepath.Join(outDir, "report.json"))

cmd/gbrgames/firstboot.go

@@ -20,15 +20,11 @@ import (
 	"time"
 )
 
-// first-boot starts a game server once with none of a template's settings applied, and records what it
+// first-boot starts a game server once with none of a template's settings applied (or, -as-deployed,
-// does on its own: its output (to find the "ready" and error lines) and the files it creates (to learn
+// with exactly the settings a new server gets), and records what it does on its own: its output (to find
-// the real config file, format and defaults). It mirrors how the node-agent runs servers: same volumes,
+// the "ready" and error lines) and the files it creates (to learn the real config file, format and
-// user, SteamCMD install and startup prefix, but without a restart policy, so a crash shows as a crash.
+// defaults, and what would be lost on a restart). The node-agent's own deploy code starts it, so it runs
-
+// exactly as on a node: volumes, user, limits, SteamCMD install and startup command.
-// Same prefix the node-agent puts in front of every startup command (DockerService)
-const agentStartupPrefix = "mkdir -p ~/.steam/sdk64 /server/.steam/sdk64 2>/dev/null; " +
-	"[ -f /home/steam/steamcmd/linux64/steamclient.so ] && cp -f /home/steam/steamcmd/linux64/steamclient.so ~/.steam/sdk64/ 2>/dev/null; " +
-	"[ -f /server/linux64/steamclient.so ] && cp -f /server/linux64/steamclient.so ~/.steam/sdk64/ 2>/dev/null; cd /server && "
 
 var (
 	ansi       = regexp.MustCompile(`\x1b\[[0-9;?]*[A-Za-z]|\x1b\][^\a]*\a`)
@@ -104,6 +100,8 @@ func runFirstBoot(args []string) error {
 	quiet := fs.Duration("quiet", 2*time.Minute, "stop once the server has printed nothing for this long")
 	outDir := fs.String("out", "", "where to write the report (default firstboot-out/<name>)")
 	keep := fs.Bool("keep", false, "keep the container and volumes for a closer look")
+	asDeployed := fs.Bool("as-deployed", false, "with -game: boot with the template's settings exactly as a new server gets "+
+		"them (default: untouched, so the server writes its own defaults)")
 	var envFlags multiFlag
 	fs.Var(&envFlags, "env", "KEY=VALUE for the container (repeatable), e.g. EULA=TRUE")
 	fs.Parse(args)
@@ -138,7 +136,19 @@ func runFirstBoot(args []string) error {
 		return err
 	}
 
-	b := &booter{spec: spec, facts: facts, container: "firstboot-" + spec.Name, outDir: *outDir}
+	b := &booter{spec: spec, facts: facts, container: "firstboot-" + spec.Name, outDir: *outDir, agentImage: agentImageFromEnv()}
+	b.payload = untouchedPayload(spec)
+	if *asDeployed {
+		if *gameID == "" {
+			return errors.New("-as-deployed needs -game")
+		}
+		if b.payload, err = bootTestPayload(*gamesDir, *gameID, false); err != nil {
+			return err
+		}
+	}
+	if out, err := runCmd("docker", "pull", "-q", b.agentImage); err != nil {
+		return fmt.Errorf("docker pull %s: %v: %s", b.agentImage, err, out)
+	}
 	if !*keep {
 		defer b.cleanup()
 	}
@@ -221,11 +231,13 @@ func buildBootSpec(gamesDir, gameID, image, startup, steamApp string, memory int
 }
 
 type booter struct {
-	spec      *bootSpec
+	spec       *bootSpec
-	facts     *imageFacts
+	facts      *imageFacts
-	container string
+	container  string
-	outDir    string
+	outDir     string
-	mounts    map[string]string // mount path -> volume name
+	agentImage string
+	payload    map[string]any    // what the node-agent deploys
+	mounts     map[string]string // mount path -> volume name
 
 	mu      sync.Mutex
 	lines   []logLine
@@ -233,64 +245,56 @@ type booter struct {
 	started time.Time
 }
 
-func (b *booter) run(maxRun, quiet time.Duration) (*bootReport, error) {
+// untouchedPayload deploys an image as a node would, but with none of a template's settings: only the
-	exec.Command("docker", "rm", "-f", b.container).Run() // leftovers from an interrupted run
+// install, the startup command and the -env values, so the server writes its own defaults.
+func untouchedPayload(spec *bootSpec) map[string]any {
+	config := map[string]any{}
+	for k, v := range spec.Env {
+		config[k] = v
+	}
+	p := map[string]any{
+		"command": "DEPLOY_SERVER", "dockerImage": spec.Image, "gameConfig": config, "ports": []any{},
+		"installMethod": "NONE", "memoryLimitMb": spec.MemoryMb, "containerUser": spec.User,
+		"dockerNetwork": bootTestNetwork, "timeoutSec": 900,
+	}
+	if spec.Startup != "" {
+		p["startupCommand"] = spec.Startup
+	}
+	if spec.SteamAppID != "" {
+		p["installMethod"], p["installConfig"], p["timeoutSec"] = "STEAMCMD", map[string]any{"appId": spec.SteamAppID}, 2400
+	}
+	return p
+}
 
-	// Same layout as the node: one volume at /server and /data, plus a fresh one per declared VOLUME
+func (b *booter) run(maxRun, quiet time.Duration) (*bootReport, error) {
+	// Exactly what nodes mount: one volume at /server and /data. Anything else the server writes, the
+	// image's own VOLUME folders included, is gone when the node recreates the container on the next start
 	data := b.container + "-data"
 	b.mounts = map[string]string{"/server": data, "/data": data}
-	for i, v := range b.facts.Volumes {
-		if _, ok := b.mounts[v]; !ok {
-			b.mounts[v] = fmt.Sprintf("%s-v%d", b.container, i)
-		}
-	}
-	for _, vol := range uniqueValues(b.mounts) {
-		if err := docker("volume", "create", vol); err != nil {
-			return nil, err
-		}
-		if err := docker("run", "--rm", "-v", vol+":/v", "alpine", "chown", "-R", b.spec.User, "/v"); err != nil {
-			return nil, err
-		}
-	}
 
-	if b.spec.SteamAppID != "" {
+	// The node-agent deploys it (install, config, startup, user, limits) and leaves the container to us.
-		fmt.Printf("[first-boot] installing Steam app %s\n", b.spec.SteamAppID)
+	// It only has to get it running: watching it until it goes quiet is done here
-		// Like the node-agent: steamcmd's image runs as uid 1000, then the volume goes to the game's user
+	p := map[string]any{}
-		steam := "/home/steam/steamcmd/steamcmd.sh +login anonymous +quit && /home/steam/steamcmd/steamcmd.sh +force_install_dir /server +login anonymous +app_update " +
+	for k, v := range b.payload {
-			b.spec.SteamAppID + " validate +quit"
+		p[k] = v
-		if err := docker("run", "--rm", "-v", data+":/server", "alpine", "chown", "-R", nodeUser, "/server"); err != nil {
-			return nil, err
-		}
-		cmd := exec.Command("docker", "run", "--rm", "--network", bootTestNetwork, "-v", data+":/server", "cm2network/steamcmd", "bash", "-c", steam)
-		cmd.Stdout, cmd.Stderr = os.Stdout, os.Stderr
-		if err := cmd.Run(); err != nil {
-			return nil, fmt.Errorf("steamcmd install of %s failed: %w", b.spec.SteamAppID, err)
-		}
-		if err := docker("run", "--rm", "-v", data+":/server", "alpine", "chown", "-R", b.spec.User, "/server"); err != nil {
-			return nil, err
-		}
 	}
+	p["containerName"], p["serverId"], p["keepContainer"] = b.container, b.container, true
+	p["settleSec"], p["holdSec"], p["expectPorts"] = 15, 0, []any{}
+	delete(p, "readyPattern")
+	delete(p, "failPatterns")
+	fmt.Printf("[first-boot] deploying %s as %v with the node-agent (memory %v MB)\n", b.spec.Image, displayUser(p["containerUser"]), p["memoryLimitMb"])
+	output, _, err := runAgentBootTest(p, b.agentImage)
+	if err != nil {
+		return nil, err
+	}
+	created, err := time.Parse(time.RFC3339Nano, inspectField(b.container, "{{.Created}}"))
+	if err != nil {
+		return nil, fmt.Errorf("the node-agent could not deploy %s: %s", b.spec.Image, bootVerdict(output))
+	}
+	b.started, b.lastAt = created, created
 
-	runArgs := []string{"run", "-d", "-t", "--init", "--name", b.container, "--user", b.spec.User,
+	// The whole console from the start, with Docker's timestamps
-		"--security-opt", "no-new-privileges:true", "--pids-limit", "1024", "--network", bootTestNetwork,
+	logs := exec.Command("docker", "logs", "-t", "-f", b.container)
-		"-m", fmt.Sprintf("%dm", b.spec.MemoryMb), "--memory-swap", fmt.Sprintf("%dm", b.spec.MemoryMb)}
-	for mount, vol := range b.mounts {
-		runArgs = append(runArgs, "-v", vol+":"+mount)
-	}
-	for _, k := range sortedKeysString(b.spec.Env) {
-		runArgs = append(runArgs, "-e", k+"="+b.spec.Env[k])
-	}
-	runArgs = append(runArgs, b.spec.Image)
-	if b.spec.Startup != "" {
-		runArgs = append(runArgs, "bash", "-c", agentStartupPrefix+b.spec.Startup)
-	}
-	fmt.Printf("[first-boot] starting %s as %s (memory %d MB)\n", b.spec.Image, b.spec.User, b.spec.MemoryMb)
-	if out, err := exec.Command("docker", runArgs...).CombinedOutput(); err != nil {
-		return nil, fmt.Errorf("docker run: %v: %s", err, strings.TrimSpace(string(out)))
-	}
-	b.started, b.lastAt = time.Now(), time.Now()
-
-	logs := exec.Command("docker", "logs", "-f", b.container)
 	pr, pw := io.Pipe()
 	logs.Stdout, logs.Stderr = pw, pw
 	if err := logs.Start(); err != nil {
@@ -301,27 +305,31 @@ func (b *booter) run(maxRun, quiet time.Duration) (*bootReport, error) {
 		sc := bufio.NewScanner(pr)
 		sc.Buffer(make([]byte, 1024*1024), 1024*1024)
 		for sc.Scan() {
-			b.addLine("console", sc.Text())
+			b.addTimestamped(sc.Text())
 		}
 		close(done)
 	}()
 
-	report := &bootReport{Image: b.spec.Image, User: b.spec.User}
+	report := &bootReport{Image: b.spec.Image, User: displayUser(p["containerUser"])}
 	minRun := time.Minute
 	for {
 		time.Sleep(2 * time.Second)
 		elapsed := time.Since(b.started)
 		running := inspectField(b.container, "{{.State.Running}}") == "true"
+		restarts, _ := strconv.Atoi(inspectField(b.container, "{{.RestartCount}}"))
 		b.mu.Lock()
 		silent, count := time.Since(b.lastAt), len(b.lines)
 		b.mu.Unlock()
 		switch {
-		case !running:
+		case !running || restarts > 0:
 			code, _ := strconv.Atoi(inspectField(b.container, "{{.State.ExitCode}}"))
 			report.ExitCode = &code
 			report.Outcome = fmt.Sprintf("exited by itself with code %d after %.0fs", code, elapsed.Seconds())
+			if restarts > 0 {
+				report.Outcome = fmt.Sprintf("crashed (exit code %d) and was restarted %d time(s) within %.0fs", code, restarts, elapsed.Seconds())
+			}
 			if inspectField(b.container, "{{.State.OOMKilled}}") == "true" {
-				report.Outcome += fmt.Sprintf(" (killed: over the %d MB memory limit)", b.spec.MemoryMb)
+				report.Outcome += fmt.Sprintf(" (killed: over the %v MB memory limit)", p["memoryLimitMb"])
 			}
 		case elapsed >= minRun && count > 0 && silent >= quiet:
 			report.Outcome = fmt.Sprintf("running and quiet for %.0fs after %.0fs; stopped gracefully", silent.Seconds(), elapsed.Seconds())
@@ -335,6 +343,7 @@ func (b *booter) run(maxRun, quiet time.Duration) (*bootReport, error) {
 	report.RanSeconds = time.Since(b.started).Seconds()
 	// Many servers only write their settings when they shut down
 	exec.Command("docker", "stop", "-t", "60", b.container).Run()
+	logs.Process.Kill()
 	pw.Close()
 	<-done
 
@@ -354,17 +363,33 @@ func (b *booter) run(maxRun, quiet time.Duration) (*bootReport, error) {
 	return report, nil
 }
 
-func (b *booter) addLine(source, raw string) {
+// addTimestamped records one line of `docker logs -t`: "2026-10-26T01:11:52.123456789Z text".
+func (b *booter) addTimestamped(raw string) {
+	at := time.Now()
+	if stamp, rest, ok := strings.Cut(raw, " "); ok {
+		if t, err := time.Parse(time.RFC3339Nano, stamp); err == nil {
+			at, raw = t, rest
+		}
+	}
 	text := strings.TrimRight(ansi.ReplaceAllString(raw, ""), "\r \t")
 	if text == "" {
 		return
 	}
 	b.mu.Lock()
-	b.lines = append(b.lines, logLine{At: time.Since(b.started).Round(100 * time.Millisecond).Seconds(), Source: source, Text: text})
+	b.lines = append(b.lines, logLine{At: at.Sub(b.started).Round(100 * time.Millisecond).Seconds(), Source: "console", Text: text})
-	b.lastAt = time.Now()
+	if at.After(b.lastAt) {
+		b.lastAt = at
+	}
 	b.mu.Unlock()
 }
 
+func displayUser(u any) string {
+	if s, ok := u.(string); ok && s != "" {
+		return s
+	}
+	return nodeUser
+}
+
 func (b *booter) console() []logLine {
 	b.mu.Lock()
 	defer b.mu.Unlock()
@@ -487,39 +512,88 @@ func (b *booter) fileLines(report *bootReport) []logLine {
 	return out
 }
 
-// containerChanges lists files the server wrote outside its volumes: settings there are lost when the
+// System folders and files that aren't the game's state: what's written there doesn't matter when lost
-// node recreates the container, which a template needs to know about.
+var (
+	systemPrefixes = []string{"/tmp", "/proc", "/sys", "/dev", "/run", "/etc", "/var/log", "/var/cache", "/var/tmp", "/var/lib/apt"}
+	transientExt   = map[string]bool{".log": true, ".pid": true, ".lock": true, ".tmp": true, ".swp": true}
+)
+
+// containerChanges lists the files the server wrote anywhere but /server and /data. All of them are
+// lost when the node recreates the container (every start): in the container's own filesystem, and in
+// the image's VOLUME folders, which Docker gives fresh anonymous volumes each time. Logs, lock and pid
+// files are left out; saves of any kind are not.
 func (b *booter) containerChanges() []string {
-	out, err := exec.Command("docker", "diff", b.container).Output()
-	if err != nil {
-		return nil
-	}
 	var changes []string
-	for _, line := range strings.Split(strings.TrimSpace(string(out)), "\n") {
+	if out, err := exec.Command("docker", "diff", b.container).Output(); err == nil {
-		kind, p, ok := strings.Cut(line, " ")
+		changes = append(changes, outsideFiles(string(out), b.mounts)...)
-		if !ok || kind == "D" {
+	}
+	// The image's VOLUME folders: Docker mounted anonymous volumes there, which docker diff doesn't show
+	mounts, _ := exec.Command("docker", "inspect", "-f", `{{range .Mounts}}{{if eq .Type "volume"}}{{.Name}}|{{.Destination}}{{"\n"}}{{end}}{{end}}`, b.container).Output()
+	for _, line := range strings.Split(strings.TrimSpace(string(mounts)), "\n") {
+		vol, dest, ok := strings.Cut(line, "|")
+		if !ok || b.mounts[dest] != "" {
 			continue
 		}
-		skip := false
+		files, err := exec.Command("docker", "run", "--rm", "-v", vol+":/v:ro", "alpine", "find", "/v", "-type", "f").Output()
-		for _, prefix := range []string{"/tmp", "/proc", "/sys", "/dev", "/run", "/var/log", "/var/cache", "/var/tmp"} {
+		if err != nil {
-			if p == prefix || strings.HasPrefix(p, prefix+"/") {
+			continue
-				skip = true
-			}
 		}
-		for mount := range b.mounts {
+		n := 0
-			if p == mount || strings.HasPrefix(p, mount+"/") {
+		for _, f := range strings.Split(strings.TrimSpace(string(files)), "\n") {
-				skip = true
+			rel := strings.TrimPrefix(f, "/v/")
+			if f == "" || transientExt[strings.ToLower(path.Ext(rel))] {
+				continue
 			}
-		}
+			if n++; n > 200 {
-		if !skip && configExt[strings.ToLower(path.Ext(p))] {
+				break
-			changes = append(changes, kind+" "+p)
+			}
+			changes = append(changes, "A "+path.Join(dest, rel))
 		}
 	}
 	return changes
 }
 
+// outsideFiles picks the files out of `docker diff` output: added or changed leaves (docker diff lists
+// folders too) outside the mounted volumes and system folders, without transient files.
+func outsideFiles(diff string, mounts map[string]string) []string {
+	var paths []string
+	kinds := map[string]string{}
+	for _, line := range strings.Split(strings.TrimSpace(diff), "\n") {
+		kind, p, ok := strings.Cut(strings.TrimSpace(line), " ")
+		if !ok || kind == "D" || p == "" {
+			continue
+		}
+		skip := transientExt[strings.ToLower(path.Ext(p))]
+		for _, prefix := range systemPrefixes {
+			if p == prefix || strings.HasPrefix(p, prefix+"/") {
+				skip = true
+			}
+		}
+		for mount := range mounts {
+			if p == mount || strings.HasPrefix(p, mount+"/") {
+				skip = true
+			}
+		}
+		if !skip {
+			paths = append(paths, p)
+			kinds[p] = kind
+		}
+	}
+	sort.Strings(paths)
+	var leaves []string
+	for i, p := range paths {
+		// A folder is followed (sorted) by the entries inside it
+		if i+1 < len(paths) && strings.HasPrefix(paths[i+1], p+"/") {
+			continue
+		}
+		leaves = append(leaves, kinds[p]+" "+p)
+	}
+	return leaves
+}
+
 func (b *booter) cleanup() {
-	exec.Command("docker", "rm", "-f", b.container).Run()
+	// -v: the anonymous volumes of the image's VOLUME folders go with it
+	exec.Command("docker", "rm", "-f", "-v", b.container).Run()
 	for _, vol := range uniqueValues(b.mounts) {
 		exec.Command("docker", "volume", "rm", vol).Run()
 	}
@@ -730,13 +804,6 @@ func describeLine(l logLine) string {
 	return fmt.Sprintf("[%s] %s", where, text)
 }
 
-func docker(args ...string) error {
-	if out, err := exec.Command("docker", args...).CombinedOutput(); err != nil {
-		return fmt.Errorf("docker %s: %v: %s", args[0], err, strings.TrimSpace(string(out)))
-	}
-	return nil
-}
-
 func inspectField(container, format string) string {
 	out, _ := exec.Command("docker", "inspect", "-f", format, container).Output()
 	return strings.TrimSpace(string(out))

cmd/gbrgames/firstboot_test.go

@@ -2,6 +2,7 @@ package main
 
 import (
 	"regexp"
+	"strings"
 	"testing"
 )
 
@@ -103,3 +104,39 @@ func TestConfigCandidates(t *testing.T) {
 		}
 	}
 }
+
+func TestOutsideFilesCountsEverySaveButNotSystemOrTransientFiles(t *testing.T) {
+	diff := `C /home
+C /home/steam
+A /home/steam/.config
+A /home/steam/.config/example
+A /home/steam/.config/example/worlds
+A /home/steam/.config/example/worlds/world1.db
+A /home/steam/.config/example/worlds/world1.fwl
+A /home/steam/.config/example/Player.log
+C /etc
+C /etc/hostname
+A /tmp/x.sock
+A /server/should-not-count.sav
+D /home/steam/old.txt
+A /home/steam/server.pid`
+	got := outsideFiles(diff, map[string]string{"/server": "v", "/data": "v"})
+	want := []string{"A /home/steam/.config/example/worlds/world1.db", "A /home/steam/.config/example/worlds/world1.fwl"}
+	if strings.Join(got, "|") != strings.Join(want, "|") {
+		t.Errorf("got %q\nwant %q", got, want)
+	}
+}
+
+func TestUntouchedPayloadDeploysWithoutTemplateSettings(t *testing.T) {
+	p := untouchedPayload(&bootSpec{Image: "example/image", Startup: "./server", SteamAppID: "1000000", MemoryMb: 4096,
+		User: "1000:1000", Env: map[string]string{"EULA": "TRUE"}})
+	if p["installMethod"] != "STEAMCMD" || p["installConfig"].(map[string]any)["appId"] != "1000000" || p["timeoutSec"] != 2400 {
+		t.Errorf("steam install: %v", p)
+	}
+	if p["startupCommand"] != "./server" || p["dockerNetwork"] != bootTestNetwork || p["gameConfig"].(map[string]any)["EULA"] != "TRUE" {
+		t.Errorf("payload: %v", p)
+	}
+	if _, ok := untouchedPayload(&bootSpec{Image: "x"})["startupCommand"]; ok {
+		t.Error("no startup command means the image's own entrypoint")
+	}
+}

schema/test.schema.json

@@ -32,6 +32,12 @@
       "type": "integer",
       "minimum": 10,
       "maximum": 3600
+    },
+    "holdSec": {
+      "description": "How long the server must stay up after its ready line to pass (default 60). Its published ports are checked after it.",
+      "type": "integer",
+      "minimum": 0,
+      "maximum": 3600
     }
   }
 }