From 6baf4796fa0f7b7a36d65bbb764c42324ce403d6 Mon Sep 17 00:00:00 2001 From: Greg Date: Mon, 26 Oct 2026 19:12:20 -0400 Subject: [PATCH] feat(boot-test): CI boot tests use the Steam download cache, updated once a day CI deployed every Steam game from scratch on each run, downloading the whole game from Steam each time. scripts/boot-test.sh now runs the new `gbrgames boot-test ID`, the builder's own deploy path: the same payload and node-agent BootTest, but Steam games start from the cache volume (gbr-steamcache-, kept by RUNNER.md's prune). The cache is updated at most once a day (a timestamp in the cache volume, not copied into servers), so a repeat run within a day doesn't ask Steam for an update at all; the node-agent's own install still checks the files and fetches anything newer. Co-Authored-By: Claude Opus 5.5 --- RUNNER.md | 12 +++++-- cmd/gbrgames/agentboot.go | 63 +++++++++++++++++++++++++++++++++-- cmd/gbrgames/boottest_test.go | 11 ++++++ cmd/gbrgames/main.go | 5 ++- scripts/boot-test.sh | 24 ++++--------- 5 files changed, 93 insertions(+), 22 deletions(-) diff --git a/RUNNER.md b/RUNNER.md index c7f19ac..9c8dce2 100644 --- a/RUNNER.md +++ b/RUNNER.md @@ -61,5 +61,13 @@ docker volume prune -af --filter "label!=gbr.cache" ``` Not `docker system prune`: it also deletes the `gbr-boottest` network whenever no container is using it, -and boot tests then refuse to run until `systemctl restart gbr-boottest-network`. Volumes labelled -`gbr.cache` are the AI builder's Steam download cache. +and boot tests then refuse to run until `systemctl restart gbr-boottest-network`. + +### Steam download cache + +Boot tests of Steam games (`installMethod: STEAMCMD`) start from a cached install, one Docker volume per +app (`gbr-steamcache-`, label `gbr.cache=steam`), the same cache the AI builder uses. The +cache is updated at most once a day; the node-agent's own install then only checks the files and +fetches what changed. So Steam isn't asked for the whole game on every run, which keeps traffic from +this machine's address low. The prune above keeps these volumes. To drop one game's cache: +`docker volume rm gbr-steamcache-`; `STEAM_CACHE=off` turns the cache off. diff --git a/cmd/gbrgames/agentboot.go b/cmd/gbrgames/agentboot.go index d145077..4d1b9e1 100644 --- a/cmd/gbrgames/agentboot.go +++ b/cmd/gbrgames/agentboot.go @@ -4,10 +4,14 @@ import ( "bytes" "encoding/json" "errors" + "flag" "fmt" "os" "os/exec" "regexp" + "strconv" + "strings" + "time" ) // Every server the builder starts, first boots included, is deployed by the node-agent's own code @@ -73,14 +77,25 @@ var steamCacheFresh = map[string]bool{} var numericAppID = regexp.MustCompile(`^\d+$`) +// When the cache was last updated, in the cache volume (not copied into servers) +const steamCacheStamp = ".gbr-cache-updated" + +// steamCacheMaxAge: a cache updated more recently isn't updated again, which saves Steam a login and an +// update check per boot; the node-agent's own install still fetches anything newer +const steamCacheMaxAge = 24 * time.Hour + // seedSteamApp fills a server volume from a cached SteamCMD install of the app, so a boot doesn't // download the whole game again: the node-agent's own install then only checks the files. The cache is -// updated once per run. Without it (STEAM_CACHE=off, or the update failing) the agent downloads as usual. +// updated at most once a day. Without it (STEAM_CACHE=off, or the update failing) the agent downloads as usual. func seedSteamApp(appID, volume string) { if os.Getenv("STEAM_CACHE") == "off" || !numericAppID.MatchString(appID) { return } cache := "gbr-steamcache-" + appID + if !steamCacheFresh[appID] && steamCacheUpdatedWithin(cache, steamCacheMaxAge) { + fmt.Printf("[steam-cache] the cached install of Steam app %s was updated in the last %s\n", appID, steamCacheMaxAge) + steamCacheFresh[appID] = true + } if !steamCacheFresh[appID] { fmt.Printf("[steam-cache] updating the cached install of Steam app %s\n", appID) if out, err := runCmd("docker", "volume", "create", "--label", "gbr.cache=steam", cache); err != nil { @@ -98,12 +113,56 @@ func seedSteamApp(appID, volume string) { fmt.Printf("[steam-cache] update failed (%v); the boot downloads the app itself\n", err) return } + runCmd("docker", "run", "--rm", "-v", cache+":/server", "alpine", "sh", "-c", + fmt.Sprintf("date +%%s > /server/%s", steamCacheStamp)) steamCacheFresh[appID] = true } runCmd("docker", "volume", "create", volume) - if out, err := runCmd("docker", "run", "--rm", "-v", cache+":/from:ro", "-v", volume+":/to", "alpine", "cp", "-a", "/from/.", "/to/"); err != nil { + if out, err := runCmd("docker", "run", "--rm", "-v", cache+":/from:ro", "-v", volume+":/to", "alpine", "sh", "-c", + "cp -a /from/. /to/ && rm -f /to/"+steamCacheStamp); err != nil { fmt.Printf("[steam-cache] copying into %s failed (%v: %s); the boot downloads the app itself\n", volume, err, out) return } fmt.Printf("[steam-cache] %s seeded from the cached install of app %s\n", volume, appID) } + +func steamCacheUpdatedWithin(cache string, age time.Duration) bool { + out, err := runCmd("docker", "run", "--rm", "-v", cache+":/c:ro", "alpine", "cat", "/c/"+steamCacheStamp) + if err != nil { + return false + } + return stampWithin(out, time.Now(), age) +} + +func stampWithin(stamp string, now time.Time, age time.Duration) bool { + secs, err := strconv.ParseInt(strings.TrimSpace(stamp), 10, 64) + return err == nil && now.Sub(time.Unix(secs, 0)) < age +} + +// runBootTest boot-tests one game the way CI does: its real payload, deployed by the node-agent, with +// the Steam download cache. Exits 3 when a secret it needs isn't set, 1 when it fails. +func runBootTest(args []string) error { + fs := flag.NewFlagSet("boot-test", flag.ExitOnError) + gamesDir := fs.String("games", "games", "directory holding one folder per game") + fs.Parse(args) + if fs.NArg() != 1 { + return errors.New("boot-test needs a game id") + } + payload, err := bootTestPayload(*gamesDir, fs.Arg(0), false) + var missing *missingSecretError + if errors.As(err, &missing) { + fmt.Fprintln(os.Stderr, "skip: "+missing.Error()) + os.Exit(exitMissingSecret) + } + if err != nil { + return err + } + _, passed, err := runAgentBootTest(payload, agentImageFromEnv()) + if err != nil { + return err + } + if !passed { + os.Exit(1) + } + return nil +} diff --git a/cmd/gbrgames/boottest_test.go b/cmd/gbrgames/boottest_test.go index 60fd5ba..0d1f4ac 100644 --- a/cmd/gbrgames/boottest_test.go +++ b/cmd/gbrgames/boottest_test.go @@ -1,6 +1,7 @@ package main import ( + "time" "encoding/json" "errors" "os" @@ -121,3 +122,13 @@ func TestBootTestPayloadLeavesOutUnboundPorts(t *testing.T) { t.Errorf("expectPorts %v", ports) } } + +func TestSteamCacheStampAge(t *testing.T) { + now := time.Unix(1_800_000_000, 0) + if !stampWithin("1799990000\n", now, 24*time.Hour) { + t.Error("updated 10000s ago should be fresh") + } + if stampWithin("1700000000", now, 24*time.Hour) || stampWithin("", now, 24*time.Hour) || stampWithin("x", now, 24*time.Hour) { + t.Error("old, missing or garbled stamps are not fresh") + } +} diff --git a/cmd/gbrgames/main.go b/cmd/gbrgames/main.go index 9b95e90..805a65f 100644 --- a/cmd/gbrgames/main.go +++ b/cmd/gbrgames/main.go @@ -4,6 +4,7 @@ // go run ./cmd/gbrgames import export.json # write games// from GET /api/v1/admin/games // go run ./cmd/gbrgames export -out p.json # build the games_published payload CI sends to the panel // go run ./cmd/gbrgames boottest-payload ID # build the deploy message the node-agent's BootTest runs +// go run ./cmd/gbrgames boot-test ID # boot-test a game with the node-agent, as CI does (Steam cache) // go run ./cmd/gbrgames inspect-image -game ID # read the image's facts and check the template against them // go run ./cmd/gbrgames first-boot -game ID # start the server once, untouched; record its log and files // go run ./cmd/gbrgames draft -name N -image I # have an AI model write a new game's template from those facts @@ -33,6 +34,8 @@ func main() { err = runExport(os.Args[2:]) case "boottest-payload": err = runBootTestPayload(os.Args[2:]) + case "boot-test": + err = runBootTest(os.Args[2:]) case "inspect-image": err = runInspectImage(os.Args[2:]) case "first-boot": @@ -59,6 +62,6 @@ func main() { } func usage() { - fmt.Fprintln(os.Stderr, "usage: gbrgames validate [-games DIR] | import [-games DIR] [-all] EXPORT.json | export [-repo R] [-commit SHA] [-out FILE] | boottest-payload [-out FILE] ID | inspect-image [-game ID] [-json] [-no-pull] [IMAGE] | first-boot [-game ID | -image IMAGE] [-env K=V] [-max D] [-quiet D] [-out DIR] | draft -name NAME -image IMAGE [-steam-app ID] [-first-boot REPORT] [-provider ollama|gemini] [-model M] | research -name NAME [-notes N] [-out FILE] | build -name NAME -variant JSON|@FILE [-out FILE] | repair -id ID [-reason R] | eval [-ids all] [-out eval.md] | worker [-once]") + fmt.Fprintln(os.Stderr, "usage: gbrgames validate [-games DIR] | import [-games DIR] [-all] EXPORT.json | export [-repo R] [-commit SHA] [-out FILE] | boottest-payload [-out FILE] ID | boot-test ID | inspect-image [-game ID] [-json] [-no-pull] [IMAGE] | first-boot [-game ID | -image IMAGE] [-env K=V] [-max D] [-quiet D] [-out DIR] | draft -name NAME -image IMAGE [-steam-app ID] [-first-boot REPORT] [-provider ollama|gemini] [-model M] | research -name NAME [-notes N] [-out FILE] | build -name NAME -variant JSON|@FILE [-out FILE] | repair -id ID [-reason R] | eval [-ids all] [-out eval.md] | worker [-once]") os.Exit(2) } diff --git a/scripts/boot-test.sh b/scripts/boot-test.sh index 6e12708..ffbc78e 100755 --- a/scripts/boot-test.sh +++ b/scripts/boot-test.sh @@ -30,24 +30,14 @@ for id in "$@"; do echo "::endgroup::" continue fi - payload=$("$gbrgames" boottest-payload "$id") + # The node-agent's BootTest with the game's real payload. Steam games start from the runner's download + # cache (volumes gbr-steamcache-, kept by RUNNER.md's prune), so Steam isn't asked for the whole + # game on every run + AGENT_IMAGE="$AGENT_IMAGE" "$gbrgames" boot-test "$id" code=$? - if [ $code -eq 3 ]; then - skipped+=("$id") - echo "::endgroup::" - continue - elif [ $code -ne 0 ]; then - failed+=("$id") - echo "::endgroup::" - continue - fi - - docker run --rm -i \ - -v /var/run/docker.sock:/var/run/docker.sock \ - --entrypoint java "$AGENT_IMAGE" \ - -Dloader.main=com.gbreed.gbr.nodeagent.BootTest -cp app.jar \ - org.springframework.boot.loader.launch.PropertiesLauncher <<< "$payload" - if [ $? -eq 0 ]; then passed+=("$id"); else failed+=("$id"); fi + if [ $code -eq 3 ]; then skipped+=("$id") + elif [ $code -eq 0 ]; then passed+=("$id") + else failed+=("$id"); fi echo "::endgroup::" done