Merge pull request 'feat(boot-test): CI boot tests use the Steam download cache, updated once a day' (#26) from feat/ci-steam-cache into main

RUNNER.md

@@ -61,5 +61,13 @@ docker volume prune -af --filter "label!=gbr.cache"
 ```
 
 Not `docker system prune`: it also deletes the `gbr-boottest` network whenever no container is using it,
-and boot tests then refuse to run until `systemctl restart gbr-boottest-network`. Volumes labelled
+and boot tests then refuse to run until `systemctl restart gbr-boottest-network`.
-`gbr.cache` are the AI builder's Steam download cache.
+
+### Steam download cache
+
+Boot tests of Steam games (`installMethod: STEAMCMD`) start from a cached install, one Docker volume per
+app (`gbr-steamcache-<app id>`, label `gbr.cache=steam`), the same cache the AI builder uses. The
+cache is updated at most once a day; the node-agent's own install then only checks the files and
+fetches what changed. So Steam isn't asked for the whole game on every run, which keeps traffic from
+this machine's address low. The prune above keeps these volumes. To drop one game's cache:
+`docker volume rm gbr-steamcache-<app id>`; `STEAM_CACHE=off` turns the cache off.

cmd/gbrgames/agentboot.go

@@ -4,10 +4,14 @@ import (
 	"bytes"
 	"encoding/json"
 	"errors"
+	"flag"
 	"fmt"
 	"os"
 	"os/exec"
 	"regexp"
+	"strconv"
+	"strings"
+	"time"
 )
 
 // Every server the builder starts, first boots included, is deployed by the node-agent's own code
@@ -73,14 +77,25 @@ var steamCacheFresh = map[string]bool{}
 
 var numericAppID = regexp.MustCompile(`^\d+$`)
 
+// When the cache was last updated, in the cache volume (not copied into servers)
+const steamCacheStamp = ".gbr-cache-updated"
+
+// steamCacheMaxAge: a cache updated more recently isn't updated again, which saves Steam a login and an
+// update check per boot; the node-agent's own install still fetches anything newer
+const steamCacheMaxAge = 24 * time.Hour
+
 // seedSteamApp fills a server volume from a cached SteamCMD install of the app, so a boot doesn't
 // download the whole game again: the node-agent's own install then only checks the files. The cache is
-// updated once per run. Without it (STEAM_CACHE=off, or the update failing) the agent downloads as usual.
+// updated at most once a day. Without it (STEAM_CACHE=off, or the update failing) the agent downloads as usual.
 func seedSteamApp(appID, volume string) {
 	if os.Getenv("STEAM_CACHE") == "off" || !numericAppID.MatchString(appID) {
 		return
 	}
 	cache := "gbr-steamcache-" + appID
+	if !steamCacheFresh[appID] && steamCacheUpdatedWithin(cache, steamCacheMaxAge) {
+		fmt.Printf("[steam-cache] the cached install of Steam app %s was updated in the last %s\n", appID, steamCacheMaxAge)
+		steamCacheFresh[appID] = true
+	}
 	if !steamCacheFresh[appID] {
 		fmt.Printf("[steam-cache] updating the cached install of Steam app %s\n", appID)
 		if out, err := runCmd("docker", "volume", "create", "--label", "gbr.cache=steam", cache); err != nil {
@@ -98,12 +113,56 @@ func seedSteamApp(appID, volume string) {
 			fmt.Printf("[steam-cache] update failed (%v); the boot downloads the app itself\n", err)
 			return
 		}
+		runCmd("docker", "run", "--rm", "-v", cache+":/server", "alpine", "sh", "-c",
+			fmt.Sprintf("date +%%s > /server/%s", steamCacheStamp))
 		steamCacheFresh[appID] = true
 	}
 	runCmd("docker", "volume", "create", volume)
-	if out, err := runCmd("docker", "run", "--rm", "-v", cache+":/from:ro", "-v", volume+":/to", "alpine", "cp", "-a", "/from/.", "/to/"); err != nil {
+	if out, err := runCmd("docker", "run", "--rm", "-v", cache+":/from:ro", "-v", volume+":/to", "alpine", "sh", "-c",
+		"cp -a /from/. /to/ && rm -f /to/"+steamCacheStamp); err != nil {
 		fmt.Printf("[steam-cache] copying into %s failed (%v: %s); the boot downloads the app itself\n", volume, err, out)
 		return
 	}
 	fmt.Printf("[steam-cache] %s seeded from the cached install of app %s\n", volume, appID)
 }
+
+func steamCacheUpdatedWithin(cache string, age time.Duration) bool {
+	out, err := runCmd("docker", "run", "--rm", "-v", cache+":/c:ro", "alpine", "cat", "/c/"+steamCacheStamp)
+	if err != nil {
+		return false
+	}
+	return stampWithin(out, time.Now(), age)
+}
+
+func stampWithin(stamp string, now time.Time, age time.Duration) bool {
+	secs, err := strconv.ParseInt(strings.TrimSpace(stamp), 10, 64)
+	return err == nil && now.Sub(time.Unix(secs, 0)) < age
+}
+
+// runBootTest boot-tests one game the way CI does: its real payload, deployed by the node-agent, with
+// the Steam download cache. Exits 3 when a secret it needs isn't set, 1 when it fails.
+func runBootTest(args []string) error {
+	fs := flag.NewFlagSet("boot-test", flag.ExitOnError)
+	gamesDir := fs.String("games", "games", "directory holding one folder per game")
+	fs.Parse(args)
+	if fs.NArg() != 1 {
+		return errors.New("boot-test needs a game id")
+	}
+	payload, err := bootTestPayload(*gamesDir, fs.Arg(0), false)
+	var missing *missingSecretError
+	if errors.As(err, &missing) {
+		fmt.Fprintln(os.Stderr, "skip: "+missing.Error())
+		os.Exit(exitMissingSecret)
+	}
+	if err != nil {
+		return err
+	}
+	_, passed, err := runAgentBootTest(payload, agentImageFromEnv())
+	if err != nil {
+		return err
+	}
+	if !passed {
+		os.Exit(1)
+	}
+	return nil
+}

cmd/gbrgames/boottest_test.go

@@ -1,6 +1,7 @@
 package main
 
 import (
+	"time"
 	"encoding/json"
 	"errors"
 	"os"
@@ -121,3 +122,13 @@ func TestBootTestPayloadLeavesOutUnboundPorts(t *testing.T) {
 		t.Errorf("expectPorts %v", ports)
 	}
 }
+
+func TestSteamCacheStampAge(t *testing.T) {
+	now := time.Unix(1_800_000_000, 0)
+	if !stampWithin("1799990000\n", now, 24*time.Hour) {
+		t.Error("updated 10000s ago should be fresh")
+	}
+	if stampWithin("1700000000", now, 24*time.Hour) || stampWithin("", now, 24*time.Hour) || stampWithin("x", now, 24*time.Hour) {
+		t.Error("old, missing or garbled stamps are not fresh")
+	}
+}

cmd/gbrgames/main.go

@@ -4,6 +4,7 @@
 //	go run ./cmd/gbrgames import export.json  # write games/<id>/ from GET /api/v1/admin/games
 //	go run ./cmd/gbrgames export -out p.json  # build the games_published payload CI sends to the panel
 //	go run ./cmd/gbrgames boottest-payload ID # build the deploy message the node-agent's BootTest runs
+//	go run ./cmd/gbrgames boot-test ID        # boot-test a game with the node-agent, as CI does (Steam cache)
 //	go run ./cmd/gbrgames inspect-image -game ID   # read the image's facts and check the template against them
 //	go run ./cmd/gbrgames first-boot -game ID      # start the server once, untouched; record its log and files
 //	go run ./cmd/gbrgames draft -name N -image I   # have an AI model write a new game's template from those facts
@@ -33,6 +34,8 @@ func main() {
 		err = runExport(os.Args[2:])
 	case "boottest-payload":
 		err = runBootTestPayload(os.Args[2:])
+	case "boot-test":
+		err = runBootTest(os.Args[2:])
 	case "inspect-image":
 		err = runInspectImage(os.Args[2:])
 	case "first-boot":
@@ -59,6 +62,6 @@ func main() {
 }
 
 func usage() {
-	fmt.Fprintln(os.Stderr, "usage: gbrgames validate [-games DIR] | import [-games DIR] [-all] EXPORT.json | export [-repo R] [-commit SHA] [-out FILE] | boottest-payload [-out FILE] ID | inspect-image [-game ID] [-json] [-no-pull] [IMAGE] | first-boot [-game ID | -image IMAGE] [-env K=V] [-max D] [-quiet D] [-out DIR] | draft -name NAME -image IMAGE [-steam-app ID] [-first-boot REPORT] [-provider ollama|gemini] [-model M] | research -name NAME [-notes N] [-out FILE] | build -name NAME -variant JSON|@FILE [-out FILE] | repair -id ID [-reason R] | eval [-ids all] [-out eval.md] | worker [-once]")
+	fmt.Fprintln(os.Stderr, "usage: gbrgames validate [-games DIR] | import [-games DIR] [-all] EXPORT.json | export [-repo R] [-commit SHA] [-out FILE] | boottest-payload [-out FILE] ID | boot-test ID | inspect-image [-game ID] [-json] [-no-pull] [IMAGE] | first-boot [-game ID | -image IMAGE] [-env K=V] [-max D] [-quiet D] [-out DIR] | draft -name NAME -image IMAGE [-steam-app ID] [-first-boot REPORT] [-provider ollama|gemini] [-model M] | research -name NAME [-notes N] [-out FILE] | build -name NAME -variant JSON|@FILE [-out FILE] | repair -id ID [-reason R] | eval [-ids all] [-out eval.md] | worker [-once]")
 	os.Exit(2)
 }

scripts/boot-test.sh

@@ -30,24 +30,14 @@ for id in "$@"; do
     echo "::endgroup::"
     continue
   fi
-  payload=$("$gbrgames" boottest-payload "$id")
+  # The node-agent's BootTest with the game's real payload. Steam games start from the runner's download
+  # cache (volumes gbr-steamcache-<app>, kept by RUNNER.md's prune), so Steam isn't asked for the whole
+  # game on every run
+  AGENT_IMAGE="$AGENT_IMAGE" "$gbrgames" boot-test "$id"
   code=$?
-  if [ $code -eq 3 ]; then
+  if [ $code -eq 3 ]; then skipped+=("$id")
-    skipped+=("$id")
+  elif [ $code -eq 0 ]; then passed+=("$id")
-    echo "::endgroup::"
+  else failed+=("$id"); fi
-    continue
-  elif [ $code -ne 0 ]; then
-    failed+=("$id")
-    echo "::endgroup::"
-    continue
-  fi
-
-  docker run --rm -i \
-    -v /var/run/docker.sock:/var/run/docker.sock \
-    --entrypoint java "$AGENT_IMAGE" \
-    -Dloader.main=com.gbreed.gbr.nodeagent.BootTest -cp app.jar \
-    org.springframework.boot.loader.launch.PropertiesLauncher <<< "$payload"
-  if [ $? -eq 0 ]; then passed+=("$id"); else failed+=("$id"); fi
   echo "::endgroup::"
 done