feat(builder): every boot through the node-agent, every lost file counted, Steam cache
What a builder boot sees is now what a node does: - First boots (untouched, or -as-deployed with exactly a new server's settings) are deployed by the node-agent's BootTest in keep mode; the builder only watches the console and inspects what's left. The Go copy of the deploy (its own docker run, mounts and startup prefix) is gone. It had drifted: it gave the image's VOLUME folders their own volumes, which nodes don't, so saves there looked safe while every start loses them. - Files written outside /server and /data are counted whatever their type (it only counted config-like extensions, so .sav or .db saves went unseen), including the image's VOLUME folders; logs, pid and lock files and system folders are left out. - Boot tests ask the server to stay up holdSec (60 s, test.yaml can set it) after its ready line and to listen on every published container port (needs the node-agent's matching BootTest). - Steam apps are downloaded once per build into a gbr.cache volume and copied into each boot, so later boots only check the files (STEAM_CACHE=off). - RUNNER.md no longer suggests docker system prune, which deletes the gbr-boottest network when nothing is using it. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
gregandClaude Opus 5.5 committed
1 parent
e07025f03d
commit
04967a8a16
10 files changed
+424
-171
No files matched your search
+169
-102
@@ -20,15 +20,11 @@ import (
|
||||
"time"
|
||||
)
|
||||
|
||||
// first-boot starts a game server once with none of a template's settings applied, and records what it
|
||||
// does on its own: its output (to find the "ready" and error lines) and the files it creates (to learn
|
||||
// the real config file, format and defaults). It mirrors how the node-agent runs servers: same volumes,
|
||||
// user, SteamCMD install and startup prefix, but without a restart policy, so a crash shows as a crash.
|
||||
|
||||
// Same prefix the node-agent puts in front of every startup command (DockerService)
|
||||
const agentStartupPrefix = "mkdir -p ~/.steam/sdk64 /server/.steam/sdk64 2>/dev/null; " +
|
||||
"[ -f /home/steam/steamcmd/linux64/steamclient.so ] && cp -f /home/steam/steamcmd/linux64/steamclient.so ~/.steam/sdk64/ 2>/dev/null; " +
|
||||
"[ -f /server/linux64/steamclient.so ] && cp -f /server/linux64/steamclient.so ~/.steam/sdk64/ 2>/dev/null; cd /server && "
|
||||
// first-boot starts a game server once with none of a template's settings applied (or, -as-deployed,
|
||||
// with exactly the settings a new server gets), and records what it does on its own: its output (to find
|
||||
// the "ready" and error lines) and the files it creates (to learn the real config file, format and
|
||||
// defaults, and what would be lost on a restart). The node-agent's own deploy code starts it, so it runs
|
||||
// exactly as on a node: volumes, user, limits, SteamCMD install and startup command.
|
||||
|
||||
var (
|
||||
ansi = regexp.MustCompile(`\x1b\[[0-9;?]*[A-Za-z]|\x1b\][^\a]*\a`)
|
||||
@@ -104,6 +100,8 @@ func runFirstBoot(args []string) error {
|
||||
quiet := fs.Duration("quiet", 2*time.Minute, "stop once the server has printed nothing for this long")
|
||||
outDir := fs.String("out", "", "where to write the report (default firstboot-out/<name>)")
|
||||
keep := fs.Bool("keep", false, "keep the container and volumes for a closer look")
|
||||
asDeployed := fs.Bool("as-deployed", false, "with -game: boot with the template's settings exactly as a new server gets "+
|
||||
"them (default: untouched, so the server writes its own defaults)")
|
||||
var envFlags multiFlag
|
||||
fs.Var(&envFlags, "env", "KEY=VALUE for the container (repeatable), e.g. EULA=TRUE")
|
||||
fs.Parse(args)
|
||||
@@ -138,7 +136,19 @@ func runFirstBoot(args []string) error {
|
||||
return err
|
||||
}
|
||||
|
||||
b := &booter{spec: spec, facts: facts, container: "firstboot-" + spec.Name, outDir: *outDir}
|
||||
b := &booter{spec: spec, facts: facts, container: "firstboot-" + spec.Name, outDir: *outDir, agentImage: agentImageFromEnv()}
|
||||
b.payload = untouchedPayload(spec)
|
||||
if *asDeployed {
|
||||
if *gameID == "" {
|
||||
return errors.New("-as-deployed needs -game")
|
||||
}
|
||||
if b.payload, err = bootTestPayload(*gamesDir, *gameID, false); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
if out, err := runCmd("docker", "pull", "-q", b.agentImage); err != nil {
|
||||
return fmt.Errorf("docker pull %s: %v: %s", b.agentImage, err, out)
|
||||
}
|
||||
if !*keep {
|
||||
defer b.cleanup()
|
||||
}
|
||||
@@ -221,11 +231,13 @@ func buildBootSpec(gamesDir, gameID, image, startup, steamApp string, memory int
|
||||
}
|
||||
|
||||
type booter struct {
|
||||
spec *bootSpec
|
||||
facts *imageFacts
|
||||
container string
|
||||
outDir string
|
||||
mounts map[string]string // mount path -> volume name
|
||||
spec *bootSpec
|
||||
facts *imageFacts
|
||||
container string
|
||||
outDir string
|
||||
agentImage string
|
||||
payload map[string]any // what the node-agent deploys
|
||||
mounts map[string]string // mount path -> volume name
|
||||
|
||||
mu sync.Mutex
|
||||
lines []logLine
|
||||
@@ -233,64 +245,56 @@ type booter struct {
|
||||
started time.Time
|
||||
}
|
||||
|
||||
func (b *booter) run(maxRun, quiet time.Duration) (*bootReport, error) {
|
||||
exec.Command("docker", "rm", "-f", b.container).Run() // leftovers from an interrupted run
|
||||
// untouchedPayload deploys an image as a node would, but with none of a template's settings: only the
|
||||
// install, the startup command and the -env values, so the server writes its own defaults.
|
||||
func untouchedPayload(spec *bootSpec) map[string]any {
|
||||
config := map[string]any{}
|
||||
for k, v := range spec.Env {
|
||||
config[k] = v
|
||||
}
|
||||
p := map[string]any{
|
||||
"command": "DEPLOY_SERVER", "dockerImage": spec.Image, "gameConfig": config, "ports": []any{},
|
||||
"installMethod": "NONE", "memoryLimitMb": spec.MemoryMb, "containerUser": spec.User,
|
||||
"dockerNetwork": bootTestNetwork, "timeoutSec": 900,
|
||||
}
|
||||
if spec.Startup != "" {
|
||||
p["startupCommand"] = spec.Startup
|
||||
}
|
||||
if spec.SteamAppID != "" {
|
||||
p["installMethod"], p["installConfig"], p["timeoutSec"] = "STEAMCMD", map[string]any{"appId": spec.SteamAppID}, 2400
|
||||
}
|
||||
return p
|
||||
}
|
||||
|
||||
// Same layout as the node: one volume at /server and /data, plus a fresh one per declared VOLUME
|
||||
func (b *booter) run(maxRun, quiet time.Duration) (*bootReport, error) {
|
||||
// Exactly what nodes mount: one volume at /server and /data. Anything else the server writes, the
|
||||
// image's own VOLUME folders included, is gone when the node recreates the container on the next start
|
||||
data := b.container + "-data"
|
||||
b.mounts = map[string]string{"/server": data, "/data": data}
|
||||
for i, v := range b.facts.Volumes {
|
||||
if _, ok := b.mounts[v]; !ok {
|
||||
b.mounts[v] = fmt.Sprintf("%s-v%d", b.container, i)
|
||||
}
|
||||
}
|
||||
for _, vol := range uniqueValues(b.mounts) {
|
||||
if err := docker("volume", "create", vol); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if err := docker("run", "--rm", "-v", vol+":/v", "alpine", "chown", "-R", b.spec.User, "/v"); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
}
|
||||
|
||||
if b.spec.SteamAppID != "" {
|
||||
fmt.Printf("[first-boot] installing Steam app %s\n", b.spec.SteamAppID)
|
||||
// Like the node-agent: steamcmd's image runs as uid 1000, then the volume goes to the game's user
|
||||
steam := "/home/steam/steamcmd/steamcmd.sh +login anonymous +quit && /home/steam/steamcmd/steamcmd.sh +force_install_dir /server +login anonymous +app_update " +
|
||||
b.spec.SteamAppID + " validate +quit"
|
||||
if err := docker("run", "--rm", "-v", data+":/server", "alpine", "chown", "-R", nodeUser, "/server"); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
cmd := exec.Command("docker", "run", "--rm", "--network", bootTestNetwork, "-v", data+":/server", "cm2network/steamcmd", "bash", "-c", steam)
|
||||
cmd.Stdout, cmd.Stderr = os.Stdout, os.Stderr
|
||||
if err := cmd.Run(); err != nil {
|
||||
return nil, fmt.Errorf("steamcmd install of %s failed: %w", b.spec.SteamAppID, err)
|
||||
}
|
||||
if err := docker("run", "--rm", "-v", data+":/server", "alpine", "chown", "-R", b.spec.User, "/server"); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
// The node-agent deploys it (install, config, startup, user, limits) and leaves the container to us.
|
||||
// It only has to get it running: watching it until it goes quiet is done here
|
||||
p := map[string]any{}
|
||||
for k, v := range b.payload {
|
||||
p[k] = v
|
||||
}
|
||||
p["containerName"], p["serverId"], p["keepContainer"] = b.container, b.container, true
|
||||
p["settleSec"], p["holdSec"], p["expectPorts"] = 15, 0, []any{}
|
||||
delete(p, "readyPattern")
|
||||
delete(p, "failPatterns")
|
||||
fmt.Printf("[first-boot] deploying %s as %v with the node-agent (memory %v MB)\n", b.spec.Image, displayUser(p["containerUser"]), p["memoryLimitMb"])
|
||||
output, _, err := runAgentBootTest(p, b.agentImage)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
created, err := time.Parse(time.RFC3339Nano, inspectField(b.container, "{{.Created}}"))
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("the node-agent could not deploy %s: %s", b.spec.Image, bootVerdict(output))
|
||||
}
|
||||
b.started, b.lastAt = created, created
|
||||
|
||||
runArgs := []string{"run", "-d", "-t", "--init", "--name", b.container, "--user", b.spec.User,
|
||||
"--security-opt", "no-new-privileges:true", "--pids-limit", "1024", "--network", bootTestNetwork,
|
||||
"-m", fmt.Sprintf("%dm", b.spec.MemoryMb), "--memory-swap", fmt.Sprintf("%dm", b.spec.MemoryMb)}
|
||||
for mount, vol := range b.mounts {
|
||||
runArgs = append(runArgs, "-v", vol+":"+mount)
|
||||
}
|
||||
for _, k := range sortedKeysString(b.spec.Env) {
|
||||
runArgs = append(runArgs, "-e", k+"="+b.spec.Env[k])
|
||||
}
|
||||
runArgs = append(runArgs, b.spec.Image)
|
||||
if b.spec.Startup != "" {
|
||||
runArgs = append(runArgs, "bash", "-c", agentStartupPrefix+b.spec.Startup)
|
||||
}
|
||||
fmt.Printf("[first-boot] starting %s as %s (memory %d MB)\n", b.spec.Image, b.spec.User, b.spec.MemoryMb)
|
||||
if out, err := exec.Command("docker", runArgs...).CombinedOutput(); err != nil {
|
||||
return nil, fmt.Errorf("docker run: %v: %s", err, strings.TrimSpace(string(out)))
|
||||
}
|
||||
b.started, b.lastAt = time.Now(), time.Now()
|
||||
|
||||
logs := exec.Command("docker", "logs", "-f", b.container)
|
||||
// The whole console from the start, with Docker's timestamps
|
||||
logs := exec.Command("docker", "logs", "-t", "-f", b.container)
|
||||
pr, pw := io.Pipe()
|
||||
logs.Stdout, logs.Stderr = pw, pw
|
||||
if err := logs.Start(); err != nil {
|
||||
@@ -301,27 +305,31 @@ func (b *booter) run(maxRun, quiet time.Duration) (*bootReport, error) {
|
||||
sc := bufio.NewScanner(pr)
|
||||
sc.Buffer(make([]byte, 1024*1024), 1024*1024)
|
||||
for sc.Scan() {
|
||||
b.addLine("console", sc.Text())
|
||||
b.addTimestamped(sc.Text())
|
||||
}
|
||||
close(done)
|
||||
}()
|
||||
|
||||
report := &bootReport{Image: b.spec.Image, User: b.spec.User}
|
||||
report := &bootReport{Image: b.spec.Image, User: displayUser(p["containerUser"])}
|
||||
minRun := time.Minute
|
||||
for {
|
||||
time.Sleep(2 * time.Second)
|
||||
elapsed := time.Since(b.started)
|
||||
running := inspectField(b.container, "{{.State.Running}}") == "true"
|
||||
restarts, _ := strconv.Atoi(inspectField(b.container, "{{.RestartCount}}"))
|
||||
b.mu.Lock()
|
||||
silent, count := time.Since(b.lastAt), len(b.lines)
|
||||
b.mu.Unlock()
|
||||
switch {
|
||||
case !running:
|
||||
case !running || restarts > 0:
|
||||
code, _ := strconv.Atoi(inspectField(b.container, "{{.State.ExitCode}}"))
|
||||
report.ExitCode = &code
|
||||
report.Outcome = fmt.Sprintf("exited by itself with code %d after %.0fs", code, elapsed.Seconds())
|
||||
if restarts > 0 {
|
||||
report.Outcome = fmt.Sprintf("crashed (exit code %d) and was restarted %d time(s) within %.0fs", code, restarts, elapsed.Seconds())
|
||||
}
|
||||
if inspectField(b.container, "{{.State.OOMKilled}}") == "true" {
|
||||
report.Outcome += fmt.Sprintf(" (killed: over the %d MB memory limit)", b.spec.MemoryMb)
|
||||
report.Outcome += fmt.Sprintf(" (killed: over the %v MB memory limit)", p["memoryLimitMb"])
|
||||
}
|
||||
case elapsed >= minRun && count > 0 && silent >= quiet:
|
||||
report.Outcome = fmt.Sprintf("running and quiet for %.0fs after %.0fs; stopped gracefully", silent.Seconds(), elapsed.Seconds())
|
||||
@@ -335,6 +343,7 @@ func (b *booter) run(maxRun, quiet time.Duration) (*bootReport, error) {
|
||||
report.RanSeconds = time.Since(b.started).Seconds()
|
||||
// Many servers only write their settings when they shut down
|
||||
exec.Command("docker", "stop", "-t", "60", b.container).Run()
|
||||
logs.Process.Kill()
|
||||
pw.Close()
|
||||
<-done
|
||||
|
||||
@@ -354,17 +363,33 @@ func (b *booter) run(maxRun, quiet time.Duration) (*bootReport, error) {
|
||||
return report, nil
|
||||
}
|
||||
|
||||
func (b *booter) addLine(source, raw string) {
|
||||
// addTimestamped records one line of `docker logs -t`: "2026-10-26T01:11:52.123456789Z text".
|
||||
func (b *booter) addTimestamped(raw string) {
|
||||
at := time.Now()
|
||||
if stamp, rest, ok := strings.Cut(raw, " "); ok {
|
||||
if t, err := time.Parse(time.RFC3339Nano, stamp); err == nil {
|
||||
at, raw = t, rest
|
||||
}
|
||||
}
|
||||
text := strings.TrimRight(ansi.ReplaceAllString(raw, ""), "\r \t")
|
||||
if text == "" {
|
||||
return
|
||||
}
|
||||
b.mu.Lock()
|
||||
b.lines = append(b.lines, logLine{At: time.Since(b.started).Round(100 * time.Millisecond).Seconds(), Source: source, Text: text})
|
||||
b.lastAt = time.Now()
|
||||
b.lines = append(b.lines, logLine{At: at.Sub(b.started).Round(100 * time.Millisecond).Seconds(), Source: "console", Text: text})
|
||||
if at.After(b.lastAt) {
|
||||
b.lastAt = at
|
||||
}
|
||||
b.mu.Unlock()
|
||||
}
|
||||
|
||||
func displayUser(u any) string {
|
||||
if s, ok := u.(string); ok && s != "" {
|
||||
return s
|
||||
}
|
||||
return nodeUser
|
||||
}
|
||||
|
||||
func (b *booter) console() []logLine {
|
||||
b.mu.Lock()
|
||||
defer b.mu.Unlock()
|
||||
@@ -487,39 +512,88 @@ func (b *booter) fileLines(report *bootReport) []logLine {
|
||||
return out
|
||||
}
|
||||
|
||||
// containerChanges lists files the server wrote outside its volumes: settings there are lost when the
|
||||
// node recreates the container, which a template needs to know about.
|
||||
// System folders and files that aren't the game's state: what's written there doesn't matter when lost
|
||||
var (
|
||||
systemPrefixes = []string{"/tmp", "/proc", "/sys", "/dev", "/run", "/etc", "/var/log", "/var/cache", "/var/tmp", "/var/lib/apt"}
|
||||
transientExt = map[string]bool{".log": true, ".pid": true, ".lock": true, ".tmp": true, ".swp": true}
|
||||
)
|
||||
|
||||
// containerChanges lists the files the server wrote anywhere but /server and /data. All of them are
|
||||
// lost when the node recreates the container (every start): in the container's own filesystem, and in
|
||||
// the image's VOLUME folders, which Docker gives fresh anonymous volumes each time. Logs, lock and pid
|
||||
// files are left out; saves of any kind are not.
|
||||
func (b *booter) containerChanges() []string {
|
||||
out, err := exec.Command("docker", "diff", b.container).Output()
|
||||
if err != nil {
|
||||
return nil
|
||||
}
|
||||
var changes []string
|
||||
for _, line := range strings.Split(strings.TrimSpace(string(out)), "\n") {
|
||||
kind, p, ok := strings.Cut(line, " ")
|
||||
if !ok || kind == "D" {
|
||||
if out, err := exec.Command("docker", "diff", b.container).Output(); err == nil {
|
||||
changes = append(changes, outsideFiles(string(out), b.mounts)...)
|
||||
}
|
||||
// The image's VOLUME folders: Docker mounted anonymous volumes there, which docker diff doesn't show
|
||||
mounts, _ := exec.Command("docker", "inspect", "-f", `{{range .Mounts}}{{if eq .Type "volume"}}{{.Name}}|{{.Destination}}{{"\n"}}{{end}}{{end}}`, b.container).Output()
|
||||
for _, line := range strings.Split(strings.TrimSpace(string(mounts)), "\n") {
|
||||
vol, dest, ok := strings.Cut(line, "|")
|
||||
if !ok || b.mounts[dest] != "" {
|
||||
continue
|
||||
}
|
||||
skip := false
|
||||
for _, prefix := range []string{"/tmp", "/proc", "/sys", "/dev", "/run", "/var/log", "/var/cache", "/var/tmp"} {
|
||||
if p == prefix || strings.HasPrefix(p, prefix+"/") {
|
||||
skip = true
|
||||
}
|
||||
files, err := exec.Command("docker", "run", "--rm", "-v", vol+":/v:ro", "alpine", "find", "/v", "-type", "f").Output()
|
||||
if err != nil {
|
||||
continue
|
||||
}
|
||||
for mount := range b.mounts {
|
||||
if p == mount || strings.HasPrefix(p, mount+"/") {
|
||||
skip = true
|
||||
n := 0
|
||||
for _, f := range strings.Split(strings.TrimSpace(string(files)), "\n") {
|
||||
rel := strings.TrimPrefix(f, "/v/")
|
||||
if f == "" || transientExt[strings.ToLower(path.Ext(rel))] {
|
||||
continue
|
||||
}
|
||||
}
|
||||
if !skip && configExt[strings.ToLower(path.Ext(p))] {
|
||||
changes = append(changes, kind+" "+p)
|
||||
if n++; n > 200 {
|
||||
break
|
||||
}
|
||||
changes = append(changes, "A "+path.Join(dest, rel))
|
||||
}
|
||||
}
|
||||
return changes
|
||||
}
|
||||
|
||||
// outsideFiles picks the files out of `docker diff` output: added or changed leaves (docker diff lists
|
||||
// folders too) outside the mounted volumes and system folders, without transient files.
|
||||
func outsideFiles(diff string, mounts map[string]string) []string {
|
||||
var paths []string
|
||||
kinds := map[string]string{}
|
||||
for _, line := range strings.Split(strings.TrimSpace(diff), "\n") {
|
||||
kind, p, ok := strings.Cut(strings.TrimSpace(line), " ")
|
||||
if !ok || kind == "D" || p == "" {
|
||||
continue
|
||||
}
|
||||
skip := transientExt[strings.ToLower(path.Ext(p))]
|
||||
for _, prefix := range systemPrefixes {
|
||||
if p == prefix || strings.HasPrefix(p, prefix+"/") {
|
||||
skip = true
|
||||
}
|
||||
}
|
||||
for mount := range mounts {
|
||||
if p == mount || strings.HasPrefix(p, mount+"/") {
|
||||
skip = true
|
||||
}
|
||||
}
|
||||
if !skip {
|
||||
paths = append(paths, p)
|
||||
kinds[p] = kind
|
||||
}
|
||||
}
|
||||
sort.Strings(paths)
|
||||
var leaves []string
|
||||
for i, p := range paths {
|
||||
// A folder is followed (sorted) by the entries inside it
|
||||
if i+1 < len(paths) && strings.HasPrefix(paths[i+1], p+"/") {
|
||||
continue
|
||||
}
|
||||
leaves = append(leaves, kinds[p]+" "+p)
|
||||
}
|
||||
return leaves
|
||||
}
|
||||
|
||||
func (b *booter) cleanup() {
|
||||
exec.Command("docker", "rm", "-f", b.container).Run()
|
||||
// -v: the anonymous volumes of the image's VOLUME folders go with it
|
||||
exec.Command("docker", "rm", "-f", "-v", b.container).Run()
|
||||
for _, vol := range uniqueValues(b.mounts) {
|
||||
exec.Command("docker", "volume", "rm", vol).Run()
|
||||
}
|
||||
@@ -730,13 +804,6 @@ func describeLine(l logLine) string {
|
||||
return fmt.Sprintf("[%s] %s", where, text)
|
||||
}
|
||||
|
||||
func docker(args ...string) error {
|
||||
if out, err := exec.Command("docker", args...).CombinedOutput(); err != nil {
|
||||
return fmt.Errorf("docker %s: %v: %s", args[0], err, strings.TrimSpace(string(out)))
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func inspectField(container, format string) string {
|
||||
out, _ := exec.Command("docker", "inspect", "-f", format, container).Output()
|
||||
return strings.TrimSpace(string(out))
|
||||
|
||||
Reference in new issue
Block a user