gbr-games/cmd/gbrgames/firstboot.go

package main

import (
	"archive/tar"
	"bufio"
	"encoding/json"
	"errors"
	"flag"
	"fmt"
	"io"
	"os"
	"os/exec"
	"path"
	"path/filepath"
	"regexp"
	"sort"
	"strconv"
	"strings"
	"sync"
	"time"
)

// first-boot starts a game server once with none of a template's settings applied (or, -as-deployed,
// with exactly the settings a new server gets), and records what it does on its own: its output (to find
// the "ready" and error lines) and the files it creates (to learn the real config file, format and
// defaults, and what would be lost on a restart). The node-agent's own deploy code starts it, so it runs
// exactly as on a node: volumes, user, limits, SteamCMD install and startup command.

var (
	ansi       = regexp.MustCompile(`\x1b\[[0-9;?]*[A-Za-z]|\x1b\][^\a]*\a`)
	readyWords = regexp.MustCompile(`(?i)\b(ready|started|listening|done|advertis\w*|online|accepting connections|initiali[sz]ed|server is running|now running|startup complete|full startup)\b`)
	// Words that mean the whole server is up, not just one part of it ("RCON Listener started")
	strongReady = regexp.MustCompile(`(?i)\b(ready|done|advertis\w*|accepting connections|server is running|startup complete|full startup|server started)\b`)
	errorWords  = regexp.MustCompile(`(?i)\b(error|exception|fatal|failed|permission denied|segmentation fault|could not|unable to|cannot)\b`)
	// Leading timestamps and [bracketed] prefixes that change on every run
	linePrefix = regexp.MustCompile(`^\s*(?:(?:\[[^\]]*\]|\d{4}-\d\d-\d\d[T ][0-9:.,]+Z?|\d\d:\d\d:\d\d(?:[.,]\d+)?)[\s:|-]*)+`)
	numberRun  = regexp.MustCompile(`[0-9]+(?:\\\.[0-9]+)*`)
)

// Files worth copying out: what a server writes as its settings, plus its own log files
var configExt = map[string]bool{".ini": true, ".cfg": true, ".conf": true, ".json": true, ".properties": true,
	".yaml": true, ".yml": true, ".toml": true, ".xml": true, ".txt": true, ".log": true}

// Game data, engine files and caches: never settings
var skipDirs = []string{"steamapps/", "Binaries/", "Content/", "Engine/", "linux64/", "linux32/", ".steam/", "Steam/",
	"compatdata/", "libraries/", "cache/", "steamcmd/", "versions/", "plugins/", "mods/", "jre/", "java/"}

type bootSpec struct {
	Name, Image, User, Startup, SteamAppID string
	MemoryMb                               int
	Env                                    map[string]string
}

type logLine struct {
	At     float64 `json:"at"`     // seconds after the container started
	Source string  `json:"source"` // "console" or a file path
	Text   string  `json:"text"`
}

type createdFile struct {
	Path string `json:"path"`
	Size int64  `json:"size"`
}

type setting struct {
	File    string `json:"file"`
	Section string `json:"section,omitempty"`
	Key     string `json:"key"`
	Value   string `json:"value"`
}

type bootReport struct {
	Image            string        `json:"image"`
	User             string        `json:"user"`
	Outcome          string        `json:"outcome"`
	ExitCode         *int          `json:"exitCode,omitempty"`
	RanSeconds       float64       `json:"ranSeconds"`
	LastOutputAt     float64       `json:"lastOutputAt"`
	ConsoleLines     int           `json:"consoleLines"`
	ReadyCandidates  []logLine     `json:"readyCandidates"`
	ErrorLines       []logLine     `json:"errorLines"`
	SuggestedReady   string        `json:"suggestedReadyPattern,omitempty"`
	SuggestedExample string        `json:"suggestedReadyExample,omitempty"`
	FilesCreated     []createdFile `json:"filesCreated"`
	ConfigFiles      []string      `json:"configFiles"`
	Settings         []setting     `json:"settings"`
	ContainerChanges []string      `json:"containerChanges"`
}

func runFirstBoot(args []string) error {
	fs := flag.NewFlagSet("first-boot", flag.ExitOnError)
	gamesDir := fs.String("games", "games", "directory holding one folder per game")
	gameID := fs.String("game", "", "take image, install, startup and user from this game's template (not its settings)")
	image := fs.String("image", "", "image to boot when there is no template yet")
	user := fs.String("user", "", "uid:gid to run as (default: template containerUser, else what the image needs, else 1000:1000)")
	startup := fs.String("startup", "", "startup command (default: the image's own entrypoint)")
	steamApp := fs.String("steam-app", "", "install this SteamCMD app into /server first")
	memory := fs.Int("memory", 0, "memory limit in MB (default: template requiredMemoryMb, else 4096)")
	maxRun := fs.Duration("max", 20*time.Minute, "stop after this long even if the server keeps logging")
	quiet := fs.Duration("quiet", 2*time.Minute, "stop once the server has printed nothing for this long")
	outDir := fs.String("out", "", "where to write the report (default firstboot-out/<name>)")
	keep := fs.Bool("keep", false, "keep the container and volumes for a closer look")
	asDeployed := fs.Bool("as-deployed", false, "with -game: boot with the template's settings exactly as a new server gets "+
		"them (default: untouched, so the server writes its own defaults)")
	var envFlags multiFlag
	fs.Var(&envFlags, "env", "KEY=VALUE for the container (repeatable), e.g. EULA=TRUE")
	fs.Parse(args)

	spec, err := buildBootSpec(*gamesDir, *gameID, *image, *startup, *steamApp, *memory, envFlags)
	if err != nil {
		return err
	}
	if out, err := exec.Command("docker", "network", "inspect", bootTestNetwork).CombinedOutput(); err != nil {
		return fmt.Errorf("docker network %s is missing; first-boot only runs isolated (see RUNNER.md): %s", bootTestNetwork, strings.TrimSpace(string(out)))
	}
	if out, err := exec.Command("docker", "pull", "-q", spec.Image).CombinedOutput(); err != nil {
		return fmt.Errorf("docker pull %s: %v: %s", spec.Image, err, strings.TrimSpace(string(out)))
	}
	facts, err := inspectImage(spec.Image, nil)
	if err != nil {
		return err
	}
	switch {
	case *user != "":
		spec.User = *user
	case spec.User == "" && facts.SuggestedContainerUser != "":
		spec.User = facts.SuggestedContainerUser
		fmt.Printf("[first-boot] running as %s: %s\n", spec.User, facts.SuggestionReason)
	case spec.User == "":
		spec.User = nodeUser
	}
	if *outDir == "" {
		*outDir = filepath.Join("firstboot-out", spec.Name)
	}
	if err := os.MkdirAll(filepath.Join(*outDir, "files"), 0o755); err != nil {
		return err
	}

	b := &booter{spec: spec, facts: facts, container: "firstboot-" + spec.Name, outDir: *outDir, agentImage: agentImageFromEnv()}
	b.payload = untouchedPayload(spec)
	if *asDeployed {
		if *gameID == "" {
			return errors.New("-as-deployed needs -game")
		}
		if b.payload, err = bootTestPayload(*gamesDir, *gameID, false); err != nil {
			return err
		}
	}
	if err := pullAgentImage(b.agentImage); err != nil {
		return err
	}
	if !*keep {
		defer b.cleanup()
	}
	report, err := b.run(*maxRun, *quiet)
	if err != nil {
		return err
	}
	if err := writeBootReport(*outDir, report, b.console()); err != nil {
		return err
	}
	printBootReport(report, *outDir)
	return nil
}

func buildBootSpec(gamesDir, gameID, image, startup, steamApp string, memory int, envFlags []string) (*bootSpec, error) {
	spec := &bootSpec{Image: image, Startup: startup, SteamAppID: steamApp, MemoryMb: memory, Env: map[string]string{}}
	if gameID != "" {
		g, err := loadMap(filepath.Join(gamesDir, gameID, "game.yaml"))
		if err != nil {
			return nil, err
		}
		spec.Name = gameID
		if spec.Image == "" {
			spec.Image, _ = g["defaultDockerImage"].(string)
		}
		if spec.Startup == "" {
			spec.Startup, _ = g["defaultStartupCommand"].(string)
		}
		spec.User, _ = g["containerUser"].(string)
		switch g["installMethod"] {
		case "STEAMCMD":
			if ic, ok := g["installConfig"].(map[string]any); ok && spec.SteamAppID == "" {
				spec.SteamAppID = fmt.Sprint(ic["appId"])
			}
		case "BASH_SCRIPT":
			return nil, errors.New("first-boot does not run BASH_SCRIPT installs yet")
		}
		if spec.MemoryMb == 0 {
			if n, ok := g["requiredMemoryMb"].(json.Number); ok {
				m, _ := n.Int64()
				spec.MemoryMb = int(m)
			}
		}
		// Only what the server can't start without (test.yaml): EULAs, required IDs. Not the template's
		// settings, so the server writes its own defaults.
		if test, err := loadMap(filepath.Join(gamesDir, gameID, "test.yaml")); err == nil {
			if cfg, ok := test["config"].(map[string]any); ok {
				for k, v := range cfg {
					spec.Env[envName(k)] = resolvePlaceholders(fmt.Sprint(v))
				}
			}
			if secrets, ok := test["secrets"].(map[string]any); ok {
				for k, name := range secrets {
					val, ok := os.LookupEnv(fmt.Sprint(name))
					if !ok || val == "" {
						return nil, fmt.Errorf("%s needs secret %s for %s; set it in the environment", gameID, name, k)
					}
					spec.Env[envName(k)] = val
				}
			}
		}
	}
	if spec.Image == "" {
		return nil, errors.New("first-boot needs -game ID or -image IMAGE")
	}
	if spec.Name == "" {
		spec.Name = strings.Trim(regexp.MustCompile(`[^a-z0-9]+`).ReplaceAllString(strings.ToLower(spec.Image), "-"), "-")
	}
	if spec.MemoryMb == 0 {
		spec.MemoryMb = 4096
	}
	for _, kv := range envFlags {
		k, v, ok := strings.Cut(kv, "=")
		if !ok {
			return nil, fmt.Errorf("-env %q: want KEY=VALUE", kv)
		}
		spec.Env[k] = v
	}
	return spec, nil
}

type booter struct {
	spec       *bootSpec
	facts      *imageFacts
	container  string
	outDir     string
	agentImage string
	payload    map[string]any    // what the node-agent deploys
	mounts     map[string]string // mount path -> volume name

	mu      sync.Mutex
	lines   []logLine
	lastAt  time.Time
	started time.Time
}

// untouchedPayload deploys an image as a node would, but with none of a template's settings: only the
// install, the startup command and the -env values, so the server writes its own defaults.
func untouchedPayload(spec *bootSpec) map[string]any {
	config := map[string]any{}
	for k, v := range spec.Env {
		config[k] = v
	}
	p := map[string]any{
		"command": "DEPLOY_SERVER", "dockerImage": spec.Image, "gameConfig": config, "ports": []any{},
		"installMethod": "NONE", "memoryLimitMb": spec.MemoryMb, "containerUser": spec.User,
		"dockerNetwork": bootTestNetwork, "timeoutSec": 900,
	}
	if spec.Startup != "" {
		p["startupCommand"] = spec.Startup
	}
	if spec.SteamAppID != "" {
		p["installMethod"], p["installConfig"], p["timeoutSec"] = "STEAMCMD", map[string]any{"appId": spec.SteamAppID}, 2400
	}
	return p
}

func (b *booter) run(maxRun, quiet time.Duration) (*bootReport, error) {
	// Exactly what nodes mount: one volume at /server and /data. Anything else the server writes, the
	// image's own VOLUME folders included, is gone when the node recreates the container on the next start
	data := b.container + "-data"
	b.mounts = map[string]string{"/server": data, "/data": data}

	// The node-agent deploys it (install, config, startup, user, limits) and leaves the container to us.
	// It only has to get it running: watching it until it goes quiet is done here
	p := map[string]any{}
	for k, v := range b.payload {
		p[k] = v
	}
	p["containerName"], p["serverId"], p["keepContainer"] = b.container, b.container, true
	p["settleSec"], p["holdSec"], p["expectPorts"] = 15, 0, []any{}
	delete(p, "readyPattern")
	delete(p, "failPatterns")
	fmt.Printf("[first-boot] deploying %s as %v with the node-agent (memory %v MB)\n", b.spec.Image, displayUser(p["containerUser"]), p["memoryLimitMb"])
	output, _, err := runAgentBootTest(p, b.agentImage)
	if err != nil {
		return nil, err
	}
	created, err := time.Parse(time.RFC3339Nano, inspectField(b.container, "{{.Created}}"))
	if err != nil {
		return nil, fmt.Errorf("the node-agent could not deploy %s: %s", b.spec.Image, bootVerdict(output))
	}
	b.started, b.lastAt = created, created

	// The whole console from the start, with Docker's timestamps
	logs := exec.Command("docker", "logs", "-t", "-f", b.container)
	pr, pw := io.Pipe()
	logs.Stdout, logs.Stderr = pw, pw
	if err := logs.Start(); err != nil {
		return nil, err
	}
	done := make(chan struct{})
	go func() {
		sc := bufio.NewScanner(pr)
		sc.Buffer(make([]byte, 1024*1024), 1024*1024)
		for sc.Scan() {
			b.addTimestamped(sc.Text())
		}
		close(done)
	}()

	report := &bootReport{Image: b.spec.Image, User: displayUser(p["containerUser"])}
	minRun := time.Minute
	for {
		time.Sleep(2 * time.Second)
		elapsed := time.Since(b.started)
		running := inspectField(b.container, "{{.State.Running}}") == "true"
		restarts, _ := strconv.Atoi(inspectField(b.container, "{{.RestartCount}}"))
		b.mu.Lock()
		silent, count := time.Since(b.lastAt), len(b.lines)
		b.mu.Unlock()
		switch {
		case !running || restarts > 0:
			code, _ := strconv.Atoi(inspectField(b.container, "{{.State.ExitCode}}"))
			report.ExitCode = &code
			report.Outcome = fmt.Sprintf("exited by itself with code %d after %.0fs", code, elapsed.Seconds())
			if restarts > 0 {
				report.Outcome = fmt.Sprintf("crashed (exit code %d) and was restarted %d time(s) within %.0fs", code, restarts, elapsed.Seconds())
			}
			if inspectField(b.container, "{{.State.OOMKilled}}") == "true" {
				report.Outcome += fmt.Sprintf(" (killed: over the %v MB memory limit)", p["memoryLimitMb"])
			}
		case elapsed >= minRun && count > 0 && silent >= quiet:
			report.Outcome = fmt.Sprintf("running and quiet for %.0fs after %.0fs; stopped gracefully", silent.Seconds(), elapsed.Seconds())
		case elapsed >= maxRun:
			report.Outcome = fmt.Sprintf("still running and logging after %.0fs (the -max limit); stopped gracefully", elapsed.Seconds())
		default:
			continue
		}
		break
	}
	report.RanSeconds = time.Since(b.started).Seconds()
	// Many servers only write their settings when they shut down
	exec.Command("docker", "stop", "-t", "60", b.container).Run()
	logs.Process.Kill()
	pw.Close()
	<-done

	b.mu.Lock()
	report.ConsoleLines = len(b.lines)
	if n := len(b.lines); n > 0 {
		report.LastOutputAt = b.lines[n-1].At
	}
	console := append([]logLine{}, b.lines...)
	b.mu.Unlock()

	if err := b.collectFiles(report); err != nil {
		return nil, err
	}
	report.ContainerChanges = b.containerChanges()
	analyzeLines(report, console, b.fileLines(report), report.RanSeconds)
	return report, nil
}

// addTimestamped records one line of `docker logs -t`: "2026-10-26T01:11:52.123456789Z text".
func (b *booter) addTimestamped(raw string) {
	at := time.Now()
	if stamp, rest, ok := strings.Cut(raw, " "); ok {
		if t, err := time.Parse(time.RFC3339Nano, stamp); err == nil {
			at, raw = t, rest
		}
	}
	text := strings.TrimRight(ansi.ReplaceAllString(raw, ""), "\r \t")
	if text == "" {
		return
	}
	b.mu.Lock()
	b.lines = append(b.lines, logLine{At: at.Sub(b.started).Round(100 * time.Millisecond).Seconds(), Source: "console", Text: text})
	if at.After(b.lastAt) {
		b.lastAt = at
	}
	b.mu.Unlock()
}

func displayUser(u any) string {
	if s, ok := u.(string); ok && s != "" {
		return s
	}
	return nodeUser
}

func (b *booter) console() []logLine {
	b.mu.Lock()
	defer b.mu.Unlock()
	return append([]logLine{}, b.lines...)
}

// collectFiles lists everything the server left in its volumes and copies out the config and log files.
func (b *booter) collectFiles(report *bootReport) error {
	for _, vol := range uniqueValues(b.mounts) {
		mount := mountFor(b.mounts, vol)
		out, err := exec.Command("docker", "run", "--rm", "-v", vol+":/v:ro", "alpine", "find", "/v", "-type", "f", "-exec", "stat", "-c", "%s|%n", "{}", "+").Output()
		if err != nil {
			return fmt.Errorf("listing %s: %w", mount, err)
		}
		var wanted []string
		for _, line := range strings.Split(strings.TrimSpace(string(out)), "\n") {
			sizeStr, p, ok := strings.Cut(line, "|")
			if !ok {
				continue
			}
			size, _ := strconv.ParseInt(sizeStr, 10, 64)
			rel := strings.TrimPrefix(p, "/v/")
			report.FilesCreated = append(report.FilesCreated, createdFile{Path: path.Join(mount, rel), Size: size})
			if isConfigCandidate(rel, size) {
				wanted = append(wanted, rel)
			}
		}
		if len(wanted) > 150 {
			wanted = wanted[:150]
		}
		if len(wanted) == 0 {
			continue
		}
		if err := b.copyOut(vol, mount, wanted, report); err != nil {
			return err
		}
	}
	sort.Slice(report.FilesCreated, func(i, j int) bool { return report.FilesCreated[i].Path < report.FilesCreated[j].Path })
	return nil
}

func isConfigCandidate(rel string, size int64) bool {
	ext := strings.ToLower(path.Ext(rel))
	if !configExt[ext] {
		return false
	}
	for _, d := range skipDirs {
		if strings.HasPrefix(rel, d) || strings.Contains(rel, "/"+d) {
			return false
		}
	}
	switch ext {
	case ".log":
		return size <= 5<<20
	case ".txt":
		return size <= 16<<10
	}
	return size <= 256<<10
}

func (b *booter) copyOut(vol, mount string, files []string, report *bootReport) error {
	args := append([]string{"run", "--rm", "-v", vol + ":/v:ro", "alpine", "tar", "cf", "-", "-C", "/v"}, files...)
	cmd := exec.Command("docker", args...)
	stdout, err := cmd.StdoutPipe()
	if err != nil {
		return err
	}
	if err := cmd.Start(); err != nil {
		return err
	}
	tr := tar.NewReader(stdout)
	for {
		h, err := tr.Next()
		if err == io.EOF {
			break
		}
		if err != nil {
			return err
		}
		if h.Typeflag != tar.TypeReg {
			continue
		}
		content, err := io.ReadAll(tr)
		if err != nil {
			return err
		}
		full := path.Join(mount, h.Name)
		dest := filepath.Join(b.outDir, "files", filepath.FromSlash(strings.TrimPrefix(full, "/")))
		os.MkdirAll(filepath.Dir(dest), 0o755)
		if err := os.WriteFile(dest, content, 0o644); err != nil {
			return err
		}
		if strings.HasSuffix(full, ".log") {
			continue
		}
		report.ConfigFiles = append(report.ConfigFiles, full)
		report.Settings = append(report.Settings, parseConfig(full, string(content))...)
	}
	return cmd.Wait()
}

// fileLines reads the copied log files, so ready and error lines that never reach the console (servers
// that only log to a file) are found too.
func (b *booter) fileLines(report *bootReport) []logLine {
	var out []logLine
	for _, f := range report.FilesCreated {
		if !strings.HasSuffix(f.Path, ".log") {
			continue
		}
		raw, err := os.ReadFile(filepath.Join(b.outDir, "files", filepath.FromSlash(strings.TrimPrefix(f.Path, "/"))))
		if err != nil {
			continue
		}
		for _, l := range strings.Split(string(raw), "\n") {
			if t := strings.TrimSpace(ansi.ReplaceAllString(l, "")); t != "" {
				out = append(out, logLine{At: -1, Source: f.Path, Text: t})
			}
		}
	}
	return out
}

// Files that aren't the game's state, so losing them doesn't matter: system folders, Docker's own init
// (in every container started with --init), logs, lock and pid files, and what the Steam client, Proton
// and the OS keep for themselves in the home folder
var (
	systemPrefixes = []string{"/tmp", "/proc", "/sys", "/dev", "/run", "/etc", "/var/log", "/var/cache", "/var/tmp", "/var/lib/apt", "/usr/sbin/docker-init"}
	transientExt   = map[string]bool{".log": true, ".pid": true, ".lock": true, ".tmp": true, ".swp": true}
	clientNoise    = []string{"/Steam/logs/", "/Steam/config/", "/Steam/appcache/", "/.steam/", "/steamcmd/", "/.cache/", "/tmp/", "/.local/share/umu"}
)

func notGameState(p string) bool {
	if transientExt[strings.ToLower(path.Ext(p))] {
		return true
	}
	for _, prefix := range systemPrefixes {
		if p == prefix || strings.HasPrefix(p, prefix+"/") {
			return true
		}
	}
	for _, n := range clientNoise {
		if strings.Contains(p, n) {
			return true
		}
	}
	return false
}

// containerChanges lists the files the server wrote anywhere but /server and /data. All of them are
// lost when the node recreates the container (every start): in the container's own filesystem, and in
// the image's VOLUME folders, which Docker gives fresh anonymous volumes each time. Files that aren't
// the game's state are left out; saves of any kind are not.
func (b *booter) containerChanges() []string {
	var changes []string
	if out, err := exec.Command("docker", "diff", b.container).Output(); err == nil {
		changes = append(changes, outsideFiles(string(out), b.mounts)...)
	}
	// The image's VOLUME folders: Docker mounted anonymous volumes there, which docker diff doesn't show
	mounts, _ := exec.Command("docker", "inspect", "-f", `{{range .Mounts}}{{if eq .Type "volume"}}{{.Name}}|{{.Destination}}{{"\n"}}{{end}}{{end}}`, b.container).Output()
	for _, line := range strings.Split(strings.TrimSpace(string(mounts)), "\n") {
		vol, dest, ok := strings.Cut(line, "|")
		if !ok || b.mounts[dest] != "" {
			continue
		}
		files, err := exec.Command("docker", "run", "--rm", "-v", vol+":/v:ro", "alpine", "find", "/v", "-type", "f").Output()
		if err != nil {
			continue
		}
		n := 0
		for _, f := range strings.Split(strings.TrimSpace(string(files)), "\n") {
			rel := strings.TrimPrefix(f, "/v/")
			if f == "" || notGameState(path.Join(dest, rel)) {
				continue
			}
			if n++; n > 200 {
				break
			}
			changes = append(changes, "A "+path.Join(dest, rel))
		}
	}
	return changes
}

// outsideFiles picks the game's files out of `docker diff` output: added or changed outside the mounted
// volumes. docker diff lists folders too, as changed when anything inside them changed, so a folder only
// counts when nothing inside it did (it was created empty).
func outsideFiles(diff string, mounts map[string]string) []string {
	kinds := map[string]string{}
	parents := map[string]bool{}
	for _, line := range strings.Split(strings.TrimSpace(diff), "\n") {
		kind, p, ok := strings.Cut(strings.TrimSpace(line), " ")
		if !ok || p == "" {
			continue
		}
		kinds[p] = kind
		for d := path.Dir(p); d != "/" && d != "."; d = path.Dir(d) {
			parents[d] = true
		}
	}
	var files []string
	for p, kind := range kinds {
		if kind != "D" && !parents[p] && !notGameState(p) && !inMounts(p, mounts) {
			files = append(files, p)
		}
	}
	sort.Strings(files)
	for i, p := range files {
		files[i] = kinds[p] + " " + p
	}
	return files
}

func inMounts(p string, mounts map[string]string) bool {
	for mount := range mounts {
		if p == mount || strings.HasPrefix(p, mount+"/") {
			return true
		}
	}
	return false
}

func (b *booter) cleanup() {
	// -v: the anonymous volumes of the image's VOLUME folders go with it
	exec.Command("docker", "rm", "-f", "-v", b.container).Run()
	for _, vol := range uniqueValues(b.mounts) {
		exec.Command("docker", "volume", "rm", vol).Run()
	}
}

// analyzeLines picks out ready-looking and error lines and turns the last ready line into a readyPattern
// suggestion (plus the line itself as readyExample, which validation checks the pattern against).
// Console lines printed after stopAt are the server shutting down ("Done", "RCON client started" from
// the stop command), so they never count as ready.
func analyzeLines(report *bootReport, console, files []logLine, stopAt float64) {
	var running []logLine
	for _, l := range console {
		if l.At < stopAt {
			running = append(running, l)
		}
	}
	for _, l := range append(append([]logLine{}, console...), files...) {
		switch {
		case errorWords.MatchString(l.Text):
			report.ErrorLines = append(report.ErrorLines, l)
		case readyWords.MatchString(l.Text):
			report.ReadyCandidates = append(report.ReadyCandidates, l)
		}
	}
	report.ErrorLines = lastN(report.ErrorLines, 25)
	report.ReadyCandidates = lastN(report.ReadyCandidates, 15)

	// The last ready-looking console line before the stop; log files (no timestamps of ours, and they
	// contain the shutdown too) only when the console has none
	var pick *logLine
	for _, words := range []*regexp.Regexp{strongReady, readyWords} {
		for i := range running {
			if !errorWords.MatchString(running[i].Text) && words.MatchString(running[i].Text) {
				pick = &running[i]
			}
		}
		if pick != nil {
			break
		}
		for i := range files {
			if !errorWords.MatchString(files[i].Text) && words.MatchString(files[i].Text) {
				pick = &files[i]
				break // the first one in a file is the startup, later ones may be the shutdown
			}
		}
		if pick != nil {
			break
		}
	}
	if pick != nil {
		if pattern := suggestPattern(pick.Text); pattern != "" {
			report.SuggestedReady, report.SuggestedExample = pattern, pick.Text
		}
	}
}

// suggestPattern turns a concrete log line into a regex that matches it on every run: timestamps and
// [prefixes] dropped, numbers generalized.
func suggestPattern(line string) string {
	core := strings.TrimSpace(linePrefix.ReplaceAllString(line, ""))
	if len(core) < 4 {
		return ""
	}
	if len(core) > 80 {
		core = core[:80]
		if i := strings.LastIndex(core, " "); i > 20 {
			core = core[:i]
		}
	}
	pattern := numberRun.ReplaceAllString(regexp.QuoteMeta(core), `[0-9.]+`)
	if re, err := regexp.Compile(pattern); err != nil || !re.MatchString(line) {
		return ""
	}
	return pattern
}

// parseConfig reads settings out of a config file: JSON objects, INI/properties-style key=value lines
// (including Unreal's OptionSettings=(A=1,B="x") lists) and top-level YAML keys.
func parseConfig(file, content string) []setting {
	var out []setting
	switch strings.ToLower(path.Ext(file)) {
	case ".json":
		var obj map[string]any
		if json.Unmarshal([]byte(content), &obj) != nil {
			return nil
		}
		for _, k := range sortedKeysAny(obj) {
			if sub, ok := obj[k].(map[string]any); ok {
				for _, sk := range sortedKeysAny(sub) {
					out = append(out, setting{File: file, Section: k, Key: sk, Value: jsonValue(sub[sk])})
				}
				continue
			}
			out = append(out, setting{File: file, Key: k, Value: jsonValue(obj[k])})
		}
		return out
	case ".yaml", ".yml":
		for _, line := range strings.Split(content, "\n") {
			if line == "" || line[0] == ' ' || line[0] == '#' || line[0] == '-' {
				continue
			}
			if k, v, ok := strings.Cut(line, ":"); ok {
				out = append(out, setting{File: file, Key: strings.TrimSpace(k), Value: strings.Trim(strings.TrimSpace(v), `"'`)})
			}
		}
		return out
	case ".xml":
		return nil // listed and copied; structure varies too much to flatten
	}
	section := ""
	for _, raw := range strings.Split(content, "\n") {
		line := strings.TrimSpace(strings.TrimPrefix(raw, "\ufeff"))
		if line == "" || strings.HasPrefix(line, "#") || strings.HasPrefix(line, ";") || strings.HasPrefix(line, "//") {
			continue
		}
		if strings.HasPrefix(line, "[") && strings.HasSuffix(line, "]") {
			section = strings.Trim(line, "[]")
			continue
		}
		k, v, ok := strings.Cut(line, "=")
		if !ok {
			continue
		}
		k, v = strings.TrimSpace(k), strings.TrimSpace(v)
		if strings.HasPrefix(v, "(") && strings.HasSuffix(v, ")") {
			for _, item := range splitTopLevel(v[1 : len(v)-1]) {
				if ik, iv, ok := strings.Cut(item, "="); ok {
					out = append(out, setting{File: file, Section: k, Key: strings.TrimSpace(ik), Value: strings.Trim(strings.TrimSpace(iv), `"`)})
				}
			}
			continue
		}
		out = append(out, setting{File: file, Section: section, Key: k, Value: strings.Trim(v, `"`)})
	}
	return out
}

// splitTopLevel splits on commas that are not inside quotes or parentheses.
func splitTopLevel(s string) []string {
	var parts []string
	depth, quoted, start := 0, false, 0
	for i, c := range s {
		switch {
		case c == '"':
			quoted = !quoted
		case quoted:
		case c == '(':
			depth++
		case c == ')':
			depth--
		case c == ',' && depth == 0:
			parts = append(parts, s[start:i])
			start = i + 1
		}
	}
	return append(parts, s[start:])
}

func writeBootReport(dir string, report *bootReport, console []logLine) error {
	var log strings.Builder
	for _, l := range console {
		fmt.Fprintf(&log, "[%7.1fs] %s\n", l.At, l.Text)
	}
	if err := os.WriteFile(filepath.Join(dir, "console.log"), []byte(log.String()), 0o644); err != nil {
		return err
	}
	b, _ := json.MarshalIndent(report, "", "  ")
	return os.WriteFile(filepath.Join(dir, "report.json"), b, 0o644)
}

func printBootReport(r *bootReport, dir string) {
	fmt.Printf("\n[first-boot] %s as %s: %s\n", r.Image, r.User, r.Outcome)
	fmt.Printf("  console: %d lines, last one at %.0fs\n", r.ConsoleLines, r.LastOutputAt)
	if r.SuggestedReady != "" {
		fmt.Printf("  readyPattern suggestion: %s\n  readyExample:            %s\n", r.SuggestedReady, r.SuggestedExample)
	}
	for _, l := range lastN(r.ReadyCandidates, 5) {
		fmt.Printf("  ready?  %s\n", describeLine(l))
	}
	for _, l := range lastN(r.ErrorLines, 5) {
		fmt.Printf("  error?  %s\n", describeLine(l))
	}
	fmt.Printf("  files created: %d; config files read: %d; settings found: %d\n", len(r.FilesCreated), len(r.ConfigFiles), len(r.Settings))
	for _, f := range r.ConfigFiles {
		n := 0
		for _, s := range r.Settings {
			if s.File == f {
				n++
			}
		}
		fmt.Printf("    %s (%d settings)\n", f, n)
	}
	if len(r.ContainerChanges) > 0 {
		fmt.Printf("  written outside the volumes (lost when the node recreates the container): %s\n", strings.Join(r.ContainerChanges, ", "))
	}
	fmt.Printf("  full report: %s\n", filepath.Join(dir, "report.json"))
}

func describeLine(l logLine) string {
	where := l.Source
	if l.At >= 0 {
		where = fmt.Sprintf("%.0fs", l.At)
	}
	text := l.Text
	if len(text) > 140 {
		text = text[:140] + "..."
	}
	return fmt.Sprintf("[%s] %s", where, text)
}

func inspectField(container, format string) string {
	out, _ := exec.Command("docker", "inspect", "-f", format, container).Output()
	return strings.TrimSpace(string(out))
}

func uniqueValues(m map[string]string) []string {
	seen := map[string]bool{}
	var out []string
	for _, v := range m {
		if !seen[v] {
			seen[v] = true
			out = append(out, v)
		}
	}
	sort.Strings(out)
	return out
}

// mountFor names a volume by its first mount path (the data volume is both /server and /data).
func mountFor(mounts map[string]string, vol string) string {
	var paths []string
	for p, v := range mounts {
		if v == vol {
			paths = append(paths, p)
		}
	}
	sort.Strings(paths)
	if len(paths) > 1 && paths[0] == "/data" {
		return "/server"
	}
	return paths[0]
}

func lastN[T any](s []T, n int) []T {
	if len(s) > n {
		return s[len(s)-n:]
	}
	return s
}

func jsonValue(v any) string {
	if s, ok := v.(string); ok {
		return s
	}
	b, _ := json.Marshal(v)
	return string(b)
}

func sortedKeysAny(m map[string]any) []string {
	out := make([]string, 0, len(m))
	for k := range m {
		out = append(out, k)
	}
	sort.Strings(out)
	return out
}

type multiFlag []string

func (m *multiFlag) String() string     { return strings.Join(*m, ",") }
func (m *multiFlag) Set(v string) error { *m = append(*m, v); return nil }