diff --git a/.gitea/workflows/builder-image.yml b/.gitea/workflows/builder-image.yml new file mode 100644 index 0000000..e8f4a23 --- /dev/null +++ b/.gitea/workflows/builder-image.yml @@ -0,0 +1,37 @@ +name: Builder image + +# Publishes the AI builder worker image (gbrgames worker, see BUILDER.md). +on: + push: + branches: ["main"] + paths: + - "cmd/**" + - "go.mod" + - "go.sum" + - "Dockerfile.builder" + workflow_dispatch: + +env: + REGISTRY: www.gitea.gbreed.com + IMAGE_NAME: gbr-builder + +jobs: + publish: + runs-on: ubuntu-latest + steps: + - name: Checkout Code + uses: actions/checkout@v4 + + - name: Login to Gitea Container Registry + run: | + echo "${{ secrets.GIT_SERVER_TOKEN }}" | docker login ${{ env.REGISTRY }} \ + -u ${{ secrets.GIT_TOKEN_USERNAME }} \ + --password-stdin + + - name: Build and Push Docker Image + run: | + IMAGE_PATH="${{ env.REGISTRY }}/${{ github.repository_owner }}/${{ env.IMAGE_NAME }}" + SHORT_SHA=$(echo "${{ github.sha }}" | cut -c1-7) + docker build -f Dockerfile.builder -t $IMAGE_PATH:$SHORT_SHA -t $IMAGE_PATH:latest . + docker push $IMAGE_PATH:$SHORT_SHA + docker push $IMAGE_PATH:latest diff --git a/.gitea/workflows/validate.yml b/.gitea/workflows/validate.yml index 5ec67d9..14c15ea 100644 --- a/.gitea/workflows/validate.yml +++ b/.gitea/workflows/validate.yml @@ -62,7 +62,7 @@ jobs: boot-test: needs: validate if: needs.validate.outputs.games != '' - # A dedicated runner on VM 116 whose game containers can reach the internet but not the LAN + # A dedicated runner whose game containers can reach the internet but not private networks runs-on: game-test steps: - name: Checkout Code diff --git a/AUTHORING.md b/AUTHORING.md index b595a5d..e21e185 100644 --- a/AUTHORING.md +++ b/AUTHORING.md @@ -41,6 +41,9 @@ Know this before writing anything: - `defaultPorts` lists **only what players connect to** (game and query ports). RCON, REST and admin ports are **never** published. - Use `"7777/udp"` or `"8211:8211/udp"`. Every port needs its protocol. +- Nodes always publish these container ports (only the host side moves when servers share a node). + So the game's own port is **not a setting**: write it into the startup command or config file. A + game that must announce the port players use gets `{PUBLIC_PORT_}`. ### The container user - Run `gbrgames inspect-image -game `. If the image is built around its own user (it declares @@ -51,6 +54,9 @@ Know this before writing anything: - Read settings under the names the agent gives them (see above). `${RCON_PORT}` when the setting is `RCONPort` silently ignores the owner's value; it arrives as `RCONPORT`. - Use `${VAR:-default}` for anything that may be unset. +- Quote text settings where they're used: `-name "${SERVER_NAME}"`. `ARGS="-name ${SERVER_NAME}"; ./server $ARGS` + splits "GBRHosting - My Server" into words; build optional arguments in a bash array instead + (`args=(-name "$SERVER_NAME"); [ -n "$PASSWORD" ] && args+=(-password "$PASSWORD"); ./server "${args[@]}"`). - If the image installs the game itself on start, make sure a failed install is retried rather than starting a server that isn't there (see ARK's template). diff --git a/BUILDER.md b/BUILDER.md new file mode 100644 index 0000000..c82fb3f --- /dev/null +++ b/BUILDER.md @@ -0,0 +1,61 @@ +# AI builder (gbrgames worker) + +The panel's **AI builder** (Admin → Games → AI Config Generator) is run by a worker we host ourselves. +The panel only keeps a queue; every result is a pull request to this repo, which CI validates and +boot-tests like any other. + +``` +Panel Builder worker (gbrgames worker, Docker) +───── ───────────────────────────────────────── +admin starts a job ──► queued ◄─poll── research: web search + page reader + model +progress / log / variants ◄─post── build: first boot → draft → boot with the node-agent +"Review draft PR" → branch draft/- + PR as the bot ──► CI (validate + boot test) +``` + +1. **Research** (`gbrgames research -name N`): searches the web, reads the best pages, and has the model list + the server's variants (edition, image or SteamCMD app, startup, ports). Every image and Steam app is + checked for real (`docker manifest inspect`, anonymous `steamcmd +app_info_print`) before the admin + sees it. +2. The admin picks a variant in the panel. +3. **Build** (`gbrgames build -name N -variant @v.json`): first boot of the image untouched (`first-boot`), a + draft by the model (`draft`, rules in `AUTHORING.md`), then the draft is deployed with the node-agent's + `BootTest`, the same check CI runs. Failures (crash, error lines, saves written outside `/server`) go + back to the model, three boots at most. +4. The worker pushes `games//` to a branch and opens a pull request as the bot account. A draft that + validates but never booted still gets a PR, titled `WIP:`, with what's left listed. Nothing reaches + players until a person merges it and enables the game. + +The worker only connects out (to the panel, Gitea, the model and the search services). + +## Requirements + +A Docker host with the isolated `gbr-boottest` network (`RUNNER.md`, step 1), so servers booted by the +builder can't reach private networks. Enough memory for the largest server it builds (~16 GB for ARK) and +disk for SteamCMD downloads. The image `gbr-builder:latest` is published by +`.gitea/workflows/builder-image.yml` whenever `cmd/` changes on main. + +## Configuration + +All through environment variables; nothing about where it runs lives in this repo. + +| Variable | What | +| --- | --- | +| `PANEL_URL` | The panel the worker takes jobs from | +| `BUILDER_TOKEN` | Shared secret the panel checks on `/api/v1/builder/**` (same value in the panel's env) | +| `GITEA_TOKEN` | The bot account's token (repository read/write on this repo): pushes draft branches, opens PRs | +| `OLLAMA_URL` | Ollama for the model (or `MODEL_PROVIDER=gemini` with `GEMINI_API_KEY`) | +| `SEARXNG_URL`, `READER_URL` | A SearXNG instance (JSON API) and a page-to-text reader (`READER_URL/`) | +| `MODEL`, `POLL_INTERVAL`, `WORKER_NAME`, `WORK_DIR` | Optional: model name, poll interval (20s), name shown in the panel, work folder | + +## Running a build by hand + +On any machine with Docker and the `gbr-boottest` network, with `OLLAMA_URL`, `SEARXNG_URL` and +`READER_URL` set: + +```bash +go run ./cmd/gbrgames research -name Valheim -out research.json +# pick a variant from research.json into v.json, then: +go run ./cmd/gbrgames build -name Valheim -variant @v.json -out build.json +``` + +The draft lands in `games//`, first-boot reports in `firstboot-out/`. diff --git a/Dockerfile.builder b/Dockerfile.builder new file mode 100644 index 0000000..dc483aa --- /dev/null +++ b/Dockerfile.builder @@ -0,0 +1,14 @@ +# The AI builder worker (gbrgames worker); see BUILDER.md. +FROM golang:1.24-alpine AS build +WORKDIR /src +COPY go.mod go.sum ./ +RUN go mod download +COPY cmd ./cmd +RUN CGO_ENABLED=0 go build -o /gbrgames ./cmd/gbrgames + +FROM alpine:3.22 +# docker CLI for first boots and boot tests (through the host's socket), git to clone and open PRs +RUN apk add --no-cache docker-cli git ca-certificates +COPY --from=build /gbrgames /usr/local/bin/gbrgames +WORKDIR /work +ENTRYPOINT ["gbrgames", "worker"] diff --git a/cmd/gbrgames/build.go b/cmd/gbrgames/build.go new file mode 100644 index 0000000..2bfcf4e --- /dev/null +++ b/cmd/gbrgames/build.go @@ -0,0 +1,372 @@ +package main + +import ( + "bytes" + "encoding/json" + "errors" + "flag" + "fmt" + "os" + "os/exec" + "path/filepath" + "strings" +) + +// build turns one researched variant into a template that has really booted: first boot of the image +// untouched (facts), a draft by the model, then the draft deployed with the node-agent's own code, exactly +// as CI's boot test does. What goes wrong in that boot (a crash, an error line, saves outside the volume) +// goes back to the model, a few rounds at most. The result is a games// folder ready for a pull request. + +type buildResult struct { + GameID string `json:"gameId"` + Dir string `json:"dir"` + Image string `json:"image"` + Passed bool `json:"passed"` + Rounds int `json:"rounds"` + Verified []string `json:"verified"` + Problems []string `json:"problems"` + Summary string `json:"summary"` +} + +const defaultAgentImage = "www.gitea.gbreed.com/gbreed-public/gbr-node-agent:latest" + +func runBuild(args []string) error { + fs := flag.NewFlagSet("build", flag.ExitOnError) + gamesDir := fs.String("games", "games", "directory holding one folder per game") + schemaDir := fs.String("schemas", "schema", "directory holding the JSON Schemas") + gameName := fs.String("name", "", "the game's name") + id := fs.String("id", "", "folder/id for the new game (default: from -name)") + variantArg := fs.String("variant", "", "the chosen variant: a research result variant as JSON, or @file") + notes := fs.String("notes", "", "what the admin asked for") + provider := fs.String("provider", "ollama", "ollama or gemini") + modelName := fs.String("model", "", "model name (default as for draft)") + attempts := fs.Int("attempts", 3, "fix rounds for validation problems, each time") + bootRounds := fs.Int("boot-rounds", 3, "boots of the draft before giving up") + out := fs.String("out", "", "write the result JSON here") + fs.Parse(args) + if *gameName == "" || *variantArg == "" { + return errors.New("build needs -name and -variant") + } + raw := []byte(*variantArg) + if strings.HasPrefix(*variantArg, "@") { + var err error + if raw, err = os.ReadFile((*variantArg)[1:]); err != nil { + return err + } + } + var v variant + if err := json.Unmarshal(raw, &v); err != nil { + return fmt.Errorf("-variant: %w", err) + } + if *id == "" { + *id = gameID(*gameName) + } + agentImage := os.Getenv("AGENT_IMAGE") + if agentImage == "" { + agentImage = defaultAgentImage + } + + res, err := build(buildOptions{gamesDir: *gamesDir, schemaDir: *schemaDir, name: *gameName, id: *id, variant: v, + notes: *notes, provider: *provider, model: *modelName, attempts: *attempts, bootRounds: *bootRounds, agentImage: agentImage}) + if res != nil && *out != "" { + raw, _ := json.MarshalIndent(res, "", " ") + if werr := os.WriteFile(*out, raw, 0o644); werr != nil && err == nil { + err = werr + } + } + if err != nil { + return err + } + if !res.Passed { + return fmt.Errorf("the draft in %s still has problems:\n- %s", res.Dir, strings.Join(res.Problems, "\n- ")) + } + fmt.Printf("[build] %s booted with the node-agent and is ready for a pull request\n", res.Dir) + return nil +} + +type buildOptions struct { + gamesDir, schemaDir, name, id string + variant variant + notes, provider, model string + attempts, bootRounds int + agentImage string +} + +func build(o buildOptions) (*buildResult, error) { + if len(o.variant.Images) == 0 { + return nil, errors.New("the variant has no image to build from") + } + res := &buildResult{GameID: o.id, Dir: filepath.Join(o.gamesDir, o.id)} + res.Verified = append(res.Verified, o.variant.Verified...) + + // 1. First boot of each candidate image until one runs as a non-root user + var report *bootReport + for _, image := range o.variant.Images { + steam := isSteamCMDImage(image) && o.variant.SteamAppID != "" + if isSteamCMDImage(image) && !steam { + continue // the SteamCMD image alone runs nothing + } + res.Image = image + if steam && o.variant.StartupHint == "" { + fmt.Printf("[build] no startup command known for Steam app %s; drafting without a first boot\n", o.variant.SteamAppID) + report = nil + break + } + fmt.Printf("[build] first boot of %s\n", image) + r, err := firstBoot(o.id+"-probe", image, o.variant.SteamAppID, steam, o.variant.StartupHint) + if err != nil { + return nil, err + } + if why := imageUnusable(r); why != "" { + fmt.Printf("[build] %s can't be used: %s\n", image, why) + res.Problems = append(res.Problems, why) + res.Image = "" + continue + } + report = r + res.Verified = append(res.Verified, fmt.Sprintf("first boot of %s: %s", image, r.Outcome)) + break + } + if res.Image == "" { + return res, errors.New("no image of this variant runs as a non-root user") + } + res.Problems = nil + + // 2. Draft until validation and the image checks pass + steamApp := "" + if isSteamCMDImage(res.Image) { + steamApp = o.variant.SteamAppID + } + d, err := newDrafter(draftJob{GamesDir: o.gamesDir, SchemaDir: o.schemaDir, Name: o.name, ID: o.id, Image: res.Image, + SteamApp: steamApp, Notes: variantNotes(o.variant, o.notes), Report: report, Provider: o.provider, Model: o.model, NoPull: true}) + if err != nil { + return nil, err + } + problems, err := d.refine(o.attempts, nil) + res.Rounds = d.rounds + if err != nil { + return res, err + } + if len(problems) > 0 { + res.Problems = problems + res.Summary = buildSummary(o, res) + return res, nil + } + + // 3. Boot the draft the way nodes run it; failures go back to the model + wantOutside := len(outsideDirs(report)) > 0 + for boot := 1; boot <= o.bootRounds; boot++ { + fmt.Printf("[build] boot %d of the draft with the node-agent\n", boot) + bootProblems, notes, err := bootCheck(o.gamesDir, o.id, o.agentImage, wantOutside) + if err != nil { + return res, err + } + if len(bootProblems) == 0 { + res.Passed = true + res.Verified = append(res.Verified, notes...) + break + } + fmt.Printf("[build] boot %d failed:\n", boot) + for _, p := range bootProblems { + fmt.Println(" - " + firstLine(p)) + } + res.Problems = bootProblems + if boot == o.bootRounds { + break + } + problems, err := d.refine(o.attempts, bootProblems) + res.Rounds = d.rounds + if err != nil { + return res, err + } + if len(problems) > 0 { + res.Problems = problems + break + } + } + if res.Passed { + res.Problems = nil + } + res.Summary = buildSummary(o, res) + return res, nil +} + +func isSteamCMDImage(image string) bool { + return image == "cm2network/steamcmd" || strings.HasPrefix(image, "cm2network/steamcmd:") +} + +func variantNotes(v variant, admin string) string { + var parts []string + if v.Title != "" { + parts = append(parts, "variant: "+v.Title+" ("+v.Description+")") + } + if v.StartupHint != "" { + parts = append(parts, "documented startup: "+v.StartupHint) + } + if len(v.Ports) > 0 { + parts = append(parts, "documented ports: "+strings.Join(v.Ports, ", ")) + } + if v.Notes != "" { + parts = append(parts, v.Notes) + } + if admin != "" { + parts = append(parts, "the admin asked for: "+admin) + } + return strings.Join(parts, "; ") +} + +// firstBoot runs first-boot and reads back its report. +func firstBoot(name, image, steamApp string, steam bool, startup string) (*bootReport, error) { + outDir := filepath.Join("firstboot-out", name) + args := []string{"-image", image, "-out", outDir} + if steam { + args = append(args, "-steam-app", steamApp, "-startup", startup) + } + if err := runFirstBoot(args); err != nil { + return nil, err + } + return readBootReport(filepath.Join(outDir, "report.json")) +} + +// bootCheck deploys the template with the node-agent's BootTest, the same check CI runs, and when the +// game wrote files outside its volume on the first boot, boots it once more to see that it no longer does. +func bootCheck(gamesDir, id, agentImage string, wantOutside bool) (problems, verified []string, err error) { + self, err := os.Executable() + if err != nil { + return nil, nil, err + } + payloadCmd := exec.Command(self, "boottest-payload", "-games", gamesDir, id) + var stderr bytes.Buffer + payloadCmd.Stderr = &stderr + payload, err := payloadCmd.Output() + if err != nil { + var exitErr *exec.ExitError + if errors.As(err, &exitErr) && exitErr.ExitCode() == exitMissingSecret { + // The builder has no CI secrets: that's the template's problem to fix, not a reason to stop + return []string{"the boot test can't run: " + strings.TrimSpace(stderr.String()) + ". Remove 'secrets' from " + + "test.yaml; boot tests fill password settings with a throwaway value"}, nil, nil + } + return nil, nil, fmt.Errorf("boottest-payload: %v: %s", err, strings.TrimSpace(stderr.String())) + } + if out, err := runCmd("docker", "pull", "-q", agentImage); err != nil { + return nil, nil, fmt.Errorf("docker pull %s: %v: %s", agentImage, err, out) + } + cmd := exec.Command("docker", "run", "--rm", "-i", "-v", "/var/run/docker.sock:/var/run/docker.sock", + "--entrypoint", "java", agentImage, "-Dloader.main=com.gbreed.gbr.nodeagent.BootTest", "-cp", "app.jar", + "org.springframework.boot.loader.launch.PropertiesLauncher") + cmd.Stdin = bytes.NewReader(payload) + var output bytes.Buffer + cmd.Stdout = &teeWriter{&output} + cmd.Stderr = cmd.Stdout + runErr := cmd.Run() + if runErr != nil { + var exitErr *exec.ExitError + if !errors.As(runErr, &exitErr) { + return nil, nil, runErr + } + return []string{bootFailure(output.String())}, nil, nil + } + verified = append(verified, "boot test with the node-agent: "+bootVerdict(output.String())) + + if wantOutside { + fmt.Println("[build] first boot wrote saves outside the volume; booting the draft to see that it no longer does") + r, err := firstBootGame(gamesDir, id) + if err != nil { + return nil, nil, err + } + if dirs := outsideDirs(r); len(dirs) > 0 { + return []string{fmt.Sprintf("booting your template, the server still wrote its own files outside the server volume, "+ + "in %s, so your startup command or settings did not move them. Check the exact launch option name and "+ + "syntax this server documents for its save/data folder (a wrong option is usually ignored silently) "+ + "and point it at /server", strings.Join(dirs, ", "))}, nil, nil + } + verified = append(verified, "the server keeps its files inside /server") + } + return nil, verified, nil +} + +func firstBootGame(gamesDir, id string) (*bootReport, error) { + outDir := filepath.Join("firstboot-out", id+"-draft") + if err := runFirstBoot([]string{"-games", gamesDir, "-game", id, "-out", outDir}); err != nil { + return nil, err + } + return readBootReport(filepath.Join(outDir, "report.json")) +} + +// bootFailure turns BootTest's output into feedback: its verdict and the server's last lines, which hold +// the actual error (an unknown option, a missing file, a port in use). +func bootFailure(output string) string { + verdict, tail := bootVerdict(output), output + if i := strings.Index(output, "---- last "); i >= 0 { + tail = output[i:] + } + lines := strings.Split(strings.TrimSpace(tail), "\n") + if len(lines) > 60 { + lines = append(lines[:1], lines[len(lines)-59:]...) + } + return "booting your template with the node-agent (the real deploy code) failed: " + verdict + + ". Find the cause in the server's output below and fix the template (startup command, settings, " + + "ready pattern, memory):\n" + strings.Join(lines, "\n") +} + +func bootVerdict(output string) string { + for _, l := range strings.Split(output, "\n") { + for _, k := range []string{"[boot-test] FAILED: ", "[boot-test] PASSED: "} { + if i := strings.Index(l, k); i >= 0 { + return strings.TrimSpace(l[i+len("[boot-test] "):]) + } + } + } + return "no verdict" +} + +func firstLine(s string) string { + line, _, _ := strings.Cut(s, "\n") + return line +} + +// teeWriter shows the boot as it happens (the worker streams it to the panel) while keeping it for the verdict. +type teeWriter struct{ buf *bytes.Buffer } + +func (t *teeWriter) Write(p []byte) (int, error) { + os.Stdout.Write(p) + return t.buf.Write(p) +} + +// buildSummary is the pull request's description: where the facts came from and what was really checked. +func buildSummary(o buildOptions, res *buildResult) string { + var b strings.Builder + fmt.Fprintf(&b, "Drafted by the GBR AI builder. **Review before merging**: check the settings, "+ + "ports and descriptions against the game's documentation.\n\n") + fmt.Fprintf(&b, "- Game: **%s**", o.name) + if o.variant.Title != "" { + fmt.Fprintf(&b, " — %s", o.variant.Title) + } + fmt.Fprintf(&b, "\n- Image: `%s`", res.Image) + if o.variant.SteamAppID != "" && isSteamCMDImage(res.Image) { + fmt.Fprintf(&b, ", Steam app %s", o.variant.SteamAppID) + } + fmt.Fprintf(&b, "\n- Model rounds: %d\n", res.Rounds) + if len(res.Verified) > 0 { + b.WriteString("\n**Checked for real**\n") + for _, v := range res.Verified { + fmt.Fprintf(&b, "- %s\n", v) + } + } + if len(res.Problems) > 0 { + b.WriteString("\n**Still open** (the builder could not fix these)\n") + for _, p := range res.Problems { + fmt.Fprintf(&b, "- %s\n", firstLine(p)) + } + } + if o.variant.Notes != "" { + fmt.Fprintf(&b, "\n**Notes from research:** %s\n", o.variant.Notes) + } + if len(o.variant.Sources) > 0 { + b.WriteString("\n**Sources**\n") + for _, s := range o.variant.Sources { + fmt.Fprintf(&b, "- %s\n", s) + } + } + return b.String() +} diff --git a/cmd/gbrgames/build_test.go b/cmd/gbrgames/build_test.go new file mode 100644 index 0000000..75e9039 --- /dev/null +++ b/cmd/gbrgames/build_test.go @@ -0,0 +1,50 @@ +package main + +import ( + "strings" + "testing" +) + +func TestBootFailureKeepsVerdictAndServerOutput(t *testing.T) { + out := "[boot-test] deploying boottest-valheim (cm2network/steamcmd)\n" + + "[boot-test] container started; waiting for ready pattern /Game server connected/\n" + + "[boot-test] FAILED: container exited before it was ready\n" + + "---- last 2 log lines ----\n" + + "Unknown argument --save-dir\n" + + "Usage: valheim_server.x86_64 ...\n" + got := bootFailure(out) + if !strings.Contains(got, "failed: FAILED: container exited before it was ready") { + t.Errorf("verdict missing: %s", got) + } + if !strings.Contains(got, "Unknown argument --save-dir") || strings.Contains(got, "deploying boottest") { + t.Errorf("want only the server's last lines, got: %s", got) + } + if v := bootVerdict("[boot-test] PASSED: READY: Game server connected"); v != "PASSED: READY: Game server connected" { + t.Errorf("verdict = %q", v) + } +} + +func TestOutsideDirsSkipsSteamNoise(t *testing.T) { + r := &bootReport{ContainerChanges: []string{ + "A /home/steam/.config/unity3d/IronGate/Valheim/worlds_local/w.db", + "A /home/steam/Steam/logs/connection_log.txt", + "C /home/steam/.steam/registry.vdf", + }} + got := outsideDirs(r) + if len(got) != 1 || got[0] != "/home/steam/.config/unity3d/IronGate/Valheim/worlds_local" { + t.Errorf("got %v", got) + } + if outsideDirs(nil) != nil { + t.Error("nil report should have no dirs") + } +} + +func TestVariantNotes(t *testing.T) { + v := variant{Title: "Crossplay", Description: "PC + Xbox", StartupHint: "./valheim_server.x86_64 -crossplay", Ports: []string{"2456/udp"}} + got := variantNotes(v, "needs Xbox") + for _, want := range []string{"variant: Crossplay (PC + Xbox)", "documented startup: ./valheim_server.x86_64 -crossplay", "2456/udp", "the admin asked for: needs Xbox"} { + if !strings.Contains(got, want) { + t.Errorf("missing %q in %q", want, got) + } + } +} diff --git a/cmd/gbrgames/draft.go b/cmd/gbrgames/draft.go index e6cf232..dc62d25 100644 --- a/cmd/gbrgames/draft.go +++ b/cmd/gbrgames/draft.go @@ -16,6 +16,8 @@ import ( "sort" "strings" "time" + + "github.com/santhosh-tekuri/jsonschema/v6" ) // draft writes a first template for a new game with an AI model: it gives the model AUTHORING.md as its @@ -52,102 +54,167 @@ func runDraft(args []string) error { if *gameName == "" || *image == "" { return errors.New("draft needs -name and -image") } - if *id == "" { - *id = strings.Trim(regexp.MustCompile(`[^a-z0-9]+`).ReplaceAllString(strings.ToLower(*gameName), "-"), "-") + var report *bootReport + if *reportPath != "" { + var err error + if report, err = readBootReport(*reportPath); err != nil { + return err + } } - dir := filepath.Join(*gamesDir, *id) - if _, err := os.Stat(dir); err == nil { - return fmt.Errorf("%s already exists; drafts only create new games", dir) - } - - m, err := newModel(*provider, *modelName) + d, err := newDrafter(draftJob{GamesDir: *gamesDir, SchemaDir: *schemaDir, Name: *gameName, ID: *id, Image: *image, + SteamApp: *steamApp, Notes: *notes, Report: report, Provider: *provider, Model: *modelName, NoPull: *noPull}) if err != nil { return err } + if problems, err := d.refine(*attempts, nil); err != nil { + return err + } else if len(problems) > 0 { + return fmt.Errorf("draft still has problems after %d fix rounds; it is in %s for a person to finish", *attempts, d.dir) + } + fmt.Println("[draft] next: review it, then boot-test it (build / CI) before opening a PR") + return nil +} + +type draftJob struct { + GamesDir, SchemaDir string + Name, ID, Image string + SteamApp, Notes string + Report *bootReport + Provider, Model string + NoPull bool +} + +// drafter holds one conversation with the model about one game, so later problems (a failed boot) can +// be sent back to it with everything it was told before. +type drafter struct { + job draftJob + dir, id string + m model + facts *imageFacts + gameSchema, testSchema *jsonschema.Schema + answerSchema map[string]any + messages []chatMessage + rounds int +} + +func newDrafter(j draftJob) (*drafter, error) { + id := j.ID + if id == "" { + id = gameID(j.Name) + } + d := &drafter{job: j, id: id, dir: filepath.Join(j.GamesDir, id)} + if _, err := os.Stat(d.dir); err == nil { + return nil, fmt.Errorf("%s already exists; drafts only create new games", d.dir) + } + var err error + if d.m, err = newModel(j.Provider, j.Model); err != nil { + return nil, err + } rules, err := os.ReadFile("AUTHORING.md") if err != nil { - return fmt.Errorf("AUTHORING.md (the model's rules): %w", err) + return nil, fmt.Errorf("AUTHORING.md (the model's rules): %w", err) } - gameSchema, err := compileSchema(filepath.Join(*schemaDir, "game.schema.json")) - if err != nil { - return err + if d.gameSchema, err = compileSchema(filepath.Join(j.SchemaDir, "game.schema.json")); err != nil { + return nil, err } - testSchema, err := compileSchema(filepath.Join(*schemaDir, "test.schema.json")) - if err != nil { - return err + if d.testSchema, err = compileSchema(filepath.Join(j.SchemaDir, "test.schema.json")); err != nil { + return nil, err } - answerSchema, err := draftAnswerSchema(*schemaDir) - if err != nil { - return err + if d.answerSchema, err = draftAnswerSchema(j.SchemaDir); err != nil { + return nil, err } - - if !*noPull { - fmt.Printf("[draft] pulling %s\n", *image) - if out, err := runCmd("docker", "pull", "-q", *image); err != nil { - return fmt.Errorf("docker pull: %v: %s", err, out) + if !j.NoPull { + fmt.Printf("[draft] pulling %s\n", j.Image) + if out, err := runCmd("docker", "pull", "-q", j.Image); err != nil { + return nil, fmt.Errorf("docker pull: %v: %s", err, out) } } - facts, err := inspectImage(*image, nil) - if err != nil { - return err + if d.facts, err = inspectImage(j.Image, nil); err != nil { + return nil, err } - var report *bootReport - if *reportPath != "" { - raw, err := os.ReadFile(*reportPath) - if err != nil { - return err - } - report = &bootReport{} - if err := json.Unmarshal(raw, report); err != nil { - return fmt.Errorf("%s: %w", *reportPath, err) - } - if why := imageUnusable(report); why != "" { - return errors.New(why) - } + if why := imageUnusable(j.Report); why != "" { + return nil, errors.New(why) } - - messages := []chatMessage{ + d.messages = []chatMessage{ {Role: "system", Content: draftSystemPrompt(string(rules))}, - {Role: "user", Content: draftRequest(*gameName, *id, *image, *steamApp, *notes, facts, report, exampleTemplates(*gamesDir, 2))}, + {Role: "user", Content: draftRequest(j.Name, id, j.Image, j.SteamApp, j.Notes, d.facts, j.Report, exampleTemplates(j.GamesDir, 2))}, } + return d, nil +} - for round := 0; round <= *attempts; round++ { - fmt.Printf("[draft] round %d: asking %s\n", round+1, m.name()) +func gameID(name string) string { + return strings.Trim(regexp.MustCompile(`[^a-z0-9]+`).ReplaceAllString(strings.ToLower(name), "-"), "-") +} + +// refine asks for a (corrected) template until it passes validation and the image checks, at most +// attempts+1 answers. feedback, when given, is what went wrong since the last answer (e.g. its boot). +// It returns the problems still left, nil when the draft passes. +func (d *drafter) refine(attempts int, feedback []string) ([]string, error) { + if len(feedback) > 0 { + d.ask(feedback) + } + var problems []string + for round := 0; round <= attempts; round++ { + d.rounds++ + fmt.Printf("[draft] round %d: asking %s\n", d.rounds, d.m.name()) start := time.Now() - answer, err := m.chat(messages, answerSchema) + answer, err := d.m.chat(d.messages, d.answerSchema) if err != nil { - return err + return nil, err } fmt.Printf("[draft] answer in %.0fs\n", time.Since(start).Seconds()) - messages = append(messages, chatMessage{Role: "assistant", Content: answer}) + d.messages = append(d.messages, chatMessage{Role: "assistant", Content: answer}) - problems := writeDraft(dir, *id, answer, report) - if len(problems) == 0 { - problems = validateGame(dir, *id, gameSchema, testSchema) - } - if len(problems) == 0 { - if g, err := loadMap(filepath.Join(dir, "game.yaml")); err == nil { - imageProblems, _ := checkTemplate(g, facts) - problems = append(problems, imageProblems...) - t, _ := loadMap(filepath.Join(dir, "test.yaml")) - problems = append(problems, draftCompleteness(g, t)...) - problems = append(problems, unreadSettings(g, facts)...) - problems = append(problems, savesOutsideVolumes(g, report)...) - } - } - if len(problems) == 0 { - fmt.Printf("[draft] %s passes validation and the image checks\n", dir) - fmt.Println("[draft] next: review it, then boot-test it (first-boot / CI) before opening a PR") - return nil + if problems = d.check(answer); len(problems) == 0 { + fmt.Printf("[draft] %s passes validation and the image checks\n", d.dir) + return nil, nil } fmt.Printf("[draft] %d problem(s):\n", len(problems)) for _, p := range problems { fmt.Println(" - " + p) } - messages = append(messages, chatMessage{Role: "user", Content: "Your template has these problems. Fix all of them and " + - "answer with the complete corrected JSON object (both \"game\" and \"test\"):\n- " + strings.Join(problems, "\n- ")}) + if round < attempts { + d.ask(problems) + } } - return fmt.Errorf("draft still has problems after %d fix rounds; it is in %s for a person to finish", *attempts, dir) + return problems, nil +} + +func (d *drafter) ask(problems []string) { + d.messages = append(d.messages, chatMessage{Role: "user", Content: "Your template has these problems. Fix all of them and " + + "answer with the complete corrected JSON object (both \"game\" and \"test\"):\n- " + strings.Join(problems, "\n- ")}) +} + +func (d *drafter) check(answer string) []string { + problems := writeDraft(d.dir, d.id, answer, d.job.Report) + if len(problems) == 0 { + problems = validateGame(d.dir, d.id, d.gameSchema, d.testSchema) + } + if len(problems) == 0 { + if g, err := loadMap(filepath.Join(d.dir, "game.yaml")); err == nil { + imageProblems, _ := checkTemplate(g, d.facts) + problems = append(problems, imageProblems...) + t, _ := loadMap(filepath.Join(d.dir, "test.yaml")) + problems = append(problems, draftCompleteness(g, t)...) + problems = append(problems, unreadSettings(g, d.facts)...) + problems = append(problems, savesOutsideVolumes(g, d.job.Report)...) + problems = append(problems, splitSettings(g)...) + problems = append(problems, portSettings(g)...) + } + } + return problems +} + +func readBootReport(p string) (*bootReport, error) { + raw, err := os.ReadFile(p) + if err != nil { + return nil, err + } + r := &bootReport{} + if err := json.Unmarshal(raw, r); err != nil { + return nil, fmt.Errorf("%s: %w", p, err) + } + return r, nil } func draftSystemPrompt(rules string) string { @@ -203,6 +270,11 @@ func draftCompleteness(game, test map[string]any) []string { } else if v, _ := n.Int64(); v < 1024 { problems = append(problems, "requiredMemoryMb is below 1024") } + // Secrets only exist as CI settings a person adds; a draft with one can't be boot-tested anywhere + if secrets, _ := test["secrets"].(map[string]any); len(secrets) > 0 { + problems = append(problems, "test.yaml has 'secrets' ("+strings.Join(sortedKeys(secrets), ", ")+"), which no boot test of a "+ + "new game has: remove them. Password settings get a throwaway value in boot tests automatically") + } if _, ok := game["readyPattern"]; !ok { if _, ok := test["settleSec"]; !ok { problems = append(problems, "no readyPattern and no test.settleSec: boot tests need one of them") @@ -269,22 +341,7 @@ var outsideNoise = []string{"/Steam/logs/", "/.steam/", "/steamcmd/", "/.cache/" // node recreates the container. A template passes once its startup command or settings point the game at // /server or /data; a boot test then shows whether that really worked. func savesOutsideVolumes(game map[string]any, report *bootReport) []string { - if report == nil { - return nil - } - dirs := map[string]bool{} - for _, c := range report.ContainerChanges { - _, p, _ := strings.Cut(c, " ") - noise := false - for _, n := range outsideNoise { - if strings.Contains(p, n) { - noise = true - } - } - if !noise && p != "" { - dirs[path.Dir(p)] = true - } - } + dirs := outsideDirs(report) if len(dirs) == 0 { return nil } @@ -302,7 +359,28 @@ func savesOutsideVolumes(game map[string]any, report *bootReport) []string { return []string{fmt.Sprintf("first boot: the server wrote its own files outside the server volume, in %s. Those are lost "+ "whenever the node recreates the container (every start). Make the game keep them under /server, e.g. with "+ "its save-directory launch option or by pointing that folder at /server, and include it in backupPaths", - strings.Join(sortedKeysBool(dirs), ", "))} + strings.Join(dirs, ", "))} +} + +// outsideDirs lists the folders outside the volumes where the game itself wrote files during a boot. +func outsideDirs(report *bootReport) []string { + if report == nil { + return nil + } + dirs := map[string]bool{} + for _, c := range report.ContainerChanges { + _, p, _ := strings.Cut(c, " ") + noise := false + for _, n := range outsideNoise { + if strings.Contains(p, n) { + noise = true + } + } + if !noise && p != "" { + dirs[path.Dir(p)] = true + } + } + return sortedKeysBool(dirs) } // imageUnusable recognises a first boot where the image died within seconds because it needs root (or @@ -400,13 +478,7 @@ func exampleTemplates(gamesDir string, n int) string { // writeDraft turns the model's JSON answer into games//game.yaml and test.yaml, using the importer's // writer so drafts look like every other template. func writeDraft(dir, id, answer string, report *bootReport) []string { - answer = strings.TrimSpace(answer) - if i := strings.Index(answer, "{"); i > 0 { - answer = answer[i:] // some models still wrap JSON in prose or code fences - } - if i := strings.LastIndex(answer, "}"); i >= 0 { - answer = answer[:i+1] - } + answer = trimToJSON(answer) var parsed struct { Game map[string]any `json:"game"` Test map[string]any `json:"test"` diff --git a/cmd/gbrgames/draft_test.go b/cmd/gbrgames/draft_test.go index 81a8fe9..062fa7c 100644 --- a/cmd/gbrgames/draft_test.go +++ b/cmd/gbrgames/draft_test.go @@ -84,3 +84,12 @@ func TestImageUnusable(t *testing.T) { t.Errorf("long run flagged: %q", why) } } + +func TestDraftCompletenessRejectsSecrets(t *testing.T) { + // The Valheim draft that couldn't be boot-tested: the password mapped to a CI secret nobody created + test := map[string]any{"settleSec": json.Number("120"), "secrets": map[string]any{"Password": "VALHEIM_TEST_PASSWORD"}} + problems := strings.Join(draftCompleteness(map[string]any{}, test), "\n") + if !strings.Contains(problems, "test.yaml has 'secrets' (Password)") { + t.Errorf("got:\n%s", problems) + } +} diff --git a/cmd/gbrgames/draftchecks.go b/cmd/gbrgames/draftchecks.go new file mode 100644 index 0000000..5db39a0 --- /dev/null +++ b/cmd/gbrgames/draftchecks.go @@ -0,0 +1,250 @@ +package main + +import ( + "encoding/json" + "fmt" + "regexp" + "sort" + "strconv" + "strings" +) + +// Checks a booting draft can still fail: things that only go wrong once an owner changes a setting, so +// neither the validator nor a boot test with the defaults sees them. + +// splitSettings finds text settings that reach the server's command line unquoted, so a value with spaces +// ("GBRHosting - My Server") arrives as several arguments and the game only sees the first word. Covers +// direct use (-name ${SERVER_NAME}) and the common ARGS="-name ${SERVER_NAME}"; ./server $ARGS, where the +// quotes in the assignment don't survive the unquoted $ARGS. +func splitSettings(game map[string]any) []string { + startup, _ := game["defaultStartupCommand"].(string) + if startup == "" { + return nil + } + textVars := map[string]string{} // env name -> setting key + for key, p := range schemaProperties(game) { + if prop, _ := p.(map[string]any); prop["type"] == "string" { + textVars[envName(key)] = key + } + } + if len(textVars) == 0 { + return nil + } + s := scanShell(unwrapShell(startup)) + + split := map[string]string{} // setting key -> how it gets split + var reach func(holder string, seen map[string]bool) []string + reach = func(holder string, seen map[string]bool) []string { + if seen[holder] { + return nil + } + seen[holder] = true + var out []string + for _, v := range s.held[holder] { + if _, ok := textVars[v]; ok { + out = append(out, v) + } + out = append(out, reach(v, seen)...) + } + return out + } + for _, e := range s.uses { + if e.quoted { + continue + } + if key, ok := textVars[e.name]; ok { + split[key] = "${" + e.name + "} is not in double quotes" + } + if len(s.held[e.name]) > 0 { + for _, v := range reach(e.name, map[string]bool{}) { + if _, done := split[textVars[v]]; !done { + split[textVars[v]] = fmt.Sprintf("${%s} is inside $%s, which is used without double quotes", v, e.name) + } + } + } + } + var problems []string + for _, key := range sortedKeysString(split) { + problems = append(problems, fmt.Sprintf("setting %s is split into words on the command line (%s): a value with spaces like "+ + "\"GBRHosting - My Server\" reaches the game as several arguments. Quote it where it is used (-name \"${%s}\"), or build "+ + "optional arguments in a bash array: args=(-name \"$%s\"); [ -n \"$X\" ] && args+=(-x \"$X\"); ./server \"${args[@]}\"", + key, split[key], envName(key), envName(key))) + } + return problems +} + +var shellWrap = regexp.MustCompile(`^\s*(?:ba)?sh\s+-c\s+'(.*)'\s*$`) + +// unwrapShell looks inside a startup command that is one bash -c '...' (ARK's), which would otherwise be +// all single-quoted text. +func unwrapShell(cmd string) string { + if m := shellWrap.FindStringSubmatch(cmd); m != nil && !strings.Contains(m[1], "'") { + return m[1] + } + return cmd +} + +type shellUse struct { + name string + quoted bool +} + +type shellScan struct { + uses []shellUse // expansions where the value is used: words, arguments + held map[string][]string // variable -> variables its assigned value contains +} + +var ( + assignWord = regexp.MustCompile(`^([A-Za-z_][A-Za-z0-9_]*)\+?=`) + varName = regexp.MustCompile(`^[A-Za-z_][A-Za-z0-9_]*`) + nestedVar = regexp.MustCompile(`\$\{?([A-Za-z_][A-Za-z0-9_]*)`) +) + +// scanShell is just enough of a shell parser for splitSettings: quotes, $VAR / ${VAR...} expansions, and +// assignments (NAME=..., NAME+=..., NAME=(...) arrays). +func scanShell(cmd string) *shellScan { + s := &shellScan{held: map[string][]string{}} + var single, double, wordStart = false, false, true + assign, inArray := "", false + record := func(name string, quoted bool) { + switch { + case assign != "" && !inArray: + // NAME="...$X..." : no splitting yet, but $NAME carries X along + s.held[assign] = append(s.held[assign], name) + case inArray && quoted: + s.held[assign] = append(s.held[assign], name) // safe unless the array is expanded unquoted + default: + s.uses = append(s.uses, shellUse{name, quoted}) + } + } + for i := 0; i < len(cmd); i++ { + c := cmd[i] + if wordStart && !single && !double { + if m := assignWord.FindString(cmd[i:]); m != "" { + assign = strings.TrimRight(strings.TrimSuffix(m, "="), "+") + i += len(m) + if i < len(cmd) && cmd[i] == '(' { + inArray = true + } else { + i-- + } + wordStart = false + continue + } + } + wordStart = false + switch { + case c == '\\' && !single: + i++ + case c == '\'' && !double: + single = !single + case c == '"' && !single: + double = !double + case c == '$' && !single && i+1 < len(cmd): + if cmd[i+1] == '{' { + end := matchingBrace(cmd, i+1) + body := cmd[i+2 : end] + name := varName.FindString(strings.TrimLeft(body, "#!")) + if name != "" { + record(name, double) + } + // Defaults can hold more expansions: ${A:-${B}} + for _, m := range nestedVar.FindAllStringSubmatch(body[len(name):], -1) { + record(m[1], double) + } + i = end + } else if name := varName.FindString(cmd[i+1:]); name != "" { + record(name, double) + i += len(name) + } + case single || double: + case inArray && c == ')': + inArray, assign = false, "" + case inArray: + case c == ' ' || c == '\t' || c == '\n' || c == ';' || c == '&' || c == '|' || c == '(' || c == ')': + assign, wordStart = "", true + } + } + return s +} + +func matchingBrace(s string, open int) int { + depth := 0 + for i := open; i < len(s); i++ { + switch s[i] { + case '{': + depth++ + case '}': + depth-- + if depth == 0 { + return i + } + } + } + return len(s) - 1 +} + +var ( + portSetting = regexp.MustCompile(`(?i)(port$|(^|_)port(_|$))`) + notAPort = regexp.MustCompile(`(?i)(sup|re|ex|im|trans)port$`) + // Reached over the Docker network, never published (AUTHORING.md) + internalPort = regexp.MustCompile(`(?i)rcon|rest|api|admin|telnet|web|http|metrics`) +) + +// portSettings checks settings that set a port players connect to against defaultPorts. The node always +// publishes defaultPorts' container ports (moving only the host side when servers share a node), so a +// game listening anywhere else can't be reached: a default that isn't published is broken from the +// start, and an editable one breaks the moment an owner changes it. +func portSettings(game map[string]any) []string { + published := map[int]bool{} + ports, _ := game["defaultPorts"].([]any) + for _, p := range ports { + n, _ := strconv.Atoi(strings.Split(containerPort(fmt.Sprint(p)), "/")[0]) + published[n] = true + } + defaults, _ := game["defaultConfig"].(map[string]any) + props := schemaProperties(game) + var problems []string + keys := make([]string, 0, len(defaults)) + for k := range defaults { + keys = append(keys, k) + } + sort.Strings(keys) + for _, key := range keys { + if !portSetting.MatchString(key) || notAPort.MatchString(key) || internalPort.MatchString(key) { + continue + } + port, ok := portNumber(defaults[key]) + if !ok { + continue // {PUBLIC_PORT_n} and other placeholders are filled per server + } + _, editable := props[key] + switch { + case !published[port]: + problems = append(problems, fmt.Sprintf("setting %s defaults to port %d, which defaultPorts does not publish: players can't reach it. "+ + "Use one of the published ports, or add this port to defaultPorts if players connect to it", key, port)) + case editable: + problems = append(problems, fmt.Sprintf("setting %s lets owners change the port, but the node always publishes container port %d "+ + "(defaultPorts), so any other value makes the server unreachable. Don't make it a setting: remove %s from "+ + "defaultConfig and configSchema and write %d directly in the startup command or config file. If the game must "+ + "announce the port players use, give it {PUBLIC_PORT_%d} instead", key, port, key, port, port)) + } + } + return problems +} + +func portNumber(v any) (int, bool) { + var s string + switch n := v.(type) { + case json.Number: + s = n.String() + case string: + s = n + case int, int64, float64: + s = fmt.Sprint(n) + default: + return 0, false + } + p, err := strconv.Atoi(strings.TrimSpace(s)) + return p, err == nil && p > 0 && p < 65536 +} diff --git a/cmd/gbrgames/draftchecks_test.go b/cmd/gbrgames/draftchecks_test.go new file mode 100644 index 0000000..148ea9f --- /dev/null +++ b/cmd/gbrgames/draftchecks_test.go @@ -0,0 +1,107 @@ +package main + +import ( + "encoding/json" + "strings" + "testing" +) + +func textSettings(keys ...string) map[string]any { + props := map[string]any{} + for _, k := range keys { + props[k] = map[string]any{"type": "string"} + } + return map[string]any{"properties": props} +} + +func TestSplitSettingsCatchesTheValheimDraft(t *testing.T) { + // What the builder drafted for Valheim: quoted in the assignment, split by the unquoted $ARGS + game := map[string]any{ + "defaultStartupCommand": `ARGS="-name ${SERVER_NAME:-Valheim Server} -port 2456 -world ${WORLD_NAME:-Dedicated}"; ` + + `[ -n "$PASSWORD" ] && ARGS="$ARGS -password $PASSWORD"; [ "$CROSSPLAY" = "true" ] && ARGS="$ARGS -crossplay"; ` + + `HOME=/server ./valheim_server.x86_64 $ARGS`, + "configSchema": textSettings("ServerName", "WorldName", "Password"), + } + problems := splitSettings(game) + got := strings.Join(problems, "\n") + if len(problems) != 3 { + t.Fatalf("want ServerName, WorldName and Password, got:\n%s", got) + } + for _, key := range []string{"setting Password", "setting ServerName", "setting WorldName", "inside $ARGS"} { + if !strings.Contains(got, key) { + t.Errorf("missing %q in:\n%s", key, got) + } + } +} + +func TestSplitSettingsAcceptsQuotedAndArrays(t *testing.T) { + for _, startup := range []string{ + `./valheim_server.x86_64 -name "${SERVER_NAME}" -world "$WORLD_NAME" -port 2456`, + `args=(-name "$SERVER_NAME" -world "${WORLD_NAME:-Dedicated}"); [ -n "$PASSWORD" ] && args+=(-password "$PASSWORD"); ./valheim_server.x86_64 "${args[@]}"`, + `bash -c 'exec ./server -name "$SERVER_NAME"'`, + } { + game := map[string]any{"defaultStartupCommand": startup, "configSchema": textSettings("ServerName", "WorldName", "Password")} + if p := splitSettings(game); len(p) != 0 { + t.Errorf("%s\n flagged: %v", startup, p) + } + } +} + +func TestSplitSettingsDirectAndArrayMisuse(t *testing.T) { + cases := map[string]string{ + `./server -name ${SERVER_NAME}`: "is not in double quotes", + `bash -c 'exec ./server -name $SERVER_NAME'`: "is not in double quotes", + `args=(-name "$SERVER_NAME"); ./server ${args[@]}`: "inside $args", + `args=(-name $SERVER_NAME); ./server "${args[@]}"`: "is not in double quotes", + `export OPTS="-name ${SERVER_NAME}" && ./run $OPTS`: "inside $OPTS", + } + for startup, want := range cases { + game := map[string]any{"defaultStartupCommand": startup, "configSchema": textSettings("ServerName")} + got := strings.Join(splitSettings(game), "\n") + if !strings.Contains(got, want) { + t.Errorf("%s\n want %q, got %q", startup, want, got) + } + } + // Numbers and booleans can't contain spaces + game := map[string]any{"defaultStartupCommand": `./server -port ${GAME_PORT}`, "configSchema": map[string]any{ + "properties": map[string]any{"GamePort": map[string]any{"type": "integer"}}}} + if p := splitSettings(game); len(p) != 0 { + t.Errorf("integer setting flagged: %v", p) + } +} + +func TestPortSettings(t *testing.T) { + game := map[string]any{ + "defaultPorts": []any{"2456/udp", "2457/udp"}, + "defaultConfig": map[string]any{ + "GamePort": json.Number("2456"), + "QueryPort": json.Number("27015"), + "RCONPort": json.Number("25575"), + "PublicPort": "{PUBLIC_PORT_2456}", + "Transport": "udp", + "Export": json.Number("9"), + }, + "configSchema": map[string]any{"properties": map[string]any{ + "GamePort": map[string]any{"type": "integer"}, "QueryPort": map[string]any{"type": "integer"}, + }}, + } + got := strings.Join(portSettings(game), "\n") + if !strings.Contains(got, "setting GamePort lets owners change the port") || !strings.Contains(got, "write 2456 directly") { + t.Errorf("editable published port not caught:\n%s", got) + } + if !strings.Contains(got, "setting QueryPort defaults to port 27015, which defaultPorts does not publish") { + t.Errorf("unpublished port not caught:\n%s", got) + } + for _, key := range []string{"RCONPort", "PublicPort", "Transport", "Export"} { + if strings.Contains(got, "setting "+key+" ") { + t.Errorf("%s should be left alone:\n%s", key, got) + } + } + + // A published port written by the template itself, not offered to owners, is fine + delete(game["configSchema"].(map[string]any)["properties"].(map[string]any), "GamePort") + delete(game["defaultConfig"].(map[string]any), "QueryPort") + if p := portSettings(game); len(p) != 0 { + t.Errorf("got %v", p) + } +} diff --git a/cmd/gbrgames/firstboot.go b/cmd/gbrgames/firstboot.go index 3b1342e..2724716 100644 --- a/cmd/gbrgames/firstboot.go +++ b/cmd/gbrgames/firstboot.go @@ -35,7 +35,7 @@ var ( readyWords = regexp.MustCompile(`(?i)\b(ready|started|listening|done|advertis\w*|online|accepting connections|initiali[sz]ed|server is running|now running|startup complete|full startup)\b`) // Words that mean the whole server is up, not just one part of it ("RCON Listener started") strongReady = regexp.MustCompile(`(?i)\b(ready|done|advertis\w*|accepting connections|server is running|startup complete|full startup|server started)\b`) - errorWords = regexp.MustCompile(`(?i)\b(error|exception|fatal|failed|permission denied|segmentation fault|could not|unable to|cannot)\b`) + errorWords = regexp.MustCompile(`(?i)\b(error|exception|fatal|failed|permission denied|segmentation fault|could not|unable to|cannot)\b`) // Leading timestamps and [bracketed] prefixes that change on every run linePrefix = regexp.MustCompile(`^\s*(?:(?:\[[^\]]*\]|\d{4}-\d\d-\d\d[T ][0-9:.,]+Z?|\d\d:\d\d:\d\d(?:[.,]\d+)?)[\s:|-]*)+`) numberRun = regexp.MustCompile(`[0-9]+(?:\\\.[0-9]+)*`) diff --git a/cmd/gbrgames/main.go b/cmd/gbrgames/main.go index 3510eb9..8c37d0f 100644 --- a/cmd/gbrgames/main.go +++ b/cmd/gbrgames/main.go @@ -7,6 +7,9 @@ // go run ./cmd/gbrgames inspect-image -game ID # read the image's facts and check the template against them // go run ./cmd/gbrgames first-boot -game ID # start the server once, untouched; record its log and files // go run ./cmd/gbrgames draft -name N -image I # have an AI model write a new game's template from those facts +// go run ./cmd/gbrgames research -name N # find how a game's server runs: variants, images, Steam app +// go run ./cmd/gbrgames build -name N -variant V # first boot, draft, and boot the draft until it works +// go run ./cmd/gbrgames worker # take research/build jobs from the panel (see BUILDER.md) package main import ( @@ -34,6 +37,12 @@ func main() { err = runFirstBoot(os.Args[2:]) case "draft": err = runDraft(os.Args[2:]) + case "research": + err = runResearch(os.Args[2:]) + case "build": + err = runBuild(os.Args[2:]) + case "worker": + err = runWorker(os.Args[2:]) default: usage() } @@ -44,6 +53,6 @@ func main() { } func usage() { - fmt.Fprintln(os.Stderr, "usage: gbrgames validate [-games DIR] | import [-games DIR] [-all] EXPORT.json | export [-repo R] [-commit SHA] [-out FILE] | boottest-payload [-out FILE] ID | inspect-image [-game ID] [-json] [-no-pull] [IMAGE] | first-boot [-game ID | -image IMAGE] [-env K=V] [-max D] [-quiet D] [-out DIR] | draft -name NAME -image IMAGE [-steam-app ID] [-first-boot REPORT] [-provider ollama|gemini] [-model M]") + fmt.Fprintln(os.Stderr, "usage: gbrgames validate [-games DIR] | import [-games DIR] [-all] EXPORT.json | export [-repo R] [-commit SHA] [-out FILE] | boottest-payload [-out FILE] ID | inspect-image [-game ID] [-json] [-no-pull] [IMAGE] | first-boot [-game ID | -image IMAGE] [-env K=V] [-max D] [-quiet D] [-out DIR] | draft -name NAME -image IMAGE [-steam-app ID] [-first-boot REPORT] [-provider ollama|gemini] [-model M] | research -name NAME [-notes N] [-out FILE] | build -name NAME -variant JSON|@FILE [-out FILE] | worker [-once]") os.Exit(2) } diff --git a/cmd/gbrgames/research.go b/cmd/gbrgames/research.go new file mode 100644 index 0000000..167fa7a --- /dev/null +++ b/cmd/gbrgames/research.go @@ -0,0 +1,398 @@ +package main + +import ( + "encoding/json" + "errors" + "flag" + "fmt" + "io" + "net/http" + "net/url" + "os" + "regexp" + "sort" + "strings" + "time" +) + +// research finds out how a game's dedicated server can be run before anything is drafted: it searches the +// web (a self-hosted SearXNG), reads the most useful pages (a reader that turns pages into text), and has +// the model list the server's variants (editions, crossplay builds, docker image or SteamCMD app). Every +// image and Steam app it names is then checked for real, so a variant shown to an admin exists. + +type variant struct { + ID string `json:"id"` + Title string `json:"title"` + Description string `json:"description"` + Recommended bool `json:"recommended"` + Images []string `json:"images"` + SteamAppID string `json:"steamAppId"` + StartupHint string `json:"startupHint"` + Ports []string `json:"ports"` + Notes string `json:"notes"` + Sources []string `json:"sources"` + // Filled by the checks, not the model + Verified []string `json:"verified,omitempty"` + Problems []string `json:"problems,omitempty"` +} + +type researchResult struct { + GameName string `json:"gameName"` + Supported bool `json:"supported"` + Reason string `json:"reason,omitempty"` + Message string `json:"message"` + Variants []variant `json:"variants"` + Sources []string `json:"sources"` +} + +var researchSchema = map[string]any{ + "type": "object", + "required": []string{"supported", "reason", "message", "variants"}, + "properties": map[string]any{ + "supported": map[string]any{"type": "boolean"}, + "reason": map[string]any{"enum": []string{"ok", "unreleased", "single_player", "no_dedicated_server", "sunset", "unknown"}}, + "message": map[string]any{"type": "string"}, + "variants": map[string]any{ + "type": "array", + "items": map[string]any{ + "type": "object", + "required": []string{"id", "title", "description", "recommended", "images", "steamAppId", "startupHint", "ports", "notes", "sources"}, + "properties": map[string]any{ + "id": map[string]any{"type": "string"}, + "title": map[string]any{"type": "string"}, + "description": map[string]any{"type": "string"}, + "recommended": map[string]any{"type": "boolean"}, + "images": map[string]any{"type": "array", "items": map[string]any{"type": "string"}}, + "steamAppId": map[string]any{"type": "string"}, + "startupHint": map[string]any{"type": "string"}, + "ports": map[string]any{"type": "array", "items": map[string]any{"type": "string"}}, + "notes": map[string]any{"type": "string"}, + "sources": map[string]any{"type": "array", "items": map[string]any{"type": "string"}}, + }, + }, + }, + }, +} + +func runResearch(args []string) error { + fs := flag.NewFlagSet("research", flag.ExitOnError) + gameName := fs.String("name", "", "the game's name, e.g. \"Valheim\"") + notes := fs.String("notes", "", "what the admin asked for (crossplay, a modded edition...)") + provider := fs.String("provider", "ollama", "ollama or gemini") + modelName := fs.String("model", "", "model name (default as for draft)") + pages := fs.Int("pages", 6, "how many search results to read") + out := fs.String("out", "", "write the result JSON here (default stdout)") + fs.Parse(args) + if *gameName == "" { + return errors.New("research needs -name") + } + m, err := newModel(*provider, *modelName) + if err != nil { + return err + } + res, err := research(m, *gameName, *notes, *pages) + if err != nil { + return err + } + raw, _ := json.MarshalIndent(res, "", " ") + if *out == "" { + fmt.Println(string(raw)) + return nil + } + return os.WriteFile(*out, raw, 0o644) +} + +func research(m model, gameName, notes string, pages int) (*researchResult, error) { + searx, reader := os.Getenv("SEARXNG_URL"), os.Getenv("READER_URL") + if searx == "" || reader == "" { + return nil, errors.New("set SEARXNG_URL (e.g. http://host:8080) and READER_URL (e.g. http://host:3000)") + } + queries := []string{ + gameName + " dedicated server docker image", + gameName + " dedicated server steamcmd app id linux", + gameName + " dedicated server setup ports launch options", + gameName + " crossplay console dedicated server", + } + var hits []searchHit + for _, q := range queries { + fmt.Printf("[research] searching: %s\n", q) + h, err := searxSearch(searx, q) + if err != nil { + fmt.Printf("[research] search failed: %v\n", err) + continue + } + hits = append(hits, h...) + } + if len(hits) == 0 { + return nil, errors.New("no search results (is SearXNG reachable?)") + } + picked := pickPages(hits, pages) + + var b strings.Builder + fmt.Fprintf(&b, "Game: **%s**\n", gameName) + if notes != "" { + fmt.Fprintf(&b, "The admin asked for: %s\n", notes) + } + b.WriteString("\n## Search results\n") + for _, h := range dedupeHits(hits) { + fmt.Fprintf(&b, "- %s — %s: %.200s\n", h.URL, h.Title, h.Content) + } + var sources []string + for _, h := range picked { + fmt.Printf("[research] reading %s\n", h.URL) + text, err := readPage(reader, h.URL) + if err != nil { + fmt.Printf("[research] could not read it: %v\n", err) + continue + } + sources = append(sources, h.URL) + fmt.Fprintf(&b, "\n## Page: %s\n%s\n", h.URL, text) + } + + messages := []chatMessage{ + {Role: "system", Content: researchPrompt}, + {Role: "user", Content: b.String()}, + } + fmt.Printf("[research] asking %s\n", m.name()) + answer, err := m.chat(messages, researchSchema) + if err != nil { + return nil, err + } + res := &researchResult{} + if err := json.Unmarshal([]byte(trimToJSON(answer)), res); err != nil { + return nil, fmt.Errorf("the model's answer is not valid JSON: %w", err) + } + res.GameName = gameName + res.Sources = sources + + verifyVariants(res) + return res, nil +} + +const researchPrompt = "You research how to host a game's dedicated server for GBR Hosting, a game server host. Servers run " + + "in Docker on Linux, as a non-root user, with a volume at /server. From the search results and pages given, list " + + "the ways this game's dedicated server can be run (variants): different editions (e.g. Java vs Bedrock, Evolved vs " + + "Ascended), crossplay builds, or notably different server images. For each variant give:\n" + + "- images: Docker Hub images that run this server, best first (prefer images that run as a non-root user, and the " + + "plain SteamCMD route cm2network/steamcmd when the game ships a Linux dedicated server on Steam). Only images that " + + "appear in the pages.\n" + + "- steamAppId: the Steam app id of the DEDICATED SERVER (not the game itself), if installed via SteamCMD; else \"\".\n" + + "- startupHint: the command that starts the server from its install folder /server, with the launch options " + + "for name, port, world and save location exactly as the pages document them; \"\" if unknown.\n" + + "- ports: every port players connect to, as \"port/proto\".\n" + + "- notes: anything a template author must know (Windows-only server needing Proton, crossplay flags, mods).\n" + + "- sources: the page URLs that support it.\n" + + "Mark one variant recommended. Use only what the pages say or what you know for certain; leave fields empty " + + "instead of guessing. If no dedicated server can be hosted (single-player only, unreleased, official servers " + + "only, shut down), set supported false, the matching reason, no variants, and explain in message. Otherwise " + + "reason \"ok\" and a one-sentence message." + +type searchHit struct { + URL string `json:"url"` + Title string `json:"title"` + Content string `json:"content"` +} + +func searxSearch(base, query string) ([]searchHit, error) { + u := strings.TrimRight(base, "/") + "/search?format=json&q=" + url.QueryEscape(query) + raw, err := httpGet(u, 30*time.Second) + if err != nil { + return nil, err + } + var resp struct { + Results []searchHit `json:"results"` + } + if err := json.Unmarshal(raw, &resp); err != nil { + return nil, fmt.Errorf("unreadable search answer: %.200s", raw) + } + if len(resp.Results) > 8 { + resp.Results = resp.Results[:8] + } + return resp.Results, nil +} + +// Pages that hold the facts a template needs, read first +var usefulPage = regexp.MustCompile(`(?i)hub\.docker\.com|github\.com|steamdb\.info|developer\.valvesoftware|wiki|fandom|reddit\.com/r/\w*(server|admin)`) + +// pickPages reads one page per site, docs and image pages first, so a handful of reads covers the most ground. +func pickPages(hits []searchHit, n int) []searchHit { + hits = dedupeHits(hits) + sort.SliceStable(hits, func(i, j int) bool { + return usefulPage.MatchString(hits[i].URL) && !usefulPage.MatchString(hits[j].URL) + }) + seen := map[string]bool{} + var out []searchHit + for _, h := range hits { + u, err := url.Parse(h.URL) + if err != nil || seen[u.Host] { + continue + } + seen[u.Host] = true + out = append(out, h) + if len(out) == n { + break + } + } + return out +} + +func dedupeHits(hits []searchHit) []searchHit { + seen := map[string]bool{} + var out []searchHit + for _, h := range hits { + if h.URL == "" || seen[h.URL] { + continue + } + seen[h.URL] = true + out = append(out, h) + } + return out +} + +func readPage(reader, pageURL string) (string, error) { + raw, err := httpGet(strings.TrimRight(reader, "/")+"/"+pageURL, 60*time.Second) + if err != nil { + return "", err + } + text := string(raw) + if len(text) > 9000 { + text = text[:9000] + "\n...(truncated)" + } + return text, nil +} + +func httpGet(u string, timeout time.Duration) ([]byte, error) { + client := &http.Client{Timeout: timeout} + resp, err := client.Get(u) + if err != nil { + return nil, err + } + defer resp.Body.Close() + raw, err := io.ReadAll(io.LimitReader(resp.Body, 4<<20)) + if err != nil { + return nil, err + } + if resp.StatusCode >= 300 { + return nil, fmt.Errorf("HTTP %d: %.200s", resp.StatusCode, raw) + } + return raw, nil +} + +// trimToJSON cuts prose or code fences some models put around a JSON answer. +func trimToJSON(answer string) string { + answer = strings.TrimSpace(answer) + if i := strings.Index(answer, "{"); i > 0 { + answer = answer[i:] + } + if i := strings.LastIndex(answer, "}"); i >= 0 { + answer = answer[:i+1] + } + return answer +} + +// verifyVariants checks every image and Steam app the model named: images must exist in their registry, +// Steam apps must exist and be readable anonymously (what SteamCMD installs need). What fails is dropped +// and noted; a variant left with nothing runnable can't be recommended. +func verifyVariants(res *researchResult) { + var apps []string + for _, v := range res.Variants { + if v.SteamAppID != "" { + apps = append(apps, v.SteamAppID) + } + } + names := steamAppNames(apps) + anyRunnable := false + for i := range res.Variants { + v := &res.Variants[i] + var images []string + for _, img := range v.Images { + img = strings.TrimSpace(img) + if img == "" { + continue + } + if out, err := runCmd("docker", "manifest", "inspect", img); err != nil { + v.Problems = append(v.Problems, fmt.Sprintf("image %s not found: %.120s", img, out)) + continue + } + v.Verified = append(v.Verified, "image "+img+" exists") + images = append(images, img) + } + v.Images = images + if v.SteamAppID != "" { + if name, ok := names[v.SteamAppID]; ok { + v.Verified = append(v.Verified, fmt.Sprintf("Steam app %s is %q", v.SteamAppID, name)) + if !hasImage(v.Images, "cm2network/steamcmd") { + v.Images = append(v.Images, "cm2network/steamcmd") + } + } else { + v.Problems = append(v.Problems, "Steam app "+v.SteamAppID+" not found for anonymous SteamCMD") + v.SteamAppID = "" + } + } + if len(v.Images) == 0 { + v.Problems = append(v.Problems, "no image or Steam app that could be checked; it can't be built") + v.Recommended = false + } else { + anyRunnable = true + } + } + if !anyRunnable && res.Supported { + res.Supported = false + res.Reason = "unknown" + res.Message = "No variant has a Docker image or Steam app that could be confirmed. " + res.Message + } + // Exactly one recommendation, among the runnable ones + rec := -1 + for i, v := range res.Variants { + if v.Recommended && rec < 0 && len(v.Images) > 0 { + rec = i + } + res.Variants[i].Recommended = false + } + if rec < 0 { + for i, v := range res.Variants { + if len(v.Images) > 0 { + rec = i + break + } + } + } + if rec >= 0 { + res.Variants[rec].Recommended = true + } +} + +func hasImage(images []string, name string) bool { + for _, i := range images { + if i == name || strings.HasPrefix(i, name+":") { + return true + } + } + return false +} + +var appName = regexp.MustCompile(`"common"\s*\{[^}]*?"name"\s+"([^"]+)"`) + +// steamAppNames asks SteamCMD (anonymously, in its official image) for each app's info: an app that +// anonymous SteamCMD can't see can't be installed by the node-agent either. +func steamAppNames(apps []string) map[string]string { + names := map[string]string{} + if len(apps) == 0 { + return names + } + for _, app := range apps { + if !regexp.MustCompile(`^\d+$`).MatchString(app) { + continue + } + out, err := runCmd("docker", "run", "--rm", "--network", bootTestNetwork, "cm2network/steamcmd", "bash", "-c", + "/home/steam/steamcmd/steamcmd.sh +login anonymous +app_info_update 1 +app_info_print "+app+" +quit") + if err != nil { + fmt.Printf("[research] steamcmd for app %s: %v\n", app, err) + continue + } + if m := appName.FindStringSubmatch(out); m != nil { + names[app] = m[1] + } + } + return names +} diff --git a/cmd/gbrgames/research_test.go b/cmd/gbrgames/research_test.go new file mode 100644 index 0000000..28c1a5c --- /dev/null +++ b/cmd/gbrgames/research_test.go @@ -0,0 +1,41 @@ +package main + +import "testing" + +func TestPickPagesPrefersDocsAndOnePerSite(t *testing.T) { + hits := []searchHit{ + {URL: "https://blog.example.com/valheim-server"}, + {URL: "https://github.com/a/valheim-docker"}, + {URL: "https://github.com/b/other"}, + {URL: "https://hub.docker.com/r/x/valheim"}, + {URL: "https://blog.example.com/valheim-server"}, + } + got := pickPages(hits, 3) + if len(got) != 3 { + t.Fatalf("got %d pages", len(got)) + } + if got[0].URL != "https://github.com/a/valheim-docker" || got[1].URL != "https://hub.docker.com/r/x/valheim" || got[2].URL != "https://blog.example.com/valheim-server" { + t.Errorf("order: %v", got) + } +} + +func TestTrimToJSON(t *testing.T) { + if got := trimToJSON("Here you go:\n```json\n{\"a\": 1}\n```"); got != `{"a": 1}` { + t.Errorf("got %q", got) + } +} + +func TestAppNameFromAppInfo(t *testing.T) { + out := `"896660" +{ + "common" + { + "name" "Valheim Dedicated Server" + "type" "Tool" + } +}` + m := appName.FindStringSubmatch(out) + if m == nil || m[1] != "Valheim Dedicated Server" { + t.Errorf("got %v", m) + } +} diff --git a/cmd/gbrgames/validate.go b/cmd/gbrgames/validate.go index 394fce3..225b13e 100644 --- a/cmd/gbrgames/validate.go +++ b/cmd/gbrgames/validate.go @@ -168,7 +168,8 @@ func validateGame(dir, folder string, gameSchema, testSchema *jsonschema.Schema) add("test.yaml config.%s is not a key of this game", key) } if sensitiveKey.MatchString(key) { - add("test.yaml config.%s looks like a credential; map it under 'secrets' to a Gitea secret instead", key) + add("test.yaml config.%s looks like a credential; remove it (boot tests fill password settings with a throwaway "+ + "value), or, only if the server can't start without a real one, map it under 'secrets' to a CI secret", key) } } for _, key := range sortedKeys(secrets) { diff --git a/cmd/gbrgames/worker.go b/cmd/gbrgames/worker.go new file mode 100644 index 0000000..4e54e6d --- /dev/null +++ b/cmd/gbrgames/worker.go @@ -0,0 +1,362 @@ +package main + +import ( + "bufio" + "bytes" + "encoding/json" + "errors" + "flag" + "fmt" + "io" + "net/http" + "os" + "os/exec" + "path/filepath" + "strings" + "sync" + "time" +) + +// worker runs the AI builder. It asks the panel for queued jobs (it only connects out, +// like node-agents do), runs research or build in a fresh clone of this repo, streams the output to the +// panel as the job's log, and finishes a build with a pull request opened as the bot account. CI then +// validates and boot-tests that pull request like any other. +// +// Panel API (headers X-Builder-Token and X-Builder-Worker): +// +// POST /api/v1/builder/jobs/claim {worker} -> 200 builderJob | 204 nothing queued +// POST /api/v1/builder/jobs/{id}/progress {stage, lines} -> 200 {cancelled} +// POST /api/v1/builder/jobs/{id}/result {research | build+prUrl | error} + +type builderJob struct { + ID string `json:"id"` + Kind string `json:"kind"` // "research" or "build" + GameName string `json:"gameName"` + Notes string `json:"notes"` + Variant json.RawMessage `json:"variant"` +} + +type jobResult struct { + Research *researchResult `json:"research,omitempty"` + Build *buildResult `json:"build,omitempty"` + PRURL string `json:"prUrl,omitempty"` + Error string `json:"error,omitempty"` +} + +type workerConfig struct { + panel, token string + gitea, repo, botToken string + workDir, name string + poll time.Duration + provider, model string +} + +func runWorker(args []string) error { + fs := flag.NewFlagSet("worker", flag.ExitOnError) + once := fs.Bool("once", false, "handle at most one job, then exit") + fs.Parse(args) + host, _ := os.Hostname() + c := workerConfig{ + panel: strings.TrimRight(os.Getenv("PANEL_URL"), "/"), token: os.Getenv("BUILDER_TOKEN"), + gitea: strings.TrimRight(envOr("GITEA_URL", "https://www.gitea.gbreed.com"), "/"), repo: envOr("GAMES_REPO", "gbreed-public/gbr-games"), + botToken: os.Getenv("GITEA_TOKEN"), workDir: envOr("WORK_DIR", "/work"), name: envOr("WORKER_NAME", host), + provider: envOr("MODEL_PROVIDER", "ollama"), model: os.Getenv("MODEL"), + } + var err error + if c.poll, err = time.ParseDuration(envOr("POLL_INTERVAL", "20s")); err != nil { + return err + } + for _, v := range []struct{ name, value string }{{"PANEL_URL", c.panel}, {"BUILDER_TOKEN", c.token}, {"GITEA_TOKEN", c.botToken}} { + if v.value == "" { + return fmt.Errorf("set %s", v.name) + } + } + fmt.Printf("[worker] %s polling %s every %s\n", c.name, c.panel, c.poll) + for { + job, err := c.claim() + if err != nil { + fmt.Printf("[worker] claim: %v\n", err) + } + if job != nil { + fmt.Printf("[worker] job %s: %s %q\n", job.ID, job.Kind, job.GameName) + res := c.handle(job) + if err := c.post("/jobs/"+job.ID+"/result", res, nil); err != nil { + fmt.Printf("[worker] posting the result of %s: %v\n", job.ID, err) + } + if *once { + return nil + } + continue + } + if *once { + return nil + } + time.Sleep(c.poll) + } +} + +func envOr(name, def string) string { + if v := os.Getenv(name); v != "" { + return v + } + return def +} + +func (c *workerConfig) claim() (*builderJob, error) { + var job builderJob + status, err := c.request("/jobs/claim", map[string]string{"worker": c.name}, &job) + if err != nil || status == http.StatusNoContent { + return nil, err + } + return &job, nil +} + +func (c *workerConfig) post(p string, body, out any) error { + _, err := c.request(p, body, out) + return err +} + +func (c *workerConfig) request(p string, body, out any) (int, error) { + payload, err := json.Marshal(body) + if err != nil { + return 0, err + } + req, err := http.NewRequest(http.MethodPost, c.panel+"/api/v1/builder"+p, bytes.NewReader(payload)) + if err != nil { + return 0, err + } + req.Header.Set("Content-Type", "application/json") + req.Header.Set("X-Builder-Token", c.token) + req.Header.Set("X-Builder-Worker", c.name) + resp, err := (&http.Client{Timeout: 30 * time.Second}).Do(req) + if err != nil { + return 0, err + } + defer resp.Body.Close() + raw, _ := io.ReadAll(resp.Body) + if resp.StatusCode >= 300 { + return resp.StatusCode, fmt.Errorf("HTTP %d: %.200s", resp.StatusCode, raw) + } + if out != nil && len(bytes.TrimSpace(raw)) > 0 { + if err := json.Unmarshal(raw, out); err != nil { + return resp.StatusCode, err + } + } + return resp.StatusCode, nil +} + +// handle runs one job in its own clone and returns what to report; it never panics the worker. +func (c *workerConfig) handle(job *builderJob) *jobResult { + dir := filepath.Join(c.workDir, "job-"+job.ID) + os.RemoveAll(dir) + defer os.RemoveAll(dir) + if out, err := runCmd("git", "clone", "-q", "--depth", "1", c.gitea+"/"+c.repo+".git", dir); err != nil { + return &jobResult{Error: "git clone: " + out} + } + self, err := os.Executable() + if err != nil { + return &jobResult{Error: err.Error()} + } + resultFile := filepath.Join(dir, "job-result.json") + modelArgs := []string{"-provider", c.provider} + if c.model != "" { + modelArgs = append(modelArgs, "-model", c.model) + } + + switch job.Kind { + case "research": + args := append([]string{"research", "-name", job.GameName, "-notes", job.Notes, "-out", resultFile}, modelArgs...) + if err := c.runStreamed(job.ID, dir, self, args, "RESEARCH"); err != nil { + return &jobResult{Error: err.Error()} + } + res := &researchResult{} + if err := readJSON(resultFile, res); err != nil { + return &jobResult{Error: err.Error()} + } + return &jobResult{Research: res} + + case "build": + variantFile := filepath.Join(dir, "job-variant.json") + if err := os.WriteFile(variantFile, job.Variant, 0o644); err != nil { + return &jobResult{Error: err.Error()} + } + args := append([]string{"build", "-name", job.GameName, "-notes", job.Notes, "-variant", "@" + variantFile, "-out", resultFile}, modelArgs...) + runErr := c.runStreamed(job.ID, dir, self, args, "FIRST_BOOT") + res := &buildResult{} + if err := readJSON(resultFile, res); err != nil { + msg := "the build produced no result" + if runErr != nil { + msg = runErr.Error() + } + return &jobResult{Error: msg} + } + // A draft that validates but didn't boot still goes up as a draft PR: a person can finish it, and + // CI shows the same boot failure + if _, err := os.Stat(filepath.Join(dir, res.Dir, "game.yaml")); err != nil { + return &jobResult{Build: res, Error: "no template was written: " + strings.Join(res.Problems, "; ")} + } + c.progress(job.ID, "OPENING_PR", []string{"[worker] opening the pull request"}) + pr, err := c.openPR(dir, job, res) + if err != nil { + return &jobResult{Build: res, Error: "opening the pull request: " + err.Error()} + } + return &jobResult{Build: res, PRURL: pr} + } + return &jobResult{Error: "unknown job kind " + job.Kind} +} + +// Output prefixes that mark a new stage of a build, shown in the panel's progress bar +var stagePrefixes = []struct{ prefix, stage string }{ + {"[build] first boot", "FIRST_BOOT"}, + {"[first-boot]", ""}, + {"[draft]", "DRAFTING"}, + {"[build] boot ", "BOOT_TEST"}, +} + +// runStreamed runs a gbrgames command and sends its output to the panel every few seconds (which also +// tells the panel the worker is alive). The panel can answer "cancelled" to stop the job. +func (c *workerConfig) runStreamed(jobID, dir, self string, args []string, stage string) error { + cmd := exec.Command(self, args...) + cmd.Dir = dir + pr, pw := io.Pipe() + cmd.Stdout, cmd.Stderr = pw, pw + if err := cmd.Start(); err != nil { + return err + } + var mu sync.Mutex + var pending []string + var lastErr string + done := make(chan struct{}) + go func() { + sc := bufio.NewScanner(pr) + sc.Buffer(make([]byte, 1024*1024), 1024*1024) + for sc.Scan() { + line := strings.TrimRight(ansi.ReplaceAllString(sc.Text(), ""), "\r ") + fmt.Println(line) + mu.Lock() + pending = append(pending, line) + for _, s := range stagePrefixes { + if s.stage != "" && strings.HasPrefix(line, s.prefix) { + stage = s.stage + } + } + if strings.HasPrefix(line, "error: ") { + lastErr = strings.TrimPrefix(line, "error: ") + } + mu.Unlock() + } + close(done) + }() + wait := make(chan error, 1) + go func() { + err := cmd.Wait() + pw.Close() + <-done + wait <- err + }() + + tick := time.NewTicker(5 * time.Second) + defer tick.Stop() + flush := func() bool { + mu.Lock() + lines, st := pending, stage + pending = nil + mu.Unlock() + return c.progress(jobID, st, lines) + } + for { + select { + case err := <-wait: + flush() + if err != nil { + if lastErr != "" { + return errors.New(lastErr) + } + return err + } + return nil + case <-tick.C: + if cancelled := flush(); cancelled { + cmd.Process.Kill() + <-wait + return errors.New("cancelled from the panel") + } + } + } +} + +// progress sends log lines (possibly none, as a heartbeat) and reports whether the job was cancelled. +func (c *workerConfig) progress(jobID, stage string, lines []string) bool { + var resp struct { + Cancelled bool `json:"cancelled"` + } + if lines == nil { + lines = []string{} + } + if err := c.post("/jobs/"+jobID+"/progress", map[string]any{"stage": stage, "lines": lines}, &resp); err != nil { + fmt.Printf("[worker] progress: %v\n", err) + } + return resp.Cancelled +} + +// openPR commits the new game folder to its own branch as the bot and opens a pull request to main. +func (c *workerConfig) openPR(dir string, job *builderJob, res *buildResult) (string, error) { + short := job.ID + if len(short) > 8 { + short = short[:8] + } + branch := "draft/" + res.GameID + "-" + short + auth := "http.extraHeader=Authorization: token " + c.botToken + steps := []struct { + name string + args []string + }{ + {"checkout", []string{"checkout", "-q", "-b", branch}}, + {"add", []string{"add", filepath.ToSlash(res.Dir)}}, + {"commit", []string{"-c", "user.name=gbr-bot", "-c", "user.email=gbr-bot@users.noreply.gitea.gbreed.com", "commit", "-q", "-m", + fmt.Sprintf("feat(games): draft %s\n\nDrafted and boot-tested by the GBR builder (job %s).", job.GameName, job.ID)}}, + {"push", []string{"-c", auth, "push", "-q", "origin", branch}}, + } + for _, s := range steps { + cmd := exec.Command("git", s.args...) + cmd.Dir = dir + if out, err := cmd.CombinedOutput(); err != nil { + // The push command holds the token: name the step only, and mask it in git's output + return "", fmt.Errorf("git %s: %s", s.name, strings.ReplaceAll(string(out), c.botToken, "***")) + } + } + title := "feat(games): add " + job.GameName + if !res.Passed { + title = "WIP: " + title + " (did not boot yet)" + } + body := map[string]any{"title": title, "head": branch, "base": "main", "body": res.Summary} + payload, _ := json.Marshal(body) + req, err := http.NewRequest(http.MethodPost, c.gitea+"/api/v1/repos/"+c.repo+"/pulls", bytes.NewReader(payload)) + if err != nil { + return "", err + } + req.Header.Set("Content-Type", "application/json") + req.Header.Set("Authorization", "token "+c.botToken) + resp, err := (&http.Client{Timeout: 30 * time.Second}).Do(req) + if err != nil { + return "", err + } + defer resp.Body.Close() + var pr struct { + HTMLURL string `json:"html_url"` + Message string `json:"message"` + } + json.NewDecoder(resp.Body).Decode(&pr) + if resp.StatusCode >= 300 { + return "", fmt.Errorf("HTTP %d: %s", resp.StatusCode, pr.Message) + } + return pr.HTMLURL, nil +} + +func readJSON(p string, out any) error { + raw, err := os.ReadFile(p) + if err != nil { + return err + } + return json.Unmarshal(raw, out) +}