Files
gbr-games/cmd/gbrgames/diagnose_test.go
T
gregandClaude Opus 5.5 487dfbdf68
Validate game templates / validate (push) Successful in 21s
Validate game templates / boot-test (push) Skipped
Validate game templates / publish (push) Skipped
feat(builder): diagnose a failed boot before changing the template
A failed boot went straight back to the model as "fix it", so it guessed:
one draft raised its memory three rounds in a row while the real cause, an
empty setting the server can't run without, stayed untouched.

Now a failed boot is investigated first:
- Evidence: the server's distinct error lines (timestamps and repeats
  removed), and a web search on the most telling one, with the top answer
  read through the reader.
- Diagnosis: the model states the cause, the evidence for it, its
  confidence and the exact fix. If unsure, it asks for up to 3 checks: a
  search, a page, the files the server writes (a first boot of the draft),
  or a boot test with settings or the startup command changed. Results go
  back to it, two rounds at most.
- Fix: the model applies exactly the diagnosed fix and nothing unrelated.
- Attempt log: each cause, fix and outcome. A fix that changed nothing
  rules its cause out; if the diagnosis returns to a ruled-out cause, the
  build stops instead of guessing, and the PR lists what the builder found.

Boot rounds go up to 5, since rounds now end on a ruled-out cause rather
than a fixed count.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-10-25 23:15:33 -04:00

129 lines
5.1 KiB
Go

package main
import (
"net/http"
"net/http/httptest"
"os"
"path/filepath"
"strings"
"testing"
)
// fakeModel answers with the given replies in order and keeps what it was asked.
type fakeModel struct {
replies []string
asked [][]chatMessage
}
func (f *fakeModel) name() string { return "fake" }
func (f *fakeModel) chat(m []chatMessage, _ map[string]any) (string, error) {
f.asked = append(f.asked, append([]chatMessage{}, m...))
r := f.replies[0]
f.replies = f.replies[1:]
return r, nil
}
func TestDiagnoseRunsRequestedChecksBeforeAnswering(t *testing.T) {
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
if strings.HasPrefix(r.URL.Path, "/search") {
w.Write([]byte(`{"results":[{"url":"https://example.com/fix","title":"Server stops at start","content":"set a password"}]}`))
return
}
w.Write([]byte("The server refuses to run without a join password."))
}))
defer srv.Close()
games := t.TempDir()
os.MkdirAll(filepath.Join(games, "example"), 0o755)
os.WriteFile(filepath.Join(games, "example", "game.yaml"), []byte("id: example\nname: Example\n"), 0o644)
m := &fakeModel{replies: []string{
`{"cause":"unsure","evidence":"no error line","confidence":"low","fix":"","checks":[{"type":"search","query":"example server stops at start"}]}`,
`{"cause":"the server needs a join password","evidence":"the search result says so","confidence":"high","fix":"give JoinPassword x-generate","checks":[]}`,
}}
iv := &investigator{m: m, gameName: "Example", gamesDir: games, id: "example", searx: srv.URL, reader: srv.URL}
d, err := iv.diagnose("FAILED: server crashed and was restarted 1 time(s)\nError: could not start")
if err != nil {
t.Fatal(err)
}
if d.Confidence != "high" || !strings.Contains(d.Fix, "x-generate") {
t.Errorf("diagnosis: %+v", d)
}
second := m.asked[1]
last := second[len(second)-1].Content
if !strings.Contains(last, "Results of your checks") || !strings.Contains(last, "https://example.com/fix") ||
!strings.Contains(last, "refuses to run without a join password") {
t.Errorf("check results not sent back:\n%s", last)
}
// The first question already carried the web evidence for the error line
if first := m.asked[0][1].Content; !strings.Contains(first, "What the web says") {
t.Errorf("no web evidence in:\n%s", first)
}
}
func TestAttemptLogRulesOutCausesThatChangedNothing(t *testing.T) {
iv := &investigator{}
iv.record(&diagnosis{Cause: "Not enough memory.", Fix: "requiredMemoryMb 8192"})
iv.outcome(false, "FAILED: server crashed", "FAILED: server crashed")
if !strings.HasPrefix(iv.log[0].Outcome, "unchanged") {
t.Errorf("outcome %q", iv.log[0].Outcome)
}
if !iv.ruledOut(&diagnosis{Cause: "not enough memory"}) {
t.Error("the same cause again should be ruled out")
}
if iv.ruledOut(&diagnosis{Cause: "the join password is empty"}) {
t.Error("a new cause is not ruled out")
}
if !strings.Contains(iv.logText(), "ruled its cause out") {
t.Error("log text should tell the model")
}
iv.record(&diagnosis{Cause: "empty join password", Fix: "x-generate"})
iv.outcome(true, "FAILED: server crashed", "")
if !strings.HasPrefix(iv.log[1].Outcome, "fixed") || iv.ruledOut(&diagnosis{Cause: "empty join password"}) {
t.Errorf("fixed cause: %+v", iv.log[1])
}
}
func TestDistinctErrorsDropsTimestampsAndRepeats(t *testing.T) {
out := "[boot-test] FAILED: container exited\n" +
"10/26/2026 01:12:02: Error: failed to bind port 30000\n" +
"10/26/2026 01:12:03: Error: failed to bind port 30001\n" +
"all fine here\n" +
"[12:00:01] Fatal: missing config file"
got := distinctErrors(out, 5)
if len(got) != 2 || got[0] != "Error: failed to bind port 30000" || got[1] != "Fatal: missing config file" {
t.Errorf("got %q", got)
}
}
func TestCopyTemplateWithAppliesTheChange(t *testing.T) {
src, dst := filepath.Join(t.TempDir(), "example"), filepath.Join(t.TempDir(), "example")
os.MkdirAll(src, 0o755)
os.WriteFile(filepath.Join(src, "game.yaml"), []byte("id: example\nname: Example\ndefaultStartupCommand: ./server\ndefaultConfig:\n JoinPassword: \"\"\n"), 0o644)
os.WriteFile(filepath.Join(src, "test.yaml"), []byte("timeoutSec: 600\n"), 0o644)
if err := copyTemplateWith(src, dst, checkRequest{Config: map[string]string{"JoinPassword": "secret1"}, Startup: "./server -verbose"}); err != nil {
t.Fatal(err)
}
g, err := loadMap(filepath.Join(dst, "game.yaml"))
if err != nil {
t.Fatal(err)
}
if g["defaultStartupCommand"] != "./server -verbose" || g["defaultConfig"].(map[string]any)["JoinPassword"] != "secret1" {
t.Errorf("got %v", g)
}
if _, err := os.Stat(filepath.Join(dst, "test.yaml")); err != nil {
t.Error("test.yaml not copied")
}
if got := describeChange(checkRequest{Config: map[string]string{"JoinPassword": "secret1"}}); got != `JoinPassword="secret1"` {
t.Errorf("describe: %s", got)
}
}
func TestFixRequestCarriesTheDiagnosis(t *testing.T) {
got := strings.Join(fixRequest("FAILED: server crashed\nlog...", &diagnosis{Cause: "c", Evidence: "e", Fix: "f"}), "")
for _, want := range []string{"FAILED: server crashed", "- cause: c", "- fix: f", "Apply exactly this fix"} {
if !strings.Contains(got, want) {
t.Errorf("missing %q in %s", want, got)
}
}
}