feat(builder): diagnose a failed boot before changing the template
A failed boot went straight back to the model as "fix it", so it guessed: one draft raised its memory three rounds in a row while the real cause, an empty setting the server can't run without, stayed untouched. Now a failed boot is investigated first: - Evidence: the server's distinct error lines (timestamps and repeats removed), and a web search on the most telling one, with the top answer read through the reader. - Diagnosis: the model states the cause, the evidence for it, its confidence and the exact fix. If unsure, it asks for up to 3 checks: a search, a page, the files the server writes (a first boot of the draft), or a boot test with settings or the startup command changed. Results go back to it, two rounds at most. - Fix: the model applies exactly the diagnosed fix and nothing unrelated. - Attempt log: each cause, fix and outcome. A fix that changed nothing rules its cause out; if the diagnosis returns to a ruled-out cause, the build stops instead of guessing, and the PR lists what the builder found. Boot rounds go up to 5, since rounds now end on a ruled-out cause rather than a fixed count. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
1 parent
4874868ce6
commit
487dfbdf68
3 files changed
+542
-3
No files matched your search
@@ -0,0 +1,128 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"testing"
|
||||
)
|
||||
|
||||
// fakeModel answers with the given replies in order and keeps what it was asked.
|
||||
type fakeModel struct {
|
||||
replies []string
|
||||
asked [][]chatMessage
|
||||
}
|
||||
|
||||
func (f *fakeModel) name() string { return "fake" }
|
||||
func (f *fakeModel) chat(m []chatMessage, _ map[string]any) (string, error) {
|
||||
f.asked = append(f.asked, append([]chatMessage{}, m...))
|
||||
r := f.replies[0]
|
||||
f.replies = f.replies[1:]
|
||||
return r, nil
|
||||
}
|
||||
|
||||
func TestDiagnoseRunsRequestedChecksBeforeAnswering(t *testing.T) {
|
||||
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
if strings.HasPrefix(r.URL.Path, "/search") {
|
||||
w.Write([]byte(`{"results":[{"url":"https://example.com/fix","title":"Server stops at start","content":"set a password"}]}`))
|
||||
return
|
||||
}
|
||||
w.Write([]byte("The server refuses to run without a join password."))
|
||||
}))
|
||||
defer srv.Close()
|
||||
games := t.TempDir()
|
||||
os.MkdirAll(filepath.Join(games, "example"), 0o755)
|
||||
os.WriteFile(filepath.Join(games, "example", "game.yaml"), []byte("id: example\nname: Example\n"), 0o644)
|
||||
|
||||
m := &fakeModel{replies: []string{
|
||||
`{"cause":"unsure","evidence":"no error line","confidence":"low","fix":"","checks":[{"type":"search","query":"example server stops at start"}]}`,
|
||||
`{"cause":"the server needs a join password","evidence":"the search result says so","confidence":"high","fix":"give JoinPassword x-generate","checks":[]}`,
|
||||
}}
|
||||
iv := &investigator{m: m, gameName: "Example", gamesDir: games, id: "example", searx: srv.URL, reader: srv.URL}
|
||||
d, err := iv.diagnose("FAILED: server crashed and was restarted 1 time(s)\nError: could not start")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if d.Confidence != "high" || !strings.Contains(d.Fix, "x-generate") {
|
||||
t.Errorf("diagnosis: %+v", d)
|
||||
}
|
||||
second := m.asked[1]
|
||||
last := second[len(second)-1].Content
|
||||
if !strings.Contains(last, "Results of your checks") || !strings.Contains(last, "https://example.com/fix") ||
|
||||
!strings.Contains(last, "refuses to run without a join password") {
|
||||
t.Errorf("check results not sent back:\n%s", last)
|
||||
}
|
||||
// The first question already carried the web evidence for the error line
|
||||
if first := m.asked[0][1].Content; !strings.Contains(first, "What the web says") {
|
||||
t.Errorf("no web evidence in:\n%s", first)
|
||||
}
|
||||
}
|
||||
|
||||
func TestAttemptLogRulesOutCausesThatChangedNothing(t *testing.T) {
|
||||
iv := &investigator{}
|
||||
iv.record(&diagnosis{Cause: "Not enough memory.", Fix: "requiredMemoryMb 8192"})
|
||||
iv.outcome(false, "FAILED: server crashed", "FAILED: server crashed")
|
||||
if !strings.HasPrefix(iv.log[0].Outcome, "unchanged") {
|
||||
t.Errorf("outcome %q", iv.log[0].Outcome)
|
||||
}
|
||||
if !iv.ruledOut(&diagnosis{Cause: "not enough memory"}) {
|
||||
t.Error("the same cause again should be ruled out")
|
||||
}
|
||||
if iv.ruledOut(&diagnosis{Cause: "the join password is empty"}) {
|
||||
t.Error("a new cause is not ruled out")
|
||||
}
|
||||
if !strings.Contains(iv.logText(), "ruled its cause out") {
|
||||
t.Error("log text should tell the model")
|
||||
}
|
||||
iv.record(&diagnosis{Cause: "empty join password", Fix: "x-generate"})
|
||||
iv.outcome(true, "FAILED: server crashed", "")
|
||||
if !strings.HasPrefix(iv.log[1].Outcome, "fixed") || iv.ruledOut(&diagnosis{Cause: "empty join password"}) {
|
||||
t.Errorf("fixed cause: %+v", iv.log[1])
|
||||
}
|
||||
}
|
||||
|
||||
func TestDistinctErrorsDropsTimestampsAndRepeats(t *testing.T) {
|
||||
out := "[boot-test] FAILED: container exited\n" +
|
||||
"10/26/2026 01:12:02: Error: failed to bind port 30000\n" +
|
||||
"10/26/2026 01:12:03: Error: failed to bind port 30001\n" +
|
||||
"all fine here\n" +
|
||||
"[12:00:01] Fatal: missing config file"
|
||||
got := distinctErrors(out, 5)
|
||||
if len(got) != 2 || got[0] != "Error: failed to bind port 30000" || got[1] != "Fatal: missing config file" {
|
||||
t.Errorf("got %q", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestCopyTemplateWithAppliesTheChange(t *testing.T) {
|
||||
src, dst := filepath.Join(t.TempDir(), "example"), filepath.Join(t.TempDir(), "example")
|
||||
os.MkdirAll(src, 0o755)
|
||||
os.WriteFile(filepath.Join(src, "game.yaml"), []byte("id: example\nname: Example\ndefaultStartupCommand: ./server\ndefaultConfig:\n JoinPassword: \"\"\n"), 0o644)
|
||||
os.WriteFile(filepath.Join(src, "test.yaml"), []byte("timeoutSec: 600\n"), 0o644)
|
||||
if err := copyTemplateWith(src, dst, checkRequest{Config: map[string]string{"JoinPassword": "secret1"}, Startup: "./server -verbose"}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
g, err := loadMap(filepath.Join(dst, "game.yaml"))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if g["defaultStartupCommand"] != "./server -verbose" || g["defaultConfig"].(map[string]any)["JoinPassword"] != "secret1" {
|
||||
t.Errorf("got %v", g)
|
||||
}
|
||||
if _, err := os.Stat(filepath.Join(dst, "test.yaml")); err != nil {
|
||||
t.Error("test.yaml not copied")
|
||||
}
|
||||
if got := describeChange(checkRequest{Config: map[string]string{"JoinPassword": "secret1"}}); got != `JoinPassword="secret1"` {
|
||||
t.Errorf("describe: %s", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestFixRequestCarriesTheDiagnosis(t *testing.T) {
|
||||
got := strings.Join(fixRequest("FAILED: server crashed\nlog...", &diagnosis{Cause: "c", Evidence: "e", Fix: "f"}), "")
|
||||
for _, want := range []string{"FAILED: server crashed", "- cause: c", "- fix: f", "Apply exactly this fix"} {
|
||||
if !strings.Contains(got, want) {
|
||||
t.Errorf("missing %q in %s", want, got)
|
||||
}
|
||||
}
|
||||
}
|
||||
Reference in new issue
Block a user