feat(builder): diagnose a failed boot before changing the template
A failed boot went straight back to the model as "fix it", so it guessed: one draft raised its memory three rounds in a row while the real cause, an empty setting the server can't run without, stayed untouched. Now a failed boot is investigated first: - Evidence: the server's distinct error lines (timestamps and repeats removed), and a web search on the most telling one, with the top answer read through the reader. - Diagnosis: the model states the cause, the evidence for it, its confidence and the exact fix. If unsure, it asks for up to 3 checks: a search, a page, the files the server writes (a first boot of the draft), or a boot test with settings or the startup command changed. Results go back to it, two rounds at most. - Fix: the model applies exactly the diagnosed fix and nothing unrelated. - Attempt log: each cause, fix and outcome. A fix that changed nothing rules its cause out; if the diagnosis returns to a ruled-out cause, the build stops instead of guessing, and the PR lists what the builder found. Boot rounds go up to 5, since rounds now end on a ruled-out cause rather than a fixed count. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
gregandClaude Opus 5.5 committed
1 parent
4874868ce6
commit
487dfbdf68
3 files changed
+542
-3
No files matched your search
@@ -0,0 +1,385 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"regexp"
|
||||
"strings"
|
||||
)
|
||||
|
||||
// When a draft fails its boot, the builder investigates before anything is changed, the way a person
|
||||
// would: it collects evidence (the server's errors, what others found about that error on the web), has
|
||||
// the model state the cause and the evidence for it, lets the model ask for more checks (a search, a
|
||||
// page, the files the server wrote, a boot with one thing changed), and only then asks for exactly that
|
||||
// fix. Every attempt is logged, so a fix that changed nothing rules its cause out instead of being tried
|
||||
// again, and the build stops when a ruled-out cause comes back.
|
||||
|
||||
type attempt struct {
|
||||
Cause string `json:"cause"`
|
||||
Fix string `json:"fix"`
|
||||
Outcome string `json:"outcome"`
|
||||
}
|
||||
|
||||
type checkRequest struct {
|
||||
Type string `json:"type"` // search, read, files, boot_with
|
||||
Query string `json:"query,omitempty"`
|
||||
URL string `json:"url,omitempty"`
|
||||
Config map[string]string `json:"config,omitempty"`
|
||||
Startup string `json:"startup,omitempty"`
|
||||
}
|
||||
|
||||
type diagnosis struct {
|
||||
Cause string `json:"cause"`
|
||||
Evidence string `json:"evidence"`
|
||||
Confidence string `json:"confidence"`
|
||||
Fix string `json:"fix"`
|
||||
Checks []checkRequest `json:"checks"`
|
||||
}
|
||||
|
||||
var diagnosisSchema = map[string]any{
|
||||
"type": "object",
|
||||
"required": []string{"cause", "evidence", "confidence", "fix", "checks"},
|
||||
"properties": map[string]any{
|
||||
"cause": map[string]any{"type": "string"},
|
||||
"evidence": map[string]any{"type": "string"},
|
||||
"confidence": map[string]any{"enum": []string{"high", "medium", "low"}},
|
||||
"fix": map[string]any{"type": "string"},
|
||||
"checks": map[string]any{
|
||||
"type": "array",
|
||||
"maxItems": 3,
|
||||
"items": map[string]any{
|
||||
"type": "object",
|
||||
"required": []string{"type"},
|
||||
"properties": map[string]any{
|
||||
"type": map[string]any{"enum": []string{"search", "read", "files", "boot_with"}},
|
||||
"query": map[string]any{"type": "string"},
|
||||
"url": map[string]any{"type": "string"},
|
||||
"config": map[string]any{"type": "object", "additionalProperties": map[string]any{"type": "string"}},
|
||||
"startup": map[string]any{"type": "string"},
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
}
|
||||
|
||||
const diagnosePrompt = "You debug why a game server template fails on GBR Hosting nodes. Work like an engineer: find the " +
|
||||
"cause from the evidence, never by guessing or by changing things to see what happens. The house rules below " +
|
||||
"describe how servers really run; the boot test starts the server with exactly the settings a new server gets " +
|
||||
"(settings left empty stay empty; only x-generate fields are filled).\n\n" +
|
||||
"Answer with JSON: cause (what is wrong, specifically), evidence (which lines, files or results show it), " +
|
||||
"confidence, fix (the exact change to the template), and checks. If the evidence doesn't settle the cause, ask " +
|
||||
"for up to 3 checks instead of guessing:\n" +
|
||||
"- {\"type\":\"search\",\"query\":\"...\"}: a web search, e.g. the exact error message with the game's name\n" +
|
||||
"- {\"type\":\"read\",\"url\":\"...\"}: read a page\n" +
|
||||
"- {\"type\":\"files\"}: boot the template once more and list the files and config files the server writes, " +
|
||||
"where it writes them, and its error lines\n" +
|
||||
"- {\"type\":\"boot_with\",\"config\":{\"Key\":\"value\"},\"startup\":\"...\"}: boot test with those settings " +
|
||||
"and/or that startup command changed, to see whether that change makes the difference\n" +
|
||||
"Leave checks empty once you are sure. Never propose a cause the attempt log has ruled out.\n\n"
|
||||
|
||||
type investigator struct {
|
||||
m model
|
||||
gameName, gamesDir, id, agentImage string
|
||||
searx, reader, rules string
|
||||
log []attempt
|
||||
}
|
||||
|
||||
func newInvestigator(m model, gameName, gamesDir, id, agentImage string) *investigator {
|
||||
rules, _ := os.ReadFile("AUTHORING.md")
|
||||
return &investigator{m: m, gameName: gameName, gamesDir: gamesDir, id: id, agentImage: agentImage,
|
||||
searx: os.Getenv("SEARXNG_URL"), reader: os.Getenv("READER_URL"), rules: string(rules)}
|
||||
}
|
||||
|
||||
// diagnose investigates one failure and returns the model's diagnosis (checks answered).
|
||||
func (iv *investigator) diagnose(failure string) (*diagnosis, error) {
|
||||
template, _ := os.ReadFile(filepath.Join(iv.gamesDir, iv.id, "game.yaml"))
|
||||
var b strings.Builder
|
||||
fmt.Fprintf(&b, "Game: %s\n\n## The template\n```yaml\n%s```\n\n## What failed\n%s\n", iv.gameName, template, failure)
|
||||
if ev := iv.webEvidence(failure); ev != "" {
|
||||
b.WriteString("\n## What the web says about its errors\n" + ev)
|
||||
}
|
||||
b.WriteString("\n## Attempt log\n" + iv.logText())
|
||||
messages := []chatMessage{
|
||||
{Role: "system", Content: diagnosePrompt + iv.rules},
|
||||
{Role: "user", Content: b.String()},
|
||||
}
|
||||
var d diagnosis
|
||||
for round := 0; ; round++ {
|
||||
fmt.Printf("[diagnose] round %d: asking %s what is wrong\n", round+1, iv.m.name())
|
||||
answer, err := iv.m.chat(messages, diagnosisSchema)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
d = diagnosis{}
|
||||
if err := json.Unmarshal([]byte(trimToJSON(answer)), &d); err != nil {
|
||||
return nil, fmt.Errorf("unreadable diagnosis: %w", err)
|
||||
}
|
||||
fmt.Printf("[diagnose] cause (%s confidence): %s\n", d.Confidence, firstLine(d.Cause))
|
||||
if len(d.Checks) == 0 || round == 2 {
|
||||
return &d, nil
|
||||
}
|
||||
messages = append(messages, chatMessage{Role: "assistant", Content: answer})
|
||||
messages = append(messages, chatMessage{Role: "user", Content: "Results of your checks:\n\n" + iv.runChecks(d.Checks) +
|
||||
"\nNow give your diagnosis. Ask for more checks only if these still don't settle the cause."})
|
||||
}
|
||||
}
|
||||
|
||||
// webEvidence searches for the server's most telling error line and reads the best answer.
|
||||
func (iv *investigator) webEvidence(failure string) string {
|
||||
errs := distinctErrors(failure, 3)
|
||||
if len(errs) == 0 || iv.searx == "" {
|
||||
return ""
|
||||
}
|
||||
return iv.search(iv.gameName + " dedicated server " + errs[0])
|
||||
}
|
||||
|
||||
func (iv *investigator) search(query string) string {
|
||||
if iv.searx == "" {
|
||||
return "(no search configured)\n"
|
||||
}
|
||||
fmt.Printf("[diagnose] searching: %s\n", query)
|
||||
hits, err := searxSearch(iv.searx, query)
|
||||
if err != nil || len(hits) == 0 {
|
||||
return fmt.Sprintf("search %q: no results\n", query)
|
||||
}
|
||||
var b strings.Builder
|
||||
fmt.Fprintf(&b, "Search %q:\n", query)
|
||||
for i, h := range hits {
|
||||
if i == 5 {
|
||||
break
|
||||
}
|
||||
fmt.Fprintf(&b, "- %s — %s: %.300s\n", h.URL, h.Title, h.Content)
|
||||
}
|
||||
if iv.reader != "" {
|
||||
if text, err := readPageLimit(iv.reader, hits[0].URL, 6000); err == nil {
|
||||
fmt.Fprintf(&b, "\nTop result (%s):\n%s\n", hits[0].URL, text)
|
||||
}
|
||||
}
|
||||
return b.String()
|
||||
}
|
||||
|
||||
func (iv *investigator) runChecks(checks []checkRequest) string {
|
||||
var b strings.Builder
|
||||
boots := 0
|
||||
for _, c := range checks {
|
||||
switch c.Type {
|
||||
case "search":
|
||||
b.WriteString(iv.search(c.Query))
|
||||
case "read":
|
||||
fmt.Printf("[diagnose] reading %s\n", c.URL)
|
||||
if text, err := readPageLimit(iv.reader, c.URL, 8000); err != nil {
|
||||
fmt.Fprintf(&b, "read %s: %v\n", c.URL, err)
|
||||
} else {
|
||||
fmt.Fprintf(&b, "Page %s:\n%s\n", c.URL, text)
|
||||
}
|
||||
case "files", "boot_with":
|
||||
// Boots take minutes: at most two per round
|
||||
if boots++; boots > 2 {
|
||||
fmt.Fprintf(&b, "%s: skipped (at most two boots per round)\n", c.Type)
|
||||
continue
|
||||
}
|
||||
if c.Type == "files" {
|
||||
b.WriteString(iv.filesCheck())
|
||||
} else {
|
||||
b.WriteString(iv.bootWith(c))
|
||||
}
|
||||
}
|
||||
b.WriteString("\n")
|
||||
}
|
||||
return b.String()
|
||||
}
|
||||
|
||||
// filesCheck boots the template and reports what the server wrote and where.
|
||||
func (iv *investigator) filesCheck() string {
|
||||
fmt.Println("[diagnose] booting once more to see what the server writes")
|
||||
r, err := firstBootGame(iv.gamesDir, iv.id)
|
||||
if err != nil {
|
||||
return "files: the boot could not run: " + err.Error() + "\n"
|
||||
}
|
||||
var b strings.Builder
|
||||
fmt.Fprintf(&b, "Files check: %s\n", r.Outcome)
|
||||
for i, f := range r.FilesCreated {
|
||||
if i == 60 {
|
||||
fmt.Fprintf(&b, " ... and %d more files\n", len(r.FilesCreated)-60)
|
||||
break
|
||||
}
|
||||
fmt.Fprintf(&b, " file %s (%d bytes)\n", f.Path, f.Size)
|
||||
}
|
||||
for i, s := range r.Settings {
|
||||
if i == 80 {
|
||||
break
|
||||
}
|
||||
fmt.Fprintf(&b, " setting %s | %s | %s = %s\n", s.File, s.Section, s.Key, s.Value)
|
||||
}
|
||||
if dirs := outsideDirs(r); len(dirs) > 0 {
|
||||
fmt.Fprintf(&b, " written outside /server and /data (lost on every restart): %s\n", strings.Join(dirs, ", "))
|
||||
}
|
||||
for i, l := range r.ErrorLines {
|
||||
if i == 10 {
|
||||
break
|
||||
}
|
||||
fmt.Fprintf(&b, " error line: %s\n", l.Text)
|
||||
}
|
||||
return b.String()
|
||||
}
|
||||
|
||||
// bootWith boot-tests a copy of the template with settings and/or the startup command changed.
|
||||
func (iv *investigator) bootWith(c checkRequest) string {
|
||||
desc := describeChange(c)
|
||||
fmt.Printf("[diagnose] boot test with %s\n", desc)
|
||||
games, err := os.MkdirTemp("", "gbr-diagnose-")
|
||||
if err != nil {
|
||||
return "boot_with: " + err.Error() + "\n"
|
||||
}
|
||||
defer os.RemoveAll(games)
|
||||
if err := copyTemplateWith(filepath.Join(iv.gamesDir, iv.id), filepath.Join(games, iv.id), c); err != nil {
|
||||
return "boot_with: " + err.Error() + "\n"
|
||||
}
|
||||
output, passed, problems, err := agentBoot(games, iv.id, iv.agentImage, false)
|
||||
switch {
|
||||
case err != nil:
|
||||
return "boot_with: the boot could not run: " + err.Error() + "\n"
|
||||
case problems != nil:
|
||||
return "boot_with: " + strings.Join(problems, "; ") + "\n"
|
||||
case passed:
|
||||
return fmt.Sprintf("Boot with %s: PASSED (%s)\n", desc, bootVerdict(output))
|
||||
}
|
||||
return fmt.Sprintf("Boot with %s: %s\n", desc, strings.TrimPrefix(bootFailure(output), "booting your template with the node-agent (the real deploy code) failed: "))
|
||||
}
|
||||
|
||||
func describeChange(c checkRequest) string {
|
||||
var parts []string
|
||||
for _, k := range sortedKeysString(c.Config) {
|
||||
parts = append(parts, fmt.Sprintf("%s=%q", k, c.Config[k]))
|
||||
}
|
||||
if c.Startup != "" {
|
||||
parts = append(parts, fmt.Sprintf("startup %q", c.Startup))
|
||||
}
|
||||
if len(parts) == 0 {
|
||||
return "no change"
|
||||
}
|
||||
return strings.Join(parts, ", ")
|
||||
}
|
||||
|
||||
// copyTemplateWith copies games/<id>/ to dst with the requested settings and startup command applied.
|
||||
func copyTemplateWith(src, dst string, c checkRequest) error {
|
||||
g, err := loadMap(filepath.Join(src, "game.yaml"))
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if len(c.Config) > 0 {
|
||||
defaults, _ := g["defaultConfig"].(map[string]any)
|
||||
if defaults == nil {
|
||||
defaults = map[string]any{}
|
||||
}
|
||||
for k, v := range c.Config {
|
||||
defaults[k] = v
|
||||
}
|
||||
g["defaultConfig"] = defaults
|
||||
}
|
||||
if c.Startup != "" {
|
||||
g["defaultStartupCommand"] = c.Startup
|
||||
}
|
||||
if err := os.MkdirAll(dst, 0o755); err != nil {
|
||||
return err
|
||||
}
|
||||
if err := writeGame(filepath.Join(dst, "game.yaml"), g); err != nil {
|
||||
return err
|
||||
}
|
||||
if raw, err := os.ReadFile(filepath.Join(src, "test.yaml")); err == nil {
|
||||
return os.WriteFile(filepath.Join(dst, "test.yaml"), raw, 0o644)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
var (
|
||||
digits = regexp.MustCompile(`\d+`)
|
||||
// Timestamps the shared linePrefix doesn't cover, like 10/26/2026 01:12:02:
|
||||
slashDate = regexp.MustCompile(`^\s*\d{1,2}/\d{1,2}/\d{2,4}[ T][0-9:.,]+:?\s*`)
|
||||
)
|
||||
|
||||
// distinctErrors picks the server's error-looking lines, without timestamps and with repeats removed.
|
||||
// The boot test's own verdict lines are not the server's errors.
|
||||
func distinctErrors(output string, n int) []string {
|
||||
seen := map[string]bool{}
|
||||
var out []string
|
||||
for _, raw := range strings.Split(output, "\n") {
|
||||
raw = strings.TrimSpace(ansi.ReplaceAllString(raw, ""))
|
||||
if strings.HasPrefix(raw, "[boot-test]") || strings.HasPrefix(raw, "booting your template") {
|
||||
continue
|
||||
}
|
||||
line := strings.TrimSpace(slashDate.ReplaceAllString(linePrefix.ReplaceAllString(raw, ""), ""))
|
||||
if line == "" || !errorWords.MatchString(line) {
|
||||
continue
|
||||
}
|
||||
key := digits.ReplaceAllString(strings.ToLower(line), "N")
|
||||
if seen[key] {
|
||||
continue
|
||||
}
|
||||
seen[key] = true
|
||||
if len(line) > 160 {
|
||||
line = line[:160]
|
||||
}
|
||||
out = append(out, line)
|
||||
if len(out) == n {
|
||||
break
|
||||
}
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
func (iv *investigator) logText() string {
|
||||
if len(iv.log) == 0 {
|
||||
return "(first attempt)\n"
|
||||
}
|
||||
var b strings.Builder
|
||||
for i, a := range iv.log {
|
||||
fmt.Fprintf(&b, "%d. cause: %s\n fix: %s\n outcome: %s\n", i+1, a.Cause, a.Fix, a.Outcome)
|
||||
}
|
||||
b.WriteString("A fix whose outcome is unchanged ruled its cause out: look for another cause.\n")
|
||||
return b.String()
|
||||
}
|
||||
|
||||
// ruledOut reports whether the diagnosis names a cause an earlier fix already disproved.
|
||||
func (iv *investigator) ruledOut(d *diagnosis) bool {
|
||||
key := causeKey(d.Cause)
|
||||
for _, a := range iv.log {
|
||||
if causeKey(a.Cause) == key && !strings.HasPrefix(a.Outcome, "fixed") {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
var nonWord = regexp.MustCompile(`[^a-z0-9]+`)
|
||||
|
||||
func causeKey(s string) string {
|
||||
return strings.Trim(nonWord.ReplaceAllString(strings.ToLower(s), " "), " ")
|
||||
}
|
||||
|
||||
func (iv *investigator) record(d *diagnosis) {
|
||||
iv.log = append(iv.log, attempt{Cause: d.Cause, Fix: d.Fix, Outcome: "(not booted yet)"})
|
||||
}
|
||||
|
||||
// outcome fills in how the last fix went: fixed, the same failure, or a new one.
|
||||
func (iv *investigator) outcome(passed bool, before, after string) {
|
||||
if len(iv.log) == 0 {
|
||||
return
|
||||
}
|
||||
a := &iv.log[len(iv.log)-1]
|
||||
switch {
|
||||
case passed:
|
||||
a.Outcome = "fixed: the boot test passed"
|
||||
case causeKey(firstLine(before)) == causeKey(firstLine(after)):
|
||||
a.Outcome = "unchanged: " + firstLine(after)
|
||||
default:
|
||||
a.Outcome = "a different failure: " + firstLine(after)
|
||||
}
|
||||
}
|
||||
|
||||
func fixRequest(failure string, d *diagnosis) []string {
|
||||
return []string{fmt.Sprintf("%s\n\nDiagnosis:\n- cause: %s\n- evidence: %s\n- fix: %s\n\nApply exactly this fix and change "+
|
||||
"nothing unrelated to it.", firstLine(failure), d.Cause, d.Evidence, d.Fix)}
|
||||
}
|
||||
Reference in new issue
Block a user