feat(builder): repair existing templates, a weekly check that queues repairs, and eval
The builder could only create games; nothing noticed when a live template stopped working, and there was no way to tell whether a builder change made it better or worse. - repair -id ID: boots an existing template as nodes run it; if it fails, the same investigation as for a draft runs and the model changes only what the diagnosis says. The worker handles "repair" jobs (gameId from the panel) and opens "fix(games): repair ..." on draft/<id>; a template that boots as it is reports "nothing to repair" and opens no PR. The boot and fix loop is now shared by builds and repairs. - validate.yml runs weekly and boot-tests every game; failed games are sent to the panel (X-GBR-Event: games_boot_failed) to queue repairs. - eval -ids all: builds games that already have a template from scratch, blind, and scores each draft against it (boots, same image and app, ports, ready line, settings found) into eval.md and eval.json. - research skips our own sites, so neither research nor eval reads our templates back. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
gregandClaude Opus 5.5 committed
1 parent
00cc61aba2
commit
17f6477b71
12 files changed
+535
-20
No files matched your search
@@ -0,0 +1,225 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"flag"
|
||||
"fmt"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"regexp"
|
||||
"sort"
|
||||
"strings"
|
||||
"time"
|
||||
)
|
||||
|
||||
// eval measures the builder: it builds games that already have a working template from scratch (research,
|
||||
// first boot, draft, boots), without opening a pull request and without the builder seeing that template,
|
||||
// and scores each draft against the merged one. Run it before and after changing the builder or the model
|
||||
// to see whether the change helped.
|
||||
|
||||
type evalScore struct {
|
||||
Game string `json:"game"`
|
||||
Boots bool `json:"boots"`
|
||||
SameRoute bool `json:"sameRoute"` // same image, and same Steam app when installed with SteamCMD
|
||||
PortsSame bool `json:"portsSame"`
|
||||
PortsDiff []string `json:"portsDiff,omitempty"`
|
||||
ReadyMatch bool `json:"readyMatch"` // the draft's readyPattern matches the real server's ready line
|
||||
Settings int `json:"settings"`
|
||||
SettingsHit int `json:"settingsHit"` // of the real template's settings, how many the draft has too
|
||||
RealSetting int `json:"realSettings"`
|
||||
Rounds int `json:"rounds"`
|
||||
Minutes float64 `json:"minutes"`
|
||||
Error string `json:"error,omitempty"`
|
||||
}
|
||||
|
||||
func runEval(args []string) error {
|
||||
fs := flag.NewFlagSet("eval", flag.ExitOnError)
|
||||
gamesDir := fs.String("games", "games", "the repo's games directory: the templates to compare against")
|
||||
schemaDir := fs.String("schemas", "schema", "directory holding the JSON Schemas")
|
||||
which := fs.String("ids", "all", "comma-separated game ids, or all")
|
||||
provider := fs.String("provider", "ollama", "ollama or gemini")
|
||||
modelName := fs.String("model", "", "model name (default as for draft)")
|
||||
out := fs.String("out", "eval.md", "write the report here (and the scores as JSON next to it)")
|
||||
fs.Parse(args)
|
||||
|
||||
ids, err := evalIDs(*gamesDir, *which)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
m, err := newModel(*provider, *modelName)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
var scores []evalScore
|
||||
for _, id := range ids {
|
||||
fmt.Printf("[eval] ===== %s =====\n", id)
|
||||
s := evalGame(m, *gamesDir, *schemaDir, id, *provider, *modelName)
|
||||
fmt.Printf("[eval] %s: boots %v, same route %v, ports same %v, ready matches %v, settings %d/%d\n",
|
||||
id, s.Boots, s.SameRoute, s.PortsSame, s.ReadyMatch, s.SettingsHit, s.RealSetting)
|
||||
scores = append(scores, s)
|
||||
}
|
||||
if err := os.WriteFile(*out, []byte(evalReport(scores)), 0o644); err != nil {
|
||||
return err
|
||||
}
|
||||
raw, _ := json.MarshalIndent(scores, "", " ")
|
||||
return os.WriteFile(strings.TrimSuffix(*out, filepath.Ext(*out))+".json", raw, 0o644)
|
||||
}
|
||||
|
||||
func evalIDs(gamesDir, which string) ([]string, error) {
|
||||
if which != "all" {
|
||||
return strings.Split(which, ","), nil
|
||||
}
|
||||
entries, err := os.ReadDir(gamesDir)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
var ids []string
|
||||
for _, e := range entries {
|
||||
if _, err := os.Stat(filepath.Join(gamesDir, e.Name(), "game.yaml")); err == nil {
|
||||
ids = append(ids, e.Name())
|
||||
}
|
||||
}
|
||||
if len(ids) == 0 {
|
||||
return nil, errors.New("no templates to evaluate against")
|
||||
}
|
||||
return ids, nil
|
||||
}
|
||||
|
||||
// evalGame builds one game blind, in an empty games folder, and compares the draft with the real template.
|
||||
func evalGame(m model, gamesDir, schemaDir, id, provider, modelName string) evalScore {
|
||||
s := evalScore{Game: id}
|
||||
start := time.Now()
|
||||
defer func() { s.Minutes = time.Since(start).Minutes() }()
|
||||
real, err := loadMap(filepath.Join(gamesDir, id, "game.yaml"))
|
||||
if err != nil {
|
||||
s.Error = err.Error()
|
||||
return s
|
||||
}
|
||||
realTest, _ := loadMap(filepath.Join(gamesDir, id, "test.yaml"))
|
||||
name, _ := real["name"].(string)
|
||||
|
||||
res, err := research(m, name, "", 6)
|
||||
if err != nil {
|
||||
s.Error = "research: " + err.Error()
|
||||
return s
|
||||
}
|
||||
var chosen *variant
|
||||
for i := range res.Variants {
|
||||
if res.Variants[i].Recommended {
|
||||
chosen = &res.Variants[i]
|
||||
}
|
||||
}
|
||||
if chosen == nil {
|
||||
s.Error = "research recommended no runnable option: " + res.Message
|
||||
return s
|
||||
}
|
||||
blind, err := os.MkdirTemp("", "gbr-eval-")
|
||||
if err != nil {
|
||||
s.Error = err.Error()
|
||||
return s
|
||||
}
|
||||
defer os.RemoveAll(blind)
|
||||
built, err := build(buildOptions{gamesDir: blind, schemaDir: schemaDir, name: name, id: id, variant: *chosen,
|
||||
provider: provider, model: modelName, attempts: 3, bootRounds: 5, agentImage: agentImageFromEnv()})
|
||||
if built != nil {
|
||||
s.Boots, s.Rounds = built.Passed, built.Rounds
|
||||
}
|
||||
if err != nil {
|
||||
s.Error = "build: " + err.Error()
|
||||
}
|
||||
draft, derr := loadMap(filepath.Join(blind, id, "game.yaml"))
|
||||
if derr != nil {
|
||||
if s.Error == "" {
|
||||
s.Error = "no draft written"
|
||||
}
|
||||
return s
|
||||
}
|
||||
scoreDraft(&s, real, realTest, draft)
|
||||
return s
|
||||
}
|
||||
|
||||
// scoreDraft compares a draft with the real template.
|
||||
func scoreDraft(s *evalScore, real, realTest, draft map[string]any) {
|
||||
s.SameRoute = real["defaultDockerImage"] == draft["defaultDockerImage"] && steamAppOf(real) == steamAppOf(draft)
|
||||
|
||||
want, got := portSet(real), portSet(draft)
|
||||
for p := range want {
|
||||
if !got[p] {
|
||||
s.PortsDiff = append(s.PortsDiff, "missing "+p)
|
||||
}
|
||||
}
|
||||
for p := range got {
|
||||
if !want[p] {
|
||||
s.PortsDiff = append(s.PortsDiff, "extra "+p)
|
||||
}
|
||||
}
|
||||
sort.Strings(s.PortsDiff)
|
||||
s.PortsSame = len(s.PortsDiff) == 0
|
||||
|
||||
if pattern, _ := draft["readyPattern"].(string); pattern != "" {
|
||||
example, _ := realTest["readyExample"].(string)
|
||||
if re, err := regexp.Compile(pattern); err == nil && example != "" {
|
||||
s.ReadyMatch = re.MatchString(example)
|
||||
}
|
||||
}
|
||||
|
||||
realKeys := map[string]bool{}
|
||||
for k := range schemaProperties(real) {
|
||||
realKeys[envName(k)] = true
|
||||
}
|
||||
s.RealSetting = len(realKeys)
|
||||
for k := range schemaProperties(draft) {
|
||||
s.Settings++
|
||||
if realKeys[envName(k)] {
|
||||
s.SettingsHit++
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func steamAppOf(g map[string]any) string {
|
||||
if g["installMethod"] != "STEAMCMD" {
|
||||
return ""
|
||||
}
|
||||
ic, _ := g["installConfig"].(map[string]any)
|
||||
return fmt.Sprint(ic["appId"])
|
||||
}
|
||||
|
||||
// portSet is a template's published container ports, as "port/proto".
|
||||
func portSet(g map[string]any) map[string]bool {
|
||||
out := map[string]bool{}
|
||||
ports, _ := g["defaultPorts"].([]any)
|
||||
for _, p := range ports {
|
||||
out[containerPort(fmt.Sprint(p))] = true
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
func evalReport(scores []evalScore) string {
|
||||
var b strings.Builder
|
||||
b.WriteString("# Builder evaluation\n\nEach game was built blind (research, first boot, draft, boots) and compared " +
|
||||
"with its merged template.\n\n")
|
||||
b.WriteString("| Game | Boots | Same image/app | Ports | Ready line | Settings found | Rounds | Minutes | Note |\n")
|
||||
b.WriteString("| --- | --- | --- | --- | --- | --- | --- | --- | --- |\n")
|
||||
passed := 0
|
||||
for _, s := range scores {
|
||||
if s.Boots {
|
||||
passed++
|
||||
}
|
||||
ports := "same"
|
||||
if !s.PortsSame {
|
||||
ports = strings.Join(s.PortsDiff, ", ")
|
||||
}
|
||||
fmt.Fprintf(&b, "| %s | %s | %s | %s | %s | %d of %d (draft has %d) | %d | %.0f | %s |\n", s.Game, mark(s.Boots), mark(s.SameRoute),
|
||||
ports, mark(s.ReadyMatch), s.SettingsHit, s.RealSetting, s.Settings, s.Rounds, s.Minutes, s.Error)
|
||||
}
|
||||
fmt.Fprintf(&b, "\n%d of %d drafts booted.\n", passed, len(scores))
|
||||
return b.String()
|
||||
}
|
||||
|
||||
func mark(ok bool) string {
|
||||
if ok {
|
||||
return "yes"
|
||||
}
|
||||
return "no"
|
||||
}
|
||||
Reference in new issue
Block a user