gbr-games/cmd/gbrgames/eval.go

package main

import (
	"encoding/json"
	"errors"
	"flag"
	"fmt"
	"os"
	"path/filepath"
	"regexp"
	"sort"
	"strings"
	"time"
)

// eval measures the builder: it builds games that already have a working template from scratch (research,
// first boot, draft, boots), without opening a pull request and without the builder seeing that template,
// and scores each draft against the merged one. Run it before and after changing the builder or the model
// to see whether the change helped.

type evalScore struct {
	Game        string   `json:"game"`
	Boots       bool     `json:"boots"`
	SameRoute   bool     `json:"sameRoute"` // same image, and same Steam app when installed with SteamCMD
	PortsSame   bool     `json:"portsSame"`
	PortsDiff   []string `json:"portsDiff,omitempty"`
	ReadyMatch  bool     `json:"readyMatch"` // the draft's readyPattern matches the real server's ready line
	Settings    int      `json:"settings"`
	SettingsHit int      `json:"settingsHit"` // of the real template's settings, how many the draft has too
	RealSetting int      `json:"realSettings"`
	Rounds      int      `json:"rounds"`
	Minutes     float64  `json:"minutes"`
	Error       string   `json:"error,omitempty"`
}

func runEval(args []string) error {
	fs := flag.NewFlagSet("eval", flag.ExitOnError)
	gamesDir := fs.String("games", "games", "the repo's games directory: the templates to compare against")
	schemaDir := fs.String("schemas", "schema", "directory holding the JSON Schemas")
	which := fs.String("ids", "all", "comma-separated game ids, or all")
	provider := fs.String("provider", "ollama", "ollama or gemini")
	modelName := fs.String("model", "", "model name (default as for draft)")
	out := fs.String("out", "eval.md", "write the report here (and the scores as JSON next to it)")
	fs.Parse(args)

	ids, err := evalIDs(*gamesDir, *which)
	if err != nil {
		return err
	}
	m, err := newModel(*provider, *modelName)
	if err != nil {
		return err
	}
	var scores []evalScore
	for _, id := range ids {
		fmt.Printf("[eval] ===== %s =====\n", id)
		s := evalGame(m, *gamesDir, *schemaDir, id, *provider, *modelName)
		fmt.Printf("[eval] %s: boots %v, same route %v, ports same %v, ready matches %v, settings %d/%d\n",
			id, s.Boots, s.SameRoute, s.PortsSame, s.ReadyMatch, s.SettingsHit, s.RealSetting)
		scores = append(scores, s)
	}
	if err := os.WriteFile(*out, []byte(evalReport(scores)), 0o644); err != nil {
		return err
	}
	raw, _ := json.MarshalIndent(scores, "", "  ")
	return os.WriteFile(strings.TrimSuffix(*out, filepath.Ext(*out))+".json", raw, 0o644)
}

func evalIDs(gamesDir, which string) ([]string, error) {
	if which != "all" {
		return strings.Split(which, ","), nil
	}
	entries, err := os.ReadDir(gamesDir)
	if err != nil {
		return nil, err
	}
	var ids []string
	for _, e := range entries {
		if _, err := os.Stat(filepath.Join(gamesDir, e.Name(), "game.yaml")); err == nil {
			ids = append(ids, e.Name())
		}
	}
	if len(ids) == 0 {
		return nil, errors.New("no templates to evaluate against")
	}
	return ids, nil
}

// evalGame builds one game blind, in an empty games folder, and compares the draft with the real template.
func evalGame(m model, gamesDir, schemaDir, id, provider, modelName string) evalScore {
	s := evalScore{Game: id}
	start := time.Now()
	defer func() { s.Minutes = time.Since(start).Minutes() }()
	real, err := loadMap(filepath.Join(gamesDir, id, "game.yaml"))
	if err != nil {
		s.Error = err.Error()
		return s
	}
	realTest, _ := loadMap(filepath.Join(gamesDir, id, "test.yaml"))
	name, _ := real["name"].(string)

	res, err := research(m, name, "", 6)
	if err != nil {
		s.Error = "research: " + err.Error()
		return s
	}
	var chosen *variant
	for i := range res.Variants {
		if res.Variants[i].Recommended {
			chosen = &res.Variants[i]
		}
	}
	if chosen == nil {
		s.Error = "research recommended no runnable option: " + res.Message
		return s
	}
	blind, err := os.MkdirTemp("", "gbr-eval-")
	if err != nil {
		s.Error = err.Error()
		return s
	}
	defer os.RemoveAll(blind)
	built, err := build(buildOptions{gamesDir: blind, schemaDir: schemaDir, name: name, id: id, variant: *chosen,
		provider: provider, model: modelName, attempts: 3, bootRounds: 5, agentImage: agentImageFromEnv()})
	if built != nil {
		s.Boots, s.Rounds = built.Passed, built.Rounds
	}
	if err != nil {
		s.Error = "build: " + err.Error()
	}
	draft, derr := loadMap(filepath.Join(blind, id, "game.yaml"))
	if derr != nil {
		if s.Error == "" {
			s.Error = "no draft written"
		}
		return s
	}
	scoreDraft(&s, real, realTest, draft)
	return s
}

// scoreDraft compares a draft with the real template.
func scoreDraft(s *evalScore, real, realTest, draft map[string]any) {
	s.SameRoute = real["defaultDockerImage"] == draft["defaultDockerImage"] && steamAppOf(real) == steamAppOf(draft)

	want, got := portSet(real), portSet(draft)
	for p := range want {
		if !got[p] {
			s.PortsDiff = append(s.PortsDiff, "missing "+p)
		}
	}
	for p := range got {
		if !want[p] {
			s.PortsDiff = append(s.PortsDiff, "extra "+p)
		}
	}
	sort.Strings(s.PortsDiff)
	s.PortsSame = len(s.PortsDiff) == 0

	if pattern, _ := draft["readyPattern"].(string); pattern != "" {
		example, _ := realTest["readyExample"].(string)
		if re, err := regexp.Compile(pattern); err == nil && example != "" {
			s.ReadyMatch = re.MatchString(example)
		}
	}

	realKeys := map[string]bool{}
	for k := range schemaProperties(real) {
		realKeys[envName(k)] = true
	}
	s.RealSetting = len(realKeys)
	for k := range schemaProperties(draft) {
		s.Settings++
		if realKeys[envName(k)] {
			s.SettingsHit++
		}
	}
}

func steamAppOf(g map[string]any) string {
	if g["installMethod"] != "STEAMCMD" {
		return ""
	}
	ic, _ := g["installConfig"].(map[string]any)
	return fmt.Sprint(ic["appId"])
}

// portSet is a template's published container ports, as "port/proto".
func portSet(g map[string]any) map[string]bool {
	out := map[string]bool{}
	ports, _ := g["defaultPorts"].([]any)
	for _, p := range ports {
		out[containerPort(fmt.Sprint(p))] = true
	}
	return out
}

func evalReport(scores []evalScore) string {
	var b strings.Builder
	b.WriteString("# Builder evaluation\n\nEach game was built blind (research, first boot, draft, boots) and compared " +
		"with its merged template.\n\n")
	b.WriteString("| Game | Boots | Same image/app | Ports | Ready line | Settings found | Rounds | Minutes | Note |\n")
	b.WriteString("| --- | --- | --- | --- | --- | --- | --- | --- | --- |\n")
	passed := 0
	for _, s := range scores {
		if s.Boots {
			passed++
		}
		ports := "same"
		if !s.PortsSame {
			ports = strings.Join(s.PortsDiff, ", ")
		}
		fmt.Fprintf(&b, "| %s | %s | %s | %s | %s | %d of %d (draft has %d) | %d | %.0f | %s |\n", s.Game, mark(s.Boots), mark(s.SameRoute),
			ports, mark(s.ReadyMatch), s.SettingsHit, s.RealSetting, s.Settings, s.Rounds, s.Minutes, s.Error)
	}
	fmt.Fprintf(&b, "\n%d of %d drafts booted.\n", passed, len(scores))
	return b.String()
}

func mark(ok bool) string {
	if ok {
		return "yes"
	}
	return "no"
}