- The drafter no longer grows one conversation. Each request is the rules and facts, the model's latest template, what is wrong now and one line per earlier round, so long builds can't push the facts out of the model's 32K context (Ollama drops the oldest part silently). The image's scripts in the facts are capped at 24000 characters in all, and diagnosis check results at 16000. - Every model call logs its prompt and answer tokens, with a warning near the context limit. - unsafeCommands: a draft whose startup command or installConfig pipes a download into a shell, decodes base64, uses eval, raw network tools or /dev/tcp, sudo/su, setuid bits, writes /etc, touches cron/systemd/firewall, or looks like a miner goes back to the model. AUTHORING.md has the rule. - The PR summary lists every address nodes would download from under "Review carefully", marks the measured checks as measured, and labels the research notes as the model's reading, not checked. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
166 lines
6.7 KiB
Go
166 lines
6.7 KiB
Go
package main
|
|
|
|
import (
|
|
"encoding/json"
|
|
"os"
|
|
"path/filepath"
|
|
"strings"
|
|
"testing"
|
|
)
|
|
|
|
func TestDraftCompletenessNamesMissingSchemaEntries(t *testing.T) {
|
|
// A draft that went round in circles: settings in defaultConfig, no configSchema
|
|
game := map[string]any{
|
|
"defaultPorts": []any{"30000/udp"},
|
|
"requiredMemoryMb": json.Number("4096"),
|
|
"defaultConfig": map[string]any{"ServerName": "x", "ServerPassword": "", "MaxPlayers": json.Number("8")},
|
|
}
|
|
problems := strings.Join(draftCompleteness(game, map[string]any{"settleSec": json.Number("120")}), "\n")
|
|
if !strings.Contains(problems, "no configSchema.properties entry") || !strings.Contains(problems, "MaxPlayers, ServerName, ServerPassword") {
|
|
t.Errorf("want the missing schema entries named, got:\n%s", problems)
|
|
}
|
|
|
|
game["configSchema"] = map[string]any{"properties": map[string]any{
|
|
"ServerName": map[string]any{"title": "Name", "description": "d", "type": "string", "category": "System & General"},
|
|
"ServerPassword": map[string]any{"title": "Password", "description": "d", "type": "string", "category": "Security & Access"},
|
|
"MaxPlayers": map[string]any{"title": "Players", "description": "d", "type": "integer", "category": "System & General"},
|
|
}}
|
|
if p := draftCompleteness(game, map[string]any{"settleSec": json.Number("120")}); len(p) != 0 {
|
|
t.Errorf("complete draft still has problems: %v", p)
|
|
}
|
|
}
|
|
|
|
func TestUnreadSettings(t *testing.T) {
|
|
facts := &imageFacts{EnvRead: []string{"SERVER_NAME", "SERVER_PASS", "SERVER_PUBLIC"}, Env: map[string]string{"TZ": "UTC"}}
|
|
game := map[string]any{"defaultConfig": map[string]any{"ServerName": "x", "ServerPassword": "", "Tz": "UTC"}}
|
|
problems := unreadSettings(game, facts)
|
|
if len(problems) != 1 || !strings.Contains(problems[0], "SERVER_PASSWORD") || !strings.Contains(problems[0], "did you mean SERVER_PASS") {
|
|
t.Errorf("got %v", problems)
|
|
}
|
|
|
|
// Settings written to a config file aren't environment-only, so nothing to check
|
|
game["configFilePath"] = "server.cfg"
|
|
if p := unreadSettings(game, facts); p != nil {
|
|
t.Errorf("config-file game: %v", p)
|
|
}
|
|
}
|
|
|
|
func TestSavesOutsideVolumes(t *testing.T) {
|
|
// A real first boot: the game's own files in ~/.config, plus Steam client logs (noise)
|
|
home := &bootReport{ContainerChanges: []string{
|
|
"A /home/steam/.config/example-game/adminlist.txt",
|
|
"A /home/steam/.config/example-game/bannedlist.txt",
|
|
"A /home/steam/Steam/logs/connection_log_30000.txt",
|
|
}}
|
|
game := map[string]any{"defaultStartupCommand": "./example_server -nographics -batchmode -world X"}
|
|
problems := savesOutsideVolumes(game, home)
|
|
if len(problems) != 1 || !strings.Contains(problems[0], "/home/steam/.config/example-game") || strings.Contains(problems[0], "Steam/logs") {
|
|
t.Fatalf("got %v", problems)
|
|
}
|
|
|
|
// Fixed with a save-directory option under /server
|
|
game["defaultStartupCommand"] = "./example_server -nographics -batchmode -world X -datadir /server/saves"
|
|
if p := savesOutsideVolumes(game, home); p != nil {
|
|
t.Errorf("redirected template still flagged: %v", p)
|
|
}
|
|
|
|
// A Proton server's first boot: only Proton's cache, which is not game data
|
|
proton := &bootReport{ContainerChanges: []string{"A /home/gameserver/.cache/umu-protonfixes/protonfixes_test.log"}}
|
|
if p := savesOutsideVolumes(map[string]any{}, proton); p != nil {
|
|
t.Errorf("proton cache flagged: %v", p)
|
|
}
|
|
}
|
|
|
|
func TestImageUnusable(t *testing.T) {
|
|
code := 2
|
|
r := &bootReport{User: "1000:1000", ExitCode: &code, RanSeconds: 2.4, ErrorLines: []logLine{
|
|
{Text: "ln: failed to create symbolic link '/etc/localtime': Permission denied"},
|
|
}}
|
|
if why := imageUnusable(r); !strings.Contains(why, "needs root") {
|
|
t.Errorf("want a needs-root verdict, got %q", why)
|
|
}
|
|
|
|
// A server that ran for minutes and then stopped is not this case
|
|
r.RanSeconds = 600
|
|
if why := imageUnusable(r); why != "" {
|
|
t.Errorf("long run flagged: %q", why)
|
|
}
|
|
}
|
|
|
|
func TestDraftCompletenessRejectsSecrets(t *testing.T) {
|
|
// A draft that couldn't be boot-tested: the password mapped to a CI secret nobody created
|
|
test := map[string]any{"settleSec": json.Number("120"), "secrets": map[string]any{"Password": "EXAMPLE_TEST_PASSWORD"}}
|
|
problems := strings.Join(draftCompleteness(map[string]any{}, test), "\n")
|
|
if !strings.Contains(problems, "test.yaml has 'secrets' (Password)") {
|
|
t.Errorf("got:\n%s", problems)
|
|
}
|
|
}
|
|
|
|
// The model copies the example's format, so it must pass everything a draft has to pass
|
|
func TestExampleTemplateFollowsTheRules(t *testing.T) {
|
|
dir := filepath.Join(t.TempDir(), "example-server")
|
|
if err := os.MkdirAll(dir, 0o755); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if err := os.WriteFile(filepath.Join(dir, "game.yaml"), []byte(exampleTemplate), 0o644); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if err := os.WriteFile(filepath.Join(dir, "test.yaml"), []byte("timeoutSec: 2400\n"), 0o644); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
gs, err := compileSchema(filepath.Join("..", "..", "schema", "game.schema.json"))
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
ts, err := compileSchema(filepath.Join("..", "..", "schema", "test.schema.json"))
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
problems := validateGame(dir, "example-server", gs, ts)
|
|
g, err := loadMap(filepath.Join(dir, "game.yaml"))
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
problems = append(problems, draftCompleteness(g, map[string]any{})...)
|
|
problems = append(problems, splitSettings(g)...)
|
|
problems = append(problems, portSettings(g)...)
|
|
for _, p := range problems {
|
|
t.Error(p)
|
|
}
|
|
}
|
|
|
|
func TestDrafterRequestStaysTheSameSizeEveryRound(t *testing.T) {
|
|
d := &drafter{base: []chatMessage{{Role: "system", Content: "rules"}, {Role: "user", Content: "facts"}}}
|
|
if got := d.request(nil); len(got) != 2 {
|
|
t.Fatalf("first request should be the rules and facts only, got %d messages", len(got))
|
|
}
|
|
d.last = `{"game":{"id":"example"}}`
|
|
d.history = []string{"round 1 fixed: defaultPorts is empty", "round 2 fixed: setting X has no title"}
|
|
got := d.request([]string{"booting your template failed: FAILED: server crashed"})
|
|
if len(got) != 4 || got[2].Role != "assistant" || got[2].Content != d.last {
|
|
t.Fatalf("want rules, facts, the latest template and the problems; got %+v", got)
|
|
}
|
|
last := got[3].Content
|
|
for _, want := range []string{"server crashed", "round 1 fixed: defaultPorts is empty", "don't undo"} {
|
|
if !strings.Contains(last, want) {
|
|
t.Errorf("missing %q in %s", want, last)
|
|
}
|
|
}
|
|
// However many rounds there were, still four messages: nothing piles up
|
|
for i := 0; i < 10; i++ {
|
|
d.history = append(d.history, "another round")
|
|
}
|
|
if n := len(d.request([]string{"x"})); n != 4 {
|
|
t.Errorf("request grew to %d messages", n)
|
|
}
|
|
}
|
|
|
|
func TestContextNoteWarnsNearTheLimit(t *testing.T) {
|
|
if strings.Contains(contextNote(12000, 2000), "WARNING") {
|
|
t.Error("warned too early")
|
|
}
|
|
if !strings.Contains(contextNote(28000, 2000), "WARNING") {
|
|
t.Error("no warning near the limit")
|
|
}
|
|
}
|