feat(builder): bounded model requests, a safety check, and a clearer PR summary
- The drafter no longer grows one conversation. Each request is the rules and facts, the model's latest template, what is wrong now and one line per earlier round, so long builds can't push the facts out of the model's 32K context (Ollama drops the oldest part silently). The image's scripts in the facts are capped at 24000 characters in all, and diagnosis check results at 16000. - Every model call logs its prompt and answer tokens, with a warning near the context limit. - unsafeCommands: a draft whose startup command or installConfig pipes a download into a shell, decodes base64, uses eval, raw network tools or /dev/tcp, sudo/su, setuid bits, writes /etc, touches cron/systemd/firewall, or looks like a miner goes back to the model. AUTHORING.md has the rule. - The PR summary lists every address nodes would download from under "Review carefully", marks the measured checks as measured, and labels the research notes as the model's reading, not checked. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
1 parent
709cdb51e8
commit
f11c5eaf2f
7 files changed
+249
-24
No files matched your search
+71
-22
@@ -86,8 +86,10 @@ type draftJob struct {
|
||||
Docs string
|
||||
}
|
||||
|
||||
// drafter holds one conversation with the model about one game, so later problems (a failed boot) can
|
||||
// be sent back to it with everything it was told before.
|
||||
// drafter keeps what the model needs about one game across rounds. Each request is rebuilt from it
|
||||
// instead of growing one conversation: a local model's context is limited, and when a conversation
|
||||
// outgrows it the oldest part, the facts, is silently dropped. A request is the rules and the facts,
|
||||
// the model's latest template, what is wrong with it now, and one line per earlier round.
|
||||
type drafter struct {
|
||||
job draftJob
|
||||
dir, id string
|
||||
@@ -95,7 +97,9 @@ type drafter struct {
|
||||
facts *imageFacts
|
||||
gameSchema, testSchema *jsonschema.Schema
|
||||
answerSchema map[string]any
|
||||
messages []chatMessage
|
||||
base []chatMessage // the rules and the facts
|
||||
last string // the model's latest answer (or the template being repaired)
|
||||
history []string // one line per earlier round
|
||||
rounds int
|
||||
}
|
||||
|
||||
@@ -137,13 +141,29 @@ func newDrafter(j draftJob) (*drafter, error) {
|
||||
if why := imageUnusable(j.Report); why != "" {
|
||||
return nil, errors.New(why)
|
||||
}
|
||||
d.messages = []chatMessage{
|
||||
d.base = []chatMessage{
|
||||
{Role: "system", Content: draftSystemPrompt(string(rules))},
|
||||
{Role: "user", Content: draftRequest(j.Name, id, j.Image, j.SteamApp, j.Notes, d.facts, j.Report)},
|
||||
}
|
||||
return d, nil
|
||||
}
|
||||
|
||||
// request builds one request: the rules and facts, and once there is a template, that template with
|
||||
// what is wrong with it and a line per earlier round.
|
||||
func (d *drafter) request(problems []string) []chatMessage {
|
||||
msgs := append([]chatMessage{}, d.base...)
|
||||
if d.last == "" {
|
||||
return msgs
|
||||
}
|
||||
var b strings.Builder
|
||||
b.WriteString("Your template (above) has these problems. Fix all of them and answer with the complete corrected " +
|
||||
"JSON object (both \"game\" and \"test\"):\n- " + strings.Join(problems, "\n- "))
|
||||
if len(d.history) > 0 {
|
||||
b.WriteString("\n\nEarlier rounds (already dealt with: don't undo those fixes):\n- " + strings.Join(d.history, "\n- "))
|
||||
}
|
||||
return append(msgs, chatMessage{Role: "assistant", Content: d.last}, chatMessage{Role: "user", Content: b.String()})
|
||||
}
|
||||
|
||||
func gameID(name string) string {
|
||||
return strings.Trim(regexp.MustCompile(`[^a-z0-9]+`).ReplaceAllString(strings.ToLower(name), "-"), "-")
|
||||
}
|
||||
@@ -152,20 +172,20 @@ func gameID(name string) string {
|
||||
// attempts+1 answers. feedback, when given, is what went wrong since the last answer (e.g. its boot).
|
||||
// It returns the problems still left, nil when the draft passes.
|
||||
func (d *drafter) refine(attempts int, feedback []string) ([]string, error) {
|
||||
if len(feedback) > 0 {
|
||||
d.ask(feedback)
|
||||
}
|
||||
var problems []string
|
||||
problems := feedback
|
||||
for round := 0; round <= attempts; round++ {
|
||||
d.rounds++
|
||||
fmt.Printf("[draft] round %d: asking %s\n", d.rounds, d.m.name())
|
||||
start := time.Now()
|
||||
answer, err := d.m.chat(d.messages, d.answerSchema)
|
||||
answer, err := d.m.chat(d.request(problems), d.answerSchema)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
fmt.Printf("[draft] answer in %.0fs\n", time.Since(start).Seconds())
|
||||
d.messages = append(d.messages, chatMessage{Role: "assistant", Content: answer})
|
||||
if len(problems) > 0 {
|
||||
d.history = append(d.history, fmt.Sprintf("round %d fixed: %s", d.rounds, summarize(problems)))
|
||||
}
|
||||
d.last = answer
|
||||
|
||||
if problems = d.check(answer); len(problems) == 0 {
|
||||
fmt.Printf("[draft] %s passes validation and the image checks\n", d.dir)
|
||||
@@ -175,16 +195,20 @@ func (d *drafter) refine(attempts int, feedback []string) ([]string, error) {
|
||||
for _, p := range problems {
|
||||
fmt.Println(" - " + p)
|
||||
}
|
||||
if round < attempts {
|
||||
d.ask(problems)
|
||||
}
|
||||
}
|
||||
return problems, nil
|
||||
}
|
||||
|
||||
func (d *drafter) ask(problems []string) {
|
||||
d.messages = append(d.messages, chatMessage{Role: "user", Content: "Your template has these problems. Fix all of them and " +
|
||||
"answer with the complete corrected JSON object (both \"game\" and \"test\"):\n- " + strings.Join(problems, "\n- ")})
|
||||
// summarize is one line for the round history: the first problem, and how many more.
|
||||
func summarize(problems []string) string {
|
||||
s := firstLine(problems[0])
|
||||
if len(s) > 160 {
|
||||
s = s[:160] + "…"
|
||||
}
|
||||
if len(problems) > 1 {
|
||||
s += fmt.Sprintf(" (and %d more)", len(problems)-1)
|
||||
}
|
||||
return s
|
||||
}
|
||||
|
||||
func (d *drafter) check(answer string) []string {
|
||||
@@ -203,6 +227,7 @@ func (d *drafter) check(answer string) []string {
|
||||
problems = append(problems, splitSettings(g)...)
|
||||
problems = append(problems, portSettings(g)...)
|
||||
problems = append(problems, undocumentedOptions(g, d.job.Docs, d.facts)...)
|
||||
problems = append(problems, unsafeCommands(g)...)
|
||||
}
|
||||
}
|
||||
return problems
|
||||
@@ -407,12 +432,22 @@ func draftRequest(gameName, id, image, steamApp, notes string, facts *imageFacts
|
||||
fmt.Fprintf(&b, "- notes: %s\n", notes)
|
||||
}
|
||||
|
||||
// The image's scripts are the biggest part of the facts: 8000 characters each, 24000 in all, so the
|
||||
// request leaves the model room for the rest and for its answer
|
||||
f := *facts
|
||||
f.Scripts = map[string]string{}
|
||||
for p, s := range facts.Scripts {
|
||||
if len(s) > 8000 {
|
||||
s = s[:8000] + "\n...(truncated)"
|
||||
budget := 24000
|
||||
for _, p := range sortedKeysString(facts.Scripts) {
|
||||
s := facts.Scripts[p]
|
||||
limit := min(8000, budget)
|
||||
if limit <= 0 {
|
||||
f.Scripts[p] = "(left out: too long for the request)"
|
||||
continue
|
||||
}
|
||||
if len(s) > limit {
|
||||
s = s[:limit] + "\n...(truncated)"
|
||||
}
|
||||
budget -= len(s)
|
||||
f.Scripts[p] = s
|
||||
}
|
||||
fj, _ := json.MarshalIndent(f, "", " ")
|
||||
@@ -660,17 +695,22 @@ type ollamaModel struct{ url, model string }
|
||||
|
||||
func (o *ollamaModel) name() string { return "ollama " + o.model + " at " + o.url }
|
||||
|
||||
// The model's context in tokens. Ollama silently drops the oldest part of a prompt that doesn't fit, so
|
||||
// every request reports how much of it is used
|
||||
const ollamaContext = 32768
|
||||
|
||||
func (o *ollamaModel) chat(messages []chatMessage, schema map[string]any) (string, error) {
|
||||
body := map[string]any{
|
||||
"model": o.model, "messages": messages, "stream": false, "format": schema,
|
||||
// Ollama silently truncates input beyond its default context
|
||||
"options": map[string]any{"num_ctx": 32768, "temperature": 0.2},
|
||||
"options": map[string]any{"num_ctx": ollamaContext, "temperature": 0.2},
|
||||
}
|
||||
var resp struct {
|
||||
Message struct {
|
||||
Content string `json:"content"`
|
||||
} `json:"message"`
|
||||
Error string `json:"error"`
|
||||
PromptTokens int `json:"prompt_eval_count"`
|
||||
AnswerTokens int `json:"eval_count"`
|
||||
Error string `json:"error"`
|
||||
}
|
||||
if err := postJSON(o.url+"/api/chat", body, &resp, 30*time.Minute); err != nil {
|
||||
return "", err
|
||||
@@ -678,9 +718,18 @@ func (o *ollamaModel) chat(messages []chatMessage, schema map[string]any) (strin
|
||||
if resp.Error != "" {
|
||||
return "", errors.New("ollama: " + resp.Error)
|
||||
}
|
||||
fmt.Println(contextNote(resp.PromptTokens, resp.AnswerTokens))
|
||||
return resp.Message.Content, nil
|
||||
}
|
||||
|
||||
func contextNote(prompt, answer int) string {
|
||||
note := fmt.Sprintf("[model] %d prompt + %d answer tokens of %d", prompt, answer, ollamaContext)
|
||||
if prompt+answer >= ollamaContext*9/10 {
|
||||
note += ": WARNING, close to the model's context; the start of the prompt may have been cut"
|
||||
}
|
||||
return note
|
||||
}
|
||||
|
||||
type geminiModel struct{ key, model string }
|
||||
|
||||
func (g *geminiModel) name() string { return "gemini " + g.model }
|
||||
|
||||
Reference in new issue
Block a user