feat(builder): research, build and a worker for the AI builder
- research: searches SearXNG, reads the best pages through the reader, has the model list the server's variants, then checks every Docker image and Steam app it named for real before an admin sees it. - build: first boot of the variant's images (skipping ones that need root), a draft by the model, then the draft deployed with the node-agent's BootTest like CI does; crashes, error lines and saves outside /server go back to the model, three boots at most. - worker: polls the panel's builder queue (outbound only), runs research or build in a fresh clone, streams the output as the job log, and opens the result as a pull request from the bot account. - Dockerfile.builder, a workflow publishing gbr-builder:latest, and BUILDER.md. Everything about where it runs comes from environment variables; deployment files live outside this public repo. draft now keeps its conversation in a drafter, so build can send boot failures back to the same model session. The boot-test runner comment no longer names its VM. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
gregandClaude Opus 5.5 committed
1 parent
05bfc9b061
commit
c75190d8c4
12 files changed
+1495
-95
No files matched your search
@@ -0,0 +1,398 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"flag"
|
||||
"fmt"
|
||||
"io"
|
||||
"net/http"
|
||||
"net/url"
|
||||
"os"
|
||||
"regexp"
|
||||
"sort"
|
||||
"strings"
|
||||
"time"
|
||||
)
|
||||
|
||||
// research finds out how a game's dedicated server can be run before anything is drafted: it searches the
|
||||
// web (a self-hosted SearXNG), reads the most useful pages (a reader that turns pages into text), and has
|
||||
// the model list the server's variants (editions, crossplay builds, docker image or SteamCMD app). Every
|
||||
// image and Steam app it names is then checked for real, so a variant shown to an admin exists.
|
||||
|
||||
type variant struct {
|
||||
ID string `json:"id"`
|
||||
Title string `json:"title"`
|
||||
Description string `json:"description"`
|
||||
Recommended bool `json:"recommended"`
|
||||
Images []string `json:"images"`
|
||||
SteamAppID string `json:"steamAppId"`
|
||||
StartupHint string `json:"startupHint"`
|
||||
Ports []string `json:"ports"`
|
||||
Notes string `json:"notes"`
|
||||
Sources []string `json:"sources"`
|
||||
// Filled by the checks, not the model
|
||||
Verified []string `json:"verified,omitempty"`
|
||||
Problems []string `json:"problems,omitempty"`
|
||||
}
|
||||
|
||||
type researchResult struct {
|
||||
GameName string `json:"gameName"`
|
||||
Supported bool `json:"supported"`
|
||||
Reason string `json:"reason,omitempty"`
|
||||
Message string `json:"message"`
|
||||
Variants []variant `json:"variants"`
|
||||
Sources []string `json:"sources"`
|
||||
}
|
||||
|
||||
var researchSchema = map[string]any{
|
||||
"type": "object",
|
||||
"required": []string{"supported", "reason", "message", "variants"},
|
||||
"properties": map[string]any{
|
||||
"supported": map[string]any{"type": "boolean"},
|
||||
"reason": map[string]any{"enum": []string{"ok", "unreleased", "single_player", "no_dedicated_server", "sunset", "unknown"}},
|
||||
"message": map[string]any{"type": "string"},
|
||||
"variants": map[string]any{
|
||||
"type": "array",
|
||||
"items": map[string]any{
|
||||
"type": "object",
|
||||
"required": []string{"id", "title", "description", "recommended", "images", "steamAppId", "startupHint", "ports", "notes", "sources"},
|
||||
"properties": map[string]any{
|
||||
"id": map[string]any{"type": "string"},
|
||||
"title": map[string]any{"type": "string"},
|
||||
"description": map[string]any{"type": "string"},
|
||||
"recommended": map[string]any{"type": "boolean"},
|
||||
"images": map[string]any{"type": "array", "items": map[string]any{"type": "string"}},
|
||||
"steamAppId": map[string]any{"type": "string"},
|
||||
"startupHint": map[string]any{"type": "string"},
|
||||
"ports": map[string]any{"type": "array", "items": map[string]any{"type": "string"}},
|
||||
"notes": map[string]any{"type": "string"},
|
||||
"sources": map[string]any{"type": "array", "items": map[string]any{"type": "string"}},
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
}
|
||||
|
||||
func runResearch(args []string) error {
|
||||
fs := flag.NewFlagSet("research", flag.ExitOnError)
|
||||
gameName := fs.String("name", "", "the game's name, e.g. \"Valheim\"")
|
||||
notes := fs.String("notes", "", "what the admin asked for (crossplay, a modded edition...)")
|
||||
provider := fs.String("provider", "ollama", "ollama or gemini")
|
||||
modelName := fs.String("model", "", "model name (default as for draft)")
|
||||
pages := fs.Int("pages", 6, "how many search results to read")
|
||||
out := fs.String("out", "", "write the result JSON here (default stdout)")
|
||||
fs.Parse(args)
|
||||
if *gameName == "" {
|
||||
return errors.New("research needs -name")
|
||||
}
|
||||
m, err := newModel(*provider, *modelName)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
res, err := research(m, *gameName, *notes, *pages)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
raw, _ := json.MarshalIndent(res, "", " ")
|
||||
if *out == "" {
|
||||
fmt.Println(string(raw))
|
||||
return nil
|
||||
}
|
||||
return os.WriteFile(*out, raw, 0o644)
|
||||
}
|
||||
|
||||
func research(m model, gameName, notes string, pages int) (*researchResult, error) {
|
||||
searx, reader := os.Getenv("SEARXNG_URL"), os.Getenv("READER_URL")
|
||||
if searx == "" || reader == "" {
|
||||
return nil, errors.New("set SEARXNG_URL (e.g. http://host:8080) and READER_URL (e.g. http://host:3000)")
|
||||
}
|
||||
queries := []string{
|
||||
gameName + " dedicated server docker image",
|
||||
gameName + " dedicated server steamcmd app id linux",
|
||||
gameName + " dedicated server setup ports launch options",
|
||||
gameName + " crossplay console dedicated server",
|
||||
}
|
||||
var hits []searchHit
|
||||
for _, q := range queries {
|
||||
fmt.Printf("[research] searching: %s\n", q)
|
||||
h, err := searxSearch(searx, q)
|
||||
if err != nil {
|
||||
fmt.Printf("[research] search failed: %v\n", err)
|
||||
continue
|
||||
}
|
||||
hits = append(hits, h...)
|
||||
}
|
||||
if len(hits) == 0 {
|
||||
return nil, errors.New("no search results (is SearXNG reachable?)")
|
||||
}
|
||||
picked := pickPages(hits, pages)
|
||||
|
||||
var b strings.Builder
|
||||
fmt.Fprintf(&b, "Game: **%s**\n", gameName)
|
||||
if notes != "" {
|
||||
fmt.Fprintf(&b, "The admin asked for: %s\n", notes)
|
||||
}
|
||||
b.WriteString("\n## Search results\n")
|
||||
for _, h := range dedupeHits(hits) {
|
||||
fmt.Fprintf(&b, "- %s — %s: %.200s\n", h.URL, h.Title, h.Content)
|
||||
}
|
||||
var sources []string
|
||||
for _, h := range picked {
|
||||
fmt.Printf("[research] reading %s\n", h.URL)
|
||||
text, err := readPage(reader, h.URL)
|
||||
if err != nil {
|
||||
fmt.Printf("[research] could not read it: %v\n", err)
|
||||
continue
|
||||
}
|
||||
sources = append(sources, h.URL)
|
||||
fmt.Fprintf(&b, "\n## Page: %s\n%s\n", h.URL, text)
|
||||
}
|
||||
|
||||
messages := []chatMessage{
|
||||
{Role: "system", Content: researchPrompt},
|
||||
{Role: "user", Content: b.String()},
|
||||
}
|
||||
fmt.Printf("[research] asking %s\n", m.name())
|
||||
answer, err := m.chat(messages, researchSchema)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
res := &researchResult{}
|
||||
if err := json.Unmarshal([]byte(trimToJSON(answer)), res); err != nil {
|
||||
return nil, fmt.Errorf("the model's answer is not valid JSON: %w", err)
|
||||
}
|
||||
res.GameName = gameName
|
||||
res.Sources = sources
|
||||
|
||||
verifyVariants(res)
|
||||
return res, nil
|
||||
}
|
||||
|
||||
const researchPrompt = "You research how to host a game's dedicated server for GBR Hosting, a game server host. Servers run " +
|
||||
"in Docker on Linux, as a non-root user, with a volume at /server. From the search results and pages given, list " +
|
||||
"the ways this game's dedicated server can be run (variants): different editions (e.g. Java vs Bedrock, Evolved vs " +
|
||||
"Ascended), crossplay builds, or notably different server images. For each variant give:\n" +
|
||||
"- images: Docker Hub images that run this server, best first (prefer images that run as a non-root user, and the " +
|
||||
"plain SteamCMD route cm2network/steamcmd when the game ships a Linux dedicated server on Steam). Only images that " +
|
||||
"appear in the pages.\n" +
|
||||
"- steamAppId: the Steam app id of the DEDICATED SERVER (not the game itself), if installed via SteamCMD; else \"\".\n" +
|
||||
"- startupHint: the command that starts the server from its install folder /server, with the launch options " +
|
||||
"for name, port, world and save location exactly as the pages document them; \"\" if unknown.\n" +
|
||||
"- ports: every port players connect to, as \"port/proto\".\n" +
|
||||
"- notes: anything a template author must know (Windows-only server needing Proton, crossplay flags, mods).\n" +
|
||||
"- sources: the page URLs that support it.\n" +
|
||||
"Mark one variant recommended. Use only what the pages say or what you know for certain; leave fields empty " +
|
||||
"instead of guessing. If no dedicated server can be hosted (single-player only, unreleased, official servers " +
|
||||
"only, shut down), set supported false, the matching reason, no variants, and explain in message. Otherwise " +
|
||||
"reason \"ok\" and a one-sentence message."
|
||||
|
||||
type searchHit struct {
|
||||
URL string `json:"url"`
|
||||
Title string `json:"title"`
|
||||
Content string `json:"content"`
|
||||
}
|
||||
|
||||
func searxSearch(base, query string) ([]searchHit, error) {
|
||||
u := strings.TrimRight(base, "/") + "/search?format=json&q=" + url.QueryEscape(query)
|
||||
raw, err := httpGet(u, 30*time.Second)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
var resp struct {
|
||||
Results []searchHit `json:"results"`
|
||||
}
|
||||
if err := json.Unmarshal(raw, &resp); err != nil {
|
||||
return nil, fmt.Errorf("unreadable search answer: %.200s", raw)
|
||||
}
|
||||
if len(resp.Results) > 8 {
|
||||
resp.Results = resp.Results[:8]
|
||||
}
|
||||
return resp.Results, nil
|
||||
}
|
||||
|
||||
// Pages that hold the facts a template needs, read first
|
||||
var usefulPage = regexp.MustCompile(`(?i)hub\.docker\.com|github\.com|steamdb\.info|developer\.valvesoftware|wiki|fandom|reddit\.com/r/\w*(server|admin)`)
|
||||
|
||||
// pickPages reads one page per site, docs and image pages first, so a handful of reads covers the most ground.
|
||||
func pickPages(hits []searchHit, n int) []searchHit {
|
||||
hits = dedupeHits(hits)
|
||||
sort.SliceStable(hits, func(i, j int) bool {
|
||||
return usefulPage.MatchString(hits[i].URL) && !usefulPage.MatchString(hits[j].URL)
|
||||
})
|
||||
seen := map[string]bool{}
|
||||
var out []searchHit
|
||||
for _, h := range hits {
|
||||
u, err := url.Parse(h.URL)
|
||||
if err != nil || seen[u.Host] {
|
||||
continue
|
||||
}
|
||||
seen[u.Host] = true
|
||||
out = append(out, h)
|
||||
if len(out) == n {
|
||||
break
|
||||
}
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
func dedupeHits(hits []searchHit) []searchHit {
|
||||
seen := map[string]bool{}
|
||||
var out []searchHit
|
||||
for _, h := range hits {
|
||||
if h.URL == "" || seen[h.URL] {
|
||||
continue
|
||||
}
|
||||
seen[h.URL] = true
|
||||
out = append(out, h)
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
func readPage(reader, pageURL string) (string, error) {
|
||||
raw, err := httpGet(strings.TrimRight(reader, "/")+"/"+pageURL, 60*time.Second)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
text := string(raw)
|
||||
if len(text) > 9000 {
|
||||
text = text[:9000] + "\n...(truncated)"
|
||||
}
|
||||
return text, nil
|
||||
}
|
||||
|
||||
func httpGet(u string, timeout time.Duration) ([]byte, error) {
|
||||
client := &http.Client{Timeout: timeout}
|
||||
resp, err := client.Get(u)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer resp.Body.Close()
|
||||
raw, err := io.ReadAll(io.LimitReader(resp.Body, 4<<20))
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if resp.StatusCode >= 300 {
|
||||
return nil, fmt.Errorf("HTTP %d: %.200s", resp.StatusCode, raw)
|
||||
}
|
||||
return raw, nil
|
||||
}
|
||||
|
||||
// trimToJSON cuts prose or code fences some models put around a JSON answer.
|
||||
func trimToJSON(answer string) string {
|
||||
answer = strings.TrimSpace(answer)
|
||||
if i := strings.Index(answer, "{"); i > 0 {
|
||||
answer = answer[i:]
|
||||
}
|
||||
if i := strings.LastIndex(answer, "}"); i >= 0 {
|
||||
answer = answer[:i+1]
|
||||
}
|
||||
return answer
|
||||
}
|
||||
|
||||
// verifyVariants checks every image and Steam app the model named: images must exist in their registry,
|
||||
// Steam apps must exist and be readable anonymously (what SteamCMD installs need). What fails is dropped
|
||||
// and noted; a variant left with nothing runnable can't be recommended.
|
||||
func verifyVariants(res *researchResult) {
|
||||
var apps []string
|
||||
for _, v := range res.Variants {
|
||||
if v.SteamAppID != "" {
|
||||
apps = append(apps, v.SteamAppID)
|
||||
}
|
||||
}
|
||||
names := steamAppNames(apps)
|
||||
anyRunnable := false
|
||||
for i := range res.Variants {
|
||||
v := &res.Variants[i]
|
||||
var images []string
|
||||
for _, img := range v.Images {
|
||||
img = strings.TrimSpace(img)
|
||||
if img == "" {
|
||||
continue
|
||||
}
|
||||
if out, err := runCmd("docker", "manifest", "inspect", img); err != nil {
|
||||
v.Problems = append(v.Problems, fmt.Sprintf("image %s not found: %.120s", img, out))
|
||||
continue
|
||||
}
|
||||
v.Verified = append(v.Verified, "image "+img+" exists")
|
||||
images = append(images, img)
|
||||
}
|
||||
v.Images = images
|
||||
if v.SteamAppID != "" {
|
||||
if name, ok := names[v.SteamAppID]; ok {
|
||||
v.Verified = append(v.Verified, fmt.Sprintf("Steam app %s is %q", v.SteamAppID, name))
|
||||
if !hasImage(v.Images, "cm2network/steamcmd") {
|
||||
v.Images = append(v.Images, "cm2network/steamcmd")
|
||||
}
|
||||
} else {
|
||||
v.Problems = append(v.Problems, "Steam app "+v.SteamAppID+" not found for anonymous SteamCMD")
|
||||
v.SteamAppID = ""
|
||||
}
|
||||
}
|
||||
if len(v.Images) == 0 {
|
||||
v.Problems = append(v.Problems, "no image or Steam app that could be checked; it can't be built")
|
||||
v.Recommended = false
|
||||
} else {
|
||||
anyRunnable = true
|
||||
}
|
||||
}
|
||||
if !anyRunnable && res.Supported {
|
||||
res.Supported = false
|
||||
res.Reason = "unknown"
|
||||
res.Message = "No variant has a Docker image or Steam app that could be confirmed. " + res.Message
|
||||
}
|
||||
// Exactly one recommendation, among the runnable ones
|
||||
rec := -1
|
||||
for i, v := range res.Variants {
|
||||
if v.Recommended && rec < 0 && len(v.Images) > 0 {
|
||||
rec = i
|
||||
}
|
||||
res.Variants[i].Recommended = false
|
||||
}
|
||||
if rec < 0 {
|
||||
for i, v := range res.Variants {
|
||||
if len(v.Images) > 0 {
|
||||
rec = i
|
||||
break
|
||||
}
|
||||
}
|
||||
}
|
||||
if rec >= 0 {
|
||||
res.Variants[rec].Recommended = true
|
||||
}
|
||||
}
|
||||
|
||||
func hasImage(images []string, name string) bool {
|
||||
for _, i := range images {
|
||||
if i == name || strings.HasPrefix(i, name+":") {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
var appName = regexp.MustCompile(`"common"\s*\{[^}]*?"name"\s+"([^"]+)"`)
|
||||
|
||||
// steamAppNames asks SteamCMD (anonymously, in its official image) for each app's info: an app that
|
||||
// anonymous SteamCMD can't see can't be installed by the node-agent either.
|
||||
func steamAppNames(apps []string) map[string]string {
|
||||
names := map[string]string{}
|
||||
if len(apps) == 0 {
|
||||
return names
|
||||
}
|
||||
for _, app := range apps {
|
||||
if !regexp.MustCompile(`^\d+$`).MatchString(app) {
|
||||
continue
|
||||
}
|
||||
out, err := runCmd("docker", "run", "--rm", "--network", bootTestNetwork, "cm2network/steamcmd", "bash", "-c",
|
||||
"/home/steam/steamcmd/steamcmd.sh +login anonymous +app_info_update 1 +app_info_print "+app+" +quit")
|
||||
if err != nil {
|
||||
fmt.Printf("[research] steamcmd for app %s: %v\n", app, err)
|
||||
continue
|
||||
}
|
||||
if m := appName.FindStringSubmatch(out); m != nil {
|
||||
names[app] = m[1]
|
||||
}
|
||||
}
|
||||
return names
|
||||
}
|
||||
Reference in new issue
Block a user