cmd/gbrgames/build.go

@@ -235,9 +235,18 @@ func bootCheck(gamesDir, id, agentImage string, wantOutside bool) (problems, ver
 	if err != nil {
 		return nil, nil, err
 	}
-	payload, err := exec.Command(self, "boottest-payload", "-games", gamesDir, id).Output()
+	payloadCmd := exec.Command(self, "boottest-payload", "-games", gamesDir, id)
+	var stderr bytes.Buffer
+	payloadCmd.Stderr = &stderr
+	payload, err := payloadCmd.Output()
 	if err != nil {
-		return nil, nil, fmt.Errorf("boottest-payload: %w", err)
+		var exitErr *exec.ExitError
+		if errors.As(err, &exitErr) && exitErr.ExitCode() == exitMissingSecret {
+			// The builder has no CI secrets: that's the template's problem to fix, not a reason to stop
+			return []string{"the boot test can't run: " + strings.TrimSpace(stderr.String()) + ". Remove 'secrets' from " +
+				"test.yaml; boot tests fill password settings with a throwaway value"}, nil, nil
+		}
+		return nil, nil, fmt.Errorf("boottest-payload: %v: %s", err, strings.TrimSpace(stderr.String()))
 	}
 	if out, err := runCmd("docker", "pull", "-q", agentImage); err != nil {
 		return nil, nil, fmt.Errorf("docker pull %s: %v: %s", agentImage, err, out)

cmd/gbrgames/draft.go

@@ -270,6 +270,11 @@ func draftCompleteness(game, test map[string]any) []string {
 	} else if v, _ := n.Int64(); v < 1024 {
 		problems = append(problems, "requiredMemoryMb is below 1024")
 	}
+	// Secrets only exist as CI settings a person adds; a draft with one can't be boot-tested anywhere
+	if secrets, _ := test["secrets"].(map[string]any); len(secrets) > 0 {
+		problems = append(problems, "test.yaml has 'secrets' ("+strings.Join(sortedKeys(secrets), ", ")+"), which no boot test of a "+
+			"new game has: remove them. Password settings get a throwaway value in boot tests automatically")
+	}
 	if _, ok := game["readyPattern"]; !ok {
 		if _, ok := test["settleSec"]; !ok {
 			problems = append(problems, "no readyPattern and no test.settleSec: boot tests need one of them")

cmd/gbrgames/draft_test.go

@@ -84,3 +84,12 @@ func TestImageUnusable(t *testing.T) {
 		t.Errorf("long run flagged: %q", why)
 	}
 }
+
+func TestDraftCompletenessRejectsSecrets(t *testing.T) {
+	// The Valheim draft that couldn't be boot-tested: the password mapped to a CI secret nobody created
+	test := map[string]any{"settleSec": json.Number("120"), "secrets": map[string]any{"Password": "VALHEIM_TEST_PASSWORD"}}
+	problems := strings.Join(draftCompleteness(map[string]any{}, test), "\n")
+	if !strings.Contains(problems, "test.yaml has 'secrets' (Password)") {
+		t.Errorf("got:\n%s", problems)
+	}
+}

cmd/gbrgames/validate.go

@@ -168,7 +168,8 @@ func validateGame(dir, folder string, gameSchema, testSchema *jsonschema.Schema)
 			add("test.yaml config.%s is not a key of this game", key)
 		}
 		if sensitiveKey.MatchString(key) {
-			add("test.yaml config.%s looks like a credential; map it under 'secrets' to a Gitea secret instead", key)
+			add("test.yaml config.%s looks like a credential; remove it (boot tests fill password settings with a throwaway "+
+				"value), or, only if the server can't start without a real one, map it under 'secrets' to a CI secret", key)
 		}
 	}
 	for _, key := range sortedKeys(secrets) {