Files
archie/modules/runs/corpus_test.go
T
samandCopilot 64dfad6d53 feat(runs): add typed results and model aliases (#2043)
Also fix repository-free message.send validation (#2046) and add a single-binary development smoke path (#1977).

Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com>
2026-11-18 23:47:35 +11:00

236 lines
6.8 KiB
Go

package runs
import (
"errors"
"os"
"slices"
"strings"
"testing"
)
// TestCorpusChecksAsItsStepTypesLand guards the workflow step contract against
// today's workflows: every shipped document and reference rule of the old
// vocabulary either checks clean or is refused for a reason naming what hasn't
// landed yet. A row's refusals shrink as each step type's owner ships, and a
// regression shows as a reference, control or setting that stops checking.
func TestCorpusChecksAsItsStepTypesLand(t *testing.T) {
notYet := func(types ...string) []string {
out := make([]string, 0, len(types))
for _, ty := range types {
out = append(out, `unknown type "`+ty+`"`)
}
return out
}
// Settings agent.run takes from today's vocabulary later on.
agentLater := func(settings ...string) []string {
out := make([]string, 0, len(settings))
for _, s := range settings {
out = append(out, "unknown setting "+s)
}
return out
}
tests := []struct {
name string
file string
yaml string
// want is every problem the document has now; none means it checks.
want []string
}{
{name: "investigate checks", file: "investigate"},
{
name: "feasibility waits for repositories, switch and approve", file: "feasibility",
want: slices.Concat(notYet("repo.prepare", "workflow.approve"),
[]string{"switch steps are not supported yet"}),
},
{
name: "implement waits for repositories and gates", file: "implement",
want: slices.Concat(notYet("repo.prepare", "repo.tooling", "gate.repository", "repo.commit", "repo.open-pr"),
agentLater("gate")),
},
{name: "pr-review waits for review steps", file: "pr-review", want: notYet("review.pr")},
{
name: "remediate waits for review and repository steps", file: "remediate",
want: slices.Concat(notYet("review.start-round", "repo.prepare", "repo.commit", "review.reply"), agentLater("gate")),
},
{
name: "tdd waits for repositories, gates and protected tests", file: "tdd",
want: slices.Concat(notYet("repo.prepare", "repo.tooling", "gate.repository", "gate.repository", "repo.commit", "repo.commit", "repo.open-pr"),
agentLater("gate", "gate", "protect_tests")),
},
{
name: "triage's references and conditions check; it waits for forge and outcome steps", file: "triage",
want: notYet("forge.comment", "forge.close-issue", "workflow.finish", "workflow.handoff"),
},
{
name: "a result field the agent's schema doesn't declare is refused",
yaml: `steps:
- id: assess
type: agent.run
settings:
mission: m
result: {type: object, properties: {fit: {type: boolean}}}
- type: agent.run
settings: {mission: "{{ steps.assess.result.fit }} {{ steps.assess.result.verdict }}"}
`,
want: []string{`step "assess"'s result has no field "verdict"`},
},
{
name: "a result schema must describe an object",
yaml: `steps:
- type: agent.run
settings: {mission: m, result: {type: string}}
`,
want: []string{"result: a JSON Schema with type object"},
},
{
name: "an earlier step's summary and result fields resolve",
yaml: `steps:
- id: plan
type: agent.run
settings: {mission: "plan {{ task.title }}"}
- type: agent.run
settings: {mission: "build {{ steps.plan.summary }} / {{ steps.plan.result.verdict }}"}
`,
},
{
name: "a later step cannot be referenced",
yaml: `steps:
- type: agent.run
settings: {mission: "{{ steps.plan.summary }}"}
- id: plan
type: agent.run
settings: {mission: x}
`,
want: []string{`no earlier step has id "plan"`},
},
{
name: "an unknown root is refused",
yaml: `steps:
- type: agent.run
settings: {mission: "{{ secrets.token }}"}
`,
want: []string{"must start with task, inputs or steps"},
},
{
name: "the event is read as an input, not a root of its own",
yaml: `steps:
- type: connector.action
settings: {connector: host, name: clean, input: {host: "{{ event.fields.host }}"}}
`,
want: []string{"must start with task, inputs or steps"},
},
{
name: "a declared event input is read through inputs",
yaml: `inputs: {event: {type: object}}
steps:
- type: connector.action
settings: {connector: host, name: clean, input: {host: "{{ inputs.event.fields.host }}"}}
`,
},
{
name: "a duplicate step id is refused",
yaml: `steps:
- {id: a, type: run.notify}
- {id: a, type: run.notify}
`,
want: []string{`step id "a" is declared twice`},
},
{
name: "when may only read earlier steps",
yaml: `steps:
- {type: run.notify, when: "!steps.later.result.fit"}
- {id: later, type: run.notify}
`,
want: []string{`reference {{ steps.later.result.fit }}: no earlier step has id "later"`},
},
{
name: "an undeclared input is refused",
yaml: `steps:
- type: agent.run
settings: {mission: "{{ inputs.ticket }}"}
`,
want: []string{`the workflow declares no input "ticket"`},
},
{
name: "when may only read a declared input",
yaml: `inputs: {ticket: {type: string}}
steps:
- {type: run.notify, when: "inputs.other"}
`,
want: []string{`the workflow declares no input "other"`},
},
{
name: "a task field that doesn't exist is refused",
yaml: `steps:
- type: agent.run
settings: {mission: "{{ task.secret }}"}
`,
want: []string{`task has no field "secret"`},
},
{
name: "on_failure is park or continue",
yaml: `steps:
- {type: run.notify, on_failure: ignore}
`,
want: []string{`park or continue, not "ignore"`},
},
{
name: "outputs are refused until something writes them",
yaml: `outputs: {verdict: {type: string}}
steps:
- type: run.notify
`,
want: []string{"workflow outputs are not supported yet"},
},
{
name: "any number of approval steps and agents",
yaml: `steps:
- {id: a, type: agent.run, settings: {mission: m}}
- {type: approval.wait}
- {id: b, type: agent.run, settings: {mission: m}}
- {type: approval.wait}
`,
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
body := tt.yaml
if tt.file != "" {
raw, err := os.ReadFile("testdata/corpus/" + tt.file + ".yaml")
if err != nil {
t.Fatal(err)
}
body = string(raw)
}
got := problemMessages(t, body)
if len(got) != len(tt.want) {
t.Fatalf("problems:\n %s\nwant %d:\n %s", strings.Join(got, "\n "), len(tt.want), strings.Join(tt.want, "\n "))
}
for _, w := range tt.want {
i := slices.IndexFunc(got, func(g string) bool { return strings.Contains(g, w) })
if i < 0 {
t.Fatalf("no problem says %q in:\n %s", w, strings.Join(got, "\n "))
}
got = slices.Delete(got, i, i+1)
}
})
}
}
func problemMessages(t *testing.T, body string) []string {
t.Helper()
_, err := ParseDocument(body)
if err == nil {
return nil
}
var invalid *InvalidError
if !errors.As(err, &invalid) {
t.Fatalf("ParseDocument: %v", err)
}
out := make([]string, 0, len(invalid.Problems))
for _, p := range invalid.Problems {
out = append(out, p.String())
}
return out
}