Files
archie/modules/runs/corpus_test.go
T

267 lines
8.1 KiB
Go

package runs
import (
"errors"
"os"
"slices"
"strings"
"testing"
)
// TestCorpusChecksAsItsStepTypesLand guards the workflow step contract against
// today's workflows: every shipped document and reference rule of the old
// vocabulary either checks clean or is refused for a reason naming what hasn't
// landed yet. A row's refusals shrink as each step type's owner ships, and a
// regression shows as a reference, control or setting that stops checking.
func TestCorpusChecksAsItsStepTypesLand(t *testing.T) {
notYet := func(types ...string) []string {
out := make([]string, 0, len(types))
for _, ty := range types {
out = append(out, `unknown type "`+ty+`"`)
}
return out
}
// Settings agent.run takes from today's vocabulary later on.
agentLater := func(settings ...string) []string {
out := make([]string, 0, len(settings))
for _, s := range settings {
out = append(out, "unknown setting "+s)
}
return out
}
tests := []struct {
name string
file string
yaml string
// want is every problem the document has now; none means it checks.
want []string
}{
{name: "investigate checks", file: "investigate"},
{
name: "feasibility waits for repositories and switch", file: "feasibility",
want: slices.Concat(notYet("repo.prepare"),
[]string{"switch steps are not supported yet"}),
},
{
name: "implement waits for repositories and gates", file: "implement",
want: slices.Concat(notYet("repo.prepare", "repo.tooling", "gate.repository", "repo.commit", "repo.open-pr"),
agentLater("gate")),
},
{name: "pr-review waits for review steps", file: "pr-review", want: notYet("review.pr")},
{
name: "remediate waits for review and repository steps", file: "remediate",
want: slices.Concat(notYet("review.start-round", "repo.prepare", "repo.commit", "review.reply"), agentLater("gate")),
},
{
name: "tdd waits for repositories, gates and protected tests", file: "tdd",
want: slices.Concat(notYet("repo.prepare", "repo.tooling", "gate.repository", "gate.repository", "repo.commit", "repo.commit", "repo.open-pr"),
agentLater("gate", "gate", "protect_tests")),
},
{
name: "triage's references and conditions check; it waits for forge steps", file: "triage",
want: notYet("forge.comment", "forge.close-issue"),
},
{
name: "a finish step with an unknown status is refused when the workflow is saved",
yaml: `steps:
- {type: workflow.finish, settings: {status: finished}}
`,
want: []string{`status is completed, parked, waiting_approval or declined, not "finished"`},
},
{
name: "a call, handoff, approval and message check their settings when the workflow is saved",
yaml: `steps:
- {type: workflow.call, settings: {workflow: w, inputs: {a: {b: 1}}}}
- {type: workflow.call, settings: {workflow: w, outputs: {x: outputs.x}}}
- {type: workflow.call, settings: {workflow: w, inputs: {a: inputs.9}}}
- {type: workflow.handoff, settings: {detail: d}}
- {type: workflow.approve, settings: {plan: p}}
- {type: message.send, settings: {channel: c, chat_id: " ", text: t}}
`,
want: []string{
"not a nested document", "outputs on a call that does not wait can never be read",
`input "a" reference "inputs.9" must name an input as inputs.<name>`,
"workflow is required", "plan or document, and then, are required",
"channel, chat_id and text are required",
},
},
{
name: "a result field the agent's schema doesn't declare is refused",
yaml: `steps:
- id: assess
type: agent.run
settings:
mission: m
result: {type: object, properties: {fit: {type: boolean}}}
- type: agent.run
settings: {mission: "{{ steps.assess.result.fit }} {{ steps.assess.result.verdict }}"}
`,
want: []string{`step "assess"'s result has no field "verdict"`},
},
{
name: "a result schema must describe an object",
yaml: `steps:
- type: agent.run
settings: {mission: m, result: {type: string}}
`,
want: []string{"result: a JSON Schema with type object"},
},
{
name: "an earlier step's summary and result fields resolve",
yaml: `steps:
- id: plan
type: agent.run
settings: {mission: "plan {{ task.title }}"}
- type: agent.run
settings: {mission: "build {{ steps.plan.summary }} / {{ steps.plan.result.verdict }}"}
`,
},
{
name: "a later step cannot be referenced",
yaml: `steps:
- type: agent.run
settings: {mission: "{{ steps.plan.summary }}"}
- id: plan
type: agent.run
settings: {mission: x}
`,
want: []string{`no earlier step has id "plan"`},
},
{
name: "an unknown root is refused",
yaml: `steps:
- type: agent.run
settings: {mission: "{{ secrets.token }}"}
`,
want: []string{"must start with task, inputs or steps"},
},
{
name: "the event is read as an input, not a root of its own",
yaml: `steps:
- type: connector.action
settings: {connector: host, name: clean, input: {host: "{{ event.fields.host }}"}}
`,
want: []string{"must start with task, inputs or steps"},
},
{
name: "a declared event input is read through inputs",
yaml: `inputs: {event: {type: object}}
steps:
- type: connector.action
settings: {connector: host, name: clean, input: {host: "{{ inputs.event.fields.host }}"}}
`,
},
{
name: "a step id with a dot is refused, so a reference reads one way",
yaml: `steps:
- {id: plan.v2, type: run.notify}
`,
want: []string{`step id "plan.v2" must be lowercase letters and digits, joined by hyphens`},
},
{
name: "a duplicate step id is refused",
yaml: `steps:
- {id: a, type: run.notify}
- {id: a, type: run.notify}
`,
want: []string{`step id "a" is declared twice`},
},
{
name: "when may only read earlier steps",
yaml: `steps:
- {type: run.notify, when: "!steps.later.result.fit"}
- {id: later, type: run.notify}
`,
want: []string{`reference {{ steps.later.result.fit }}: no earlier step has id "later"`},
},
{
name: "an undeclared input is refused",
yaml: `steps:
- type: agent.run
settings: {mission: "{{ inputs.ticket }}"}
`,
want: []string{`the workflow declares no input "ticket"`},
},
{
name: "when may only read a declared input",
yaml: `inputs: {ticket: {type: string}}
steps:
- {type: run.notify, when: "inputs.other"}
`,
want: []string{`the workflow declares no input "other"`},
},
{
name: "a task field that doesn't exist is refused",
yaml: `steps:
- type: agent.run
settings: {mission: "{{ task.secret }}"}
`,
want: []string{`task has no field "secret"`},
},
{
name: "on_failure is park or continue",
yaml: `steps:
- {type: run.notify, on_failure: ignore}
`,
want: []string{`park or continue, not "ignore"`},
},
{
name: "outputs are refused until something writes them",
yaml: `outputs: {verdict: {type: string}}
steps:
- type: run.notify
`,
want: []string{"workflow outputs are not supported yet"},
},
{
name: "any number of approval steps and agents",
yaml: `steps:
- {id: a, type: agent.run, settings: {mission: m}}
- {type: approval.wait}
- {id: b, type: agent.run, settings: {mission: m}}
- {type: approval.wait}
`,
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
body := tt.yaml
if tt.file != "" {
raw, err := os.ReadFile("testdata/corpus/" + tt.file + ".yaml")
if err != nil {
t.Fatal(err)
}
body = string(raw)
}
got := problemMessages(t, body)
if len(got) != len(tt.want) {
t.Fatalf("problems:\n %s\nwant %d:\n %s", strings.Join(got, "\n "), len(tt.want), strings.Join(tt.want, "\n "))
}
for _, w := range tt.want {
i := slices.IndexFunc(got, func(g string) bool { return strings.Contains(g, w) })
if i < 0 {
t.Fatalf("no problem says %q in:\n %s", w, strings.Join(got, "\n "))
}
got = slices.Delete(got, i, i+1)
}
})
}
}
func problemMessages(t *testing.T, body string) []string {
t.Helper()
_, err := ParseDocument(body)
if err == nil {
return nil
}
var invalid *InvalidError
if !errors.As(err, &invalid) {
t.Fatalf("ParseDocument: %v", err)
}
out := make([]string, 0, len(invalid.Problems))
for _, p := range invalid.Problems {
out = append(out, p.String())
}
return out
}