Also fix repository-free message.send validation (#2046) and add a single-binary development smoke path (#1977). Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com>
236 lines
6.8 KiB
Go
236 lines
6.8 KiB
Go
package runs
|
|
|
|
import (
|
|
"errors"
|
|
"os"
|
|
"slices"
|
|
"strings"
|
|
"testing"
|
|
)
|
|
|
|
// TestCorpusChecksAsItsStepTypesLand guards the workflow step contract against
|
|
// today's workflows: every shipped document and reference rule of the old
|
|
// vocabulary either checks clean or is refused for a reason naming what hasn't
|
|
// landed yet. A row's refusals shrink as each step type's owner ships, and a
|
|
// regression shows as a reference, control or setting that stops checking.
|
|
func TestCorpusChecksAsItsStepTypesLand(t *testing.T) {
|
|
notYet := func(types ...string) []string {
|
|
out := make([]string, 0, len(types))
|
|
for _, ty := range types {
|
|
out = append(out, `unknown type "`+ty+`"`)
|
|
}
|
|
return out
|
|
}
|
|
// Settings agent.run takes from today's vocabulary later on.
|
|
agentLater := func(settings ...string) []string {
|
|
out := make([]string, 0, len(settings))
|
|
for _, s := range settings {
|
|
out = append(out, "unknown setting "+s)
|
|
}
|
|
return out
|
|
}
|
|
tests := []struct {
|
|
name string
|
|
file string
|
|
yaml string
|
|
// want is every problem the document has now; none means it checks.
|
|
want []string
|
|
}{
|
|
{name: "investigate checks", file: "investigate"},
|
|
{
|
|
name: "feasibility waits for repositories, switch and approve", file: "feasibility",
|
|
want: slices.Concat(notYet("repo.prepare", "workflow.approve"),
|
|
[]string{"switch steps are not supported yet"}),
|
|
},
|
|
{
|
|
name: "implement waits for repositories and gates", file: "implement",
|
|
want: slices.Concat(notYet("repo.prepare", "repo.tooling", "gate.repository", "repo.commit", "repo.open-pr"),
|
|
agentLater("gate")),
|
|
},
|
|
{name: "pr-review waits for review steps", file: "pr-review", want: notYet("review.pr")},
|
|
{
|
|
name: "remediate waits for review and repository steps", file: "remediate",
|
|
want: slices.Concat(notYet("review.start-round", "repo.prepare", "repo.commit", "review.reply"), agentLater("gate")),
|
|
},
|
|
{
|
|
name: "tdd waits for repositories, gates and protected tests", file: "tdd",
|
|
want: slices.Concat(notYet("repo.prepare", "repo.tooling", "gate.repository", "gate.repository", "repo.commit", "repo.commit", "repo.open-pr"),
|
|
agentLater("gate", "gate", "protect_tests")),
|
|
},
|
|
{
|
|
name: "triage's references and conditions check; it waits for forge and outcome steps", file: "triage",
|
|
want: notYet("forge.comment", "forge.close-issue", "workflow.finish", "workflow.handoff"),
|
|
},
|
|
{
|
|
name: "a result field the agent's schema doesn't declare is refused",
|
|
yaml: `steps:
|
|
- id: assess
|
|
type: agent.run
|
|
settings:
|
|
mission: m
|
|
result: {type: object, properties: {fit: {type: boolean}}}
|
|
- type: agent.run
|
|
settings: {mission: "{{ steps.assess.result.fit }} {{ steps.assess.result.verdict }}"}
|
|
`,
|
|
want: []string{`step "assess"'s result has no field "verdict"`},
|
|
},
|
|
{
|
|
name: "a result schema must describe an object",
|
|
yaml: `steps:
|
|
- type: agent.run
|
|
settings: {mission: m, result: {type: string}}
|
|
`,
|
|
want: []string{"result: a JSON Schema with type object"},
|
|
},
|
|
{
|
|
name: "an earlier step's summary and result fields resolve",
|
|
yaml: `steps:
|
|
- id: plan
|
|
type: agent.run
|
|
settings: {mission: "plan {{ task.title }}"}
|
|
- type: agent.run
|
|
settings: {mission: "build {{ steps.plan.summary }} / {{ steps.plan.result.verdict }}"}
|
|
`,
|
|
},
|
|
{
|
|
name: "a later step cannot be referenced",
|
|
yaml: `steps:
|
|
- type: agent.run
|
|
settings: {mission: "{{ steps.plan.summary }}"}
|
|
- id: plan
|
|
type: agent.run
|
|
settings: {mission: x}
|
|
`,
|
|
want: []string{`no earlier step has id "plan"`},
|
|
},
|
|
{
|
|
name: "an unknown root is refused",
|
|
yaml: `steps:
|
|
- type: agent.run
|
|
settings: {mission: "{{ secrets.token }}"}
|
|
`,
|
|
want: []string{"must start with task, inputs or steps"},
|
|
},
|
|
{
|
|
name: "the event is read as an input, not a root of its own",
|
|
yaml: `steps:
|
|
- type: connector.action
|
|
settings: {connector: host, name: clean, input: {host: "{{ event.fields.host }}"}}
|
|
`,
|
|
want: []string{"must start with task, inputs or steps"},
|
|
},
|
|
{
|
|
name: "a declared event input is read through inputs",
|
|
yaml: `inputs: {event: {type: object}}
|
|
steps:
|
|
- type: connector.action
|
|
settings: {connector: host, name: clean, input: {host: "{{ inputs.event.fields.host }}"}}
|
|
`,
|
|
},
|
|
{
|
|
name: "a duplicate step id is refused",
|
|
yaml: `steps:
|
|
- {id: a, type: run.notify}
|
|
- {id: a, type: run.notify}
|
|
`,
|
|
want: []string{`step id "a" is declared twice`},
|
|
},
|
|
{
|
|
name: "when may only read earlier steps",
|
|
yaml: `steps:
|
|
- {type: run.notify, when: "!steps.later.result.fit"}
|
|
- {id: later, type: run.notify}
|
|
`,
|
|
want: []string{`reference {{ steps.later.result.fit }}: no earlier step has id "later"`},
|
|
},
|
|
{
|
|
name: "an undeclared input is refused",
|
|
yaml: `steps:
|
|
- type: agent.run
|
|
settings: {mission: "{{ inputs.ticket }}"}
|
|
`,
|
|
want: []string{`the workflow declares no input "ticket"`},
|
|
},
|
|
{
|
|
name: "when may only read a declared input",
|
|
yaml: `inputs: {ticket: {type: string}}
|
|
steps:
|
|
- {type: run.notify, when: "inputs.other"}
|
|
`,
|
|
want: []string{`the workflow declares no input "other"`},
|
|
},
|
|
{
|
|
name: "a task field that doesn't exist is refused",
|
|
yaml: `steps:
|
|
- type: agent.run
|
|
settings: {mission: "{{ task.secret }}"}
|
|
`,
|
|
want: []string{`task has no field "secret"`},
|
|
},
|
|
{
|
|
name: "on_failure is park or continue",
|
|
yaml: `steps:
|
|
- {type: run.notify, on_failure: ignore}
|
|
`,
|
|
want: []string{`park or continue, not "ignore"`},
|
|
},
|
|
{
|
|
name: "outputs are refused until something writes them",
|
|
yaml: `outputs: {verdict: {type: string}}
|
|
steps:
|
|
- type: run.notify
|
|
`,
|
|
want: []string{"workflow outputs are not supported yet"},
|
|
},
|
|
{
|
|
name: "any number of approval steps and agents",
|
|
yaml: `steps:
|
|
- {id: a, type: agent.run, settings: {mission: m}}
|
|
- {type: approval.wait}
|
|
- {id: b, type: agent.run, settings: {mission: m}}
|
|
- {type: approval.wait}
|
|
`,
|
|
},
|
|
}
|
|
for _, tt := range tests {
|
|
t.Run(tt.name, func(t *testing.T) {
|
|
body := tt.yaml
|
|
if tt.file != "" {
|
|
raw, err := os.ReadFile("testdata/corpus/" + tt.file + ".yaml")
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
body = string(raw)
|
|
}
|
|
got := problemMessages(t, body)
|
|
if len(got) != len(tt.want) {
|
|
t.Fatalf("problems:\n %s\nwant %d:\n %s", strings.Join(got, "\n "), len(tt.want), strings.Join(tt.want, "\n "))
|
|
}
|
|
for _, w := range tt.want {
|
|
i := slices.IndexFunc(got, func(g string) bool { return strings.Contains(g, w) })
|
|
if i < 0 {
|
|
t.Fatalf("no problem says %q in:\n %s", w, strings.Join(got, "\n "))
|
|
}
|
|
got = slices.Delete(got, i, i+1)
|
|
}
|
|
})
|
|
}
|
|
}
|
|
|
|
func problemMessages(t *testing.T, body string) []string {
|
|
t.Helper()
|
|
_, err := ParseDocument(body)
|
|
if err == nil {
|
|
return nil
|
|
}
|
|
var invalid *InvalidError
|
|
if !errors.As(err, &invalid) {
|
|
t.Fatalf("ParseDocument: %v", err)
|
|
}
|
|
out := make([]string, 0, len(invalid.Problems))
|
|
for _, p := range invalid.Problems {
|
|
out = append(out, p.String())
|
|
}
|
|
return out
|
|
}
|