Files
archie/modules/runs/steps_test.go
T

546 lines
20 KiB
Go

package runs
import (
"cmp"
"context"
"encoding/json/jsontext"
json "encoding/json/v2"
"errors"
"slices"
"strings"
"testing"
"github.com/google/uuid"
"github.com/samcharles93/ai-sdk/chat"
"github.com/samcharles93/archie-core/modules/approvals"
"github.com/samcharles93/archie-core/modules/deliveries"
"github.com/samcharles93/archie-core/modules/governance"
"github.com/samcharles93/archie-core/modules/judgement"
"github.com/samcharles93/archie-core/modules/runs/internal/store"
)
// actionJudge decides each action at the level named for it and remembers
// every request, so a test sees what a step asked for.
type actionJudge struct {
levels map[string]judgement.Level
fail error
// failAction limits fail to the action with this name; empty fails them all.
failAction string
assessed []judgement.Request
decisions []judgement.Decision
executed int
}
func (j *actionJudge) Assess(_ context.Context, req judgement.Request) (judgement.Decision, error) {
if j.fail != nil && (j.failAction == "" || j.failAction == req.Action) {
return judgement.Decision{}, j.fail
}
j.assessed = append(j.assessed, req)
d := judgement.Decision{Key: req.Key, Level: j.levels[req.Action], Action: judgement.Action{
Connector: req.Connector, Name: req.Action, Input: req.Input, Digest: req.Action,
}}
j.decisions = append(j.decisions, d)
return d, nil
}
func (j *actionJudge) Execute(context.Context, judgement.Decision, jsontext.Value, string) (string, error) {
j.executed++
return "done", nil
}
func (j *actionJudge) ExecuteApproved(context.Context, judgement.Decision, judgement.Approval, jsontext.Value, string) (string, error) {
j.executed++
return "done", nil
}
func (j *actionJudge) ForRun(context.Context, uuid.UUID) ([]judgement.Decision, error) {
return j.decisions, nil
}
// askCounter approves every request at once and counts them. failFirst makes
// that many requests fail first, as an approvals store that is briefly down.
type askCounter struct {
asked int
failFirst int
}
func (a *askCounter) Ask(_ context.Context, _ uuid.UUID, _ string, d judgement.Decision, _ []governance.PathStep) (approvals.Request, error) {
a.asked++
if a.failFirst > 0 {
a.failFirst--
return approvals.Request{}, errors.New("approvals unavailable")
}
return approvals.Request{ID: uuid.New(), State: approvals.StatePending, Decision: d}, nil
}
func (a *askCounter) Escalate(context.Context, uuid.UUID, int, governance.PathStep) error { return nil }
func (a *askCounter) Expire(context.Context, uuid.UUID) error { return nil }
func (a *askCounter) Accept(context.Context, uuid.UUID) error { return nil }
func (a *askCounter) Discard(context.Context, uuid.UUID, string) error { return nil }
// onePersonPath sends every request to one person.
type onePersonPath struct{}
func (onePersonPath) AgentModelForAlias(context.Context, string) (governance.Model, error) {
return governance.Model{}, errors.New("no agent in these workflows")
}
func (onePersonPath) AgentProvider(context.Context, governance.Model) (chat.Provider, chat.ModelInfo, error) {
return nil, chat.ModelInfo{}, errors.New("no agent in these workflows")
}
func (onePersonPath) EscalationPath(context.Context, string, bool) ([]governance.PathStep, error) {
return []governance.PathStep{{Person: "p"}}, nil
}
func (onePersonPath) RecurrenceThreshold(context.Context) (int, error) { return 0, nil }
func (onePersonPath) AttemptsPerRun(context.Context) (int, error) { return 1, nil }
type notices struct{ sent []deliveries.Notification }
func (n *notices) Notify(_ context.Context, msg deliveries.Notification) error {
n.sent = append(n.sent, msg)
return nil
}
// chats records the messages a workflow sends.
type chats struct{ sent []string }
func (c *chats) Send(_ context.Context, channel, chatID, text, _ string) error {
c.sent = append(c.sent, channel+"/"+chatID+": "+text)
return nil
}
// callWork is a worker store whose started child runs end when a test says
// so: until ending is set, a call waiting on its child has nothing to read.
type callWork struct {
signalAfter
ending jsontext.Value
}
func (c *callWork) Signalled(ctx context.Context, id uuid.UUID, key string) (jsontext.Value, bool, error) {
if !strings.HasPrefix(key, callSignalPrefix) {
return c.signalAfter.Signalled(ctx, id, key)
}
return c.ending, c.ending != nil, nil
}
const callSignalPrefix = "call:"
// TestDocumentStepsRun guards the workflow step contract on the engine: a
// step's settings are rendered from the run's inputs and earlier results, its
// condition decides whether it runs, a failure ends the run unless the step
// continues, and every action judgement holds is put to people exactly once,
// by an approval step or before the run ends. The first step that sets an
// outcome ends the run, a failed step parks it, and a parked run retried from a
// step keeps the results of the steps before it. A call starts one child run
// however often the run restarts, and a waiting call resumes from the child's
// ending; a handoff starts the next workflow on the same input; an approval
// delivers the plan and leaves the run waiting; a message is sent once, and
// not at all in a dry run.
func TestDocumentStepsRun(t *testing.T) {
input := jsontext.Value(`{"event":{"fields":{"host":"web-1"}}}`)
tests := []struct {
name string
doc string
levels map[string]judgement.Level
fail error
failAction string
failFirst int
// retry retries the parked run from the named step once the failure
// has cleared; the counts below cover both attempts.
retry string
wantState string
wantDetail string
wantInput string
wantAssessed int
wantAsked int
wantExecuted int
wantNotice string
// dry runs the workflow as a dry run.
dry bool
// bodies are the stored workflows a step can start.
bodies map[string]string
// crash kills the worker as it records its first step, so the run
// restarts having done that step's effect without recording it.
crash bool
// childEnding is how the started child ends: the run is expected to
// wait for it, and a restart then finds it ended.
childEnding string
// wantStarted is the child runs started, in all attempts, and
// wantChildInput the first one's input.
wantStarted int
wantChildInput string
// wantSignal is the key the first child run signals when it ends.
wantSignal string
wantSent []string
}{
{
name: "an action's input is rendered from the run's inputs",
doc: `inputs: {event: {type: object}}
steps:
- {type: connector.action, settings: {connector: host, name: clean, input: {host: "{{ inputs.event.fields.host }}"}}}
`,
levels: map[string]judgement.Level{"clean": judgement.ActQuietly},
wantInput: `{"host":"web-1"}`, wantAssessed: 1, wantExecuted: 1,
},
{
name: "a step whose condition doesn't hold is skipped",
doc: `inputs: {event: {type: object}}
steps:
- {type: connector.action, when: "inputs.event.fields.missing", settings: {connector: host, name: clean}}
`,
},
{
name: "a failing step parks the run and later steps do not run",
doc: `steps:
- {type: connector.action, settings: {connector: host, name: clean}}
- {type: run.notify}
`,
fail: errors.New("judgement is down"), wantState: store.StateParked, wantDetail: "judgement is down",
},
{
name: "a retry from a step keeps the results of the steps before it",
doc: `steps:
- {id: first, type: connector.action, settings: {connector: host, name: clean}}
- {id: second, type: connector.action, settings: {connector: host, name: restart}}
- {type: run.notify, settings: {title: "{{ steps.first.summary }} then {{ steps.second.summary }}"}}
`,
levels: map[string]judgement.Level{"clean": judgement.ActQuietly, "restart": judgement.ActQuietly},
fail: errors.New("judgement is down"), failAction: "restart", retry: "second",
wantState: store.StateSucceeded, wantAssessed: 2, wantExecuted: 2, wantNotice: "done then done",
},
{
name: "a finish step ends the run with its status and detail",
doc: `steps:
- {type: workflow.finish, settings: {status: declined, detail: not worth fixing}}
- {type: run.notify, settings: {title: too late}}
`,
wantState: store.StateDeclined, wantDetail: "not worth fixing",
},
{
name: "a finish step whose condition does not hold lets the run go on",
doc: `inputs: {event: {type: object}}
steps:
- {type: workflow.finish, when: "inputs.event.fields.missing", settings: {status: parked, detail: stop}}
- {type: run.notify, settings: {title: carried on}}
`,
wantNotice: "carried on",
},
{
name: "a finish step can leave the run waiting for approval",
doc: `steps:
- {type: workflow.finish, settings: {status: waiting_approval, detail: needs a person}}
`,
wantState: store.StateWaitingApproval, wantDetail: "needs a person",
},
{
name: "a finish step may name the steps before it",
doc: `steps:
- {id: clean, type: connector.action, settings: {connector: host, name: clean}}
- {type: workflow.finish, settings: {detail: "cleaned: {{ steps.clean.summary }}"}}
`,
levels: map[string]judgement.Level{"clean": judgement.ActQuietly},
wantAssessed: 1, wantExecuted: 1, wantState: store.StateSucceeded, wantDetail: "cleaned: done",
},
{
name: "a run that reaches the end of its steps completes with the last summary",
doc: `steps:
- {id: clean, type: connector.action, settings: {connector: host, name: clean}}
`,
levels: map[string]judgement.Level{"clean": judgement.ActQuietly},
wantAssessed: 1, wantExecuted: 1, wantState: store.StateSucceeded, wantDetail: "done",
},
{
name: "a failing step that continues lets the run go on",
doc: `steps:
- {type: connector.action, on_failure: continue, settings: {connector: host, name: clean}}
- {type: run.notify, settings: {title: carried on}}
`,
fail: errors.New("judgement is down"), wantNotice: "carried on",
},
{
name: "an action held after the last approval step is put to people before the run ends",
doc: `steps:
- {type: connector.action, settings: {connector: host, name: clean}}
`,
levels: map[string]judgement.Level{"clean": judgement.ProposeAndWait},
wantAssessed: 2, wantAsked: 1, wantExecuted: 1,
},
{
name: "a held action is settled when the last approval step is skipped",
doc: `inputs: {skip: {type: bool}}
steps:
- {type: connector.action, settings: {connector: host, name: clean}}
- {type: approval.wait, when: "inputs.skip"}
`,
levels: map[string]judgement.Level{"clean": judgement.ProposeAndWait},
wantAssessed: 2, wantAsked: 1, wantExecuted: 1,
},
{
name: "an approval that fails and continues is settled before the run ends",
doc: `steps:
- {type: connector.action, settings: {connector: host, name: clean}}
- {type: approval.wait, on_failure: continue}
`,
levels: map[string]judgement.Level{"clean": judgement.ProposeAndWait},
failFirst: 1,
wantAssessed: 2, wantAsked: 2, wantExecuted: 1,
},
{
name: "a later approval step asks only about what was held since the last",
doc: `steps:
- {type: connector.action, settings: {connector: host, name: clean}}
- {type: approval.wait}
- {type: connector.action, settings: {connector: host, name: restart}}
- {type: approval.wait}
`,
levels: map[string]judgement.Level{"clean": judgement.ProposeAndWait, "restart": judgement.ProposeAndWait},
wantAssessed: 4, wantAsked: 2, wantExecuted: 2,
},
{
name: "a call that does not wait starts a child with its inputs and moves on",
doc: `inputs: {event: {type: object}}
steps:
- {id: child, type: workflow.call, settings: {workflow: remediate, inputs: {event: inputs.event, mode: fast}}}
- {type: run.notify, settings: {title: carried on}}
`,
bodies: map[string]string{"remediate": "inputs: {event: {type: object}, mode: {type: string}}\nsteps:\n - type: run.notify\n"},
wantStarted: 1, wantChildInput: `{"event":{"fields":{"host":"web-1"}},"mode":"fast"}`, wantSignal: "call:child", wantNotice: "carried on",
},
{
name: "a waiting call resumes after a restart without starting a second child",
doc: `steps:
- {id: child, type: workflow.call, settings: {workflow: remediate, wait: true}}
- {type: run.notify, settings: {title: "{{ steps.child.result.verdict }}"}}
`,
childEnding: `{"state":"succeeded","detail":"","error":"","result":{"verdict":"fixed"}}`,
wantStarted: 1, wantNotice: "fixed",
},
{
name: "a call restarted after the worker died starts the child once",
doc: `steps:
- {id: child, type: workflow.call, settings: {workflow: remediate, wait: true}}
- {type: run.notify, settings: {title: "{{ steps.child.result.verdict }}"}}
`,
crash: true,
childEnding: `{"state":"succeeded","detail":"","error":"","result":{"verdict":"fixed"}}`,
wantStarted: 1, wantNotice: "fixed",
},
{
name: "a waiting call parks the run when the child parks",
doc: `steps:
- {id: child, type: workflow.call, settings: {workflow: remediate, wait: true}}
- {type: run.notify, settings: {title: too late}}
`,
childEnding: `{"state":"parked","detail":"needs an operator","error":"","result":null}`,
wantStarted: 1, wantState: store.StateParked, wantDetail: "ended parked: needs an operator",
},
{
name: "a call is not started in a dry run",
doc: `steps:
- {type: workflow.call, settings: {workflow: remediate, wait: true}}
`,
dry: true,
},
{
name: "a handoff ends the run and starts the next workflow on the same input",
doc: `inputs: {event: {type: object}}
steps:
- {type: workflow.handoff, settings: {workflow: remediate, detail: needs a fix}}
- {type: run.notify, settings: {title: too late}}
`,
bodies: map[string]string{"remediate": "inputs: {event: {type: object}}\nsteps:\n - type: run.notify\n"},
wantStarted: 1, wantChildInput: `{"event":{"fields":{"host":"web-1"}}}`, wantDetail: "needs a fix",
},
{
name: "a handoff whose condition does not hold starts nothing",
doc: `inputs: {event: {type: object}}
steps:
- {type: workflow.handoff, when: "inputs.event.fields.missing", settings: {workflow: remediate}}
- {type: run.notify, settings: {title: stayed}}
`,
wantNotice: "stayed",
},
{
name: "an approval delivers the plan and leaves the run waiting for approval",
doc: `steps:
- {type: workflow.approve, settings: {plan: restart the host, then: remediate}}
- {type: run.notify, settings: {title: too late}}
`,
wantState: store.StateWaitingApproval, wantDetail: "awaiting go/no-go", wantNotice: "Plan awaiting approval",
},
{
name: "an approval needs a plan",
doc: `inputs: {event: {type: object}}
steps:
- {type: workflow.approve, settings: {document: "{{ inputs.event.fields.missing }}", then: remediate}}
`,
wantState: store.StateParked, wantDetail: "the plan is empty",
},
{
name: "a message is sent with its text rendered",
doc: `inputs: {event: {type: object}}
steps:
- {type: message.send, settings: {channel: chat, chat_id: ops, text: "cleaning {{ inputs.event.fields.host }}"}}
`,
wantSent: []string{"chat/ops: cleaning web-1"},
},
{
name: "a message is not sent in a dry run",
doc: `steps:
- {type: message.send, settings: {channel: chat, chat_id: ops, text: hello}}
`,
dry: true,
},
{
name: "a later step reads an earlier step's result",
doc: `steps:
- {id: clean, type: connector.action, settings: {connector: host, name: clean}}
- {type: run.notify, settings: {title: "cleaned: {{ steps.clean.summary }}"}}
`,
levels: map[string]judgement.Level{"clean": judgement.ActQuietly},
wantAssessed: 1, wantExecuted: 1, wantNotice: "cleaned: done",
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
doc, err := ParseDocument(tt.doc)
if err != nil {
t.Fatalf("ParseDocument: %v", err)
}
judge := &actionJudge{levels: tt.levels, fail: tt.fail, failAction: tt.failAction}
ask := &askCounter{failFirst: tt.failFirst}
told := &notices{}
sent := &chats{}
wf := Deps{Settings: onePersonPath{}, Toolbox: anyBox{}, Judge: judge, Approver: ask, Notifier: told, Messenger: sent}.Compile(doc)
work := &callWork{signalAfter: signalAfter{arrives: 1}}
work.bodies = tt.bodies
run := store.Run{ID: uuid.New(), Dry: tt.dry}
rc := newContext(t.Context(), work, run, uuid.New(), nil)
restart := func() {
rc = newContext(t.Context(), work, run, uuid.New(), slices.Clone(work.steps))
}
if tt.crash {
first := 1
work.rejectFrom = &first
}
_, err = wf(rc, input)
if tt.crash {
if !errors.Is(err, store.ErrLeaseLost) {
t.Fatalf("first run error = %v, want the lease lost", err)
}
work.rejectFrom = nil
restart()
_, err = wf(rc, input)
}
if tt.childEnding != "" {
if !errors.Is(err, errSuspended) {
t.Fatalf("waiting run error = %v, want it suspended", err)
}
work.ending = jsontext.Value(tt.childEnding)
restart()
_, err = wf(rc, input)
}
if err != nil {
t.Fatalf("run error = %v", err)
}
if tt.retry != "" {
if rc.ending == nil || rc.ending.State != store.StateParked {
t.Fatalf("ending = %+v, want the run parked before its retry", rc.ending)
}
judge.fail = nil
kept := stepsBeforeRetry(t, work.steps, tt.retry)
rc = newContext(t.Context(), work, run, uuid.New(), kept)
work.steps = kept
if _, err = wf(rc, input); err != nil {
t.Fatalf("retry error = %v", err)
}
}
wantState := cmp.Or(tt.wantState, store.StateSucceeded)
if rc.ending == nil || rc.ending.State != wantState || !strings.Contains(rc.ending.Detail, tt.wantDetail) {
t.Fatalf("ending = %+v, want %s with detail containing %q", rc.ending, wantState, tt.wantDetail)
}
if len(judge.assessed) != tt.wantAssessed || ask.asked != tt.wantAsked || judge.executed != tt.wantExecuted {
t.Fatalf("assessed %d, asked %d, executed %d; want %d, %d, %d",
len(judge.assessed), ask.asked, judge.executed, tt.wantAssessed, tt.wantAsked, tt.wantExecuted)
}
if tt.wantInput != "" {
var got, want any
if err := json.Unmarshal(judge.assessed[0].Input, &got); err != nil {
t.Fatal(err)
}
if err := json.Unmarshal([]byte(tt.wantInput), &want); err != nil {
t.Fatal(err)
}
if a, b := mustJSON(t, got), mustJSON(t, want); a != b {
t.Fatalf("action input = %s, want %s", a, b)
}
}
if len(work.started) != tt.wantStarted {
t.Fatalf("started %d runs, want %d", len(work.started), tt.wantStarted)
}
if tt.wantChildInput != "" {
var got, want any
if err := json.Unmarshal(work.started[0].Input, &got); err != nil {
t.Fatal(err)
}
if err := json.Unmarshal([]byte(tt.wantChildInput), &want); err != nil {
t.Fatal(err)
}
if a, b := mustJSON(t, got), mustJSON(t, want); a != b {
t.Fatalf("child input = %s, want %s", a, b)
}
}
if tt.wantSignal != "" && work.started[0].ParentSignal != tt.wantSignal {
t.Fatalf("child signals %q, want %q", work.started[0].ParentSignal, tt.wantSignal)
}
if !slices.Equal(sent.sent, tt.wantSent) {
t.Fatalf("messages = %q, want %q", sent.sent, tt.wantSent)
}
if tt.wantNotice != "" && (len(told.sent) != 1 || told.sent[0].Title != tt.wantNotice) {
t.Fatalf("notices = %+v, want one titled %q", told.sent, tt.wantNotice)
}
})
}
}
// stepsBeforeRetry is what a retry from the named step keeps: the steps
// recorded before it began, and the markers written before it was entered. It
// mirrors store.Retry.
func stepsBeforeRetry(t *testing.T, steps []store.Step, from string) []store.Step {
t.Helper()
first, marker := -1, -1
for _, s := range steps {
if s.Name == enteredPrefix+from {
marker = s.Seq
if err := json.Unmarshal(s.Output, &first); err != nil {
t.Fatal(err)
}
}
}
if marker < 0 {
t.Fatalf("the run never reached step %q", from)
}
var kept []store.Step
for _, s := range steps {
if (s.Seq < first) || (s.Seq >= markerBase && s.Seq < marker) {
kept = append(kept, s)
}
}
return kept
}
func mustJSON(t *testing.T, v any) string {
t.Helper()
b, err := json.Marshal(v, json.Deterministic(true))
if err != nil {
t.Fatal(err)
}
return string(b)
}