Files
archie/modules/runs/steps_test.go
T
sam 90637aa457 fix(runs): settle held actions a skipped or failed approval would leave
The end-of-run decision read the document's last approval step alone, so a conditional approval that was skipped, or one that continued past a failure, let a run finish without ever putting its held actions to people. Those approvals no longer count as settling. Resume still replays the agent and the request exactly once.

Triggers and dry runs now refuse a document that names an identity or profile, dry runs check required inputs, and step ids cannot contain dots that references read as separators.

The shipped investigate document gains a test that holds an action, suspends, resumes through the journal and reports one approved action.

Refs #2039
2026-11-21 01:20:21 +11:00

253 lines
8.8 KiB
Go

package runs
import (
"context"
"encoding/json/jsontext"
json "encoding/json/v2"
"errors"
"testing"
"github.com/google/uuid"
"github.com/samcharles93/ai-sdk/chat"
"github.com/samcharles93/archie-core/modules/approvals"
"github.com/samcharles93/archie-core/modules/deliveries"
"github.com/samcharles93/archie-core/modules/governance"
"github.com/samcharles93/archie-core/modules/judgement"
"github.com/samcharles93/archie-core/modules/runs/internal/store"
)
// actionJudge decides each action at the level named for it and remembers
// every request, so a test sees what a step asked for.
type actionJudge struct {
levels map[string]judgement.Level
fail error
assessed []judgement.Request
decisions []judgement.Decision
executed int
}
func (j *actionJudge) Assess(_ context.Context, req judgement.Request) (judgement.Decision, error) {
if j.fail != nil {
return judgement.Decision{}, j.fail
}
j.assessed = append(j.assessed, req)
d := judgement.Decision{Key: req.Key, Level: j.levels[req.Action], Action: judgement.Action{
Connector: req.Connector, Name: req.Action, Input: req.Input, Digest: req.Action,
}}
j.decisions = append(j.decisions, d)
return d, nil
}
func (j *actionJudge) Execute(context.Context, judgement.Decision, jsontext.Value, string) (string, error) {
j.executed++
return "done", nil
}
func (j *actionJudge) ExecuteApproved(context.Context, judgement.Decision, judgement.Approval, jsontext.Value, string) (string, error) {
j.executed++
return "done", nil
}
func (j *actionJudge) ForRun(context.Context, uuid.UUID) ([]judgement.Decision, error) {
return j.decisions, nil
}
// askCounter approves every request at once and counts them. failFirst makes
// that many requests fail first, as an approvals store that is briefly down.
type askCounter struct {
asked int
failFirst int
}
func (a *askCounter) Ask(_ context.Context, _ uuid.UUID, _ string, d judgement.Decision, _ []governance.PathStep) (approvals.Request, error) {
a.asked++
if a.failFirst > 0 {
a.failFirst--
return approvals.Request{}, errors.New("approvals unavailable")
}
return approvals.Request{ID: uuid.New(), State: approvals.StatePending, Decision: d}, nil
}
func (a *askCounter) Escalate(context.Context, uuid.UUID, int, governance.PathStep) error { return nil }
func (a *askCounter) Expire(context.Context, uuid.UUID) error { return nil }
func (a *askCounter) Accept(context.Context, uuid.UUID) error { return nil }
func (a *askCounter) Discard(context.Context, uuid.UUID, string) error { return nil }
// onePersonPath sends every request to one person.
type onePersonPath struct{}
func (onePersonPath) AgentModelForAlias(context.Context, string) (governance.Model, error) {
return governance.Model{}, errors.New("no agent in these workflows")
}
func (onePersonPath) AgentProvider(context.Context, governance.Model) (chat.Provider, chat.ModelInfo, error) {
return nil, chat.ModelInfo{}, errors.New("no agent in these workflows")
}
func (onePersonPath) EscalationPath(context.Context, string, bool) ([]governance.PathStep, error) {
return []governance.PathStep{{Person: "p"}}, nil
}
func (onePersonPath) RecurrenceThreshold(context.Context) (int, error) { return 0, nil }
func (onePersonPath) AttemptsPerRun(context.Context) (int, error) { return 1, nil }
type notices struct{ sent []deliveries.Notification }
func (n *notices) Notify(_ context.Context, msg deliveries.Notification) error {
n.sent = append(n.sent, msg)
return nil
}
// TestDocumentStepsRun guards the workflow step contract on the engine: a
// step's settings are rendered from the run's inputs and earlier results, its
// condition decides whether it runs, a failure ends the run unless the step
// continues, and every action judgement holds is put to people exactly once,
// by an approval step or before the run ends.
func TestDocumentStepsRun(t *testing.T) {
input := jsontext.Value(`{"event":{"fields":{"host":"web-1"}}}`)
tests := []struct {
name string
doc string
levels map[string]judgement.Level
fail error
failFirst int
wantErr bool
wantInput string
wantAssessed int
wantAsked int
wantExecuted int
wantNotice string
}{
{
name: "an action's input is rendered from the run's inputs",
doc: `inputs: {event: {type: object}}
steps:
- {type: connector.action, settings: {connector: host, name: clean, input: {host: "{{ inputs.event.fields.host }}"}}}
`,
levels: map[string]judgement.Level{"clean": judgement.ActQuietly},
wantInput: `{"host":"web-1"}`, wantAssessed: 1, wantExecuted: 1,
},
{
name: "a step whose condition doesn't hold is skipped",
doc: `inputs: {event: {type: object}}
steps:
- {type: connector.action, when: "inputs.event.fields.missing", settings: {connector: host, name: clean}}
`,
},
{
name: "a failing step fails the run",
doc: `steps:
- {type: connector.action, settings: {connector: host, name: clean}}
- {type: run.notify}
`,
fail: errors.New("judgement is down"), wantErr: true,
},
{
name: "a failing step that continues lets the run go on",
doc: `steps:
- {type: connector.action, on_failure: continue, settings: {connector: host, name: clean}}
- {type: run.notify, settings: {title: carried on}}
`,
fail: errors.New("judgement is down"), wantNotice: "carried on",
},
{
name: "an action held after the last approval step is put to people before the run ends",
doc: `steps:
- {type: connector.action, settings: {connector: host, name: clean}}
`,
levels: map[string]judgement.Level{"clean": judgement.ProposeAndWait},
wantAssessed: 2, wantAsked: 1, wantExecuted: 1,
},
{
name: "a held action is settled when the last approval step is skipped",
doc: `inputs: {skip: {type: bool}}
steps:
- {type: connector.action, settings: {connector: host, name: clean}}
- {type: approval.wait, when: "inputs.skip"}
`,
levels: map[string]judgement.Level{"clean": judgement.ProposeAndWait},
wantAssessed: 2, wantAsked: 1, wantExecuted: 1,
},
{
name: "an approval that fails and continues is settled before the run ends",
doc: `steps:
- {type: connector.action, settings: {connector: host, name: clean}}
- {type: approval.wait, on_failure: continue}
`,
levels: map[string]judgement.Level{"clean": judgement.ProposeAndWait},
failFirst: 1,
wantAssessed: 2, wantAsked: 2, wantExecuted: 1,
},
{
name: "a later approval step asks only about what was held since the last",
doc: `steps:
- {type: connector.action, settings: {connector: host, name: clean}}
- {type: approval.wait}
- {type: connector.action, settings: {connector: host, name: restart}}
- {type: approval.wait}
`,
levels: map[string]judgement.Level{"clean": judgement.ProposeAndWait, "restart": judgement.ProposeAndWait},
wantAssessed: 4, wantAsked: 2, wantExecuted: 2,
},
{
name: "a later step reads an earlier step's result",
doc: `steps:
- {id: clean, type: connector.action, settings: {connector: host, name: clean}}
- {type: run.notify, settings: {title: "cleaned: {{ steps.clean.summary }}"}}
`,
levels: map[string]judgement.Level{"clean": judgement.ActQuietly},
wantAssessed: 1, wantExecuted: 1, wantNotice: "cleaned: done",
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
doc, err := ParseDocument(tt.doc)
if err != nil {
t.Fatalf("ParseDocument: %v", err)
}
judge := &actionJudge{levels: tt.levels, fail: tt.fail}
ask := &askCounter{failFirst: tt.failFirst}
told := &notices{}
wf := Deps{Settings: onePersonPath{}, Toolbox: anyBox{}, Judge: judge, Approver: ask, Notifier: told}.Compile(doc)
work := &signalAfter{arrives: 1}
rc := newContext(t.Context(), work, store.Run{ID: uuid.New()}, uuid.New(), nil)
_, err = wf(rc, input)
if (err != nil) != tt.wantErr {
t.Fatalf("run error = %v, want error %v", err, tt.wantErr)
}
if len(judge.assessed) != tt.wantAssessed || ask.asked != tt.wantAsked || judge.executed != tt.wantExecuted {
t.Fatalf("assessed %d, asked %d, executed %d; want %d, %d, %d",
len(judge.assessed), ask.asked, judge.executed, tt.wantAssessed, tt.wantAsked, tt.wantExecuted)
}
if tt.wantInput != "" {
var got, want any
if err := json.Unmarshal(judge.assessed[0].Input, &got); err != nil {
t.Fatal(err)
}
if err := json.Unmarshal([]byte(tt.wantInput), &want); err != nil {
t.Fatal(err)
}
if a, b := mustJSON(t, got), mustJSON(t, want); a != b {
t.Fatalf("action input = %s, want %s", a, b)
}
}
if tt.wantNotice != "" && (len(told.sent) != 1 || told.sent[0].Title != tt.wantNotice) {
t.Fatalf("notices = %+v, want one titled %q", told.sent, tt.wantNotice)
}
})
}
}
func mustJSON(t *testing.T, v any) string {
t.Helper()
b, err := json.Marshal(v, json.Deterministic(true))
if err != nil {
t.Fatal(err)
}
return string(b)
}