Files
archie/modules/runs/steps_test.go
T
samandCopilot 64dfad6d53 feat(runs): add typed results and model aliases (#2043)
Also fix repository-free message.send validation (#2046) and add a single-binary development smoke path (#1977).

Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com>
2026-11-18 23:47:35 +11:00

224 lines
7.8 KiB
Go

package runs
import (
"context"
"encoding/json/jsontext"
json "encoding/json/v2"
"errors"
"testing"
"github.com/google/uuid"
"github.com/samcharles93/ai-sdk/chat"
"github.com/samcharles93/archie-core/modules/approvals"
"github.com/samcharles93/archie-core/modules/deliveries"
"github.com/samcharles93/archie-core/modules/governance"
"github.com/samcharles93/archie-core/modules/judgement"
"github.com/samcharles93/archie-core/modules/runs/internal/store"
)
// actionJudge decides each action at the level named for it and remembers
// every request, so a test sees what a step asked for.
type actionJudge struct {
levels map[string]judgement.Level
fail error
assessed []judgement.Request
decisions []judgement.Decision
executed int
}
func (j *actionJudge) Assess(_ context.Context, req judgement.Request) (judgement.Decision, error) {
if j.fail != nil {
return judgement.Decision{}, j.fail
}
j.assessed = append(j.assessed, req)
d := judgement.Decision{Key: req.Key, Level: j.levels[req.Action], Action: judgement.Action{
Connector: req.Connector, Name: req.Action, Input: req.Input, Digest: req.Action,
}}
j.decisions = append(j.decisions, d)
return d, nil
}
func (j *actionJudge) Execute(context.Context, judgement.Decision, jsontext.Value, string) (string, error) {
j.executed++
return "done", nil
}
func (j *actionJudge) ExecuteApproved(context.Context, judgement.Decision, judgement.Approval, jsontext.Value, string) (string, error) {
j.executed++
return "done", nil
}
func (j *actionJudge) ForRun(context.Context, uuid.UUID) ([]judgement.Decision, error) {
return j.decisions, nil
}
// askCounter approves every request at once and counts them.
type askCounter struct{ asked int }
func (a *askCounter) Ask(_ context.Context, _ uuid.UUID, _ string, d judgement.Decision, _ []governance.PathStep) (approvals.Request, error) {
a.asked++
return approvals.Request{ID: uuid.New(), State: approvals.StatePending, Decision: d}, nil
}
func (a *askCounter) Escalate(context.Context, uuid.UUID, int, governance.PathStep) error { return nil }
func (a *askCounter) Expire(context.Context, uuid.UUID) error { return nil }
func (a *askCounter) Accept(context.Context, uuid.UUID) error { return nil }
func (a *askCounter) Discard(context.Context, uuid.UUID, string) error { return nil }
// onePersonPath sends every request to one person.
type onePersonPath struct{}
func (onePersonPath) AgentModelForAlias(context.Context, string) (governance.Model, error) {
return governance.Model{}, errors.New("no agent in these workflows")
}
func (onePersonPath) AgentProvider(context.Context, governance.Model) (chat.Provider, chat.ModelInfo, error) {
return nil, chat.ModelInfo{}, errors.New("no agent in these workflows")
}
func (onePersonPath) EscalationPath(context.Context, string, bool) ([]governance.PathStep, error) {
return []governance.PathStep{{Person: "p"}}, nil
}
func (onePersonPath) RecurrenceThreshold(context.Context) (int, error) { return 0, nil }
func (onePersonPath) AttemptsPerRun(context.Context) (int, error) { return 1, nil }
type notices struct{ sent []deliveries.Notification }
func (n *notices) Notify(_ context.Context, msg deliveries.Notification) error {
n.sent = append(n.sent, msg)
return nil
}
// TestDocumentStepsRun guards the workflow step contract on the engine: a
// step's settings are rendered from the run's inputs and earlier results, its
// condition decides whether it runs, a failure ends the run unless the step
// continues, and every action judgement holds is put to people exactly once,
// by an approval step or before the run ends.
func TestDocumentStepsRun(t *testing.T) {
input := jsontext.Value(`{"event":{"fields":{"host":"web-1"}}}`)
tests := []struct {
name string
doc string
levels map[string]judgement.Level
fail error
wantErr bool
wantInput string
wantAssessed int
wantAsked int
wantExecuted int
wantNotice string
}{
{
name: "an action's input is rendered from the run's inputs",
doc: `inputs: {event: {type: object}}
steps:
- {type: connector.action, settings: {connector: host, name: clean, input: {host: "{{ inputs.event.fields.host }}"}}}
`,
levels: map[string]judgement.Level{"clean": judgement.ActQuietly},
wantInput: `{"host":"web-1"}`, wantAssessed: 1, wantExecuted: 1,
},
{
name: "a step whose condition doesn't hold is skipped",
doc: `inputs: {event: {type: object}}
steps:
- {type: connector.action, when: "inputs.event.fields.missing", settings: {connector: host, name: clean}}
`,
},
{
name: "a failing step fails the run",
doc: `steps:
- {type: connector.action, settings: {connector: host, name: clean}}
- {type: run.notify}
`,
fail: errors.New("judgement is down"), wantErr: true,
},
{
name: "a failing step that continues lets the run go on",
doc: `steps:
- {type: connector.action, on_failure: continue, settings: {connector: host, name: clean}}
- {type: run.notify, settings: {title: carried on}}
`,
fail: errors.New("judgement is down"), wantNotice: "carried on",
},
{
name: "an action held after the last approval step is put to people before the run ends",
doc: `steps:
- {type: connector.action, settings: {connector: host, name: clean}}
`,
levels: map[string]judgement.Level{"clean": judgement.ProposeAndWait},
wantAssessed: 2, wantAsked: 1, wantExecuted: 1,
},
{
name: "a later approval step asks only about what was held since the last",
doc: `steps:
- {type: connector.action, settings: {connector: host, name: clean}}
- {type: approval.wait}
- {type: connector.action, settings: {connector: host, name: restart}}
- {type: approval.wait}
`,
levels: map[string]judgement.Level{"clean": judgement.ProposeAndWait, "restart": judgement.ProposeAndWait},
wantAssessed: 4, wantAsked: 2, wantExecuted: 2,
},
{
name: "a later step reads an earlier step's result",
doc: `steps:
- {id: clean, type: connector.action, settings: {connector: host, name: clean}}
- {type: run.notify, settings: {title: "cleaned: {{ steps.clean.summary }}"}}
`,
levels: map[string]judgement.Level{"clean": judgement.ActQuietly},
wantAssessed: 1, wantExecuted: 1, wantNotice: "cleaned: done",
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
doc, err := ParseDocument(tt.doc)
if err != nil {
t.Fatalf("ParseDocument: %v", err)
}
judge := &actionJudge{levels: tt.levels, fail: tt.fail}
ask := &askCounter{}
told := &notices{}
wf := Deps{Settings: onePersonPath{}, Toolbox: anyBox{}, Judge: judge, Approver: ask, Notifier: told}.Compile(doc)
work := &signalAfter{arrives: 1}
rc := newContext(t.Context(), work, store.Run{ID: uuid.New()}, uuid.New(), nil)
_, err = wf(rc, input)
if (err != nil) != tt.wantErr {
t.Fatalf("run error = %v, want error %v", err, tt.wantErr)
}
if len(judge.assessed) != tt.wantAssessed || ask.asked != tt.wantAsked || judge.executed != tt.wantExecuted {
t.Fatalf("assessed %d, asked %d, executed %d; want %d, %d, %d",
len(judge.assessed), ask.asked, judge.executed, tt.wantAssessed, tt.wantAsked, tt.wantExecuted)
}
if tt.wantInput != "" {
var got, want any
if err := json.Unmarshal(judge.assessed[0].Input, &got); err != nil {
t.Fatal(err)
}
if err := json.Unmarshal([]byte(tt.wantInput), &want); err != nil {
t.Fatal(err)
}
if a, b := mustJSON(t, got), mustJSON(t, want); a != b {
t.Fatalf("action input = %s, want %s", a, b)
}
}
if tt.wantNotice != "" && (len(told.sent) != 1 || told.sent[0].Title != tt.wantNotice) {
t.Fatalf("notices = %+v, want one titled %q", told.sent, tt.wantNotice)
}
})
}
}
func mustJSON(t *testing.T, v any) string {
t.Helper()
b, err := json.Marshal(v, json.Deterministic(true))
if err != nil {
t.Fatal(err)
}
return string(b)
}