The end-of-run decision read the document's last approval step alone, so a conditional approval that was skipped, or one that continued past a failure, let a run finish without ever putting its held actions to people. Those approvals no longer count as settling. Resume still replays the agent and the request exactly once. Triggers and dry runs now refuse a document that names an identity or profile, dry runs check required inputs, and step ids cannot contain dots that references read as separators. The shipped investigate document gains a test that holds an action, suspends, resumes through the journal and reports one approved action. Refs #2039
253 lines
8.8 KiB
Go
253 lines
8.8 KiB
Go
package runs
|
|
|
|
import (
|
|
"context"
|
|
"encoding/json/jsontext"
|
|
json "encoding/json/v2"
|
|
"errors"
|
|
"testing"
|
|
|
|
"github.com/google/uuid"
|
|
"github.com/samcharles93/ai-sdk/chat"
|
|
|
|
"github.com/samcharles93/archie-core/modules/approvals"
|
|
"github.com/samcharles93/archie-core/modules/deliveries"
|
|
"github.com/samcharles93/archie-core/modules/governance"
|
|
"github.com/samcharles93/archie-core/modules/judgement"
|
|
"github.com/samcharles93/archie-core/modules/runs/internal/store"
|
|
)
|
|
|
|
// actionJudge decides each action at the level named for it and remembers
|
|
// every request, so a test sees what a step asked for.
|
|
type actionJudge struct {
|
|
levels map[string]judgement.Level
|
|
fail error
|
|
assessed []judgement.Request
|
|
decisions []judgement.Decision
|
|
executed int
|
|
}
|
|
|
|
func (j *actionJudge) Assess(_ context.Context, req judgement.Request) (judgement.Decision, error) {
|
|
if j.fail != nil {
|
|
return judgement.Decision{}, j.fail
|
|
}
|
|
j.assessed = append(j.assessed, req)
|
|
d := judgement.Decision{Key: req.Key, Level: j.levels[req.Action], Action: judgement.Action{
|
|
Connector: req.Connector, Name: req.Action, Input: req.Input, Digest: req.Action,
|
|
}}
|
|
j.decisions = append(j.decisions, d)
|
|
return d, nil
|
|
}
|
|
|
|
func (j *actionJudge) Execute(context.Context, judgement.Decision, jsontext.Value, string) (string, error) {
|
|
j.executed++
|
|
return "done", nil
|
|
}
|
|
|
|
func (j *actionJudge) ExecuteApproved(context.Context, judgement.Decision, judgement.Approval, jsontext.Value, string) (string, error) {
|
|
j.executed++
|
|
return "done", nil
|
|
}
|
|
|
|
func (j *actionJudge) ForRun(context.Context, uuid.UUID) ([]judgement.Decision, error) {
|
|
return j.decisions, nil
|
|
}
|
|
|
|
// askCounter approves every request at once and counts them. failFirst makes
|
|
// that many requests fail first, as an approvals store that is briefly down.
|
|
type askCounter struct {
|
|
asked int
|
|
failFirst int
|
|
}
|
|
|
|
func (a *askCounter) Ask(_ context.Context, _ uuid.UUID, _ string, d judgement.Decision, _ []governance.PathStep) (approvals.Request, error) {
|
|
a.asked++
|
|
if a.failFirst > 0 {
|
|
a.failFirst--
|
|
return approvals.Request{}, errors.New("approvals unavailable")
|
|
}
|
|
return approvals.Request{ID: uuid.New(), State: approvals.StatePending, Decision: d}, nil
|
|
}
|
|
|
|
func (a *askCounter) Escalate(context.Context, uuid.UUID, int, governance.PathStep) error { return nil }
|
|
|
|
func (a *askCounter) Expire(context.Context, uuid.UUID) error { return nil }
|
|
|
|
func (a *askCounter) Accept(context.Context, uuid.UUID) error { return nil }
|
|
|
|
func (a *askCounter) Discard(context.Context, uuid.UUID, string) error { return nil }
|
|
|
|
// onePersonPath sends every request to one person.
|
|
type onePersonPath struct{}
|
|
|
|
func (onePersonPath) AgentModelForAlias(context.Context, string) (governance.Model, error) {
|
|
return governance.Model{}, errors.New("no agent in these workflows")
|
|
}
|
|
|
|
func (onePersonPath) AgentProvider(context.Context, governance.Model) (chat.Provider, chat.ModelInfo, error) {
|
|
return nil, chat.ModelInfo{}, errors.New("no agent in these workflows")
|
|
}
|
|
|
|
func (onePersonPath) EscalationPath(context.Context, string, bool) ([]governance.PathStep, error) {
|
|
return []governance.PathStep{{Person: "p"}}, nil
|
|
}
|
|
|
|
func (onePersonPath) RecurrenceThreshold(context.Context) (int, error) { return 0, nil }
|
|
func (onePersonPath) AttemptsPerRun(context.Context) (int, error) { return 1, nil }
|
|
|
|
type notices struct{ sent []deliveries.Notification }
|
|
|
|
func (n *notices) Notify(_ context.Context, msg deliveries.Notification) error {
|
|
n.sent = append(n.sent, msg)
|
|
return nil
|
|
}
|
|
|
|
// TestDocumentStepsRun guards the workflow step contract on the engine: a
|
|
// step's settings are rendered from the run's inputs and earlier results, its
|
|
// condition decides whether it runs, a failure ends the run unless the step
|
|
// continues, and every action judgement holds is put to people exactly once,
|
|
// by an approval step or before the run ends.
|
|
func TestDocumentStepsRun(t *testing.T) {
|
|
input := jsontext.Value(`{"event":{"fields":{"host":"web-1"}}}`)
|
|
tests := []struct {
|
|
name string
|
|
doc string
|
|
levels map[string]judgement.Level
|
|
fail error
|
|
failFirst int
|
|
wantErr bool
|
|
wantInput string
|
|
wantAssessed int
|
|
wantAsked int
|
|
wantExecuted int
|
|
wantNotice string
|
|
}{
|
|
{
|
|
name: "an action's input is rendered from the run's inputs",
|
|
doc: `inputs: {event: {type: object}}
|
|
steps:
|
|
- {type: connector.action, settings: {connector: host, name: clean, input: {host: "{{ inputs.event.fields.host }}"}}}
|
|
`,
|
|
levels: map[string]judgement.Level{"clean": judgement.ActQuietly},
|
|
wantInput: `{"host":"web-1"}`, wantAssessed: 1, wantExecuted: 1,
|
|
},
|
|
{
|
|
name: "a step whose condition doesn't hold is skipped",
|
|
doc: `inputs: {event: {type: object}}
|
|
steps:
|
|
- {type: connector.action, when: "inputs.event.fields.missing", settings: {connector: host, name: clean}}
|
|
`,
|
|
},
|
|
{
|
|
name: "a failing step fails the run",
|
|
doc: `steps:
|
|
- {type: connector.action, settings: {connector: host, name: clean}}
|
|
- {type: run.notify}
|
|
`,
|
|
fail: errors.New("judgement is down"), wantErr: true,
|
|
},
|
|
{
|
|
name: "a failing step that continues lets the run go on",
|
|
doc: `steps:
|
|
- {type: connector.action, on_failure: continue, settings: {connector: host, name: clean}}
|
|
- {type: run.notify, settings: {title: carried on}}
|
|
`,
|
|
fail: errors.New("judgement is down"), wantNotice: "carried on",
|
|
},
|
|
{
|
|
name: "an action held after the last approval step is put to people before the run ends",
|
|
doc: `steps:
|
|
- {type: connector.action, settings: {connector: host, name: clean}}
|
|
`,
|
|
levels: map[string]judgement.Level{"clean": judgement.ProposeAndWait},
|
|
wantAssessed: 2, wantAsked: 1, wantExecuted: 1,
|
|
},
|
|
{
|
|
name: "a held action is settled when the last approval step is skipped",
|
|
doc: `inputs: {skip: {type: bool}}
|
|
steps:
|
|
- {type: connector.action, settings: {connector: host, name: clean}}
|
|
- {type: approval.wait, when: "inputs.skip"}
|
|
`,
|
|
levels: map[string]judgement.Level{"clean": judgement.ProposeAndWait},
|
|
wantAssessed: 2, wantAsked: 1, wantExecuted: 1,
|
|
},
|
|
{
|
|
name: "an approval that fails and continues is settled before the run ends",
|
|
doc: `steps:
|
|
- {type: connector.action, settings: {connector: host, name: clean}}
|
|
- {type: approval.wait, on_failure: continue}
|
|
`,
|
|
levels: map[string]judgement.Level{"clean": judgement.ProposeAndWait},
|
|
failFirst: 1,
|
|
wantAssessed: 2, wantAsked: 2, wantExecuted: 1,
|
|
},
|
|
{
|
|
name: "a later approval step asks only about what was held since the last",
|
|
doc: `steps:
|
|
- {type: connector.action, settings: {connector: host, name: clean}}
|
|
- {type: approval.wait}
|
|
- {type: connector.action, settings: {connector: host, name: restart}}
|
|
- {type: approval.wait}
|
|
`,
|
|
levels: map[string]judgement.Level{"clean": judgement.ProposeAndWait, "restart": judgement.ProposeAndWait},
|
|
wantAssessed: 4, wantAsked: 2, wantExecuted: 2,
|
|
},
|
|
{
|
|
name: "a later step reads an earlier step's result",
|
|
doc: `steps:
|
|
- {id: clean, type: connector.action, settings: {connector: host, name: clean}}
|
|
- {type: run.notify, settings: {title: "cleaned: {{ steps.clean.summary }}"}}
|
|
`,
|
|
levels: map[string]judgement.Level{"clean": judgement.ActQuietly},
|
|
wantAssessed: 1, wantExecuted: 1, wantNotice: "cleaned: done",
|
|
},
|
|
}
|
|
for _, tt := range tests {
|
|
t.Run(tt.name, func(t *testing.T) {
|
|
doc, err := ParseDocument(tt.doc)
|
|
if err != nil {
|
|
t.Fatalf("ParseDocument: %v", err)
|
|
}
|
|
judge := &actionJudge{levels: tt.levels, fail: tt.fail}
|
|
ask := &askCounter{failFirst: tt.failFirst}
|
|
told := ¬ices{}
|
|
wf := Deps{Settings: onePersonPath{}, Toolbox: anyBox{}, Judge: judge, Approver: ask, Notifier: told}.Compile(doc)
|
|
work := &signalAfter{arrives: 1}
|
|
rc := newContext(t.Context(), work, store.Run{ID: uuid.New()}, uuid.New(), nil)
|
|
_, err = wf(rc, input)
|
|
if (err != nil) != tt.wantErr {
|
|
t.Fatalf("run error = %v, want error %v", err, tt.wantErr)
|
|
}
|
|
if len(judge.assessed) != tt.wantAssessed || ask.asked != tt.wantAsked || judge.executed != tt.wantExecuted {
|
|
t.Fatalf("assessed %d, asked %d, executed %d; want %d, %d, %d",
|
|
len(judge.assessed), ask.asked, judge.executed, tt.wantAssessed, tt.wantAsked, tt.wantExecuted)
|
|
}
|
|
if tt.wantInput != "" {
|
|
var got, want any
|
|
if err := json.Unmarshal(judge.assessed[0].Input, &got); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if err := json.Unmarshal([]byte(tt.wantInput), &want); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if a, b := mustJSON(t, got), mustJSON(t, want); a != b {
|
|
t.Fatalf("action input = %s, want %s", a, b)
|
|
}
|
|
}
|
|
if tt.wantNotice != "" && (len(told.sent) != 1 || told.sent[0].Title != tt.wantNotice) {
|
|
t.Fatalf("notices = %+v, want one titled %q", told.sent, tt.wantNotice)
|
|
}
|
|
})
|
|
}
|
|
}
|
|
|
|
func mustJSON(t *testing.T, v any) string {
|
|
t.Helper()
|
|
b, err := json.Marshal(v, json.Deterministic(true))
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
return string(b)
|
|
}
|