Also fix repository-free message.send validation (#2046) and add a single-binary development smoke path (#1977). Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com>
224 lines
7.8 KiB
Go
224 lines
7.8 KiB
Go
package runs
|
|
|
|
import (
|
|
"context"
|
|
"encoding/json/jsontext"
|
|
json "encoding/json/v2"
|
|
"errors"
|
|
"testing"
|
|
|
|
"github.com/google/uuid"
|
|
"github.com/samcharles93/ai-sdk/chat"
|
|
|
|
"github.com/samcharles93/archie-core/modules/approvals"
|
|
"github.com/samcharles93/archie-core/modules/deliveries"
|
|
"github.com/samcharles93/archie-core/modules/governance"
|
|
"github.com/samcharles93/archie-core/modules/judgement"
|
|
"github.com/samcharles93/archie-core/modules/runs/internal/store"
|
|
)
|
|
|
|
// actionJudge decides each action at the level named for it and remembers
|
|
// every request, so a test sees what a step asked for.
|
|
type actionJudge struct {
|
|
levels map[string]judgement.Level
|
|
fail error
|
|
assessed []judgement.Request
|
|
decisions []judgement.Decision
|
|
executed int
|
|
}
|
|
|
|
func (j *actionJudge) Assess(_ context.Context, req judgement.Request) (judgement.Decision, error) {
|
|
if j.fail != nil {
|
|
return judgement.Decision{}, j.fail
|
|
}
|
|
j.assessed = append(j.assessed, req)
|
|
d := judgement.Decision{Key: req.Key, Level: j.levels[req.Action], Action: judgement.Action{
|
|
Connector: req.Connector, Name: req.Action, Input: req.Input, Digest: req.Action,
|
|
}}
|
|
j.decisions = append(j.decisions, d)
|
|
return d, nil
|
|
}
|
|
|
|
func (j *actionJudge) Execute(context.Context, judgement.Decision, jsontext.Value, string) (string, error) {
|
|
j.executed++
|
|
return "done", nil
|
|
}
|
|
|
|
func (j *actionJudge) ExecuteApproved(context.Context, judgement.Decision, judgement.Approval, jsontext.Value, string) (string, error) {
|
|
j.executed++
|
|
return "done", nil
|
|
}
|
|
|
|
func (j *actionJudge) ForRun(context.Context, uuid.UUID) ([]judgement.Decision, error) {
|
|
return j.decisions, nil
|
|
}
|
|
|
|
// askCounter approves every request at once and counts them.
|
|
type askCounter struct{ asked int }
|
|
|
|
func (a *askCounter) Ask(_ context.Context, _ uuid.UUID, _ string, d judgement.Decision, _ []governance.PathStep) (approvals.Request, error) {
|
|
a.asked++
|
|
return approvals.Request{ID: uuid.New(), State: approvals.StatePending, Decision: d}, nil
|
|
}
|
|
|
|
func (a *askCounter) Escalate(context.Context, uuid.UUID, int, governance.PathStep) error { return nil }
|
|
|
|
func (a *askCounter) Expire(context.Context, uuid.UUID) error { return nil }
|
|
|
|
func (a *askCounter) Accept(context.Context, uuid.UUID) error { return nil }
|
|
|
|
func (a *askCounter) Discard(context.Context, uuid.UUID, string) error { return nil }
|
|
|
|
// onePersonPath sends every request to one person.
|
|
type onePersonPath struct{}
|
|
|
|
func (onePersonPath) AgentModelForAlias(context.Context, string) (governance.Model, error) {
|
|
return governance.Model{}, errors.New("no agent in these workflows")
|
|
}
|
|
|
|
func (onePersonPath) AgentProvider(context.Context, governance.Model) (chat.Provider, chat.ModelInfo, error) {
|
|
return nil, chat.ModelInfo{}, errors.New("no agent in these workflows")
|
|
}
|
|
|
|
func (onePersonPath) EscalationPath(context.Context, string, bool) ([]governance.PathStep, error) {
|
|
return []governance.PathStep{{Person: "p"}}, nil
|
|
}
|
|
|
|
func (onePersonPath) RecurrenceThreshold(context.Context) (int, error) { return 0, nil }
|
|
func (onePersonPath) AttemptsPerRun(context.Context) (int, error) { return 1, nil }
|
|
|
|
type notices struct{ sent []deliveries.Notification }
|
|
|
|
func (n *notices) Notify(_ context.Context, msg deliveries.Notification) error {
|
|
n.sent = append(n.sent, msg)
|
|
return nil
|
|
}
|
|
|
|
// TestDocumentStepsRun guards the workflow step contract on the engine: a
|
|
// step's settings are rendered from the run's inputs and earlier results, its
|
|
// condition decides whether it runs, a failure ends the run unless the step
|
|
// continues, and every action judgement holds is put to people exactly once,
|
|
// by an approval step or before the run ends.
|
|
func TestDocumentStepsRun(t *testing.T) {
|
|
input := jsontext.Value(`{"event":{"fields":{"host":"web-1"}}}`)
|
|
tests := []struct {
|
|
name string
|
|
doc string
|
|
levels map[string]judgement.Level
|
|
fail error
|
|
wantErr bool
|
|
wantInput string
|
|
wantAssessed int
|
|
wantAsked int
|
|
wantExecuted int
|
|
wantNotice string
|
|
}{
|
|
{
|
|
name: "an action's input is rendered from the run's inputs",
|
|
doc: `inputs: {event: {type: object}}
|
|
steps:
|
|
- {type: connector.action, settings: {connector: host, name: clean, input: {host: "{{ inputs.event.fields.host }}"}}}
|
|
`,
|
|
levels: map[string]judgement.Level{"clean": judgement.ActQuietly},
|
|
wantInput: `{"host":"web-1"}`, wantAssessed: 1, wantExecuted: 1,
|
|
},
|
|
{
|
|
name: "a step whose condition doesn't hold is skipped",
|
|
doc: `inputs: {event: {type: object}}
|
|
steps:
|
|
- {type: connector.action, when: "inputs.event.fields.missing", settings: {connector: host, name: clean}}
|
|
`,
|
|
},
|
|
{
|
|
name: "a failing step fails the run",
|
|
doc: `steps:
|
|
- {type: connector.action, settings: {connector: host, name: clean}}
|
|
- {type: run.notify}
|
|
`,
|
|
fail: errors.New("judgement is down"), wantErr: true,
|
|
},
|
|
{
|
|
name: "a failing step that continues lets the run go on",
|
|
doc: `steps:
|
|
- {type: connector.action, on_failure: continue, settings: {connector: host, name: clean}}
|
|
- {type: run.notify, settings: {title: carried on}}
|
|
`,
|
|
fail: errors.New("judgement is down"), wantNotice: "carried on",
|
|
},
|
|
{
|
|
name: "an action held after the last approval step is put to people before the run ends",
|
|
doc: `steps:
|
|
- {type: connector.action, settings: {connector: host, name: clean}}
|
|
`,
|
|
levels: map[string]judgement.Level{"clean": judgement.ProposeAndWait},
|
|
wantAssessed: 2, wantAsked: 1, wantExecuted: 1,
|
|
},
|
|
{
|
|
name: "a later approval step asks only about what was held since the last",
|
|
doc: `steps:
|
|
- {type: connector.action, settings: {connector: host, name: clean}}
|
|
- {type: approval.wait}
|
|
- {type: connector.action, settings: {connector: host, name: restart}}
|
|
- {type: approval.wait}
|
|
`,
|
|
levels: map[string]judgement.Level{"clean": judgement.ProposeAndWait, "restart": judgement.ProposeAndWait},
|
|
wantAssessed: 4, wantAsked: 2, wantExecuted: 2,
|
|
},
|
|
{
|
|
name: "a later step reads an earlier step's result",
|
|
doc: `steps:
|
|
- {id: clean, type: connector.action, settings: {connector: host, name: clean}}
|
|
- {type: run.notify, settings: {title: "cleaned: {{ steps.clean.summary }}"}}
|
|
`,
|
|
levels: map[string]judgement.Level{"clean": judgement.ActQuietly},
|
|
wantAssessed: 1, wantExecuted: 1, wantNotice: "cleaned: done",
|
|
},
|
|
}
|
|
for _, tt := range tests {
|
|
t.Run(tt.name, func(t *testing.T) {
|
|
doc, err := ParseDocument(tt.doc)
|
|
if err != nil {
|
|
t.Fatalf("ParseDocument: %v", err)
|
|
}
|
|
judge := &actionJudge{levels: tt.levels, fail: tt.fail}
|
|
ask := &askCounter{}
|
|
told := ¬ices{}
|
|
wf := Deps{Settings: onePersonPath{}, Toolbox: anyBox{}, Judge: judge, Approver: ask, Notifier: told}.Compile(doc)
|
|
work := &signalAfter{arrives: 1}
|
|
rc := newContext(t.Context(), work, store.Run{ID: uuid.New()}, uuid.New(), nil)
|
|
_, err = wf(rc, input)
|
|
if (err != nil) != tt.wantErr {
|
|
t.Fatalf("run error = %v, want error %v", err, tt.wantErr)
|
|
}
|
|
if len(judge.assessed) != tt.wantAssessed || ask.asked != tt.wantAsked || judge.executed != tt.wantExecuted {
|
|
t.Fatalf("assessed %d, asked %d, executed %d; want %d, %d, %d",
|
|
len(judge.assessed), ask.asked, judge.executed, tt.wantAssessed, tt.wantAsked, tt.wantExecuted)
|
|
}
|
|
if tt.wantInput != "" {
|
|
var got, want any
|
|
if err := json.Unmarshal(judge.assessed[0].Input, &got); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if err := json.Unmarshal([]byte(tt.wantInput), &want); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if a, b := mustJSON(t, got), mustJSON(t, want); a != b {
|
|
t.Fatalf("action input = %s, want %s", a, b)
|
|
}
|
|
}
|
|
if tt.wantNotice != "" && (len(told.sent) != 1 || told.sent[0].Title != tt.wantNotice) {
|
|
t.Fatalf("notices = %+v, want one titled %q", told.sent, tt.wantNotice)
|
|
}
|
|
})
|
|
}
|
|
}
|
|
|
|
func mustJSON(t *testing.T, v any) string {
|
|
t.Helper()
|
|
b, err := json.Marshal(v, json.Deterministic(true))
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
return string(b)
|
|
}
|