Files
archie/internal/domain/workflow/workflow_call_test.go
T
sam c265b07ec7 refactor(workflow): stop assuming a human decides
An approval, ask or park can be resolved by an operator, a rule, a
classifier or automatically, so the vocabulary stops naming a person:

- step type human.approve -> workflow.approve
- park class needs_human -> needs_decision (ParkNeedsDecision)
- event kinds human_* -> operator_*
- alert kind task.needs_human -> task.needs_decision
- status grouping "needs you" -> "needs attention"

0061 rewrites the stored values, including the approval step type in the
workflow-definitions resource and each task's pinned definition, with the
digest recomputed so resumed tasks still match.
2026-10-25 17:31:16 +11:00

512 lines
17 KiB
Go

package workflow
import (
"context"
"errors"
"fmt"
"log/slog"
"maps"
"slices"
"strings"
"sync"
"testing"
"time"
"github.com/samcharles93/archie-core/internal/domain/storecontract"
"github.com/samcharles93/archie-core/internal/domain/workflow/task"
"github.com/samcharles93/archie-core/internal/events"
)
// callSpy records the key every call site started its child under, and models
// the store's own idempotency: one child per call key.
type callSpy struct {
mu sync.Mutex
keys []string
children map[string]*task.Task
}
func (c *callSpy) StartCall(_ context.Context, callerTaskID int64, callKey, _ string, _ map[string]any) (*task.Task, error) {
c.mu.Lock()
defer c.mu.Unlock()
c.keys = append(c.keys, callKey)
child, ok := c.children[callKey]
if !ok {
child = &task.Task{ID: int64(len(c.children) + 1), CallParentTaskID: callerTaskID}
c.children[callKey] = child
}
return child, nil
}
func (c *callSpy) CallStatus(context.Context, int64, int64) (string, string, map[string]any, error) {
return StatusCompleted, "", nil, nil
}
func (c *callSpy) startedKeys() []string {
c.mu.Lock()
defer c.mu.Unlock()
return slices.Clone(c.keys)
}
// TestWorkflowCallKeyIsTheDeclaredPath pins the call's durable identity: the
// step's declared path plus the callee it starts. Two runs of one call site --
// a retry of the stage, a lost EnqueueCallTask reply -- key on the same value
// and start one child, and neither a sibling call site nor the same path under
// a re-pinned definition reuses another call's child.
func TestWorkflowCallKeyIsTheDeclaredPath(t *testing.T) {
registry := StepRegistry{WorkflowCallStepName: newWorkflowCallStage}
tests := []struct {
name string
def string
// repin is the definition a later run compiles instead. One task can
// run a second definition -- workflow.handoff and workflow.approve set the
// task's workflow, and the daemon re-pins when the pin's id differs --
// so a call path that collides there must not reuse the first callee.
repin string
wantKeys []string
wantRepin []string
wantErr string
}{
{
name: "a call step keys on its declared id and the callee it starts",
def: `id: caller
repository: none
steps:
- {id: check, type: workflow.call, settings: {workflow: callee}}
`,
wantKeys: []string{"check@callee"},
},
{
name: "a call in a parallel branch keys on its path within the run",
def: `id: caller
repository: none
steps:
- id: fanout
parallel:
a: [{id: check, type: workflow.call, settings: {workflow: callee}}]
b: [{id: verify, type: workflow.call, settings: {workflow: callee}}]
`,
wantKeys: []string{"fanout/a/check@callee", "fanout/b/verify@callee"},
},
{
name: "the same path under a re-pinned definition keys on its callee",
def: `id: caller
repository: none
steps:
- {id: classify, type: workflow.call, settings: {workflow: callee-a}}
`,
repin: `id: caller
repository: none
steps:
- {id: classify, type: workflow.call, settings: {workflow: callee-b}}
`,
wantKeys: []string{"classify@callee-a"},
wantRepin: []string{"classify@callee-b"},
},
{
// A step id may spell a step type: it is a stable identifier like
// any other, and the path it declares is the call's identity.
name: "a step id that spells the step type is accepted",
def: `id: caller
repository: none
steps:
- {id: workflow.call, type: workflow.call, settings: {workflow: callee}}
`,
wantKeys: []string{"workflow.call@callee"},
},
{
// The step's id is the call's identity, so an id-less call step is
// refused where its definition is compiled -- which is also the path
// a pinned definition takes.
name: "a call step with no id is refused",
def: `id: caller
repository: none
steps:
- type: workflow.call
settings: {workflow: callee}
`,
// The whole message, so the refusal still names the step it
// refused: this is where a pinned definition is compiled.
wantErr: `workflow "caller" step 1: "workflow.call" needs an id`,
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
spy := &callSpy{children: map[string]*task.Task{}}
tc := &TaskContext{Task: &task.Task{ID: 7}, Calls: spy, Log: slog.Default()}
// Twice: a requeued attempt re-runs the call site.
second := tt.def
if tt.repin != "" {
second = tt.repin
}
for i, def := range []string{tt.def, second} {
wf, err := ParseAndCompile(def, registry)
if tt.wantErr != "" {
if err == nil || !strings.Contains(err.Error(), tt.wantErr) {
t.Fatalf("compile err = %v, want %q", err, tt.wantErr)
}
continue
}
if err != nil {
t.Fatalf("compile: %v", err)
}
before := len(spy.startedKeys())
if err := wf.Stages[0].Run(context.Background(), tc); err != nil {
t.Fatalf("run: %v", err)
}
want := tt.wantKeys
if i == 1 && tt.repin != "" {
want = tt.wantRepin
}
got := spy.startedKeys()[before:]
// Branches finish in whatever order; the keys a run used, not
// their order, are the behaviour.
slices.Sort(got)
if !slices.Equal(got, want) {
t.Fatalf("run %d keys = %q, want %q", i+1, got, want)
}
}
if tt.wantErr != "" {
if keys := spy.startedKeys(); len(keys) != 0 {
t.Fatalf("started %q, want no call", keys)
}
return
}
if want := len(tt.wantKeys) + len(tt.wantRepin); len(spy.children) != want {
t.Fatalf("started %d children, want %d", len(spy.children), want)
}
})
}
}
// TestWorkflowCallWithoutADeclaredPathRefuses pins the one state a call cannot
// key on: a step run with no declared path. The store's unique index ignores
// an empty key, so without the refusal every attempt would start a fresh child
// -- the duplicate the key exists to prevent.
func TestWorkflowCallWithoutADeclaredPathRefuses(t *testing.T) {
spy := &callSpy{children: map[string]*task.Task{}}
tc := &TaskContext{Task: &task.Task{ID: 7}, Calls: spy, Log: slog.Default()}
err := runWorkflowCall(context.Background(), workflowCallSettings{Workflow: "callee"}, tc)
if err == nil || !strings.Contains(err.Error(), "no declared path") {
t.Fatalf("err = %v, want a refusal naming the missing path", err)
}
if keys := spy.startedKeys(); len(keys) != 0 {
t.Fatalf("started %q, want no call", keys)
}
}
// TestWorkflowCallRecordsTheSiteNotTheKey pins the two values apart: the key
// that identifies the child carries the callee, while the step row and the
// call's events are named and staged for the call site, so a rail pairing an
// event with its step by stage can still attach them.
func TestWorkflowCallRecordsTheSiteNotTheKey(t *testing.T) {
registry := StepRegistry{WorkflowCallStepName: newWorkflowCallStage}
wf, err := ParseAndCompile(`id: caller
repository: none
steps:
- {id: check, type: workflow.call, settings: {workflow: callee}}
`, registry)
if err != nil {
t.Fatalf("compile: %v", err)
}
store := &fakeStepStore{}
spy := &callSpy{children: map[string]*task.Task{}}
tc := &TaskContext{Task: &task.Task{ID: 7, Attempt: 1}, Store: store, StepID: 3, Calls: spy, Log: slog.Default()}
if err := wf.Stages[0].Run(context.Background(), tc); err != nil {
t.Fatalf("run: %v", err)
}
if keys := spy.startedKeys(); !slices.Equal(keys, []string{"check@callee"}) {
t.Fatalf("keys = %q, want [check@callee]", keys)
}
if len(store.started) != 1 || store.started[0].Kind != task.StepKindCall || store.started[0].Name != "check" {
t.Fatalf("started = %#v, want one call step named check", store.started)
}
if len(store.events) != 1 || store.events[0].Kind != events.KindWorkflowCallStarted || store.events[0].Stage != "check" {
t.Fatalf("events = %#v, want one %s staged check", store.events, events.KindWorkflowCallStarted)
}
}
func TestValidateWorkflowCallsWalksBranches(t *testing.T) {
registry := StepRegistry{
AgentRunStepName: newAgentRunStage,
FinishStepName: newFinishStage,
WorkflowCallStepName: newWorkflowCallStage,
}
tests := []struct {
name string
defs map[string]string
wantErr string
}{
{
name: "a branch call to a defined workflow is accepted",
defs: map[string]string{
"caller": `id: caller
repository: none
steps:
- parallel:
a: [{id: check, type: workflow.call, settings: {workflow: callee}}]
b: [{type: agent.run, settings: {mission: m, read_only: true}}]
`,
"callee": `id: callee
repository: none
steps:
- type: workflow.finish
`,
},
},
{
name: "a branch call to an unknown workflow is refused naming the step",
defs: map[string]string{
"caller": `id: caller
repository: none
steps:
- parallel:
a: [{id: check, type: workflow.call, settings: {workflow: missing}}]
b: [{type: agent.run, settings: {mission: m, read_only: true}}]
`,
},
wantErr: `workflow "caller" step 1 branch "a" step 1 calls "missing", which is not defined`,
},
{
name: "a cycle through branch calls is refused",
defs: map[string]string{
"a": `id: a
repository: none
steps:
- parallel:
x: [{id: check, type: workflow.call, settings: {workflow: b}}]
y: [{type: agent.run, settings: {mission: m, read_only: true}}]
`,
"b": `id: b
repository: none
steps:
- id: call-back
type: workflow.call
settings: {workflow: a}
`,
},
wantErr: "calls itself through",
},
{
name: "a later step reads a waited call's declared output",
defs: map[string]string{
"caller": `id: caller
repository: none
steps:
- {id: check, type: workflow.call, settings: {workflow: callee, wait: true}}
- type: agent.run
settings: {mission: "{{ steps.check.result.verdict }}"}
`,
"callee": `id: callee
repository: none
outputs: {verdict: {type: string}}
steps:
- type: workflow.finish
`,
},
},
{
name: "a call result field the callee does not declare is refused",
defs: map[string]string{
"caller": `id: caller
repository: none
steps:
- {id: check, type: workflow.call, settings: {workflow: callee, wait: true}}
- type: agent.run
settings: {mission: "{{ steps.check.result.verdit }}"}
`,
"callee": `id: callee
repository: none
outputs: {verdict: {type: string}}
steps:
- type: workflow.finish
`,
},
wantErr: `the workflow call "check" declares no output "verdit"`,
},
{
name: "a call that does not wait has no result",
defs: map[string]string{
"caller": `id: caller
repository: none
steps:
- {id: check, type: workflow.call, settings: {workflow: callee, wait: false}}
- type: agent.run
settings: {mission: "{{ steps.check.result.verdict }}"}
`,
"callee": `id: callee
repository: none
outputs: {verdict: {type: string}}
steps:
- type: workflow.finish
`,
},
wantErr: `call "check" does not wait, so it has no result`,
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
collection := task.WorkflowDefinitionCollection{}
for _, id := range slices.Sorted(maps.Keys(tt.defs)) {
collection.Definitions = append(collection.Definitions, task.WorkflowDefinitionEntry{ID: id, YAML: tt.defs[id]})
}
err := ValidateDefinitionCollection(collection, registry)
if tt.wantErr == "" {
if err != nil {
t.Fatalf("err = %v, want nil", err)
}
return
}
if err == nil || !strings.Contains(err.Error(), tt.wantErr) {
t.Fatalf("err = %v, want %q", err, tt.wantErr)
}
})
}
}
// wobblyCaller models a dependency that is briefly unavailable: CallStatus
// fails failReads times before it answers, and counts the reads and the
// children it was asked to start.
type wobblyCaller struct {
mu sync.Mutex
failReads int
failWith error
reads int
started int
status string
detail string
outputs map[string]any
}
func (c *wobblyCaller) StartCall(_ context.Context, callerTaskID int64, _, _ string, _ map[string]any) (*task.Task, error) {
c.mu.Lock()
defer c.mu.Unlock()
c.started++
return &task.Task{ID: 42, CallParentTaskID: callerTaskID}, nil
}
func (c *wobblyCaller) CallStatus(context.Context, int64, int64) (string, string, map[string]any, error) {
c.mu.Lock()
defer c.mu.Unlock()
c.reads++
if c.reads <= c.failReads {
return "", "", nil, c.failWith
}
return c.status, c.detail, c.outputs, nil
}
func (c *wobblyCaller) counts() (reads, started int) {
c.mu.Lock()
defer c.mu.Unlock()
return c.reads, c.started
}
// shortenCallWaits makes the caller's polls and retry backoff immediate, so a
// test drives the retry loop without sleeping. Restored by t.Cleanup.
func shortenCallWaits(t *testing.T) {
t.Helper()
oldPoll, oldBackoff, oldMax := callPollInterval, callReadBackoff, callReadMaxBackoff
callPollInterval, callReadBackoff, callReadMaxBackoff = time.Millisecond, time.Millisecond, 2*time.Millisecond
t.Cleanup(func() { callPollInterval, callReadBackoff, callReadMaxBackoff = oldPoll, oldBackoff, oldMax })
}
// TestAwaitCalleeRetriesTransientReadFailures pins that a callee status read
// that fails because the dependency is briefly unavailable is retried, not
// failed: the child is still running, and the caller reads it again.
func TestAwaitCalleeRetriesTransientReadFailures(t *testing.T) {
shortenCallWaits(t)
caller := &wobblyCaller{
failReads: 2,
failWith: fmt.Errorf("dial state store: %w", storecontract.ErrUnavailable),
status: StatusCompleted,
detail: "done",
outputs: map[string]any{"report": "written"},
}
tc := &TaskContext{Task: &task.Task{ID: 7}, Calls: caller, Log: slog.Default()}
outputs, err := awaitCallee(context.Background(), workflowCallSettings{Workflow: "callee", Wait: true}, tc, "check", 42)
if err != nil {
t.Fatalf("awaitCallee: %v", err)
}
if outputs["report"] != "written" {
t.Fatalf("outputs = %v, want report=written", outputs)
}
if reads, _ := caller.counts(); reads != 3 {
t.Fatalf("reads = %d, want 3 (two retries, then the answer)", reads)
}
}
// TestAwaitCalleeFailsPermanentReadErrorAtOnce pins the other half: a contract
// error is not retried, because it will fail the same way every time.
func TestAwaitCalleeFailsPermanentReadErrorAtOnce(t *testing.T) {
shortenCallWaits(t)
caller := &wobblyCaller{failReads: 5, failWith: storecontract.ErrCallNotYours, status: StatusCompleted}
tc := &TaskContext{Task: &task.Task{ID: 7}, Calls: caller, Log: slog.Default()}
_, err := awaitCallee(context.Background(), workflowCallSettings{Workflow: "callee", Wait: true}, tc, "check", 42)
if !errors.Is(err, storecontract.ErrCallNotYours) {
t.Fatalf("err = %v, want ErrCallNotYours", err)
}
if reads, _ := caller.counts(); reads != 1 {
t.Fatalf("reads = %d, want 1 (no retry of a contract error)", reads)
}
}
// TestAwaitCalleeRetryStaysInsideTheDeadline pins the bound on recovery: an
// outage that outlasts the caller's budget ends the wait there rather than
// retrying past it.
func TestAwaitCalleeRetryStaysInsideTheDeadline(t *testing.T) {
shortenCallWaits(t)
caller := &wobblyCaller{failReads: 1 << 30, failWith: fmt.Errorf("dial: %w", storecontract.ErrUnavailable)}
tc := &TaskContext{Task: &task.Task{ID: 7}, Calls: caller, Log: slog.Default()}
ctx, cancel := context.WithTimeout(context.Background(), 30*time.Millisecond)
defer cancel()
_, err := awaitCallee(ctx, workflowCallSettings{Workflow: "callee", Wait: true}, tc, "check", 42)
if !errors.Is(err, context.DeadlineExceeded) {
t.Fatalf("err = %v, want the caller's deadline", err)
}
}
// TestWorkflowCallRetriesARefusedReadWithoutASecondChild pins the whole step:
// a wait:true call whose status reads are briefly refused still succeeds, the
// caller starts exactly one child, and the call step is closed once.
func TestWorkflowCallRetriesARefusedReadWithoutASecondChild(t *testing.T) {
shortenCallWaits(t)
registry := StepRegistry{WorkflowCallStepName: newWorkflowCallStage}
wf, err := ParseAndCompile(`id: caller
repository: none
steps:
- {id: check, type: workflow.call, settings: {workflow: callee, wait: true}}
`, registry)
if err != nil {
t.Fatalf("compile: %v", err)
}
store := &fakeStepStore{}
caller := &wobblyCaller{
failReads: 1,
failWith: fmt.Errorf("dial: %w", storecontract.ErrUnavailable),
status: StatusCompleted,
outputs: map[string]any{"verdict": "go"},
}
tc := &TaskContext{Task: &task.Task{ID: 7, Attempt: 1}, Store: store, StepID: 3, Calls: caller, Log: slog.Default()}
if err := wf.Stages[0].Run(context.Background(), tc); err != nil {
t.Fatalf("run: %v", err)
}
reads, started := caller.counts()
if started != 1 {
t.Fatalf("started %d children, want 1: a retried read must not start another", started)
}
if reads != 2 {
t.Fatalf("reads = %d, want 2 (one refused, one answer)", reads)
}
if tc.stepResult.Result["verdict"] != "go" {
t.Fatalf("step result = %v, want the callee's outputs", tc.stepResult.Result)
}
calls := 0
for _, s := range store.started {
if s.Kind == task.StepKindCall {
calls++
}
}
if calls != 1 {
t.Fatalf("recorded %d call steps, want 1", calls)
}
}