fix(workflow): key a call on its callee, and free a terminal child's key
A call's key was the step's declared path alone, so one task running a second definition at a colliding path -- workflow.handoff, human.approve and a definition re-pin all change it -- was handed the first callee's child, waited on the wrong workflow and published its outputs. The key now carries the callee too, while the step row and the call's events stay named and staged for the call site, so a rail pairing an event with its step still attaches them. A child that reached dead or closed_wont_do can never do the work and holds no result the caller can use, so it leaves the call key's index and frees the key for the caller's retry to start a fresh child. parked and rejected keep theirs: a parked child is recoverable by retrying it, and a rejected child ran. The id a call step must declare is now refused where the definition is compiled, which is also the path a pinned definition takes, so an id that spells a step type is accepted again and no stage needs a heuristic.
This commit is contained in:
1 parent
0101947131
commit
703aea4568
13 files changed
+302
-109
No files matched your search
@@ -252,6 +252,10 @@ _runs without a repository._
|
||||
| `wait` | bool | Wait for the child run to finish before continuing. |
|
||||
| `workflow` | text | The workflow to start as a child run. |
|
||||
|
||||
The step must declare an `id`: the step's id is the key of the call it starts,
|
||||
so a retry of that call site returns the child already started instead of
|
||||
starting a second one.
|
||||
|
||||
### `workflow.finish`
|
||||
|
||||
_runs without a repository._
|
||||
|
||||
@@ -12114,10 +12114,12 @@ type EnqueueCallTaskRequest struct {
|
||||
Workflow string `protobuf:"bytes,2,opt,name=workflow,proto3" json:"workflow,omitempty"`
|
||||
InputsJson string `protobuf:"bytes,3,opt,name=inputs_json,json=inputsJson,proto3" json:"inputs_json,omitempty"`
|
||||
// call_key is the call site's durable identity: the call step's declared
|
||||
// path within the run. The store stamps it on the child and, when the
|
||||
// caller already started a child under it, returns that child instead of
|
||||
// starting a second one. Required: a retry of one call site carries the
|
||||
// same key, so the retry cannot duplicate the child's work.
|
||||
// path within the run, plus the workflow it starts (so one task running a
|
||||
// second definition at a colliding path does not reuse the first callee's
|
||||
// child). The store stamps it on the child and, when the caller already
|
||||
// started a child under it, returns that child instead of starting a second
|
||||
// one. Required: a retry of one call site carries the same key, so the retry
|
||||
// cannot duplicate the child's work.
|
||||
CallKey string `protobuf:"bytes,4,opt,name=call_key,json=callKey,proto3" json:"call_key,omitempty"`
|
||||
unknownFields protoimpl.UnknownFields
|
||||
sizeCache protoimpl.SizeCache
|
||||
|
||||
@@ -72,6 +72,14 @@ func checkTypedStep(step StepRecord, mode task.RepositoryMode, earlier *refScope
|
||||
if !ok {
|
||||
return fmt.Errorf("unknown type %q", step.Type)
|
||||
}
|
||||
// A workflow.call's step id is the identity of the call it starts -- it
|
||||
// keys the callee, so a retry of the call site returns the child already
|
||||
// started rather than a second one. It is refused here rather than in
|
||||
// validateWorkflowCalls because a pinned definition is compiled on its own
|
||||
// (ParseAndCompile), never through the collection.
|
||||
if step.Type == WorkflowCallStepName && step.ID == "" {
|
||||
return fmt.Errorf("%q needs an id: the step's id is the key of the call it starts", step.Type)
|
||||
}
|
||||
if mode != task.RepositoryRequired && NeedsRepository(step.Type) {
|
||||
return fmt.Errorf("%q needs a repository, but the workflow's repository is %s", step.Type, mode)
|
||||
}
|
||||
|
||||
@@ -15,6 +15,7 @@ import (
|
||||
type fakeStepStore struct {
|
||||
started []task.StepStart
|
||||
finished []task.StepFinish
|
||||
events []events.Event
|
||||
nextID int64
|
||||
}
|
||||
|
||||
@@ -24,7 +25,10 @@ func (f *fakeStepStore) Transition(context.Context, int64, string, string, strin
|
||||
return nil
|
||||
}
|
||||
|
||||
func (f *fakeStepStore) InsertEvent(context.Context, events.Event) (int64, error) { return 0, nil }
|
||||
func (f *fakeStepStore) InsertEvent(_ context.Context, e events.Event) (int64, error) {
|
||||
f.events = append(f.events, e)
|
||||
return 0, nil
|
||||
}
|
||||
|
||||
func (f *fakeStepStore) StartStep(_ context.Context, s task.StepStart) (int64, events.Event, error) {
|
||||
f.nextID++
|
||||
|
||||
@@ -6,8 +6,9 @@ import "context"
|
||||
type Caller interface {
|
||||
// StartCall enqueues the callee for callerTaskID's call step at depth + 1.
|
||||
// callKey is the call site's durable identity -- the call step's declared
|
||||
// path within the run -- and is required: the same caller and key return
|
||||
// the child already started, so a retry of a call site starts one child.
|
||||
// path within the run, plus the workflow it starts -- and is required: the
|
||||
// same caller and key return the child already started, so a retry of a
|
||||
// call site starts one child.
|
||||
// It fails if the caller is not running or the depth limit is exceeded.
|
||||
StartCall(ctx context.Context, callerTaskID int64, callKey, workflow string, inputs map[string]any) (*Task, error)
|
||||
// CallStatus reads one call's callee: its status, latest transition detail
|
||||
|
||||
@@ -103,13 +103,18 @@ func runWorkflowCall(ctx context.Context, s workflowCallSettings, tc *TaskContex
|
||||
if tc.Calls == nil {
|
||||
return fmt.Errorf("%s: this runner carries no callee capability", WorkflowCallStepName)
|
||||
}
|
||||
// The key is what makes a retry of this call site return the same child,
|
||||
// so a call that cannot name itself refuses rather than sharing a key with
|
||||
// its siblings.
|
||||
key, err := workflowCallKey(tc)
|
||||
// The key is what makes a retry of this call site return the same child: the
|
||||
// call site's declared path, plus the callee. The callee belongs in it
|
||||
// because one task can run a second definition at a colliding path
|
||||
// (workflow.handoff and human.approve set the task's workflow, and the
|
||||
// daemon re-pins when the pin's id differs), and that run must not be
|
||||
// handed the first callee's child. A step path cannot contain "@": its
|
||||
// segments are stable identifiers joined with "/".
|
||||
path, err := workflowCallPath(tc)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
key := path + "@" + s.Workflow
|
||||
if tc.Task.CallDepth+1 > MaxCallDepth {
|
||||
return fmt.Errorf("%s: call depth %d would pass the limit of %d", WorkflowCallStepName, tc.Task.CallDepth+1, MaxCallDepth)
|
||||
}
|
||||
@@ -122,13 +127,14 @@ func runWorkflowCall(ctx context.Context, s workflowCallSettings, tc *TaskContex
|
||||
return fmt.Errorf("%s: start %q: %w", WorkflowCallStepName, s.Workflow, err)
|
||||
}
|
||||
started := map[string]any{"callee_task_id": callee.ID, "workflow": s.Workflow, "wait": s.Wait}
|
||||
if err := tc.EmitDurable(ctx, events.KindWorkflowCallStarted, WorkflowCallStepName,
|
||||
if err := tc.EmitDurable(ctx, events.KindWorkflowCallStarted, path,
|
||||
fmt.Sprintf("started %q as task %d", s.Workflow, callee.ID), started); err != nil {
|
||||
tc.Log.Warn("workflow call start not persisted", "err", err)
|
||||
}
|
||||
// Record the call as a step holding the callee's execution ID, named for
|
||||
// the call site so the dashboard shows which call started it.
|
||||
callStep, _, err := tc.startCallStep(ctx, key, callee.ID)
|
||||
// the call site -- the path, not the key that also carries the callee --
|
||||
// so the dashboard shows which call started it.
|
||||
callStep, _, err := tc.startCallStep(ctx, path, callee.ID)
|
||||
if err != nil {
|
||||
return fmt.Errorf("%s: could not be recorded: %w", WorkflowCallStepName, err)
|
||||
}
|
||||
@@ -143,7 +149,7 @@ func runWorkflowCall(ctx context.Context, s workflowCallSettings, tc *TaskContex
|
||||
}
|
||||
// Close the call step with the callee's outcome, unless the caller itself was
|
||||
// cancelled.
|
||||
calleeOutputs, waitErr := awaitCallee(ctx, s, tc, callee.ID)
|
||||
calleeOutputs, waitErr := awaitCallee(ctx, s, tc, path, callee.ID)
|
||||
if waitErr != nil {
|
||||
if ctx.Err() == nil {
|
||||
if ferr := tc.finishChildStep(ctx, callStep, taskstate.StepFailed, waitErr.Error(), 0); ferr != nil {
|
||||
@@ -170,28 +176,26 @@ func runWorkflowCall(ctx context.Context, s workflowCallSettings, tc *TaskContex
|
||||
return nil
|
||||
}
|
||||
|
||||
// workflowCallKey is the running call's durable identity: the step's declared
|
||||
// path within the run. validateWorkflowCalls refuses a call step with no id at
|
||||
// save, but a definition pinned before that rule still compiles and runs: its
|
||||
// path is the bare step type, one key shared by every call site in the run, so
|
||||
// the second call site would be handed the first's child. A stage run outside
|
||||
// the step wrapper has no path at all, which is the same failure.
|
||||
func workflowCallKey(tc *TaskContext) (string, error) {
|
||||
path := tc.stepPath
|
||||
if path == "" || path == WorkflowCallStepName || strings.HasSuffix(path, "/"+WorkflowCallStepName) {
|
||||
return "", fmt.Errorf("%s: the step declares no id, and the step's id is the call's key", WorkflowCallStepName)
|
||||
// workflowCallPath is the running call site's declared path within the run,
|
||||
// which names the step and identifies the call. Every registry stage is
|
||||
// wrapped, so a run reaching here without a path has no identity to key its
|
||||
// child on: the store's index ignores an empty key, so keying on one would
|
||||
// start a fresh child on every attempt -- the duplicate the key prevents.
|
||||
func workflowCallPath(tc *TaskContext) (string, error) {
|
||||
if tc.stepPath == "" {
|
||||
return "", fmt.Errorf("%s: the running step has no declared path, so this call has no key", WorkflowCallStepName)
|
||||
}
|
||||
return path, nil
|
||||
return tc.stepPath, nil
|
||||
}
|
||||
|
||||
// startCallStep records the call step for a callee this run just started.
|
||||
func (tc *TaskContext) startCallStep(ctx context.Context, key string, calleeTaskID int64) (int64, events.Event, error) {
|
||||
func (tc *TaskContext) startCallStep(ctx context.Context, path string, calleeTaskID int64) (int64, events.Event, error) {
|
||||
if tc.Store == nil {
|
||||
return 0, events.Event{}, nil
|
||||
}
|
||||
stepID, event, err := tc.Store.StartStep(ctx, task.StepStart{
|
||||
ExecutionID: tc.Task.ID, Attempt: tc.Task.Attempt, ParentID: tc.StepID,
|
||||
Kind: task.StepKindCall, Name: key, CalledExecutionID: calleeTaskID,
|
||||
Kind: task.StepKindCall, Name: path, CalledExecutionID: calleeTaskID,
|
||||
})
|
||||
if err != nil {
|
||||
return 0, events.Event{}, err
|
||||
@@ -244,7 +248,7 @@ func callEnded(status string) bool {
|
||||
// the callee's detail when the callee did not succeed, and returning the
|
||||
// callee's written outputs when it did. The caller's own wall clock bounds
|
||||
// the wait.
|
||||
func awaitCallee(ctx context.Context, s workflowCallSettings, tc *TaskContext, callTaskID int64) (map[string]any, error) {
|
||||
func awaitCallee(ctx context.Context, s workflowCallSettings, tc *TaskContext, path string, callTaskID int64) (map[string]any, error) {
|
||||
var outputs map[string]any
|
||||
for {
|
||||
status, detail, latestOutputs, err := tc.Calls.CallStatus(ctx, tc.Task.ID, callTaskID)
|
||||
@@ -254,7 +258,7 @@ func awaitCallee(ctx context.Context, s workflowCallSettings, tc *TaskContext, c
|
||||
outputs = latestOutputs
|
||||
if callEnded(status) {
|
||||
if callSucceeded(status) {
|
||||
if err := tc.EmitDurable(ctx, events.KindWorkflowCallFinished, WorkflowCallStepName,
|
||||
if err := tc.EmitDurable(ctx, events.KindWorkflowCallFinished, path,
|
||||
fmt.Sprintf("%q (task %d) finished: %s", s.Workflow, callTaskID, detail),
|
||||
map[string]any{"callee_task_id": callTaskID, "workflow": s.Workflow, "status": status, "detail": detail}); err != nil {
|
||||
tc.Log.Warn("workflow call finish not persisted", "err", err)
|
||||
@@ -294,12 +298,6 @@ func validateWorkflowCalls(parsed map[string]YAMLDefinition) error {
|
||||
if step.Type != WorkflowCallStepName {
|
||||
return nil
|
||||
}
|
||||
// The step's id is the call's key, so a call without one has no way
|
||||
// to say which call site it is, and two of them in one run would
|
||||
// share a child.
|
||||
if step.ID == "" {
|
||||
return fmt.Errorf("%s: %s declares no id, and the step's id is the call's key", where, WorkflowCallStepName)
|
||||
}
|
||||
s, err := decodeCallSettings(step)
|
||||
if err != nil {
|
||||
return fmt.Errorf("%s: %w", where, err)
|
||||
|
||||
@@ -10,6 +10,7 @@ import (
|
||||
"testing"
|
||||
|
||||
"github.com/samcharles93/archie-core/internal/domain/workflow/task"
|
||||
"github.com/samcharles93/archie-core/internal/events"
|
||||
)
|
||||
|
||||
// callSpy records the key every call site started its child under, and models
|
||||
@@ -42,26 +43,33 @@ func (c *callSpy) startedKeys() []string {
|
||||
return slices.Clone(c.keys)
|
||||
}
|
||||
|
||||
// TestWorkflowCallKeyIsTheDeclaredPath pins the call's durable identity: two
|
||||
// runs of one call site -- a retry of the stage, a lost EnqueueCallTask reply
|
||||
// -- key on the same declared path and start one child, and two call sites in
|
||||
// one run key on different paths so neither returns the other's child.
|
||||
// TestWorkflowCallKeyIsTheDeclaredPath pins the call's durable identity: the
|
||||
// step's declared path plus the callee it starts. Two runs of one call site --
|
||||
// a retry of the stage, a lost EnqueueCallTask reply -- key on the same value
|
||||
// and start one child, and neither a sibling call site nor the same path under
|
||||
// a re-pinned definition reuses another call's child.
|
||||
func TestWorkflowCallKeyIsTheDeclaredPath(t *testing.T) {
|
||||
registry := StepRegistry{WorkflowCallStepName: newWorkflowCallStage}
|
||||
tests := []struct {
|
||||
name string
|
||||
def string
|
||||
wantKeys []string
|
||||
wantErr string
|
||||
name string
|
||||
def string
|
||||
// repin is the definition a later run compiles instead. One task can
|
||||
// run a second definition -- workflow.handoff and human.approve set the
|
||||
// task's workflow, and the daemon re-pins when the pin's id differs --
|
||||
// so a call path that collides there must not reuse the first callee.
|
||||
repin string
|
||||
wantKeys []string
|
||||
wantRepin []string
|
||||
wantErr string
|
||||
}{
|
||||
{
|
||||
name: "a call step keys on its declared id",
|
||||
name: "a call step keys on its declared id and the callee it starts",
|
||||
def: `id: caller
|
||||
repository: none
|
||||
steps:
|
||||
- {id: check, type: workflow.call, settings: {workflow: callee}}
|
||||
`,
|
||||
wantKeys: []string{"check"},
|
||||
wantKeys: []string{"check@callee"},
|
||||
},
|
||||
{
|
||||
name: "a call in a parallel branch keys on its path within the run",
|
||||
@@ -73,63 +81,144 @@ steps:
|
||||
a: [{id: check, type: workflow.call, settings: {workflow: callee}}]
|
||||
b: [{id: verify, type: workflow.call, settings: {workflow: callee}}]
|
||||
`,
|
||||
wantKeys: []string{"fanout/a/check", "fanout/b/verify"},
|
||||
wantKeys: []string{"fanout/a/check@callee", "fanout/b/verify@callee"},
|
||||
},
|
||||
{
|
||||
// A definition pinned before the rule that a call declares an id
|
||||
// compiles and runs: were its path the bare step type, every call
|
||||
// site in the run would share one key and the second would return
|
||||
// the first's child.
|
||||
name: "a call step with no id is refused, not keyed on its type",
|
||||
name: "the same path under a re-pinned definition keys on its callee",
|
||||
def: `id: caller
|
||||
repository: none
|
||||
steps:
|
||||
- {id: classify, type: workflow.call, settings: {workflow: callee-a}}
|
||||
`,
|
||||
repin: `id: caller
|
||||
repository: none
|
||||
steps:
|
||||
- {id: classify, type: workflow.call, settings: {workflow: callee-b}}
|
||||
`,
|
||||
wantKeys: []string{"classify@callee-a"},
|
||||
wantRepin: []string{"classify@callee-b"},
|
||||
},
|
||||
{
|
||||
// A step id may spell a step type: it is a stable identifier like
|
||||
// any other, and the path it declares is the call's identity.
|
||||
name: "a step id that spells the step type is accepted",
|
||||
def: `id: caller
|
||||
repository: none
|
||||
steps:
|
||||
- {id: workflow.call, type: workflow.call, settings: {workflow: callee}}
|
||||
`,
|
||||
wantKeys: []string{"workflow.call@callee"},
|
||||
},
|
||||
{
|
||||
// The step's id is the call's identity, so an id-less call step is
|
||||
// refused where its definition is compiled -- which is also the path
|
||||
// a pinned definition takes.
|
||||
name: "a call step with no id is refused",
|
||||
def: `id: caller
|
||||
repository: none
|
||||
steps:
|
||||
- type: workflow.call
|
||||
settings: {workflow: callee}
|
||||
`,
|
||||
wantErr: "declares no id",
|
||||
wantErr: "needs an id",
|
||||
},
|
||||
}
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
wf, err := ParseAndCompile(tt.def, registry)
|
||||
if err != nil {
|
||||
t.Fatalf("compile: %v", err)
|
||||
}
|
||||
spy := &callSpy{children: map[string]*task.Task{}}
|
||||
tc := &TaskContext{Task: &task.Task{ID: 7}, Calls: spy, Log: slog.Default()}
|
||||
stage := wf.Stages[0]
|
||||
// The same stage twice: a requeued attempt re-runs the call site.
|
||||
for range 2 {
|
||||
if err := stage.Run(context.Background(), tc); err != nil {
|
||||
if tt.wantErr == "" || !strings.Contains(err.Error(), tt.wantErr) {
|
||||
t.Fatalf("err = %v, want %q", err, tt.wantErr)
|
||||
}
|
||||
if keys := spy.startedKeys(); len(keys) != 0 {
|
||||
t.Fatalf("started %q, want no call", keys)
|
||||
}
|
||||
return
|
||||
}
|
||||
// Twice: a requeued attempt re-runs the call site.
|
||||
second := tt.def
|
||||
if tt.repin != "" {
|
||||
second = tt.repin
|
||||
}
|
||||
for i, def := range []string{tt.def, second} {
|
||||
wf, err := ParseAndCompile(def, registry)
|
||||
if tt.wantErr != "" {
|
||||
t.Fatalf("err = nil, want %q", tt.wantErr)
|
||||
if err == nil || !strings.Contains(err.Error(), tt.wantErr) {
|
||||
t.Fatalf("compile err = %v, want %q", err, tt.wantErr)
|
||||
}
|
||||
continue
|
||||
}
|
||||
if err != nil {
|
||||
t.Fatalf("compile: %v", err)
|
||||
}
|
||||
before := len(spy.startedKeys())
|
||||
if err := wf.Stages[0].Run(context.Background(), tc); err != nil {
|
||||
t.Fatalf("run: %v", err)
|
||||
}
|
||||
want := tt.wantKeys
|
||||
if i == 1 && tt.repin != "" {
|
||||
want = tt.wantRepin
|
||||
}
|
||||
got := spy.startedKeys()[before:]
|
||||
// Branches finish in whatever order; the keys a run used, not
|
||||
// their order, are the behaviour.
|
||||
slices.Sort(got)
|
||||
if !slices.Equal(got, want) {
|
||||
t.Fatalf("run %d keys = %q, want %q", i+1, got, want)
|
||||
}
|
||||
}
|
||||
keys := spy.startedKeys()
|
||||
first, second := keys[:len(keys)/2], keys[len(keys)/2:]
|
||||
// Branches finish in whatever order; the keys a run used, not their
|
||||
// order, are the behaviour.
|
||||
slices.Sort(first)
|
||||
slices.Sort(second)
|
||||
if !slices.Equal(first, tt.wantKeys) || !slices.Equal(second, tt.wantKeys) {
|
||||
t.Fatalf("keys = %q then %q, want %q both times", first, second, tt.wantKeys)
|
||||
if tt.wantErr != "" {
|
||||
if keys := spy.startedKeys(); len(keys) != 0 {
|
||||
t.Fatalf("started %q, want no call", keys)
|
||||
}
|
||||
return
|
||||
}
|
||||
if len(spy.children) != len(tt.wantKeys) {
|
||||
t.Fatalf("started %d children, want %d", len(spy.children), len(tt.wantKeys))
|
||||
if want := len(tt.wantKeys) + len(tt.wantRepin); len(spy.children) != want {
|
||||
t.Fatalf("started %d children, want %d", len(spy.children), want)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// TestWorkflowCallWithoutADeclaredPathRefuses pins the one state a call cannot
|
||||
// key on: a step run with no declared path. The store's unique index ignores
|
||||
// an empty key, so without the refusal every attempt would start a fresh child
|
||||
// -- the duplicate the key exists to prevent.
|
||||
func TestWorkflowCallWithoutADeclaredPathRefuses(t *testing.T) {
|
||||
spy := &callSpy{children: map[string]*task.Task{}}
|
||||
tc := &TaskContext{Task: &task.Task{ID: 7}, Calls: spy, Log: slog.Default()}
|
||||
err := runWorkflowCall(context.Background(), workflowCallSettings{Workflow: "callee"}, tc)
|
||||
if err == nil || !strings.Contains(err.Error(), "no declared path") {
|
||||
t.Fatalf("err = %v, want a refusal naming the missing path", err)
|
||||
}
|
||||
if keys := spy.startedKeys(); len(keys) != 0 {
|
||||
t.Fatalf("started %q, want no call", keys)
|
||||
}
|
||||
}
|
||||
|
||||
// TestWorkflowCallRecordsTheSiteNotTheKey pins the two values apart: the key
|
||||
// that identifies the child carries the callee, while the step row and the
|
||||
// call's events are named and staged for the call site, so a rail pairing an
|
||||
// event with its step by stage can still attach them.
|
||||
func TestWorkflowCallRecordsTheSiteNotTheKey(t *testing.T) {
|
||||
registry := StepRegistry{WorkflowCallStepName: newWorkflowCallStage}
|
||||
wf, err := ParseAndCompile(`id: caller
|
||||
repository: none
|
||||
steps:
|
||||
- {id: check, type: workflow.call, settings: {workflow: callee}}
|
||||
`, registry)
|
||||
if err != nil {
|
||||
t.Fatalf("compile: %v", err)
|
||||
}
|
||||
store := &fakeStepStore{}
|
||||
spy := &callSpy{children: map[string]*task.Task{}}
|
||||
tc := &TaskContext{Task: &task.Task{ID: 7, Attempt: 1}, Store: store, StepID: 3, Calls: spy, Log: slog.Default()}
|
||||
if err := wf.Stages[0].Run(context.Background(), tc); err != nil {
|
||||
t.Fatalf("run: %v", err)
|
||||
}
|
||||
if keys := spy.startedKeys(); !slices.Equal(keys, []string{"check@callee"}) {
|
||||
t.Fatalf("keys = %q, want [check@callee]", keys)
|
||||
}
|
||||
if len(store.started) != 1 || store.started[0].Kind != task.StepKindCall || store.started[0].Name != "check" {
|
||||
t.Fatalf("started = %#v, want one call step named check", store.started)
|
||||
}
|
||||
if len(store.events) != 1 || store.events[0].Kind != events.KindWorkflowCallStarted || store.events[0].Stage != "check" {
|
||||
t.Fatalf("events = %#v, want one %s staged check", store.events, events.KindWorkflowCallStarted)
|
||||
}
|
||||
}
|
||||
|
||||
func TestValidateWorkflowCallsWalksBranches(t *testing.T) {
|
||||
registry := StepRegistry{
|
||||
AgentRunStepName: newAgentRunStage,
|
||||
@@ -185,7 +274,7 @@ steps:
|
||||
- type: workflow.finish
|
||||
`,
|
||||
},
|
||||
wantErr: `workflow "caller" step 1: workflow.call declares no id`,
|
||||
wantErr: `workflow "caller" step 1: "workflow.call" needs an id`,
|
||||
},
|
||||
{
|
||||
name: "a cycle through branch calls is refused",
|
||||
|
||||
@@ -1,14 +1,23 @@
|
||||
-- tasks.call_key is a workflow.call call site's durable identity: the call
|
||||
-- step's declared path within the calling run. With the parent it is the key
|
||||
-- EnqueueCallTask upserts on, so a retry of one call site -- a lost
|
||||
-- EnqueueCallTask reply, a failed step record, a requeued attempt -- returns
|
||||
-- the child already started instead of starting a second one. The index is
|
||||
-- partial: every task that is not a workflow.call callee carries the empty key.
|
||||
-- step's declared path within the calling run, plus the callee it starts. With
|
||||
-- the parent it is the key EnqueueCallTask upserts on, so a retry of one call
|
||||
-- site -- a lost EnqueueCallTask reply, a failed step record, a requeued
|
||||
-- attempt -- returns the child already started instead of starting a second
|
||||
-- one. The index is partial: every task that is not a workflow.call callee
|
||||
-- carries the empty key.
|
||||
--
|
||||
-- A child that reached dead or closed_wont_do (taskstate.Dead and
|
||||
-- taskstate.Declined) can never do the work and holds no result the caller can
|
||||
-- use, so it leaves the index and frees its key for the caller's retry to
|
||||
-- start a fresh child. parked and rejected stay in: a parked child is
|
||||
-- recoverable by retrying the child -- which is what the caller's own park
|
||||
-- names -- and a rejected child ran, so reusing it is what the call site wants.
|
||||
|
||||
-- +goose Up
|
||||
|
||||
ALTER TABLE tasks ADD COLUMN call_key text NOT NULL DEFAULT '';
|
||||
CREATE UNIQUE INDEX idx_tasks_call_key ON tasks (call_parent_task_id, call_key) WHERE call_key <> '';
|
||||
CREATE UNIQUE INDEX idx_tasks_call_key ON tasks (call_parent_task_id, call_key)
|
||||
WHERE call_key <> '' AND status NOT IN ('dead', 'closed_wont_do');
|
||||
|
||||
-- +goose Down
|
||||
|
||||
|
||||
@@ -41,7 +41,7 @@ SELECT
|
||||
caller.id, caller.call_depth + 1, $7
|
||||
FROM caller
|
||||
WHERE caller.status = 'running' AND caller.call_depth + 1 <= $8::int
|
||||
ON CONFLICT (call_parent_task_id, call_key) WHERE call_key <> '' DO UPDATE SET call_key = EXCLUDED.call_key
|
||||
ON CONFLICT (call_parent_task_id, call_key) WHERE call_key <> '' AND status NOT IN ('dead', 'closed_wont_do') DO UPDATE SET call_key = EXCLUDED.call_key
|
||||
RETURNING id, owner, repo, issue_number, title, body, labels, status, workflow, branch, plan, notes, pr_number, tokens_used, iterations, attempt, park_reason, watch_comment_id, park_class, remediation_rounds, retry_count, source, identity, binding_id, binding_version, review_payload, workflow_definition_version, workflow_definition_digest, workflow_definition_yaml, created_at, updated_at, review_cursor, inputs, org_id, workspace_id, call_parent_task_id, call_depth, outputs, review_gate, rereview_rounds, retry_mode, resume_from, resume_results, pending_reviews, origin, call_key
|
||||
`
|
||||
|
||||
@@ -60,9 +60,11 @@ type EnqueueCallTaskParams struct {
|
||||
//
|
||||
// The call key makes this the one write that can be retried: a call site that
|
||||
// already started its child gets that child back rather than a second one. The
|
||||
// upsert only fires for a keyed callee, and the WHERE below still decides
|
||||
// whether the caller may call at all -- a caller that is missing, not running
|
||||
// or past the depth limit inserts nothing and returns no row, conflict or not.
|
||||
// upsert only fires for a keyed callee whose child can still do the work -- a
|
||||
// child that ended dead or closed_wont_do freed its key -- and the WHERE below
|
||||
// still decides whether the caller may call at all: a caller that is missing,
|
||||
// not running or past the depth limit inserts nothing and returns no row,
|
||||
// conflict or not.
|
||||
func (q *Queries) EnqueueCallTask(ctx context.Context, arg EnqueueCallTaskParams) (Task, error) {
|
||||
row := q.db.QueryRow(ctx, enqueueCallTask,
|
||||
arg.ID,
|
||||
|
||||
@@ -3,9 +3,11 @@
|
||||
--
|
||||
-- The call key makes this the one write that can be retried: a call site that
|
||||
-- already started its child gets that child back rather than a second one. The
|
||||
-- upsert only fires for a keyed callee, and the WHERE below still decides
|
||||
-- whether the caller may call at all -- a caller that is missing, not running
|
||||
-- or past the depth limit inserts nothing and returns no row, conflict or not.
|
||||
-- upsert only fires for a keyed callee whose child can still do the work -- a
|
||||
-- child that ended dead or closed_wont_do freed its key -- and the WHERE below
|
||||
-- still decides whether the caller may call at all: a caller that is missing,
|
||||
-- not running or past the depth limit inserts nothing and returns no row,
|
||||
-- conflict or not.
|
||||
WITH caller AS (
|
||||
SELECT t.* FROM tasks t WHERE t.id = $1 FOR UPDATE
|
||||
)
|
||||
@@ -24,7 +26,7 @@ SELECT
|
||||
caller.id, caller.call_depth + 1, sqlc.arg('call_key')
|
||||
FROM caller
|
||||
WHERE caller.status = 'running' AND caller.call_depth + 1 <= sqlc.arg('max_depth')::int
|
||||
ON CONFLICT (call_parent_task_id, call_key) WHERE call_key <> '' DO UPDATE SET call_key = EXCLUDED.call_key
|
||||
ON CONFLICT (call_parent_task_id, call_key) WHERE call_key <> '' AND status NOT IN ('dead', 'closed_wont_do') DO UPDATE SET call_key = EXCLUDED.call_key
|
||||
RETURNING *;
|
||||
|
||||
-- name: CallStatusDetail :one
|
||||
|
||||
@@ -783,8 +783,9 @@ func (s *Store) TaskByID(ctx context.Context, taskID int64) (*workflow.Task, err
|
||||
|
||||
// StartCall enqueues a workflow.call callee inheriting the caller's org,
|
||||
// workspace, identity and repo, and returns the callee already started under
|
||||
// callerTaskID and callKey when there is one. The caller must be running and
|
||||
// within workflow.MaxCallDepth.
|
||||
// callerTaskID and callKey when there is one. A child that ended dead or
|
||||
// declined freed its key, so such a call starts a fresh child. The caller must
|
||||
// be running and within workflow.MaxCallDepth.
|
||||
func (s *Store) StartCall(ctx context.Context, callerTaskID int64, callKey string, wf string, inputs map[string]any) (*workflow.Task, error) {
|
||||
encoded, err := task.EncodeInputs(inputs)
|
||||
if err != nil {
|
||||
|
||||
@@ -4,12 +4,14 @@ import (
|
||||
"testing"
|
||||
|
||||
"github.com/samcharles93/archie-core/internal/infrastructure/postgres/pgstore"
|
||||
"github.com/samcharles93/archie-core/internal/taskstate"
|
||||
)
|
||||
|
||||
// StartCall is idempotent per call site: the same caller and call key return
|
||||
// the child already started, so a retry of a lost EnqueueCallTask reply, or of
|
||||
// the stage that made the call, does not start a second child and run its work
|
||||
// twice. A different call site in the same run still gets its own child.
|
||||
// StartCall is idempotent per call key: the same caller and key return the
|
||||
// child already started, so a retry of a lost EnqueueCallTask reply, or of the
|
||||
// stage that made the call, does not start a second child and run its work
|
||||
// twice. Another call site -- the same declared path starting a different
|
||||
// workflow -- still gets its own child.
|
||||
func TestStartCallReturnsOneChildPerCallKey(t *testing.T) {
|
||||
ctx := t.Context()
|
||||
db := pgstore.Open(t)
|
||||
@@ -21,23 +23,23 @@ func TestStartCallReturnsOneChildPerCallKey(t *testing.T) {
|
||||
t.Fatalf("claim: %v", err)
|
||||
}
|
||||
|
||||
first, err := db.StartCall(ctx, caller.ID, "check", "callee", nil)
|
||||
first, err := db.StartCall(ctx, caller.ID, "check@callee-a", "callee-a", nil)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
retry, err := db.StartCall(ctx, caller.ID, "check", "callee", nil)
|
||||
retry, err := db.StartCall(ctx, caller.ID, "check@callee-a", "callee-a", nil)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if retry.ID != first.ID {
|
||||
t.Fatalf("the retry started task %d, want the child already started, %d", retry.ID, first.ID)
|
||||
}
|
||||
other, err := db.StartCall(ctx, caller.ID, "verify", "callee", nil)
|
||||
other, err := db.StartCall(ctx, caller.ID, "check@callee-b", "callee-b", nil)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if other.ID == first.ID {
|
||||
t.Fatalf("a different call key returned the first child, %d", first.ID)
|
||||
t.Fatalf("the same path starting another workflow returned the first child, %d", first.ID)
|
||||
}
|
||||
|
||||
var rows int
|
||||
@@ -49,3 +51,72 @@ func TestStartCallReturnsOneChildPerCallKey(t *testing.T) {
|
||||
t.Fatalf("the caller started %d children, want one per call key (2)", rows)
|
||||
}
|
||||
}
|
||||
|
||||
// A child that reached a hard terminal failure can never do the work, so it
|
||||
// frees its key: it left no result the caller can use, and the caller's retry
|
||||
// starting a fresh child duplicates nothing. A parked child keeps the key --
|
||||
// it is recoverable by retrying the child, which is what the caller's own park
|
||||
// names.
|
||||
func TestStartCallFreesTheKeyOfAHardTerminalChild(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
// move takes the child out of queued the way its real ending does.
|
||||
move func(t *testing.T, db *pgstore.TaskDB, childID int64)
|
||||
wantNew bool
|
||||
}{
|
||||
{
|
||||
name: "a declined child frees its key",
|
||||
move: func(t *testing.T, db *pgstore.TaskDB, childID int64) {
|
||||
t.Helper()
|
||||
if err := db.Transition(t.Context(), childID, taskstate.Queued, taskstate.Declined, "the operator declined it"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
},
|
||||
wantNew: true,
|
||||
},
|
||||
{
|
||||
name: "a parked child keeps its key",
|
||||
move: func(t *testing.T, db *pgstore.TaskDB, childID int64) {
|
||||
t.Helper()
|
||||
// Parked is reachable only from running, so the child is
|
||||
// claimed first, as a worker would claim it.
|
||||
claimed, err := db.ClaimNext(t.Context())
|
||||
if err != nil || claimed == nil || claimed.ID != childID {
|
||||
t.Fatalf("claim child %d: %v, got %v", childID, err, claimed)
|
||||
}
|
||||
if err := db.ParkTask(t.Context(), childID, taskstate.Running, "needs an operator", taskstate.ParkNeedsHuman); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
},
|
||||
},
|
||||
}
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
ctx := t.Context()
|
||||
db := pgstore.Open(t)
|
||||
if _, err := db.EnqueueChatTask(ctx, "", "", "caller", "", "tdd", "", "", nil); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
caller, err := db.ClaimNext(ctx)
|
||||
if err != nil || caller == nil {
|
||||
t.Fatalf("claim: %v", err)
|
||||
}
|
||||
first, err := db.StartCall(ctx, caller.ID, "check@callee", "callee", nil)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
tt.move(t, db, first.ID)
|
||||
|
||||
retry, err := db.StartCall(ctx, caller.ID, "check@callee", "callee", nil)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if tt.wantNew && retry.ID == first.ID {
|
||||
t.Fatalf("the retry returned the child that can never do the work, %d, want a fresh child", first.ID)
|
||||
}
|
||||
if !tt.wantNew && retry.ID != first.ID {
|
||||
t.Fatalf("the retry started task %d, want the parked child it can retry, %d", retry.ID, first.ID)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
@@ -1261,10 +1261,12 @@ message EnqueueCallTaskRequest {
|
||||
string workflow = 2;
|
||||
string inputs_json = 3;
|
||||
// call_key is the call site's durable identity: the call step's declared
|
||||
// path within the run. The store stamps it on the child and, when the
|
||||
// caller already started a child under it, returns that child instead of
|
||||
// starting a second one. Required: a retry of one call site carries the
|
||||
// same key, so the retry cannot duplicate the child's work.
|
||||
// path within the run, plus the workflow it starts (so one task running a
|
||||
// second definition at a colliding path does not reuse the first callee's
|
||||
// child). The store stamps it on the child and, when the caller already
|
||||
// started a child under it, returns that child instead of starting a second
|
||||
// one. Required: a retry of one call site carries the same key, so the retry
|
||||
// cannot duplicate the child's work.
|
||||
string call_key = 4;
|
||||
}
|
||||
message EnqueueCallTaskResponse {
|
||||
|
||||
Reference in new issue
Block a user