diff --git a/WORKFLOWS.md b/WORKFLOWS.md index 97caf1faf..3c572334c 100644 --- a/WORKFLOWS.md +++ b/WORKFLOWS.md @@ -252,6 +252,10 @@ _runs without a repository._ | `wait` | bool | Wait for the child run to finish before continuing. | | `workflow` | text | The workflow to start as a child run. | +The step must declare an `id`: the step's id is the key of the call it starts, +so a retry of that call site returns the child already started instead of +starting a second one. + ### `workflow.finish` _runs without a repository._ diff --git a/internal/contracts/state/v1/state.pb.go b/internal/contracts/state/v1/state.pb.go index 9744ee35d..c6c90c3c4 100644 --- a/internal/contracts/state/v1/state.pb.go +++ b/internal/contracts/state/v1/state.pb.go @@ -12114,10 +12114,12 @@ type EnqueueCallTaskRequest struct { Workflow string `protobuf:"bytes,2,opt,name=workflow,proto3" json:"workflow,omitempty"` InputsJson string `protobuf:"bytes,3,opt,name=inputs_json,json=inputsJson,proto3" json:"inputs_json,omitempty"` // call_key is the call site's durable identity: the call step's declared - // path within the run. The store stamps it on the child and, when the - // caller already started a child under it, returns that child instead of - // starting a second one. Required: a retry of one call site carries the - // same key, so the retry cannot duplicate the child's work. + // path within the run, plus the workflow it starts (so one task running a + // second definition at a colliding path does not reuse the first callee's + // child). The store stamps it on the child and, when the caller already + // started a child under it, returns that child instead of starting a second + // one. Required: a retry of one call site carries the same key, so the retry + // cannot duplicate the child's work. CallKey string `protobuf:"bytes,4,opt,name=call_key,json=callKey,proto3" json:"call_key,omitempty"` unknownFields protoimpl.UnknownFields sizeCache protoimpl.SizeCache diff --git a/internal/domain/workflow/step_flow.go b/internal/domain/workflow/step_flow.go index f27b69a50..5eff39a03 100644 --- a/internal/domain/workflow/step_flow.go +++ b/internal/domain/workflow/step_flow.go @@ -72,6 +72,14 @@ func checkTypedStep(step StepRecord, mode task.RepositoryMode, earlier *refScope if !ok { return fmt.Errorf("unknown type %q", step.Type) } + // A workflow.call's step id is the identity of the call it starts -- it + // keys the callee, so a retry of the call site returns the child already + // started rather than a second one. It is refused here rather than in + // validateWorkflowCalls because a pinned definition is compiled on its own + // (ParseAndCompile), never through the collection. + if step.Type == WorkflowCallStepName && step.ID == "" { + return fmt.Errorf("%q needs an id: the step's id is the key of the call it starts", step.Type) + } if mode != task.RepositoryRequired && NeedsRepository(step.Type) { return fmt.Errorf("%q needs a repository, but the workflow's repository is %s", step.Type, mode) } diff --git a/internal/domain/workflow/step_flow_test.go b/internal/domain/workflow/step_flow_test.go index eedce853e..59733d978 100644 --- a/internal/domain/workflow/step_flow_test.go +++ b/internal/domain/workflow/step_flow_test.go @@ -15,6 +15,7 @@ import ( type fakeStepStore struct { started []task.StepStart finished []task.StepFinish + events []events.Event nextID int64 } @@ -24,7 +25,10 @@ func (f *fakeStepStore) Transition(context.Context, int64, string, string, strin return nil } -func (f *fakeStepStore) InsertEvent(context.Context, events.Event) (int64, error) { return 0, nil } +func (f *fakeStepStore) InsertEvent(_ context.Context, e events.Event) (int64, error) { + f.events = append(f.events, e) + return 0, nil +} func (f *fakeStepStore) StartStep(_ context.Context, s task.StepStart) (int64, events.Event, error) { f.nextID++ diff --git a/internal/domain/workflow/task/caller.go b/internal/domain/workflow/task/caller.go index 9416c560c..92ef0b64d 100644 --- a/internal/domain/workflow/task/caller.go +++ b/internal/domain/workflow/task/caller.go @@ -6,8 +6,9 @@ import "context" type Caller interface { // StartCall enqueues the callee for callerTaskID's call step at depth + 1. // callKey is the call site's durable identity -- the call step's declared - // path within the run -- and is required: the same caller and key return - // the child already started, so a retry of a call site starts one child. + // path within the run, plus the workflow it starts -- and is required: the + // same caller and key return the child already started, so a retry of a + // call site starts one child. // It fails if the caller is not running or the depth limit is exceeded. StartCall(ctx context.Context, callerTaskID int64, callKey, workflow string, inputs map[string]any) (*Task, error) // CallStatus reads one call's callee: its status, latest transition detail diff --git a/internal/domain/workflow/workflow_call.go b/internal/domain/workflow/workflow_call.go index 4025956f6..f2297a138 100644 --- a/internal/domain/workflow/workflow_call.go +++ b/internal/domain/workflow/workflow_call.go @@ -103,13 +103,18 @@ func runWorkflowCall(ctx context.Context, s workflowCallSettings, tc *TaskContex if tc.Calls == nil { return fmt.Errorf("%s: this runner carries no callee capability", WorkflowCallStepName) } - // The key is what makes a retry of this call site return the same child, - // so a call that cannot name itself refuses rather than sharing a key with - // its siblings. - key, err := workflowCallKey(tc) + // The key is what makes a retry of this call site return the same child: the + // call site's declared path, plus the callee. The callee belongs in it + // because one task can run a second definition at a colliding path + // (workflow.handoff and human.approve set the task's workflow, and the + // daemon re-pins when the pin's id differs), and that run must not be + // handed the first callee's child. A step path cannot contain "@": its + // segments are stable identifiers joined with "/". + path, err := workflowCallPath(tc) if err != nil { return err } + key := path + "@" + s.Workflow if tc.Task.CallDepth+1 > MaxCallDepth { return fmt.Errorf("%s: call depth %d would pass the limit of %d", WorkflowCallStepName, tc.Task.CallDepth+1, MaxCallDepth) } @@ -122,13 +127,14 @@ func runWorkflowCall(ctx context.Context, s workflowCallSettings, tc *TaskContex return fmt.Errorf("%s: start %q: %w", WorkflowCallStepName, s.Workflow, err) } started := map[string]any{"callee_task_id": callee.ID, "workflow": s.Workflow, "wait": s.Wait} - if err := tc.EmitDurable(ctx, events.KindWorkflowCallStarted, WorkflowCallStepName, + if err := tc.EmitDurable(ctx, events.KindWorkflowCallStarted, path, fmt.Sprintf("started %q as task %d", s.Workflow, callee.ID), started); err != nil { tc.Log.Warn("workflow call start not persisted", "err", err) } // Record the call as a step holding the callee's execution ID, named for - // the call site so the dashboard shows which call started it. - callStep, _, err := tc.startCallStep(ctx, key, callee.ID) + // the call site -- the path, not the key that also carries the callee -- + // so the dashboard shows which call started it. + callStep, _, err := tc.startCallStep(ctx, path, callee.ID) if err != nil { return fmt.Errorf("%s: could not be recorded: %w", WorkflowCallStepName, err) } @@ -143,7 +149,7 @@ func runWorkflowCall(ctx context.Context, s workflowCallSettings, tc *TaskContex } // Close the call step with the callee's outcome, unless the caller itself was // cancelled. - calleeOutputs, waitErr := awaitCallee(ctx, s, tc, callee.ID) + calleeOutputs, waitErr := awaitCallee(ctx, s, tc, path, callee.ID) if waitErr != nil { if ctx.Err() == nil { if ferr := tc.finishChildStep(ctx, callStep, taskstate.StepFailed, waitErr.Error(), 0); ferr != nil { @@ -170,28 +176,26 @@ func runWorkflowCall(ctx context.Context, s workflowCallSettings, tc *TaskContex return nil } -// workflowCallKey is the running call's durable identity: the step's declared -// path within the run. validateWorkflowCalls refuses a call step with no id at -// save, but a definition pinned before that rule still compiles and runs: its -// path is the bare step type, one key shared by every call site in the run, so -// the second call site would be handed the first's child. A stage run outside -// the step wrapper has no path at all, which is the same failure. -func workflowCallKey(tc *TaskContext) (string, error) { - path := tc.stepPath - if path == "" || path == WorkflowCallStepName || strings.HasSuffix(path, "/"+WorkflowCallStepName) { - return "", fmt.Errorf("%s: the step declares no id, and the step's id is the call's key", WorkflowCallStepName) +// workflowCallPath is the running call site's declared path within the run, +// which names the step and identifies the call. Every registry stage is +// wrapped, so a run reaching here without a path has no identity to key its +// child on: the store's index ignores an empty key, so keying on one would +// start a fresh child on every attempt -- the duplicate the key prevents. +func workflowCallPath(tc *TaskContext) (string, error) { + if tc.stepPath == "" { + return "", fmt.Errorf("%s: the running step has no declared path, so this call has no key", WorkflowCallStepName) } - return path, nil + return tc.stepPath, nil } // startCallStep records the call step for a callee this run just started. -func (tc *TaskContext) startCallStep(ctx context.Context, key string, calleeTaskID int64) (int64, events.Event, error) { +func (tc *TaskContext) startCallStep(ctx context.Context, path string, calleeTaskID int64) (int64, events.Event, error) { if tc.Store == nil { return 0, events.Event{}, nil } stepID, event, err := tc.Store.StartStep(ctx, task.StepStart{ ExecutionID: tc.Task.ID, Attempt: tc.Task.Attempt, ParentID: tc.StepID, - Kind: task.StepKindCall, Name: key, CalledExecutionID: calleeTaskID, + Kind: task.StepKindCall, Name: path, CalledExecutionID: calleeTaskID, }) if err != nil { return 0, events.Event{}, err @@ -244,7 +248,7 @@ func callEnded(status string) bool { // the callee's detail when the callee did not succeed, and returning the // callee's written outputs when it did. The caller's own wall clock bounds // the wait. -func awaitCallee(ctx context.Context, s workflowCallSettings, tc *TaskContext, callTaskID int64) (map[string]any, error) { +func awaitCallee(ctx context.Context, s workflowCallSettings, tc *TaskContext, path string, callTaskID int64) (map[string]any, error) { var outputs map[string]any for { status, detail, latestOutputs, err := tc.Calls.CallStatus(ctx, tc.Task.ID, callTaskID) @@ -254,7 +258,7 @@ func awaitCallee(ctx context.Context, s workflowCallSettings, tc *TaskContext, c outputs = latestOutputs if callEnded(status) { if callSucceeded(status) { - if err := tc.EmitDurable(ctx, events.KindWorkflowCallFinished, WorkflowCallStepName, + if err := tc.EmitDurable(ctx, events.KindWorkflowCallFinished, path, fmt.Sprintf("%q (task %d) finished: %s", s.Workflow, callTaskID, detail), map[string]any{"callee_task_id": callTaskID, "workflow": s.Workflow, "status": status, "detail": detail}); err != nil { tc.Log.Warn("workflow call finish not persisted", "err", err) @@ -294,12 +298,6 @@ func validateWorkflowCalls(parsed map[string]YAMLDefinition) error { if step.Type != WorkflowCallStepName { return nil } - // The step's id is the call's key, so a call without one has no way - // to say which call site it is, and two of them in one run would - // share a child. - if step.ID == "" { - return fmt.Errorf("%s: %s declares no id, and the step's id is the call's key", where, WorkflowCallStepName) - } s, err := decodeCallSettings(step) if err != nil { return fmt.Errorf("%s: %w", where, err) diff --git a/internal/domain/workflow/workflow_call_test.go b/internal/domain/workflow/workflow_call_test.go index 29752d19f..2438e680d 100644 --- a/internal/domain/workflow/workflow_call_test.go +++ b/internal/domain/workflow/workflow_call_test.go @@ -10,6 +10,7 @@ import ( "testing" "github.com/samcharles93/archie-core/internal/domain/workflow/task" + "github.com/samcharles93/archie-core/internal/events" ) // callSpy records the key every call site started its child under, and models @@ -42,26 +43,33 @@ func (c *callSpy) startedKeys() []string { return slices.Clone(c.keys) } -// TestWorkflowCallKeyIsTheDeclaredPath pins the call's durable identity: two -// runs of one call site -- a retry of the stage, a lost EnqueueCallTask reply -// -- key on the same declared path and start one child, and two call sites in -// one run key on different paths so neither returns the other's child. +// TestWorkflowCallKeyIsTheDeclaredPath pins the call's durable identity: the +// step's declared path plus the callee it starts. Two runs of one call site -- +// a retry of the stage, a lost EnqueueCallTask reply -- key on the same value +// and start one child, and neither a sibling call site nor the same path under +// a re-pinned definition reuses another call's child. func TestWorkflowCallKeyIsTheDeclaredPath(t *testing.T) { registry := StepRegistry{WorkflowCallStepName: newWorkflowCallStage} tests := []struct { - name string - def string - wantKeys []string - wantErr string + name string + def string + // repin is the definition a later run compiles instead. One task can + // run a second definition -- workflow.handoff and human.approve set the + // task's workflow, and the daemon re-pins when the pin's id differs -- + // so a call path that collides there must not reuse the first callee. + repin string + wantKeys []string + wantRepin []string + wantErr string }{ { - name: "a call step keys on its declared id", + name: "a call step keys on its declared id and the callee it starts", def: `id: caller repository: none steps: - {id: check, type: workflow.call, settings: {workflow: callee}} `, - wantKeys: []string{"check"}, + wantKeys: []string{"check@callee"}, }, { name: "a call in a parallel branch keys on its path within the run", @@ -73,63 +81,144 @@ steps: a: [{id: check, type: workflow.call, settings: {workflow: callee}}] b: [{id: verify, type: workflow.call, settings: {workflow: callee}}] `, - wantKeys: []string{"fanout/a/check", "fanout/b/verify"}, + wantKeys: []string{"fanout/a/check@callee", "fanout/b/verify@callee"}, }, { - // A definition pinned before the rule that a call declares an id - // compiles and runs: were its path the bare step type, every call - // site in the run would share one key and the second would return - // the first's child. - name: "a call step with no id is refused, not keyed on its type", + name: "the same path under a re-pinned definition keys on its callee", + def: `id: caller +repository: none +steps: + - {id: classify, type: workflow.call, settings: {workflow: callee-a}} +`, + repin: `id: caller +repository: none +steps: + - {id: classify, type: workflow.call, settings: {workflow: callee-b}} +`, + wantKeys: []string{"classify@callee-a"}, + wantRepin: []string{"classify@callee-b"}, + }, + { + // A step id may spell a step type: it is a stable identifier like + // any other, and the path it declares is the call's identity. + name: "a step id that spells the step type is accepted", + def: `id: caller +repository: none +steps: + - {id: workflow.call, type: workflow.call, settings: {workflow: callee}} +`, + wantKeys: []string{"workflow.call@callee"}, + }, + { + // The step's id is the call's identity, so an id-less call step is + // refused where its definition is compiled -- which is also the path + // a pinned definition takes. + name: "a call step with no id is refused", def: `id: caller repository: none steps: - type: workflow.call settings: {workflow: callee} `, - wantErr: "declares no id", + wantErr: "needs an id", }, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - wf, err := ParseAndCompile(tt.def, registry) - if err != nil { - t.Fatalf("compile: %v", err) - } spy := &callSpy{children: map[string]*task.Task{}} tc := &TaskContext{Task: &task.Task{ID: 7}, Calls: spy, Log: slog.Default()} - stage := wf.Stages[0] - // The same stage twice: a requeued attempt re-runs the call site. - for range 2 { - if err := stage.Run(context.Background(), tc); err != nil { - if tt.wantErr == "" || !strings.Contains(err.Error(), tt.wantErr) { - t.Fatalf("err = %v, want %q", err, tt.wantErr) - } - if keys := spy.startedKeys(); len(keys) != 0 { - t.Fatalf("started %q, want no call", keys) - } - return - } + // Twice: a requeued attempt re-runs the call site. + second := tt.def + if tt.repin != "" { + second = tt.repin + } + for i, def := range []string{tt.def, second} { + wf, err := ParseAndCompile(def, registry) if tt.wantErr != "" { - t.Fatalf("err = nil, want %q", tt.wantErr) + if err == nil || !strings.Contains(err.Error(), tt.wantErr) { + t.Fatalf("compile err = %v, want %q", err, tt.wantErr) + } + continue + } + if err != nil { + t.Fatalf("compile: %v", err) + } + before := len(spy.startedKeys()) + if err := wf.Stages[0].Run(context.Background(), tc); err != nil { + t.Fatalf("run: %v", err) + } + want := tt.wantKeys + if i == 1 && tt.repin != "" { + want = tt.wantRepin + } + got := spy.startedKeys()[before:] + // Branches finish in whatever order; the keys a run used, not + // their order, are the behaviour. + slices.Sort(got) + if !slices.Equal(got, want) { + t.Fatalf("run %d keys = %q, want %q", i+1, got, want) } } - keys := spy.startedKeys() - first, second := keys[:len(keys)/2], keys[len(keys)/2:] - // Branches finish in whatever order; the keys a run used, not their - // order, are the behaviour. - slices.Sort(first) - slices.Sort(second) - if !slices.Equal(first, tt.wantKeys) || !slices.Equal(second, tt.wantKeys) { - t.Fatalf("keys = %q then %q, want %q both times", first, second, tt.wantKeys) + if tt.wantErr != "" { + if keys := spy.startedKeys(); len(keys) != 0 { + t.Fatalf("started %q, want no call", keys) + } + return } - if len(spy.children) != len(tt.wantKeys) { - t.Fatalf("started %d children, want %d", len(spy.children), len(tt.wantKeys)) + if want := len(tt.wantKeys) + len(tt.wantRepin); len(spy.children) != want { + t.Fatalf("started %d children, want %d", len(spy.children), want) } }) } } +// TestWorkflowCallWithoutADeclaredPathRefuses pins the one state a call cannot +// key on: a step run with no declared path. The store's unique index ignores +// an empty key, so without the refusal every attempt would start a fresh child +// -- the duplicate the key exists to prevent. +func TestWorkflowCallWithoutADeclaredPathRefuses(t *testing.T) { + spy := &callSpy{children: map[string]*task.Task{}} + tc := &TaskContext{Task: &task.Task{ID: 7}, Calls: spy, Log: slog.Default()} + err := runWorkflowCall(context.Background(), workflowCallSettings{Workflow: "callee"}, tc) + if err == nil || !strings.Contains(err.Error(), "no declared path") { + t.Fatalf("err = %v, want a refusal naming the missing path", err) + } + if keys := spy.startedKeys(); len(keys) != 0 { + t.Fatalf("started %q, want no call", keys) + } +} + +// TestWorkflowCallRecordsTheSiteNotTheKey pins the two values apart: the key +// that identifies the child carries the callee, while the step row and the +// call's events are named and staged for the call site, so a rail pairing an +// event with its step by stage can still attach them. +func TestWorkflowCallRecordsTheSiteNotTheKey(t *testing.T) { + registry := StepRegistry{WorkflowCallStepName: newWorkflowCallStage} + wf, err := ParseAndCompile(`id: caller +repository: none +steps: + - {id: check, type: workflow.call, settings: {workflow: callee}} +`, registry) + if err != nil { + t.Fatalf("compile: %v", err) + } + store := &fakeStepStore{} + spy := &callSpy{children: map[string]*task.Task{}} + tc := &TaskContext{Task: &task.Task{ID: 7, Attempt: 1}, Store: store, StepID: 3, Calls: spy, Log: slog.Default()} + if err := wf.Stages[0].Run(context.Background(), tc); err != nil { + t.Fatalf("run: %v", err) + } + if keys := spy.startedKeys(); !slices.Equal(keys, []string{"check@callee"}) { + t.Fatalf("keys = %q, want [check@callee]", keys) + } + if len(store.started) != 1 || store.started[0].Kind != task.StepKindCall || store.started[0].Name != "check" { + t.Fatalf("started = %#v, want one call step named check", store.started) + } + if len(store.events) != 1 || store.events[0].Kind != events.KindWorkflowCallStarted || store.events[0].Stage != "check" { + t.Fatalf("events = %#v, want one %s staged check", store.events, events.KindWorkflowCallStarted) + } +} + func TestValidateWorkflowCallsWalksBranches(t *testing.T) { registry := StepRegistry{ AgentRunStepName: newAgentRunStage, @@ -185,7 +274,7 @@ steps: - type: workflow.finish `, }, - wantErr: `workflow "caller" step 1: workflow.call declares no id`, + wantErr: `workflow "caller" step 1: "workflow.call" needs an id`, }, { name: "a cycle through branch calls is refused", diff --git a/internal/infrastructure/postgres/migrations/0038_workflow_call_key.sql b/internal/infrastructure/postgres/migrations/0038_workflow_call_key.sql index 46c647ba7..d6faf524f 100644 --- a/internal/infrastructure/postgres/migrations/0038_workflow_call_key.sql +++ b/internal/infrastructure/postgres/migrations/0038_workflow_call_key.sql @@ -1,14 +1,23 @@ -- tasks.call_key is a workflow.call call site's durable identity: the call --- step's declared path within the calling run. With the parent it is the key --- EnqueueCallTask upserts on, so a retry of one call site -- a lost --- EnqueueCallTask reply, a failed step record, a requeued attempt -- returns --- the child already started instead of starting a second one. The index is --- partial: every task that is not a workflow.call callee carries the empty key. +-- step's declared path within the calling run, plus the callee it starts. With +-- the parent it is the key EnqueueCallTask upserts on, so a retry of one call +-- site -- a lost EnqueueCallTask reply, a failed step record, a requeued +-- attempt -- returns the child already started instead of starting a second +-- one. The index is partial: every task that is not a workflow.call callee +-- carries the empty key. +-- +-- A child that reached dead or closed_wont_do (taskstate.Dead and +-- taskstate.Declined) can never do the work and holds no result the caller can +-- use, so it leaves the index and frees its key for the caller's retry to +-- start a fresh child. parked and rejected stay in: a parked child is +-- recoverable by retrying the child -- which is what the caller's own park +-- names -- and a rejected child ran, so reusing it is what the call site wants. -- +goose Up ALTER TABLE tasks ADD COLUMN call_key text NOT NULL DEFAULT ''; -CREATE UNIQUE INDEX idx_tasks_call_key ON tasks (call_parent_task_id, call_key) WHERE call_key <> ''; +CREATE UNIQUE INDEX idx_tasks_call_key ON tasks (call_parent_task_id, call_key) + WHERE call_key <> '' AND status NOT IN ('dead', 'closed_wont_do'); -- +goose Down diff --git a/internal/infrastructure/postgres/postgresdb/call_tasks.sql.go b/internal/infrastructure/postgres/postgresdb/call_tasks.sql.go index 6cfb82386..b68951bd3 100644 --- a/internal/infrastructure/postgres/postgresdb/call_tasks.sql.go +++ b/internal/infrastructure/postgres/postgresdb/call_tasks.sql.go @@ -41,7 +41,7 @@ SELECT caller.id, caller.call_depth + 1, $7 FROM caller WHERE caller.status = 'running' AND caller.call_depth + 1 <= $8::int -ON CONFLICT (call_parent_task_id, call_key) WHERE call_key <> '' DO UPDATE SET call_key = EXCLUDED.call_key +ON CONFLICT (call_parent_task_id, call_key) WHERE call_key <> '' AND status NOT IN ('dead', 'closed_wont_do') DO UPDATE SET call_key = EXCLUDED.call_key RETURNING id, owner, repo, issue_number, title, body, labels, status, workflow, branch, plan, notes, pr_number, tokens_used, iterations, attempt, park_reason, watch_comment_id, park_class, remediation_rounds, retry_count, source, identity, binding_id, binding_version, review_payload, workflow_definition_version, workflow_definition_digest, workflow_definition_yaml, created_at, updated_at, review_cursor, inputs, org_id, workspace_id, call_parent_task_id, call_depth, outputs, review_gate, rereview_rounds, retry_mode, resume_from, resume_results, pending_reviews, origin, call_key ` @@ -60,9 +60,11 @@ type EnqueueCallTaskParams struct { // // The call key makes this the one write that can be retried: a call site that // already started its child gets that child back rather than a second one. The -// upsert only fires for a keyed callee, and the WHERE below still decides -// whether the caller may call at all -- a caller that is missing, not running -// or past the depth limit inserts nothing and returns no row, conflict or not. +// upsert only fires for a keyed callee whose child can still do the work -- a +// child that ended dead or closed_wont_do freed its key -- and the WHERE below +// still decides whether the caller may call at all: a caller that is missing, +// not running or past the depth limit inserts nothing and returns no row, +// conflict or not. func (q *Queries) EnqueueCallTask(ctx context.Context, arg EnqueueCallTaskParams) (Task, error) { row := q.db.QueryRow(ctx, enqueueCallTask, arg.ID, diff --git a/internal/infrastructure/postgres/queries/call_tasks.sql b/internal/infrastructure/postgres/queries/call_tasks.sql index d0203c211..dfa25fce1 100644 --- a/internal/infrastructure/postgres/queries/call_tasks.sql +++ b/internal/infrastructure/postgres/queries/call_tasks.sql @@ -3,9 +3,11 @@ -- -- The call key makes this the one write that can be retried: a call site that -- already started its child gets that child back rather than a second one. The --- upsert only fires for a keyed callee, and the WHERE below still decides --- whether the caller may call at all -- a caller that is missing, not running --- or past the depth limit inserts nothing and returns no row, conflict or not. +-- upsert only fires for a keyed callee whose child can still do the work -- a +-- child that ended dead or closed_wont_do freed its key -- and the WHERE below +-- still decides whether the caller may call at all: a caller that is missing, +-- not running or past the depth limit inserts nothing and returns no row, +-- conflict or not. WITH caller AS ( SELECT t.* FROM tasks t WHERE t.id = $1 FOR UPDATE ) @@ -24,7 +26,7 @@ SELECT caller.id, caller.call_depth + 1, sqlc.arg('call_key') FROM caller WHERE caller.status = 'running' AND caller.call_depth + 1 <= sqlc.arg('max_depth')::int -ON CONFLICT (call_parent_task_id, call_key) WHERE call_key <> '' DO UPDATE SET call_key = EXCLUDED.call_key +ON CONFLICT (call_parent_task_id, call_key) WHERE call_key <> '' AND status NOT IN ('dead', 'closed_wont_do') DO UPDATE SET call_key = EXCLUDED.call_key RETURNING *; -- name: CallStatusDetail :one diff --git a/internal/infrastructure/postgres/store.go b/internal/infrastructure/postgres/store.go index 1409cc9ca..aa287d868 100644 --- a/internal/infrastructure/postgres/store.go +++ b/internal/infrastructure/postgres/store.go @@ -783,8 +783,9 @@ func (s *Store) TaskByID(ctx context.Context, taskID int64) (*workflow.Task, err // StartCall enqueues a workflow.call callee inheriting the caller's org, // workspace, identity and repo, and returns the callee already started under -// callerTaskID and callKey when there is one. The caller must be running and -// within workflow.MaxCallDepth. +// callerTaskID and callKey when there is one. A child that ended dead or +// declined freed its key, so such a call starts a fresh child. The caller must +// be running and within workflow.MaxCallDepth. func (s *Store) StartCall(ctx context.Context, callerTaskID int64, callKey string, wf string, inputs map[string]any) (*workflow.Task, error) { encoded, err := task.EncodeInputs(inputs) if err != nil { diff --git a/internal/infrastructure/postgres/workflow_call_test.go b/internal/infrastructure/postgres/workflow_call_test.go index eb224cad7..39ce936a7 100644 --- a/internal/infrastructure/postgres/workflow_call_test.go +++ b/internal/infrastructure/postgres/workflow_call_test.go @@ -4,12 +4,14 @@ import ( "testing" "github.com/samcharles93/archie-core/internal/infrastructure/postgres/pgstore" + "github.com/samcharles93/archie-core/internal/taskstate" ) -// StartCall is idempotent per call site: the same caller and call key return -// the child already started, so a retry of a lost EnqueueCallTask reply, or of -// the stage that made the call, does not start a second child and run its work -// twice. A different call site in the same run still gets its own child. +// StartCall is idempotent per call key: the same caller and key return the +// child already started, so a retry of a lost EnqueueCallTask reply, or of the +// stage that made the call, does not start a second child and run its work +// twice. Another call site -- the same declared path starting a different +// workflow -- still gets its own child. func TestStartCallReturnsOneChildPerCallKey(t *testing.T) { ctx := t.Context() db := pgstore.Open(t) @@ -21,23 +23,23 @@ func TestStartCallReturnsOneChildPerCallKey(t *testing.T) { t.Fatalf("claim: %v", err) } - first, err := db.StartCall(ctx, caller.ID, "check", "callee", nil) + first, err := db.StartCall(ctx, caller.ID, "check@callee-a", "callee-a", nil) if err != nil { t.Fatal(err) } - retry, err := db.StartCall(ctx, caller.ID, "check", "callee", nil) + retry, err := db.StartCall(ctx, caller.ID, "check@callee-a", "callee-a", nil) if err != nil { t.Fatal(err) } if retry.ID != first.ID { t.Fatalf("the retry started task %d, want the child already started, %d", retry.ID, first.ID) } - other, err := db.StartCall(ctx, caller.ID, "verify", "callee", nil) + other, err := db.StartCall(ctx, caller.ID, "check@callee-b", "callee-b", nil) if err != nil { t.Fatal(err) } if other.ID == first.ID { - t.Fatalf("a different call key returned the first child, %d", first.ID) + t.Fatalf("the same path starting another workflow returned the first child, %d", first.ID) } var rows int @@ -49,3 +51,72 @@ func TestStartCallReturnsOneChildPerCallKey(t *testing.T) { t.Fatalf("the caller started %d children, want one per call key (2)", rows) } } + +// A child that reached a hard terminal failure can never do the work, so it +// frees its key: it left no result the caller can use, and the caller's retry +// starting a fresh child duplicates nothing. A parked child keeps the key -- +// it is recoverable by retrying the child, which is what the caller's own park +// names. +func TestStartCallFreesTheKeyOfAHardTerminalChild(t *testing.T) { + tests := []struct { + name string + // move takes the child out of queued the way its real ending does. + move func(t *testing.T, db *pgstore.TaskDB, childID int64) + wantNew bool + }{ + { + name: "a declined child frees its key", + move: func(t *testing.T, db *pgstore.TaskDB, childID int64) { + t.Helper() + if err := db.Transition(t.Context(), childID, taskstate.Queued, taskstate.Declined, "the operator declined it"); err != nil { + t.Fatal(err) + } + }, + wantNew: true, + }, + { + name: "a parked child keeps its key", + move: func(t *testing.T, db *pgstore.TaskDB, childID int64) { + t.Helper() + // Parked is reachable only from running, so the child is + // claimed first, as a worker would claim it. + claimed, err := db.ClaimNext(t.Context()) + if err != nil || claimed == nil || claimed.ID != childID { + t.Fatalf("claim child %d: %v, got %v", childID, err, claimed) + } + if err := db.ParkTask(t.Context(), childID, taskstate.Running, "needs an operator", taskstate.ParkNeedsHuman); err != nil { + t.Fatal(err) + } + }, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + ctx := t.Context() + db := pgstore.Open(t) + if _, err := db.EnqueueChatTask(ctx, "", "", "caller", "", "tdd", "", "", nil); err != nil { + t.Fatal(err) + } + caller, err := db.ClaimNext(ctx) + if err != nil || caller == nil { + t.Fatalf("claim: %v", err) + } + first, err := db.StartCall(ctx, caller.ID, "check@callee", "callee", nil) + if err != nil { + t.Fatal(err) + } + tt.move(t, db, first.ID) + + retry, err := db.StartCall(ctx, caller.ID, "check@callee", "callee", nil) + if err != nil { + t.Fatal(err) + } + if tt.wantNew && retry.ID == first.ID { + t.Fatalf("the retry returned the child that can never do the work, %d, want a fresh child", first.ID) + } + if !tt.wantNew && retry.ID != first.ID { + t.Fatalf("the retry started task %d, want the parked child it can retry, %d", retry.ID, first.ID) + } + }) + } +} diff --git a/proto/state/v1/state.proto b/proto/state/v1/state.proto index 0ee587a49..73d70b226 100644 --- a/proto/state/v1/state.proto +++ b/proto/state/v1/state.proto @@ -1261,10 +1261,12 @@ message EnqueueCallTaskRequest { string workflow = 2; string inputs_json = 3; // call_key is the call site's durable identity: the call step's declared - // path within the run. The store stamps it on the child and, when the - // caller already started a child under it, returns that child instead of - // starting a second one. Required: a retry of one call site carries the - // same key, so the retry cannot duplicate the child's work. + // path within the run, plus the workflow it starts (so one task running a + // second definition at a colliding path does not reuse the first callee's + // child). The store stamps it on the child and, when the caller already + // started a child under it, returns that child instead of starting a second + // one. Required: a retry of one call site carries the same key, so the retry + // cannot duplicate the child's work. string call_key = 4; } message EnqueueCallTaskResponse {