From 455de3d7997169c0ed085fc0db481fb60b38cecf Mon Sep 17 00:00:00 2001 From: Sam Catlow Date: Fri, 9 Oct 2026 02:21:49 +1100 Subject: [PATCH] fix(workflow): retry a transient read while awaiting a callee --- .../domain/storecontract/storecontract.go | 8 + internal/domain/workflow/workflow_call.go | 73 +++++++-- .../domain/workflow/workflow_call_test.go | 151 ++++++++++++++++++ internal/infrastructure/staterpc/values.go | 5 + 4 files changed, 228 insertions(+), 9 deletions(-) diff --git a/internal/domain/storecontract/storecontract.go b/internal/domain/storecontract/storecontract.go index 2a223f578..be8769ac9 100644 --- a/internal/domain/storecontract/storecontract.go +++ b/internal/domain/storecontract/storecontract.go @@ -527,6 +527,14 @@ var ( ErrResourceVersionConflict = errors.New("resource version conflict") ) +// ErrUnavailable means the store could not serve a request now: it is +// restarting or unreachable. It is not paired with a canonical message the way +// the sentinels above are -- staterpc derives it from the Unavailable code +// alone, whose text varies by transport -- and it is the one store error a +// caller with budget may answer by retrying rather than by failing its own +// work. The wrapping keeps the gRPC status code readable through the chain. +var ErrUnavailable = errors.New("store: state store unavailable") + // Resource is one control-plane resource document: the stored value for an // org's kind, its revision, and the audit record written with the last write. // Resource and ResourceWrite live here so the UI-side consumers can reference diff --git a/internal/domain/workflow/workflow_call.go b/internal/domain/workflow/workflow_call.go index bc68ec9ea..ae82ef894 100644 --- a/internal/domain/workflow/workflow_call.go +++ b/internal/domain/workflow/workflow_call.go @@ -2,6 +2,7 @@ package workflow import ( "context" + "errors" "fmt" "maps" "regexp" @@ -11,6 +12,7 @@ import ( "gopkg.in/yaml.v3" + "github.com/samcharles93/archie-core/internal/domain/storecontract" "github.com/samcharles93/archie-core/internal/domain/workflow/task" "github.com/samcharles93/archie-core/internal/events" "github.com/samcharles93/archie-core/internal/taskstate" @@ -32,6 +34,14 @@ const MaxCallDepth = 5 // status. A var so tests do not sleep. var callPollInterval = 2 * time.Second +// callReadBackoff is the wait before re-reading a callee's status after a +// transient dependency error, doubling each further failure up to +// callReadMaxBackoff. Vars so tests do not sleep. +var ( + callReadBackoff = 500 * time.Millisecond + callReadMaxBackoff = 5 * time.Second +) + // workflowCallSettings are the workflow.call step's settings. An inputs value is // a literal or "inputs.", a reference to the calling workflow's declared // input. @@ -250,16 +260,34 @@ func callEnded(status string) bool { // awaitCallee polls the callee until it is terminal, failing the stage with // the callee's detail when the callee did not succeed, and returning the -// callee's written outputs when it did. The caller's own wall clock bounds -// the wait. +// callee's written outputs when it did. The caller's own wall clock bounds the +// wait. +// +// A status read that fails because the dependency is briefly unavailable is +// retried rather than failed: the callee may still be running, and the caller +// keeps the child it already started instead of starting another. Every retry +// stays inside the caller's deadline. func awaitCallee(ctx context.Context, s workflowCallSettings, tc *TaskContext, path string, callTaskID int64) (map[string]any, error) { - var outputs map[string]any + retries := 0 for { - status, detail, latestOutputs, err := tc.Calls.CallStatus(ctx, tc.Task.ID, callTaskID) + status, detail, outputs, err := tc.Calls.CallStatus(ctx, tc.Task.ID, callTaskID) if err != nil { - return nil, fmt.Errorf("%s: read task %d: %w", WorkflowCallStepName, callTaskID, err) + if ctx.Err() != nil { + return nil, ctx.Err() + } + if !transientCallError(err) { + return nil, fmt.Errorf("%s: read task %d: %w", WorkflowCallStepName, callTaskID, err) + } + retries++ + wait := callReadBackoffFor(retries) + tc.Log.Warn("callee status read failed; retrying", + "callee_task_id", callTaskID, "attempt", retries, "in", wait, "err", err) + if !sleepOrDone(ctx, wait) { + return nil, ctx.Err() + } + continue } - outputs = latestOutputs + retries = 0 if callEnded(status) { if callSucceeded(status) { if err := tc.EmitDurable(ctx, events.KindWorkflowCallFinished, path, @@ -271,14 +299,41 @@ func awaitCallee(ctx context.Context, s workflowCallSettings, tc *TaskContext, p } return nil, fmt.Errorf("%s: callee %q (task %d) ended %s: %s", WorkflowCallStepName, s.Workflow, callTaskID, status, detail) } - select { - case <-ctx.Done(): + if !sleepOrDone(ctx, callPollInterval) { return nil, ctx.Err() - case <-time.After(callPollInterval): } } } +// sleepOrDone waits for d, reporting false when the caller's context ended +// first. +func sleepOrDone(ctx context.Context, d time.Duration) bool { + select { + case <-ctx.Done(): + return false + case <-time.After(d): + return true + } +} + +// transientCallError reports whether a failed callee status read is worth +// retrying: the dependency could not serve the request now, or the read's own +// timeout elapsed while the caller still had budget. A contract error -- the +// call is not this caller's, or the caller is not running -- is not transient. +func transientCallError(err error) bool { + return errors.Is(err, storecontract.ErrUnavailable) || errors.Is(err, context.DeadlineExceeded) +} + +// callReadBackoffFor is the wait before retry n (1-based): doubling from +// callReadBackoff, capped at callReadMaxBackoff. +func callReadBackoffFor(attempt int) time.Duration { + backoff := callReadBackoff << (attempt - 1) + if backoff <= 0 || backoff > callReadMaxBackoff { + return callReadMaxBackoff + } + return backoff +} + // callSucceeded reports whether a callee's terminal state satisfies its // caller. pr_open and merged are successes: a repository callee opens its PR // and reports on it as its own outcome. diff --git a/internal/domain/workflow/workflow_call_test.go b/internal/domain/workflow/workflow_call_test.go index 8e063c892..f5b2a50a6 100644 --- a/internal/domain/workflow/workflow_call_test.go +++ b/internal/domain/workflow/workflow_call_test.go @@ -2,13 +2,17 @@ package workflow import ( "context" + "errors" + "fmt" "log/slog" "maps" "slices" "strings" "sync" "testing" + "time" + "github.com/samcharles93/archie-core/internal/domain/storecontract" "github.com/samcharles93/archie-core/internal/domain/workflow/task" "github.com/samcharles93/archie-core/internal/events" ) @@ -358,3 +362,150 @@ steps: }) } } + +// wobblyCaller models a dependency that is briefly unavailable: CallStatus +// fails failReads times before it answers, and counts the reads and the +// children it was asked to start. +type wobblyCaller struct { + mu sync.Mutex + failReads int + failWith error + reads int + started int + status string + detail string + outputs map[string]any +} + +func (c *wobblyCaller) StartCall(_ context.Context, callerTaskID int64, _, _ string, _ map[string]any) (*task.Task, error) { + c.mu.Lock() + defer c.mu.Unlock() + c.started++ + return &task.Task{ID: 42, CallParentTaskID: callerTaskID}, nil +} + +func (c *wobblyCaller) CallStatus(context.Context, int64, int64) (string, string, map[string]any, error) { + c.mu.Lock() + defer c.mu.Unlock() + c.reads++ + if c.reads <= c.failReads { + return "", "", nil, c.failWith + } + return c.status, c.detail, c.outputs, nil +} + +func (c *wobblyCaller) counts() (reads, started int) { + c.mu.Lock() + defer c.mu.Unlock() + return c.reads, c.started +} + +// shortenCallWaits makes the caller's polls and retry backoff immediate, so a +// test drives the retry loop without sleeping. Restored by t.Cleanup. +func shortenCallWaits(t *testing.T) { + t.Helper() + oldPoll, oldBackoff, oldMax := callPollInterval, callReadBackoff, callReadMaxBackoff + callPollInterval, callReadBackoff, callReadMaxBackoff = time.Millisecond, time.Millisecond, 2*time.Millisecond + t.Cleanup(func() { callPollInterval, callReadBackoff, callReadMaxBackoff = oldPoll, oldBackoff, oldMax }) +} + +// TestAwaitCalleeRetriesTransientReadFailures pins that a callee status read +// that fails because the dependency is briefly unavailable is retried, not +// failed: the child is still running, and the caller reads it again. +func TestAwaitCalleeRetriesTransientReadFailures(t *testing.T) { + shortenCallWaits(t) + caller := &wobblyCaller{ + failReads: 2, + failWith: fmt.Errorf("dial state store: %w", storecontract.ErrUnavailable), + status: StatusCompleted, + detail: "done", + outputs: map[string]any{"report": "written"}, + } + tc := &TaskContext{Task: &task.Task{ID: 7}, Calls: caller, Log: slog.Default()} + outputs, err := awaitCallee(context.Background(), workflowCallSettings{Workflow: "callee", Wait: true}, tc, "check", 42) + if err != nil { + t.Fatalf("awaitCallee: %v", err) + } + if outputs["report"] != "written" { + t.Fatalf("outputs = %v, want report=written", outputs) + } + if reads, _ := caller.counts(); reads != 3 { + t.Fatalf("reads = %d, want 3 (two retries, then the answer)", reads) + } +} + +// TestAwaitCalleeFailsPermanentReadErrorAtOnce pins the other half: a contract +// error is not retried, because it will fail the same way every time. +func TestAwaitCalleeFailsPermanentReadErrorAtOnce(t *testing.T) { + shortenCallWaits(t) + caller := &wobblyCaller{failReads: 5, failWith: storecontract.ErrCallNotYours, status: StatusCompleted} + tc := &TaskContext{Task: &task.Task{ID: 7}, Calls: caller, Log: slog.Default()} + _, err := awaitCallee(context.Background(), workflowCallSettings{Workflow: "callee", Wait: true}, tc, "check", 42) + if !errors.Is(err, storecontract.ErrCallNotYours) { + t.Fatalf("err = %v, want ErrCallNotYours", err) + } + if reads, _ := caller.counts(); reads != 1 { + t.Fatalf("reads = %d, want 1 (no retry of a contract error)", reads) + } +} + +// TestAwaitCalleeRetryStaysInsideTheDeadline pins the bound on recovery: an +// outage that outlasts the caller's budget ends the wait there rather than +// retrying past it. +func TestAwaitCalleeRetryStaysInsideTheDeadline(t *testing.T) { + shortenCallWaits(t) + caller := &wobblyCaller{failReads: 1 << 30, failWith: fmt.Errorf("dial: %w", storecontract.ErrUnavailable)} + tc := &TaskContext{Task: &task.Task{ID: 7}, Calls: caller, Log: slog.Default()} + ctx, cancel := context.WithTimeout(context.Background(), 30*time.Millisecond) + defer cancel() + _, err := awaitCallee(ctx, workflowCallSettings{Workflow: "callee", Wait: true}, tc, "check", 42) + if !errors.Is(err, context.DeadlineExceeded) { + t.Fatalf("err = %v, want the caller's deadline", err) + } +} + +// TestWorkflowCallRetriesARefusedReadWithoutASecondChild pins the whole step: +// a wait:true call whose status reads are briefly refused still succeeds, the +// caller starts exactly one child, and the call step is closed once. +func TestWorkflowCallRetriesARefusedReadWithoutASecondChild(t *testing.T) { + shortenCallWaits(t) + registry := StepRegistry{WorkflowCallStepName: newWorkflowCallStage} + wf, err := ParseAndCompile(`id: caller +repository: none +steps: + - {id: check, type: workflow.call, settings: {workflow: callee, wait: true}} +`, registry) + if err != nil { + t.Fatalf("compile: %v", err) + } + store := &fakeStepStore{} + caller := &wobblyCaller{ + failReads: 1, + failWith: fmt.Errorf("dial: %w", storecontract.ErrUnavailable), + status: StatusCompleted, + outputs: map[string]any{"verdict": "go"}, + } + tc := &TaskContext{Task: &task.Task{ID: 7, Attempt: 1}, Store: store, StepID: 3, Calls: caller, Log: slog.Default()} + if err := wf.Stages[0].Run(context.Background(), tc); err != nil { + t.Fatalf("run: %v", err) + } + reads, started := caller.counts() + if started != 1 { + t.Fatalf("started %d children, want 1: a retried read must not start another", started) + } + if reads != 2 { + t.Fatalf("reads = %d, want 2 (one refused, one answer)", reads) + } + if tc.stepResult.Result["verdict"] != "go" { + t.Fatalf("step result = %v, want the callee's outputs", tc.stepResult.Result) + } + calls := 0 + for _, s := range store.started { + if s.Kind == task.StepKindCall { + calls++ + } + } + if calls != 1 { + t.Fatalf("recorded %d call steps, want 1", calls) + } +} diff --git a/internal/infrastructure/staterpc/values.go b/internal/infrastructure/staterpc/values.go index f65e57768..f2ecaebd0 100644 --- a/internal/infrastructure/staterpc/values.go +++ b/internal/infrastructure/staterpc/values.go @@ -456,6 +456,11 @@ func unmapError(err error) error { return context.Canceled case codes.DeadlineExceeded: return context.DeadlineExceeded + case codes.Unavailable: + // Wrap both, so a caller can retry on errors.Is(ErrUnavailable) and a + // caller that maps status codes still reads Unavailable through the + // chain. + return fmt.Errorf("%w: %w", storecontract.ErrUnavailable, err) } if sentinel := sentinelForStatus(st); sentinel != nil { return sentinel