fix(workflow): retry a transient read while awaiting a callee

This commit is contained in:
sam committed 2026-10-09 02:21:49 +11:00
1 parent 4e74ed37b4
commit 455de3d799
4 files changed
+228 -9

No files matched your search

@@ -527,6 +527,14 @@ var (
ErrResourceVersionConflict = errors.New("resource version conflict")
)
// ErrUnavailable means the store could not serve a request now: it is
// restarting or unreachable. It is not paired with a canonical message the way
// the sentinels above are -- staterpc derives it from the Unavailable code
// alone, whose text varies by transport -- and it is the one store error a
// caller with budget may answer by retrying rather than by failing its own
// work. The wrapping keeps the gRPC status code readable through the chain.
var ErrUnavailable = errors.New("store: state store unavailable")
// Resource is one control-plane resource document: the stored value for an
// org's kind, its revision, and the audit record written with the last write.
// Resource and ResourceWrite live here so the UI-side consumers can reference
+64 -9
View File
@@ -2,6 +2,7 @@ package workflow
import (
"context"
"errors"
"fmt"
"maps"
"regexp"
@@ -11,6 +12,7 @@ import (
"gopkg.in/yaml.v3"
"github.com/samcharles93/archie-core/internal/domain/storecontract"
"github.com/samcharles93/archie-core/internal/domain/workflow/task"
"github.com/samcharles93/archie-core/internal/events"
"github.com/samcharles93/archie-core/internal/taskstate"
@@ -32,6 +34,14 @@ const MaxCallDepth = 5
// status. A var so tests do not sleep.
var callPollInterval = 2 * time.Second
// callReadBackoff is the wait before re-reading a callee's status after a
// transient dependency error, doubling each further failure up to
// callReadMaxBackoff. Vars so tests do not sleep.
var (
callReadBackoff = 500 * time.Millisecond
callReadMaxBackoff = 5 * time.Second
)
// workflowCallSettings are the workflow.call step's settings. An inputs value is
// a literal or "inputs.<name>", a reference to the calling workflow's declared
// input.
@@ -250,16 +260,34 @@ func callEnded(status string) bool {
// awaitCallee polls the callee until it is terminal, failing the stage with
// the callee's detail when the callee did not succeed, and returning the
// callee's written outputs when it did. The caller's own wall clock bounds
// the wait.
// callee's written outputs when it did. The caller's own wall clock bounds the
// wait.
//
// A status read that fails because the dependency is briefly unavailable is
// retried rather than failed: the callee may still be running, and the caller
// keeps the child it already started instead of starting another. Every retry
// stays inside the caller's deadline.
func awaitCallee(ctx context.Context, s workflowCallSettings, tc *TaskContext, path string, callTaskID int64) (map[string]any, error) {
var outputs map[string]any
retries := 0
for {
status, detail, latestOutputs, err := tc.Calls.CallStatus(ctx, tc.Task.ID, callTaskID)
status, detail, outputs, err := tc.Calls.CallStatus(ctx, tc.Task.ID, callTaskID)
if err != nil {
return nil, fmt.Errorf("%s: read task %d: %w", WorkflowCallStepName, callTaskID, err)
if ctx.Err() != nil {
return nil, ctx.Err()
}
if !transientCallError(err) {
return nil, fmt.Errorf("%s: read task %d: %w", WorkflowCallStepName, callTaskID, err)
}
retries++
wait := callReadBackoffFor(retries)
tc.Log.Warn("callee status read failed; retrying",
"callee_task_id", callTaskID, "attempt", retries, "in", wait, "err", err)
if !sleepOrDone(ctx, wait) {
return nil, ctx.Err()
}
continue
}
outputs = latestOutputs
retries = 0
if callEnded(status) {
if callSucceeded(status) {
if err := tc.EmitDurable(ctx, events.KindWorkflowCallFinished, path,
@@ -271,14 +299,41 @@ func awaitCallee(ctx context.Context, s workflowCallSettings, tc *TaskContext, p
}
return nil, fmt.Errorf("%s: callee %q (task %d) ended %s: %s", WorkflowCallStepName, s.Workflow, callTaskID, status, detail)
}
select {
case <-ctx.Done():
if !sleepOrDone(ctx, callPollInterval) {
return nil, ctx.Err()
case <-time.After(callPollInterval):
}
}
}
// sleepOrDone waits for d, reporting false when the caller's context ended
// first.
func sleepOrDone(ctx context.Context, d time.Duration) bool {
select {
case <-ctx.Done():
return false
case <-time.After(d):
return true
}
}
// transientCallError reports whether a failed callee status read is worth
// retrying: the dependency could not serve the request now, or the read's own
// timeout elapsed while the caller still had budget. A contract error -- the
// call is not this caller's, or the caller is not running -- is not transient.
func transientCallError(err error) bool {
return errors.Is(err, storecontract.ErrUnavailable) || errors.Is(err, context.DeadlineExceeded)
}
// callReadBackoffFor is the wait before retry n (1-based): doubling from
// callReadBackoff, capped at callReadMaxBackoff.
func callReadBackoffFor(attempt int) time.Duration {
backoff := callReadBackoff << (attempt - 1)
if backoff <= 0 || backoff > callReadMaxBackoff {
return callReadMaxBackoff
}
return backoff
}
// callSucceeded reports whether a callee's terminal state satisfies its
// caller. pr_open and merged are successes: a repository callee opens its PR
// and reports on it as its own outcome.
@@ -2,13 +2,17 @@ package workflow
import (
"context"
"errors"
"fmt"
"log/slog"
"maps"
"slices"
"strings"
"sync"
"testing"
"time"
"github.com/samcharles93/archie-core/internal/domain/storecontract"
"github.com/samcharles93/archie-core/internal/domain/workflow/task"
"github.com/samcharles93/archie-core/internal/events"
)
@@ -358,3 +362,150 @@ steps:
})
}
}
// wobblyCaller models a dependency that is briefly unavailable: CallStatus
// fails failReads times before it answers, and counts the reads and the
// children it was asked to start.
type wobblyCaller struct {
mu sync.Mutex
failReads int
failWith error
reads int
started int
status string
detail string
outputs map[string]any
}
func (c *wobblyCaller) StartCall(_ context.Context, callerTaskID int64, _, _ string, _ map[string]any) (*task.Task, error) {
c.mu.Lock()
defer c.mu.Unlock()
c.started++
return &task.Task{ID: 42, CallParentTaskID: callerTaskID}, nil
}
func (c *wobblyCaller) CallStatus(context.Context, int64, int64) (string, string, map[string]any, error) {
c.mu.Lock()
defer c.mu.Unlock()
c.reads++
if c.reads <= c.failReads {
return "", "", nil, c.failWith
}
return c.status, c.detail, c.outputs, nil
}
func (c *wobblyCaller) counts() (reads, started int) {
c.mu.Lock()
defer c.mu.Unlock()
return c.reads, c.started
}
// shortenCallWaits makes the caller's polls and retry backoff immediate, so a
// test drives the retry loop without sleeping. Restored by t.Cleanup.
func shortenCallWaits(t *testing.T) {
t.Helper()
oldPoll, oldBackoff, oldMax := callPollInterval, callReadBackoff, callReadMaxBackoff
callPollInterval, callReadBackoff, callReadMaxBackoff = time.Millisecond, time.Millisecond, 2*time.Millisecond
t.Cleanup(func() { callPollInterval, callReadBackoff, callReadMaxBackoff = oldPoll, oldBackoff, oldMax })
}
// TestAwaitCalleeRetriesTransientReadFailures pins that a callee status read
// that fails because the dependency is briefly unavailable is retried, not
// failed: the child is still running, and the caller reads it again.
func TestAwaitCalleeRetriesTransientReadFailures(t *testing.T) {
shortenCallWaits(t)
caller := &wobblyCaller{
failReads: 2,
failWith: fmt.Errorf("dial state store: %w", storecontract.ErrUnavailable),
status: StatusCompleted,
detail: "done",
outputs: map[string]any{"report": "written"},
}
tc := &TaskContext{Task: &task.Task{ID: 7}, Calls: caller, Log: slog.Default()}
outputs, err := awaitCallee(context.Background(), workflowCallSettings{Workflow: "callee", Wait: true}, tc, "check", 42)
if err != nil {
t.Fatalf("awaitCallee: %v", err)
}
if outputs["report"] != "written" {
t.Fatalf("outputs = %v, want report=written", outputs)
}
if reads, _ := caller.counts(); reads != 3 {
t.Fatalf("reads = %d, want 3 (two retries, then the answer)", reads)
}
}
// TestAwaitCalleeFailsPermanentReadErrorAtOnce pins the other half: a contract
// error is not retried, because it will fail the same way every time.
func TestAwaitCalleeFailsPermanentReadErrorAtOnce(t *testing.T) {
shortenCallWaits(t)
caller := &wobblyCaller{failReads: 5, failWith: storecontract.ErrCallNotYours, status: StatusCompleted}
tc := &TaskContext{Task: &task.Task{ID: 7}, Calls: caller, Log: slog.Default()}
_, err := awaitCallee(context.Background(), workflowCallSettings{Workflow: "callee", Wait: true}, tc, "check", 42)
if !errors.Is(err, storecontract.ErrCallNotYours) {
t.Fatalf("err = %v, want ErrCallNotYours", err)
}
if reads, _ := caller.counts(); reads != 1 {
t.Fatalf("reads = %d, want 1 (no retry of a contract error)", reads)
}
}
// TestAwaitCalleeRetryStaysInsideTheDeadline pins the bound on recovery: an
// outage that outlasts the caller's budget ends the wait there rather than
// retrying past it.
func TestAwaitCalleeRetryStaysInsideTheDeadline(t *testing.T) {
shortenCallWaits(t)
caller := &wobblyCaller{failReads: 1 << 30, failWith: fmt.Errorf("dial: %w", storecontract.ErrUnavailable)}
tc := &TaskContext{Task: &task.Task{ID: 7}, Calls: caller, Log: slog.Default()}
ctx, cancel := context.WithTimeout(context.Background(), 30*time.Millisecond)
defer cancel()
_, err := awaitCallee(ctx, workflowCallSettings{Workflow: "callee", Wait: true}, tc, "check", 42)
if !errors.Is(err, context.DeadlineExceeded) {
t.Fatalf("err = %v, want the caller's deadline", err)
}
}
// TestWorkflowCallRetriesARefusedReadWithoutASecondChild pins the whole step:
// a wait:true call whose status reads are briefly refused still succeeds, the
// caller starts exactly one child, and the call step is closed once.
func TestWorkflowCallRetriesARefusedReadWithoutASecondChild(t *testing.T) {
shortenCallWaits(t)
registry := StepRegistry{WorkflowCallStepName: newWorkflowCallStage}
wf, err := ParseAndCompile(`id: caller
repository: none
steps:
- {id: check, type: workflow.call, settings: {workflow: callee, wait: true}}
`, registry)
if err != nil {
t.Fatalf("compile: %v", err)
}
store := &fakeStepStore{}
caller := &wobblyCaller{
failReads: 1,
failWith: fmt.Errorf("dial: %w", storecontract.ErrUnavailable),
status: StatusCompleted,
outputs: map[string]any{"verdict": "go"},
}
tc := &TaskContext{Task: &task.Task{ID: 7, Attempt: 1}, Store: store, StepID: 3, Calls: caller, Log: slog.Default()}
if err := wf.Stages[0].Run(context.Background(), tc); err != nil {
t.Fatalf("run: %v", err)
}
reads, started := caller.counts()
if started != 1 {
t.Fatalf("started %d children, want 1: a retried read must not start another", started)
}
if reads != 2 {
t.Fatalf("reads = %d, want 2 (one refused, one answer)", reads)
}
if tc.stepResult.Result["verdict"] != "go" {
t.Fatalf("step result = %v, want the callee's outputs", tc.stepResult.Result)
}
calls := 0
for _, s := range store.started {
if s.Kind == task.StepKindCall {
calls++
}
}
if calls != 1 {
t.Fatalf("recorded %d call steps, want 1", calls)
}
}
@@ -456,6 +456,11 @@ func unmapError(err error) error {
return context.Canceled
case codes.DeadlineExceeded:
return context.DeadlineExceeded
case codes.Unavailable:
// Wrap both, so a caller can retry on errors.Is(ErrUnavailable) and a
// caller that maps status codes still reads Unavailable through the
// chain.
return fmt.Errorf("%w: %w", storecontract.ErrUnavailable, err)
}
if sentinel := sentinelForStatus(st); sentinel != nil {
return sentinel