fix(workflow): retry a transient read while awaiting a callee
This commit is contained in:
1 parent
4e74ed37b4
commit
455de3d799
4 files changed
+228
-9
No files matched your search
@@ -527,6 +527,14 @@ var (
|
||||
ErrResourceVersionConflict = errors.New("resource version conflict")
|
||||
)
|
||||
|
||||
// ErrUnavailable means the store could not serve a request now: it is
|
||||
// restarting or unreachable. It is not paired with a canonical message the way
|
||||
// the sentinels above are -- staterpc derives it from the Unavailable code
|
||||
// alone, whose text varies by transport -- and it is the one store error a
|
||||
// caller with budget may answer by retrying rather than by failing its own
|
||||
// work. The wrapping keeps the gRPC status code readable through the chain.
|
||||
var ErrUnavailable = errors.New("store: state store unavailable")
|
||||
|
||||
// Resource is one control-plane resource document: the stored value for an
|
||||
// org's kind, its revision, and the audit record written with the last write.
|
||||
// Resource and ResourceWrite live here so the UI-side consumers can reference
|
||||
|
||||
@@ -2,6 +2,7 @@ package workflow
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"maps"
|
||||
"regexp"
|
||||
@@ -11,6 +12,7 @@ import (
|
||||
|
||||
"gopkg.in/yaml.v3"
|
||||
|
||||
"github.com/samcharles93/archie-core/internal/domain/storecontract"
|
||||
"github.com/samcharles93/archie-core/internal/domain/workflow/task"
|
||||
"github.com/samcharles93/archie-core/internal/events"
|
||||
"github.com/samcharles93/archie-core/internal/taskstate"
|
||||
@@ -32,6 +34,14 @@ const MaxCallDepth = 5
|
||||
// status. A var so tests do not sleep.
|
||||
var callPollInterval = 2 * time.Second
|
||||
|
||||
// callReadBackoff is the wait before re-reading a callee's status after a
|
||||
// transient dependency error, doubling each further failure up to
|
||||
// callReadMaxBackoff. Vars so tests do not sleep.
|
||||
var (
|
||||
callReadBackoff = 500 * time.Millisecond
|
||||
callReadMaxBackoff = 5 * time.Second
|
||||
)
|
||||
|
||||
// workflowCallSettings are the workflow.call step's settings. An inputs value is
|
||||
// a literal or "inputs.<name>", a reference to the calling workflow's declared
|
||||
// input.
|
||||
@@ -250,16 +260,34 @@ func callEnded(status string) bool {
|
||||
|
||||
// awaitCallee polls the callee until it is terminal, failing the stage with
|
||||
// the callee's detail when the callee did not succeed, and returning the
|
||||
// callee's written outputs when it did. The caller's own wall clock bounds
|
||||
// the wait.
|
||||
// callee's written outputs when it did. The caller's own wall clock bounds the
|
||||
// wait.
|
||||
//
|
||||
// A status read that fails because the dependency is briefly unavailable is
|
||||
// retried rather than failed: the callee may still be running, and the caller
|
||||
// keeps the child it already started instead of starting another. Every retry
|
||||
// stays inside the caller's deadline.
|
||||
func awaitCallee(ctx context.Context, s workflowCallSettings, tc *TaskContext, path string, callTaskID int64) (map[string]any, error) {
|
||||
var outputs map[string]any
|
||||
retries := 0
|
||||
for {
|
||||
status, detail, latestOutputs, err := tc.Calls.CallStatus(ctx, tc.Task.ID, callTaskID)
|
||||
status, detail, outputs, err := tc.Calls.CallStatus(ctx, tc.Task.ID, callTaskID)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("%s: read task %d: %w", WorkflowCallStepName, callTaskID, err)
|
||||
if ctx.Err() != nil {
|
||||
return nil, ctx.Err()
|
||||
}
|
||||
if !transientCallError(err) {
|
||||
return nil, fmt.Errorf("%s: read task %d: %w", WorkflowCallStepName, callTaskID, err)
|
||||
}
|
||||
retries++
|
||||
wait := callReadBackoffFor(retries)
|
||||
tc.Log.Warn("callee status read failed; retrying",
|
||||
"callee_task_id", callTaskID, "attempt", retries, "in", wait, "err", err)
|
||||
if !sleepOrDone(ctx, wait) {
|
||||
return nil, ctx.Err()
|
||||
}
|
||||
continue
|
||||
}
|
||||
outputs = latestOutputs
|
||||
retries = 0
|
||||
if callEnded(status) {
|
||||
if callSucceeded(status) {
|
||||
if err := tc.EmitDurable(ctx, events.KindWorkflowCallFinished, path,
|
||||
@@ -271,14 +299,41 @@ func awaitCallee(ctx context.Context, s workflowCallSettings, tc *TaskContext, p
|
||||
}
|
||||
return nil, fmt.Errorf("%s: callee %q (task %d) ended %s: %s", WorkflowCallStepName, s.Workflow, callTaskID, status, detail)
|
||||
}
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
if !sleepOrDone(ctx, callPollInterval) {
|
||||
return nil, ctx.Err()
|
||||
case <-time.After(callPollInterval):
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// sleepOrDone waits for d, reporting false when the caller's context ended
|
||||
// first.
|
||||
func sleepOrDone(ctx context.Context, d time.Duration) bool {
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return false
|
||||
case <-time.After(d):
|
||||
return true
|
||||
}
|
||||
}
|
||||
|
||||
// transientCallError reports whether a failed callee status read is worth
|
||||
// retrying: the dependency could not serve the request now, or the read's own
|
||||
// timeout elapsed while the caller still had budget. A contract error -- the
|
||||
// call is not this caller's, or the caller is not running -- is not transient.
|
||||
func transientCallError(err error) bool {
|
||||
return errors.Is(err, storecontract.ErrUnavailable) || errors.Is(err, context.DeadlineExceeded)
|
||||
}
|
||||
|
||||
// callReadBackoffFor is the wait before retry n (1-based): doubling from
|
||||
// callReadBackoff, capped at callReadMaxBackoff.
|
||||
func callReadBackoffFor(attempt int) time.Duration {
|
||||
backoff := callReadBackoff << (attempt - 1)
|
||||
if backoff <= 0 || backoff > callReadMaxBackoff {
|
||||
return callReadMaxBackoff
|
||||
}
|
||||
return backoff
|
||||
}
|
||||
|
||||
// callSucceeded reports whether a callee's terminal state satisfies its
|
||||
// caller. pr_open and merged are successes: a repository callee opens its PR
|
||||
// and reports on it as its own outcome.
|
||||
|
||||
@@ -2,13 +2,17 @@ package workflow
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"log/slog"
|
||||
"maps"
|
||||
"slices"
|
||||
"strings"
|
||||
"sync"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/samcharles93/archie-core/internal/domain/storecontract"
|
||||
"github.com/samcharles93/archie-core/internal/domain/workflow/task"
|
||||
"github.com/samcharles93/archie-core/internal/events"
|
||||
)
|
||||
@@ -358,3 +362,150 @@ steps:
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// wobblyCaller models a dependency that is briefly unavailable: CallStatus
|
||||
// fails failReads times before it answers, and counts the reads and the
|
||||
// children it was asked to start.
|
||||
type wobblyCaller struct {
|
||||
mu sync.Mutex
|
||||
failReads int
|
||||
failWith error
|
||||
reads int
|
||||
started int
|
||||
status string
|
||||
detail string
|
||||
outputs map[string]any
|
||||
}
|
||||
|
||||
func (c *wobblyCaller) StartCall(_ context.Context, callerTaskID int64, _, _ string, _ map[string]any) (*task.Task, error) {
|
||||
c.mu.Lock()
|
||||
defer c.mu.Unlock()
|
||||
c.started++
|
||||
return &task.Task{ID: 42, CallParentTaskID: callerTaskID}, nil
|
||||
}
|
||||
|
||||
func (c *wobblyCaller) CallStatus(context.Context, int64, int64) (string, string, map[string]any, error) {
|
||||
c.mu.Lock()
|
||||
defer c.mu.Unlock()
|
||||
c.reads++
|
||||
if c.reads <= c.failReads {
|
||||
return "", "", nil, c.failWith
|
||||
}
|
||||
return c.status, c.detail, c.outputs, nil
|
||||
}
|
||||
|
||||
func (c *wobblyCaller) counts() (reads, started int) {
|
||||
c.mu.Lock()
|
||||
defer c.mu.Unlock()
|
||||
return c.reads, c.started
|
||||
}
|
||||
|
||||
// shortenCallWaits makes the caller's polls and retry backoff immediate, so a
|
||||
// test drives the retry loop without sleeping. Restored by t.Cleanup.
|
||||
func shortenCallWaits(t *testing.T) {
|
||||
t.Helper()
|
||||
oldPoll, oldBackoff, oldMax := callPollInterval, callReadBackoff, callReadMaxBackoff
|
||||
callPollInterval, callReadBackoff, callReadMaxBackoff = time.Millisecond, time.Millisecond, 2*time.Millisecond
|
||||
t.Cleanup(func() { callPollInterval, callReadBackoff, callReadMaxBackoff = oldPoll, oldBackoff, oldMax })
|
||||
}
|
||||
|
||||
// TestAwaitCalleeRetriesTransientReadFailures pins that a callee status read
|
||||
// that fails because the dependency is briefly unavailable is retried, not
|
||||
// failed: the child is still running, and the caller reads it again.
|
||||
func TestAwaitCalleeRetriesTransientReadFailures(t *testing.T) {
|
||||
shortenCallWaits(t)
|
||||
caller := &wobblyCaller{
|
||||
failReads: 2,
|
||||
failWith: fmt.Errorf("dial state store: %w", storecontract.ErrUnavailable),
|
||||
status: StatusCompleted,
|
||||
detail: "done",
|
||||
outputs: map[string]any{"report": "written"},
|
||||
}
|
||||
tc := &TaskContext{Task: &task.Task{ID: 7}, Calls: caller, Log: slog.Default()}
|
||||
outputs, err := awaitCallee(context.Background(), workflowCallSettings{Workflow: "callee", Wait: true}, tc, "check", 42)
|
||||
if err != nil {
|
||||
t.Fatalf("awaitCallee: %v", err)
|
||||
}
|
||||
if outputs["report"] != "written" {
|
||||
t.Fatalf("outputs = %v, want report=written", outputs)
|
||||
}
|
||||
if reads, _ := caller.counts(); reads != 3 {
|
||||
t.Fatalf("reads = %d, want 3 (two retries, then the answer)", reads)
|
||||
}
|
||||
}
|
||||
|
||||
// TestAwaitCalleeFailsPermanentReadErrorAtOnce pins the other half: a contract
|
||||
// error is not retried, because it will fail the same way every time.
|
||||
func TestAwaitCalleeFailsPermanentReadErrorAtOnce(t *testing.T) {
|
||||
shortenCallWaits(t)
|
||||
caller := &wobblyCaller{failReads: 5, failWith: storecontract.ErrCallNotYours, status: StatusCompleted}
|
||||
tc := &TaskContext{Task: &task.Task{ID: 7}, Calls: caller, Log: slog.Default()}
|
||||
_, err := awaitCallee(context.Background(), workflowCallSettings{Workflow: "callee", Wait: true}, tc, "check", 42)
|
||||
if !errors.Is(err, storecontract.ErrCallNotYours) {
|
||||
t.Fatalf("err = %v, want ErrCallNotYours", err)
|
||||
}
|
||||
if reads, _ := caller.counts(); reads != 1 {
|
||||
t.Fatalf("reads = %d, want 1 (no retry of a contract error)", reads)
|
||||
}
|
||||
}
|
||||
|
||||
// TestAwaitCalleeRetryStaysInsideTheDeadline pins the bound on recovery: an
|
||||
// outage that outlasts the caller's budget ends the wait there rather than
|
||||
// retrying past it.
|
||||
func TestAwaitCalleeRetryStaysInsideTheDeadline(t *testing.T) {
|
||||
shortenCallWaits(t)
|
||||
caller := &wobblyCaller{failReads: 1 << 30, failWith: fmt.Errorf("dial: %w", storecontract.ErrUnavailable)}
|
||||
tc := &TaskContext{Task: &task.Task{ID: 7}, Calls: caller, Log: slog.Default()}
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 30*time.Millisecond)
|
||||
defer cancel()
|
||||
_, err := awaitCallee(ctx, workflowCallSettings{Workflow: "callee", Wait: true}, tc, "check", 42)
|
||||
if !errors.Is(err, context.DeadlineExceeded) {
|
||||
t.Fatalf("err = %v, want the caller's deadline", err)
|
||||
}
|
||||
}
|
||||
|
||||
// TestWorkflowCallRetriesARefusedReadWithoutASecondChild pins the whole step:
|
||||
// a wait:true call whose status reads are briefly refused still succeeds, the
|
||||
// caller starts exactly one child, and the call step is closed once.
|
||||
func TestWorkflowCallRetriesARefusedReadWithoutASecondChild(t *testing.T) {
|
||||
shortenCallWaits(t)
|
||||
registry := StepRegistry{WorkflowCallStepName: newWorkflowCallStage}
|
||||
wf, err := ParseAndCompile(`id: caller
|
||||
repository: none
|
||||
steps:
|
||||
- {id: check, type: workflow.call, settings: {workflow: callee, wait: true}}
|
||||
`, registry)
|
||||
if err != nil {
|
||||
t.Fatalf("compile: %v", err)
|
||||
}
|
||||
store := &fakeStepStore{}
|
||||
caller := &wobblyCaller{
|
||||
failReads: 1,
|
||||
failWith: fmt.Errorf("dial: %w", storecontract.ErrUnavailable),
|
||||
status: StatusCompleted,
|
||||
outputs: map[string]any{"verdict": "go"},
|
||||
}
|
||||
tc := &TaskContext{Task: &task.Task{ID: 7, Attempt: 1}, Store: store, StepID: 3, Calls: caller, Log: slog.Default()}
|
||||
if err := wf.Stages[0].Run(context.Background(), tc); err != nil {
|
||||
t.Fatalf("run: %v", err)
|
||||
}
|
||||
reads, started := caller.counts()
|
||||
if started != 1 {
|
||||
t.Fatalf("started %d children, want 1: a retried read must not start another", started)
|
||||
}
|
||||
if reads != 2 {
|
||||
t.Fatalf("reads = %d, want 2 (one refused, one answer)", reads)
|
||||
}
|
||||
if tc.stepResult.Result["verdict"] != "go" {
|
||||
t.Fatalf("step result = %v, want the callee's outputs", tc.stepResult.Result)
|
||||
}
|
||||
calls := 0
|
||||
for _, s := range store.started {
|
||||
if s.Kind == task.StepKindCall {
|
||||
calls++
|
||||
}
|
||||
}
|
||||
if calls != 1 {
|
||||
t.Fatalf("recorded %d call steps, want 1", calls)
|
||||
}
|
||||
}
|
||||
@@ -456,6 +456,11 @@ func unmapError(err error) error {
|
||||
return context.Canceled
|
||||
case codes.DeadlineExceeded:
|
||||
return context.DeadlineExceeded
|
||||
case codes.Unavailable:
|
||||
// Wrap both, so a caller can retry on errors.Is(ErrUnavailable) and a
|
||||
// caller that maps status codes still reads Unavailable through the
|
||||
// chain.
|
||||
return fmt.Errorf("%w: %w", storecontract.ErrUnavailable, err)
|
||||
}
|
||||
if sentinel := sentinelForStatus(st); sentinel != nil {
|
||||
return sentinel
|
||||
|
||||
Reference in new issue
Block a user