Files
archie/modules/runs/context.go
T

273 lines
8.6 KiB
Go

package runs
import (
"context"
"database/sql"
"encoding/json/jsontext"
json "encoding/json/v2"
"errors"
"fmt"
"strconv"
"strings"
"time"
"github.com/google/uuid"
"github.com/samcharles93/archie-core/modules/runs/internal/store"
)
var (
// ErrCancelled ends a run whose cancellation was requested.
ErrCancelled = errors.New("run cancelled")
// ErrOutcomeUnknown is returned by a step that can't confirm what its
// effect did. The step is recorded as unknown and is never repeated (R15).
ErrOutcomeUnknown = errors.New("outcome unknown")
// ErrWaitTimedOut is returned by Wait when its deadline passes.
ErrWaitTimedOut = errors.New("wait timed out")
// ErrNondeterministic means replay met a step the saved history doesn't
// have at that position: the workflow's code took a different path.
ErrNondeterministic = errors.New("workflow replay diverged from its saved steps")
errSuspended = errors.New("run suspended")
)
// Context is a run's handle on the engine while its workflow executes.
type Context struct {
// The run owns its context, as an http.Request does: every step takes its
// cancellation from it, and a second context passed beside it could disagree.
ctx context.Context //nolint:containedctx // see above
work workerStore
run store.Run
token uuid.UUID
saved map[int]store.Step
next int
// started holds the keys of effects an earlier attempt began; new markers
// are numbered from nextMarker.
started map[string]bool
nextMarker int
// overlay is what a dry run carries instead of a stored version.
overlay *dryOverlay
}
// markerBase is where started markers are numbered, far above any step a
// workflow takes, so they never shift the positions a replay depends on.
const markerBase = 1 << 20
const startedPrefix = "started:"
func newContext(ctx context.Context, work workerStore, run store.Run, token uuid.UUID, steps []store.Step) *Context {
saved := make(map[int]store.Step, len(steps))
started := map[string]bool{}
nextMarker := markerBase
for _, s := range steps {
saved[s.Seq] = s
if key, ok := strings.CutPrefix(s.Name, startedPrefix); ok {
started[key] = true
nextMarker = max(nextMarker, s.Seq+1)
}
}
rc := &Context{ctx: ctx, work: work, run: run, token: token, saved: saved, started: started, nextMarker: nextMarker}
if run.Dry {
rc.overlay, _ = readOverlay(run.Overlay)
}
return rc
}
// Begin durably records that the effect behind key is about to happen, before
// it does. It reports true when an earlier attempt already began it: that
// attempt may have died after the effect and before its result was recorded,
// so the caller must not run it again and treats its outcome as unknown (R15).
func (rc *Context) Begin(ctx context.Context, key string) (bool, error) {
if rc.started[key] {
return true, nil
}
seq := rc.nextMarker
if _, err := rc.record(ctx, seq, startedPrefix+key, true, nil); err != nil {
return false, err
}
rc.started[key] = true
rc.nextMarker = seq + 1
return false, nil
}
// RunID is the run's id.
func (rc *Context) RunID() uuid.UUID { return rc.run.ID }
// Workflow is the name of the workflow this run executes.
func (rc *Context) Workflow() string { return rc.run.Workflow }
// OrgID is the organisation the run belongs to.
func (rc *Context) OrgID() string { return rc.run.OrgID }
// Step runs fn once for this position in the workflow and saves its result.
// It returns the result as saved, and on replay returns it without calling fn,
// so both paths see the same value. fn receives an idempotency key unique to
// this run and step, for connectors to honour.
func (rc *Context) Step[T any](name string, fn func(ctx context.Context, key string) (T, error)) (T, error) {
var zero T
seq := rc.advance()
if saved, ok := rc.saved[seq]; ok {
return replay[T](saved, name)
}
if err := rc.checkCancel(rc.ctx); err != nil {
return zero, err
}
out, err := fn(rc.ctx, rc.run.ID.String()+":"+strconv.Itoa(seq))
step, recErr := rc.record(rc.ctx, seq, name, out, err)
if errors.Is(recErr, store.ErrStepExists) {
// Another holder recorded this step first; its result is the one that
// counts.
return rc.reload[T](seq, name)
}
if recErr != nil {
return zero, recErr
}
if step.Outcome != store.OutcomeDone {
return zero, err
}
// Hand back the value as recorded, not as fn returned it: encoding can
// rewrite it (raw JSON is compacted), and a resumed run must see exactly
// what this one sees.
return replay[T](step, name)
}
// Sleep suspends the run durably until d has passed since the first time this
// step ran.
func (rc *Context) Sleep(d time.Duration) error {
wake, err := rc.Step("sleep", func(context.Context, string) (time.Time, error) {
return time.Now().Add(d).UTC(), nil
})
if err != nil {
return err
}
if time.Now().Before(wake) {
return rc.suspend(&wake, "")
}
return nil
}
// Wait returns the payload signalled to this run under key, suspending the run
// durably until it arrives or timeout passes from the first time it waited.
func (rc *Context) Wait(key string, timeout time.Duration) (jsontext.Value, error) {
deadline, err := rc.Step("wait-start:"+key, func(context.Context, string) (time.Time, error) {
return time.Now().Add(timeout).UTC(), nil
})
if err != nil {
return nil, err
}
return rc.Step("wait:"+key, func(ctx context.Context, _ string) (jsontext.Value, error) {
payload, ok, err := rc.work.Signalled(ctx, rc.run.ID, key)
switch {
case err != nil:
return nil, err
case ok:
return payload, nil
case !time.Now().Before(deadline):
return nil, ErrWaitTimedOut
}
return nil, rc.suspend(&deadline, key)
})
}
// Context is the run's context: it ends when the run's lease is lost or the
// binary shuts down.
func (rc *Context) Context() context.Context { return rc.ctx }
func (rc *Context) advance() int {
seq := rc.next
rc.next++
return seq
}
func (rc *Context) checkCancel(ctx context.Context) error {
cancelled, err := rc.work.CancelRequested(ctx, rc.run.ID)
if err != nil {
return err
}
if cancelled {
return ErrCancelled
}
return nil
}
// record saves a step's outcome. A suspension, a lost lease or a cancelled
// context is not an outcome: nothing is recorded and the error passes through.
func (rc *Context) record(ctx context.Context, seq int, name string, out any, stepErr error) (store.Step, error) {
if errors.Is(stepErr, errSuspended) || errors.Is(stepErr, store.ErrLeaseLost) || errors.Is(stepErr, context.Canceled) {
return store.Step{}, stepErr
}
step := store.Step{RunID: rc.run.ID, Seq: seq, Name: name, Outcome: store.OutcomeDone}
switch {
case errors.Is(stepErr, ErrOutcomeUnknown):
step.Outcome = store.OutcomeUnknown
step.Error = sql.NullString{String: stepErr.Error(), Valid: true}
case stepErr != nil:
step.Outcome = store.OutcomeFailed
step.Error = sql.NullString{String: stepErr.Error(), Valid: true}
default:
raw, err := json.Marshal(out)
if err != nil {
return store.Step{}, fmt.Errorf("encode step %q: %w", name, err)
}
step.Output = raw
}
if err := rc.work.RecordStep(ctx, rc.token, step); err != nil {
return store.Step{}, err
}
rc.saved[seq] = step
return step, nil
}
// reload reads the step another holder recorded at seq and returns its result.
func (rc *Context) reload[T any](seq int, name string) (T, error) {
var zero T
steps, err := rc.work.Steps(rc.ctx, rc.run.ID)
if err != nil {
return zero, err
}
for _, s := range steps {
if s.Seq == seq {
rc.saved[seq] = s
return replay[T](s, name)
}
}
return zero, fmt.Errorf("step %d reported as recorded but not found", seq)
}
func (rc *Context) suspend(wakeAt *time.Time, waitingFor string) error {
if err := rc.work.Suspend(rc.ctx, rc.run.ID, rc.token, wakeAt, waitingFor); err != nil {
return err
}
return errSuspended
}
// replay returns a saved step's result as T.
func replay[T any](saved store.Step, name string) (T, error) {
var zero T
if saved.Name != name {
return zero, fmt.Errorf("%w: step %d is %q, replay asked for %q", ErrNondeterministic, saved.Seq, saved.Name, name)
}
switch saved.Outcome {
case store.OutcomeUnknown:
return zero, fmt.Errorf("%w: %s", ErrOutcomeUnknown, saved.Error.String)
case store.OutcomeFailed:
return zero, savedError(saved.Error.String)
}
var out T
if err := json.Unmarshal(saved.Output, &out); err != nil {
return zero, fmt.Errorf("decode step %q: %w", name, err)
}
return out, nil
}
// savedError rebuilds a recorded step error, keeping the engine's own
// sentinels comparable with errors.Is after a replay.
func savedError(text string) error {
for _, sentinel := range []error{ErrWaitTimedOut, ErrCancelled} {
if text == sentinel.Error() {
return sentinel
}
}
return errors.New(text)
}