Files
archie/modules/runs/context.go
T

323 lines
10 KiB
Go

package runs
import (
"context"
"database/sql"
"encoding/json/jsontext"
json "encoding/json/v2"
"errors"
"fmt"
"strconv"
"strings"
"time"
"github.com/google/uuid"
"github.com/samcharles93/archie-core/modules/runs/internal/store"
)
var (
// ErrCancelled ends a run whose cancellation was requested.
ErrCancelled = errors.New("run cancelled")
// ErrOutcomeUnknown is returned by a step that can't confirm what its
// effect did. The step is recorded as unknown and is never repeated (R15).
ErrOutcomeUnknown = errors.New("outcome unknown")
// ErrWaitTimedOut is returned by Wait when its deadline passes.
ErrWaitTimedOut = errors.New("wait timed out")
// ErrNondeterministic means replay met a step the saved history doesn't
// have at that position: the workflow's code took a different path.
ErrNondeterministic = errors.New("workflow replay diverged from its saved steps")
errSuspended = errors.New("run suspended")
)
// Context is a run's handle on the engine while its workflow executes.
type Context struct {
// The run owns its context, as an http.Request does: every step takes its
// cancellation from it, and a second context passed beside it could disagree.
ctx context.Context //nolint:containedctx // see above
work runWork
run store.Run
token uuid.UUID
saved map[int]store.Step
next int
// started holds the keys of effects an earlier attempt began; new markers
// are numbered from nextMarker.
started map[string]bool
nextMarker int
// entered holds the top-level steps an earlier attempt reached.
entered map[string]bool
// ending is set when the run ends before its last step. A run that
// completes every step has no ending and succeeds.
ending *Ending
// overlay is what a dry run carries instead of a stored version.
overlay *dryOverlay
// resolve checks a workflow to start, and returns the run to queue. It is a
// field because the step types, which are initialised before it, can't
// name the document checks that depend on them.
resolve func(ctx context.Context, req StartRequest) (store.NewRun, error)
}
// markerBase is where started markers are numbered, far above any step a
// workflow takes, so they never shift the positions a replay depends on.
const markerBase = store.MarkerBase
const (
startedPrefix = "started:"
// enteredPrefix names the marker that records the position a top-level
// step began at, so a parked run can retry from it.
enteredPrefix = "at:"
)
func newContext(ctx context.Context, work runWork, run store.Run, token uuid.UUID, steps []store.Step) *Context {
saved := make(map[int]store.Step, len(steps))
started, entered := map[string]bool{}, map[string]bool{}
nextMarker := markerBase
for _, s := range steps {
saved[s.Seq] = s
if key, ok := strings.CutPrefix(s.Name, startedPrefix); ok {
started[key] = true
nextMarker = max(nextMarker, s.Seq+1)
}
if name, ok := strings.CutPrefix(s.Name, enteredPrefix); ok {
entered[name] = true
nextMarker = max(nextMarker, s.Seq+1)
}
}
rc := &Context{ctx: ctx, work: work, run: run, token: token, saved: saved, started: started, entered: entered, nextMarker: nextMarker}
rc.resolve = func(ctx context.Context, req StartRequest) (store.NewRun, error) { return newRun(ctx, work, req) }
if run.Dry {
rc.overlay, _ = readOverlay(run.Overlay)
}
return rc
}
// Begin durably records that the effect behind key is about to happen, before
// it does. It reports true when an earlier attempt already began it: that
// attempt may have died after the effect and before its result was recorded,
// so the caller must not run it again and treats its outcome as unknown (R15).
func (rc *Context) Begin(ctx context.Context, key string) (bool, error) {
if rc.started[key] {
return true, nil
}
seq := rc.nextMarker
if _, err := rc.record(ctx, seq, startedPrefix+key, true, nil); err != nil {
return false, err
}
rc.started[key] = true
rc.nextMarker = seq + 1
return false, nil
}
// Enter records that the top-level step name begins at the run's next position,
// unless an earlier attempt already did. Retrying a parked run from the step
// discards everything recorded from that position onwards.
func (rc *Context) Enter(ctx context.Context, name string) error {
if rc.entered[name] {
return nil
}
seq := rc.nextMarker
if _, err := rc.record(ctx, seq, enteredPrefix+name, rc.next, nil); err != nil && !errors.Is(err, store.ErrStepExists) {
return err
}
rc.entered[name] = true
rc.nextMarker = seq + 1
return nil
}
// End records that the run ends before its last step, with this state and
// detail.
func (rc *Context) End(state, detail string) {
rc.ending = &Ending{State: state, Detail: detail}
}
// Ending describes how a run ended before its last step.
type Ending struct {
// State is the run state that the ending sets.
State string
// Detail is the finish step's detail, or the reason a step parked the
// run.
Detail string
}
// RunID is the run's id.
func (rc *Context) RunID() uuid.UUID { return rc.run.ID }
// Workflow is the name of the workflow this run executes.
func (rc *Context) Workflow() string { return rc.run.Workflow }
// OrgID is the organisation the run belongs to.
func (rc *Context) OrgID() string { return rc.run.OrgID }
// Step runs fn once for this position in the workflow and saves its result.
// It returns the result as saved, and on replay returns it without calling fn,
// so both paths see the same value. fn receives an idempotency key unique to
// this run and step, for connectors to honour.
func (rc *Context) Step[T any](name string, fn func(ctx context.Context, key string) (T, error)) (T, error) {
var zero T
seq := rc.advance()
if saved, ok := rc.saved[seq]; ok {
return replay[T](saved, name)
}
if err := rc.checkCancel(rc.ctx); err != nil {
return zero, err
}
out, err := fn(rc.ctx, rc.run.ID.String()+":"+strconv.Itoa(seq))
step, recErr := rc.record(rc.ctx, seq, name, out, err)
if errors.Is(recErr, store.ErrStepExists) {
// Another holder recorded this step first; its result is the one that
// counts.
return rc.reload[T](seq, name)
}
if recErr != nil {
return zero, recErr
}
if step.Outcome != store.OutcomeDone {
return zero, err
}
// Hand back the value as recorded, not as fn returned it: encoding can
// rewrite it (raw JSON is compacted), and a resumed run must see exactly
// what this one sees.
return replay[T](step, name)
}
// Sleep suspends the run durably until d has passed since the first time this
// step ran.
func (rc *Context) Sleep(d time.Duration) error {
wake, err := rc.Step("sleep", func(context.Context, string) (time.Time, error) {
return time.Now().Add(d).UTC(), nil
})
if err != nil {
return err
}
if time.Now().Before(wake) {
return rc.suspend(&wake, "")
}
return nil
}
// Wait returns the payload signalled to this run under key, suspending the run
// durably until it arrives or timeout passes from the first time it waited.
func (rc *Context) Wait(key string, timeout time.Duration) (jsontext.Value, error) {
deadline, err := rc.Step("wait-start:"+key, func(context.Context, string) (time.Time, error) {
return time.Now().Add(timeout).UTC(), nil
})
if err != nil {
return nil, err
}
return rc.Step("wait:"+key, func(ctx context.Context, _ string) (jsontext.Value, error) {
payload, ok, err := rc.work.Signalled(ctx, rc.run.ID, key)
switch {
case err != nil:
return nil, err
case ok:
return payload, nil
case !time.Now().Before(deadline):
return nil, ErrWaitTimedOut
}
return nil, rc.suspend(&deadline, key)
})
}
// Context is the run's context: it ends when the run's lease is lost or the
// binary shuts down.
func (rc *Context) Context() context.Context { return rc.ctx }
func (rc *Context) advance() int {
seq := rc.next
rc.next++
return seq
}
func (rc *Context) checkCancel(ctx context.Context) error {
cancelled, err := rc.work.CancelRequested(ctx, rc.run.ID)
if err != nil {
return err
}
if cancelled {
return ErrCancelled
}
return nil
}
// record saves a step's outcome. A suspension, a lost lease or a cancelled
// context is not an outcome: nothing is recorded and the error passes through.
func (rc *Context) record(ctx context.Context, seq int, name string, out any, stepErr error) (store.Step, error) {
if errors.Is(stepErr, errSuspended) || errors.Is(stepErr, store.ErrLeaseLost) || errors.Is(stepErr, context.Canceled) {
return store.Step{}, stepErr
}
step := store.Step{RunID: rc.run.ID, Seq: seq, Name: name, Outcome: store.OutcomeDone}
switch {
case errors.Is(stepErr, ErrOutcomeUnknown):
step.Outcome = store.OutcomeUnknown
step.Error = sql.NullString{String: stepErr.Error(), Valid: true}
case stepErr != nil:
step.Outcome = store.OutcomeFailed
step.Error = sql.NullString{String: stepErr.Error(), Valid: true}
default:
raw, err := json.Marshal(out)
if err != nil {
return store.Step{}, fmt.Errorf("encode step %q: %w", name, err)
}
step.Output = raw
}
if err := rc.work.RecordStep(ctx, rc.token, step); err != nil {
return store.Step{}, err
}
rc.saved[seq] = step
return step, nil
}
// reload reads the step another holder recorded at seq and returns its result.
func (rc *Context) reload[T any](seq int, name string) (T, error) {
var zero T
steps, err := rc.work.Steps(rc.ctx, rc.run.ID)
if err != nil {
return zero, err
}
for _, s := range steps {
if s.Seq == seq {
rc.saved[seq] = s
return replay[T](s, name)
}
}
return zero, fmt.Errorf("step %d reported as recorded but not found", seq)
}
func (rc *Context) suspend(wakeAt *time.Time, waitingFor string) error {
if err := rc.work.Suspend(rc.ctx, rc.run.ID, rc.token, wakeAt, waitingFor); err != nil {
return err
}
return errSuspended
}
// replay returns a saved step's result as T.
func replay[T any](saved store.Step, name string) (T, error) {
var zero T
if saved.Name != name {
return zero, fmt.Errorf("%w: step %d is %q, replay asked for %q", ErrNondeterministic, saved.Seq, saved.Name, name)
}
switch saved.Outcome {
case store.OutcomeUnknown:
return zero, fmt.Errorf("%w: %s", ErrOutcomeUnknown, saved.Error.String)
case store.OutcomeFailed:
return zero, savedError(saved.Error.String)
}
var out T
if err := json.Unmarshal(saved.Output, &out); err != nil {
return zero, fmt.Errorf("decode step %q: %w", name, err)
}
return out, nil
}
// savedError rebuilds a recorded step error, keeping the engine's own
// sentinels comparable with errors.Is after a replay.
func savedError(text string) error {
for _, sentinel := range []error{ErrWaitTimedOut, ErrCancelled} {
if text == sentinel.Error() {
return sentinel
}
}
return errors.New(text)
}