323 lines
10 KiB
Go
323 lines
10 KiB
Go
package runs
|
|
|
|
import (
|
|
"context"
|
|
"database/sql"
|
|
"encoding/json/jsontext"
|
|
json "encoding/json/v2"
|
|
"errors"
|
|
"fmt"
|
|
"strconv"
|
|
"strings"
|
|
"time"
|
|
|
|
"github.com/google/uuid"
|
|
|
|
"github.com/samcharles93/archie-core/modules/runs/internal/store"
|
|
)
|
|
|
|
var (
|
|
// ErrCancelled ends a run whose cancellation was requested.
|
|
ErrCancelled = errors.New("run cancelled")
|
|
// ErrOutcomeUnknown is returned by a step that can't confirm what its
|
|
// effect did. The step is recorded as unknown and is never repeated (R15).
|
|
ErrOutcomeUnknown = errors.New("outcome unknown")
|
|
// ErrWaitTimedOut is returned by Wait when its deadline passes.
|
|
ErrWaitTimedOut = errors.New("wait timed out")
|
|
// ErrNondeterministic means replay met a step the saved history doesn't
|
|
// have at that position: the workflow's code took a different path.
|
|
ErrNondeterministic = errors.New("workflow replay diverged from its saved steps")
|
|
|
|
errSuspended = errors.New("run suspended")
|
|
)
|
|
|
|
// Context is a run's handle on the engine while its workflow executes.
|
|
type Context struct {
|
|
// The run owns its context, as an http.Request does: every step takes its
|
|
// cancellation from it, and a second context passed beside it could disagree.
|
|
ctx context.Context //nolint:containedctx // see above
|
|
work runWork
|
|
run store.Run
|
|
token uuid.UUID
|
|
saved map[int]store.Step
|
|
next int
|
|
// started holds the keys of effects an earlier attempt began; new markers
|
|
// are numbered from nextMarker.
|
|
started map[string]bool
|
|
nextMarker int
|
|
// entered holds the top-level steps an earlier attempt reached.
|
|
entered map[string]bool
|
|
// ending is set when the run ends before its last step. A run that
|
|
// completes every step has no ending and succeeds.
|
|
ending *Ending
|
|
// overlay is what a dry run carries instead of a stored version.
|
|
overlay *dryOverlay
|
|
// resolve checks a workflow to start, and returns the run to queue. It is a
|
|
// field because the step types, which are initialised before it, can't
|
|
// name the document checks that depend on them.
|
|
resolve func(ctx context.Context, req StartRequest) (store.NewRun, error)
|
|
}
|
|
|
|
// markerBase is where started markers are numbered, far above any step a
|
|
// workflow takes, so they never shift the positions a replay depends on.
|
|
const markerBase = store.MarkerBase
|
|
|
|
const (
|
|
startedPrefix = "started:"
|
|
// enteredPrefix names the marker that records the position a top-level
|
|
// step began at, so a parked run can retry from it.
|
|
enteredPrefix = "at:"
|
|
)
|
|
|
|
func newContext(ctx context.Context, work runWork, run store.Run, token uuid.UUID, steps []store.Step) *Context {
|
|
saved := make(map[int]store.Step, len(steps))
|
|
started, entered := map[string]bool{}, map[string]bool{}
|
|
nextMarker := markerBase
|
|
for _, s := range steps {
|
|
saved[s.Seq] = s
|
|
if key, ok := strings.CutPrefix(s.Name, startedPrefix); ok {
|
|
started[key] = true
|
|
nextMarker = max(nextMarker, s.Seq+1)
|
|
}
|
|
if name, ok := strings.CutPrefix(s.Name, enteredPrefix); ok {
|
|
entered[name] = true
|
|
nextMarker = max(nextMarker, s.Seq+1)
|
|
}
|
|
}
|
|
rc := &Context{ctx: ctx, work: work, run: run, token: token, saved: saved, started: started, entered: entered, nextMarker: nextMarker}
|
|
rc.resolve = func(ctx context.Context, req StartRequest) (store.NewRun, error) { return newRun(ctx, work, req) }
|
|
if run.Dry {
|
|
rc.overlay, _ = readOverlay(run.Overlay)
|
|
}
|
|
return rc
|
|
}
|
|
|
|
// Begin durably records that the effect behind key is about to happen, before
|
|
// it does. It reports true when an earlier attempt already began it: that
|
|
// attempt may have died after the effect and before its result was recorded,
|
|
// so the caller must not run it again and treats its outcome as unknown (R15).
|
|
func (rc *Context) Begin(ctx context.Context, key string) (bool, error) {
|
|
if rc.started[key] {
|
|
return true, nil
|
|
}
|
|
seq := rc.nextMarker
|
|
if _, err := rc.record(ctx, seq, startedPrefix+key, true, nil); err != nil {
|
|
return false, err
|
|
}
|
|
rc.started[key] = true
|
|
rc.nextMarker = seq + 1
|
|
return false, nil
|
|
}
|
|
|
|
// Enter records that the top-level step name begins at the run's next position,
|
|
// unless an earlier attempt already did. Retrying a parked run from the step
|
|
// discards everything recorded from that position onwards.
|
|
func (rc *Context) Enter(ctx context.Context, name string) error {
|
|
if rc.entered[name] {
|
|
return nil
|
|
}
|
|
seq := rc.nextMarker
|
|
if _, err := rc.record(ctx, seq, enteredPrefix+name, rc.next, nil); err != nil && !errors.Is(err, store.ErrStepExists) {
|
|
return err
|
|
}
|
|
rc.entered[name] = true
|
|
rc.nextMarker = seq + 1
|
|
return nil
|
|
}
|
|
|
|
// End records that the run ends before its last step, with this state and
|
|
// detail.
|
|
func (rc *Context) End(state, detail string) {
|
|
rc.ending = &Ending{State: state, Detail: detail}
|
|
}
|
|
|
|
// Ending describes how a run ended before its last step.
|
|
type Ending struct {
|
|
// State is the run state that the ending sets.
|
|
State string
|
|
// Detail is the finish step's detail, or the reason a step parked the
|
|
// run.
|
|
Detail string
|
|
}
|
|
|
|
// RunID is the run's id.
|
|
func (rc *Context) RunID() uuid.UUID { return rc.run.ID }
|
|
|
|
// Workflow is the name of the workflow this run executes.
|
|
func (rc *Context) Workflow() string { return rc.run.Workflow }
|
|
|
|
// OrgID is the organisation the run belongs to.
|
|
func (rc *Context) OrgID() string { return rc.run.OrgID }
|
|
|
|
// Step runs fn once for this position in the workflow and saves its result.
|
|
// It returns the result as saved, and on replay returns it without calling fn,
|
|
// so both paths see the same value. fn receives an idempotency key unique to
|
|
// this run and step, for connectors to honour.
|
|
func (rc *Context) Step[T any](name string, fn func(ctx context.Context, key string) (T, error)) (T, error) {
|
|
var zero T
|
|
seq := rc.advance()
|
|
if saved, ok := rc.saved[seq]; ok {
|
|
return replay[T](saved, name)
|
|
}
|
|
if err := rc.checkCancel(rc.ctx); err != nil {
|
|
return zero, err
|
|
}
|
|
out, err := fn(rc.ctx, rc.run.ID.String()+":"+strconv.Itoa(seq))
|
|
step, recErr := rc.record(rc.ctx, seq, name, out, err)
|
|
if errors.Is(recErr, store.ErrStepExists) {
|
|
// Another holder recorded this step first; its result is the one that
|
|
// counts.
|
|
return rc.reload[T](seq, name)
|
|
}
|
|
if recErr != nil {
|
|
return zero, recErr
|
|
}
|
|
if step.Outcome != store.OutcomeDone {
|
|
return zero, err
|
|
}
|
|
// Hand back the value as recorded, not as fn returned it: encoding can
|
|
// rewrite it (raw JSON is compacted), and a resumed run must see exactly
|
|
// what this one sees.
|
|
return replay[T](step, name)
|
|
}
|
|
|
|
// Sleep suspends the run durably until d has passed since the first time this
|
|
// step ran.
|
|
func (rc *Context) Sleep(d time.Duration) error {
|
|
wake, err := rc.Step("sleep", func(context.Context, string) (time.Time, error) {
|
|
return time.Now().Add(d).UTC(), nil
|
|
})
|
|
if err != nil {
|
|
return err
|
|
}
|
|
if time.Now().Before(wake) {
|
|
return rc.suspend(&wake, "")
|
|
}
|
|
return nil
|
|
}
|
|
|
|
// Wait returns the payload signalled to this run under key, suspending the run
|
|
// durably until it arrives or timeout passes from the first time it waited.
|
|
func (rc *Context) Wait(key string, timeout time.Duration) (jsontext.Value, error) {
|
|
deadline, err := rc.Step("wait-start:"+key, func(context.Context, string) (time.Time, error) {
|
|
return time.Now().Add(timeout).UTC(), nil
|
|
})
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
return rc.Step("wait:"+key, func(ctx context.Context, _ string) (jsontext.Value, error) {
|
|
payload, ok, err := rc.work.Signalled(ctx, rc.run.ID, key)
|
|
switch {
|
|
case err != nil:
|
|
return nil, err
|
|
case ok:
|
|
return payload, nil
|
|
case !time.Now().Before(deadline):
|
|
return nil, ErrWaitTimedOut
|
|
}
|
|
return nil, rc.suspend(&deadline, key)
|
|
})
|
|
}
|
|
|
|
// Context is the run's context: it ends when the run's lease is lost or the
|
|
// binary shuts down.
|
|
func (rc *Context) Context() context.Context { return rc.ctx }
|
|
|
|
func (rc *Context) advance() int {
|
|
seq := rc.next
|
|
rc.next++
|
|
return seq
|
|
}
|
|
|
|
func (rc *Context) checkCancel(ctx context.Context) error {
|
|
cancelled, err := rc.work.CancelRequested(ctx, rc.run.ID)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
if cancelled {
|
|
return ErrCancelled
|
|
}
|
|
return nil
|
|
}
|
|
|
|
// record saves a step's outcome. A suspension, a lost lease or a cancelled
|
|
// context is not an outcome: nothing is recorded and the error passes through.
|
|
func (rc *Context) record(ctx context.Context, seq int, name string, out any, stepErr error) (store.Step, error) {
|
|
if errors.Is(stepErr, errSuspended) || errors.Is(stepErr, store.ErrLeaseLost) || errors.Is(stepErr, context.Canceled) {
|
|
return store.Step{}, stepErr
|
|
}
|
|
step := store.Step{RunID: rc.run.ID, Seq: seq, Name: name, Outcome: store.OutcomeDone}
|
|
switch {
|
|
case errors.Is(stepErr, ErrOutcomeUnknown):
|
|
step.Outcome = store.OutcomeUnknown
|
|
step.Error = sql.NullString{String: stepErr.Error(), Valid: true}
|
|
case stepErr != nil:
|
|
step.Outcome = store.OutcomeFailed
|
|
step.Error = sql.NullString{String: stepErr.Error(), Valid: true}
|
|
default:
|
|
raw, err := json.Marshal(out)
|
|
if err != nil {
|
|
return store.Step{}, fmt.Errorf("encode step %q: %w", name, err)
|
|
}
|
|
step.Output = raw
|
|
}
|
|
if err := rc.work.RecordStep(ctx, rc.token, step); err != nil {
|
|
return store.Step{}, err
|
|
}
|
|
rc.saved[seq] = step
|
|
return step, nil
|
|
}
|
|
|
|
// reload reads the step another holder recorded at seq and returns its result.
|
|
func (rc *Context) reload[T any](seq int, name string) (T, error) {
|
|
var zero T
|
|
steps, err := rc.work.Steps(rc.ctx, rc.run.ID)
|
|
if err != nil {
|
|
return zero, err
|
|
}
|
|
for _, s := range steps {
|
|
if s.Seq == seq {
|
|
rc.saved[seq] = s
|
|
return replay[T](s, name)
|
|
}
|
|
}
|
|
return zero, fmt.Errorf("step %d reported as recorded but not found", seq)
|
|
}
|
|
|
|
func (rc *Context) suspend(wakeAt *time.Time, waitingFor string) error {
|
|
if err := rc.work.Suspend(rc.ctx, rc.run.ID, rc.token, wakeAt, waitingFor); err != nil {
|
|
return err
|
|
}
|
|
return errSuspended
|
|
}
|
|
|
|
// replay returns a saved step's result as T.
|
|
func replay[T any](saved store.Step, name string) (T, error) {
|
|
var zero T
|
|
if saved.Name != name {
|
|
return zero, fmt.Errorf("%w: step %d is %q, replay asked for %q", ErrNondeterministic, saved.Seq, saved.Name, name)
|
|
}
|
|
switch saved.Outcome {
|
|
case store.OutcomeUnknown:
|
|
return zero, fmt.Errorf("%w: %s", ErrOutcomeUnknown, saved.Error.String)
|
|
case store.OutcomeFailed:
|
|
return zero, savedError(saved.Error.String)
|
|
}
|
|
var out T
|
|
if err := json.Unmarshal(saved.Output, &out); err != nil {
|
|
return zero, fmt.Errorf("decode step %q: %w", name, err)
|
|
}
|
|
return out, nil
|
|
}
|
|
|
|
// savedError rebuilds a recorded step error, keeping the engine's own
|
|
// sentinels comparable with errors.Is after a replay.
|
|
func savedError(text string) error {
|
|
for _, sentinel := range []error{ErrWaitTimedOut, ErrCancelled} {
|
|
if text == sentinel.Error() {
|
|
return sentinel
|
|
}
|
|
}
|
|
return errors.New(text)
|
|
}
|