package runs import ( "context" "database/sql" "encoding/json/jsontext" json "encoding/json/v2" "errors" "fmt" "strconv" "strings" "time" "github.com/google/uuid" "github.com/samcharles93/archie-core/modules/runs/internal/store" ) var ( // ErrCancelled ends a run whose cancellation was requested. ErrCancelled = errors.New("run cancelled") // ErrOutcomeUnknown is returned by a step that can't confirm what its // effect did. The step is recorded as unknown and is never repeated (R15). ErrOutcomeUnknown = errors.New("outcome unknown") // ErrWaitTimedOut is returned by Wait when its deadline passes. ErrWaitTimedOut = errors.New("wait timed out") // ErrNondeterministic means replay met a step the saved history doesn't // have at that position: the workflow's code took a different path. ErrNondeterministic = errors.New("workflow replay diverged from its saved steps") errSuspended = errors.New("run suspended") ) // Context is a run's handle on the engine while its workflow executes. type Context struct { // The run owns its context, as an http.Request does: every step takes its // cancellation from it, and a second context passed beside it could disagree. ctx context.Context //nolint:containedctx // see above work runWork run store.Run token uuid.UUID saved map[int]store.Step next int // started holds the keys of effects an earlier attempt began; new markers // are numbered from nextMarker. started map[string]bool nextMarker int // entered holds the top-level steps an earlier attempt reached. entered map[string]bool // ending is set when the run ends before its last step. A run that // completes every step has no ending and succeeds. ending *Ending // overlay is what a dry run carries instead of a stored version. overlay *dryOverlay // resolve checks a workflow to start, and returns the run to queue. It is a // field because the step types, which are initialised before it, can't // name the document checks that depend on them. resolve func(ctx context.Context, req StartRequest) (store.NewRun, error) } // markerBase is where started markers are numbered, far above any step a // workflow takes, so they never shift the positions a replay depends on. const markerBase = store.MarkerBase const ( startedPrefix = "started:" // enteredPrefix names the marker that records the position a top-level // step began at, so a parked run can retry from it. enteredPrefix = "at:" ) func newContext(ctx context.Context, work runWork, run store.Run, token uuid.UUID, steps []store.Step) *Context { saved := make(map[int]store.Step, len(steps)) started, entered := map[string]bool{}, map[string]bool{} nextMarker := markerBase for _, s := range steps { saved[s.Seq] = s if key, ok := strings.CutPrefix(s.Name, startedPrefix); ok { started[key] = true nextMarker = max(nextMarker, s.Seq+1) } if name, ok := strings.CutPrefix(s.Name, enteredPrefix); ok { entered[name] = true nextMarker = max(nextMarker, s.Seq+1) } } rc := &Context{ctx: ctx, work: work, run: run, token: token, saved: saved, started: started, entered: entered, nextMarker: nextMarker} rc.resolve = func(ctx context.Context, req StartRequest) (store.NewRun, error) { return newRun(ctx, work, req) } if run.Dry { rc.overlay, _ = readOverlay(run.Overlay) } return rc } // Begin durably records that the effect behind key is about to happen, before // it does. It reports true when an earlier attempt already began it: that // attempt may have died after the effect and before its result was recorded, // so the caller must not run it again and treats its outcome as unknown (R15). func (rc *Context) Begin(ctx context.Context, key string) (bool, error) { if rc.started[key] { return true, nil } seq := rc.nextMarker if _, err := rc.record(ctx, seq, startedPrefix+key, true, nil); err != nil { return false, err } rc.started[key] = true rc.nextMarker = seq + 1 return false, nil } // Enter records that the top-level step name begins at the run's next position, // unless an earlier attempt already did. Retrying a parked run from the step // discards everything recorded from that position onwards. func (rc *Context) Enter(ctx context.Context, name string) error { if rc.entered[name] { return nil } seq := rc.nextMarker if _, err := rc.record(ctx, seq, enteredPrefix+name, rc.next, nil); err != nil && !errors.Is(err, store.ErrStepExists) { return err } rc.entered[name] = true rc.nextMarker = seq + 1 return nil } // End records that the run ends before its last step, with this state and // detail. func (rc *Context) End(state, detail string) { rc.ending = &Ending{State: state, Detail: detail} } // Ending describes how a run ended before its last step. type Ending struct { // State is the run state that the ending sets. State string // Detail is the finish step's detail, or the reason a step parked the // run. Detail string } // RunID is the run's id. func (rc *Context) RunID() uuid.UUID { return rc.run.ID } // Workflow is the name of the workflow this run executes. func (rc *Context) Workflow() string { return rc.run.Workflow } // OrgID is the organisation the run belongs to. func (rc *Context) OrgID() string { return rc.run.OrgID } // Step runs fn once for this position in the workflow and saves its result. // It returns the result as saved, and on replay returns it without calling fn, // so both paths see the same value. fn receives an idempotency key unique to // this run and step, for connectors to honour. func (rc *Context) Step[T any](name string, fn func(ctx context.Context, key string) (T, error)) (T, error) { var zero T seq := rc.advance() if saved, ok := rc.saved[seq]; ok { return replay[T](saved, name) } if err := rc.checkCancel(rc.ctx); err != nil { return zero, err } out, err := fn(rc.ctx, rc.run.ID.String()+":"+strconv.Itoa(seq)) step, recErr := rc.record(rc.ctx, seq, name, out, err) if errors.Is(recErr, store.ErrStepExists) { // Another holder recorded this step first; its result is the one that // counts. return rc.reload[T](seq, name) } if recErr != nil { return zero, recErr } if step.Outcome != store.OutcomeDone { return zero, err } // Hand back the value as recorded, not as fn returned it: encoding can // rewrite it (raw JSON is compacted), and a resumed run must see exactly // what this one sees. return replay[T](step, name) } // Sleep suspends the run durably until d has passed since the first time this // step ran. func (rc *Context) Sleep(d time.Duration) error { wake, err := rc.Step("sleep", func(context.Context, string) (time.Time, error) { return time.Now().Add(d).UTC(), nil }) if err != nil { return err } if time.Now().Before(wake) { return rc.suspend(&wake, "") } return nil } // Wait returns the payload signalled to this run under key, suspending the run // durably until it arrives or timeout passes from the first time it waited. func (rc *Context) Wait(key string, timeout time.Duration) (jsontext.Value, error) { deadline, err := rc.Step("wait-start:"+key, func(context.Context, string) (time.Time, error) { return time.Now().Add(timeout).UTC(), nil }) if err != nil { return nil, err } return rc.Step("wait:"+key, func(ctx context.Context, _ string) (jsontext.Value, error) { payload, ok, err := rc.work.Signalled(ctx, rc.run.ID, key) switch { case err != nil: return nil, err case ok: return payload, nil case !time.Now().Before(deadline): return nil, ErrWaitTimedOut } return nil, rc.suspend(&deadline, key) }) } // Context is the run's context: it ends when the run's lease is lost or the // binary shuts down. func (rc *Context) Context() context.Context { return rc.ctx } func (rc *Context) advance() int { seq := rc.next rc.next++ return seq } func (rc *Context) checkCancel(ctx context.Context) error { cancelled, err := rc.work.CancelRequested(ctx, rc.run.ID) if err != nil { return err } if cancelled { return ErrCancelled } return nil } // record saves a step's outcome. A suspension, a lost lease or a cancelled // context is not an outcome: nothing is recorded and the error passes through. func (rc *Context) record(ctx context.Context, seq int, name string, out any, stepErr error) (store.Step, error) { if errors.Is(stepErr, errSuspended) || errors.Is(stepErr, store.ErrLeaseLost) || errors.Is(stepErr, context.Canceled) { return store.Step{}, stepErr } step := store.Step{RunID: rc.run.ID, Seq: seq, Name: name, Outcome: store.OutcomeDone} switch { case errors.Is(stepErr, ErrOutcomeUnknown): step.Outcome = store.OutcomeUnknown step.Error = sql.NullString{String: stepErr.Error(), Valid: true} case stepErr != nil: step.Outcome = store.OutcomeFailed step.Error = sql.NullString{String: stepErr.Error(), Valid: true} default: raw, err := json.Marshal(out) if err != nil { return store.Step{}, fmt.Errorf("encode step %q: %w", name, err) } step.Output = raw } if err := rc.work.RecordStep(ctx, rc.token, step); err != nil { return store.Step{}, err } rc.saved[seq] = step return step, nil } // reload reads the step another holder recorded at seq and returns its result. func (rc *Context) reload[T any](seq int, name string) (T, error) { var zero T steps, err := rc.work.Steps(rc.ctx, rc.run.ID) if err != nil { return zero, err } for _, s := range steps { if s.Seq == seq { rc.saved[seq] = s return replay[T](s, name) } } return zero, fmt.Errorf("step %d reported as recorded but not found", seq) } func (rc *Context) suspend(wakeAt *time.Time, waitingFor string) error { if err := rc.work.Suspend(rc.ctx, rc.run.ID, rc.token, wakeAt, waitingFor); err != nil { return err } return errSuspended } // replay returns a saved step's result as T. func replay[T any](saved store.Step, name string) (T, error) { var zero T if saved.Name != name { return zero, fmt.Errorf("%w: step %d is %q, replay asked for %q", ErrNondeterministic, saved.Seq, saved.Name, name) } switch saved.Outcome { case store.OutcomeUnknown: return zero, fmt.Errorf("%w: %s", ErrOutcomeUnknown, saved.Error.String) case store.OutcomeFailed: return zero, savedError(saved.Error.String) } var out T if err := json.Unmarshal(saved.Output, &out); err != nil { return zero, fmt.Errorf("decode step %q: %w", name, err) } return out, nil } // savedError rebuilds a recorded step error, keeping the engine's own // sentinels comparable with errors.Is after a replay. func savedError(text string) error { for _, sentinel := range []error{ErrWaitTimedOut, ErrCancelled} { if text == sentinel.Error() { return sentinel } } return errors.New(text) }