Killing the binary mid-investigation and restarting it failed the run: the first model request after the resume didn't match the recorded one. The tool schemas pinned in a step reached the live attempt as written but the resume as stored, and both the v2 encoder (compacting raw JSON) and jsonb (reordering keys) rewrite them. Step now returns the decoded recorded value on the live path too, and runs.steps.output is json, which keeps the text as written.
230 lines
7.1 KiB
Go
230 lines
7.1 KiB
Go
package runs
|
|
|
|
import (
|
|
"context"
|
|
"database/sql"
|
|
"encoding/json/jsontext"
|
|
json "encoding/json/v2"
|
|
"errors"
|
|
"fmt"
|
|
"strconv"
|
|
"time"
|
|
|
|
"github.com/google/uuid"
|
|
|
|
"github.com/samcharles93/archie-core/modules/runs/internal/store"
|
|
)
|
|
|
|
var (
|
|
// ErrCancelled ends a run whose cancellation was requested.
|
|
ErrCancelled = errors.New("run cancelled")
|
|
// ErrOutcomeUnknown is returned by a step that can't confirm what its
|
|
// effect did. The step is recorded as unknown and is never repeated (R15).
|
|
ErrOutcomeUnknown = errors.New("outcome unknown")
|
|
// ErrWaitTimedOut is returned by Wait when its deadline passes.
|
|
ErrWaitTimedOut = errors.New("wait timed out")
|
|
// ErrNondeterministic means replay met a step the saved history doesn't
|
|
// have at that position: the workflow's code took a different path.
|
|
ErrNondeterministic = errors.New("workflow replay diverged from its saved steps")
|
|
|
|
errSuspended = errors.New("run suspended")
|
|
)
|
|
|
|
// Context is a run's handle on the engine while its workflow executes.
|
|
type Context struct {
|
|
// The run owns its context, as an http.Request does: every step takes its
|
|
// cancellation from it, and a second context passed beside it could disagree.
|
|
ctx context.Context //nolint:containedctx // see above
|
|
work workerStore
|
|
run store.Run
|
|
token uuid.UUID
|
|
saved map[int]store.Step
|
|
next int
|
|
}
|
|
|
|
func newContext(ctx context.Context, work workerStore, run store.Run, token uuid.UUID, steps []store.Step) *Context {
|
|
saved := make(map[int]store.Step, len(steps))
|
|
for _, s := range steps {
|
|
saved[s.Seq] = s
|
|
}
|
|
return &Context{ctx: ctx, work: work, run: run, token: token, saved: saved}
|
|
}
|
|
|
|
// RunID is the run's id.
|
|
func (rc *Context) RunID() uuid.UUID { return rc.run.ID }
|
|
|
|
// OrgID is the organisation the run belongs to.
|
|
func (rc *Context) OrgID() string { return rc.run.OrgID }
|
|
|
|
// Step runs fn once for this position in the workflow and saves its result.
|
|
// It returns the result as saved, and on replay returns it without calling fn,
|
|
// so both paths see the same value. fn receives an idempotency key unique to
|
|
// this run and step, for connectors to honour.
|
|
func (rc *Context) Step[T any](name string, fn func(ctx context.Context, key string) (T, error)) (T, error) {
|
|
var zero T
|
|
seq := rc.advance()
|
|
if saved, ok := rc.saved[seq]; ok {
|
|
return replay[T](saved, name)
|
|
}
|
|
if err := rc.checkCancel(rc.ctx); err != nil {
|
|
return zero, err
|
|
}
|
|
out, err := fn(rc.ctx, rc.run.ID.String()+":"+strconv.Itoa(seq))
|
|
step, recErr := rc.record(rc.ctx, seq, name, out, err)
|
|
if errors.Is(recErr, store.ErrStepExists) {
|
|
// Another holder recorded this step first; its result is the one that
|
|
// counts.
|
|
return rc.reload[T](seq, name)
|
|
}
|
|
if recErr != nil {
|
|
return zero, recErr
|
|
}
|
|
if step.Outcome != store.OutcomeDone {
|
|
return zero, err
|
|
}
|
|
// Hand back the value as recorded, not as fn returned it: encoding can
|
|
// rewrite it (raw JSON is compacted), and a resumed run must see exactly
|
|
// what this one sees.
|
|
return replay[T](step, name)
|
|
}
|
|
|
|
// Sleep suspends the run durably until d has passed since the first time this
|
|
// step ran.
|
|
func (rc *Context) Sleep(d time.Duration) error {
|
|
wake, err := rc.Step("sleep", func(context.Context, string) (time.Time, error) {
|
|
return time.Now().Add(d).UTC(), nil
|
|
})
|
|
if err != nil {
|
|
return err
|
|
}
|
|
if time.Now().Before(wake) {
|
|
return rc.suspend(&wake, "")
|
|
}
|
|
return nil
|
|
}
|
|
|
|
// Wait returns the payload signalled to this run under key, suspending the run
|
|
// durably until it arrives or timeout passes from the first time it waited.
|
|
func (rc *Context) Wait(key string, timeout time.Duration) (jsontext.Value, error) {
|
|
deadline, err := rc.Step("wait-start:"+key, func(context.Context, string) (time.Time, error) {
|
|
return time.Now().Add(timeout).UTC(), nil
|
|
})
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
return rc.Step("wait:"+key, func(ctx context.Context, _ string) (jsontext.Value, error) {
|
|
payload, ok, err := rc.work.Signalled(ctx, rc.run.ID, key)
|
|
switch {
|
|
case err != nil:
|
|
return nil, err
|
|
case ok:
|
|
return payload, nil
|
|
case !time.Now().Before(deadline):
|
|
return nil, ErrWaitTimedOut
|
|
}
|
|
return nil, rc.suspend(&deadline, key)
|
|
})
|
|
}
|
|
|
|
// Context is the run's context: it ends when the run's lease is lost or the
|
|
// binary shuts down.
|
|
func (rc *Context) Context() context.Context { return rc.ctx }
|
|
|
|
func (rc *Context) advance() int {
|
|
seq := rc.next
|
|
rc.next++
|
|
return seq
|
|
}
|
|
|
|
func (rc *Context) checkCancel(ctx context.Context) error {
|
|
cancelled, err := rc.work.CancelRequested(ctx, rc.run.ID)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
if cancelled {
|
|
return ErrCancelled
|
|
}
|
|
return nil
|
|
}
|
|
|
|
// record saves a step's outcome. A suspension, a lost lease or a cancelled
|
|
// context is not an outcome: nothing is recorded and the error passes through.
|
|
func (rc *Context) record(ctx context.Context, seq int, name string, out any, stepErr error) (store.Step, error) {
|
|
if errors.Is(stepErr, errSuspended) || errors.Is(stepErr, store.ErrLeaseLost) || errors.Is(stepErr, context.Canceled) {
|
|
return store.Step{}, stepErr
|
|
}
|
|
step := store.Step{RunID: rc.run.ID, Seq: seq, Name: name, Outcome: store.OutcomeDone}
|
|
switch {
|
|
case errors.Is(stepErr, ErrOutcomeUnknown):
|
|
step.Outcome = store.OutcomeUnknown
|
|
step.Error = sql.NullString{String: stepErr.Error(), Valid: true}
|
|
case stepErr != nil:
|
|
step.Outcome = store.OutcomeFailed
|
|
step.Error = sql.NullString{String: stepErr.Error(), Valid: true}
|
|
default:
|
|
raw, err := json.Marshal(out)
|
|
if err != nil {
|
|
return store.Step{}, fmt.Errorf("encode step %q: %w", name, err)
|
|
}
|
|
step.Output = raw
|
|
}
|
|
if err := rc.work.RecordStep(ctx, rc.token, step); err != nil {
|
|
return store.Step{}, err
|
|
}
|
|
rc.saved[seq] = step
|
|
return step, nil
|
|
}
|
|
|
|
// reload reads the step another holder recorded at seq and returns its result.
|
|
func (rc *Context) reload[T any](seq int, name string) (T, error) {
|
|
var zero T
|
|
steps, err := rc.work.Steps(rc.ctx, rc.run.ID)
|
|
if err != nil {
|
|
return zero, err
|
|
}
|
|
for _, s := range steps {
|
|
if s.Seq == seq {
|
|
rc.saved[seq] = s
|
|
return replay[T](s, name)
|
|
}
|
|
}
|
|
return zero, fmt.Errorf("step %d reported as recorded but not found", seq)
|
|
}
|
|
|
|
func (rc *Context) suspend(wakeAt *time.Time, waitingFor string) error {
|
|
if err := rc.work.Suspend(rc.ctx, rc.run.ID, rc.token, wakeAt, waitingFor); err != nil {
|
|
return err
|
|
}
|
|
return errSuspended
|
|
}
|
|
|
|
// replay returns a saved step's result as T.
|
|
func replay[T any](saved store.Step, name string) (T, error) {
|
|
var zero T
|
|
if saved.Name != name {
|
|
return zero, fmt.Errorf("%w: step %d is %q, replay asked for %q", ErrNondeterministic, saved.Seq, saved.Name, name)
|
|
}
|
|
switch saved.Outcome {
|
|
case store.OutcomeUnknown:
|
|
return zero, fmt.Errorf("%w: %s", ErrOutcomeUnknown, saved.Error.String)
|
|
case store.OutcomeFailed:
|
|
return zero, savedError(saved.Error.String)
|
|
}
|
|
var out T
|
|
if err := json.Unmarshal(saved.Output, &out); err != nil {
|
|
return zero, fmt.Errorf("decode step %q: %w", name, err)
|
|
}
|
|
return out, nil
|
|
}
|
|
|
|
// savedError rebuilds a recorded step error, keeping the engine's own
|
|
// sentinels comparable with errors.Is after a replay.
|
|
func savedError(text string) error {
|
|
for _, sentinel := range []error{ErrWaitTimedOut, ErrCancelled} {
|
|
if text == sentinel.Error() {
|
|
return sentinel
|
|
}
|
|
}
|
|
return errors.New(text)
|
|
}
|