Files
archie/modules/runs/context.go
T
sam 4877362223 fix(runs): a step hands back its value as recorded, so a resumed agent doesn't diverge from its journal (#1977)
Killing the binary mid-investigation and restarting it failed the run: the
first model request after the resume didn't match the recorded one. The tool
schemas pinned in a step reached the live attempt as written but the resume
as stored, and both the v2 encoder (compacting raw JSON) and jsonb (reordering
keys) rewrite them. Step now returns the decoded recorded value on the live
path too, and runs.steps.output is json, which keeps the text as written.
2026-11-16 00:36:19 +11:00

230 lines
7.1 KiB
Go

package runs
import (
"context"
"database/sql"
"encoding/json/jsontext"
json "encoding/json/v2"
"errors"
"fmt"
"strconv"
"time"
"github.com/google/uuid"
"github.com/samcharles93/archie-core/modules/runs/internal/store"
)
var (
// ErrCancelled ends a run whose cancellation was requested.
ErrCancelled = errors.New("run cancelled")
// ErrOutcomeUnknown is returned by a step that can't confirm what its
// effect did. The step is recorded as unknown and is never repeated (R15).
ErrOutcomeUnknown = errors.New("outcome unknown")
// ErrWaitTimedOut is returned by Wait when its deadline passes.
ErrWaitTimedOut = errors.New("wait timed out")
// ErrNondeterministic means replay met a step the saved history doesn't
// have at that position: the workflow's code took a different path.
ErrNondeterministic = errors.New("workflow replay diverged from its saved steps")
errSuspended = errors.New("run suspended")
)
// Context is a run's handle on the engine while its workflow executes.
type Context struct {
// The run owns its context, as an http.Request does: every step takes its
// cancellation from it, and a second context passed beside it could disagree.
ctx context.Context //nolint:containedctx // see above
work workerStore
run store.Run
token uuid.UUID
saved map[int]store.Step
next int
}
func newContext(ctx context.Context, work workerStore, run store.Run, token uuid.UUID, steps []store.Step) *Context {
saved := make(map[int]store.Step, len(steps))
for _, s := range steps {
saved[s.Seq] = s
}
return &Context{ctx: ctx, work: work, run: run, token: token, saved: saved}
}
// RunID is the run's id.
func (rc *Context) RunID() uuid.UUID { return rc.run.ID }
// OrgID is the organisation the run belongs to.
func (rc *Context) OrgID() string { return rc.run.OrgID }
// Step runs fn once for this position in the workflow and saves its result.
// It returns the result as saved, and on replay returns it without calling fn,
// so both paths see the same value. fn receives an idempotency key unique to
// this run and step, for connectors to honour.
func (rc *Context) Step[T any](name string, fn func(ctx context.Context, key string) (T, error)) (T, error) {
var zero T
seq := rc.advance()
if saved, ok := rc.saved[seq]; ok {
return replay[T](saved, name)
}
if err := rc.checkCancel(rc.ctx); err != nil {
return zero, err
}
out, err := fn(rc.ctx, rc.run.ID.String()+":"+strconv.Itoa(seq))
step, recErr := rc.record(rc.ctx, seq, name, out, err)
if errors.Is(recErr, store.ErrStepExists) {
// Another holder recorded this step first; its result is the one that
// counts.
return rc.reload[T](seq, name)
}
if recErr != nil {
return zero, recErr
}
if step.Outcome != store.OutcomeDone {
return zero, err
}
// Hand back the value as recorded, not as fn returned it: encoding can
// rewrite it (raw JSON is compacted), and a resumed run must see exactly
// what this one sees.
return replay[T](step, name)
}
// Sleep suspends the run durably until d has passed since the first time this
// step ran.
func (rc *Context) Sleep(d time.Duration) error {
wake, err := rc.Step("sleep", func(context.Context, string) (time.Time, error) {
return time.Now().Add(d).UTC(), nil
})
if err != nil {
return err
}
if time.Now().Before(wake) {
return rc.suspend(&wake, "")
}
return nil
}
// Wait returns the payload signalled to this run under key, suspending the run
// durably until it arrives or timeout passes from the first time it waited.
func (rc *Context) Wait(key string, timeout time.Duration) (jsontext.Value, error) {
deadline, err := rc.Step("wait-start:"+key, func(context.Context, string) (time.Time, error) {
return time.Now().Add(timeout).UTC(), nil
})
if err != nil {
return nil, err
}
return rc.Step("wait:"+key, func(ctx context.Context, _ string) (jsontext.Value, error) {
payload, ok, err := rc.work.Signalled(ctx, rc.run.ID, key)
switch {
case err != nil:
return nil, err
case ok:
return payload, nil
case !time.Now().Before(deadline):
return nil, ErrWaitTimedOut
}
return nil, rc.suspend(&deadline, key)
})
}
// Context is the run's context: it ends when the run's lease is lost or the
// binary shuts down.
func (rc *Context) Context() context.Context { return rc.ctx }
func (rc *Context) advance() int {
seq := rc.next
rc.next++
return seq
}
func (rc *Context) checkCancel(ctx context.Context) error {
cancelled, err := rc.work.CancelRequested(ctx, rc.run.ID)
if err != nil {
return err
}
if cancelled {
return ErrCancelled
}
return nil
}
// record saves a step's outcome. A suspension, a lost lease or a cancelled
// context is not an outcome: nothing is recorded and the error passes through.
func (rc *Context) record(ctx context.Context, seq int, name string, out any, stepErr error) (store.Step, error) {
if errors.Is(stepErr, errSuspended) || errors.Is(stepErr, store.ErrLeaseLost) || errors.Is(stepErr, context.Canceled) {
return store.Step{}, stepErr
}
step := store.Step{RunID: rc.run.ID, Seq: seq, Name: name, Outcome: store.OutcomeDone}
switch {
case errors.Is(stepErr, ErrOutcomeUnknown):
step.Outcome = store.OutcomeUnknown
step.Error = sql.NullString{String: stepErr.Error(), Valid: true}
case stepErr != nil:
step.Outcome = store.OutcomeFailed
step.Error = sql.NullString{String: stepErr.Error(), Valid: true}
default:
raw, err := json.Marshal(out)
if err != nil {
return store.Step{}, fmt.Errorf("encode step %q: %w", name, err)
}
step.Output = raw
}
if err := rc.work.RecordStep(ctx, rc.token, step); err != nil {
return store.Step{}, err
}
rc.saved[seq] = step
return step, nil
}
// reload reads the step another holder recorded at seq and returns its result.
func (rc *Context) reload[T any](seq int, name string) (T, error) {
var zero T
steps, err := rc.work.Steps(rc.ctx, rc.run.ID)
if err != nil {
return zero, err
}
for _, s := range steps {
if s.Seq == seq {
rc.saved[seq] = s
return replay[T](s, name)
}
}
return zero, fmt.Errorf("step %d reported as recorded but not found", seq)
}
func (rc *Context) suspend(wakeAt *time.Time, waitingFor string) error {
if err := rc.work.Suspend(rc.ctx, rc.run.ID, rc.token, wakeAt, waitingFor); err != nil {
return err
}
return errSuspended
}
// replay returns a saved step's result as T.
func replay[T any](saved store.Step, name string) (T, error) {
var zero T
if saved.Name != name {
return zero, fmt.Errorf("%w: step %d is %q, replay asked for %q", ErrNondeterministic, saved.Seq, saved.Name, name)
}
switch saved.Outcome {
case store.OutcomeUnknown:
return zero, fmt.Errorf("%w: %s", ErrOutcomeUnknown, saved.Error.String)
case store.OutcomeFailed:
return zero, savedError(saved.Error.String)
}
var out T
if err := json.Unmarshal(saved.Output, &out); err != nil {
return zero, fmt.Errorf("decode step %q: %w", name, err)
}
return out, nil
}
// savedError rebuilds a recorded step error, keeping the engine's own
// sentinels comparable with errors.Is after a replay.
func savedError(text string) error {
for _, sentinel := range []error{ErrWaitTimedOut, ErrCancelled} {
if text == sentinel.Error() {
return sentinel
}
}
return errors.New(text)
}