Files
archie/modules/runs/runs.go
T

413 lines
12 KiB
Go

// Package runs is Archie's durable run engine. A run executes a stored
// workflow document step by step; each step's result is saved before the next starts,
// so after a crash the run replays from saved results without repeating a
// finished step. Every write is fenced by the worker's lease.
package runs
import (
"context"
"encoding/json/jsontext"
"errors"
"fmt"
"log/slog"
"strings"
"sync"
"time"
"github.com/google/uuid"
"github.com/jmoiron/sqlx"
"github.com/samcharles93/archie-core/modules/governance"
"github.com/samcharles93/archie-core/modules/runs/internal/store"
)
// StartRequest specifies the workflow and organisation for a new run.
// A non-empty Key makes the start idempotent within the organisation.
type StartRequest struct {
OrgID string
Workflow string
Key string
Input jsontext.Value
}
// Options tune the engine. Zero values take the defaults.
type Options struct {
Owner string
Workers int
LeaseTTL time.Duration
PollInterval time.Duration
}
// workerStore is what a worker needs from storage; every write is fenced.
type workerStore interface {
Claim(ctx context.Context, owner string, token uuid.UUID, ttl time.Duration) (store.Run, bool, error)
RenewLease(ctx context.Context, id, token uuid.UUID, ttl time.Duration) (bool, error)
Steps(ctx context.Context, id uuid.UUID) ([]store.Step, error)
RecordStep(ctx context.Context, token uuid.UUID, step store.Step) error
Suspend(ctx context.Context, id, token uuid.UUID, wakeAt *time.Time, waitingFor string) error
Finish(ctx context.Context, id, token uuid.UUID, ended store.Ended) error
Signalled(ctx context.Context, id uuid.UUID, key string) (jsontext.Value, bool, error)
CancelRequested(ctx context.Context, id uuid.UUID) (bool, error)
}
// runWork is everything a running workflow needs from storage.
type runWork interface {
workerStore
runStarter
}
// runStarter is what a step that starts another run needs from storage.
type runStarter interface {
workflowFinder
StartFor(ctx context.Context, token, parent uuid.UUID, n store.NewRun) (uuid.UUID, error)
}
// Engine starts, executes and resumes runs.
type Engine struct {
db *store.Store
work runWork
log *slog.Logger
opts Options
mu sync.RWMutex
deps *Deps
// load turns a claimed run into the code that executes it.
load func(ctx context.Context, run store.Run) (Workflow, error)
}
// New returns an engine on db.
func New(db *sqlx.DB, log *slog.Logger, opts Options) *Engine {
s := store.New(db)
return newEngine(s, s, log, opts)
}
func newEngine(db *store.Store, work runWork, log *slog.Logger, opts Options) *Engine {
if opts.Workers <= 0 {
opts.Workers = 4
}
if opts.LeaseTTL <= 0 {
opts.LeaseTTL = 30 * time.Second
}
if opts.PollInterval <= 0 {
opts.PollInterval = time.Second
}
if opts.Owner == "" {
opts.Owner = "archie"
}
e := &Engine{db: db, work: work, log: log, opts: opts}
e.load = e.loadDocument
return e
}
// Use gives the engine what a document's steps need. It must be called before
// Work, and is separate from New because the modules those steps use are built
// with the engine.
func (e *Engine) Use(d Deps) {
e.mu.Lock()
defer e.mu.Unlock()
e.deps = &d
}
func (e *Engine) steps() (Deps, error) {
e.mu.RLock()
defer e.mu.RUnlock()
if e.deps == nil {
return Deps{}, errNoDeps
}
return *e.deps, nil
}
// Install stores body as the next version of the workflow name, unless it is
// what the latest version already is, and returns the version in force. A body
// that doesn't validate is stored as nothing.
func (e *Engine) Install(ctx context.Context, orgID, name, body, by string) (int, error) {
if _, err := e.validateDocument(ctx, body); err != nil {
return 0, err
}
latest, err := e.db.Workflow(ctx, orgID, name, 0)
switch {
case err == nil && normalise(latest.Body) == normalise(body):
return latest.Version, nil
case err != nil && !errors.Is(err, store.ErrNoWorkflow):
return 0, err
}
w, err := e.db.AddWorkflow(ctx, orgID, name, body, by)
if err != nil {
return 0, err
}
return w.Version, nil
}
func (e *Engine) validateDocument(ctx context.Context, body string) (Document, error) {
doc, err := ParseDocument(body)
if err != nil {
return Document{}, err
}
var problems []Problem
for i, step := range doc.Steps {
if step.Type != typeAgent {
continue
}
var s agentSettings
if err := decodeSettings(step.Settings, &s); err != nil {
return Document{}, err
}
if modelAlias(s.Model) == "" {
continue
}
deps, err := e.steps()
if err != nil {
return Document{}, err
}
if _, err := deps.Settings.AgentModelForAlias(ctx, s.Model); err != nil {
if !errors.Is(err, governance.ErrNoAgentModel) {
return Document{}, err
}
problems = append(problems, Problem{Path: fmt.Sprintf("steps[%d].settings.model", i), Message: fmt.Sprintf("unknown agent model alias %q", s.Model)})
}
}
if len(problems) > 0 {
return Document{}, &InvalidError{Problems: problems}
}
return doc, nil
}
// Has reports whether a workflow is stored under name.
func (e *Engine) Has(ctx context.Context, orgID, name string) (bool, error) {
_, err := e.db.Workflow(ctx, orgID, name, 0)
if errors.Is(err, store.ErrNoWorkflow) {
return false, nil
}
return err == nil, err
}
// Start queues a run under the latest stored version of its workflow and
// returns its id.
func (e *Engine) Start(ctx context.Context, req StartRequest) (uuid.UUID, error) {
n, err := newRun(ctx, e.db, req)
if err != nil {
return uuid.Nil, err
}
return e.db.Start(ctx, n)
}
// workflowFinder reads stored workflow versions.
type workflowFinder interface {
Workflow(ctx context.Context, orgID, name string, version int) (store.Workflow, error)
}
// newRun checks that the latest version of the workflow can start with the
// request's input, and returns the run to queue.
func newRun(ctx context.Context, finder workflowFinder, req StartRequest) (store.NewRun, error) {
w, err := finder.Workflow(ctx, req.OrgID, req.Workflow, 0)
if errors.Is(err, store.ErrNoWorkflow) {
return store.NewRun{}, fmt.Errorf("no workflow named %q", req.Workflow)
}
if err != nil {
return store.NewRun{}, err
}
doc, err := ParseDocument(w.Body)
if err != nil {
return store.NewRun{}, fmt.Errorf("workflow %q version %d: %w", req.Workflow, w.Version, err)
}
if err := doc.unsupported(); err != nil {
return store.NewRun{}, err
}
input := req.Input
if len(input) == 0 {
input = jsontext.Value("null")
}
if err := doc.checkInputs(decodeInputs(input)); err != nil {
return store.NewRun{}, fmt.Errorf("workflow %q: %w", req.Workflow, err)
}
return store.NewRun{
OrgID: req.OrgID, Workflow: req.Workflow, Version: w.Version, StartKey: req.Key, Input: input,
}, nil
}
// CheckTrigger refuses a trigger for a workflow it couldn't start: one that
// doesn't exist, or one that requires an input a trigger doesn't supply. A
// trigger supplies only the event that fired it.
func (e *Engine) CheckTrigger(ctx context.Context, orgID, workflow string) error {
w, err := e.db.Workflow(ctx, orgID, workflow, 0)
refuse := func(format string, args ...any) error {
return &InvalidError{Problems: []Problem{{Path: "workflow", Message: fmt.Sprintf(format, args...)}}}
}
if errors.Is(err, store.ErrNoWorkflow) {
return refuse("no workflow named %q", workflow)
}
if err != nil {
return err
}
doc, err := ParseDocument(w.Body)
if err != nil {
return refuse("workflow %q is not valid: %v", workflow, err)
}
if err := doc.unsupported(); err != nil {
return refuse("workflow %q can't start: %v", workflow, err)
}
if missing := doc.requires("event"); len(missing) > 0 {
return refuse("workflow %q requires %s, and a trigger supplies only event", workflow, strings.Join(missing, ", "))
}
return nil
}
// Signal delivers a payload to a run waiting on key, now or later.
func (e *Engine) Signal(ctx context.Context, id uuid.UUID, key string, payload jsontext.Value) error {
return e.db.Signal(ctx, id, key, payload)
}
// Cancel asks a run to stop.
func (e *Engine) Cancel(ctx context.Context, id uuid.UUID) error {
return e.db.Cancel(ctx, id)
}
// ErrNotRetryable is returned by Retry for a run that is not parked, or that
// never reached the step to retry from.
var ErrNotRetryable = errors.New("run can't be retried from there")
// Retry queues a parked run to run again from a top-level step, named by its id
// or, without one, its type and position as the trail shows it. An empty from
// is the step that parked the run. The steps before it keep their results.
func (e *Engine) Retry(ctx context.Context, orgID string, id uuid.UUID, from string) error {
err := e.db.Retry(ctx, orgID, id, from)
if errors.Is(err, store.ErrNotParked) || errors.Is(err, store.ErrNoStep) {
return fmt.Errorf("%w: %w", ErrNotRetryable, err)
}
return err
}
// Work runs the engine's workers until ctx ends.
func (e *Engine) Work(ctx context.Context) {
var wg sync.WaitGroup
for range e.opts.Workers {
wg.Go(func() { e.workerLoop(ctx) })
}
wg.Wait()
}
func (e *Engine) workerLoop(ctx context.Context) {
for ctx.Err() == nil {
claimed, err := e.claimAndExecute(ctx)
if err != nil {
e.log.ErrorContext(ctx, "run worker", "error", err)
}
if claimed && err == nil {
continue
}
select {
case <-ctx.Done():
case <-time.After(e.opts.PollInterval):
}
}
}
func (e *Engine) claimAndExecute(ctx context.Context) (bool, error) {
token := uuid.New()
run, ok, err := e.work.Claim(ctx, e.opts.Owner, token, e.opts.LeaseTTL)
if err != nil || !ok {
return false, err
}
e.execute(ctx, run, token)
return true, nil
}
// execute runs one claimed run to its next suspension or end. The run's
// context ends if the lease can't be renewed, so a stale worker stops.
func (e *Engine) execute(ctx context.Context, run store.Run, token uuid.UUID) {
runCtx, cancel := context.WithCancel(ctx)
defer cancel()
go e.keepLease(runCtx, cancel, run.ID, token)
log := e.log.With("run", run.ID, "workflow", run.Workflow, "attempt", run.Attempts)
result, ending, err := e.invoke(runCtx, run, token)
switch {
case errors.Is(err, errSuspended):
log.InfoContext(ctx, "run suspended")
return
case errors.Is(err, store.ErrLeaseLost), errors.Is(err, context.Canceled):
log.WarnContext(ctx, "run stopped: lease lost or shutting down")
return
}
ended := store.Ended{State: store.StateSucceeded, Result: result}
switch {
case errors.Is(err, ErrCancelled):
ended.State, ended.Error = store.StateCancelled, err.Error()
case err != nil:
ended.State, ended.Error = store.StateFailed, err.Error()
case ending != nil:
ended.State, ended.Detail = ending.State, ending.Detail
}
if ferr := e.work.Finish(ctx, run.ID, token, ended); ferr != nil {
log.WarnContext(ctx, "finish run", "error", ferr)
return
}
log.InfoContext(ctx, "run finished", "state", ended.State)
}
// invoke runs the workflow, turning a panic in workflow or step code into a
// failure of this run rather than of the process.
func (e *Engine) invoke(ctx context.Context, run store.Run, token uuid.UUID) (result jsontext.Value, ending *Ending, err error) {
defer func() {
if p := recover(); p != nil {
err = fmt.Errorf("panic: %v", p)
}
}()
wf, err := e.load(ctx, run)
if err != nil {
return nil, nil, err
}
steps, err := e.work.Steps(ctx, run.ID)
if err != nil {
return nil, nil, err
}
rc := newContext(ctx, e.work, run, token, steps)
result, err = wf(rc, run.Input)
return result, rc.ending, err
}
// loadDocument reads the document a run executes: the version it started
// under, or for a dry run the document it carries.
func (e *Engine) loadDocument(ctx context.Context, run store.Run) (Workflow, error) {
deps, err := e.steps()
if err != nil {
return nil, err
}
var body string
if run.Dry {
o, err := readOverlay(run.Overlay)
if err != nil {
return nil, err
}
body = o.Document
} else {
w, err := e.db.Workflow(ctx, run.OrgID, run.Workflow, run.Version)
if err != nil {
return nil, fmt.Errorf("workflow %q version %d: %w", run.Workflow, run.Version, err)
}
body = w.Body
}
doc, err := ParseDocument(body)
if err != nil {
return nil, err
}
return deps.Compile(doc), nil //nolint:contextcheck // a workflow takes its context from the run handle, as a handler takes it from the request
}
func (e *Engine) keepLease(ctx context.Context, lost context.CancelFunc, id, token uuid.UUID) {
tick := time.NewTicker(e.opts.LeaseTTL / 3)
defer tick.Stop()
for {
select {
case <-ctx.Done():
return
case <-tick.C:
held, err := e.work.RenewLease(ctx, id, token, e.opts.LeaseTTL)
if err != nil || !held {
e.log.WarnContext(ctx, "lease not renewed", "run", id, "error", err)
lost()
return
}
}
}
}