Also fix repository-free message.send validation (#2046) and add a single-binary development smoke path (#1977). Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com>
363 lines
11 KiB
Go
363 lines
11 KiB
Go
// Package runs is Archie's durable run engine. A run executes a stored
|
|
// workflow document step by step; each step's result is saved before the next starts,
|
|
// so after a crash the run replays from saved results without repeating a
|
|
// finished step. Every write is fenced by the worker's lease.
|
|
package runs
|
|
|
|
import (
|
|
"context"
|
|
"encoding/json/jsontext"
|
|
"errors"
|
|
"fmt"
|
|
"log/slog"
|
|
"strings"
|
|
"sync"
|
|
"time"
|
|
|
|
"github.com/google/uuid"
|
|
"github.com/jmoiron/sqlx"
|
|
|
|
"github.com/samcharles93/archie-core/modules/governance"
|
|
"github.com/samcharles93/archie-core/modules/runs/internal/store"
|
|
)
|
|
|
|
// StartRequest specifies the workflow and organisation for a new run.
|
|
// A non-empty Key makes the start idempotent within the organisation.
|
|
type StartRequest struct {
|
|
OrgID string
|
|
Workflow string
|
|
Key string
|
|
Input jsontext.Value
|
|
}
|
|
|
|
// Options tune the engine. Zero values take the defaults.
|
|
type Options struct {
|
|
Owner string
|
|
Workers int
|
|
LeaseTTL time.Duration
|
|
PollInterval time.Duration
|
|
}
|
|
|
|
// workerStore is what a worker needs from storage; every write is fenced.
|
|
type workerStore interface {
|
|
Claim(ctx context.Context, owner string, token uuid.UUID, ttl time.Duration) (store.Run, bool, error)
|
|
RenewLease(ctx context.Context, id, token uuid.UUID, ttl time.Duration) (bool, error)
|
|
Steps(ctx context.Context, id uuid.UUID) ([]store.Step, error)
|
|
RecordStep(ctx context.Context, token uuid.UUID, step store.Step) error
|
|
Suspend(ctx context.Context, id, token uuid.UUID, wakeAt *time.Time, waitingFor string) error
|
|
Finish(ctx context.Context, id, token uuid.UUID, state string, result jsontext.Value, errText string) error
|
|
Signalled(ctx context.Context, id uuid.UUID, key string) (jsontext.Value, bool, error)
|
|
CancelRequested(ctx context.Context, id uuid.UUID) (bool, error)
|
|
}
|
|
|
|
// Engine starts, executes and resumes runs.
|
|
type Engine struct {
|
|
db *store.Store
|
|
work workerStore
|
|
log *slog.Logger
|
|
opts Options
|
|
mu sync.RWMutex
|
|
deps *Deps
|
|
// load turns a claimed run into the code that executes it.
|
|
load func(ctx context.Context, run store.Run) (Workflow, error)
|
|
}
|
|
|
|
// New returns an engine on db.
|
|
func New(db *sqlx.DB, log *slog.Logger, opts Options) *Engine {
|
|
s := store.New(db)
|
|
return newEngine(s, s, log, opts)
|
|
}
|
|
|
|
func newEngine(db *store.Store, work workerStore, log *slog.Logger, opts Options) *Engine {
|
|
if opts.Workers <= 0 {
|
|
opts.Workers = 4
|
|
}
|
|
if opts.LeaseTTL <= 0 {
|
|
opts.LeaseTTL = 30 * time.Second
|
|
}
|
|
if opts.PollInterval <= 0 {
|
|
opts.PollInterval = time.Second
|
|
}
|
|
if opts.Owner == "" {
|
|
opts.Owner = "archie"
|
|
}
|
|
e := &Engine{db: db, work: work, log: log, opts: opts}
|
|
e.load = e.loadDocument
|
|
return e
|
|
}
|
|
|
|
// Use gives the engine what a document's steps need. It must be called before
|
|
// Work, and is separate from New because the modules those steps use are built
|
|
// with the engine.
|
|
func (e *Engine) Use(d Deps) {
|
|
e.mu.Lock()
|
|
defer e.mu.Unlock()
|
|
e.deps = &d
|
|
}
|
|
|
|
func (e *Engine) steps() (Deps, error) {
|
|
e.mu.RLock()
|
|
defer e.mu.RUnlock()
|
|
if e.deps == nil {
|
|
return Deps{}, errNoDeps
|
|
}
|
|
return *e.deps, nil
|
|
}
|
|
|
|
// Install stores body as the next version of the workflow name, unless it is
|
|
// what the latest version already is, and returns the version in force. A body
|
|
// that doesn't validate is stored as nothing.
|
|
func (e *Engine) Install(ctx context.Context, orgID, name, body, by string) (int, error) {
|
|
if err := e.validateDocument(ctx, body); err != nil {
|
|
return 0, err
|
|
}
|
|
latest, err := e.db.Workflow(ctx, orgID, name, 0)
|
|
switch {
|
|
case err == nil && normalise(latest.Body) == normalise(body):
|
|
return latest.Version, nil
|
|
case err != nil && !errors.Is(err, store.ErrNoWorkflow):
|
|
return 0, err
|
|
}
|
|
w, err := e.db.AddWorkflow(ctx, orgID, name, body, by)
|
|
if err != nil {
|
|
return 0, err
|
|
}
|
|
return w.Version, nil
|
|
}
|
|
|
|
func (e *Engine) validateDocument(ctx context.Context, body string) error {
|
|
doc, err := ParseDocument(body)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
var problems []Problem
|
|
for i, step := range doc.Steps {
|
|
if step.Type != typeAgent {
|
|
continue
|
|
}
|
|
var s agentSettings
|
|
if err := decodeSettings(step.Settings, &s); err != nil {
|
|
return err
|
|
}
|
|
if modelAlias(s.Model) == "" {
|
|
continue
|
|
}
|
|
deps, err := e.steps()
|
|
if err != nil {
|
|
return err
|
|
}
|
|
if _, err := deps.Settings.AgentModelForAlias(ctx, s.Model); err != nil {
|
|
if !errors.Is(err, governance.ErrNoAgentModel) {
|
|
return err
|
|
}
|
|
problems = append(problems, Problem{Path: fmt.Sprintf("steps[%d].settings.model", i), Message: fmt.Sprintf("unknown agent model alias %q", s.Model)})
|
|
}
|
|
}
|
|
if len(problems) > 0 {
|
|
return &InvalidError{Problems: problems}
|
|
}
|
|
return nil
|
|
}
|
|
|
|
// Has reports whether a workflow is stored under name.
|
|
func (e *Engine) Has(ctx context.Context, orgID, name string) (bool, error) {
|
|
_, err := e.db.Workflow(ctx, orgID, name, 0)
|
|
if errors.Is(err, store.ErrNoWorkflow) {
|
|
return false, nil
|
|
}
|
|
return err == nil, err
|
|
}
|
|
|
|
// Start queues a run under the latest stored version of its workflow and
|
|
// returns its id.
|
|
func (e *Engine) Start(ctx context.Context, req StartRequest) (uuid.UUID, error) {
|
|
w, err := e.db.Workflow(ctx, req.OrgID, req.Workflow, 0)
|
|
if errors.Is(err, store.ErrNoWorkflow) {
|
|
return uuid.Nil, fmt.Errorf("no workflow named %q", req.Workflow)
|
|
}
|
|
if err != nil {
|
|
return uuid.Nil, err
|
|
}
|
|
doc, err := ParseDocument(w.Body)
|
|
if err != nil {
|
|
return uuid.Nil, fmt.Errorf("workflow %q version %d: %w", req.Workflow, w.Version, err)
|
|
}
|
|
if err := doc.unsupported(); err != nil {
|
|
return uuid.Nil, err
|
|
}
|
|
input := req.Input
|
|
if len(input) == 0 {
|
|
input = jsontext.Value("null")
|
|
}
|
|
if err := doc.checkInputs(decodeInputs(input)); err != nil {
|
|
return uuid.Nil, fmt.Errorf("workflow %q: %w", req.Workflow, err)
|
|
}
|
|
return e.db.Start(ctx, store.NewRun{
|
|
OrgID: req.OrgID, Workflow: req.Workflow, Version: w.Version, StartKey: req.Key, Input: input,
|
|
})
|
|
}
|
|
|
|
// CheckTrigger refuses a trigger for a workflow it couldn't start: one that
|
|
// doesn't exist, or one that requires an input a trigger doesn't supply. A
|
|
// trigger supplies only the event that fired it.
|
|
func (e *Engine) CheckTrigger(ctx context.Context, orgID, workflow string) error {
|
|
w, err := e.db.Workflow(ctx, orgID, workflow, 0)
|
|
refuse := func(format string, args ...any) error {
|
|
return &InvalidError{Problems: []Problem{{Path: "workflow", Message: fmt.Sprintf(format, args...)}}}
|
|
}
|
|
if errors.Is(err, store.ErrNoWorkflow) {
|
|
return refuse("no workflow named %q", workflow)
|
|
}
|
|
if err != nil {
|
|
return err
|
|
}
|
|
doc, err := ParseDocument(w.Body)
|
|
if err != nil {
|
|
return refuse("workflow %q is not valid: %v", workflow, err)
|
|
}
|
|
if missing := doc.requires("event"); len(missing) > 0 {
|
|
return refuse("workflow %q requires %s, and a trigger supplies only event", workflow, strings.Join(missing, ", "))
|
|
}
|
|
return nil
|
|
}
|
|
|
|
// Signal delivers a payload to a run waiting on key, now or later.
|
|
func (e *Engine) Signal(ctx context.Context, id uuid.UUID, key string, payload jsontext.Value) error {
|
|
return e.db.Signal(ctx, id, key, payload)
|
|
}
|
|
|
|
// Cancel asks a run to stop.
|
|
func (e *Engine) Cancel(ctx context.Context, id uuid.UUID) error {
|
|
return e.db.Cancel(ctx, id)
|
|
}
|
|
|
|
// Work runs the engine's workers until ctx ends.
|
|
func (e *Engine) Work(ctx context.Context) {
|
|
var wg sync.WaitGroup
|
|
for range e.opts.Workers {
|
|
wg.Go(func() { e.workerLoop(ctx) })
|
|
}
|
|
wg.Wait()
|
|
}
|
|
|
|
func (e *Engine) workerLoop(ctx context.Context) {
|
|
for ctx.Err() == nil {
|
|
claimed, err := e.claimAndExecute(ctx)
|
|
if err != nil {
|
|
e.log.ErrorContext(ctx, "run worker", "error", err)
|
|
}
|
|
if claimed && err == nil {
|
|
continue
|
|
}
|
|
select {
|
|
case <-ctx.Done():
|
|
case <-time.After(e.opts.PollInterval):
|
|
}
|
|
}
|
|
}
|
|
|
|
func (e *Engine) claimAndExecute(ctx context.Context) (bool, error) {
|
|
token := uuid.New()
|
|
run, ok, err := e.work.Claim(ctx, e.opts.Owner, token, e.opts.LeaseTTL)
|
|
if err != nil || !ok {
|
|
return false, err
|
|
}
|
|
e.execute(ctx, run, token)
|
|
return true, nil
|
|
}
|
|
|
|
// execute runs one claimed run to its next suspension or end. The run's
|
|
// context ends if the lease can't be renewed, so a stale worker stops.
|
|
func (e *Engine) execute(ctx context.Context, run store.Run, token uuid.UUID) {
|
|
runCtx, cancel := context.WithCancel(ctx)
|
|
defer cancel()
|
|
go e.keepLease(runCtx, cancel, run.ID, token)
|
|
|
|
log := e.log.With("run", run.ID, "workflow", run.Workflow, "attempt", run.Attempts)
|
|
result, err := e.invoke(runCtx, run, token)
|
|
switch {
|
|
case errors.Is(err, errSuspended):
|
|
log.InfoContext(ctx, "run suspended")
|
|
return
|
|
case errors.Is(err, store.ErrLeaseLost), errors.Is(err, context.Canceled):
|
|
log.WarnContext(ctx, "run stopped: lease lost or shutting down")
|
|
return
|
|
}
|
|
state, errText := store.StateSucceeded, ""
|
|
if errors.Is(err, ErrCancelled) {
|
|
state, errText = store.StateCancelled, err.Error()
|
|
} else if err != nil {
|
|
state, errText = store.StateFailed, err.Error()
|
|
}
|
|
if ferr := e.work.Finish(ctx, run.ID, token, state, result, errText); ferr != nil {
|
|
log.WarnContext(ctx, "finish run", "error", ferr)
|
|
return
|
|
}
|
|
log.InfoContext(ctx, "run finished", "state", state)
|
|
}
|
|
|
|
// invoke runs the workflow, turning a panic in workflow or step code into a
|
|
// failure of this run rather than of the process.
|
|
func (e *Engine) invoke(ctx context.Context, run store.Run, token uuid.UUID) (result jsontext.Value, err error) {
|
|
defer func() {
|
|
if p := recover(); p != nil {
|
|
err = fmt.Errorf("panic: %v", p)
|
|
}
|
|
}()
|
|
wf, err := e.load(ctx, run)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
steps, err := e.work.Steps(ctx, run.ID)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
return wf(newContext(ctx, e.work, run, token, steps), run.Input)
|
|
}
|
|
|
|
// loadDocument reads the document a run executes: the version it started
|
|
// under, or for a dry run the document it carries.
|
|
func (e *Engine) loadDocument(ctx context.Context, run store.Run) (Workflow, error) {
|
|
deps, err := e.steps()
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
var body string
|
|
if run.Dry {
|
|
o, err := readOverlay(run.Overlay)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
body = o.Document
|
|
} else {
|
|
w, err := e.db.Workflow(ctx, run.OrgID, run.Workflow, run.Version)
|
|
if err != nil {
|
|
return nil, fmt.Errorf("workflow %q version %d: %w", run.Workflow, run.Version, err)
|
|
}
|
|
body = w.Body
|
|
}
|
|
doc, err := ParseDocument(body)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
return deps.Compile(doc), nil //nolint:contextcheck // a workflow takes its context from the run handle, as a handler takes it from the request
|
|
}
|
|
|
|
func (e *Engine) keepLease(ctx context.Context, lost context.CancelFunc, id, token uuid.UUID) {
|
|
tick := time.NewTicker(e.opts.LeaseTTL / 3)
|
|
defer tick.Stop()
|
|
for {
|
|
select {
|
|
case <-ctx.Done():
|
|
return
|
|
case <-tick.C:
|
|
held, err := e.work.RenewLease(ctx, id, token, e.opts.LeaseTTL)
|
|
if err != nil || !held {
|
|
e.log.WarnContext(ctx, "lease not renewed", "run", id, "error", err)
|
|
lost()
|
|
return
|
|
}
|
|
}
|
|
}
|
|
}
|