feat(runs): workflow.call, workflow.handoff, workflow.approve and message.send (#2044)

This commit is contained in:
sam committed 2026-11-21 14:59:49 +11:00
1 parent 7b6154e7e8
commit 5d8c9303e0
14 files changed
+852 -36

No files matched your search

+1 -1
View File
@@ -59,7 +59,7 @@ func (o *operations) Start(ctx context.Context) error {
// from the request.
o.engine.Use(runs.Deps{
Settings: o.settings, Toolbox: o.conns, Judge: o.judge, Approver: o.allow,
Notifier: o.tell, Memory: o.remember,
Notifier: o.tell, Messenger: o.tell, Memory: o.remember,
})
if err := installStarter(ctx, o.engine); err != nil {
return err
@@ -0,0 +1,14 @@
-- +goose Up
-- A workflow.call step starts a child run. parent_run_id and parent_signal say
-- which run, and which signal key, to wake when the child ends; call_depth
-- bounds how deep calls nest. A run started by a handoff has no parent.
ALTER TABLE runs.runs ADD COLUMN parent_run_id uuid REFERENCES runs.runs (id) ON DELETE CASCADE;
ALTER TABLE runs.runs ADD COLUMN parent_signal text;
ALTER TABLE runs.runs ADD COLUMN call_depth integer NOT NULL DEFAULT 0;
CREATE INDEX runs_runs_parent ON runs.runs (parent_run_id) WHERE parent_run_id IS NOT NULL;
-- +goose Down
DROP INDEX runs.runs_runs_parent;
ALTER TABLE runs.runs DROP COLUMN call_depth;
ALTER TABLE runs.runs DROP COLUMN parent_signal;
ALTER TABLE runs.runs DROP COLUMN parent_run_id;
+22 -1
View File
@@ -37,7 +37,10 @@ const (
heartbeat = 25 * time.Second
// deliveryMessage is the delivery a connector declares to carry messages.
deliveryMessage = "message"
maxBackoff = 10 * time.Minute
// deliveryChat is the delivery a connector declares to send text to one
// chat it can address.
deliveryChat = "chat_message"
maxBackoff = 10 * time.Minute
// maxAttempts ends a delivery that keeps failing: with the backoff capped
// at maxBackoff it is about a day of trying.
maxAttempts = 150
@@ -137,6 +140,24 @@ func (m *Module) Notify(ctx context.Context, n Notification) error {
})
}
// Send delivers text to one chat on the named channel, and returns when the
// channel has accepted it. The key makes the delivery idempotent on the
// channel, so a step that is replayed after a crash does not send twice.
func (m *Module) Send(ctx context.Context, channel, chatID, text, key string) error {
c, ok := m.channels.Get(channel)
if !ok {
return fmt.Errorf("connector %q isn't running", channel)
}
raw, err := json.Marshal(map[string]string{"chat_id": chatID, "text": text})
if err != nil {
return err
}
if _, err := c.Deliver(ctx, deliveryChat, raw, key); err != nil {
return fmt.Errorf("send to %s chat %s: %w", channel, chatID, err)
}
return nil
}
// Ask delivers an approval request on every channel. Asking again for the same
// key changes nothing.
func (m *Module) Ask(ctx context.Context, a ApprovalRequest) error {
+7 -2
View File
@@ -36,7 +36,7 @@ type Context struct {
// The run owns its context, as an http.Request does: every step takes its
// cancellation from it, and a second context passed beside it could disagree.
ctx context.Context //nolint:containedctx // see above
work workerStore
work runWork
run store.Run
token uuid.UUID
saved map[int]store.Step
@@ -52,6 +52,10 @@ type Context struct {
ending *Ending
// overlay is what a dry run carries instead of a stored version.
overlay *dryOverlay
// resolve checks a workflow to start, and returns the run to queue. It is a
// field because the step types, which are initialised before it, can't
// name the document checks that depend on them.
resolve func(ctx context.Context, req StartRequest) (store.NewRun, error)
}
// markerBase is where started markers are numbered, far above any step a
@@ -65,7 +69,7 @@ const (
enteredPrefix = "at:"
)
func newContext(ctx context.Context, work workerStore, run store.Run, token uuid.UUID, steps []store.Step) *Context {
func newContext(ctx context.Context, work runWork, run store.Run, token uuid.UUID, steps []store.Step) *Context {
saved := make(map[int]store.Step, len(steps))
started, entered := map[string]bool{}, map[string]bool{}
nextMarker := markerBase
@@ -81,6 +85,7 @@ func newContext(ctx context.Context, work workerStore, run store.Run, token uuid
}
}
rc := &Context{ctx: ctx, work: work, run: run, token: token, saved: saved, started: started, entered: entered, nextMarker: nextMarker}
rc.resolve = func(ctx context.Context, req StartRequest) (store.NewRun, error) { return newRun(ctx, work, req) }
if run.Dry {
rc.overlay, _ = readOverlay(run.Overlay)
}
+21 -4
View File
@@ -38,8 +38,8 @@ func TestCorpusChecksAsItsStepTypesLand(t *testing.T) {
}{
{name: "investigate checks", file: "investigate"},
{
name: "feasibility waits for repositories, switch and approve", file: "feasibility",
want: slices.Concat(notYet("repo.prepare", "workflow.approve"),
name: "feasibility waits for repositories and switch", file: "feasibility",
want: slices.Concat(notYet("repo.prepare"),
[]string{"switch steps are not supported yet"}),
},
{
@@ -58,8 +58,8 @@ func TestCorpusChecksAsItsStepTypesLand(t *testing.T) {
agentLater("gate", "gate", "protect_tests")),
},
{
name: "triage's references and conditions check; it waits for forge and handoff steps", file: "triage",
want: notYet("forge.comment", "forge.close-issue", "workflow.handoff"),
name: "triage's references and conditions check; it waits for forge steps", file: "triage",
want: notYet("forge.comment", "forge.close-issue"),
},
{
name: "a finish step with an unknown status is refused when the workflow is saved",
@@ -68,6 +68,23 @@ func TestCorpusChecksAsItsStepTypesLand(t *testing.T) {
`,
want: []string{`status is completed, parked, waiting_approval or declined, not "finished"`},
},
{
name: "a call, handoff, approval and message check their settings when the workflow is saved",
yaml: `steps:
- {type: workflow.call, settings: {workflow: w, inputs: {a: {b: 1}}}}
- {type: workflow.call, settings: {workflow: w, outputs: {x: outputs.x}}}
- {type: workflow.call, settings: {workflow: w, inputs: {a: inputs.9}}}
- {type: workflow.handoff, settings: {detail: d}}
- {type: workflow.approve, settings: {plan: p}}
- {type: message.send, settings: {channel: c, chat_id: " ", text: t}}
`,
want: []string{
"not a nested document", "outputs on a call that does not wait can never be read",
`input "a" reference "inputs.9" must name an input as inputs.<name>`,
"workflow is required", "plan or document, and then, are required",
"channel, chat_id and text are required",
},
},
{
name: "a result field the agent's schema doesn't declare is refused",
yaml: `steps:
+268
View File
@@ -0,0 +1,268 @@
package runs
import (
"context"
"encoding/json/jsontext"
json "encoding/json/v2"
"errors"
"fmt"
"regexp"
"strings"
"time"
"github.com/google/uuid"
"github.com/samcharles93/archie-core/modules/deliveries"
"github.com/samcharles93/archie-core/modules/runs/internal/store"
)
// maxCallDepth bounds how deep calls nest. A workflow that calls itself is
// not refused when it is saved, so this is also what stops it.
const maxCallDepth = 5
// maxCallWait is how long a waiting call holds out for its child before the
// step fails.
const maxCallWait = 7 * 24 * time.Hour
// callReference prefixes a call's input value that names an input of the
// calling workflow.
const callReference = "inputs."
var refName = regexp.MustCompile(`^[A-Za-z_][A-Za-z0-9_]*$`)
// callSettings start another workflow as a child run.
type callSettings struct {
Workflow string `yaml:"workflow" json:"workflow" jsonschema:"the workflow to start as a child run"`
Inputs map[string]any `yaml:"inputs,omitempty" json:"inputs,omitempty" jsonschema:"callee input names mapped to scalar literals or inputs.<name> references to caller inputs"`
Wait bool `yaml:"wait,omitempty" json:"wait,omitempty" jsonschema:"wait for the child run to finish before continuing"`
Outputs map[string]string `yaml:"outputs,omitempty" json:"outputs,omitempty" jsonschema:"callee output names mapped to outputs.<name> references to caller outputs; requires wait: true"`
}
func (s callSettings) valid() error {
if strings.TrimSpace(s.Workflow) == "" {
return errors.New("workflow is required")
}
for name, value := range s.Inputs {
if _, ok := value.(map[string]any); ok {
return fmt.Errorf("input %q must be a literal or an inputs.<name> reference, not a nested document", name)
}
if ref, ok := value.(string); ok && strings.HasPrefix(ref, callReference) && !isInputReference(ref) {
return fmt.Errorf("input %q reference %q must name an input as inputs.<name>", name, ref)
}
}
switch {
case len(s.Outputs) > 0 && !s.Wait:
return errors.New("outputs on a call that does not wait can never be read: nothing has finished when the caller moves on")
case len(s.Outputs) > 0:
return errors.New("outputs are not supported yet: a workflow can't declare outputs")
}
return nil
}
func (s callSettings) describe() string {
line := fmt.Sprintf("Starts %s as a child run", s.Workflow)
if s.Wait {
line += " and waits for it to finish"
}
return line
}
// isInputReference reports whether a string names an input of the calling
// workflow. A literal that begins with "inputs." but names no input is a
// literal.
func isInputReference(value string) bool {
name, ok := strings.CutPrefix(value, callReference)
return ok && refName.MatchString(name)
}
// callInputs turns the call's settings into the child's input: a reference
// reads the caller's own input, a literal passes through.
func callInputs(call, callerInputs map[string]any) (map[string]any, error) {
resolved := make(map[string]any, len(call))
for name, value := range call {
if ref, ok := value.(string); ok && isInputReference(ref) {
got, present := callerInputs[strings.TrimPrefix(ref, callReference)]
if !present {
return nil, fmt.Errorf("input %q references %q, which the calling run does not carry", name, ref)
}
resolved[name] = got
continue
}
resolved[name] = value
}
return resolved, nil
}
// runCall starts the child, and with wait true suspends the run until the
// child ends. The child is keyed by this run and step, so a replay or a
// restart finds the child it already started. The child's ending signals this
// run, which holds no worker while it waits.
func runCall(_ Deps, rc *Context, p *progress, name string, s callSettings) (StepResult, error) {
if rc.Dry() {
_, err := rc.Step("call-skipped", func(context.Context, string) (string, error) {
return s.Workflow, nil
})
return StepResult{Summary: "would start " + s.Workflow}, err
}
if rc.run.CallDepth+1 > maxCallDepth {
return StepResult{}, fmt.Errorf("call depth %d would pass the limit of %d", rc.run.CallDepth+1, maxCallDepth)
}
inputs, err := callInputs(s.Inputs, p.inputs)
if err != nil {
return StepResult{}, err
}
signal := callSignal(name)
child, err := rc.Step("call-start:"+name, func(ctx context.Context, _ string) (uuid.UUID, error) {
return rc.startRun(ctx, "call:"+rc.run.ID.String()+":"+name, s.Workflow, inputs, signal, rc.run.CallDepth+1)
})
if err != nil {
return StepResult{}, fmt.Errorf("start %q: %w", s.Workflow, err)
}
if !s.Wait {
return StepResult{Summary: fmt.Sprintf("started %q (run %s); no wait", s.Workflow, child)}, nil
}
payload, err := rc.Wait(signal, maxCallWait)
if err != nil {
return StepResult{}, err
}
return calleeResult(s.Workflow, child, payload)
}
// callSignal is the key a call's child signals when it ends.
func callSignal(name string) string { return "call:" + name }
// calleeEnding is what a child run tells its caller.
type calleeEnding struct {
State string `json:"state"`
Detail string `json:"detail"`
Error string `json:"error"`
Result jsontext.Value `json:"result"`
}
// calleeResult turns a child's ending into the call step's result. A child
// that did not succeed fails the step, with its own detail as the reason: a
// parked child will not move without an operator, so the caller does not wait
// for one.
func calleeResult(workflow string, child uuid.UUID, payload jsontext.Value) (StepResult, error) {
var end calleeEnding
if err := json.Unmarshal(payload, &end); err != nil {
return StepResult{}, fmt.Errorf("decode the ending of %q: %w", workflow, err)
}
if end.State != store.StateSucceeded {
return StepResult{}, fmt.Errorf("callee %q (run %s) ended %s: %s", workflow, child, end.State, firstText(end.Error, end.Detail))
}
var fields map[string]any
// A result that is not an object has no fields for a later step to read.
_ = json.Unmarshal(end.Result, &fields)
return StepResult{Summary: fmt.Sprintf("%q (run %s) finished", workflow, child), Result: fields}, nil
}
// startRun queues a run of another workflow for this run, idempotent on key.
func (rc *Context) startRun(ctx context.Context, key, workflow string, input map[string]any, signal string, depth int) (uuid.UUID, error) {
raw, err := json.Marshal(input)
if err != nil {
return uuid.Nil, fmt.Errorf("encode input: %w", err)
}
n, err := rc.resolve(ctx, StartRequest{OrgID: rc.run.OrgID, Workflow: workflow, Input: raw})
if err != nil {
return uuid.Nil, err
}
n.StartKey, n.ParentSignal, n.CallDepth = key, signal, depth
return rc.work.StartFor(ctx, rc.token, rc.run.ID, n)
}
// handoffSettings end the run and start another workflow with the same input.
type handoffSettings struct {
Workflow string `yaml:"workflow" json:"workflow" jsonschema:"the workflow the task is requeued under"`
Detail string `yaml:"detail,omitempty" json:"detail,omitempty" jsonschema:"why the task was handed off"`
}
func (s handoffSettings) valid() error {
if strings.TrimSpace(s.Workflow) == "" {
return errors.New("workflow is required")
}
return nil
}
func (s handoffSettings) describe() string {
return fmt.Sprintf("Ends the run and starts %s with the same input", s.Workflow)
}
// runHandoff completes the run and starts a run of the target workflow on the
// same input. A dry run records the handoff without starting anything.
func runHandoff(_ Deps, rc *Context, p *progress, name string, s handoffSettings) (StepResult, error) {
target := strings.TrimSpace(s.Workflow)
detail := s.Detail
if detail == "" {
detail = "handed off to " + target
}
if rc.Dry() {
if _, err := rc.Step("handoff-skipped", func(context.Context, string) (string, error) { return target, nil }); err != nil {
return StepResult{}, err
}
} else if _, err := rc.Step("handoff:"+name, func(ctx context.Context, _ string) (uuid.UUID, error) {
return rc.startRun(ctx, "handoff:"+rc.run.ID.String()+":"+name, target, p.inputs, "", rc.run.CallDepth)
}); err != nil {
return StepResult{}, fmt.Errorf("hand off to %q: %w", target, err)
}
rc.End(store.StateSucceeded, detail)
return StepResult{Summary: detail}, nil
}
// approveSettings put a plan to an operator and end the run until they decide.
type approveSettings struct {
Plan string `yaml:"plan,omitempty" json:"plan,omitempty" jsonschema:"what the operator decides on; the next workflow reads it as task.plan"`
Document any `yaml:"document,omitempty" json:"document,omitempty" jsonschema:"the plan as structured fields (title, problem, solution, files, acceptance, non_goals, diff_estimate, risks), such as {{ steps.prd.result }}; the plan text is rendered from it"`
Then string `yaml:"then" json:"then" jsonschema:"the workflow the task runs once approved"`
}
func (s approveSettings) valid() error {
if (strings.TrimSpace(s.Plan) == "" && s.Document == nil) || strings.TrimSpace(s.Then) == "" {
return errors.New("plan or document, and then, are required")
}
return nil
}
func (s approveSettings) describe() string {
return fmt.Sprintf("Delivers the plan and waits for approval; %s runs once it is approved", s.Then)
}
// handover is what an approval step records for whoever decides: the plan, in
// text and as the document it was rendered from, and the workflow to run once
// it is approved.
type handover struct {
Plan string `json:"plan"`
Document *planDocument `json:"document,omitzero"`
Then string `json:"then"`
}
// runApprove delivers the plan and ends the run waiting for approval. A dry
// run records the plan without delivering it.
func runApprove(d Deps, rc *Context, _ *progress, name string, s approveSettings) (StepResult, error) {
doc, err := readPlanDocument(s.Document)
if err != nil {
return StepResult{}, err
}
plan := s.Plan
if doc != nil {
plan = doc.markdown()
}
if strings.TrimSpace(plan) == "" {
return StepResult{}, errors.New("the plan is empty")
}
_, err = rc.Step("approve:"+name, func(ctx context.Context, key string) (handover, error) {
out := handover{Plan: plan, Document: doc, Then: strings.TrimSpace(s.Then)}
if rc.Dry() {
return out, nil
}
return out, d.Notifier.Notify(ctx, deliveries.Notification{
RunID: rc.RunID(), Key: key, Title: "Plan awaiting approval", Body: plan,
})
})
if err != nil {
return StepResult{}, err
}
const detail = "plan delivered, awaiting go/no-go"
rc.End(store.StateWaitingApproval, detail)
return StepResult{Summary: detail}, nil
}
+113 -6
View File
@@ -70,6 +70,9 @@ type Run struct {
Dry bool `db:"dry"`
// Scan into []byte because live runs have a SQL NULL overlay.
Overlay []byte `db:"overlay"`
// CallDepth is how many workflow.call steps lie between this run and the
// run that started the chain.
CallDepth int `db:"call_depth"`
}
// Step is one recorded step of a run.
@@ -101,6 +104,11 @@ type NewRun struct {
Input jsontext.Value
Dry bool
Overlay jsontext.Value
// CallDepth is the depth of a run started by a call.
CallDepth int
// ParentSignal is the signal key to send the run that called this one
// when this run ends; empty for a run nothing waits on.
ParentSignal string
}
// Start inserts a queued run. A run with the same organisation and start key
@@ -122,6 +130,37 @@ func (s *Store) Start(ctx context.Context, n NewRun) (uuid.UUID, error) {
return id, nil
}
// StartFor inserts a queued run on behalf of a run whose lease the token
// holds. The start key makes it idempotent, so a replayed step or a restarted
// worker gets the run already started. A non-empty ParentSignal links the new
// run to the parent: its ending signals the parent under that key.
func (s *Store) StartFor(ctx context.Context, token, parent uuid.UUID, n NewRun) (uuid.UUID, error) {
if n.StartKey == "" {
return uuid.Nil, errors.New("start run: a run started for another needs a start key")
}
var link uuid.NullUUID
if n.ParentSignal != "" {
link = uuid.NullUUID{UUID: parent, Valid: true}
}
var id uuid.UUID
err := s.db.GetContext(ctx, &id, `
INSERT INTO runs.runs (id, org_id, workflow, workflow_version, start_key, input, state, parent_run_id, parent_signal, call_depth)
SELECT $1, $2, $3, $4, $5, $6, 'queued', $7, NULLIF($8, ''), $9
WHERE EXISTS (SELECT 1 FROM runs.runs WHERE id = $10 AND lease_token = $11)
ON CONFLICT (org_id, start_key) DO NOTHING
RETURNING id`, uuid.New(), n.OrgID, n.Workflow, n.Version, n.StartKey, n.Input, link, n.ParentSignal, n.CallDepth, parent, token)
if errors.Is(err, sql.ErrNoRows) {
err = s.db.GetContext(ctx, &id, `SELECT id FROM runs.runs WHERE org_id = $1 AND start_key = $2`, n.OrgID, n.StartKey)
if errors.Is(err, sql.ErrNoRows) {
return uuid.Nil, ErrLeaseLost
}
}
if err != nil {
return uuid.Nil, fmt.Errorf("start run: %w", err)
}
return id, nil
}
// Claim takes the oldest run that is queued, due to wake, or whose lease has
// expired. SKIP LOCKED only keeps claimers from waiting on each other;
// correctness rests on the lease token every later write is fenced by.
@@ -140,7 +179,7 @@ func (s *Store) Claim(ctx context.Context, owner string, token uuid.UUID, ttl ti
ORDER BY created_at
FOR UPDATE SKIP LOCKED
LIMIT 1)
RETURNING r.id, r.org_id, r.workflow, r.workflow_version, r.input, r.attempts, r.dry, r.overlay`, token, owner, ttl.Seconds())
RETURNING r.id, r.org_id, r.workflow, r.workflow_version, r.input, r.attempts, r.dry, r.overlay, r.call_depth`, token, owner, ttl.Seconds())
if errors.Is(err, sql.ErrNoRows) {
return Run{}, false, nil
}
@@ -218,9 +257,16 @@ type Ended struct {
Detail string
}
// Finish records how a run ended while the token holds the lease.
// Finish records how a run ended while the token holds the lease, and tells
// the run waiting on it, if any, in the same transaction so that a crash
// cannot leave the parent waiting for a run that has ended.
func (s *Store) Finish(ctx context.Context, id, token uuid.UUID, e Ended) error {
res, err := s.db.ExecContext(ctx, `
tx, err := s.db.BeginTxx(ctx, nil)
if err != nil {
return fmt.Errorf("finish run: %w", err)
}
defer func() { _ = tx.Rollback() }()
res, err := tx.ExecContext(ctx, `
UPDATE runs.runs
SET state = $3, result = $4, error = NULLIF($5, ''), detail = NULLIF($6, ''),
finished_at = now(), updated_at = now(),
@@ -232,6 +278,42 @@ func (s *Store) Finish(ctx context.Context, id, token uuid.UUID, e Ended) error
if affected(res) == 0 {
return ErrLeaseLost
}
if err := signalParent(ctx, tx, id); err != nil {
return err
}
if err := tx.Commit(); err != nil {
return fmt.Errorf("finish run: %w", err)
}
return nil
}
type execer interface {
ExecContext(ctx context.Context, query string, args ...any) (sql.Result, error)
}
// signalParent tells the run that called this one how it ended, and wakes it
// if it is waiting. Only an ending an operator or the caller must act on
// counts: a run waiting for approval has not ended, and its caller keeps
// waiting. The first ending wins, so a later retry of a parked child does not
// change what the parent was told.
func signalParent(ctx context.Context, q execer, id uuid.UUID) error {
if _, err := q.ExecContext(ctx, `
WITH ended AS (
SELECT parent_run_id, parent_signal,
jsonb_build_object('state', state, 'detail', coalesce(detail, ''),
'error', coalesce(error, ''), 'result', coalesce(result, 'null'::jsonb)) AS payload
FROM runs.runs
WHERE id = $1 AND parent_signal IS NOT NULL
AND state IN ('succeeded', 'failed', 'cancelled', 'parked', 'declined')),
sent AS (
INSERT INTO runs.signals (run_id, key, payload)
SELECT parent_run_id, parent_signal, payload FROM ended
ON CONFLICT (run_id, key) DO NOTHING)
UPDATE runs.runs r SET state = 'queued', wake_at = NULL, updated_at = now()
FROM ended e
WHERE r.id = e.parent_run_id AND r.state = 'waiting' AND r.waiting_for = e.parent_signal`, id); err != nil {
return fmt.Errorf("signal parent run: %w", err)
}
return nil
}
@@ -270,6 +352,20 @@ func (s *Store) Retry(ctx context.Context, orgID string, id uuid.UUID, from stri
if err != nil {
return fmt.Errorf("find step: %w", err)
}
// A call being retried starts a fresh child: the old one is detached, so
// its ending no longer reaches this run, and the signal it already sent is
// forgotten.
if _, err := tx.ExecContext(ctx, `
WITH retried AS (
SELECT 'call:' || substr(name, length('call-start:') + 1) AS signal FROM runs.steps
WHERE run_id = $1 AND name LIKE 'call-start:%' AND ((seq >= $2 AND seq < $3) OR seq >= $4)),
gone AS (
DELETE FROM runs.signals WHERE run_id = $1 AND key IN (SELECT signal FROM retried))
UPDATE runs.runs SET parent_run_id = NULL, parent_signal = NULL, start_key = NULL
WHERE parent_run_id = $1 AND parent_signal IN (SELECT signal FROM retried)`,
id, at.FirstSeq, MarkerBase, at.Marker); err != nil {
return fmt.Errorf("detach calls: %w", err)
}
if _, err := tx.ExecContext(ctx, `
DELETE FROM runs.steps WHERE run_id = $1 AND ((seq >= $2 AND seq < $3) OR seq >= $4)`,
id, at.FirstSeq, MarkerBase, at.Marker); err != nil {
@@ -324,10 +420,15 @@ func (s *Store) Signalled(ctx context.Context, id uuid.UUID, key string) (jsonte
return payload, true, nil
}
// Cancel asks a run to stop. A run not currently held ends at once; a running
// one sees the request at its next step.
// Cancel asks a run to stop. A run not currently held ends at once and tells
// the run waiting on it; a running one sees the request at its next step.
func (s *Store) Cancel(ctx context.Context, id uuid.UUID) error {
if _, err := s.db.ExecContext(ctx, `
tx, err := s.db.BeginTxx(ctx, nil)
if err != nil {
return fmt.Errorf("cancel run: %w", err)
}
defer func() { _ = tx.Rollback() }()
if _, err := tx.ExecContext(ctx, `
UPDATE runs.runs
SET cancel_requested = true, updated_at = now(),
state = CASE WHEN state IN ('queued', 'waiting', 'parked', 'waiting_approval') THEN 'cancelled' ELSE state END,
@@ -335,6 +436,12 @@ func (s *Store) Cancel(ctx context.Context, id uuid.UUID) error {
WHERE id = $1 AND state NOT IN ('succeeded', 'failed', 'cancelled', 'declined')`, id); err != nil {
return fmt.Errorf("cancel run: %w", err)
}
if err := signalParent(ctx, tx, id); err != nil {
return err
}
if err := tx.Commit(); err != nil {
return fmt.Errorf("cancel run: %w", err)
}
return nil
}
+7 -6
View File
@@ -93,12 +93,13 @@ func stepBudget(ctx context.Context, settings Settings) (agentloop.Budget, error
// the connectors and tools an agent may call, judgement, approvals, delivery
// and memory.
type Deps struct {
Settings Settings
Toolbox Toolbox
Judge Judge
Approver Approver
Notifier Notifier
Memory Memory
Settings Settings
Toolbox Toolbox
Judge Judge
Approver Approver
Notifier Notifier
Messenger Messenger
Memory Memory
}
// Workflow is a document ready to run: the code that executes its steps for
+44
View File
@@ -0,0 +1,44 @@
package runs
import (
"context"
"errors"
"fmt"
"strings"
)
// Messenger delivers text to one chat on a messaging channel.
type Messenger interface {
Send(ctx context.Context, channel, chatID, text, key string) error
}
// messageSettings send an authored message.
type messageSettings struct {
Channel string `yaml:"channel" json:"channel" jsonschema:"the messaging channel to send through"`
ChatID string `yaml:"chat_id" json:"chat_id" jsonschema:"the channel's identifier for the chat"`
Text string `yaml:"text" json:"text" jsonschema:"the message; references to earlier steps and inputs are filled in"`
}
func (s messageSettings) valid() error {
if strings.TrimSpace(s.Channel) == "" || strings.TrimSpace(s.ChatID) == "" || strings.TrimSpace(s.Text) == "" {
return errors.New("channel, chat_id and text are required")
}
return nil
}
func (s messageSettings) describe() string {
return fmt.Sprintf("Sends a message to a chat on %s", s.Channel)
}
// runMessage sends the message. A dry run records it without sending.
func runMessage(d Deps, rc *Context, _ *progress, _ string, s messageSettings) (StepResult, error) {
sent := fmt.Sprintf("send to %s chat %s: %s", s.Channel, s.ChatID, s.Text)
if rc.Dry() {
_, err := rc.Step("send-skipped", func(context.Context, string) (string, error) { return sent, nil })
return StepResult{Summary: sent}, err
}
_, err := rc.Step("send", func(ctx context.Context, key string) (bool, error) {
return true, d.Messenger.Send(ctx, s.Channel, s.ChatID, s.Text, key)
})
return StepResult{Summary: sent}, err
}
+87
View File
@@ -0,0 +1,87 @@
package runs
import (
json "encoding/json/v2"
"errors"
"fmt"
"strings"
)
// planDocument is a plan written as structured fields. The text every surface
// reads (the approval message and the next workflow's mission) and the
// dashboard's layout are both rendered from it, so what the approver approves
// is what the next workflow is handed.
type planDocument struct {
Title string `json:"title"`
Problem string `json:"problem"`
Solution string `json:"solution"`
Files []planFile `json:"files,omitempty"`
Acceptance []string `json:"acceptance,omitempty"`
NonGoals []string `json:"non_goals,omitempty"`
DiffEstimate string `json:"diff_estimate,omitempty"`
Risks []string `json:"risks,omitempty"`
}
// planFile is one file a plan adds, modifies or deletes.
type planFile struct {
Path string `json:"path"`
Change string `json:"change"`
Note string `json:"note,omitempty"`
}
// readPlanDocument decodes a rendered document setting. A value that is not an
// object, or an empty one (an absent reference renders empty), is no document.
func readPlanDocument(value any) (*planDocument, error) {
if fields, ok := value.(map[string]any); !ok || len(fields) == 0 {
return nil, nil //nolint:nilnil // no document is not an error
}
data, err := json.Marshal(value)
if err != nil {
return nil, err
}
var doc planDocument
if err := json.Unmarshal(data, &doc); err != nil {
return nil, fmt.Errorf("document: %w", err)
}
if strings.TrimSpace(doc.Problem) == "" || strings.TrimSpace(doc.Solution) == "" {
return nil, errors.New("document needs a problem and a solution")
}
return &doc, nil
}
// markdown is the plan as text.
func (d planDocument) markdown() string {
var b strings.Builder
if d.Title != "" {
fmt.Fprintf(&b, "# %s\n\n", d.Title)
}
fmt.Fprintf(&b, "## Problem\n\n%s\n\n## Proposed solution\n\n%s\n\n", strings.TrimSpace(d.Problem), strings.TrimSpace(d.Solution))
if len(d.Files) > 0 {
b.WriteString("## Files\n\n")
for _, f := range d.Files {
fmt.Fprintf(&b, "- `%s` (%s)", f.Path, f.Change)
if f.Note != "" {
fmt.Fprintf(&b, ": %s", f.Note)
}
b.WriteString("\n")
}
b.WriteString("\n")
}
list := func(heading string, items []string) {
if len(items) == 0 {
return
}
fmt.Fprintf(&b, "## %s\n\n", heading)
for _, item := range items {
fmt.Fprintf(&b, "- %s\n", item)
}
b.WriteString("\n")
}
list("Acceptance criteria", d.Acceptance)
list("Non-goals", d.NonGoals)
if d.DiffEstimate != "" {
fmt.Fprintf(&b, "## Estimated diff\n\n%s\n\n", d.DiffEstimate)
}
list("Risks", d.Risks)
return strings.TrimSpace(b.String())
}
+38 -11
View File
@@ -50,10 +50,22 @@ type workerStore interface {
CancelRequested(ctx context.Context, id uuid.UUID) (bool, error)
}
// runWork is everything a running workflow needs from storage.
type runWork interface {
workerStore
runStarter
}
// runStarter is what a step that starts another run needs from storage.
type runStarter interface {
workflowFinder
StartFor(ctx context.Context, token, parent uuid.UUID, n store.NewRun) (uuid.UUID, error)
}
// Engine starts, executes and resumes runs.
type Engine struct {
db *store.Store
work workerStore
work runWork
log *slog.Logger
opts Options
mu sync.RWMutex
@@ -68,7 +80,7 @@ func New(db *sqlx.DB, log *slog.Logger, opts Options) *Engine {
return newEngine(s, s, log, opts)
}
func newEngine(db *store.Store, work workerStore, log *slog.Logger, opts Options) *Engine {
func newEngine(db *store.Store, work runWork, log *slog.Logger, opts Options) *Engine {
if opts.Workers <= 0 {
opts.Workers = 4
}
@@ -171,30 +183,45 @@ func (e *Engine) Has(ctx context.Context, orgID, name string) (bool, error) {
// Start queues a run under the latest stored version of its workflow and
// returns its id.
func (e *Engine) Start(ctx context.Context, req StartRequest) (uuid.UUID, error) {
w, err := e.db.Workflow(ctx, req.OrgID, req.Workflow, 0)
if errors.Is(err, store.ErrNoWorkflow) {
return uuid.Nil, fmt.Errorf("no workflow named %q", req.Workflow)
}
n, err := newRun(ctx, e.db, req)
if err != nil {
return uuid.Nil, err
}
return e.db.Start(ctx, n)
}
// workflowFinder reads stored workflow versions.
type workflowFinder interface {
Workflow(ctx context.Context, orgID, name string, version int) (store.Workflow, error)
}
// newRun checks that the latest version of the workflow can start with the
// request's input, and returns the run to queue.
func newRun(ctx context.Context, finder workflowFinder, req StartRequest) (store.NewRun, error) {
w, err := finder.Workflow(ctx, req.OrgID, req.Workflow, 0)
if errors.Is(err, store.ErrNoWorkflow) {
return store.NewRun{}, fmt.Errorf("no workflow named %q", req.Workflow)
}
if err != nil {
return store.NewRun{}, err
}
doc, err := ParseDocument(w.Body)
if err != nil {
return uuid.Nil, fmt.Errorf("workflow %q version %d: %w", req.Workflow, w.Version, err)
return store.NewRun{}, fmt.Errorf("workflow %q version %d: %w", req.Workflow, w.Version, err)
}
if err := doc.unsupported(); err != nil {
return uuid.Nil, err
return store.NewRun{}, err
}
input := req.Input
if len(input) == 0 {
input = jsontext.Value("null")
}
if err := doc.checkInputs(decodeInputs(input)); err != nil {
return uuid.Nil, fmt.Errorf("workflow %q: %w", req.Workflow, err)
return store.NewRun{}, fmt.Errorf("workflow %q: %w", req.Workflow, err)
}
return e.db.Start(ctx, store.NewRun{
return store.NewRun{
OrgID: req.OrgID, Workflow: req.Workflow, Version: w.Version, StartKey: req.Key, Input: input,
})
}, nil
}
// CheckTrigger refuses a trigger for a workflow it couldn't start: one that
+29
View File
@@ -31,6 +31,11 @@ type fakeWork struct {
errors []string
details []string
results []jsontext.Value
// bodies are the stored workflows by name. A workflow not listed is a
// single notify step.
bodies map[string]string
// started is every run started for another, once per start key.
started []store.NewRun
}
func (f *fakeWork) Claim(context.Context, string, uuid.UUID, time.Duration) (store.Run, bool, error) {
@@ -86,6 +91,30 @@ func (f *fakeWork) Signalled(context.Context, uuid.UUID, string) (jsontext.Value
return nil, false, nil
}
func (f *fakeWork) Workflow(_ context.Context, _, name string, _ int) (store.Workflow, error) {
body, ok := f.bodies[name]
if !ok {
body = "steps:\n - type: run.notify\n"
}
return store.Workflow{Name: name, Version: 1, Body: body}, nil
}
// StartFor starts a run once per start key, as the store does.
func (f *fakeWork) StartFor(_ context.Context, _, _ uuid.UUID, n store.NewRun) (uuid.UUID, error) {
f.mu.Lock()
defer f.mu.Unlock()
if f.leaseLost {
return uuid.Nil, store.ErrLeaseLost
}
for _, started := range f.started {
if started.StartKey == n.StartKey {
return uuid.NewSHA1(uuid.NameSpaceURL, []byte(n.StartKey)), nil
}
}
f.started = append(f.started, n)
return uuid.NewSHA1(uuid.NameSpaceURL, []byte(n.StartKey)), nil
}
func (f *fakeWork) CancelRequested(context.Context, uuid.UUID) (bool, error) {
return false, nil
}
+193 -5
View File
@@ -6,6 +6,7 @@ import (
"encoding/json/jsontext"
json "encoding/json/v2"
"errors"
"slices"
"strings"
"testing"
@@ -106,13 +107,41 @@ func (n *notices) Notify(_ context.Context, msg deliveries.Notification) error {
return nil
}
// chats records the messages a workflow sends.
type chats struct{ sent []string }
func (c *chats) Send(_ context.Context, channel, chatID, text, _ string) error {
c.sent = append(c.sent, channel+"/"+chatID+": "+text)
return nil
}
// callWork is a worker store whose started child runs end when a test says
// so: until ending is set, a call waiting on its child has nothing to read.
type callWork struct {
signalAfter
ending jsontext.Value
}
func (c *callWork) Signalled(ctx context.Context, id uuid.UUID, key string) (jsontext.Value, bool, error) {
if !strings.HasPrefix(key, callSignalPrefix) {
return c.signalAfter.Signalled(ctx, id, key)
}
return c.ending, c.ending != nil, nil
}
const callSignalPrefix = "call:"
// TestDocumentStepsRun guards the workflow step contract on the engine: a
// step's settings are rendered from the run's inputs and earlier results, its
// condition decides whether it runs, a failure ends the run unless the step
// continues, and every action judgement holds is put to people exactly once,
// by an approval step or before the run ends. The first step that sets an
// outcome ends the run, a failed step parks it, and a parked run retried from a
// step keeps the results of the steps before it.
// step keeps the results of the steps before it. A call starts one child run
// however often the run restarts, and a waiting call resumes from the child's
// ending; a handoff starts the next workflow on the same input; an approval
// delivers the plan and leaves the run waiting; a message is sent once, and
// not at all in a dry run.
func TestDocumentStepsRun(t *testing.T) {
input := jsontext.Value(`{"event":{"fields":{"host":"web-1"}}}`)
tests := []struct {
@@ -132,6 +161,23 @@ func TestDocumentStepsRun(t *testing.T) {
wantAsked int
wantExecuted int
wantNotice string
// dry runs the workflow as a dry run.
dry bool
// bodies are the stored workflows a step can start.
bodies map[string]string
// crash kills the worker as it records its first step, so the run
// restarts having done that step's effect without recording it.
crash bool
// childEnding is how the started child ends: the run is expected to
// wait for it, and a restart then finds it ended.
childEnding string
// wantStarted is the child runs started, in all attempts, and
// wantChildInput the first one's input.
wantStarted int
wantChildInput string
// wantSignal is the key the first child run signals when it ends.
wantSignal string
wantSent []string
}{
{
name: "an action's input is rendered from the run's inputs",
@@ -256,6 +302,101 @@ steps:
levels: map[string]judgement.Level{"clean": judgement.ProposeAndWait, "restart": judgement.ProposeAndWait},
wantAssessed: 4, wantAsked: 2, wantExecuted: 2,
},
{
name: "a call that does not wait starts a child with its inputs and moves on",
doc: `inputs: {event: {type: object}}
steps:
- {id: child, type: workflow.call, settings: {workflow: remediate, inputs: {event: inputs.event, mode: fast}}}
- {type: run.notify, settings: {title: carried on}}
`,
bodies: map[string]string{"remediate": "inputs: {event: {type: object}, mode: {type: string}}\nsteps:\n - type: run.notify\n"},
wantStarted: 1, wantChildInput: `{"event":{"fields":{"host":"web-1"}},"mode":"fast"}`, wantSignal: "call:child", wantNotice: "carried on",
},
{
name: "a waiting call resumes after a restart without starting a second child",
doc: `steps:
- {id: child, type: workflow.call, settings: {workflow: remediate, wait: true}}
- {type: run.notify, settings: {title: "{{ steps.child.result.verdict }}"}}
`,
childEnding: `{"state":"succeeded","detail":"","error":"","result":{"verdict":"fixed"}}`,
wantStarted: 1, wantNotice: "fixed",
},
{
name: "a call restarted after the worker died starts the child once",
doc: `steps:
- {id: child, type: workflow.call, settings: {workflow: remediate, wait: true}}
- {type: run.notify, settings: {title: "{{ steps.child.result.verdict }}"}}
`,
crash: true,
childEnding: `{"state":"succeeded","detail":"","error":"","result":{"verdict":"fixed"}}`,
wantStarted: 1, wantNotice: "fixed",
},
{
name: "a waiting call parks the run when the child parks",
doc: `steps:
- {id: child, type: workflow.call, settings: {workflow: remediate, wait: true}}
- {type: run.notify, settings: {title: too late}}
`,
childEnding: `{"state":"parked","detail":"needs an operator","error":"","result":null}`,
wantStarted: 1, wantState: store.StateParked, wantDetail: "ended parked: needs an operator",
},
{
name: "a call is not started in a dry run",
doc: `steps:
- {type: workflow.call, settings: {workflow: remediate, wait: true}}
`,
dry: true,
},
{
name: "a handoff ends the run and starts the next workflow on the same input",
doc: `inputs: {event: {type: object}}
steps:
- {type: workflow.handoff, settings: {workflow: remediate, detail: needs a fix}}
- {type: run.notify, settings: {title: too late}}
`,
bodies: map[string]string{"remediate": "inputs: {event: {type: object}}\nsteps:\n - type: run.notify\n"},
wantStarted: 1, wantChildInput: `{"event":{"fields":{"host":"web-1"}}}`, wantDetail: "needs a fix",
},
{
name: "a handoff whose condition does not hold starts nothing",
doc: `inputs: {event: {type: object}}
steps:
- {type: workflow.handoff, when: "inputs.event.fields.missing", settings: {workflow: remediate}}
- {type: run.notify, settings: {title: stayed}}
`,
wantNotice: "stayed",
},
{
name: "an approval delivers the plan and leaves the run waiting for approval",
doc: `steps:
- {type: workflow.approve, settings: {plan: restart the host, then: remediate}}
- {type: run.notify, settings: {title: too late}}
`,
wantState: store.StateWaitingApproval, wantDetail: "awaiting go/no-go", wantNotice: "Plan awaiting approval",
},
{
name: "an approval needs a plan",
doc: `inputs: {event: {type: object}}
steps:
- {type: workflow.approve, settings: {document: "{{ inputs.event.fields.missing }}", then: remediate}}
`,
wantState: store.StateParked, wantDetail: "the plan is empty",
},
{
name: "a message is sent with its text rendered",
doc: `inputs: {event: {type: object}}
steps:
- {type: message.send, settings: {channel: chat, chat_id: ops, text: "cleaning {{ inputs.event.fields.host }}"}}
`,
wantSent: []string{"chat/ops: cleaning web-1"},
},
{
name: "a message is not sent in a dry run",
doc: `steps:
- {type: message.send, settings: {channel: chat, chat_id: ops, text: hello}}
`,
dry: true,
},
{
name: "a later step reads an earlier step's result",
doc: `steps:
@@ -275,11 +416,37 @@ steps:
judge := &actionJudge{levels: tt.levels, fail: tt.fail, failAction: tt.failAction}
ask := &askCounter{failFirst: tt.failFirst}
told := &notices{}
wf := Deps{Settings: onePersonPath{}, Toolbox: anyBox{}, Judge: judge, Approver: ask, Notifier: told}.Compile(doc)
work := &signalAfter{arrives: 1}
run := store.Run{ID: uuid.New()}
sent := &chats{}
wf := Deps{Settings: onePersonPath{}, Toolbox: anyBox{}, Judge: judge, Approver: ask, Notifier: told, Messenger: sent}.Compile(doc)
work := &callWork{signalAfter: signalAfter{arrives: 1}}
work.bodies = tt.bodies
run := store.Run{ID: uuid.New(), Dry: tt.dry}
rc := newContext(t.Context(), work, run, uuid.New(), nil)
if _, err = wf(rc, input); err != nil {
restart := func() {
rc = newContext(t.Context(), work, run, uuid.New(), slices.Clone(work.steps))
}
if tt.crash {
first := 1
work.rejectFrom = &first
}
_, err = wf(rc, input)
if tt.crash {
if !errors.Is(err, store.ErrLeaseLost) {
t.Fatalf("first run error = %v, want the lease lost", err)
}
work.rejectFrom = nil
restart()
_, err = wf(rc, input)
}
if tt.childEnding != "" {
if !errors.Is(err, errSuspended) {
t.Fatalf("waiting run error = %v, want it suspended", err)
}
work.ending = jsontext.Value(tt.childEnding)
restart()
_, err = wf(rc, input)
}
if err != nil {
t.Fatalf("run error = %v", err)
}
if tt.retry != "" {
@@ -314,6 +481,27 @@ steps:
t.Fatalf("action input = %s, want %s", a, b)
}
}
if len(work.started) != tt.wantStarted {
t.Fatalf("started %d runs, want %d", len(work.started), tt.wantStarted)
}
if tt.wantChildInput != "" {
var got, want any
if err := json.Unmarshal(work.started[0].Input, &got); err != nil {
t.Fatal(err)
}
if err := json.Unmarshal([]byte(tt.wantChildInput), &want); err != nil {
t.Fatal(err)
}
if a, b := mustJSON(t, got), mustJSON(t, want); a != b {
t.Fatalf("child input = %s, want %s", a, b)
}
}
if tt.wantSignal != "" && work.started[0].ParentSignal != tt.wantSignal {
t.Fatalf("child signals %q, want %q", work.started[0].ParentSignal, tt.wantSignal)
}
if !slices.Equal(sent.sent, tt.wantSent) {
t.Fatalf("messages = %q, want %q", sent.sent, tt.wantSent)
}
if tt.wantNotice != "" && (len(told.sent) != 1 || told.sent[0].Title != tt.wantNotice) {
t.Fatalf("notices = %+v, want one titled %q", told.sent, tt.wantNotice)
}
+8
View File
@@ -25,6 +25,10 @@ const (
typeNotify = "run.notify"
typeSleep = "workflow.sleep"
typeFinish = "workflow.finish"
typeCall = "workflow.call"
typeHandoff = "workflow.handoff"
typeApprove = "workflow.approve"
typeMessage = "message.send"
)
// StepType defines a workflow step's settings and implementation.
@@ -90,6 +94,10 @@ var stepTypes = vocabulary(
define(typeNotify, "Report the findings", notifySettings.valid, notifySettings.describe, runNotify),
define(typeSleep, "Wait", sleepSettings.valid, sleepSettings.describe, runSleep),
define(typeFinish, "Finish", finishSettings.valid, finishSettings.describe, runFinish),
define(typeCall, "Call workflow", callSettings.valid, callSettings.describe, runCall),
define(typeHandoff, "Hand off", handoffSettings.valid, handoffSettings.describe, runHandoff),
define(typeApprove, "Approval", approveSettings.valid, approveSettings.describe, runApprove),
define(typeMessage, "Send message", messageSettings.valid, messageSettings.describe, runMessage),
)
func vocabulary(types ...StepType) map[string]StepType {