diff --git a/internal/app/archie/operations.go b/internal/app/archie/operations.go index 75f0a35a6..025617a20 100644 --- a/internal/app/archie/operations.go +++ b/internal/app/archie/operations.go @@ -59,7 +59,7 @@ func (o *operations) Start(ctx context.Context) error { // from the request. o.engine.Use(runs.Deps{ Settings: o.settings, Toolbox: o.conns, Judge: o.judge, Approver: o.allow, - Notifier: o.tell, Memory: o.remember, + Notifier: o.tell, Messenger: o.tell, Memory: o.remember, }) if err := installStarter(ctx, o.engine); err != nil { return err diff --git a/internal/platform/pg/migrations/00045_runs_calls.sql b/internal/platform/pg/migrations/00045_runs_calls.sql new file mode 100644 index 000000000..a3a58818b --- /dev/null +++ b/internal/platform/pg/migrations/00045_runs_calls.sql @@ -0,0 +1,14 @@ +-- +goose Up +-- A workflow.call step starts a child run. parent_run_id and parent_signal say +-- which run, and which signal key, to wake when the child ends; call_depth +-- bounds how deep calls nest. A run started by a handoff has no parent. +ALTER TABLE runs.runs ADD COLUMN parent_run_id uuid REFERENCES runs.runs (id) ON DELETE CASCADE; +ALTER TABLE runs.runs ADD COLUMN parent_signal text; +ALTER TABLE runs.runs ADD COLUMN call_depth integer NOT NULL DEFAULT 0; +CREATE INDEX runs_runs_parent ON runs.runs (parent_run_id) WHERE parent_run_id IS NOT NULL; + +-- +goose Down +DROP INDEX runs.runs_runs_parent; +ALTER TABLE runs.runs DROP COLUMN call_depth; +ALTER TABLE runs.runs DROP COLUMN parent_signal; +ALTER TABLE runs.runs DROP COLUMN parent_run_id; diff --git a/modules/deliveries/deliveries.go b/modules/deliveries/deliveries.go index 411aedf4d..dd8f878b0 100644 --- a/modules/deliveries/deliveries.go +++ b/modules/deliveries/deliveries.go @@ -37,7 +37,10 @@ const ( heartbeat = 25 * time.Second // deliveryMessage is the delivery a connector declares to carry messages. deliveryMessage = "message" - maxBackoff = 10 * time.Minute + // deliveryChat is the delivery a connector declares to send text to one + // chat it can address. + deliveryChat = "chat_message" + maxBackoff = 10 * time.Minute // maxAttempts ends a delivery that keeps failing: with the backoff capped // at maxBackoff it is about a day of trying. maxAttempts = 150 @@ -137,6 +140,24 @@ func (m *Module) Notify(ctx context.Context, n Notification) error { }) } +// Send delivers text to one chat on the named channel, and returns when the +// channel has accepted it. The key makes the delivery idempotent on the +// channel, so a step that is replayed after a crash does not send twice. +func (m *Module) Send(ctx context.Context, channel, chatID, text, key string) error { + c, ok := m.channels.Get(channel) + if !ok { + return fmt.Errorf("connector %q isn't running", channel) + } + raw, err := json.Marshal(map[string]string{"chat_id": chatID, "text": text}) + if err != nil { + return err + } + if _, err := c.Deliver(ctx, deliveryChat, raw, key); err != nil { + return fmt.Errorf("send to %s chat %s: %w", channel, chatID, err) + } + return nil +} + // Ask delivers an approval request on every channel. Asking again for the same // key changes nothing. func (m *Module) Ask(ctx context.Context, a ApprovalRequest) error { diff --git a/modules/runs/context.go b/modules/runs/context.go index 584fafe41..cb15499ea 100644 --- a/modules/runs/context.go +++ b/modules/runs/context.go @@ -36,7 +36,7 @@ type Context struct { // The run owns its context, as an http.Request does: every step takes its // cancellation from it, and a second context passed beside it could disagree. ctx context.Context //nolint:containedctx // see above - work workerStore + work runWork run store.Run token uuid.UUID saved map[int]store.Step @@ -52,6 +52,10 @@ type Context struct { ending *Ending // overlay is what a dry run carries instead of a stored version. overlay *dryOverlay + // resolve checks a workflow to start, and returns the run to queue. It is a + // field because the step types, which are initialised before it, can't + // name the document checks that depend on them. + resolve func(ctx context.Context, req StartRequest) (store.NewRun, error) } // markerBase is where started markers are numbered, far above any step a @@ -65,7 +69,7 @@ const ( enteredPrefix = "at:" ) -func newContext(ctx context.Context, work workerStore, run store.Run, token uuid.UUID, steps []store.Step) *Context { +func newContext(ctx context.Context, work runWork, run store.Run, token uuid.UUID, steps []store.Step) *Context { saved := make(map[int]store.Step, len(steps)) started, entered := map[string]bool{}, map[string]bool{} nextMarker := markerBase @@ -81,6 +85,7 @@ func newContext(ctx context.Context, work workerStore, run store.Run, token uuid } } rc := &Context{ctx: ctx, work: work, run: run, token: token, saved: saved, started: started, entered: entered, nextMarker: nextMarker} + rc.resolve = func(ctx context.Context, req StartRequest) (store.NewRun, error) { return newRun(ctx, work, req) } if run.Dry { rc.overlay, _ = readOverlay(run.Overlay) } diff --git a/modules/runs/corpus_test.go b/modules/runs/corpus_test.go index 46d093587..a2330d407 100644 --- a/modules/runs/corpus_test.go +++ b/modules/runs/corpus_test.go @@ -38,8 +38,8 @@ func TestCorpusChecksAsItsStepTypesLand(t *testing.T) { }{ {name: "investigate checks", file: "investigate"}, { - name: "feasibility waits for repositories, switch and approve", file: "feasibility", - want: slices.Concat(notYet("repo.prepare", "workflow.approve"), + name: "feasibility waits for repositories and switch", file: "feasibility", + want: slices.Concat(notYet("repo.prepare"), []string{"switch steps are not supported yet"}), }, { @@ -58,8 +58,8 @@ func TestCorpusChecksAsItsStepTypesLand(t *testing.T) { agentLater("gate", "gate", "protect_tests")), }, { - name: "triage's references and conditions check; it waits for forge and handoff steps", file: "triage", - want: notYet("forge.comment", "forge.close-issue", "workflow.handoff"), + name: "triage's references and conditions check; it waits for forge steps", file: "triage", + want: notYet("forge.comment", "forge.close-issue"), }, { name: "a finish step with an unknown status is refused when the workflow is saved", @@ -68,6 +68,23 @@ func TestCorpusChecksAsItsStepTypesLand(t *testing.T) { `, want: []string{`status is completed, parked, waiting_approval or declined, not "finished"`}, }, + { + name: "a call, handoff, approval and message check their settings when the workflow is saved", + yaml: `steps: + - {type: workflow.call, settings: {workflow: w, inputs: {a: {b: 1}}}} + - {type: workflow.call, settings: {workflow: w, outputs: {x: outputs.x}}} + - {type: workflow.call, settings: {workflow: w, inputs: {a: inputs.9}}} + - {type: workflow.handoff, settings: {detail: d}} + - {type: workflow.approve, settings: {plan: p}} + - {type: message.send, settings: {channel: c, chat_id: " ", text: t}} +`, + want: []string{ + "not a nested document", "outputs on a call that does not wait can never be read", + `input "a" reference "inputs.9" must name an input as inputs.`, + "workflow is required", "plan or document, and then, are required", + "channel, chat_id and text are required", + }, + }, { name: "a result field the agent's schema doesn't declare is refused", yaml: `steps: diff --git a/modules/runs/handover.go b/modules/runs/handover.go new file mode 100644 index 000000000..6716e345a --- /dev/null +++ b/modules/runs/handover.go @@ -0,0 +1,268 @@ +package runs + +import ( + "context" + "encoding/json/jsontext" + json "encoding/json/v2" + "errors" + "fmt" + "regexp" + "strings" + "time" + + "github.com/google/uuid" + + "github.com/samcharles93/archie-core/modules/deliveries" + "github.com/samcharles93/archie-core/modules/runs/internal/store" +) + +// maxCallDepth bounds how deep calls nest. A workflow that calls itself is +// not refused when it is saved, so this is also what stops it. +const maxCallDepth = 5 + +// maxCallWait is how long a waiting call holds out for its child before the +// step fails. +const maxCallWait = 7 * 24 * time.Hour + +// callReference prefixes a call's input value that names an input of the +// calling workflow. +const callReference = "inputs." + +var refName = regexp.MustCompile(`^[A-Za-z_][A-Za-z0-9_]*$`) + +// callSettings start another workflow as a child run. +type callSettings struct { + Workflow string `yaml:"workflow" json:"workflow" jsonschema:"the workflow to start as a child run"` + Inputs map[string]any `yaml:"inputs,omitempty" json:"inputs,omitempty" jsonschema:"callee input names mapped to scalar literals or inputs. references to caller inputs"` + Wait bool `yaml:"wait,omitempty" json:"wait,omitempty" jsonschema:"wait for the child run to finish before continuing"` + Outputs map[string]string `yaml:"outputs,omitempty" json:"outputs,omitempty" jsonschema:"callee output names mapped to outputs. references to caller outputs; requires wait: true"` +} + +func (s callSettings) valid() error { + if strings.TrimSpace(s.Workflow) == "" { + return errors.New("workflow is required") + } + for name, value := range s.Inputs { + if _, ok := value.(map[string]any); ok { + return fmt.Errorf("input %q must be a literal or an inputs. reference, not a nested document", name) + } + if ref, ok := value.(string); ok && strings.HasPrefix(ref, callReference) && !isInputReference(ref) { + return fmt.Errorf("input %q reference %q must name an input as inputs.", name, ref) + } + } + switch { + case len(s.Outputs) > 0 && !s.Wait: + return errors.New("outputs on a call that does not wait can never be read: nothing has finished when the caller moves on") + case len(s.Outputs) > 0: + return errors.New("outputs are not supported yet: a workflow can't declare outputs") + } + return nil +} + +func (s callSettings) describe() string { + line := fmt.Sprintf("Starts %s as a child run", s.Workflow) + if s.Wait { + line += " and waits for it to finish" + } + return line +} + +// isInputReference reports whether a string names an input of the calling +// workflow. A literal that begins with "inputs." but names no input is a +// literal. +func isInputReference(value string) bool { + name, ok := strings.CutPrefix(value, callReference) + return ok && refName.MatchString(name) +} + +// callInputs turns the call's settings into the child's input: a reference +// reads the caller's own input, a literal passes through. +func callInputs(call, callerInputs map[string]any) (map[string]any, error) { + resolved := make(map[string]any, len(call)) + for name, value := range call { + if ref, ok := value.(string); ok && isInputReference(ref) { + got, present := callerInputs[strings.TrimPrefix(ref, callReference)] + if !present { + return nil, fmt.Errorf("input %q references %q, which the calling run does not carry", name, ref) + } + resolved[name] = got + continue + } + resolved[name] = value + } + return resolved, nil +} + +// runCall starts the child, and with wait true suspends the run until the +// child ends. The child is keyed by this run and step, so a replay or a +// restart finds the child it already started. The child's ending signals this +// run, which holds no worker while it waits. +func runCall(_ Deps, rc *Context, p *progress, name string, s callSettings) (StepResult, error) { + if rc.Dry() { + _, err := rc.Step("call-skipped", func(context.Context, string) (string, error) { + return s.Workflow, nil + }) + return StepResult{Summary: "would start " + s.Workflow}, err + } + if rc.run.CallDepth+1 > maxCallDepth { + return StepResult{}, fmt.Errorf("call depth %d would pass the limit of %d", rc.run.CallDepth+1, maxCallDepth) + } + inputs, err := callInputs(s.Inputs, p.inputs) + if err != nil { + return StepResult{}, err + } + signal := callSignal(name) + child, err := rc.Step("call-start:"+name, func(ctx context.Context, _ string) (uuid.UUID, error) { + return rc.startRun(ctx, "call:"+rc.run.ID.String()+":"+name, s.Workflow, inputs, signal, rc.run.CallDepth+1) + }) + if err != nil { + return StepResult{}, fmt.Errorf("start %q: %w", s.Workflow, err) + } + if !s.Wait { + return StepResult{Summary: fmt.Sprintf("started %q (run %s); no wait", s.Workflow, child)}, nil + } + payload, err := rc.Wait(signal, maxCallWait) + if err != nil { + return StepResult{}, err + } + return calleeResult(s.Workflow, child, payload) +} + +// callSignal is the key a call's child signals when it ends. +func callSignal(name string) string { return "call:" + name } + +// calleeEnding is what a child run tells its caller. +type calleeEnding struct { + State string `json:"state"` + Detail string `json:"detail"` + Error string `json:"error"` + Result jsontext.Value `json:"result"` +} + +// calleeResult turns a child's ending into the call step's result. A child +// that did not succeed fails the step, with its own detail as the reason: a +// parked child will not move without an operator, so the caller does not wait +// for one. +func calleeResult(workflow string, child uuid.UUID, payload jsontext.Value) (StepResult, error) { + var end calleeEnding + if err := json.Unmarshal(payload, &end); err != nil { + return StepResult{}, fmt.Errorf("decode the ending of %q: %w", workflow, err) + } + if end.State != store.StateSucceeded { + return StepResult{}, fmt.Errorf("callee %q (run %s) ended %s: %s", workflow, child, end.State, firstText(end.Error, end.Detail)) + } + var fields map[string]any + // A result that is not an object has no fields for a later step to read. + _ = json.Unmarshal(end.Result, &fields) + return StepResult{Summary: fmt.Sprintf("%q (run %s) finished", workflow, child), Result: fields}, nil +} + +// startRun queues a run of another workflow for this run, idempotent on key. +func (rc *Context) startRun(ctx context.Context, key, workflow string, input map[string]any, signal string, depth int) (uuid.UUID, error) { + raw, err := json.Marshal(input) + if err != nil { + return uuid.Nil, fmt.Errorf("encode input: %w", err) + } + n, err := rc.resolve(ctx, StartRequest{OrgID: rc.run.OrgID, Workflow: workflow, Input: raw}) + if err != nil { + return uuid.Nil, err + } + n.StartKey, n.ParentSignal, n.CallDepth = key, signal, depth + return rc.work.StartFor(ctx, rc.token, rc.run.ID, n) +} + +// handoffSettings end the run and start another workflow with the same input. +type handoffSettings struct { + Workflow string `yaml:"workflow" json:"workflow" jsonschema:"the workflow the task is requeued under"` + Detail string `yaml:"detail,omitempty" json:"detail,omitempty" jsonschema:"why the task was handed off"` +} + +func (s handoffSettings) valid() error { + if strings.TrimSpace(s.Workflow) == "" { + return errors.New("workflow is required") + } + return nil +} + +func (s handoffSettings) describe() string { + return fmt.Sprintf("Ends the run and starts %s with the same input", s.Workflow) +} + +// runHandoff completes the run and starts a run of the target workflow on the +// same input. A dry run records the handoff without starting anything. +func runHandoff(_ Deps, rc *Context, p *progress, name string, s handoffSettings) (StepResult, error) { + target := strings.TrimSpace(s.Workflow) + detail := s.Detail + if detail == "" { + detail = "handed off to " + target + } + if rc.Dry() { + if _, err := rc.Step("handoff-skipped", func(context.Context, string) (string, error) { return target, nil }); err != nil { + return StepResult{}, err + } + } else if _, err := rc.Step("handoff:"+name, func(ctx context.Context, _ string) (uuid.UUID, error) { + return rc.startRun(ctx, "handoff:"+rc.run.ID.String()+":"+name, target, p.inputs, "", rc.run.CallDepth) + }); err != nil { + return StepResult{}, fmt.Errorf("hand off to %q: %w", target, err) + } + rc.End(store.StateSucceeded, detail) + return StepResult{Summary: detail}, nil +} + +// approveSettings put a plan to an operator and end the run until they decide. +type approveSettings struct { + Plan string `yaml:"plan,omitempty" json:"plan,omitempty" jsonschema:"what the operator decides on; the next workflow reads it as task.plan"` + Document any `yaml:"document,omitempty" json:"document,omitempty" jsonschema:"the plan as structured fields (title, problem, solution, files, acceptance, non_goals, diff_estimate, risks), such as {{ steps.prd.result }}; the plan text is rendered from it"` + Then string `yaml:"then" json:"then" jsonschema:"the workflow the task runs once approved"` +} + +func (s approveSettings) valid() error { + if (strings.TrimSpace(s.Plan) == "" && s.Document == nil) || strings.TrimSpace(s.Then) == "" { + return errors.New("plan or document, and then, are required") + } + return nil +} + +func (s approveSettings) describe() string { + return fmt.Sprintf("Delivers the plan and waits for approval; %s runs once it is approved", s.Then) +} + +// handover is what an approval step records for whoever decides: the plan, in +// text and as the document it was rendered from, and the workflow to run once +// it is approved. +type handover struct { + Plan string `json:"plan"` + Document *planDocument `json:"document,omitzero"` + Then string `json:"then"` +} + +// runApprove delivers the plan and ends the run waiting for approval. A dry +// run records the plan without delivering it. +func runApprove(d Deps, rc *Context, _ *progress, name string, s approveSettings) (StepResult, error) { + doc, err := readPlanDocument(s.Document) + if err != nil { + return StepResult{}, err + } + plan := s.Plan + if doc != nil { + plan = doc.markdown() + } + if strings.TrimSpace(plan) == "" { + return StepResult{}, errors.New("the plan is empty") + } + _, err = rc.Step("approve:"+name, func(ctx context.Context, key string) (handover, error) { + out := handover{Plan: plan, Document: doc, Then: strings.TrimSpace(s.Then)} + if rc.Dry() { + return out, nil + } + return out, d.Notifier.Notify(ctx, deliveries.Notification{ + RunID: rc.RunID(), Key: key, Title: "Plan awaiting approval", Body: plan, + }) + }) + if err != nil { + return StepResult{}, err + } + const detail = "plan delivered, awaiting go/no-go" + rc.End(store.StateWaitingApproval, detail) + return StepResult{Summary: detail}, nil +} diff --git a/modules/runs/internal/store/store.go b/modules/runs/internal/store/store.go index 8c9a5b0c5..f15f9b705 100644 --- a/modules/runs/internal/store/store.go +++ b/modules/runs/internal/store/store.go @@ -70,6 +70,9 @@ type Run struct { Dry bool `db:"dry"` // Scan into []byte because live runs have a SQL NULL overlay. Overlay []byte `db:"overlay"` + // CallDepth is how many workflow.call steps lie between this run and the + // run that started the chain. + CallDepth int `db:"call_depth"` } // Step is one recorded step of a run. @@ -101,6 +104,11 @@ type NewRun struct { Input jsontext.Value Dry bool Overlay jsontext.Value + // CallDepth is the depth of a run started by a call. + CallDepth int + // ParentSignal is the signal key to send the run that called this one + // when this run ends; empty for a run nothing waits on. + ParentSignal string } // Start inserts a queued run. A run with the same organisation and start key @@ -122,6 +130,37 @@ func (s *Store) Start(ctx context.Context, n NewRun) (uuid.UUID, error) { return id, nil } +// StartFor inserts a queued run on behalf of a run whose lease the token +// holds. The start key makes it idempotent, so a replayed step or a restarted +// worker gets the run already started. A non-empty ParentSignal links the new +// run to the parent: its ending signals the parent under that key. +func (s *Store) StartFor(ctx context.Context, token, parent uuid.UUID, n NewRun) (uuid.UUID, error) { + if n.StartKey == "" { + return uuid.Nil, errors.New("start run: a run started for another needs a start key") + } + var link uuid.NullUUID + if n.ParentSignal != "" { + link = uuid.NullUUID{UUID: parent, Valid: true} + } + var id uuid.UUID + err := s.db.GetContext(ctx, &id, ` + INSERT INTO runs.runs (id, org_id, workflow, workflow_version, start_key, input, state, parent_run_id, parent_signal, call_depth) + SELECT $1, $2, $3, $4, $5, $6, 'queued', $7, NULLIF($8, ''), $9 + WHERE EXISTS (SELECT 1 FROM runs.runs WHERE id = $10 AND lease_token = $11) + ON CONFLICT (org_id, start_key) DO NOTHING + RETURNING id`, uuid.New(), n.OrgID, n.Workflow, n.Version, n.StartKey, n.Input, link, n.ParentSignal, n.CallDepth, parent, token) + if errors.Is(err, sql.ErrNoRows) { + err = s.db.GetContext(ctx, &id, `SELECT id FROM runs.runs WHERE org_id = $1 AND start_key = $2`, n.OrgID, n.StartKey) + if errors.Is(err, sql.ErrNoRows) { + return uuid.Nil, ErrLeaseLost + } + } + if err != nil { + return uuid.Nil, fmt.Errorf("start run: %w", err) + } + return id, nil +} + // Claim takes the oldest run that is queued, due to wake, or whose lease has // expired. SKIP LOCKED only keeps claimers from waiting on each other; // correctness rests on the lease token every later write is fenced by. @@ -140,7 +179,7 @@ func (s *Store) Claim(ctx context.Context, owner string, token uuid.UUID, ttl ti ORDER BY created_at FOR UPDATE SKIP LOCKED LIMIT 1) - RETURNING r.id, r.org_id, r.workflow, r.workflow_version, r.input, r.attempts, r.dry, r.overlay`, token, owner, ttl.Seconds()) + RETURNING r.id, r.org_id, r.workflow, r.workflow_version, r.input, r.attempts, r.dry, r.overlay, r.call_depth`, token, owner, ttl.Seconds()) if errors.Is(err, sql.ErrNoRows) { return Run{}, false, nil } @@ -218,9 +257,16 @@ type Ended struct { Detail string } -// Finish records how a run ended while the token holds the lease. +// Finish records how a run ended while the token holds the lease, and tells +// the run waiting on it, if any, in the same transaction so that a crash +// cannot leave the parent waiting for a run that has ended. func (s *Store) Finish(ctx context.Context, id, token uuid.UUID, e Ended) error { - res, err := s.db.ExecContext(ctx, ` + tx, err := s.db.BeginTxx(ctx, nil) + if err != nil { + return fmt.Errorf("finish run: %w", err) + } + defer func() { _ = tx.Rollback() }() + res, err := tx.ExecContext(ctx, ` UPDATE runs.runs SET state = $3, result = $4, error = NULLIF($5, ''), detail = NULLIF($6, ''), finished_at = now(), updated_at = now(), @@ -232,6 +278,42 @@ func (s *Store) Finish(ctx context.Context, id, token uuid.UUID, e Ended) error if affected(res) == 0 { return ErrLeaseLost } + if err := signalParent(ctx, tx, id); err != nil { + return err + } + if err := tx.Commit(); err != nil { + return fmt.Errorf("finish run: %w", err) + } + return nil +} + +type execer interface { + ExecContext(ctx context.Context, query string, args ...any) (sql.Result, error) +} + +// signalParent tells the run that called this one how it ended, and wakes it +// if it is waiting. Only an ending an operator or the caller must act on +// counts: a run waiting for approval has not ended, and its caller keeps +// waiting. The first ending wins, so a later retry of a parked child does not +// change what the parent was told. +func signalParent(ctx context.Context, q execer, id uuid.UUID) error { + if _, err := q.ExecContext(ctx, ` + WITH ended AS ( + SELECT parent_run_id, parent_signal, + jsonb_build_object('state', state, 'detail', coalesce(detail, ''), + 'error', coalesce(error, ''), 'result', coalesce(result, 'null'::jsonb)) AS payload + FROM runs.runs + WHERE id = $1 AND parent_signal IS NOT NULL + AND state IN ('succeeded', 'failed', 'cancelled', 'parked', 'declined')), + sent AS ( + INSERT INTO runs.signals (run_id, key, payload) + SELECT parent_run_id, parent_signal, payload FROM ended + ON CONFLICT (run_id, key) DO NOTHING) + UPDATE runs.runs r SET state = 'queued', wake_at = NULL, updated_at = now() + FROM ended e + WHERE r.id = e.parent_run_id AND r.state = 'waiting' AND r.waiting_for = e.parent_signal`, id); err != nil { + return fmt.Errorf("signal parent run: %w", err) + } return nil } @@ -270,6 +352,20 @@ func (s *Store) Retry(ctx context.Context, orgID string, id uuid.UUID, from stri if err != nil { return fmt.Errorf("find step: %w", err) } + // A call being retried starts a fresh child: the old one is detached, so + // its ending no longer reaches this run, and the signal it already sent is + // forgotten. + if _, err := tx.ExecContext(ctx, ` + WITH retried AS ( + SELECT 'call:' || substr(name, length('call-start:') + 1) AS signal FROM runs.steps + WHERE run_id = $1 AND name LIKE 'call-start:%' AND ((seq >= $2 AND seq < $3) OR seq >= $4)), + gone AS ( + DELETE FROM runs.signals WHERE run_id = $1 AND key IN (SELECT signal FROM retried)) + UPDATE runs.runs SET parent_run_id = NULL, parent_signal = NULL, start_key = NULL + WHERE parent_run_id = $1 AND parent_signal IN (SELECT signal FROM retried)`, + id, at.FirstSeq, MarkerBase, at.Marker); err != nil { + return fmt.Errorf("detach calls: %w", err) + } if _, err := tx.ExecContext(ctx, ` DELETE FROM runs.steps WHERE run_id = $1 AND ((seq >= $2 AND seq < $3) OR seq >= $4)`, id, at.FirstSeq, MarkerBase, at.Marker); err != nil { @@ -324,10 +420,15 @@ func (s *Store) Signalled(ctx context.Context, id uuid.UUID, key string) (jsonte return payload, true, nil } -// Cancel asks a run to stop. A run not currently held ends at once; a running -// one sees the request at its next step. +// Cancel asks a run to stop. A run not currently held ends at once and tells +// the run waiting on it; a running one sees the request at its next step. func (s *Store) Cancel(ctx context.Context, id uuid.UUID) error { - if _, err := s.db.ExecContext(ctx, ` + tx, err := s.db.BeginTxx(ctx, nil) + if err != nil { + return fmt.Errorf("cancel run: %w", err) + } + defer func() { _ = tx.Rollback() }() + if _, err := tx.ExecContext(ctx, ` UPDATE runs.runs SET cancel_requested = true, updated_at = now(), state = CASE WHEN state IN ('queued', 'waiting', 'parked', 'waiting_approval') THEN 'cancelled' ELSE state END, @@ -335,6 +436,12 @@ func (s *Store) Cancel(ctx context.Context, id uuid.UUID) error { WHERE id = $1 AND state NOT IN ('succeeded', 'failed', 'cancelled', 'declined')`, id); err != nil { return fmt.Errorf("cancel run: %w", err) } + if err := signalParent(ctx, tx, id); err != nil { + return err + } + if err := tx.Commit(); err != nil { + return fmt.Errorf("cancel run: %w", err) + } return nil } diff --git a/modules/runs/interpreter.go b/modules/runs/interpreter.go index 7ecb367ae..6063ae8cb 100644 --- a/modules/runs/interpreter.go +++ b/modules/runs/interpreter.go @@ -93,12 +93,13 @@ func stepBudget(ctx context.Context, settings Settings) (agentloop.Budget, error // the connectors and tools an agent may call, judgement, approvals, delivery // and memory. type Deps struct { - Settings Settings - Toolbox Toolbox - Judge Judge - Approver Approver - Notifier Notifier - Memory Memory + Settings Settings + Toolbox Toolbox + Judge Judge + Approver Approver + Notifier Notifier + Messenger Messenger + Memory Memory } // Workflow is a document ready to run: the code that executes its steps for diff --git a/modules/runs/message.go b/modules/runs/message.go new file mode 100644 index 000000000..07bc4cca5 --- /dev/null +++ b/modules/runs/message.go @@ -0,0 +1,44 @@ +package runs + +import ( + "context" + "errors" + "fmt" + "strings" +) + +// Messenger delivers text to one chat on a messaging channel. +type Messenger interface { + Send(ctx context.Context, channel, chatID, text, key string) error +} + +// messageSettings send an authored message. +type messageSettings struct { + Channel string `yaml:"channel" json:"channel" jsonschema:"the messaging channel to send through"` + ChatID string `yaml:"chat_id" json:"chat_id" jsonschema:"the channel's identifier for the chat"` + Text string `yaml:"text" json:"text" jsonschema:"the message; references to earlier steps and inputs are filled in"` +} + +func (s messageSettings) valid() error { + if strings.TrimSpace(s.Channel) == "" || strings.TrimSpace(s.ChatID) == "" || strings.TrimSpace(s.Text) == "" { + return errors.New("channel, chat_id and text are required") + } + return nil +} + +func (s messageSettings) describe() string { + return fmt.Sprintf("Sends a message to a chat on %s", s.Channel) +} + +// runMessage sends the message. A dry run records it without sending. +func runMessage(d Deps, rc *Context, _ *progress, _ string, s messageSettings) (StepResult, error) { + sent := fmt.Sprintf("send to %s chat %s: %s", s.Channel, s.ChatID, s.Text) + if rc.Dry() { + _, err := rc.Step("send-skipped", func(context.Context, string) (string, error) { return sent, nil }) + return StepResult{Summary: sent}, err + } + _, err := rc.Step("send", func(ctx context.Context, key string) (bool, error) { + return true, d.Messenger.Send(ctx, s.Channel, s.ChatID, s.Text, key) + }) + return StepResult{Summary: sent}, err +} diff --git a/modules/runs/plan.go b/modules/runs/plan.go new file mode 100644 index 000000000..7f7e0289c --- /dev/null +++ b/modules/runs/plan.go @@ -0,0 +1,87 @@ +package runs + +import ( + json "encoding/json/v2" + "errors" + "fmt" + "strings" +) + +// planDocument is a plan written as structured fields. The text every surface +// reads (the approval message and the next workflow's mission) and the +// dashboard's layout are both rendered from it, so what the approver approves +// is what the next workflow is handed. +type planDocument struct { + Title string `json:"title"` + Problem string `json:"problem"` + Solution string `json:"solution"` + Files []planFile `json:"files,omitempty"` + Acceptance []string `json:"acceptance,omitempty"` + NonGoals []string `json:"non_goals,omitempty"` + DiffEstimate string `json:"diff_estimate,omitempty"` + Risks []string `json:"risks,omitempty"` +} + +// planFile is one file a plan adds, modifies or deletes. +type planFile struct { + Path string `json:"path"` + Change string `json:"change"` + Note string `json:"note,omitempty"` +} + +// readPlanDocument decodes a rendered document setting. A value that is not an +// object, or an empty one (an absent reference renders empty), is no document. +func readPlanDocument(value any) (*planDocument, error) { + if fields, ok := value.(map[string]any); !ok || len(fields) == 0 { + return nil, nil //nolint:nilnil // no document is not an error + } + data, err := json.Marshal(value) + if err != nil { + return nil, err + } + var doc planDocument + if err := json.Unmarshal(data, &doc); err != nil { + return nil, fmt.Errorf("document: %w", err) + } + if strings.TrimSpace(doc.Problem) == "" || strings.TrimSpace(doc.Solution) == "" { + return nil, errors.New("document needs a problem and a solution") + } + return &doc, nil +} + +// markdown is the plan as text. +func (d planDocument) markdown() string { + var b strings.Builder + if d.Title != "" { + fmt.Fprintf(&b, "# %s\n\n", d.Title) + } + fmt.Fprintf(&b, "## Problem\n\n%s\n\n## Proposed solution\n\n%s\n\n", strings.TrimSpace(d.Problem), strings.TrimSpace(d.Solution)) + if len(d.Files) > 0 { + b.WriteString("## Files\n\n") + for _, f := range d.Files { + fmt.Fprintf(&b, "- `%s` (%s)", f.Path, f.Change) + if f.Note != "" { + fmt.Fprintf(&b, ": %s", f.Note) + } + b.WriteString("\n") + } + b.WriteString("\n") + } + list := func(heading string, items []string) { + if len(items) == 0 { + return + } + fmt.Fprintf(&b, "## %s\n\n", heading) + for _, item := range items { + fmt.Fprintf(&b, "- %s\n", item) + } + b.WriteString("\n") + } + list("Acceptance criteria", d.Acceptance) + list("Non-goals", d.NonGoals) + if d.DiffEstimate != "" { + fmt.Fprintf(&b, "## Estimated diff\n\n%s\n\n", d.DiffEstimate) + } + list("Risks", d.Risks) + return strings.TrimSpace(b.String()) +} diff --git a/modules/runs/runs.go b/modules/runs/runs.go index 96f344e26..3d0c3bfcf 100644 --- a/modules/runs/runs.go +++ b/modules/runs/runs.go @@ -50,10 +50,22 @@ type workerStore interface { CancelRequested(ctx context.Context, id uuid.UUID) (bool, error) } +// runWork is everything a running workflow needs from storage. +type runWork interface { + workerStore + runStarter +} + +// runStarter is what a step that starts another run needs from storage. +type runStarter interface { + workflowFinder + StartFor(ctx context.Context, token, parent uuid.UUID, n store.NewRun) (uuid.UUID, error) +} + // Engine starts, executes and resumes runs. type Engine struct { db *store.Store - work workerStore + work runWork log *slog.Logger opts Options mu sync.RWMutex @@ -68,7 +80,7 @@ func New(db *sqlx.DB, log *slog.Logger, opts Options) *Engine { return newEngine(s, s, log, opts) } -func newEngine(db *store.Store, work workerStore, log *slog.Logger, opts Options) *Engine { +func newEngine(db *store.Store, work runWork, log *slog.Logger, opts Options) *Engine { if opts.Workers <= 0 { opts.Workers = 4 } @@ -171,30 +183,45 @@ func (e *Engine) Has(ctx context.Context, orgID, name string) (bool, error) { // Start queues a run under the latest stored version of its workflow and // returns its id. func (e *Engine) Start(ctx context.Context, req StartRequest) (uuid.UUID, error) { - w, err := e.db.Workflow(ctx, req.OrgID, req.Workflow, 0) - if errors.Is(err, store.ErrNoWorkflow) { - return uuid.Nil, fmt.Errorf("no workflow named %q", req.Workflow) - } + n, err := newRun(ctx, e.db, req) if err != nil { return uuid.Nil, err } + return e.db.Start(ctx, n) +} + +// workflowFinder reads stored workflow versions. +type workflowFinder interface { + Workflow(ctx context.Context, orgID, name string, version int) (store.Workflow, error) +} + +// newRun checks that the latest version of the workflow can start with the +// request's input, and returns the run to queue. +func newRun(ctx context.Context, finder workflowFinder, req StartRequest) (store.NewRun, error) { + w, err := finder.Workflow(ctx, req.OrgID, req.Workflow, 0) + if errors.Is(err, store.ErrNoWorkflow) { + return store.NewRun{}, fmt.Errorf("no workflow named %q", req.Workflow) + } + if err != nil { + return store.NewRun{}, err + } doc, err := ParseDocument(w.Body) if err != nil { - return uuid.Nil, fmt.Errorf("workflow %q version %d: %w", req.Workflow, w.Version, err) + return store.NewRun{}, fmt.Errorf("workflow %q version %d: %w", req.Workflow, w.Version, err) } if err := doc.unsupported(); err != nil { - return uuid.Nil, err + return store.NewRun{}, err } input := req.Input if len(input) == 0 { input = jsontext.Value("null") } if err := doc.checkInputs(decodeInputs(input)); err != nil { - return uuid.Nil, fmt.Errorf("workflow %q: %w", req.Workflow, err) + return store.NewRun{}, fmt.Errorf("workflow %q: %w", req.Workflow, err) } - return e.db.Start(ctx, store.NewRun{ + return store.NewRun{ OrgID: req.OrgID, Workflow: req.Workflow, Version: w.Version, StartKey: req.Key, Input: input, - }) + }, nil } // CheckTrigger refuses a trigger for a workflow it couldn't start: one that diff --git a/modules/runs/runs_test.go b/modules/runs/runs_test.go index daadb972d..d409d59a5 100644 --- a/modules/runs/runs_test.go +++ b/modules/runs/runs_test.go @@ -31,6 +31,11 @@ type fakeWork struct { errors []string details []string results []jsontext.Value + // bodies are the stored workflows by name. A workflow not listed is a + // single notify step. + bodies map[string]string + // started is every run started for another, once per start key. + started []store.NewRun } func (f *fakeWork) Claim(context.Context, string, uuid.UUID, time.Duration) (store.Run, bool, error) { @@ -86,6 +91,30 @@ func (f *fakeWork) Signalled(context.Context, uuid.UUID, string) (jsontext.Value return nil, false, nil } +func (f *fakeWork) Workflow(_ context.Context, _, name string, _ int) (store.Workflow, error) { + body, ok := f.bodies[name] + if !ok { + body = "steps:\n - type: run.notify\n" + } + return store.Workflow{Name: name, Version: 1, Body: body}, nil +} + +// StartFor starts a run once per start key, as the store does. +func (f *fakeWork) StartFor(_ context.Context, _, _ uuid.UUID, n store.NewRun) (uuid.UUID, error) { + f.mu.Lock() + defer f.mu.Unlock() + if f.leaseLost { + return uuid.Nil, store.ErrLeaseLost + } + for _, started := range f.started { + if started.StartKey == n.StartKey { + return uuid.NewSHA1(uuid.NameSpaceURL, []byte(n.StartKey)), nil + } + } + f.started = append(f.started, n) + return uuid.NewSHA1(uuid.NameSpaceURL, []byte(n.StartKey)), nil +} + func (f *fakeWork) CancelRequested(context.Context, uuid.UUID) (bool, error) { return false, nil } diff --git a/modules/runs/steps_test.go b/modules/runs/steps_test.go index e5b9080d0..f11351f48 100644 --- a/modules/runs/steps_test.go +++ b/modules/runs/steps_test.go @@ -6,6 +6,7 @@ import ( "encoding/json/jsontext" json "encoding/json/v2" "errors" + "slices" "strings" "testing" @@ -106,13 +107,41 @@ func (n *notices) Notify(_ context.Context, msg deliveries.Notification) error { return nil } +// chats records the messages a workflow sends. +type chats struct{ sent []string } + +func (c *chats) Send(_ context.Context, channel, chatID, text, _ string) error { + c.sent = append(c.sent, channel+"/"+chatID+": "+text) + return nil +} + +// callWork is a worker store whose started child runs end when a test says +// so: until ending is set, a call waiting on its child has nothing to read. +type callWork struct { + signalAfter + ending jsontext.Value +} + +func (c *callWork) Signalled(ctx context.Context, id uuid.UUID, key string) (jsontext.Value, bool, error) { + if !strings.HasPrefix(key, callSignalPrefix) { + return c.signalAfter.Signalled(ctx, id, key) + } + return c.ending, c.ending != nil, nil +} + +const callSignalPrefix = "call:" + // TestDocumentStepsRun guards the workflow step contract on the engine: a // step's settings are rendered from the run's inputs and earlier results, its // condition decides whether it runs, a failure ends the run unless the step // continues, and every action judgement holds is put to people exactly once, // by an approval step or before the run ends. The first step that sets an // outcome ends the run, a failed step parks it, and a parked run retried from a -// step keeps the results of the steps before it. +// step keeps the results of the steps before it. A call starts one child run +// however often the run restarts, and a waiting call resumes from the child's +// ending; a handoff starts the next workflow on the same input; an approval +// delivers the plan and leaves the run waiting; a message is sent once, and +// not at all in a dry run. func TestDocumentStepsRun(t *testing.T) { input := jsontext.Value(`{"event":{"fields":{"host":"web-1"}}}`) tests := []struct { @@ -132,6 +161,23 @@ func TestDocumentStepsRun(t *testing.T) { wantAsked int wantExecuted int wantNotice string + // dry runs the workflow as a dry run. + dry bool + // bodies are the stored workflows a step can start. + bodies map[string]string + // crash kills the worker as it records its first step, so the run + // restarts having done that step's effect without recording it. + crash bool + // childEnding is how the started child ends: the run is expected to + // wait for it, and a restart then finds it ended. + childEnding string + // wantStarted is the child runs started, in all attempts, and + // wantChildInput the first one's input. + wantStarted int + wantChildInput string + // wantSignal is the key the first child run signals when it ends. + wantSignal string + wantSent []string }{ { name: "an action's input is rendered from the run's inputs", @@ -256,6 +302,101 @@ steps: levels: map[string]judgement.Level{"clean": judgement.ProposeAndWait, "restart": judgement.ProposeAndWait}, wantAssessed: 4, wantAsked: 2, wantExecuted: 2, }, + { + name: "a call that does not wait starts a child with its inputs and moves on", + doc: `inputs: {event: {type: object}} +steps: + - {id: child, type: workflow.call, settings: {workflow: remediate, inputs: {event: inputs.event, mode: fast}}} + - {type: run.notify, settings: {title: carried on}} +`, + bodies: map[string]string{"remediate": "inputs: {event: {type: object}, mode: {type: string}}\nsteps:\n - type: run.notify\n"}, + wantStarted: 1, wantChildInput: `{"event":{"fields":{"host":"web-1"}},"mode":"fast"}`, wantSignal: "call:child", wantNotice: "carried on", + }, + { + name: "a waiting call resumes after a restart without starting a second child", + doc: `steps: + - {id: child, type: workflow.call, settings: {workflow: remediate, wait: true}} + - {type: run.notify, settings: {title: "{{ steps.child.result.verdict }}"}} +`, + childEnding: `{"state":"succeeded","detail":"","error":"","result":{"verdict":"fixed"}}`, + wantStarted: 1, wantNotice: "fixed", + }, + { + name: "a call restarted after the worker died starts the child once", + doc: `steps: + - {id: child, type: workflow.call, settings: {workflow: remediate, wait: true}} + - {type: run.notify, settings: {title: "{{ steps.child.result.verdict }}"}} +`, + crash: true, + childEnding: `{"state":"succeeded","detail":"","error":"","result":{"verdict":"fixed"}}`, + wantStarted: 1, wantNotice: "fixed", + }, + { + name: "a waiting call parks the run when the child parks", + doc: `steps: + - {id: child, type: workflow.call, settings: {workflow: remediate, wait: true}} + - {type: run.notify, settings: {title: too late}} +`, + childEnding: `{"state":"parked","detail":"needs an operator","error":"","result":null}`, + wantStarted: 1, wantState: store.StateParked, wantDetail: "ended parked: needs an operator", + }, + { + name: "a call is not started in a dry run", + doc: `steps: + - {type: workflow.call, settings: {workflow: remediate, wait: true}} +`, + dry: true, + }, + { + name: "a handoff ends the run and starts the next workflow on the same input", + doc: `inputs: {event: {type: object}} +steps: + - {type: workflow.handoff, settings: {workflow: remediate, detail: needs a fix}} + - {type: run.notify, settings: {title: too late}} +`, + bodies: map[string]string{"remediate": "inputs: {event: {type: object}}\nsteps:\n - type: run.notify\n"}, + wantStarted: 1, wantChildInput: `{"event":{"fields":{"host":"web-1"}}}`, wantDetail: "needs a fix", + }, + { + name: "a handoff whose condition does not hold starts nothing", + doc: `inputs: {event: {type: object}} +steps: + - {type: workflow.handoff, when: "inputs.event.fields.missing", settings: {workflow: remediate}} + - {type: run.notify, settings: {title: stayed}} +`, + wantNotice: "stayed", + }, + { + name: "an approval delivers the plan and leaves the run waiting for approval", + doc: `steps: + - {type: workflow.approve, settings: {plan: restart the host, then: remediate}} + - {type: run.notify, settings: {title: too late}} +`, + wantState: store.StateWaitingApproval, wantDetail: "awaiting go/no-go", wantNotice: "Plan awaiting approval", + }, + { + name: "an approval needs a plan", + doc: `inputs: {event: {type: object}} +steps: + - {type: workflow.approve, settings: {document: "{{ inputs.event.fields.missing }}", then: remediate}} +`, + wantState: store.StateParked, wantDetail: "the plan is empty", + }, + { + name: "a message is sent with its text rendered", + doc: `inputs: {event: {type: object}} +steps: + - {type: message.send, settings: {channel: chat, chat_id: ops, text: "cleaning {{ inputs.event.fields.host }}"}} +`, + wantSent: []string{"chat/ops: cleaning web-1"}, + }, + { + name: "a message is not sent in a dry run", + doc: `steps: + - {type: message.send, settings: {channel: chat, chat_id: ops, text: hello}} +`, + dry: true, + }, { name: "a later step reads an earlier step's result", doc: `steps: @@ -275,11 +416,37 @@ steps: judge := &actionJudge{levels: tt.levels, fail: tt.fail, failAction: tt.failAction} ask := &askCounter{failFirst: tt.failFirst} told := ¬ices{} - wf := Deps{Settings: onePersonPath{}, Toolbox: anyBox{}, Judge: judge, Approver: ask, Notifier: told}.Compile(doc) - work := &signalAfter{arrives: 1} - run := store.Run{ID: uuid.New()} + sent := &chats{} + wf := Deps{Settings: onePersonPath{}, Toolbox: anyBox{}, Judge: judge, Approver: ask, Notifier: told, Messenger: sent}.Compile(doc) + work := &callWork{signalAfter: signalAfter{arrives: 1}} + work.bodies = tt.bodies + run := store.Run{ID: uuid.New(), Dry: tt.dry} rc := newContext(t.Context(), work, run, uuid.New(), nil) - if _, err = wf(rc, input); err != nil { + restart := func() { + rc = newContext(t.Context(), work, run, uuid.New(), slices.Clone(work.steps)) + } + if tt.crash { + first := 1 + work.rejectFrom = &first + } + _, err = wf(rc, input) + if tt.crash { + if !errors.Is(err, store.ErrLeaseLost) { + t.Fatalf("first run error = %v, want the lease lost", err) + } + work.rejectFrom = nil + restart() + _, err = wf(rc, input) + } + if tt.childEnding != "" { + if !errors.Is(err, errSuspended) { + t.Fatalf("waiting run error = %v, want it suspended", err) + } + work.ending = jsontext.Value(tt.childEnding) + restart() + _, err = wf(rc, input) + } + if err != nil { t.Fatalf("run error = %v", err) } if tt.retry != "" { @@ -314,6 +481,27 @@ steps: t.Fatalf("action input = %s, want %s", a, b) } } + if len(work.started) != tt.wantStarted { + t.Fatalf("started %d runs, want %d", len(work.started), tt.wantStarted) + } + if tt.wantChildInput != "" { + var got, want any + if err := json.Unmarshal(work.started[0].Input, &got); err != nil { + t.Fatal(err) + } + if err := json.Unmarshal([]byte(tt.wantChildInput), &want); err != nil { + t.Fatal(err) + } + if a, b := mustJSON(t, got), mustJSON(t, want); a != b { + t.Fatalf("child input = %s, want %s", a, b) + } + } + if tt.wantSignal != "" && work.started[0].ParentSignal != tt.wantSignal { + t.Fatalf("child signals %q, want %q", work.started[0].ParentSignal, tt.wantSignal) + } + if !slices.Equal(sent.sent, tt.wantSent) { + t.Fatalf("messages = %q, want %q", sent.sent, tt.wantSent) + } if tt.wantNotice != "" && (len(told.sent) != 1 || told.sent[0].Title != tt.wantNotice) { t.Fatalf("notices = %+v, want one titled %q", told.sent, tt.wantNotice) } diff --git a/modules/runs/vocabulary.go b/modules/runs/vocabulary.go index f2a070885..da2d42d2c 100644 --- a/modules/runs/vocabulary.go +++ b/modules/runs/vocabulary.go @@ -25,6 +25,10 @@ const ( typeNotify = "run.notify" typeSleep = "workflow.sleep" typeFinish = "workflow.finish" + typeCall = "workflow.call" + typeHandoff = "workflow.handoff" + typeApprove = "workflow.approve" + typeMessage = "message.send" ) // StepType defines a workflow step's settings and implementation. @@ -90,6 +94,10 @@ var stepTypes = vocabulary( define(typeNotify, "Report the findings", notifySettings.valid, notifySettings.describe, runNotify), define(typeSleep, "Wait", sleepSettings.valid, sleepSettings.describe, runSleep), define(typeFinish, "Finish", finishSettings.valid, finishSettings.describe, runFinish), + define(typeCall, "Call workflow", callSettings.valid, callSettings.describe, runCall), + define(typeHandoff, "Hand off", handoffSettings.valid, handoffSettings.describe, runHandoff), + define(typeApprove, "Approval", approveSettings.valid, approveSettings.describe, runApprove), + define(typeMessage, "Send message", messageSettings.valid, messageSettings.describe, runMessage), ) func vocabulary(types ...StepType) map[string]StepType {