Merge branch 'worktree-agent-a3fd604fba0fb36cc'

This commit is contained in:
sam committed 2026-11-21 13:55:00 +11:00
commit 3513dad609
14 files changed
+542 -75

No files matched your search

+41
View File
@@ -2199,6 +2199,13 @@ components:
- name
- expected_version
type: object
RetryRunInputBody:
additionalProperties: false
properties:
from:
description: The step to retry from, by id or as the trail names it; by default the step that parked the run
type: string
type: object
RollbackInputBody:
additionalProperties: false
properties:
@@ -2339,6 +2346,8 @@ components:
created_at:
format: date-time
type: string
detail:
type: string
error:
type: string
finished_at:
@@ -5100,6 +5109,38 @@ paths:
summary: Get a run's decision trail
tags:
- Runs
/api/runs/{id}/retry:
post:
operationId: retry-run
parameters:
- in: path
name: id
required: true
schema:
format: uuid
type: string
requestBody:
content:
application/json:
schema:
$ref: "#/components/schemas/RetryRunInputBody"
required: true
responses:
"200":
content:
application/json:
schema:
$ref: "#/components/schemas/RunsRunView"
description: OK
default:
content:
application/problem+json:
schema:
$ref: "#/components/schemas/ErrorModel"
description: Error
summary: Retry a parked run from a step, keeping the results of the steps before it
tags:
- Runs
/api/secret-engines:
get:
operationId: list-secret-engines
@@ -0,0 +1,17 @@
-- +goose Up
-- A run can end in four ways beyond success and failure: a step parks it for an
-- operator to retry, a workflow.finish step ends it waiting for approval or
-- declined, and detail carries what the ending says. A parked or waiting run is
-- not terminal: parked runs retry from a step.
ALTER TABLE runs.runs DROP CONSTRAINT runs_runs_state_check;
ALTER TABLE runs.runs ADD CONSTRAINT runs_runs_state_check
CHECK (state IN ('queued', 'running', 'waiting', 'succeeded', 'failed', 'cancelled',
'parked', 'waiting_approval', 'declined'));
ALTER TABLE runs.runs ADD COLUMN detail text;
-- +goose Down
UPDATE runs.runs SET state = 'failed' WHERE state IN ('parked', 'waiting_approval', 'declined');
ALTER TABLE runs.runs DROP COLUMN detail;
ALTER TABLE runs.runs DROP CONSTRAINT runs_runs_state_check;
ALTER TABLE runs.runs ADD CONSTRAINT runs_runs_state_check
CHECK (state IN ('queued', 'running', 'waiting', 'succeeded', 'failed', 'cancelled'));
+48 -4
View File
@@ -45,19 +45,29 @@ type Context struct {
// are numbered from nextMarker.
started map[string]bool
nextMarker int
// entered holds the top-level steps an earlier attempt reached.
entered map[string]bool
// ending is how the workflow ended when it ended other than by running out
// of steps, which is a success.
ending *Ending
// overlay is what a dry run carries instead of a stored version.
overlay *dryOverlay
}
// markerBase is where started markers are numbered, far above any step a
// workflow takes, so they never shift the positions a replay depends on.
const markerBase = 1 << 20
const markerBase = store.MarkerBase
const startedPrefix = "started:"
const (
startedPrefix = "started:"
// enteredPrefix names the marker that records the position a top-level
// step began at, so a parked run can retry from it.
enteredPrefix = "at:"
)
func newContext(ctx context.Context, work workerStore, run store.Run, token uuid.UUID, steps []store.Step) *Context {
saved := make(map[int]store.Step, len(steps))
started := map[string]bool{}
started, entered := map[string]bool{}, map[string]bool{}
nextMarker := markerBase
for _, s := range steps {
saved[s.Seq] = s
@@ -65,8 +75,12 @@ func newContext(ctx context.Context, work workerStore, run store.Run, token uuid
started[key] = true
nextMarker = max(nextMarker, s.Seq+1)
}
if name, ok := strings.CutPrefix(s.Name, enteredPrefix); ok {
entered[name] = true
nextMarker = max(nextMarker, s.Seq+1)
}
}
rc := &Context{ctx: ctx, work: work, run: run, token: token, saved: saved, started: started, nextMarker: nextMarker}
rc := &Context{ctx: ctx, work: work, run: run, token: token, saved: saved, started: started, entered: entered, nextMarker: nextMarker}
if run.Dry {
rc.overlay, _ = readOverlay(run.Overlay)
}
@@ -90,6 +104,36 @@ func (rc *Context) Begin(ctx context.Context, key string) (bool, error) {
return false, nil
}
// Enter records that the top-level step name begins at the run's next position,
// unless an earlier attempt already did. Retrying a parked run from the step
// forgets everything recorded from that position on.
func (rc *Context) Enter(ctx context.Context, name string) error {
if rc.entered[name] {
return nil
}
seq := rc.nextMarker
if _, err := rc.record(ctx, seq, enteredPrefix+name, rc.next, nil); err != nil && !errors.Is(err, store.ErrStepExists) {
return err
}
rc.entered[name] = true
rc.nextMarker = seq + 1
return nil
}
// End records how the workflow ends, ahead of its steps running out.
func (rc *Context) End(state, detail string) {
rc.ending = &Ending{State: state, Detail: detail}
}
// Ending is how a run that stopped before its steps ran out ended.
type Ending struct {
// State is the run state the ending sets.
State string
// Detail is what the ending says: a finish step's detail or why a step
// parked the run.
Detail string
}
// RunID is the run's id.
func (rc *Context) RunID() uuid.UUID { return rc.run.ID }
+9 -2
View File
@@ -58,8 +58,15 @@ func TestCorpusChecksAsItsStepTypesLand(t *testing.T) {
agentLater("gate", "gate", "protect_tests")),
},
{
name: "triage's references and conditions check; it waits for forge and outcome steps", file: "triage",
want: notYet("forge.comment", "forge.close-issue", "workflow.finish", "workflow.handoff"),
name: "triage's references and conditions check; it waits for forge and handoff steps", file: "triage",
want: notYet("forge.comment", "forge.close-issue", "workflow.handoff"),
},
{
name: "a finish step with an unknown status is refused when the workflow is saved",
yaml: `steps:
- {type: workflow.finish, settings: {status: finished}}
`,
want: []string{`status is completed, parked, waiting_approval or declined, not "finished"`},
},
{
name: "a result field the agent's schema doesn't declare is refused",
+90 -9
View File
@@ -22,8 +22,19 @@ const (
StateSucceeded = "succeeded"
StateFailed = "failed"
StateCancelled = "cancelled"
// StateParked is a run stopped for an operator, by a failed step or a
// workflow.finish step. It is retried from a step.
StateParked = "parked"
// StateWaitingApproval and StateDeclined are the other endings a
// workflow.finish step chooses.
StateWaitingApproval = "waiting_approval"
StateDeclined = "declined"
)
// MarkerBase is where a run's markers are numbered, far above any step a
// workflow takes, so a marker never shifts the position a replay depends on.
const MarkerBase = 1 << 20
// Step outcomes.
const (
OutcomeDone = "done"
@@ -37,6 +48,12 @@ var (
// ErrStepExists means the step was already recorded, by this run's replay
// or by a previous lease holder.
ErrStepExists = errors.New("step already recorded")
// ErrNoRun means the organisation has no such run to retry.
ErrNoRun = errors.New("no such run")
// ErrNotParked means the run is not parked, so there is nothing to retry.
ErrNotParked = errors.New("run is not parked")
// ErrNoStep means the run never reached the step it was to retry from.
ErrNoStep = errors.New("run did not reach that step")
)
// Run is a claimed run.
@@ -192,13 +209,23 @@ func (s *Store) Suspend(ctx context.Context, id, token uuid.UUID, wakeAt *time.T
return nil
}
// Finish records a terminal state while the token holds the lease.
func (s *Store) Finish(ctx context.Context, id, token uuid.UUID, state string, result jsontext.Value, errText string) error {
// Ended is how a run ended. Error is a failure's text and Detail what an
// ending says, such as a workflow.finish step's detail.
type Ended struct {
State string
Result jsontext.Value
Error string
Detail string
}
// Finish records how a run ended while the token holds the lease.
func (s *Store) Finish(ctx context.Context, id, token uuid.UUID, e Ended) error {
res, err := s.db.ExecContext(ctx, `
UPDATE runs.runs
SET state = $3, result = $4, error = NULLIF($5, ''), finished_at = now(), updated_at = now(),
SET state = $3, result = $4, error = NULLIF($5, ''), detail = NULLIF($6, ''),
finished_at = now(), updated_at = now(),
lease_token = NULL, lease_owner = NULL, lease_expires_at = NULL
WHERE id = $1 AND lease_token = $2`, id, token, state, nullJSON(result), errText)
WHERE id = $1 AND lease_token = $2`, id, token, e.State, nullJSON(e.Result), e.Error, e.Detail)
if err != nil {
return fmt.Errorf("finish run: %w", err)
}
@@ -208,6 +235,59 @@ func (s *Store) Finish(ctx context.Context, id, token uuid.UUID, state string, r
return nil
}
// Retry queues a parked run to run again from a step, keeping what the steps
// before it recorded: the run replays them without repeating their effects.
// from names the step; empty means the step the run last reached. What that
// step and every later one recorded is forgotten, including the markers that
// stop an effect starting twice, so the step runs afresh.
func (s *Store) Retry(ctx context.Context, orgID string, id uuid.UUID, from string) error {
tx, err := s.db.BeginTxx(ctx, nil)
if err != nil {
return fmt.Errorf("retry run: %w", err)
}
defer func() { _ = tx.Rollback() }()
var state string
err = tx.GetContext(ctx, &state, `SELECT state FROM runs.runs WHERE org_id = $1 AND id = $2 AND NOT dry FOR UPDATE`, orgID, id)
switch {
case errors.Is(err, sql.ErrNoRows):
return ErrNoRun
case err != nil:
return fmt.Errorf("retry run: %w", err)
case state != StateParked:
return ErrNotParked
}
var at struct {
Marker int `db:"marker"`
FirstSeq int `db:"first_seq"`
}
err = tx.GetContext(ctx, &at, `
SELECT seq AS marker, (output #>> '{}')::int AS first_seq FROM runs.steps
WHERE run_id = $1 AND name LIKE 'at:%' AND ($2 = '' OR name = 'at:' || $2)
ORDER BY seq DESC LIMIT 1`, id, from)
if errors.Is(err, sql.ErrNoRows) {
return ErrNoStep
}
if err != nil {
return fmt.Errorf("find step: %w", err)
}
if _, err := tx.ExecContext(ctx, `
DELETE FROM runs.steps WHERE run_id = $1 AND ((seq >= $2 AND seq < $3) OR seq >= $4)`,
id, at.FirstSeq, MarkerBase, at.Marker); err != nil {
return fmt.Errorf("forget steps: %w", err)
}
if _, err := tx.ExecContext(ctx, `
UPDATE runs.runs
SET state = 'queued', result = NULL, error = NULL, detail = NULL, finished_at = NULL,
wake_at = NULL, waiting_for = NULL, updated_at = now()
WHERE id = $1`, id); err != nil {
return fmt.Errorf("queue run: %w", err)
}
if err := tx.Commit(); err != nil {
return fmt.Errorf("retry run: %w", err)
}
return nil
}
// Signal stores a signal for a run and queues the run if it is waiting for it.
func (s *Store) Signal(ctx context.Context, id uuid.UUID, key string, payload jsontext.Value) error {
tx, err := s.db.BeginTxx(ctx, nil)
@@ -250,9 +330,9 @@ func (s *Store) Cancel(ctx context.Context, id uuid.UUID) error {
if _, err := s.db.ExecContext(ctx, `
UPDATE runs.runs
SET cancel_requested = true, updated_at = now(),
state = CASE WHEN state IN ('queued', 'waiting') THEN 'cancelled' ELSE state END,
finished_at = CASE WHEN state IN ('queued', 'waiting') THEN now() ELSE finished_at END
WHERE id = $1 AND state NOT IN ('succeeded', 'failed', 'cancelled')`, id); err != nil {
state = CASE WHEN state IN ('queued', 'waiting', 'parked', 'waiting_approval') THEN 'cancelled' ELSE state END,
finished_at = CASE WHEN state IN ('queued', 'waiting', 'parked', 'waiting_approval') THEN now() ELSE finished_at END
WHERE id = $1 AND state NOT IN ('succeeded', 'failed', 'cancelled', 'declined')`, id); err != nil {
return fmt.Errorf("cancel run: %w", err)
}
return nil
@@ -275,6 +355,7 @@ type Summary struct {
State string `db:"state"`
Attempts int `db:"attempts"`
Error sql.NullString `db:"error"`
Detail sql.NullString `db:"detail"`
CreatedAt time.Time `db:"created_at"`
FinishedAt sql.NullTime `db:"finished_at"`
Dry bool `db:"dry"`
@@ -299,7 +380,7 @@ type Recorded struct {
func (s *Store) Recent(ctx context.Context, orgID string, limit int) ([]Summary, error) {
var out []Summary
if err := s.db.SelectContext(ctx, &out, `
SELECT id, workflow, workflow_version, state, attempts, error, created_at, finished_at, dry FROM runs.runs
SELECT id, workflow, workflow_version, state, attempts, error, detail, created_at, finished_at, dry FROM runs.runs
WHERE org_id = $1 AND NOT dry ORDER BY created_at DESC LIMIT $2`, orgID, limit); err != nil {
return nil, fmt.Errorf("list runs: %w", err)
}
@@ -311,7 +392,7 @@ func (s *Store) Recent(ctx context.Context, orgID string, limit int) ([]Summary,
func (s *Store) Detail(ctx context.Context, orgID string, id uuid.UUID) (Detail, bool, error) {
var d Detail
err := s.db.GetContext(ctx, &d, `
SELECT id, workflow, workflow_version, state, attempts, error, created_at, finished_at, dry, input,
SELECT id, workflow, workflow_version, state, attempts, error, detail, created_at, finished_at, dry, input,
coalesce(result, 'null') AS result
FROM runs.runs WHERE org_id = $1 AND id = $2`, orgID, id)
if errors.Is(err, sql.ErrNoRows) {
+53 -13
View File
@@ -132,7 +132,10 @@ type progress struct {
agent *AgentResult
memory bool
remembered bool
outcomes []approvalOutcome
// last is what the most recent step that ran reported, which a run that
// reaches the end of its steps gives as its detail.
last string
outcomes []approvalOutcome
// settled is the actions already put to people, by digest, so a later
// approval step asks only about what was held since.
settled map[string]bool
@@ -157,26 +160,53 @@ func (d Deps) Compile(doc Document) Workflow {
return nil, err
}
}
for i, s := range doc.Steps {
if p.stopped && s.Type != typeNotify {
continue
}
if err := d.runStep(rc, p, s, i); err != nil {
return nil, err
}
if parked, err := d.runSteps(rc, p, doc.Steps); err != nil {
return nil, err
} else if parked {
return p.encode()
}
// An action still held when the document ends is put to people before
// the run finishes, so nothing judgement held runs unapproved or is
// silently dropped.
if settleAtEnd && !p.stopped {
// silently dropped. A run a finish step parks, declines or leaves
// waiting for approval ends as it was told to.
if settleAtEnd && !p.stopped && (rc.ending == nil || rc.ending.State == store.StateSucceeded) {
if _, err := d.settle(rc, p); err != nil {
return nil, err
}
}
if rc.ending == nil {
rc.End(store.StateSucceeded, p.last)
}
return p.result()
}
}
// runSteps runs the steps in order until one ends the run. It reports whether
// a step failed and parked the run, for an operator to retry from that step;
// a step that continues on failure does not.
func (d Deps) runSteps(rc *Context, p *progress, steps []Step) (parked bool, err error) {
for i, s := range steps {
if p.stopped && s.Type != typeNotify {
continue
}
name := s.name(i)
if err := rc.Enter(rc.Context(), name); err != nil {
return false, err
}
if err := d.runStep(rc, p, s, i); err != nil {
if !stepFailed(err) {
return false, err
}
rc.End(store.StateParked, fmt.Sprintf("step %s: %v", name, err))
return true, nil
}
if rc.ending != nil {
break
}
}
return false, nil
}
// runStep resolves settings from the run's inputs and earlier results, and
// saves the result under the step ID if the step runs.
func (d Deps) runStep(rc *Context, p *progress, s Step, index int) error {
@@ -202,6 +232,7 @@ func (d Deps) runStep(rc *Context, p *progress, s Step, index int) error {
if s.ID != "" {
p.results[s.ID] = res
}
p.last = res.Summary
return nil
}
@@ -409,6 +440,18 @@ func agentResult(name string, res AgentResult, returned jsontext.Value, schema *
// actions when no agent ran. An agent that did not complete fails the run once
// the owner has been told.
func (p *progress) result() (jsontext.Value, error) {
out, err := p.encode()
if err != nil {
return nil, err
}
if p.agent != nil && !p.agent.Completed {
return out, fmt.Errorf("investigation stopped: %s", p.agent.StopReason)
}
return out, nil
}
// encode is the run's findings so far, as the result a run records.
func (p *progress) encode() (jsontext.Value, error) {
var v any = p.outcomes
if p.agent != nil {
v = *p.agent
@@ -417,9 +460,6 @@ func (p *progress) result() (jsontext.Value, error) {
if err != nil {
return nil, fmt.Errorf("encode result: %w", err)
}
if p.agent != nil && !p.agent.Completed {
return out, fmt.Errorf("investigation stopped: %s", p.agent.StopReason)
}
return out, nil
}
+33 -13
View File
@@ -45,7 +45,7 @@ type workerStore interface {
Steps(ctx context.Context, id uuid.UUID) ([]store.Step, error)
RecordStep(ctx context.Context, token uuid.UUID, step store.Step) error
Suspend(ctx context.Context, id, token uuid.UUID, wakeAt *time.Time, waitingFor string) error
Finish(ctx context.Context, id, token uuid.UUID, state string, result jsontext.Value, errText string) error
Finish(ctx context.Context, id, token uuid.UUID, ended store.Ended) error
Signalled(ctx context.Context, id uuid.UUID, key string) (jsontext.Value, bool, error)
CancelRequested(ctx context.Context, id uuid.UUID) (bool, error)
}
@@ -234,6 +234,21 @@ func (e *Engine) Cancel(ctx context.Context, id uuid.UUID) error {
return e.db.Cancel(ctx, id)
}
// ErrNotRetryable is returned by Retry for a run that is not parked, or that
// never reached the step to retry from.
var ErrNotRetryable = errors.New("run can't be retried from there")
// Retry queues a parked run to run again from a top-level step, named by its id
// or, without one, its type and position as the trail shows it. An empty from
// is the step that parked the run. The steps before it keep their results.
func (e *Engine) Retry(ctx context.Context, orgID string, id uuid.UUID, from string) error {
err := e.db.Retry(ctx, orgID, id, from)
if errors.Is(err, store.ErrNotParked) || errors.Is(err, store.ErrNoStep) {
return fmt.Errorf("%w: %w", ErrNotRetryable, err)
}
return err
}
// Work runs the engine's workers until ctx ends.
func (e *Engine) Work(ctx context.Context) {
var wg sync.WaitGroup
@@ -277,7 +292,7 @@ func (e *Engine) execute(ctx context.Context, run store.Run, token uuid.UUID) {
go e.keepLease(runCtx, cancel, run.ID, token)
log := e.log.With("run", run.ID, "workflow", run.Workflow, "attempt", run.Attempts)
result, err := e.invoke(runCtx, run, token)
result, ending, err := e.invoke(runCtx, run, token)
switch {
case errors.Is(err, errSuspended):
log.InfoContext(ctx, "run suspended")
@@ -286,22 +301,25 @@ func (e *Engine) execute(ctx context.Context, run store.Run, token uuid.UUID) {
log.WarnContext(ctx, "run stopped: lease lost or shutting down")
return
}
state, errText := store.StateSucceeded, ""
if errors.Is(err, ErrCancelled) {
state, errText = store.StateCancelled, err.Error()
} else if err != nil {
state, errText = store.StateFailed, err.Error()
ended := store.Ended{State: store.StateSucceeded, Result: result}
switch {
case errors.Is(err, ErrCancelled):
ended.State, ended.Error = store.StateCancelled, err.Error()
case err != nil:
ended.State, ended.Error = store.StateFailed, err.Error()
case ending != nil:
ended.State, ended.Detail = ending.State, ending.Detail
}
if ferr := e.work.Finish(ctx, run.ID, token, state, result, errText); ferr != nil {
if ferr := e.work.Finish(ctx, run.ID, token, ended); ferr != nil {
log.WarnContext(ctx, "finish run", "error", ferr)
return
}
log.InfoContext(ctx, "run finished", "state", state)
log.InfoContext(ctx, "run finished", "state", ended.State)
}
// invoke runs the workflow, turning a panic in workflow or step code into a
// failure of this run rather than of the process.
func (e *Engine) invoke(ctx context.Context, run store.Run, token uuid.UUID) (result jsontext.Value, err error) {
func (e *Engine) invoke(ctx context.Context, run store.Run, token uuid.UUID) (result jsontext.Value, ending *Ending, err error) {
defer func() {
if p := recover(); p != nil {
err = fmt.Errorf("panic: %v", p)
@@ -309,13 +327,15 @@ func (e *Engine) invoke(ctx context.Context, run store.Run, token uuid.UUID) (re
}()
wf, err := e.load(ctx, run)
if err != nil {
return nil, err
return nil, nil, err
}
steps, err := e.work.Steps(ctx, run.ID)
if err != nil {
return nil, err
return nil, nil, err
}
return wf(newContext(ctx, e.work, run, token, steps), run.Input)
rc := newContext(ctx, e.work, run, token, steps)
result, err = wf(rc, run.Input)
return result, rc.ending, err
}
// loadDocument reads the document a run executes: the version it started
+6 -4
View File
@@ -29,6 +29,7 @@ type fakeWork struct {
steps []store.Step
finished []string
errors []string
details []string
results []jsontext.Value
}
@@ -68,15 +69,16 @@ func (f *fakeWork) Suspend(context.Context, uuid.UUID, uuid.UUID, *time.Time, st
return nil
}
func (f *fakeWork) Finish(_ context.Context, _, _ uuid.UUID, state string, result jsontext.Value, errText string) error {
func (f *fakeWork) Finish(_ context.Context, _, _ uuid.UUID, e store.Ended) error {
f.mu.Lock()
defer f.mu.Unlock()
if f.leaseLost {
return store.ErrLeaseLost
}
f.finished = append(f.finished, state)
f.errors = append(f.errors, errText)
f.results = append(f.results, result)
f.finished = append(f.finished, e.State)
f.errors = append(f.errors, e.Error)
f.details = append(f.details, e.Detail)
f.results = append(f.results, e.Result)
return nil
}
+125 -20
View File
@@ -1,10 +1,12 @@
package runs
import (
"cmp"
"context"
"encoding/json/jsontext"
json "encoding/json/v2"
"errors"
"strings"
"testing"
"github.com/google/uuid"
@@ -20,15 +22,17 @@ import (
// actionJudge decides each action at the level named for it and remembers
// every request, so a test sees what a step asked for.
type actionJudge struct {
levels map[string]judgement.Level
fail error
assessed []judgement.Request
decisions []judgement.Decision
executed int
levels map[string]judgement.Level
fail error
// failAction limits fail to the action with this name; empty fails them all.
failAction string
assessed []judgement.Request
decisions []judgement.Decision
executed int
}
func (j *actionJudge) Assess(_ context.Context, req judgement.Request) (judgement.Decision, error) {
if j.fail != nil {
if j.fail != nil && (j.failAction == "" || j.failAction == req.Action) {
return judgement.Decision{}, j.fail
}
j.assessed = append(j.assessed, req)
@@ -106,16 +110,23 @@ func (n *notices) Notify(_ context.Context, msg deliveries.Notification) error {
// step's settings are rendered from the run's inputs and earlier results, its
// condition decides whether it runs, a failure ends the run unless the step
// continues, and every action judgement holds is put to people exactly once,
// by an approval step or before the run ends.
// by an approval step or before the run ends. The first step that sets an
// outcome ends the run, a failed step parks it, and a parked run retried from a
// step keeps the results of the steps before it.
func TestDocumentStepsRun(t *testing.T) {
input := jsontext.Value(`{"event":{"fields":{"host":"web-1"}}}`)
tests := []struct {
name string
doc string
levels map[string]judgement.Level
fail error
failFirst int
wantErr bool
name string
doc string
levels map[string]judgement.Level
fail error
failAction string
failFirst int
// retry retries the parked run from the named step once the failure
// has cleared; the counts below cover both attempts.
retry string
wantState string
wantDetail string
wantInput string
wantAssessed int
wantAsked int
@@ -139,12 +150,64 @@ steps:
`,
},
{
name: "a failing step fails the run",
name: "a failing step parks the run and later steps do not run",
doc: `steps:
- {type: connector.action, settings: {connector: host, name: clean}}
- {type: run.notify}
`,
fail: errors.New("judgement is down"), wantErr: true,
fail: errors.New("judgement is down"), wantState: store.StateParked, wantDetail: "judgement is down",
},
{
name: "a retry from a step keeps the results of the steps before it",
doc: `steps:
- {id: first, type: connector.action, settings: {connector: host, name: clean}}
- {id: second, type: connector.action, settings: {connector: host, name: restart}}
- {type: run.notify, settings: {title: "{{ steps.first.summary }} then {{ steps.second.summary }}"}}
`,
levels: map[string]judgement.Level{"clean": judgement.ActQuietly, "restart": judgement.ActQuietly},
fail: errors.New("judgement is down"), failAction: "restart", retry: "second",
wantState: store.StateSucceeded, wantAssessed: 2, wantExecuted: 2, wantNotice: "done then done",
},
{
name: "a finish step ends the run with its status and detail",
doc: `steps:
- {type: workflow.finish, settings: {status: declined, detail: not worth fixing}}
- {type: run.notify, settings: {title: too late}}
`,
wantState: store.StateDeclined, wantDetail: "not worth fixing",
},
{
name: "a finish step whose condition does not hold lets the run go on",
doc: `inputs: {event: {type: object}}
steps:
- {type: workflow.finish, when: "inputs.event.fields.missing", settings: {status: parked, detail: stop}}
- {type: run.notify, settings: {title: carried on}}
`,
wantNotice: "carried on",
},
{
name: "a finish step can leave the run waiting for approval",
doc: `steps:
- {type: workflow.finish, settings: {status: waiting_approval, detail: needs a person}}
`,
wantState: store.StateWaitingApproval, wantDetail: "needs a person",
},
{
name: "a finish step may name the steps before it",
doc: `steps:
- {id: clean, type: connector.action, settings: {connector: host, name: clean}}
- {type: workflow.finish, settings: {detail: "cleaned: {{ steps.clean.summary }}"}}
`,
levels: map[string]judgement.Level{"clean": judgement.ActQuietly},
wantAssessed: 1, wantExecuted: 1, wantState: store.StateSucceeded, wantDetail: "cleaned: done",
},
{
name: "a run that reaches the end of its steps completes with the last summary",
doc: `steps:
- {id: clean, type: connector.action, settings: {connector: host, name: clean}}
`,
levels: map[string]judgement.Level{"clean": judgement.ActQuietly},
wantAssessed: 1, wantExecuted: 1, wantState: store.StateSucceeded, wantDetail: "done",
},
{
name: "a failing step that continues lets the run go on",
@@ -209,15 +272,31 @@ steps:
if err != nil {
t.Fatalf("ParseDocument: %v", err)
}
judge := &actionJudge{levels: tt.levels, fail: tt.fail}
judge := &actionJudge{levels: tt.levels, fail: tt.fail, failAction: tt.failAction}
ask := &askCounter{failFirst: tt.failFirst}
told := &notices{}
wf := Deps{Settings: onePersonPath{}, Toolbox: anyBox{}, Judge: judge, Approver: ask, Notifier: told}.Compile(doc)
work := &signalAfter{arrives: 1}
rc := newContext(t.Context(), work, store.Run{ID: uuid.New()}, uuid.New(), nil)
_, err = wf(rc, input)
if (err != nil) != tt.wantErr {
t.Fatalf("run error = %v, want error %v", err, tt.wantErr)
run := store.Run{ID: uuid.New()}
rc := newContext(t.Context(), work, run, uuid.New(), nil)
if _, err = wf(rc, input); err != nil {
t.Fatalf("run error = %v", err)
}
if tt.retry != "" {
if rc.ending == nil || rc.ending.State != store.StateParked {
t.Fatalf("ending = %+v, want the run parked before its retry", rc.ending)
}
judge.fail = nil
kept := stepsBeforeRetry(t, work.steps, tt.retry)
rc = newContext(t.Context(), work, run, uuid.New(), kept)
work.steps = kept
if _, err = wf(rc, input); err != nil {
t.Fatalf("retry error = %v", err)
}
}
wantState := cmp.Or(tt.wantState, store.StateSucceeded)
if rc.ending == nil || rc.ending.State != wantState || !strings.Contains(rc.ending.Detail, tt.wantDetail) {
t.Fatalf("ending = %+v, want %s with detail containing %q", rc.ending, wantState, tt.wantDetail)
}
if len(judge.assessed) != tt.wantAssessed || ask.asked != tt.wantAsked || judge.executed != tt.wantExecuted {
t.Fatalf("assessed %d, asked %d, executed %d; want %d, %d, %d",
@@ -242,6 +321,32 @@ steps:
}
}
// stepsBeforeRetry is what a retry from the named step keeps: the steps
// recorded before it began, and the markers written before it was entered. It
// mirrors store.Retry.
func stepsBeforeRetry(t *testing.T, steps []store.Step, from string) []store.Step {
t.Helper()
first, marker := -1, -1
for _, s := range steps {
if s.Name == enteredPrefix+from {
marker = s.Seq
if err := json.Unmarshal(s.Output, &first); err != nil {
t.Fatal(err)
}
}
}
if marker < 0 {
t.Fatalf("the run never reached step %q", from)
}
var kept []store.Step
for _, s := range steps {
if (s.Seq < first) || (s.Seq >= markerBase && s.Seq < marker) {
kept = append(kept, s)
}
}
return kept
}
func mustJSON(t *testing.T, v any) string {
t.Helper()
b, err := json.Marshal(v, json.Deterministic(true))
+44 -8
View File
@@ -4,6 +4,7 @@ import (
"context"
"encoding/json/jsontext"
json "encoding/json/v2"
"errors"
"net/http"
"strconv"
"strings"
@@ -39,12 +40,15 @@ type Origins interface {
// RunView is a run as the API shows it.
type RunView struct {
ID uuid.UUID `json:"id"`
Workflow string `json:"workflow"`
Version int `json:"workflow_version"`
State string `json:"state"`
Attempts int `json:"attempts"`
Error string `json:"error,omitzero"`
ID uuid.UUID `json:"id"`
Workflow string `json:"workflow"`
Version int `json:"workflow_version"`
State string `json:"state"`
Attempts int `json:"attempts"`
Error string `json:"error,omitzero"`
// Detail is what the run's ending says: a finish step's detail, or why a
// step parked the run.
Detail string `json:"detail,omitzero"`
CreatedAt time.Time `json:"created_at"`
FinishedAt *time.Time `json:"finished_at,omitzero"`
Result jsontext.Value `json:"result,omitzero"`
@@ -133,7 +137,7 @@ type Approvals interface {
// Register adds the run API for orgID: recent runs, each run's trail, and
// the workflows runs execute.
func (e *Engine) Register(api huma.API, orgID string, origins Origins, decisions Decisions, asked Approvals) {
a := &runAPI{db: e.db, orgID: orgID, origins: origins, decisions: decisions, approvals: asked}
a := &runAPI{engine: e, db: e.db, orgID: orgID, origins: origins, decisions: decisions, approvals: asked}
tags := []string{"Runs"}
huma.Register(api, huma.Operation{
OperationID: "list-runs", Method: http.MethodGet, Path: "/api/runs",
@@ -143,9 +147,40 @@ func (e *Engine) Register(api huma.API, orgID string, origins Origins, decisions
OperationID: "get-run", Method: http.MethodGet, Path: "/api/runs/{id}",
Summary: "Get a run's decision trail", Tags: tags,
}, a.trail)
huma.Register(api, huma.Operation{
OperationID: "retry-run", Method: http.MethodPost, Path: "/api/runs/{id}/retry",
Summary: "Retry a parked run from a step, keeping the results of the steps before it", Tags: tags,
}, a.retry)
e.registerWorkflows(api, orgID)
}
type retryRunInput struct {
ID string `path:"id" format:"uuid"`
Body struct {
From string `json:"from,omitempty" doc:"The step to retry from, by id or as the trail names it; by default the step that parked the run"`
}
}
func (a *runAPI) retry(ctx context.Context, in *retryRunInput) (*httpapi.Reply[RunView], error) {
id, err := uuid.Parse(in.ID)
if err != nil {
return nil, huma.NewError(http.StatusBadRequest, "a run id is a UUID")
}
switch err := a.engine.Retry(ctx, a.orgID, id, in.Body.From); {
case errors.Is(err, store.ErrNoRun):
return nil, huma.NewError(http.StatusNotFound, "no such run")
case errors.Is(err, ErrNotRetryable):
return nil, huma.NewError(http.StatusConflict, "only a parked run retries, and only from a step it reached")
case err != nil:
return nil, huma.NewError(http.StatusInternalServerError, "couldn't retry the run")
}
d, ok, err := a.db.Detail(ctx, a.orgID, id)
if err != nil || !ok {
return nil, huma.NewError(http.StatusInternalServerError, "couldn't read the run")
}
return httpapi.OK(view(d.Summary, d.Result)), nil
}
type recentRunsInput struct {
Limit int `query:"limit" minimum:"1" maximum:"200" default:"50"`
}
@@ -155,6 +190,7 @@ type runInput struct {
}
type runAPI struct {
engine *Engine
db *store.Store
orgID string
origins Origins
@@ -309,7 +345,7 @@ func effectOf(s store.Recorded) *Effect {
func view(s store.Summary, result jsontext.Value) RunView {
v := RunView{
ID: s.ID, Workflow: s.Workflow, Version: s.Version, State: s.State, Attempts: s.Attempts,
Error: s.Error.String, CreatedAt: s.CreatedAt, Result: result,
Error: s.Error.String, Detail: s.Detail.String, CreatedAt: s.CreatedAt, Result: result,
}
if s.FinishedAt.Valid {
v.FinishedAt = &s.FinishedAt.Time
+48
View File
@@ -14,6 +14,7 @@ import (
"github.com/samcharles93/archie-core/modules/deliveries"
"github.com/samcharles93/archie-core/modules/governance"
"github.com/samcharles93/archie-core/modules/judgement"
"github.com/samcharles93/archie-core/modules/runs/internal/store"
)
// The step types the engine runs.
@@ -23,6 +24,7 @@ const (
typeApproval = "approval.wait"
typeNotify = "run.notify"
typeSleep = "workflow.sleep"
typeFinish = "workflow.finish"
)
// StepType defines a workflow step's settings and implementation.
@@ -87,6 +89,7 @@ var stepTypes = vocabulary(
}, runApprovalWait),
define(typeNotify, "Report the findings", notifySettings.valid, notifySettings.describe, runNotify),
define(typeSleep, "Wait", sleepSettings.valid, sleepSettings.describe, runSleep),
define(typeFinish, "Finish", finishSettings.valid, finishSettings.describe, runFinish),
)
func vocabulary(types ...StepType) map[string]StepType {
@@ -299,6 +302,51 @@ func runSleep(_ Deps, rc *Context, _ *progress, _ string, s sleepSettings) (Step
return StepResult{}, rc.Sleep(s.duration())
}
// finishStatuses are the outcomes a finish step chooses, with the run state
// each one sets.
var finishStatuses = map[string]string{
"completed": store.StateSucceeded,
"parked": store.StateParked,
"waiting_approval": store.StateWaitingApproval,
"declined": store.StateDeclined,
}
// finishSettings end the run with an outcome.
type finishSettings struct {
Status string `yaml:"status,omitempty" json:"status,omitempty" jsonschema:"how the run ends: completed (the default), parked, waiting_approval or declined"`
Detail string `yaml:"detail,omitempty" json:"detail,omitempty" jsonschema:"what the operator reads about the outcome"`
}
func (s finishSettings) valid() error {
if s.Status != "" {
if _, ok := finishStatuses[s.Status]; !ok {
return fmt.Errorf("status is completed, parked, waiting_approval or declined, not %q", s.Status)
}
}
return nil
}
func (s finishSettings) state() string {
if s.Status == "" {
return store.StateSucceeded
}
return finishStatuses[s.Status]
}
func (s finishSettings) describe() string {
status := "completed"
if s.Status != "" {
status = s.Status
}
return "Ends the run as " + status + "; later steps do not run"
}
// runFinish sets the run's outcome: the first step that does ends the run.
func runFinish(_ Deps, rc *Context, _ *progress, _ string, s finishSettings) (StepResult, error) {
rc.End(s.state(), s.Detail)
return StepResult{Summary: s.Detail}, nil
}
// holds reports whether a step type can propose an action judgement may hold
// for approval.
func holds(stepType string) bool {
+5
View File
@@ -187,6 +187,9 @@ export const api = {
// The archie binary's module API; the wire types are below the object.
runs: (limit = 200) => request<Run[]>("/api/runs" + qs({ limit })),
run: (id: string) => request<RunTrail>(`/api/runs/${encodeURIComponent(id)}`),
/** Queues a parked run again from the step that parked it, or from `from`. */
retryRun: (id: string, from?: string) =>
request<Run>(`/api/runs/${encodeURIComponent(id)}/retry`, { method: "POST", body: from ? { from } : {} }),
approvals: () => request<Approval[]>("/api/approvals"),
answerApproval: (id: string, approved: boolean) =>
request<Approval>(`/api/approvals/${encodeURIComponent(id)}/answer`, { method: "POST", body: { approved } }),
@@ -662,6 +665,8 @@ export interface Run {
state: string;
attempts: number;
error?: string;
/** What the run's ending says: a finish step's detail, or why a step parked it. */
detail?: string;
created_at: string;
finished_at?: string;
result?: unknown;
+21 -2
View File
@@ -1,17 +1,19 @@
<script setup lang="ts">
import { computed, onMounted } from "vue";
import { computed, onMounted, ref } from "vue";
import { RouterLink, useRoute } from "vue-router";
import CodeView from "@/base/CodeView.vue";
import PageHeader from "@/base/PageHeader.vue";
import { Badge } from "@/components/ui/badge";
import { Button } from "@/components/ui/button";
import { Empty, EmptyHeader, EmptyTitle } from "@/components/ui/empty";
import { MetadataList } from "@/components/ui/metadata-list";
import { ReadError } from "@/components/ui/read-error";
import { Section } from "@/components/ui/section";
import { Skeleton } from "@/components/ui/skeleton";
import { Timestamp } from "@/components/ui/timestamp";
import { api, type RunAction } from "@/lib/api";
import { errors } from "@/copy/errors";
import { api, describeError, type RunAction } from "@/lib/api";
import { usePoll, useRead } from "@/lib/use-read";
import { useLiveResource } from "@/stores/session";
import ApprovalsSection from "./ApprovalsSection.vue";
@@ -37,8 +39,23 @@ const details = computed(() => {
{ label: "Attempts", value: String(run.attempts) },
{ label: "Run", value: run.id, mono: true },
...(run.error ? [{ label: "Error", value: run.error }] : []),
...(run.detail ? [{ label: "Detail", value: run.detail }] : []),
];
});
const retrying = ref(false);
const retryError = ref("");
async function retry(): Promise<void> {
retrying.value = true;
retryError.value = "";
try {
await api.retryRun(id);
await reload();
} catch (cause) {
retryError.value = describeError(cause, errors.common.unknown);
} finally {
retrying.value = false;
}
}
const origin = computed(() => {
const o = data.value?.origin;
if (!o) return [];
@@ -64,10 +81,12 @@ const result = computed(() =>
<PageHeader :title="data ? data.run.workflow : 'Run'">
<Badge v-if="data" :variant="stateTone(data.run.state)">{{ data.run.state }}</Badge>
<span v-if="data" class="text-caption text-fg-muted">Started <Timestamp :at="data.run.created_at" /></span>
<Button v-if="data?.run.state === 'parked'" size="sm" :disabled="retrying" @click="retry">Retry</Button>
</PageHeader>
<ReadError v-if="error" :message="error" retry-label="Retry" @retry="reload" />
<div v-else-if="!data" role="status" aria-label="Loading run" class="space-y-2"><Skeleton class="h-24 w-full" /><Skeleton class="h-48 w-full" /></div>
<template v-else>
<p v-if="retryError" role="alert" class="mb-2 text-caption text-danger">{{ retryError }}</p>
<ApprovalsSection :run="id" @answered="reload" />
<div class="grid gap-x-4 lg:grid-cols-2">
<Section title="Run"><MetadataList :rows="details" /></Section>
+2
View File
@@ -7,6 +7,8 @@ export function stateTone(state: string): "ok" | "warn" | "danger" | "info" | "i
case "done":
return "ok";
case "waiting":
case "waiting_approval":
case "parked":
case "pending":
case "unknown":
return "warn";