diff --git a/api/openapi.yaml b/api/openapi.yaml index 23ddcd232..a5945ee24 100644 --- a/api/openapi.yaml +++ b/api/openapi.yaml @@ -2172,6 +2172,13 @@ components: - name - expected_version type: object + RetryRunInputBody: + additionalProperties: false + properties: + from: + description: The step to retry from, by id or as the trail names it; by default the step that parked the run + type: string + type: object RollbackInputBody: additionalProperties: false properties: @@ -2312,6 +2319,8 @@ components: created_at: format: date-time type: string + detail: + type: string error: type: string finished_at: @@ -5073,6 +5082,38 @@ paths: summary: Get a run's decision trail tags: - Runs + /api/runs/{id}/retry: + post: + operationId: retry-run + parameters: + - in: path + name: id + required: true + schema: + format: uuid + type: string + requestBody: + content: + application/json: + schema: + $ref: "#/components/schemas/RetryRunInputBody" + required: true + responses: + "200": + content: + application/json: + schema: + $ref: "#/components/schemas/RunsRunView" + description: OK + default: + content: + application/problem+json: + schema: + $ref: "#/components/schemas/ErrorModel" + description: Error + summary: Retry a parked run from a step, keeping the results of the steps before it + tags: + - Runs /api/secret-engines: get: operationId: list-secret-engines diff --git a/internal/platform/pg/migrations/00044_runs_endings.sql b/internal/platform/pg/migrations/00044_runs_endings.sql new file mode 100644 index 000000000..631c79d25 --- /dev/null +++ b/internal/platform/pg/migrations/00044_runs_endings.sql @@ -0,0 +1,17 @@ +-- +goose Up +-- A run can end in four ways beyond success and failure: a step parks it for an +-- operator to retry, a workflow.finish step ends it waiting for approval or +-- declined, and detail carries what the ending says. A parked or waiting run is +-- not terminal: parked runs retry from a step. +ALTER TABLE runs.runs DROP CONSTRAINT runs_runs_state_check; +ALTER TABLE runs.runs ADD CONSTRAINT runs_runs_state_check + CHECK (state IN ('queued', 'running', 'waiting', 'succeeded', 'failed', 'cancelled', + 'parked', 'waiting_approval', 'declined')); +ALTER TABLE runs.runs ADD COLUMN detail text; + +-- +goose Down +UPDATE runs.runs SET state = 'failed' WHERE state IN ('parked', 'waiting_approval', 'declined'); +ALTER TABLE runs.runs DROP COLUMN detail; +ALTER TABLE runs.runs DROP CONSTRAINT runs_runs_state_check; +ALTER TABLE runs.runs ADD CONSTRAINT runs_runs_state_check + CHECK (state IN ('queued', 'running', 'waiting', 'succeeded', 'failed', 'cancelled')); diff --git a/modules/runs/context.go b/modules/runs/context.go index 376734e9b..0909038f5 100644 --- a/modules/runs/context.go +++ b/modules/runs/context.go @@ -45,19 +45,29 @@ type Context struct { // are numbered from nextMarker. started map[string]bool nextMarker int + // entered holds the top-level steps an earlier attempt reached. + entered map[string]bool + // ending is how the workflow ended when it ended other than by running out + // of steps, which is a success. + ending *Ending // overlay is what a dry run carries instead of a stored version. overlay *dryOverlay } // markerBase is where started markers are numbered, far above any step a // workflow takes, so they never shift the positions a replay depends on. -const markerBase = 1 << 20 +const markerBase = store.MarkerBase -const startedPrefix = "started:" +const ( + startedPrefix = "started:" + // enteredPrefix names the marker that records the position a top-level + // step began at, so a parked run can retry from it. + enteredPrefix = "at:" +) func newContext(ctx context.Context, work workerStore, run store.Run, token uuid.UUID, steps []store.Step) *Context { saved := make(map[int]store.Step, len(steps)) - started := map[string]bool{} + started, entered := map[string]bool{}, map[string]bool{} nextMarker := markerBase for _, s := range steps { saved[s.Seq] = s @@ -65,8 +75,12 @@ func newContext(ctx context.Context, work workerStore, run store.Run, token uuid started[key] = true nextMarker = max(nextMarker, s.Seq+1) } + if name, ok := strings.CutPrefix(s.Name, enteredPrefix); ok { + entered[name] = true + nextMarker = max(nextMarker, s.Seq+1) + } } - rc := &Context{ctx: ctx, work: work, run: run, token: token, saved: saved, started: started, nextMarker: nextMarker} + rc := &Context{ctx: ctx, work: work, run: run, token: token, saved: saved, started: started, entered: entered, nextMarker: nextMarker} if run.Dry { rc.overlay, _ = readOverlay(run.Overlay) } @@ -90,6 +104,36 @@ func (rc *Context) Begin(ctx context.Context, key string) (bool, error) { return false, nil } +// Enter records that the top-level step name begins at the run's next position, +// unless an earlier attempt already did. Retrying a parked run from the step +// forgets everything recorded from that position on. +func (rc *Context) Enter(ctx context.Context, name string) error { + if rc.entered[name] { + return nil + } + seq := rc.nextMarker + if _, err := rc.record(ctx, seq, enteredPrefix+name, rc.next, nil); err != nil && !errors.Is(err, store.ErrStepExists) { + return err + } + rc.entered[name] = true + rc.nextMarker = seq + 1 + return nil +} + +// End records how the workflow ends, ahead of its steps running out. +func (rc *Context) End(state, detail string) { + rc.ending = &Ending{State: state, Detail: detail} +} + +// Ending is how a run that stopped before its steps ran out ended. +type Ending struct { + // State is the run state the ending sets. + State string + // Detail is what the ending says: a finish step's detail or why a step + // parked the run. + Detail string +} + // RunID is the run's id. func (rc *Context) RunID() uuid.UUID { return rc.run.ID } diff --git a/modules/runs/corpus_test.go b/modules/runs/corpus_test.go index 647c77fc4..46d093587 100644 --- a/modules/runs/corpus_test.go +++ b/modules/runs/corpus_test.go @@ -58,8 +58,15 @@ func TestCorpusChecksAsItsStepTypesLand(t *testing.T) { agentLater("gate", "gate", "protect_tests")), }, { - name: "triage's references and conditions check; it waits for forge and outcome steps", file: "triage", - want: notYet("forge.comment", "forge.close-issue", "workflow.finish", "workflow.handoff"), + name: "triage's references and conditions check; it waits for forge and handoff steps", file: "triage", + want: notYet("forge.comment", "forge.close-issue", "workflow.handoff"), + }, + { + name: "a finish step with an unknown status is refused when the workflow is saved", + yaml: `steps: + - {type: workflow.finish, settings: {status: finished}} +`, + want: []string{`status is completed, parked, waiting_approval or declined, not "finished"`}, }, { name: "a result field the agent's schema doesn't declare is refused", diff --git a/modules/runs/internal/store/store.go b/modules/runs/internal/store/store.go index 7d5477ae4..366e1b025 100644 --- a/modules/runs/internal/store/store.go +++ b/modules/runs/internal/store/store.go @@ -22,8 +22,19 @@ const ( StateSucceeded = "succeeded" StateFailed = "failed" StateCancelled = "cancelled" + // StateParked is a run stopped for an operator, by a failed step or a + // workflow.finish step. It is retried from a step. + StateParked = "parked" + // StateWaitingApproval and StateDeclined are the other endings a + // workflow.finish step chooses. + StateWaitingApproval = "waiting_approval" + StateDeclined = "declined" ) +// MarkerBase is where a run's markers are numbered, far above any step a +// workflow takes, so a marker never shifts the position a replay depends on. +const MarkerBase = 1 << 20 + // Step outcomes. const ( OutcomeDone = "done" @@ -37,6 +48,12 @@ var ( // ErrStepExists means the step was already recorded, by this run's replay // or by a previous lease holder. ErrStepExists = errors.New("step already recorded") + // ErrNoRun means the organisation has no such run to retry. + ErrNoRun = errors.New("no such run") + // ErrNotParked means the run is not parked, so there is nothing to retry. + ErrNotParked = errors.New("run is not parked") + // ErrNoStep means the run never reached the step it was to retry from. + ErrNoStep = errors.New("run did not reach that step") ) // Run is a claimed run. @@ -192,13 +209,23 @@ func (s *Store) Suspend(ctx context.Context, id, token uuid.UUID, wakeAt *time.T return nil } -// Finish records a terminal state while the token holds the lease. -func (s *Store) Finish(ctx context.Context, id, token uuid.UUID, state string, result jsontext.Value, errText string) error { +// Ended is how a run ended. Error is a failure's text and Detail what an +// ending says, such as a workflow.finish step's detail. +type Ended struct { + State string + Result jsontext.Value + Error string + Detail string +} + +// Finish records how a run ended while the token holds the lease. +func (s *Store) Finish(ctx context.Context, id, token uuid.UUID, e Ended) error { res, err := s.db.ExecContext(ctx, ` UPDATE runs.runs - SET state = $3, result = $4, error = NULLIF($5, ''), finished_at = now(), updated_at = now(), + SET state = $3, result = $4, error = NULLIF($5, ''), detail = NULLIF($6, ''), + finished_at = now(), updated_at = now(), lease_token = NULL, lease_owner = NULL, lease_expires_at = NULL - WHERE id = $1 AND lease_token = $2`, id, token, state, nullJSON(result), errText) + WHERE id = $1 AND lease_token = $2`, id, token, e.State, nullJSON(e.Result), e.Error, e.Detail) if err != nil { return fmt.Errorf("finish run: %w", err) } @@ -208,6 +235,59 @@ func (s *Store) Finish(ctx context.Context, id, token uuid.UUID, state string, r return nil } +// Retry queues a parked run to run again from a step, keeping what the steps +// before it recorded: the run replays them without repeating their effects. +// from names the step; empty means the step the run last reached. What that +// step and every later one recorded is forgotten, including the markers that +// stop an effect starting twice, so the step runs afresh. +func (s *Store) Retry(ctx context.Context, orgID string, id uuid.UUID, from string) error { + tx, err := s.db.BeginTxx(ctx, nil) + if err != nil { + return fmt.Errorf("retry run: %w", err) + } + defer func() { _ = tx.Rollback() }() + var state string + err = tx.GetContext(ctx, &state, `SELECT state FROM runs.runs WHERE org_id = $1 AND id = $2 AND NOT dry FOR UPDATE`, orgID, id) + switch { + case errors.Is(err, sql.ErrNoRows): + return ErrNoRun + case err != nil: + return fmt.Errorf("retry run: %w", err) + case state != StateParked: + return ErrNotParked + } + var at struct { + Marker int `db:"marker"` + FirstSeq int `db:"first_seq"` + } + err = tx.GetContext(ctx, &at, ` + SELECT seq AS marker, (output #>> '{}')::int AS first_seq FROM runs.steps + WHERE run_id = $1 AND name LIKE 'at:%' AND ($2 = '' OR name = 'at:' || $2) + ORDER BY seq DESC LIMIT 1`, id, from) + if errors.Is(err, sql.ErrNoRows) { + return ErrNoStep + } + if err != nil { + return fmt.Errorf("find step: %w", err) + } + if _, err := tx.ExecContext(ctx, ` + DELETE FROM runs.steps WHERE run_id = $1 AND ((seq >= $2 AND seq < $3) OR seq >= $4)`, + id, at.FirstSeq, MarkerBase, at.Marker); err != nil { + return fmt.Errorf("forget steps: %w", err) + } + if _, err := tx.ExecContext(ctx, ` + UPDATE runs.runs + SET state = 'queued', result = NULL, error = NULL, detail = NULL, finished_at = NULL, + wake_at = NULL, waiting_for = NULL, updated_at = now() + WHERE id = $1`, id); err != nil { + return fmt.Errorf("queue run: %w", err) + } + if err := tx.Commit(); err != nil { + return fmt.Errorf("retry run: %w", err) + } + return nil +} + // Signal stores a signal for a run and queues the run if it is waiting for it. func (s *Store) Signal(ctx context.Context, id uuid.UUID, key string, payload jsontext.Value) error { tx, err := s.db.BeginTxx(ctx, nil) @@ -250,9 +330,9 @@ func (s *Store) Cancel(ctx context.Context, id uuid.UUID) error { if _, err := s.db.ExecContext(ctx, ` UPDATE runs.runs SET cancel_requested = true, updated_at = now(), - state = CASE WHEN state IN ('queued', 'waiting') THEN 'cancelled' ELSE state END, - finished_at = CASE WHEN state IN ('queued', 'waiting') THEN now() ELSE finished_at END - WHERE id = $1 AND state NOT IN ('succeeded', 'failed', 'cancelled')`, id); err != nil { + state = CASE WHEN state IN ('queued', 'waiting', 'parked', 'waiting_approval') THEN 'cancelled' ELSE state END, + finished_at = CASE WHEN state IN ('queued', 'waiting', 'parked', 'waiting_approval') THEN now() ELSE finished_at END + WHERE id = $1 AND state NOT IN ('succeeded', 'failed', 'cancelled', 'declined')`, id); err != nil { return fmt.Errorf("cancel run: %w", err) } return nil @@ -275,6 +355,7 @@ type Summary struct { State string `db:"state"` Attempts int `db:"attempts"` Error sql.NullString `db:"error"` + Detail sql.NullString `db:"detail"` CreatedAt time.Time `db:"created_at"` FinishedAt sql.NullTime `db:"finished_at"` Dry bool `db:"dry"` @@ -299,7 +380,7 @@ type Recorded struct { func (s *Store) Recent(ctx context.Context, orgID string, limit int) ([]Summary, error) { var out []Summary if err := s.db.SelectContext(ctx, &out, ` - SELECT id, workflow, workflow_version, state, attempts, error, created_at, finished_at, dry FROM runs.runs + SELECT id, workflow, workflow_version, state, attempts, error, detail, created_at, finished_at, dry FROM runs.runs WHERE org_id = $1 AND NOT dry ORDER BY created_at DESC LIMIT $2`, orgID, limit); err != nil { return nil, fmt.Errorf("list runs: %w", err) } @@ -311,7 +392,7 @@ func (s *Store) Recent(ctx context.Context, orgID string, limit int) ([]Summary, func (s *Store) Detail(ctx context.Context, orgID string, id uuid.UUID) (Detail, bool, error) { var d Detail err := s.db.GetContext(ctx, &d, ` - SELECT id, workflow, workflow_version, state, attempts, error, created_at, finished_at, dry, input, + SELECT id, workflow, workflow_version, state, attempts, error, detail, created_at, finished_at, dry, input, coalesce(result, 'null') AS result FROM runs.runs WHERE org_id = $1 AND id = $2`, orgID, id) if errors.Is(err, sql.ErrNoRows) { diff --git a/modules/runs/interpreter.go b/modules/runs/interpreter.go index 516abe1b5..50a0d7481 100644 --- a/modules/runs/interpreter.go +++ b/modules/runs/interpreter.go @@ -132,7 +132,10 @@ type progress struct { agent *AgentResult memory bool remembered bool - outcomes []approvalOutcome + // last is what the most recent step that ran reported, which a run that + // reaches the end of its steps gives as its detail. + last string + outcomes []approvalOutcome // settled is the actions already put to people, by digest, so a later // approval step asks only about what was held since. settled map[string]bool @@ -161,18 +164,35 @@ func (d Deps) Compile(doc Document) Workflow { if p.stopped && s.Type != typeNotify { continue } - if err := d.runStep(rc, p, s, i); err != nil { + name := s.name(i) + if err := rc.Enter(rc.Context(), name); err != nil { return nil, err } + if err := d.runStep(rc, p, s, i); err != nil { + if !stepFailed(err) { + return nil, err + } + // A failed step parks the run for an operator to retry from + // it, unless the step continues on failure. + rc.End(store.StateParked, fmt.Sprintf("step %s: %v", name, err)) + return p.encode() + } + if rc.ending != nil { + break + } } // An action still held when the document ends is put to people before // the run finishes, so nothing judgement held runs unapproved or is - // silently dropped. - if settleAtEnd && !p.stopped { + // silently dropped. A run a finish step parks, declines or leaves + // waiting for approval ends as it was told to. + if settleAtEnd && !p.stopped && (rc.ending == nil || rc.ending.State == store.StateSucceeded) { if _, err := d.settle(rc, p); err != nil { return nil, err } } + if rc.ending == nil { + rc.End(store.StateSucceeded, p.last) + } return p.result() } } @@ -202,6 +222,7 @@ func (d Deps) runStep(rc *Context, p *progress, s Step, index int) error { if s.ID != "" { p.results[s.ID] = res } + p.last = res.Summary return nil } @@ -409,6 +430,18 @@ func agentResult(name string, res AgentResult, returned jsontext.Value, schema * // actions when no agent ran. An agent that did not complete fails the run once // the owner has been told. func (p *progress) result() (jsontext.Value, error) { + out, err := p.encode() + if err != nil { + return nil, err + } + if p.agent != nil && !p.agent.Completed { + return out, fmt.Errorf("investigation stopped: %s", p.agent.StopReason) + } + return out, nil +} + +// encode is the run's findings so far, as the result a run records. +func (p *progress) encode() (jsontext.Value, error) { var v any = p.outcomes if p.agent != nil { v = *p.agent @@ -417,9 +450,6 @@ func (p *progress) result() (jsontext.Value, error) { if err != nil { return nil, fmt.Errorf("encode result: %w", err) } - if p.agent != nil && !p.agent.Completed { - return out, fmt.Errorf("investigation stopped: %s", p.agent.StopReason) - } return out, nil } diff --git a/modules/runs/runs.go b/modules/runs/runs.go index a77453b35..96f344e26 100644 --- a/modules/runs/runs.go +++ b/modules/runs/runs.go @@ -45,7 +45,7 @@ type workerStore interface { Steps(ctx context.Context, id uuid.UUID) ([]store.Step, error) RecordStep(ctx context.Context, token uuid.UUID, step store.Step) error Suspend(ctx context.Context, id, token uuid.UUID, wakeAt *time.Time, waitingFor string) error - Finish(ctx context.Context, id, token uuid.UUID, state string, result jsontext.Value, errText string) error + Finish(ctx context.Context, id, token uuid.UUID, ended store.Ended) error Signalled(ctx context.Context, id uuid.UUID, key string) (jsontext.Value, bool, error) CancelRequested(ctx context.Context, id uuid.UUID) (bool, error) } @@ -234,6 +234,21 @@ func (e *Engine) Cancel(ctx context.Context, id uuid.UUID) error { return e.db.Cancel(ctx, id) } +// ErrNotRetryable is returned by Retry for a run that is not parked, or that +// never reached the step to retry from. +var ErrNotRetryable = errors.New("run can't be retried from there") + +// Retry queues a parked run to run again from a top-level step, named by its id +// or, without one, its type and position as the trail shows it. An empty from +// is the step that parked the run. The steps before it keep their results. +func (e *Engine) Retry(ctx context.Context, orgID string, id uuid.UUID, from string) error { + err := e.db.Retry(ctx, orgID, id, from) + if errors.Is(err, store.ErrNotParked) || errors.Is(err, store.ErrNoStep) { + return fmt.Errorf("%w: %w", ErrNotRetryable, err) + } + return err +} + // Work runs the engine's workers until ctx ends. func (e *Engine) Work(ctx context.Context) { var wg sync.WaitGroup @@ -277,7 +292,7 @@ func (e *Engine) execute(ctx context.Context, run store.Run, token uuid.UUID) { go e.keepLease(runCtx, cancel, run.ID, token) log := e.log.With("run", run.ID, "workflow", run.Workflow, "attempt", run.Attempts) - result, err := e.invoke(runCtx, run, token) + result, ending, err := e.invoke(runCtx, run, token) switch { case errors.Is(err, errSuspended): log.InfoContext(ctx, "run suspended") @@ -286,22 +301,25 @@ func (e *Engine) execute(ctx context.Context, run store.Run, token uuid.UUID) { log.WarnContext(ctx, "run stopped: lease lost or shutting down") return } - state, errText := store.StateSucceeded, "" - if errors.Is(err, ErrCancelled) { - state, errText = store.StateCancelled, err.Error() - } else if err != nil { - state, errText = store.StateFailed, err.Error() + ended := store.Ended{State: store.StateSucceeded, Result: result} + switch { + case errors.Is(err, ErrCancelled): + ended.State, ended.Error = store.StateCancelled, err.Error() + case err != nil: + ended.State, ended.Error = store.StateFailed, err.Error() + case ending != nil: + ended.State, ended.Detail = ending.State, ending.Detail } - if ferr := e.work.Finish(ctx, run.ID, token, state, result, errText); ferr != nil { + if ferr := e.work.Finish(ctx, run.ID, token, ended); ferr != nil { log.WarnContext(ctx, "finish run", "error", ferr) return } - log.InfoContext(ctx, "run finished", "state", state) + log.InfoContext(ctx, "run finished", "state", ended.State) } // invoke runs the workflow, turning a panic in workflow or step code into a // failure of this run rather than of the process. -func (e *Engine) invoke(ctx context.Context, run store.Run, token uuid.UUID) (result jsontext.Value, err error) { +func (e *Engine) invoke(ctx context.Context, run store.Run, token uuid.UUID) (result jsontext.Value, ending *Ending, err error) { defer func() { if p := recover(); p != nil { err = fmt.Errorf("panic: %v", p) @@ -309,13 +327,15 @@ func (e *Engine) invoke(ctx context.Context, run store.Run, token uuid.UUID) (re }() wf, err := e.load(ctx, run) if err != nil { - return nil, err + return nil, nil, err } steps, err := e.work.Steps(ctx, run.ID) if err != nil { - return nil, err + return nil, nil, err } - return wf(newContext(ctx, e.work, run, token, steps), run.Input) + rc := newContext(ctx, e.work, run, token, steps) + result, err = wf(rc, run.Input) + return result, rc.ending, err } // loadDocument reads the document a run executes: the version it started diff --git a/modules/runs/runs_test.go b/modules/runs/runs_test.go index cc7d6077a..daadb972d 100644 --- a/modules/runs/runs_test.go +++ b/modules/runs/runs_test.go @@ -29,6 +29,7 @@ type fakeWork struct { steps []store.Step finished []string errors []string + details []string results []jsontext.Value } @@ -68,15 +69,16 @@ func (f *fakeWork) Suspend(context.Context, uuid.UUID, uuid.UUID, *time.Time, st return nil } -func (f *fakeWork) Finish(_ context.Context, _, _ uuid.UUID, state string, result jsontext.Value, errText string) error { +func (f *fakeWork) Finish(_ context.Context, _, _ uuid.UUID, e store.Ended) error { f.mu.Lock() defer f.mu.Unlock() if f.leaseLost { return store.ErrLeaseLost } - f.finished = append(f.finished, state) - f.errors = append(f.errors, errText) - f.results = append(f.results, result) + f.finished = append(f.finished, e.State) + f.errors = append(f.errors, e.Error) + f.details = append(f.details, e.Detail) + f.results = append(f.results, e.Result) return nil } diff --git a/modules/runs/steps_test.go b/modules/runs/steps_test.go index 9c57cd06d..e5b9080d0 100644 --- a/modules/runs/steps_test.go +++ b/modules/runs/steps_test.go @@ -1,10 +1,12 @@ package runs import ( + "cmp" "context" "encoding/json/jsontext" json "encoding/json/v2" "errors" + "strings" "testing" "github.com/google/uuid" @@ -20,15 +22,17 @@ import ( // actionJudge decides each action at the level named for it and remembers // every request, so a test sees what a step asked for. type actionJudge struct { - levels map[string]judgement.Level - fail error - assessed []judgement.Request - decisions []judgement.Decision - executed int + levels map[string]judgement.Level + fail error + // failAction limits fail to the action with this name; empty fails them all. + failAction string + assessed []judgement.Request + decisions []judgement.Decision + executed int } func (j *actionJudge) Assess(_ context.Context, req judgement.Request) (judgement.Decision, error) { - if j.fail != nil { + if j.fail != nil && (j.failAction == "" || j.failAction == req.Action) { return judgement.Decision{}, j.fail } j.assessed = append(j.assessed, req) @@ -106,16 +110,23 @@ func (n *notices) Notify(_ context.Context, msg deliveries.Notification) error { // step's settings are rendered from the run's inputs and earlier results, its // condition decides whether it runs, a failure ends the run unless the step // continues, and every action judgement holds is put to people exactly once, -// by an approval step or before the run ends. +// by an approval step or before the run ends. The first step that sets an +// outcome ends the run, a failed step parks it, and a parked run retried from a +// step keeps the results of the steps before it. func TestDocumentStepsRun(t *testing.T) { input := jsontext.Value(`{"event":{"fields":{"host":"web-1"}}}`) tests := []struct { - name string - doc string - levels map[string]judgement.Level - fail error - failFirst int - wantErr bool + name string + doc string + levels map[string]judgement.Level + fail error + failAction string + failFirst int + // retry retries the parked run from the named step once the failure + // has cleared; the counts below cover both attempts. + retry string + wantState string + wantDetail string wantInput string wantAssessed int wantAsked int @@ -139,12 +150,64 @@ steps: `, }, { - name: "a failing step fails the run", + name: "a failing step parks the run and later steps do not run", doc: `steps: - {type: connector.action, settings: {connector: host, name: clean}} - {type: run.notify} `, - fail: errors.New("judgement is down"), wantErr: true, + fail: errors.New("judgement is down"), wantState: store.StateParked, wantDetail: "judgement is down", + }, + { + name: "a retry from a step keeps the results of the steps before it", + doc: `steps: + - {id: first, type: connector.action, settings: {connector: host, name: clean}} + - {id: second, type: connector.action, settings: {connector: host, name: restart}} + - {type: run.notify, settings: {title: "{{ steps.first.summary }} then {{ steps.second.summary }}"}} +`, + levels: map[string]judgement.Level{"clean": judgement.ActQuietly, "restart": judgement.ActQuietly}, + fail: errors.New("judgement is down"), failAction: "restart", retry: "second", + wantState: store.StateSucceeded, wantAssessed: 2, wantExecuted: 2, wantNotice: "done then done", + }, + { + name: "a finish step ends the run with its status and detail", + doc: `steps: + - {type: workflow.finish, settings: {status: declined, detail: not worth fixing}} + - {type: run.notify, settings: {title: too late}} +`, + wantState: store.StateDeclined, wantDetail: "not worth fixing", + }, + { + name: "a finish step whose condition does not hold lets the run go on", + doc: `inputs: {event: {type: object}} +steps: + - {type: workflow.finish, when: "inputs.event.fields.missing", settings: {status: parked, detail: stop}} + - {type: run.notify, settings: {title: carried on}} +`, + wantNotice: "carried on", + }, + { + name: "a finish step can leave the run waiting for approval", + doc: `steps: + - {type: workflow.finish, settings: {status: waiting_approval, detail: needs a person}} +`, + wantState: store.StateWaitingApproval, wantDetail: "needs a person", + }, + { + name: "a finish step may name the steps before it", + doc: `steps: + - {id: clean, type: connector.action, settings: {connector: host, name: clean}} + - {type: workflow.finish, settings: {detail: "cleaned: {{ steps.clean.summary }}"}} +`, + levels: map[string]judgement.Level{"clean": judgement.ActQuietly}, + wantAssessed: 1, wantExecuted: 1, wantState: store.StateSucceeded, wantDetail: "cleaned: done", + }, + { + name: "a run that reaches the end of its steps completes with the last summary", + doc: `steps: + - {id: clean, type: connector.action, settings: {connector: host, name: clean}} +`, + levels: map[string]judgement.Level{"clean": judgement.ActQuietly}, + wantAssessed: 1, wantExecuted: 1, wantState: store.StateSucceeded, wantDetail: "done", }, { name: "a failing step that continues lets the run go on", @@ -209,15 +272,31 @@ steps: if err != nil { t.Fatalf("ParseDocument: %v", err) } - judge := &actionJudge{levels: tt.levels, fail: tt.fail} + judge := &actionJudge{levels: tt.levels, fail: tt.fail, failAction: tt.failAction} ask := &askCounter{failFirst: tt.failFirst} told := ¬ices{} wf := Deps{Settings: onePersonPath{}, Toolbox: anyBox{}, Judge: judge, Approver: ask, Notifier: told}.Compile(doc) work := &signalAfter{arrives: 1} - rc := newContext(t.Context(), work, store.Run{ID: uuid.New()}, uuid.New(), nil) - _, err = wf(rc, input) - if (err != nil) != tt.wantErr { - t.Fatalf("run error = %v, want error %v", err, tt.wantErr) + run := store.Run{ID: uuid.New()} + rc := newContext(t.Context(), work, run, uuid.New(), nil) + if _, err = wf(rc, input); err != nil { + t.Fatalf("run error = %v", err) + } + if tt.retry != "" { + if rc.ending == nil || rc.ending.State != store.StateParked { + t.Fatalf("ending = %+v, want the run parked before its retry", rc.ending) + } + judge.fail = nil + kept := stepsBeforeRetry(t, work.steps, tt.retry) + rc = newContext(t.Context(), work, run, uuid.New(), kept) + work.steps = kept + if _, err = wf(rc, input); err != nil { + t.Fatalf("retry error = %v", err) + } + } + wantState := cmp.Or(tt.wantState, store.StateSucceeded) + if rc.ending == nil || rc.ending.State != wantState || !strings.Contains(rc.ending.Detail, tt.wantDetail) { + t.Fatalf("ending = %+v, want %s with detail containing %q", rc.ending, wantState, tt.wantDetail) } if len(judge.assessed) != tt.wantAssessed || ask.asked != tt.wantAsked || judge.executed != tt.wantExecuted { t.Fatalf("assessed %d, asked %d, executed %d; want %d, %d, %d", @@ -242,6 +321,32 @@ steps: } } +// stepsBeforeRetry is what a retry from the named step keeps: the steps +// recorded before it began, and the markers written before it was entered. It +// mirrors store.Retry. +func stepsBeforeRetry(t *testing.T, steps []store.Step, from string) []store.Step { + t.Helper() + first, marker := -1, -1 + for _, s := range steps { + if s.Name == enteredPrefix+from { + marker = s.Seq + if err := json.Unmarshal(s.Output, &first); err != nil { + t.Fatal(err) + } + } + } + if marker < 0 { + t.Fatalf("the run never reached step %q", from) + } + var kept []store.Step + for _, s := range steps { + if (s.Seq < first) || (s.Seq >= markerBase && s.Seq < marker) { + kept = append(kept, s) + } + } + return kept +} + func mustJSON(t *testing.T, v any) string { t.Helper() b, err := json.Marshal(v, json.Deterministic(true)) diff --git a/modules/runs/trail.go b/modules/runs/trail.go index 27392b470..bdf5992d6 100644 --- a/modules/runs/trail.go +++ b/modules/runs/trail.go @@ -4,6 +4,7 @@ import ( "context" "encoding/json/jsontext" json "encoding/json/v2" + "errors" "net/http" "strconv" "strings" @@ -39,12 +40,15 @@ type Origins interface { // RunView is a run as the API shows it. type RunView struct { - ID uuid.UUID `json:"id"` - Workflow string `json:"workflow"` - Version int `json:"workflow_version"` - State string `json:"state"` - Attempts int `json:"attempts"` - Error string `json:"error,omitzero"` + ID uuid.UUID `json:"id"` + Workflow string `json:"workflow"` + Version int `json:"workflow_version"` + State string `json:"state"` + Attempts int `json:"attempts"` + Error string `json:"error,omitzero"` + // Detail is what the run's ending says: a finish step's detail, or why a + // step parked the run. + Detail string `json:"detail,omitzero"` CreatedAt time.Time `json:"created_at"` FinishedAt *time.Time `json:"finished_at,omitzero"` Result jsontext.Value `json:"result,omitzero"` @@ -133,7 +137,7 @@ type Approvals interface { // Register adds the run API for orgID: recent runs, each run's trail, and // the workflows runs execute. func (e *Engine) Register(api huma.API, orgID string, origins Origins, decisions Decisions, asked Approvals) { - a := &runAPI{db: e.db, orgID: orgID, origins: origins, decisions: decisions, approvals: asked} + a := &runAPI{engine: e, db: e.db, orgID: orgID, origins: origins, decisions: decisions, approvals: asked} tags := []string{"Runs"} huma.Register(api, huma.Operation{ OperationID: "list-runs", Method: http.MethodGet, Path: "/api/runs", @@ -143,9 +147,40 @@ func (e *Engine) Register(api huma.API, orgID string, origins Origins, decisions OperationID: "get-run", Method: http.MethodGet, Path: "/api/runs/{id}", Summary: "Get a run's decision trail", Tags: tags, }, a.trail) + huma.Register(api, huma.Operation{ + OperationID: "retry-run", Method: http.MethodPost, Path: "/api/runs/{id}/retry", + Summary: "Retry a parked run from a step, keeping the results of the steps before it", Tags: tags, + }, a.retry) e.registerWorkflows(api, orgID) } +type retryRunInput struct { + ID string `path:"id" format:"uuid"` + Body struct { + From string `json:"from,omitempty" doc:"The step to retry from, by id or as the trail names it; by default the step that parked the run"` + } +} + +func (a *runAPI) retry(ctx context.Context, in *retryRunInput) (*httpapi.Reply[RunView], error) { + id, err := uuid.Parse(in.ID) + if err != nil { + return nil, huma.NewError(http.StatusBadRequest, "a run id is a UUID") + } + switch err := a.engine.Retry(ctx, a.orgID, id, in.Body.From); { + case errors.Is(err, store.ErrNoRun): + return nil, huma.NewError(http.StatusNotFound, "no such run") + case errors.Is(err, ErrNotRetryable): + return nil, huma.NewError(http.StatusConflict, "only a parked run retries, and only from a step it reached") + case err != nil: + return nil, huma.NewError(http.StatusInternalServerError, "couldn't retry the run") + } + d, ok, err := a.db.Detail(ctx, a.orgID, id) + if err != nil || !ok { + return nil, huma.NewError(http.StatusInternalServerError, "couldn't read the run") + } + return httpapi.OK(view(d.Summary, d.Result)), nil +} + type recentRunsInput struct { Limit int `query:"limit" minimum:"1" maximum:"200" default:"50"` } @@ -155,6 +190,7 @@ type runInput struct { } type runAPI struct { + engine *Engine db *store.Store orgID string origins Origins @@ -309,7 +345,7 @@ func effectOf(s store.Recorded) *Effect { func view(s store.Summary, result jsontext.Value) RunView { v := RunView{ ID: s.ID, Workflow: s.Workflow, Version: s.Version, State: s.State, Attempts: s.Attempts, - Error: s.Error.String, CreatedAt: s.CreatedAt, Result: result, + Error: s.Error.String, Detail: s.Detail.String, CreatedAt: s.CreatedAt, Result: result, } if s.FinishedAt.Valid { v.FinishedAt = &s.FinishedAt.Time diff --git a/modules/runs/vocabulary.go b/modules/runs/vocabulary.go index 2c25190ad..86cde22c5 100644 --- a/modules/runs/vocabulary.go +++ b/modules/runs/vocabulary.go @@ -14,6 +14,7 @@ import ( "github.com/samcharles93/archie-core/modules/deliveries" "github.com/samcharles93/archie-core/modules/governance" "github.com/samcharles93/archie-core/modules/judgement" + "github.com/samcharles93/archie-core/modules/runs/internal/store" ) // The step types the engine runs. @@ -23,6 +24,7 @@ const ( typeApproval = "approval.wait" typeNotify = "run.notify" typeSleep = "workflow.sleep" + typeFinish = "workflow.finish" ) // StepType defines a workflow step's settings and implementation. @@ -87,6 +89,7 @@ var stepTypes = vocabulary( }, runApprovalWait), define(typeNotify, "Report the findings", notifySettings.valid, notifySettings.describe, runNotify), define(typeSleep, "Wait", sleepSettings.valid, sleepSettings.describe, runSleep), + define(typeFinish, "Finish", finishSettings.valid, finishSettings.describe, runFinish), ) func vocabulary(types ...StepType) map[string]StepType { @@ -299,6 +302,51 @@ func runSleep(_ Deps, rc *Context, _ *progress, _ string, s sleepSettings) (Step return StepResult{}, rc.Sleep(s.duration()) } +// finishStatuses are the outcomes a finish step chooses, with the run state +// each one sets. +var finishStatuses = map[string]string{ + "completed": store.StateSucceeded, + "parked": store.StateParked, + "waiting_approval": store.StateWaitingApproval, + "declined": store.StateDeclined, +} + +// finishSettings end the run with an outcome. +type finishSettings struct { + Status string `yaml:"status,omitempty" json:"status,omitempty" jsonschema:"how the run ends: completed (the default), parked, waiting_approval or declined"` + Detail string `yaml:"detail,omitempty" json:"detail,omitempty" jsonschema:"what the operator reads about the outcome"` +} + +func (s finishSettings) valid() error { + if s.Status != "" { + if _, ok := finishStatuses[s.Status]; !ok { + return fmt.Errorf("status is completed, parked, waiting_approval or declined, not %q", s.Status) + } + } + return nil +} + +func (s finishSettings) state() string { + if s.Status == "" { + return store.StateSucceeded + } + return finishStatuses[s.Status] +} + +func (s finishSettings) describe() string { + status := "completed" + if s.Status != "" { + status = s.Status + } + return "Ends the run as " + status + "; later steps do not run" +} + +// runFinish sets the run's outcome: the first step that does ends the run. +func runFinish(_ Deps, rc *Context, _ *progress, _ string, s finishSettings) (StepResult, error) { + rc.End(s.state(), s.Detail) + return StepResult{Summary: s.Detail}, nil +} + // holds reports whether a step type can propose an action judgement may hold // for approval. func holds(stepType string) bool { diff --git a/ui/src/lib/api.ts b/ui/src/lib/api.ts index 429dbee1b..41bcf3963 100644 --- a/ui/src/lib/api.ts +++ b/ui/src/lib/api.ts @@ -187,6 +187,9 @@ export const api = { // The archie binary's module API; the wire types are below the object. runs: (limit = 200) => request("/api/runs" + qs({ limit })), run: (id: string) => request(`/api/runs/${encodeURIComponent(id)}`), + /** Queues a parked run again from the step that parked it, or from `from`. */ + retryRun: (id: string, from?: string) => + request(`/api/runs/${encodeURIComponent(id)}/retry`, { method: "POST", body: from ? { from } : {} }), approvals: () => request("/api/approvals"), answerApproval: (id: string, approved: boolean) => request(`/api/approvals/${encodeURIComponent(id)}/answer`, { method: "POST", body: { approved } }), @@ -662,6 +665,8 @@ export interface Run { state: string; attempts: number; error?: string; + /** What the run's ending says: a finish step's detail, or why a step parked it. */ + detail?: string; created_at: string; finished_at?: string; result?: unknown; diff --git a/ui/src/runs/RunPage.vue b/ui/src/runs/RunPage.vue index 43761b9e1..742add8a6 100644 --- a/ui/src/runs/RunPage.vue +++ b/ui/src/runs/RunPage.vue @@ -1,17 +1,19 @@