feat(runs): the first step that sets an outcome ends the run (#2042)
Add workflow.finish (completed, parked, waiting_approval, declined). A failed step parks the run unless it continues on failure, and a parked run retries from a top-level step, replaying the earlier steps' recorded results. Adds the retry-run operation and a Retry button on the run page.
This commit is contained in:
14 files changed
+526
-69
No files matched your search
@@ -2172,6 +2172,13 @@ components:
|
|||||||
- name
|
- name
|
||||||
- expected_version
|
- expected_version
|
||||||
type: object
|
type: object
|
||||||
|
RetryRunInputBody:
|
||||||
|
additionalProperties: false
|
||||||
|
properties:
|
||||||
|
from:
|
||||||
|
description: The step to retry from, by id or as the trail names it; by default the step that parked the run
|
||||||
|
type: string
|
||||||
|
type: object
|
||||||
RollbackInputBody:
|
RollbackInputBody:
|
||||||
additionalProperties: false
|
additionalProperties: false
|
||||||
properties:
|
properties:
|
||||||
@@ -2312,6 +2319,8 @@ components:
|
|||||||
created_at:
|
created_at:
|
||||||
format: date-time
|
format: date-time
|
||||||
type: string
|
type: string
|
||||||
|
detail:
|
||||||
|
type: string
|
||||||
error:
|
error:
|
||||||
type: string
|
type: string
|
||||||
finished_at:
|
finished_at:
|
||||||
@@ -5073,6 +5082,38 @@ paths:
|
|||||||
summary: Get a run's decision trail
|
summary: Get a run's decision trail
|
||||||
tags:
|
tags:
|
||||||
- Runs
|
- Runs
|
||||||
|
/api/runs/{id}/retry:
|
||||||
|
post:
|
||||||
|
operationId: retry-run
|
||||||
|
parameters:
|
||||||
|
- in: path
|
||||||
|
name: id
|
||||||
|
required: true
|
||||||
|
schema:
|
||||||
|
format: uuid
|
||||||
|
type: string
|
||||||
|
requestBody:
|
||||||
|
content:
|
||||||
|
application/json:
|
||||||
|
schema:
|
||||||
|
$ref: "#/components/schemas/RetryRunInputBody"
|
||||||
|
required: true
|
||||||
|
responses:
|
||||||
|
"200":
|
||||||
|
content:
|
||||||
|
application/json:
|
||||||
|
schema:
|
||||||
|
$ref: "#/components/schemas/RunsRunView"
|
||||||
|
description: OK
|
||||||
|
default:
|
||||||
|
content:
|
||||||
|
application/problem+json:
|
||||||
|
schema:
|
||||||
|
$ref: "#/components/schemas/ErrorModel"
|
||||||
|
description: Error
|
||||||
|
summary: Retry a parked run from a step, keeping the results of the steps before it
|
||||||
|
tags:
|
||||||
|
- Runs
|
||||||
/api/secret-engines:
|
/api/secret-engines:
|
||||||
get:
|
get:
|
||||||
operationId: list-secret-engines
|
operationId: list-secret-engines
|
||||||
|
|||||||
@@ -0,0 +1,17 @@
|
|||||||
|
-- +goose Up
|
||||||
|
-- A run can end in four ways beyond success and failure: a step parks it for an
|
||||||
|
-- operator to retry, a workflow.finish step ends it waiting for approval or
|
||||||
|
-- declined, and detail carries what the ending says. A parked or waiting run is
|
||||||
|
-- not terminal: parked runs retry from a step.
|
||||||
|
ALTER TABLE runs.runs DROP CONSTRAINT runs_runs_state_check;
|
||||||
|
ALTER TABLE runs.runs ADD CONSTRAINT runs_runs_state_check
|
||||||
|
CHECK (state IN ('queued', 'running', 'waiting', 'succeeded', 'failed', 'cancelled',
|
||||||
|
'parked', 'waiting_approval', 'declined'));
|
||||||
|
ALTER TABLE runs.runs ADD COLUMN detail text;
|
||||||
|
|
||||||
|
-- +goose Down
|
||||||
|
UPDATE runs.runs SET state = 'failed' WHERE state IN ('parked', 'waiting_approval', 'declined');
|
||||||
|
ALTER TABLE runs.runs DROP COLUMN detail;
|
||||||
|
ALTER TABLE runs.runs DROP CONSTRAINT runs_runs_state_check;
|
||||||
|
ALTER TABLE runs.runs ADD CONSTRAINT runs_runs_state_check
|
||||||
|
CHECK (state IN ('queued', 'running', 'waiting', 'succeeded', 'failed', 'cancelled'));
|
||||||
+48
-4
@@ -45,19 +45,29 @@ type Context struct {
|
|||||||
// are numbered from nextMarker.
|
// are numbered from nextMarker.
|
||||||
started map[string]bool
|
started map[string]bool
|
||||||
nextMarker int
|
nextMarker int
|
||||||
|
// entered holds the top-level steps an earlier attempt reached.
|
||||||
|
entered map[string]bool
|
||||||
|
// ending is how the workflow ended when it ended other than by running out
|
||||||
|
// of steps, which is a success.
|
||||||
|
ending *Ending
|
||||||
// overlay is what a dry run carries instead of a stored version.
|
// overlay is what a dry run carries instead of a stored version.
|
||||||
overlay *dryOverlay
|
overlay *dryOverlay
|
||||||
}
|
}
|
||||||
|
|
||||||
// markerBase is where started markers are numbered, far above any step a
|
// markerBase is where started markers are numbered, far above any step a
|
||||||
// workflow takes, so they never shift the positions a replay depends on.
|
// workflow takes, so they never shift the positions a replay depends on.
|
||||||
const markerBase = 1 << 20
|
const markerBase = store.MarkerBase
|
||||||
|
|
||||||
const startedPrefix = "started:"
|
const (
|
||||||
|
startedPrefix = "started:"
|
||||||
|
// enteredPrefix names the marker that records the position a top-level
|
||||||
|
// step began at, so a parked run can retry from it.
|
||||||
|
enteredPrefix = "at:"
|
||||||
|
)
|
||||||
|
|
||||||
func newContext(ctx context.Context, work workerStore, run store.Run, token uuid.UUID, steps []store.Step) *Context {
|
func newContext(ctx context.Context, work workerStore, run store.Run, token uuid.UUID, steps []store.Step) *Context {
|
||||||
saved := make(map[int]store.Step, len(steps))
|
saved := make(map[int]store.Step, len(steps))
|
||||||
started := map[string]bool{}
|
started, entered := map[string]bool{}, map[string]bool{}
|
||||||
nextMarker := markerBase
|
nextMarker := markerBase
|
||||||
for _, s := range steps {
|
for _, s := range steps {
|
||||||
saved[s.Seq] = s
|
saved[s.Seq] = s
|
||||||
@@ -65,8 +75,12 @@ func newContext(ctx context.Context, work workerStore, run store.Run, token uuid
|
|||||||
started[key] = true
|
started[key] = true
|
||||||
nextMarker = max(nextMarker, s.Seq+1)
|
nextMarker = max(nextMarker, s.Seq+1)
|
||||||
}
|
}
|
||||||
|
if name, ok := strings.CutPrefix(s.Name, enteredPrefix); ok {
|
||||||
|
entered[name] = true
|
||||||
|
nextMarker = max(nextMarker, s.Seq+1)
|
||||||
|
}
|
||||||
}
|
}
|
||||||
rc := &Context{ctx: ctx, work: work, run: run, token: token, saved: saved, started: started, nextMarker: nextMarker}
|
rc := &Context{ctx: ctx, work: work, run: run, token: token, saved: saved, started: started, entered: entered, nextMarker: nextMarker}
|
||||||
if run.Dry {
|
if run.Dry {
|
||||||
rc.overlay, _ = readOverlay(run.Overlay)
|
rc.overlay, _ = readOverlay(run.Overlay)
|
||||||
}
|
}
|
||||||
@@ -90,6 +104,36 @@ func (rc *Context) Begin(ctx context.Context, key string) (bool, error) {
|
|||||||
return false, nil
|
return false, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// Enter records that the top-level step name begins at the run's next position,
|
||||||
|
// unless an earlier attempt already did. Retrying a parked run from the step
|
||||||
|
// forgets everything recorded from that position on.
|
||||||
|
func (rc *Context) Enter(ctx context.Context, name string) error {
|
||||||
|
if rc.entered[name] {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
seq := rc.nextMarker
|
||||||
|
if _, err := rc.record(ctx, seq, enteredPrefix+name, rc.next, nil); err != nil && !errors.Is(err, store.ErrStepExists) {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
rc.entered[name] = true
|
||||||
|
rc.nextMarker = seq + 1
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// End records how the workflow ends, ahead of its steps running out.
|
||||||
|
func (rc *Context) End(state, detail string) {
|
||||||
|
rc.ending = &Ending{State: state, Detail: detail}
|
||||||
|
}
|
||||||
|
|
||||||
|
// Ending is how a run that stopped before its steps ran out ended.
|
||||||
|
type Ending struct {
|
||||||
|
// State is the run state the ending sets.
|
||||||
|
State string
|
||||||
|
// Detail is what the ending says: a finish step's detail or why a step
|
||||||
|
// parked the run.
|
||||||
|
Detail string
|
||||||
|
}
|
||||||
|
|
||||||
// RunID is the run's id.
|
// RunID is the run's id.
|
||||||
func (rc *Context) RunID() uuid.UUID { return rc.run.ID }
|
func (rc *Context) RunID() uuid.UUID { return rc.run.ID }
|
||||||
|
|
||||||
|
|||||||
@@ -58,8 +58,15 @@ func TestCorpusChecksAsItsStepTypesLand(t *testing.T) {
|
|||||||
agentLater("gate", "gate", "protect_tests")),
|
agentLater("gate", "gate", "protect_tests")),
|
||||||
},
|
},
|
||||||
{
|
{
|
||||||
name: "triage's references and conditions check; it waits for forge and outcome steps", file: "triage",
|
name: "triage's references and conditions check; it waits for forge and handoff steps", file: "triage",
|
||||||
want: notYet("forge.comment", "forge.close-issue", "workflow.finish", "workflow.handoff"),
|
want: notYet("forge.comment", "forge.close-issue", "workflow.handoff"),
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "a finish step with an unknown status is refused when the workflow is saved",
|
||||||
|
yaml: `steps:
|
||||||
|
- {type: workflow.finish, settings: {status: finished}}
|
||||||
|
`,
|
||||||
|
want: []string{`status is completed, parked, waiting_approval or declined, not "finished"`},
|
||||||
},
|
},
|
||||||
{
|
{
|
||||||
name: "a result field the agent's schema doesn't declare is refused",
|
name: "a result field the agent's schema doesn't declare is refused",
|
||||||
|
|||||||
@@ -22,8 +22,19 @@ const (
|
|||||||
StateSucceeded = "succeeded"
|
StateSucceeded = "succeeded"
|
||||||
StateFailed = "failed"
|
StateFailed = "failed"
|
||||||
StateCancelled = "cancelled"
|
StateCancelled = "cancelled"
|
||||||
|
// StateParked is a run stopped for an operator, by a failed step or a
|
||||||
|
// workflow.finish step. It is retried from a step.
|
||||||
|
StateParked = "parked"
|
||||||
|
// StateWaitingApproval and StateDeclined are the other endings a
|
||||||
|
// workflow.finish step chooses.
|
||||||
|
StateWaitingApproval = "waiting_approval"
|
||||||
|
StateDeclined = "declined"
|
||||||
)
|
)
|
||||||
|
|
||||||
|
// MarkerBase is where a run's markers are numbered, far above any step a
|
||||||
|
// workflow takes, so a marker never shifts the position a replay depends on.
|
||||||
|
const MarkerBase = 1 << 20
|
||||||
|
|
||||||
// Step outcomes.
|
// Step outcomes.
|
||||||
const (
|
const (
|
||||||
OutcomeDone = "done"
|
OutcomeDone = "done"
|
||||||
@@ -37,6 +48,12 @@ var (
|
|||||||
// ErrStepExists means the step was already recorded, by this run's replay
|
// ErrStepExists means the step was already recorded, by this run's replay
|
||||||
// or by a previous lease holder.
|
// or by a previous lease holder.
|
||||||
ErrStepExists = errors.New("step already recorded")
|
ErrStepExists = errors.New("step already recorded")
|
||||||
|
// ErrNoRun means the organisation has no such run to retry.
|
||||||
|
ErrNoRun = errors.New("no such run")
|
||||||
|
// ErrNotParked means the run is not parked, so there is nothing to retry.
|
||||||
|
ErrNotParked = errors.New("run is not parked")
|
||||||
|
// ErrNoStep means the run never reached the step it was to retry from.
|
||||||
|
ErrNoStep = errors.New("run did not reach that step")
|
||||||
)
|
)
|
||||||
|
|
||||||
// Run is a claimed run.
|
// Run is a claimed run.
|
||||||
@@ -192,13 +209,23 @@ func (s *Store) Suspend(ctx context.Context, id, token uuid.UUID, wakeAt *time.T
|
|||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
|
||||||
// Finish records a terminal state while the token holds the lease.
|
// Ended is how a run ended. Error is a failure's text and Detail what an
|
||||||
func (s *Store) Finish(ctx context.Context, id, token uuid.UUID, state string, result jsontext.Value, errText string) error {
|
// ending says, such as a workflow.finish step's detail.
|
||||||
|
type Ended struct {
|
||||||
|
State string
|
||||||
|
Result jsontext.Value
|
||||||
|
Error string
|
||||||
|
Detail string
|
||||||
|
}
|
||||||
|
|
||||||
|
// Finish records how a run ended while the token holds the lease.
|
||||||
|
func (s *Store) Finish(ctx context.Context, id, token uuid.UUID, e Ended) error {
|
||||||
res, err := s.db.ExecContext(ctx, `
|
res, err := s.db.ExecContext(ctx, `
|
||||||
UPDATE runs.runs
|
UPDATE runs.runs
|
||||||
SET state = $3, result = $4, error = NULLIF($5, ''), finished_at = now(), updated_at = now(),
|
SET state = $3, result = $4, error = NULLIF($5, ''), detail = NULLIF($6, ''),
|
||||||
|
finished_at = now(), updated_at = now(),
|
||||||
lease_token = NULL, lease_owner = NULL, lease_expires_at = NULL
|
lease_token = NULL, lease_owner = NULL, lease_expires_at = NULL
|
||||||
WHERE id = $1 AND lease_token = $2`, id, token, state, nullJSON(result), errText)
|
WHERE id = $1 AND lease_token = $2`, id, token, e.State, nullJSON(e.Result), e.Error, e.Detail)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return fmt.Errorf("finish run: %w", err)
|
return fmt.Errorf("finish run: %w", err)
|
||||||
}
|
}
|
||||||
@@ -208,6 +235,59 @@ func (s *Store) Finish(ctx context.Context, id, token uuid.UUID, state string, r
|
|||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// Retry queues a parked run to run again from a step, keeping what the steps
|
||||||
|
// before it recorded: the run replays them without repeating their effects.
|
||||||
|
// from names the step; empty means the step the run last reached. What that
|
||||||
|
// step and every later one recorded is forgotten, including the markers that
|
||||||
|
// stop an effect starting twice, so the step runs afresh.
|
||||||
|
func (s *Store) Retry(ctx context.Context, orgID string, id uuid.UUID, from string) error {
|
||||||
|
tx, err := s.db.BeginTxx(ctx, nil)
|
||||||
|
if err != nil {
|
||||||
|
return fmt.Errorf("retry run: %w", err)
|
||||||
|
}
|
||||||
|
defer func() { _ = tx.Rollback() }()
|
||||||
|
var state string
|
||||||
|
err = tx.GetContext(ctx, &state, `SELECT state FROM runs.runs WHERE org_id = $1 AND id = $2 AND NOT dry FOR UPDATE`, orgID, id)
|
||||||
|
switch {
|
||||||
|
case errors.Is(err, sql.ErrNoRows):
|
||||||
|
return ErrNoRun
|
||||||
|
case err != nil:
|
||||||
|
return fmt.Errorf("retry run: %w", err)
|
||||||
|
case state != StateParked:
|
||||||
|
return ErrNotParked
|
||||||
|
}
|
||||||
|
var at struct {
|
||||||
|
Marker int `db:"marker"`
|
||||||
|
FirstSeq int `db:"first_seq"`
|
||||||
|
}
|
||||||
|
err = tx.GetContext(ctx, &at, `
|
||||||
|
SELECT seq AS marker, (output #>> '{}')::int AS first_seq FROM runs.steps
|
||||||
|
WHERE run_id = $1 AND name LIKE 'at:%' AND ($2 = '' OR name = 'at:' || $2)
|
||||||
|
ORDER BY seq DESC LIMIT 1`, id, from)
|
||||||
|
if errors.Is(err, sql.ErrNoRows) {
|
||||||
|
return ErrNoStep
|
||||||
|
}
|
||||||
|
if err != nil {
|
||||||
|
return fmt.Errorf("find step: %w", err)
|
||||||
|
}
|
||||||
|
if _, err := tx.ExecContext(ctx, `
|
||||||
|
DELETE FROM runs.steps WHERE run_id = $1 AND ((seq >= $2 AND seq < $3) OR seq >= $4)`,
|
||||||
|
id, at.FirstSeq, MarkerBase, at.Marker); err != nil {
|
||||||
|
return fmt.Errorf("forget steps: %w", err)
|
||||||
|
}
|
||||||
|
if _, err := tx.ExecContext(ctx, `
|
||||||
|
UPDATE runs.runs
|
||||||
|
SET state = 'queued', result = NULL, error = NULL, detail = NULL, finished_at = NULL,
|
||||||
|
wake_at = NULL, waiting_for = NULL, updated_at = now()
|
||||||
|
WHERE id = $1`, id); err != nil {
|
||||||
|
return fmt.Errorf("queue run: %w", err)
|
||||||
|
}
|
||||||
|
if err := tx.Commit(); err != nil {
|
||||||
|
return fmt.Errorf("retry run: %w", err)
|
||||||
|
}
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
// Signal stores a signal for a run and queues the run if it is waiting for it.
|
// Signal stores a signal for a run and queues the run if it is waiting for it.
|
||||||
func (s *Store) Signal(ctx context.Context, id uuid.UUID, key string, payload jsontext.Value) error {
|
func (s *Store) Signal(ctx context.Context, id uuid.UUID, key string, payload jsontext.Value) error {
|
||||||
tx, err := s.db.BeginTxx(ctx, nil)
|
tx, err := s.db.BeginTxx(ctx, nil)
|
||||||
@@ -250,9 +330,9 @@ func (s *Store) Cancel(ctx context.Context, id uuid.UUID) error {
|
|||||||
if _, err := s.db.ExecContext(ctx, `
|
if _, err := s.db.ExecContext(ctx, `
|
||||||
UPDATE runs.runs
|
UPDATE runs.runs
|
||||||
SET cancel_requested = true, updated_at = now(),
|
SET cancel_requested = true, updated_at = now(),
|
||||||
state = CASE WHEN state IN ('queued', 'waiting') THEN 'cancelled' ELSE state END,
|
state = CASE WHEN state IN ('queued', 'waiting', 'parked', 'waiting_approval') THEN 'cancelled' ELSE state END,
|
||||||
finished_at = CASE WHEN state IN ('queued', 'waiting') THEN now() ELSE finished_at END
|
finished_at = CASE WHEN state IN ('queued', 'waiting', 'parked', 'waiting_approval') THEN now() ELSE finished_at END
|
||||||
WHERE id = $1 AND state NOT IN ('succeeded', 'failed', 'cancelled')`, id); err != nil {
|
WHERE id = $1 AND state NOT IN ('succeeded', 'failed', 'cancelled', 'declined')`, id); err != nil {
|
||||||
return fmt.Errorf("cancel run: %w", err)
|
return fmt.Errorf("cancel run: %w", err)
|
||||||
}
|
}
|
||||||
return nil
|
return nil
|
||||||
@@ -275,6 +355,7 @@ type Summary struct {
|
|||||||
State string `db:"state"`
|
State string `db:"state"`
|
||||||
Attempts int `db:"attempts"`
|
Attempts int `db:"attempts"`
|
||||||
Error sql.NullString `db:"error"`
|
Error sql.NullString `db:"error"`
|
||||||
|
Detail sql.NullString `db:"detail"`
|
||||||
CreatedAt time.Time `db:"created_at"`
|
CreatedAt time.Time `db:"created_at"`
|
||||||
FinishedAt sql.NullTime `db:"finished_at"`
|
FinishedAt sql.NullTime `db:"finished_at"`
|
||||||
Dry bool `db:"dry"`
|
Dry bool `db:"dry"`
|
||||||
@@ -299,7 +380,7 @@ type Recorded struct {
|
|||||||
func (s *Store) Recent(ctx context.Context, orgID string, limit int) ([]Summary, error) {
|
func (s *Store) Recent(ctx context.Context, orgID string, limit int) ([]Summary, error) {
|
||||||
var out []Summary
|
var out []Summary
|
||||||
if err := s.db.SelectContext(ctx, &out, `
|
if err := s.db.SelectContext(ctx, &out, `
|
||||||
SELECT id, workflow, workflow_version, state, attempts, error, created_at, finished_at, dry FROM runs.runs
|
SELECT id, workflow, workflow_version, state, attempts, error, detail, created_at, finished_at, dry FROM runs.runs
|
||||||
WHERE org_id = $1 AND NOT dry ORDER BY created_at DESC LIMIT $2`, orgID, limit); err != nil {
|
WHERE org_id = $1 AND NOT dry ORDER BY created_at DESC LIMIT $2`, orgID, limit); err != nil {
|
||||||
return nil, fmt.Errorf("list runs: %w", err)
|
return nil, fmt.Errorf("list runs: %w", err)
|
||||||
}
|
}
|
||||||
@@ -311,7 +392,7 @@ func (s *Store) Recent(ctx context.Context, orgID string, limit int) ([]Summary,
|
|||||||
func (s *Store) Detail(ctx context.Context, orgID string, id uuid.UUID) (Detail, bool, error) {
|
func (s *Store) Detail(ctx context.Context, orgID string, id uuid.UUID) (Detail, bool, error) {
|
||||||
var d Detail
|
var d Detail
|
||||||
err := s.db.GetContext(ctx, &d, `
|
err := s.db.GetContext(ctx, &d, `
|
||||||
SELECT id, workflow, workflow_version, state, attempts, error, created_at, finished_at, dry, input,
|
SELECT id, workflow, workflow_version, state, attempts, error, detail, created_at, finished_at, dry, input,
|
||||||
coalesce(result, 'null') AS result
|
coalesce(result, 'null') AS result
|
||||||
FROM runs.runs WHERE org_id = $1 AND id = $2`, orgID, id)
|
FROM runs.runs WHERE org_id = $1 AND id = $2`, orgID, id)
|
||||||
if errors.Is(err, sql.ErrNoRows) {
|
if errors.Is(err, sql.ErrNoRows) {
|
||||||
|
|||||||
@@ -132,7 +132,10 @@ type progress struct {
|
|||||||
agent *AgentResult
|
agent *AgentResult
|
||||||
memory bool
|
memory bool
|
||||||
remembered bool
|
remembered bool
|
||||||
outcomes []approvalOutcome
|
// last is what the most recent step that ran reported, which a run that
|
||||||
|
// reaches the end of its steps gives as its detail.
|
||||||
|
last string
|
||||||
|
outcomes []approvalOutcome
|
||||||
// settled is the actions already put to people, by digest, so a later
|
// settled is the actions already put to people, by digest, so a later
|
||||||
// approval step asks only about what was held since.
|
// approval step asks only about what was held since.
|
||||||
settled map[string]bool
|
settled map[string]bool
|
||||||
@@ -161,18 +164,35 @@ func (d Deps) Compile(doc Document) Workflow {
|
|||||||
if p.stopped && s.Type != typeNotify {
|
if p.stopped && s.Type != typeNotify {
|
||||||
continue
|
continue
|
||||||
}
|
}
|
||||||
if err := d.runStep(rc, p, s, i); err != nil {
|
name := s.name(i)
|
||||||
|
if err := rc.Enter(rc.Context(), name); err != nil {
|
||||||
return nil, err
|
return nil, err
|
||||||
}
|
}
|
||||||
|
if err := d.runStep(rc, p, s, i); err != nil {
|
||||||
|
if !stepFailed(err) {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
// A failed step parks the run for an operator to retry from
|
||||||
|
// it, unless the step continues on failure.
|
||||||
|
rc.End(store.StateParked, fmt.Sprintf("step %s: %v", name, err))
|
||||||
|
return p.encode()
|
||||||
|
}
|
||||||
|
if rc.ending != nil {
|
||||||
|
break
|
||||||
|
}
|
||||||
}
|
}
|
||||||
// An action still held when the document ends is put to people before
|
// An action still held when the document ends is put to people before
|
||||||
// the run finishes, so nothing judgement held runs unapproved or is
|
// the run finishes, so nothing judgement held runs unapproved or is
|
||||||
// silently dropped.
|
// silently dropped. A run a finish step parks, declines or leaves
|
||||||
if settleAtEnd && !p.stopped {
|
// waiting for approval ends as it was told to.
|
||||||
|
if settleAtEnd && !p.stopped && (rc.ending == nil || rc.ending.State == store.StateSucceeded) {
|
||||||
if _, err := d.settle(rc, p); err != nil {
|
if _, err := d.settle(rc, p); err != nil {
|
||||||
return nil, err
|
return nil, err
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
if rc.ending == nil {
|
||||||
|
rc.End(store.StateSucceeded, p.last)
|
||||||
|
}
|
||||||
return p.result()
|
return p.result()
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -202,6 +222,7 @@ func (d Deps) runStep(rc *Context, p *progress, s Step, index int) error {
|
|||||||
if s.ID != "" {
|
if s.ID != "" {
|
||||||
p.results[s.ID] = res
|
p.results[s.ID] = res
|
||||||
}
|
}
|
||||||
|
p.last = res.Summary
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -409,6 +430,18 @@ func agentResult(name string, res AgentResult, returned jsontext.Value, schema *
|
|||||||
// actions when no agent ran. An agent that did not complete fails the run once
|
// actions when no agent ran. An agent that did not complete fails the run once
|
||||||
// the owner has been told.
|
// the owner has been told.
|
||||||
func (p *progress) result() (jsontext.Value, error) {
|
func (p *progress) result() (jsontext.Value, error) {
|
||||||
|
out, err := p.encode()
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
if p.agent != nil && !p.agent.Completed {
|
||||||
|
return out, fmt.Errorf("investigation stopped: %s", p.agent.StopReason)
|
||||||
|
}
|
||||||
|
return out, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// encode is the run's findings so far, as the result a run records.
|
||||||
|
func (p *progress) encode() (jsontext.Value, error) {
|
||||||
var v any = p.outcomes
|
var v any = p.outcomes
|
||||||
if p.agent != nil {
|
if p.agent != nil {
|
||||||
v = *p.agent
|
v = *p.agent
|
||||||
@@ -417,9 +450,6 @@ func (p *progress) result() (jsontext.Value, error) {
|
|||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, fmt.Errorf("encode result: %w", err)
|
return nil, fmt.Errorf("encode result: %w", err)
|
||||||
}
|
}
|
||||||
if p.agent != nil && !p.agent.Completed {
|
|
||||||
return out, fmt.Errorf("investigation stopped: %s", p.agent.StopReason)
|
|
||||||
}
|
|
||||||
return out, nil
|
return out, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
+33
-13
@@ -45,7 +45,7 @@ type workerStore interface {
|
|||||||
Steps(ctx context.Context, id uuid.UUID) ([]store.Step, error)
|
Steps(ctx context.Context, id uuid.UUID) ([]store.Step, error)
|
||||||
RecordStep(ctx context.Context, token uuid.UUID, step store.Step) error
|
RecordStep(ctx context.Context, token uuid.UUID, step store.Step) error
|
||||||
Suspend(ctx context.Context, id, token uuid.UUID, wakeAt *time.Time, waitingFor string) error
|
Suspend(ctx context.Context, id, token uuid.UUID, wakeAt *time.Time, waitingFor string) error
|
||||||
Finish(ctx context.Context, id, token uuid.UUID, state string, result jsontext.Value, errText string) error
|
Finish(ctx context.Context, id, token uuid.UUID, ended store.Ended) error
|
||||||
Signalled(ctx context.Context, id uuid.UUID, key string) (jsontext.Value, bool, error)
|
Signalled(ctx context.Context, id uuid.UUID, key string) (jsontext.Value, bool, error)
|
||||||
CancelRequested(ctx context.Context, id uuid.UUID) (bool, error)
|
CancelRequested(ctx context.Context, id uuid.UUID) (bool, error)
|
||||||
}
|
}
|
||||||
@@ -234,6 +234,21 @@ func (e *Engine) Cancel(ctx context.Context, id uuid.UUID) error {
|
|||||||
return e.db.Cancel(ctx, id)
|
return e.db.Cancel(ctx, id)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// ErrNotRetryable is returned by Retry for a run that is not parked, or that
|
||||||
|
// never reached the step to retry from.
|
||||||
|
var ErrNotRetryable = errors.New("run can't be retried from there")
|
||||||
|
|
||||||
|
// Retry queues a parked run to run again from a top-level step, named by its id
|
||||||
|
// or, without one, its type and position as the trail shows it. An empty from
|
||||||
|
// is the step that parked the run. The steps before it keep their results.
|
||||||
|
func (e *Engine) Retry(ctx context.Context, orgID string, id uuid.UUID, from string) error {
|
||||||
|
err := e.db.Retry(ctx, orgID, id, from)
|
||||||
|
if errors.Is(err, store.ErrNotParked) || errors.Is(err, store.ErrNoStep) {
|
||||||
|
return fmt.Errorf("%w: %w", ErrNotRetryable, err)
|
||||||
|
}
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
|
||||||
// Work runs the engine's workers until ctx ends.
|
// Work runs the engine's workers until ctx ends.
|
||||||
func (e *Engine) Work(ctx context.Context) {
|
func (e *Engine) Work(ctx context.Context) {
|
||||||
var wg sync.WaitGroup
|
var wg sync.WaitGroup
|
||||||
@@ -277,7 +292,7 @@ func (e *Engine) execute(ctx context.Context, run store.Run, token uuid.UUID) {
|
|||||||
go e.keepLease(runCtx, cancel, run.ID, token)
|
go e.keepLease(runCtx, cancel, run.ID, token)
|
||||||
|
|
||||||
log := e.log.With("run", run.ID, "workflow", run.Workflow, "attempt", run.Attempts)
|
log := e.log.With("run", run.ID, "workflow", run.Workflow, "attempt", run.Attempts)
|
||||||
result, err := e.invoke(runCtx, run, token)
|
result, ending, err := e.invoke(runCtx, run, token)
|
||||||
switch {
|
switch {
|
||||||
case errors.Is(err, errSuspended):
|
case errors.Is(err, errSuspended):
|
||||||
log.InfoContext(ctx, "run suspended")
|
log.InfoContext(ctx, "run suspended")
|
||||||
@@ -286,22 +301,25 @@ func (e *Engine) execute(ctx context.Context, run store.Run, token uuid.UUID) {
|
|||||||
log.WarnContext(ctx, "run stopped: lease lost or shutting down")
|
log.WarnContext(ctx, "run stopped: lease lost or shutting down")
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
state, errText := store.StateSucceeded, ""
|
ended := store.Ended{State: store.StateSucceeded, Result: result}
|
||||||
if errors.Is(err, ErrCancelled) {
|
switch {
|
||||||
state, errText = store.StateCancelled, err.Error()
|
case errors.Is(err, ErrCancelled):
|
||||||
} else if err != nil {
|
ended.State, ended.Error = store.StateCancelled, err.Error()
|
||||||
state, errText = store.StateFailed, err.Error()
|
case err != nil:
|
||||||
|
ended.State, ended.Error = store.StateFailed, err.Error()
|
||||||
|
case ending != nil:
|
||||||
|
ended.State, ended.Detail = ending.State, ending.Detail
|
||||||
}
|
}
|
||||||
if ferr := e.work.Finish(ctx, run.ID, token, state, result, errText); ferr != nil {
|
if ferr := e.work.Finish(ctx, run.ID, token, ended); ferr != nil {
|
||||||
log.WarnContext(ctx, "finish run", "error", ferr)
|
log.WarnContext(ctx, "finish run", "error", ferr)
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
log.InfoContext(ctx, "run finished", "state", state)
|
log.InfoContext(ctx, "run finished", "state", ended.State)
|
||||||
}
|
}
|
||||||
|
|
||||||
// invoke runs the workflow, turning a panic in workflow or step code into a
|
// invoke runs the workflow, turning a panic in workflow or step code into a
|
||||||
// failure of this run rather than of the process.
|
// failure of this run rather than of the process.
|
||||||
func (e *Engine) invoke(ctx context.Context, run store.Run, token uuid.UUID) (result jsontext.Value, err error) {
|
func (e *Engine) invoke(ctx context.Context, run store.Run, token uuid.UUID) (result jsontext.Value, ending *Ending, err error) {
|
||||||
defer func() {
|
defer func() {
|
||||||
if p := recover(); p != nil {
|
if p := recover(); p != nil {
|
||||||
err = fmt.Errorf("panic: %v", p)
|
err = fmt.Errorf("panic: %v", p)
|
||||||
@@ -309,13 +327,15 @@ func (e *Engine) invoke(ctx context.Context, run store.Run, token uuid.UUID) (re
|
|||||||
}()
|
}()
|
||||||
wf, err := e.load(ctx, run)
|
wf, err := e.load(ctx, run)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, err
|
return nil, nil, err
|
||||||
}
|
}
|
||||||
steps, err := e.work.Steps(ctx, run.ID)
|
steps, err := e.work.Steps(ctx, run.ID)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, err
|
return nil, nil, err
|
||||||
}
|
}
|
||||||
return wf(newContext(ctx, e.work, run, token, steps), run.Input)
|
rc := newContext(ctx, e.work, run, token, steps)
|
||||||
|
result, err = wf(rc, run.Input)
|
||||||
|
return result, rc.ending, err
|
||||||
}
|
}
|
||||||
|
|
||||||
// loadDocument reads the document a run executes: the version it started
|
// loadDocument reads the document a run executes: the version it started
|
||||||
|
|||||||
@@ -29,6 +29,7 @@ type fakeWork struct {
|
|||||||
steps []store.Step
|
steps []store.Step
|
||||||
finished []string
|
finished []string
|
||||||
errors []string
|
errors []string
|
||||||
|
details []string
|
||||||
results []jsontext.Value
|
results []jsontext.Value
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -68,15 +69,16 @@ func (f *fakeWork) Suspend(context.Context, uuid.UUID, uuid.UUID, *time.Time, st
|
|||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
|
||||||
func (f *fakeWork) Finish(_ context.Context, _, _ uuid.UUID, state string, result jsontext.Value, errText string) error {
|
func (f *fakeWork) Finish(_ context.Context, _, _ uuid.UUID, e store.Ended) error {
|
||||||
f.mu.Lock()
|
f.mu.Lock()
|
||||||
defer f.mu.Unlock()
|
defer f.mu.Unlock()
|
||||||
if f.leaseLost {
|
if f.leaseLost {
|
||||||
return store.ErrLeaseLost
|
return store.ErrLeaseLost
|
||||||
}
|
}
|
||||||
f.finished = append(f.finished, state)
|
f.finished = append(f.finished, e.State)
|
||||||
f.errors = append(f.errors, errText)
|
f.errors = append(f.errors, e.Error)
|
||||||
f.results = append(f.results, result)
|
f.details = append(f.details, e.Detail)
|
||||||
|
f.results = append(f.results, e.Result)
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
+125
-20
@@ -1,10 +1,12 @@
|
|||||||
package runs
|
package runs
|
||||||
|
|
||||||
import (
|
import (
|
||||||
|
"cmp"
|
||||||
"context"
|
"context"
|
||||||
"encoding/json/jsontext"
|
"encoding/json/jsontext"
|
||||||
json "encoding/json/v2"
|
json "encoding/json/v2"
|
||||||
"errors"
|
"errors"
|
||||||
|
"strings"
|
||||||
"testing"
|
"testing"
|
||||||
|
|
||||||
"github.com/google/uuid"
|
"github.com/google/uuid"
|
||||||
@@ -20,15 +22,17 @@ import (
|
|||||||
// actionJudge decides each action at the level named for it and remembers
|
// actionJudge decides each action at the level named for it and remembers
|
||||||
// every request, so a test sees what a step asked for.
|
// every request, so a test sees what a step asked for.
|
||||||
type actionJudge struct {
|
type actionJudge struct {
|
||||||
levels map[string]judgement.Level
|
levels map[string]judgement.Level
|
||||||
fail error
|
fail error
|
||||||
assessed []judgement.Request
|
// failAction limits fail to the action with this name; empty fails them all.
|
||||||
decisions []judgement.Decision
|
failAction string
|
||||||
executed int
|
assessed []judgement.Request
|
||||||
|
decisions []judgement.Decision
|
||||||
|
executed int
|
||||||
}
|
}
|
||||||
|
|
||||||
func (j *actionJudge) Assess(_ context.Context, req judgement.Request) (judgement.Decision, error) {
|
func (j *actionJudge) Assess(_ context.Context, req judgement.Request) (judgement.Decision, error) {
|
||||||
if j.fail != nil {
|
if j.fail != nil && (j.failAction == "" || j.failAction == req.Action) {
|
||||||
return judgement.Decision{}, j.fail
|
return judgement.Decision{}, j.fail
|
||||||
}
|
}
|
||||||
j.assessed = append(j.assessed, req)
|
j.assessed = append(j.assessed, req)
|
||||||
@@ -106,16 +110,23 @@ func (n *notices) Notify(_ context.Context, msg deliveries.Notification) error {
|
|||||||
// step's settings are rendered from the run's inputs and earlier results, its
|
// step's settings are rendered from the run's inputs and earlier results, its
|
||||||
// condition decides whether it runs, a failure ends the run unless the step
|
// condition decides whether it runs, a failure ends the run unless the step
|
||||||
// continues, and every action judgement holds is put to people exactly once,
|
// continues, and every action judgement holds is put to people exactly once,
|
||||||
// by an approval step or before the run ends.
|
// by an approval step or before the run ends. The first step that sets an
|
||||||
|
// outcome ends the run, a failed step parks it, and a parked run retried from a
|
||||||
|
// step keeps the results of the steps before it.
|
||||||
func TestDocumentStepsRun(t *testing.T) {
|
func TestDocumentStepsRun(t *testing.T) {
|
||||||
input := jsontext.Value(`{"event":{"fields":{"host":"web-1"}}}`)
|
input := jsontext.Value(`{"event":{"fields":{"host":"web-1"}}}`)
|
||||||
tests := []struct {
|
tests := []struct {
|
||||||
name string
|
name string
|
||||||
doc string
|
doc string
|
||||||
levels map[string]judgement.Level
|
levels map[string]judgement.Level
|
||||||
fail error
|
fail error
|
||||||
failFirst int
|
failAction string
|
||||||
wantErr bool
|
failFirst int
|
||||||
|
// retry retries the parked run from the named step once the failure
|
||||||
|
// has cleared; the counts below cover both attempts.
|
||||||
|
retry string
|
||||||
|
wantState string
|
||||||
|
wantDetail string
|
||||||
wantInput string
|
wantInput string
|
||||||
wantAssessed int
|
wantAssessed int
|
||||||
wantAsked int
|
wantAsked int
|
||||||
@@ -139,12 +150,64 @@ steps:
|
|||||||
`,
|
`,
|
||||||
},
|
},
|
||||||
{
|
{
|
||||||
name: "a failing step fails the run",
|
name: "a failing step parks the run and later steps do not run",
|
||||||
doc: `steps:
|
doc: `steps:
|
||||||
- {type: connector.action, settings: {connector: host, name: clean}}
|
- {type: connector.action, settings: {connector: host, name: clean}}
|
||||||
- {type: run.notify}
|
- {type: run.notify}
|
||||||
`,
|
`,
|
||||||
fail: errors.New("judgement is down"), wantErr: true,
|
fail: errors.New("judgement is down"), wantState: store.StateParked, wantDetail: "judgement is down",
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "a retry from a step keeps the results of the steps before it",
|
||||||
|
doc: `steps:
|
||||||
|
- {id: first, type: connector.action, settings: {connector: host, name: clean}}
|
||||||
|
- {id: second, type: connector.action, settings: {connector: host, name: restart}}
|
||||||
|
- {type: run.notify, settings: {title: "{{ steps.first.summary }} then {{ steps.second.summary }}"}}
|
||||||
|
`,
|
||||||
|
levels: map[string]judgement.Level{"clean": judgement.ActQuietly, "restart": judgement.ActQuietly},
|
||||||
|
fail: errors.New("judgement is down"), failAction: "restart", retry: "second",
|
||||||
|
wantState: store.StateSucceeded, wantAssessed: 2, wantExecuted: 2, wantNotice: "done then done",
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "a finish step ends the run with its status and detail",
|
||||||
|
doc: `steps:
|
||||||
|
- {type: workflow.finish, settings: {status: declined, detail: not worth fixing}}
|
||||||
|
- {type: run.notify, settings: {title: too late}}
|
||||||
|
`,
|
||||||
|
wantState: store.StateDeclined, wantDetail: "not worth fixing",
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "a finish step whose condition does not hold lets the run go on",
|
||||||
|
doc: `inputs: {event: {type: object}}
|
||||||
|
steps:
|
||||||
|
- {type: workflow.finish, when: "inputs.event.fields.missing", settings: {status: parked, detail: stop}}
|
||||||
|
- {type: run.notify, settings: {title: carried on}}
|
||||||
|
`,
|
||||||
|
wantNotice: "carried on",
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "a finish step can leave the run waiting for approval",
|
||||||
|
doc: `steps:
|
||||||
|
- {type: workflow.finish, settings: {status: waiting_approval, detail: needs a person}}
|
||||||
|
`,
|
||||||
|
wantState: store.StateWaitingApproval, wantDetail: "needs a person",
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "a finish step may name the steps before it",
|
||||||
|
doc: `steps:
|
||||||
|
- {id: clean, type: connector.action, settings: {connector: host, name: clean}}
|
||||||
|
- {type: workflow.finish, settings: {detail: "cleaned: {{ steps.clean.summary }}"}}
|
||||||
|
`,
|
||||||
|
levels: map[string]judgement.Level{"clean": judgement.ActQuietly},
|
||||||
|
wantAssessed: 1, wantExecuted: 1, wantState: store.StateSucceeded, wantDetail: "cleaned: done",
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "a run that reaches the end of its steps completes with the last summary",
|
||||||
|
doc: `steps:
|
||||||
|
- {id: clean, type: connector.action, settings: {connector: host, name: clean}}
|
||||||
|
`,
|
||||||
|
levels: map[string]judgement.Level{"clean": judgement.ActQuietly},
|
||||||
|
wantAssessed: 1, wantExecuted: 1, wantState: store.StateSucceeded, wantDetail: "done",
|
||||||
},
|
},
|
||||||
{
|
{
|
||||||
name: "a failing step that continues lets the run go on",
|
name: "a failing step that continues lets the run go on",
|
||||||
@@ -209,15 +272,31 @@ steps:
|
|||||||
if err != nil {
|
if err != nil {
|
||||||
t.Fatalf("ParseDocument: %v", err)
|
t.Fatalf("ParseDocument: %v", err)
|
||||||
}
|
}
|
||||||
judge := &actionJudge{levels: tt.levels, fail: tt.fail}
|
judge := &actionJudge{levels: tt.levels, fail: tt.fail, failAction: tt.failAction}
|
||||||
ask := &askCounter{failFirst: tt.failFirst}
|
ask := &askCounter{failFirst: tt.failFirst}
|
||||||
told := ¬ices{}
|
told := ¬ices{}
|
||||||
wf := Deps{Settings: onePersonPath{}, Toolbox: anyBox{}, Judge: judge, Approver: ask, Notifier: told}.Compile(doc)
|
wf := Deps{Settings: onePersonPath{}, Toolbox: anyBox{}, Judge: judge, Approver: ask, Notifier: told}.Compile(doc)
|
||||||
work := &signalAfter{arrives: 1}
|
work := &signalAfter{arrives: 1}
|
||||||
rc := newContext(t.Context(), work, store.Run{ID: uuid.New()}, uuid.New(), nil)
|
run := store.Run{ID: uuid.New()}
|
||||||
_, err = wf(rc, input)
|
rc := newContext(t.Context(), work, run, uuid.New(), nil)
|
||||||
if (err != nil) != tt.wantErr {
|
if _, err = wf(rc, input); err != nil {
|
||||||
t.Fatalf("run error = %v, want error %v", err, tt.wantErr)
|
t.Fatalf("run error = %v", err)
|
||||||
|
}
|
||||||
|
if tt.retry != "" {
|
||||||
|
if rc.ending == nil || rc.ending.State != store.StateParked {
|
||||||
|
t.Fatalf("ending = %+v, want the run parked before its retry", rc.ending)
|
||||||
|
}
|
||||||
|
judge.fail = nil
|
||||||
|
kept := stepsBeforeRetry(t, work.steps, tt.retry)
|
||||||
|
rc = newContext(t.Context(), work, run, uuid.New(), kept)
|
||||||
|
work.steps = kept
|
||||||
|
if _, err = wf(rc, input); err != nil {
|
||||||
|
t.Fatalf("retry error = %v", err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
wantState := cmp.Or(tt.wantState, store.StateSucceeded)
|
||||||
|
if rc.ending == nil || rc.ending.State != wantState || !strings.Contains(rc.ending.Detail, tt.wantDetail) {
|
||||||
|
t.Fatalf("ending = %+v, want %s with detail containing %q", rc.ending, wantState, tt.wantDetail)
|
||||||
}
|
}
|
||||||
if len(judge.assessed) != tt.wantAssessed || ask.asked != tt.wantAsked || judge.executed != tt.wantExecuted {
|
if len(judge.assessed) != tt.wantAssessed || ask.asked != tt.wantAsked || judge.executed != tt.wantExecuted {
|
||||||
t.Fatalf("assessed %d, asked %d, executed %d; want %d, %d, %d",
|
t.Fatalf("assessed %d, asked %d, executed %d; want %d, %d, %d",
|
||||||
@@ -242,6 +321,32 @@ steps:
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// stepsBeforeRetry is what a retry from the named step keeps: the steps
|
||||||
|
// recorded before it began, and the markers written before it was entered. It
|
||||||
|
// mirrors store.Retry.
|
||||||
|
func stepsBeforeRetry(t *testing.T, steps []store.Step, from string) []store.Step {
|
||||||
|
t.Helper()
|
||||||
|
first, marker := -1, -1
|
||||||
|
for _, s := range steps {
|
||||||
|
if s.Name == enteredPrefix+from {
|
||||||
|
marker = s.Seq
|
||||||
|
if err := json.Unmarshal(s.Output, &first); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if marker < 0 {
|
||||||
|
t.Fatalf("the run never reached step %q", from)
|
||||||
|
}
|
||||||
|
var kept []store.Step
|
||||||
|
for _, s := range steps {
|
||||||
|
if (s.Seq < first) || (s.Seq >= markerBase && s.Seq < marker) {
|
||||||
|
kept = append(kept, s)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return kept
|
||||||
|
}
|
||||||
|
|
||||||
func mustJSON(t *testing.T, v any) string {
|
func mustJSON(t *testing.T, v any) string {
|
||||||
t.Helper()
|
t.Helper()
|
||||||
b, err := json.Marshal(v, json.Deterministic(true))
|
b, err := json.Marshal(v, json.Deterministic(true))
|
||||||
|
|||||||
+44
-8
@@ -4,6 +4,7 @@ import (
|
|||||||
"context"
|
"context"
|
||||||
"encoding/json/jsontext"
|
"encoding/json/jsontext"
|
||||||
json "encoding/json/v2"
|
json "encoding/json/v2"
|
||||||
|
"errors"
|
||||||
"net/http"
|
"net/http"
|
||||||
"strconv"
|
"strconv"
|
||||||
"strings"
|
"strings"
|
||||||
@@ -39,12 +40,15 @@ type Origins interface {
|
|||||||
|
|
||||||
// RunView is a run as the API shows it.
|
// RunView is a run as the API shows it.
|
||||||
type RunView struct {
|
type RunView struct {
|
||||||
ID uuid.UUID `json:"id"`
|
ID uuid.UUID `json:"id"`
|
||||||
Workflow string `json:"workflow"`
|
Workflow string `json:"workflow"`
|
||||||
Version int `json:"workflow_version"`
|
Version int `json:"workflow_version"`
|
||||||
State string `json:"state"`
|
State string `json:"state"`
|
||||||
Attempts int `json:"attempts"`
|
Attempts int `json:"attempts"`
|
||||||
Error string `json:"error,omitzero"`
|
Error string `json:"error,omitzero"`
|
||||||
|
// Detail is what the run's ending says: a finish step's detail, or why a
|
||||||
|
// step parked the run.
|
||||||
|
Detail string `json:"detail,omitzero"`
|
||||||
CreatedAt time.Time `json:"created_at"`
|
CreatedAt time.Time `json:"created_at"`
|
||||||
FinishedAt *time.Time `json:"finished_at,omitzero"`
|
FinishedAt *time.Time `json:"finished_at,omitzero"`
|
||||||
Result jsontext.Value `json:"result,omitzero"`
|
Result jsontext.Value `json:"result,omitzero"`
|
||||||
@@ -133,7 +137,7 @@ type Approvals interface {
|
|||||||
// Register adds the run API for orgID: recent runs, each run's trail, and
|
// Register adds the run API for orgID: recent runs, each run's trail, and
|
||||||
// the workflows runs execute.
|
// the workflows runs execute.
|
||||||
func (e *Engine) Register(api huma.API, orgID string, origins Origins, decisions Decisions, asked Approvals) {
|
func (e *Engine) Register(api huma.API, orgID string, origins Origins, decisions Decisions, asked Approvals) {
|
||||||
a := &runAPI{db: e.db, orgID: orgID, origins: origins, decisions: decisions, approvals: asked}
|
a := &runAPI{engine: e, db: e.db, orgID: orgID, origins: origins, decisions: decisions, approvals: asked}
|
||||||
tags := []string{"Runs"}
|
tags := []string{"Runs"}
|
||||||
huma.Register(api, huma.Operation{
|
huma.Register(api, huma.Operation{
|
||||||
OperationID: "list-runs", Method: http.MethodGet, Path: "/api/runs",
|
OperationID: "list-runs", Method: http.MethodGet, Path: "/api/runs",
|
||||||
@@ -143,9 +147,40 @@ func (e *Engine) Register(api huma.API, orgID string, origins Origins, decisions
|
|||||||
OperationID: "get-run", Method: http.MethodGet, Path: "/api/runs/{id}",
|
OperationID: "get-run", Method: http.MethodGet, Path: "/api/runs/{id}",
|
||||||
Summary: "Get a run's decision trail", Tags: tags,
|
Summary: "Get a run's decision trail", Tags: tags,
|
||||||
}, a.trail)
|
}, a.trail)
|
||||||
|
huma.Register(api, huma.Operation{
|
||||||
|
OperationID: "retry-run", Method: http.MethodPost, Path: "/api/runs/{id}/retry",
|
||||||
|
Summary: "Retry a parked run from a step, keeping the results of the steps before it", Tags: tags,
|
||||||
|
}, a.retry)
|
||||||
e.registerWorkflows(api, orgID)
|
e.registerWorkflows(api, orgID)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
type retryRunInput struct {
|
||||||
|
ID string `path:"id" format:"uuid"`
|
||||||
|
Body struct {
|
||||||
|
From string `json:"from,omitempty" doc:"The step to retry from, by id or as the trail names it; by default the step that parked the run"`
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func (a *runAPI) retry(ctx context.Context, in *retryRunInput) (*httpapi.Reply[RunView], error) {
|
||||||
|
id, err := uuid.Parse(in.ID)
|
||||||
|
if err != nil {
|
||||||
|
return nil, huma.NewError(http.StatusBadRequest, "a run id is a UUID")
|
||||||
|
}
|
||||||
|
switch err := a.engine.Retry(ctx, a.orgID, id, in.Body.From); {
|
||||||
|
case errors.Is(err, store.ErrNoRun):
|
||||||
|
return nil, huma.NewError(http.StatusNotFound, "no such run")
|
||||||
|
case errors.Is(err, ErrNotRetryable):
|
||||||
|
return nil, huma.NewError(http.StatusConflict, "only a parked run retries, and only from a step it reached")
|
||||||
|
case err != nil:
|
||||||
|
return nil, huma.NewError(http.StatusInternalServerError, "couldn't retry the run")
|
||||||
|
}
|
||||||
|
d, ok, err := a.db.Detail(ctx, a.orgID, id)
|
||||||
|
if err != nil || !ok {
|
||||||
|
return nil, huma.NewError(http.StatusInternalServerError, "couldn't read the run")
|
||||||
|
}
|
||||||
|
return httpapi.OK(view(d.Summary, d.Result)), nil
|
||||||
|
}
|
||||||
|
|
||||||
type recentRunsInput struct {
|
type recentRunsInput struct {
|
||||||
Limit int `query:"limit" minimum:"1" maximum:"200" default:"50"`
|
Limit int `query:"limit" minimum:"1" maximum:"200" default:"50"`
|
||||||
}
|
}
|
||||||
@@ -155,6 +190,7 @@ type runInput struct {
|
|||||||
}
|
}
|
||||||
|
|
||||||
type runAPI struct {
|
type runAPI struct {
|
||||||
|
engine *Engine
|
||||||
db *store.Store
|
db *store.Store
|
||||||
orgID string
|
orgID string
|
||||||
origins Origins
|
origins Origins
|
||||||
@@ -309,7 +345,7 @@ func effectOf(s store.Recorded) *Effect {
|
|||||||
func view(s store.Summary, result jsontext.Value) RunView {
|
func view(s store.Summary, result jsontext.Value) RunView {
|
||||||
v := RunView{
|
v := RunView{
|
||||||
ID: s.ID, Workflow: s.Workflow, Version: s.Version, State: s.State, Attempts: s.Attempts,
|
ID: s.ID, Workflow: s.Workflow, Version: s.Version, State: s.State, Attempts: s.Attempts,
|
||||||
Error: s.Error.String, CreatedAt: s.CreatedAt, Result: result,
|
Error: s.Error.String, Detail: s.Detail.String, CreatedAt: s.CreatedAt, Result: result,
|
||||||
}
|
}
|
||||||
if s.FinishedAt.Valid {
|
if s.FinishedAt.Valid {
|
||||||
v.FinishedAt = &s.FinishedAt.Time
|
v.FinishedAt = &s.FinishedAt.Time
|
||||||
|
|||||||
@@ -14,6 +14,7 @@ import (
|
|||||||
"github.com/samcharles93/archie-core/modules/deliveries"
|
"github.com/samcharles93/archie-core/modules/deliveries"
|
||||||
"github.com/samcharles93/archie-core/modules/governance"
|
"github.com/samcharles93/archie-core/modules/governance"
|
||||||
"github.com/samcharles93/archie-core/modules/judgement"
|
"github.com/samcharles93/archie-core/modules/judgement"
|
||||||
|
"github.com/samcharles93/archie-core/modules/runs/internal/store"
|
||||||
)
|
)
|
||||||
|
|
||||||
// The step types the engine runs.
|
// The step types the engine runs.
|
||||||
@@ -23,6 +24,7 @@ const (
|
|||||||
typeApproval = "approval.wait"
|
typeApproval = "approval.wait"
|
||||||
typeNotify = "run.notify"
|
typeNotify = "run.notify"
|
||||||
typeSleep = "workflow.sleep"
|
typeSleep = "workflow.sleep"
|
||||||
|
typeFinish = "workflow.finish"
|
||||||
)
|
)
|
||||||
|
|
||||||
// StepType defines a workflow step's settings and implementation.
|
// StepType defines a workflow step's settings and implementation.
|
||||||
@@ -87,6 +89,7 @@ var stepTypes = vocabulary(
|
|||||||
}, runApprovalWait),
|
}, runApprovalWait),
|
||||||
define(typeNotify, "Report the findings", notifySettings.valid, notifySettings.describe, runNotify),
|
define(typeNotify, "Report the findings", notifySettings.valid, notifySettings.describe, runNotify),
|
||||||
define(typeSleep, "Wait", sleepSettings.valid, sleepSettings.describe, runSleep),
|
define(typeSleep, "Wait", sleepSettings.valid, sleepSettings.describe, runSleep),
|
||||||
|
define(typeFinish, "Finish", finishSettings.valid, finishSettings.describe, runFinish),
|
||||||
)
|
)
|
||||||
|
|
||||||
func vocabulary(types ...StepType) map[string]StepType {
|
func vocabulary(types ...StepType) map[string]StepType {
|
||||||
@@ -299,6 +302,51 @@ func runSleep(_ Deps, rc *Context, _ *progress, _ string, s sleepSettings) (Step
|
|||||||
return StepResult{}, rc.Sleep(s.duration())
|
return StepResult{}, rc.Sleep(s.duration())
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// finishStatuses are the outcomes a finish step chooses, with the run state
|
||||||
|
// each one sets.
|
||||||
|
var finishStatuses = map[string]string{
|
||||||
|
"completed": store.StateSucceeded,
|
||||||
|
"parked": store.StateParked,
|
||||||
|
"waiting_approval": store.StateWaitingApproval,
|
||||||
|
"declined": store.StateDeclined,
|
||||||
|
}
|
||||||
|
|
||||||
|
// finishSettings end the run with an outcome.
|
||||||
|
type finishSettings struct {
|
||||||
|
Status string `yaml:"status,omitempty" json:"status,omitempty" jsonschema:"how the run ends: completed (the default), parked, waiting_approval or declined"`
|
||||||
|
Detail string `yaml:"detail,omitempty" json:"detail,omitempty" jsonschema:"what the operator reads about the outcome"`
|
||||||
|
}
|
||||||
|
|
||||||
|
func (s finishSettings) valid() error {
|
||||||
|
if s.Status != "" {
|
||||||
|
if _, ok := finishStatuses[s.Status]; !ok {
|
||||||
|
return fmt.Errorf("status is completed, parked, waiting_approval or declined, not %q", s.Status)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func (s finishSettings) state() string {
|
||||||
|
if s.Status == "" {
|
||||||
|
return store.StateSucceeded
|
||||||
|
}
|
||||||
|
return finishStatuses[s.Status]
|
||||||
|
}
|
||||||
|
|
||||||
|
func (s finishSettings) describe() string {
|
||||||
|
status := "completed"
|
||||||
|
if s.Status != "" {
|
||||||
|
status = s.Status
|
||||||
|
}
|
||||||
|
return "Ends the run as " + status + "; later steps do not run"
|
||||||
|
}
|
||||||
|
|
||||||
|
// runFinish sets the run's outcome: the first step that does ends the run.
|
||||||
|
func runFinish(_ Deps, rc *Context, _ *progress, _ string, s finishSettings) (StepResult, error) {
|
||||||
|
rc.End(s.state(), s.Detail)
|
||||||
|
return StepResult{Summary: s.Detail}, nil
|
||||||
|
}
|
||||||
|
|
||||||
// holds reports whether a step type can propose an action judgement may hold
|
// holds reports whether a step type can propose an action judgement may hold
|
||||||
// for approval.
|
// for approval.
|
||||||
func holds(stepType string) bool {
|
func holds(stepType string) bool {
|
||||||
|
|||||||
@@ -187,6 +187,9 @@ export const api = {
|
|||||||
// The archie binary's module API; the wire types are below the object.
|
// The archie binary's module API; the wire types are below the object.
|
||||||
runs: (limit = 200) => request<Run[]>("/api/runs" + qs({ limit })),
|
runs: (limit = 200) => request<Run[]>("/api/runs" + qs({ limit })),
|
||||||
run: (id: string) => request<RunTrail>(`/api/runs/${encodeURIComponent(id)}`),
|
run: (id: string) => request<RunTrail>(`/api/runs/${encodeURIComponent(id)}`),
|
||||||
|
/** Queues a parked run again from the step that parked it, or from `from`. */
|
||||||
|
retryRun: (id: string, from?: string) =>
|
||||||
|
request<Run>(`/api/runs/${encodeURIComponent(id)}/retry`, { method: "POST", body: from ? { from } : {} }),
|
||||||
approvals: () => request<Approval[]>("/api/approvals"),
|
approvals: () => request<Approval[]>("/api/approvals"),
|
||||||
answerApproval: (id: string, approved: boolean) =>
|
answerApproval: (id: string, approved: boolean) =>
|
||||||
request<Approval>(`/api/approvals/${encodeURIComponent(id)}/answer`, { method: "POST", body: { approved } }),
|
request<Approval>(`/api/approvals/${encodeURIComponent(id)}/answer`, { method: "POST", body: { approved } }),
|
||||||
@@ -662,6 +665,8 @@ export interface Run {
|
|||||||
state: string;
|
state: string;
|
||||||
attempts: number;
|
attempts: number;
|
||||||
error?: string;
|
error?: string;
|
||||||
|
/** What the run's ending says: a finish step's detail, or why a step parked it. */
|
||||||
|
detail?: string;
|
||||||
created_at: string;
|
created_at: string;
|
||||||
finished_at?: string;
|
finished_at?: string;
|
||||||
result?: unknown;
|
result?: unknown;
|
||||||
|
|||||||
+21
-2
@@ -1,17 +1,19 @@
|
|||||||
<script setup lang="ts">
|
<script setup lang="ts">
|
||||||
import { computed, onMounted } from "vue";
|
import { computed, onMounted, ref } from "vue";
|
||||||
import { RouterLink, useRoute } from "vue-router";
|
import { RouterLink, useRoute } from "vue-router";
|
||||||
|
|
||||||
import CodeView from "@/base/CodeView.vue";
|
import CodeView from "@/base/CodeView.vue";
|
||||||
import PageHeader from "@/base/PageHeader.vue";
|
import PageHeader from "@/base/PageHeader.vue";
|
||||||
import { Badge } from "@/components/ui/badge";
|
import { Badge } from "@/components/ui/badge";
|
||||||
|
import { Button } from "@/components/ui/button";
|
||||||
import { Empty, EmptyHeader, EmptyTitle } from "@/components/ui/empty";
|
import { Empty, EmptyHeader, EmptyTitle } from "@/components/ui/empty";
|
||||||
import { MetadataList } from "@/components/ui/metadata-list";
|
import { MetadataList } from "@/components/ui/metadata-list";
|
||||||
import { ReadError } from "@/components/ui/read-error";
|
import { ReadError } from "@/components/ui/read-error";
|
||||||
import { Section } from "@/components/ui/section";
|
import { Section } from "@/components/ui/section";
|
||||||
import { Skeleton } from "@/components/ui/skeleton";
|
import { Skeleton } from "@/components/ui/skeleton";
|
||||||
import { Timestamp } from "@/components/ui/timestamp";
|
import { Timestamp } from "@/components/ui/timestamp";
|
||||||
import { api, type RunAction } from "@/lib/api";
|
import { errors } from "@/copy/errors";
|
||||||
|
import { api, describeError, type RunAction } from "@/lib/api";
|
||||||
import { usePoll, useRead } from "@/lib/use-read";
|
import { usePoll, useRead } from "@/lib/use-read";
|
||||||
import { useLiveResource } from "@/stores/session";
|
import { useLiveResource } from "@/stores/session";
|
||||||
import ApprovalsSection from "./ApprovalsSection.vue";
|
import ApprovalsSection from "./ApprovalsSection.vue";
|
||||||
@@ -37,8 +39,23 @@ const details = computed(() => {
|
|||||||
{ label: "Attempts", value: String(run.attempts) },
|
{ label: "Attempts", value: String(run.attempts) },
|
||||||
{ label: "Run", value: run.id, mono: true },
|
{ label: "Run", value: run.id, mono: true },
|
||||||
...(run.error ? [{ label: "Error", value: run.error }] : []),
|
...(run.error ? [{ label: "Error", value: run.error }] : []),
|
||||||
|
...(run.detail ? [{ label: "Detail", value: run.detail }] : []),
|
||||||
];
|
];
|
||||||
});
|
});
|
||||||
|
const retrying = ref(false);
|
||||||
|
const retryError = ref("");
|
||||||
|
async function retry(): Promise<void> {
|
||||||
|
retrying.value = true;
|
||||||
|
retryError.value = "";
|
||||||
|
try {
|
||||||
|
await api.retryRun(id);
|
||||||
|
await reload();
|
||||||
|
} catch (cause) {
|
||||||
|
retryError.value = describeError(cause, errors.common.unknown);
|
||||||
|
} finally {
|
||||||
|
retrying.value = false;
|
||||||
|
}
|
||||||
|
}
|
||||||
const origin = computed(() => {
|
const origin = computed(() => {
|
||||||
const o = data.value?.origin;
|
const o = data.value?.origin;
|
||||||
if (!o) return [];
|
if (!o) return [];
|
||||||
@@ -64,10 +81,12 @@ const result = computed(() =>
|
|||||||
<PageHeader :title="data ? data.run.workflow : 'Run'">
|
<PageHeader :title="data ? data.run.workflow : 'Run'">
|
||||||
<Badge v-if="data" :variant="stateTone(data.run.state)">{{ data.run.state }}</Badge>
|
<Badge v-if="data" :variant="stateTone(data.run.state)">{{ data.run.state }}</Badge>
|
||||||
<span v-if="data" class="text-caption text-fg-muted">Started <Timestamp :at="data.run.created_at" /></span>
|
<span v-if="data" class="text-caption text-fg-muted">Started <Timestamp :at="data.run.created_at" /></span>
|
||||||
|
<Button v-if="data?.run.state === 'parked'" size="sm" :disabled="retrying" @click="retry">Retry</Button>
|
||||||
</PageHeader>
|
</PageHeader>
|
||||||
<ReadError v-if="error" :message="error" retry-label="Retry" @retry="reload" />
|
<ReadError v-if="error" :message="error" retry-label="Retry" @retry="reload" />
|
||||||
<div v-else-if="!data" role="status" aria-label="Loading run" class="space-y-2"><Skeleton class="h-24 w-full" /><Skeleton class="h-48 w-full" /></div>
|
<div v-else-if="!data" role="status" aria-label="Loading run" class="space-y-2"><Skeleton class="h-24 w-full" /><Skeleton class="h-48 w-full" /></div>
|
||||||
<template v-else>
|
<template v-else>
|
||||||
|
<p v-if="retryError" role="alert" class="mb-2 text-caption text-danger">{{ retryError }}</p>
|
||||||
<ApprovalsSection :run="id" @answered="reload" />
|
<ApprovalsSection :run="id" @answered="reload" />
|
||||||
<div class="grid gap-x-4 lg:grid-cols-2">
|
<div class="grid gap-x-4 lg:grid-cols-2">
|
||||||
<Section title="Run"><MetadataList :rows="details" /></Section>
|
<Section title="Run"><MetadataList :rows="details" /></Section>
|
||||||
|
|||||||
@@ -7,6 +7,8 @@ export function stateTone(state: string): "ok" | "warn" | "danger" | "info" | "i
|
|||||||
case "done":
|
case "done":
|
||||||
return "ok";
|
return "ok";
|
||||||
case "waiting":
|
case "waiting":
|
||||||
|
case "waiting_approval":
|
||||||
|
case "parked":
|
||||||
case "pending":
|
case "pending":
|
||||||
case "unknown":
|
case "unknown":
|
||||||
return "warn";
|
return "warn";
|
||||||
|
|||||||
Reference in new issue
Block a user