Files
archie/internal/infrastructure/postgres/steps.go
T
sam cb2ca07d0b feat(statestore): CancelExecution, and one cancel path for daemon stop and operator actions
Plan item 5 of docs/prds/execution-tree-state-machine.md (archie-core-1217).
Until now a stop was a context cancel plus a caller-side transition, and the
step rows a worker still held running could never be told: the execution moved
while the work carried on, and the old interrupt-first ordering existed only
because the record could not defend itself. The step table can: the PRD's
ordering -- the store is the record, the context cancel is the delivery --
became safe the moment a worker's next step write fails ErrStaleTransition
against a cancelled step.

- storecontract.ExecutionCanceller and the CancelExecution RPC: one
  transaction moves every non-terminal StepExecution of the execution's
  current attempt to cancelled (one event row each, in the same
  transaction), and the execution itself to the status the operator action
  names, under the shared transition table with its audit row. Staleness is
  decided before legality, the way guardTransition decides it. Earlier
  attempts are never rewritten. Deliberately administrative: the
  deny-by-default grant rule covers it, pinned by a test.
- taskactions.Service routes stop, cancel, abandon and reject through it and
  the old Transition/CancelTask combinations are deleted; the daemon's
  in-memory cancel is demoted to the delivery hook the PRD names, called
  after the store record.
- The gateway's chat controller records first and delivers second, the same
  order, and records even when no runtime is wired here (a standalone
  gateway cannot reach the daemon's goroutines): the record is what unsticks
  a lost-cancel task, and the delivery is best effort. StopRunning and
  TaskRuntime.CancelRunningTasks are deleted outright -- no production
  caller remained once the task_action tool owned stopping, and the daemon's
  CancelRunningTasks/runningTasks.stopAll go with them.
- The gateway's chat adapter writes through CancelExecution instead of a
  bare Transition, so chat /cancel and the dashboard's action service land
  on the same path.

PRD decisions recorded here:
- Each cancelled step's event row is stage_finish with {cancelled: true} and
  the reason as its detail -- the exact-one-event-per-step-transition rule
  covers the pending -> cancelled transition too, even for a step that never
  ran.
- A parked cancel carries the needs_human class, matching the Transition
  path this replaces; the transient class stays with parkRunningTask's
  unwind parks, which now meet ErrStaleTransition for cancelled runs and are
  ignored there.
- Cancelling a wait:true callee is deferred to #1216's agent-call children:
  the callee's own tree and the wait flag are what make
  "cancelling a caller cancels each callee it waits on" decidable, and the
  callee writes it cannot fake are this issue's half of the contract.
- RecoverStale's interrupted sweep is plan item 6, untouched here.
2026-09-27 06:34:01 +10:00

300 lines
11 KiB
Go

package postgres
import (
"context"
"errors"
"fmt"
"time"
"github.com/jackc/pgx/v5"
"github.com/jackc/pgx/v5/pgtype"
"github.com/samcharles93/archie-core/internal/domain/storecontract"
task "github.com/samcharles93/archie-core/internal/domain/workflow/task"
"github.com/samcharles93/archie-core/internal/events"
"github.com/samcharles93/archie-core/internal/infrastructure/postgres/postgresdb"
"github.com/samcharles93/archie-core/internal/taskstate"
)
// The step-execution writes of docs/prds/execution-tree-state-machine.md.
// Each is one transaction: the guarded row write, and the domain event row
// the step transition produces, so the events table records exactly the
// transitions the step row does. The events land in the table; the caller
// publishes the returned event to its bus after the write commits, and the
// daemon's event sink skips already-persisted rows by their assigned ID --
// the EmitDurable convention.
// StartStep records a StepExecution entering running: the row is created
// pending, guarded into running under the shared step transition table, and
// the stage_start event is appended in the same transaction. The execution's
// row is locked for the whole write, so its status check cannot race the
// guarded updates, and org and workspace are stamped from it -- the request's
// ownership fields are ignored, like every owned record.
func (s *Store) StartStep(ctx context.Context, start task.StepStart) (int64, events.Event, error) {
if !task.ValidStepKind(start.Kind) {
return 0, events.Event{}, fmt.Errorf("%w: unknown step kind %q", storecontract.ErrInvalidStep, start.Kind)
}
tx, err := s.pool.Begin(ctx)
if err != nil {
return 0, events.Event{}, err
}
defer func() { _ = tx.Rollback(ctx) }()
q := postgresdb.New(tx)
execution, err := q.LockExecutionForStep(ctx, start.ExecutionID)
if errors.Is(err, pgx.ErrNoRows) {
// The execution the caller names does not exist: its view of the run
// is stale, matching what guardTransition returns for a missing row.
return 0, events.Event{}, storecontract.ErrStaleTransition
}
if err != nil {
return 0, events.Event{}, err
}
if execution.Attempt != int64(start.Attempt) {
// A step belongs to exactly one attempt; a caller naming one the
// execution no longer runs is describing a run that is over.
return 0, events.Event{}, storecontract.ErrStaleTransition
}
depth, err := s.startParent(ctx, q, start)
if err != nil {
return 0, events.Event{}, err
}
// A step cannot reach running while its execution is not running, and
// never under a terminal parent. Both are the state machine's rules, so
// they refuse as illegal transitions.
if err := taskstate.CheckStepStart(execution.Status, depth.underTerminal); err != nil {
return 0, events.Event{}, fmt.Errorf("%w: %w", storecontract.ErrIllegalTransition, err)
}
stepID, err := q.InsertStepExecution(ctx, postgresdb.InsertStepExecutionParams{
OrgID: execution.OrgID, WorkspaceID: execution.WorkspaceID,
ExecutionID: execution.ID, Attempt: execution.Attempt,
ParentID: stepParentID(start.ParentID), Depth: int32(depth.depth),
Kind: start.Kind, Name: start.Name, CalledExecutionID: 0,
})
if err != nil {
return 0, events.Event{}, err
}
n, err := q.StartStepExecution(ctx, stepID)
if err != nil {
return 0, events.Event{}, err
}
if n == 0 {
return 0, events.Event{}, storecontract.ErrStaleTransition
}
event := stepEvent(events.KindStageStart, execution.ID,
execution.Owner, execution.Repo, execution.IssueNumber, execution.Workflow,
int(execution.Attempt), start.Name, nil)
eventID, err := insertEventQ(ctx, q, event)
if err != nil {
return 0, events.Event{}, err
}
event.ID = eventID
if err := tx.Commit(ctx); err != nil {
return 0, events.Event{}, err
}
return stepID, event, nil
}
// stepParent resolves the enclosing StepExecution a child step names: its
// row is locked in the same transaction, the tree must stay inside one
// execution and one attempt, and depth is derived from the parent, never
// supplied by the caller.
func (s *Store) startParent(ctx context.Context, q *postgresdb.Queries, start task.StepStart) (stepParentState, error) {
if start.ParentID == 0 {
return stepParentState{}, nil
}
parent, err := q.LockStepExecution(ctx, start.ParentID)
if errors.Is(err, pgx.ErrNoRows) {
return stepParentState{}, storecontract.ErrStaleTransition
}
if err != nil {
return stepParentState{}, err
}
if parent.ExecutionID != start.ExecutionID || parent.Attempt != int64(start.Attempt) {
// Trees never nest across executions, and a parent from another
// attempt is a tree the caller is not running.
return stepParentState{}, storecontract.ErrStaleTransition
}
return stepParentState{
depth: int(parent.Depth) + 1,
underTerminal: taskstate.StepTerminal(taskstate.StepStatus(parent.Status)),
}, nil
}
type stepParentState struct {
depth int
underTerminal bool
}
// FinishStep moves one step to its outcome under the shared step transition
// table, writing the stage_finish event in the same transaction and returning
// it for the caller's post-commit publish. A step that is missing, that is
// not the named execution's, or whose status does not match from is stale; an
// off-table from->to pair is refused as an illegal transition.
func (s *Store) FinishStep(ctx context.Context, finish task.StepFinish) (events.Event, error) {
tx, err := s.pool.Begin(ctx)
if err != nil {
return events.Event{}, err
}
defer func() { _ = tx.Rollback(ctx) }()
q := postgresdb.New(tx)
step, err := q.LockStepExecution(ctx, finish.StepID)
if errors.Is(err, pgx.ErrNoRows) {
return events.Event{}, storecontract.ErrStaleTransition
}
if err != nil {
return events.Event{}, err
}
// The step belongs to another run, or the caller believes a state the row
// does not hold. Either way its view is stale, and writing would move
// another execution's record.
if step.ExecutionID != finish.ExecutionID || taskstate.StepStatus(step.Status) != finish.From {
return events.Event{}, storecontract.ErrStaleTransition
}
// Staleness is decided first, the way guardTransition decides it: a caller
// that is wrong about the row's state gets the stale sentinel even when
// its pair is also unroutable.
if !taskstate.CanStepTransition(finish.From, finish.To) {
return events.Event{}, fmt.Errorf("%w: %s -> %s is not a step transition",
storecontract.ErrIllegalTransition, finish.From, finish.To)
}
execution, err := q.TaskByID(ctx, step.ExecutionID)
if err != nil {
return events.Event{}, err
}
durationMS := int64(0)
if step.StartedAt.Valid {
durationMS = time.Since(step.StartedAt.Time).Milliseconds()
}
n, err := q.FinishStepExecution(ctx, postgresdb.FinishStepExecutionParams{
Status: string(finish.To), Detail: clip(finish.Detail, 4000), TokensUsed: finish.TokensUsed,
ID: finish.StepID, Status_2: string(finish.From),
})
if err != nil {
return events.Event{}, err
}
if n == 0 {
return events.Event{}, storecontract.ErrStaleTransition
}
event := stepEvent(events.KindStageFinish, execution.ID,
execution.Owner, execution.Repo, execution.IssueNumber, execution.Workflow,
int(step.Attempt), step.Name,
stepEventData(finish.To, durationMS, finish.Detail))
eventID, err := insertEventQ(ctx, q, event)
if err != nil {
return events.Event{}, err
}
event.ID = eventID
if err := tx.Commit(ctx); err != nil {
return events.Event{}, err
}
return event, nil
}
// stepEvent builds the event row one step transition writes, stamped with the
// execution's identity the way TaskContext.Emit stamped it -- the same fields
// the timeline and the stage statistics read back.
func stepEvent(kind string, executionID int64, owner, repo string, issue int64, wf string, attempt int, stage string, data map[string]any) events.Event {
return events.Event{
Kind: kind, TaskID: executionID,
Repo: owner + "/" + repo, Issue: int(issue), Workflow: wf,
Attempt: attempt, Stage: stage, Data: data,
}
}
// stepEventData reads a stage_finish event's data the way the dashboard
// (stageOutcome) and the stage statistics (StageStats) do: duration always,
// the error text only when the step reported failure, and the interruption
// marker only for an interrupted step.
func stepEventData(to taskstate.StepStatus, durationMS int64, detail string) map[string]any {
data := map[string]any{"duration_ms": durationMS}
switch to {
case taskstate.StepFailed:
data["error"] = detail
case taskstate.StepInterrupted:
data["interrupted"] = true
}
return data
}
// stepParentID encodes the nullable parent id an insert takes: 0 means "no
// parent" -- a stage at the tree's root -- which the column stores as NULL
// rather than as a reference to a row that does not exist.
func stepParentID(parentID int64) pgtype.Int8 {
return pgtype.Int8{Int64: parentID, Valid: parentID != 0}
}
// CancelExecution is the one cancel path (docs/prds/execution-tree-state-machine.md,
// "Cancellation"): one transaction moves every non-terminal StepExecution of
// the execution's current attempt to cancelled -- one event row each -- and
// the execution itself to the status the operator action names, under the
// shared execution transition table. Staleness is decided before legality, as
// guardTransition decides it, and the audit row lands in the same write. The
// caller cancels the in-memory context after this commits; the worker's next
// step write then fails ErrStaleTransition and stops.
func (s *Store) CancelExecution(ctx context.Context, taskID int64, reason, to string) ([]int64, error) {
tx, err := s.pool.Begin(ctx)
if err != nil {
return nil, err
}
defer func() { _ = tx.Rollback(ctx) }()
q := postgresdb.New(tx)
execution, err := q.LockExecutionForStep(ctx, taskID)
if errors.Is(err, pgx.ErrNoRows) {
return nil, storecontract.ErrStaleTransition
}
if err != nil {
return nil, err
}
if !taskstate.CanTransition(execution.Status, to) {
return nil, fmt.Errorf("%w: %s -> %s is not a transition",
storecontract.ErrIllegalTransition, execution.Status, to)
}
steps, err := q.CancelAttemptSteps(ctx, postgresdb.CancelAttemptStepsParams{
ExecutionID: taskID, Attempt: execution.Attempt,
})
if err != nil {
return nil, err
}
for _, step := range steps {
if _, err := insertEventQ(ctx, q, stepEvent(events.KindStageFinish, execution.ID,
execution.Owner, execution.Repo, execution.IssueNumber, execution.Workflow,
int(execution.Attempt), step.Name, map[string]any{"cancelled": true})); err != nil {
return nil, err
}
}
n, err := q.TransitionTask(ctx, postgresdb.TransitionTaskParams{
ID: taskID, Status: to, ParkReason: clip(reason, 4000),
ParkClass: taskstate.ParkNeedsHuman, Status_2: execution.Status,
})
if err != nil {
return nil, err
}
if n == 0 {
return nil, storecontract.ErrStaleTransition
}
if err := q.InsertTransition(ctx, postgresdb.InsertTransitionParams{
TaskID: taskID, FromStatus: execution.Status, ToStatus: to, Detail: clip(reason, 4000),
}); err != nil {
return nil, err
}
if err := tx.Commit(ctx); err != nil {
return nil, err
}
ids := make([]int64, 0, len(steps))
for _, step := range steps {
ids = append(ids, step.ID)
}
return ids, nil
}