feat(statestore): enforce the execution transition table and map ErrIllegalTransition over the wire

Every caller-supplied status write (Transition, ParkTask, Requeue,
RetryTask, BeginRemediation) now checks the shared taskstate transition
table inside the write's transaction, after the from guard decides
staleness. An off-table pair returns the new storecontract.ErrIllegalTransition,
carried over gRPC through mapError/unmapError on (FailedPrecondition,
"illegal transition"). Claim, BeginRemediation and RecoverStale pin
their pairs in SQL; a test pins those pairs to the table so a query
edit cannot move a write off it. Requeue/RetryTask share one guarded
requeue body. (archie-core-qdbo.5)
This commit is contained in:
sam committed 2026-09-26 13:13:20 +10:00
1 parent e7e3ea9d49
commit 71d98fe9b4
8 files changed
+274 -33

No files matched your search

@@ -399,6 +399,13 @@ var (
// ErrStaleTransition is returned when a task transition's expected
// from status does not match the task's current status.
ErrStaleTransition = errors.New("store: stale transition: task status does not match expected from status")
// ErrIllegalTransition is returned when a task's current status and the
// requested status form a pair the shared taskstate transition table
// does not route (docs/prds/execution-tree-state-machine.md): the move
// would have executed against a truthful from, but the table forbids
// it. Distinct from ErrStaleTransition, which covers a from that does
// not match the row's actual status.
ErrIllegalTransition = errors.New("store: illegal transition: status pair is not in the transition table")
// ErrBindingNotFound is returned when a binding ID does not exist.
ErrBindingNotFound = errors.New("store: binding not found")
// ErrBindingOverlap is returned when a binding's matcher overlaps an
@@ -484,6 +484,20 @@ func (q *Queries) ListTaskSummaries(ctx context.Context, limit int32) ([]ListTas
return items, nil
}
const lockTaskStatus = `-- name: LockTaskStatus :one
SELECT status FROM tasks WHERE id = $1 FOR UPDATE
`
// Locks the task's row and returns its current status, so the staleness and
// transition-table checks that decide a guarded write hold against a
// concurrent claim or transition until the update in the same transaction.
func (q *Queries) LockTaskStatus(ctx context.Context, id int64) (string, error) {
row := q.db.QueryRow(ctx, lockTaskStatus, id)
var status string
err := row.Scan(&status)
return status, err
}
const parkTask = `-- name: ParkTask :execrows
UPDATE tasks
SET status = 'parked', park_reason = $2, park_class = $3, updated_at = now()
@@ -73,6 +73,12 @@ UPDATE tasks SET workflow = $2, stage = $3, branch = $4, plan = $5, notes = $6,
updated_at = now()
WHERE id = $1;
-- name: LockTaskStatus :one
-- Locks the task's row and returns its current status, so the staleness and
-- transition-table checks that decide a guarded write hold against a
-- concurrent claim or transition until the update in the same transaction.
SELECT status FROM tasks WHERE id = $1 FOR UPDATE;
-- name: TransitionTask :execrows
-- Arriving at 'parked' also records why. The class is normalized by the
-- caller, so an unknown value persists as needs_human rather than as itself.
+68 -30
View File
@@ -37,6 +37,33 @@ func (s *Store) queries() *postgresdb.Queries {
return postgresdb.New(s.pool)
}
// guardTransition locks the task's row and checks the status write the caller
// is about to perform: a row whose status is not the expected from is stale
// (ErrStaleTransition), a from->to pair outside the shared transition table is
// refused with ErrIllegalTransition
// (docs/prds/execution-tree-state-machine.md). Staleness is decided first, so
// a caller that is wrong about the row's state gets the stale sentinel even
// when its pair is also unroutable. The row stays locked for the caller's
// transaction, so the checks cannot race the guarded write that follows.
// A missing row is stale, matching what the guarded update alone used to
// return.
func guardTransition(ctx context.Context, q *postgresdb.Queries, taskID int64, from, to string) error {
status, err := q.LockTaskStatus(ctx, taskID)
if errors.Is(err, pgx.ErrNoRows) {
return storecontract.ErrStaleTransition
}
if err != nil {
return err
}
if status != from {
return storecontract.ErrStaleTransition
}
if !taskstate.CanTransition(from, to) {
return storecontract.ErrIllegalTransition
}
return nil
}
// Close is a no-op: this store owns no file handle
// or connection -- the pool belongs to the composition that opened it.
func (s *Store) Close() error { return nil }
@@ -180,8 +207,9 @@ func (s *Store) ClaimByIssue(ctx context.Context, owner, repo string, number int
// Transition moves a task to a new status and records the audit detail. The
// from status guards the update; a mismatch returns ErrStaleTransition without
// writing an audit row. Transitioning to parked also stores detail as
// ParkReason in the same transaction.
// writing an audit row, and a from->to pair outside the shared transition
// table returns ErrIllegalTransition. Transitioning to parked also stores
// detail as ParkReason in the same transaction.
func (s *Store) Transition(ctx context.Context, taskID int64, from, to, detail string) error {
tx, err := s.pool.Begin(ctx)
if err != nil {
@@ -190,6 +218,9 @@ func (s *Store) Transition(ctx context.Context, taskID int64, from, to, detail s
defer func() { _ = tx.Rollback(ctx) }()
q := postgresdb.New(tx)
if err := guardTransition(ctx, q, taskID, from, to); err != nil {
return err
}
n, err := q.TransitionTask(ctx, postgresdb.TransitionTaskParams{
ID: taskID, Status: to, ParkReason: clip(detail, 4000),
ParkClass: taskstate.ParkNeedsHuman, Status_2: from,
@@ -210,6 +241,8 @@ func (s *Store) Transition(ctx context.Context, taskID int64, from, to, detail s
// ParkTask is the classified park write: the same guarded running->parked
// transition Transition performs, carrying the park class the site chose.
// The park is only legal from running (the transition table), so a stale or
// off-table from is refused before anything is written.
func (s *Store) ParkTask(ctx context.Context, taskID int64, from, detail, class string) error {
tx, err := s.pool.Begin(ctx)
if err != nil {
@@ -218,6 +251,9 @@ func (s *Store) ParkTask(ctx context.Context, taskID int64, from, detail, class
defer func() { _ = tx.Rollback(ctx) }()
q := postgresdb.New(tx)
if err := guardTransition(ctx, q, taskID, from, workflow.StatusParked); err != nil {
return err
}
n, err := q.ParkTask(ctx, postgresdb.ParkTaskParams{
ID: taskID, ParkReason: clip(detail, 4000),
ParkClass: taskstate.NormalizeParkClass(class), Status: from,
@@ -269,6 +305,9 @@ func (s *Store) BeginRemediation(ctx context.Context, taskID int64, payload stri
defer func() { _ = tx.Rollback(ctx) }()
q := postgresdb.New(tx)
if err := guardTransition(ctx, q, taskID, workflow.StatusPROpen, workflow.StatusQueued); err != nil {
return err
}
n, err := q.BeginRemediationTask(ctx, postgresdb.BeginRemediationTaskParams{
ID: taskID, ReviewPayload: clip(payload, 4000),
})
@@ -317,35 +356,31 @@ func (s *Store) SetReviewCursors(ctx context.Context, taskID, reviewCursor, comm
}
// Requeue puts a task back on the queue; an empty workflow keeps the task's
// current workflow.
// current workflow. The from->queued pair must be routed by the transition
// table: requeueing out of a terminal status is refused, not rewritten.
func (s *Store) Requeue(ctx context.Context, taskID int64, fromStatus, wf string) error {
tx, err := s.pool.Begin(ctx)
if err != nil {
return err
}
defer func() { _ = tx.Rollback(ctx) }()
q := postgresdb.New(tx)
n, err := q.RequeueTask(ctx, postgresdb.RequeueTaskParams{
ID: taskID, Workflow: wf, FromStatus: fromStatus,
return s.requeue(ctx, taskID, fromStatus, wf, "requeued "+wf, func(q *postgresdb.Queries) (int64, error) {
return q.RequeueTask(ctx, postgresdb.RequeueTaskParams{
ID: taskID, Workflow: wf, FromStatus: fromStatus,
})
})
if err != nil {
return err
}
if n == 0 {
return storecontract.ErrStaleTransition
}
if err := q.InsertTransition(ctx, postgresdb.InsertTransitionParams{
TaskID: taskID, FromStatus: fromStatus, ToStatus: workflow.StatusQueued, Detail: "requeued " + wf,
}); err != nil {
return err
}
return tx.Commit(ctx)
}
// RetryTask requeues a task and increments retry_count in the same guarded
// transaction.
// transaction, under the same table check Requeue applies.
func (s *Store) RetryTask(ctx context.Context, taskID int64, fromStatus, wf string) error {
return s.requeue(ctx, taskID, fromStatus, wf, "retried "+wf, func(q *postgresdb.Queries) (int64, error) {
return q.RetryTask(ctx, postgresdb.RetryTaskParams{
ID: taskID, Workflow: wf, FromStatus: fromStatus,
})
})
}
// requeue carries the guarded requeue write Requeue and RetryTask share: the
// transition-table check, the update guarded on the from status, and one audit
// row. update is the query that differs between the two callers; detail is the
// audit line it records.
func (s *Store) requeue(ctx context.Context, taskID int64, fromStatus, wf, detail string, update func(*postgresdb.Queries) (int64, error)) error {
tx, err := s.pool.Begin(ctx)
if err != nil {
return err
@@ -353,9 +388,10 @@ func (s *Store) RetryTask(ctx context.Context, taskID int64, fromStatus, wf stri
defer func() { _ = tx.Rollback(ctx) }()
q := postgresdb.New(tx)
n, err := q.RetryTask(ctx, postgresdb.RetryTaskParams{
ID: taskID, Workflow: wf, FromStatus: fromStatus,
})
if err := guardTransition(ctx, q, taskID, fromStatus, workflow.StatusQueued); err != nil {
return err
}
n, err := update(q)
if err != nil {
return err
}
@@ -363,7 +399,7 @@ func (s *Store) RetryTask(ctx context.Context, taskID int64, fromStatus, wf stri
return storecontract.ErrStaleTransition
}
if err := q.InsertTransition(ctx, postgresdb.InsertTransitionParams{
TaskID: taskID, FromStatus: fromStatus, ToStatus: workflow.StatusQueued, Detail: "retried " + wf,
TaskID: taskID, FromStatus: fromStatus, ToStatus: workflow.StatusQueued, Detail: detail,
}); err != nil {
return err
}
@@ -399,7 +435,9 @@ func (s *Store) ArchiveTask(ctx context.Context, taskID int64, fromStatus string
return eventID, nil
}
// RecoverStale re-queues tasks left running by a crashed daemon.
// RecoverStale re-queues tasks left running by a crashed daemon. The write is
// the table's crash-recovery edge (running->queued), pinned in SQL and pinned
// to the table by TestSQLPinnedTransitionsAreTableLegal.
func (s *Store) RecoverStale(ctx context.Context) (int64, error) {
return s.queries().RecoverStaleTasks(ctx)
}
@@ -0,0 +1,148 @@
package postgres
import (
"errors"
"testing"
"github.com/samcharles93/archie-core/internal/domain/storecontract"
"github.com/samcharles93/archie-core/internal/taskstate"
"github.com/samcharles93/archie-core/internal/taskstate/taskstatetest"
)
// Enforcement of the shared transition table (docs/prds/execution-tree-state-machine.md):
// a status write whose from->to pair the table does not route is refused with
// ErrIllegalTransition, leaves the row untouched and writes no audit row --
// the same three guarantees a stale from already had under ErrStaleTransition.
// TestIllegalTransitionsAreRejected drives every caller-supplied status write
// through an illegal pair and asserts the refusal is total: sentinel, status,
// audit trail.
func TestIllegalTransitionsAreRejected(t *testing.T) {
tests := []struct {
name string
seed string // status the task is parked in before the write
write func(s *Store, taskID int64) error
}{
{name: "transition queued->parked", seed: taskstate.Queued, write: func(s *Store, taskID int64) error {
return s.Transition(t.Context(), taskID, taskstate.Queued, taskstate.Parked, "why")
}},
{name: "transition running->merged", seed: taskstate.Running, write: func(s *Store, taskID int64) error {
return s.Transition(t.Context(), taskID, taskstate.Running, taskstate.Merged, "why")
}},
{name: "transition running->rejected", seed: taskstate.Running, write: func(s *Store, taskID int64) error {
return s.Transition(t.Context(), taskID, taskstate.Running, taskstate.Rejected, "why")
}},
{name: "transition parked->running", seed: taskstate.Parked, write: func(s *Store, taskID int64) error {
return s.Transition(t.Context(), taskID, taskstate.Parked, taskstate.Running, "why")
}},
{name: "transition waiting_human->pr_open", seed: taskstate.WaitingHuman, write: func(s *Store, taskID int64) error {
return s.Transition(t.Context(), taskID, taskstate.WaitingHuman, taskstate.PROpen, "why")
}},
{name: "transition out of terminal merged", seed: taskstate.Merged, write: func(s *Store, taskID int64) error {
return s.Transition(t.Context(), taskID, taskstate.Merged, taskstate.Queued, "why")
}},
{name: "park from queued", seed: taskstate.Queued, write: func(s *Store, taskID int64) error {
return s.ParkTask(t.Context(), taskID, taskstate.Queued, "why", taskstate.ParkTransient)
}},
{name: "park from pr_open", seed: taskstate.PROpen, write: func(s *Store, taskID int64) error {
return s.ParkTask(t.Context(), taskID, taskstate.PROpen, "why", taskstate.ParkTransient)
}},
{name: "requeue from terminal merged", seed: taskstate.Merged, write: func(s *Store, taskID int64) error {
return s.Requeue(t.Context(), taskID, taskstate.Merged, "implement")
}},
{name: "retry from terminal merged", seed: taskstate.Merged, write: func(s *Store, taskID int64) error {
return s.RetryTask(t.Context(), taskID, taskstate.Merged, "implement")
}},
{name: "retry from queued self-pair", seed: taskstate.Queued, write: func(s *Store, taskID int64) error {
return s.RetryTask(t.Context(), taskID, taskstate.Queued, "implement")
}},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
s := storeFor(t)
task, err := s.EnqueueChatTask(t.Context(), "acme", "widgets", "t", "b", "implement", "")
if err != nil || task == nil {
t.Fatalf("EnqueueChatTask: %+v %v", task, err)
}
taskstatetest.Seed(t.Context(), t, s, task.ID, taskstate.Queued, tt.seed, "seed")
before := len(transitions(t, s, task.ID))
err = tt.write(s, task.ID)
if !errors.Is(err, storecontract.ErrIllegalTransition) {
t.Fatalf("illegal write = %v, want ErrIllegalTransition", err)
}
got, err := s.TaskByID(t.Context(), task.ID)
if err != nil || got == nil {
t.Fatalf("TaskByID: %+v %v", got, err)
}
if got.Status != tt.seed {
t.Fatalf("status after refused write = %q, want %q", got.Status, tt.seed)
}
if n := len(transitions(t, s, task.ID)); n != before {
t.Fatalf("refused write added %d audit row(s), want none", n-before)
}
})
}
}
// Staleness still wins when the from is both untruthful and illegal: the
// row's own status is the ground truth a caller first has to be right about
// (a queued->merged write against a running task is stale, as the wire
// conformance battery has always asserted).
func TestStaleFromBeatsIllegalPair(t *testing.T) {
s := storeFor(t)
task, err := s.EnqueueChatTask(t.Context(), "acme", "widgets", "t", "b", "implement", "")
if err != nil || task == nil {
t.Fatalf("EnqueueChatTask: %+v %v", task, err)
}
taskstatetest.Seed(t.Context(), t, s, task.ID, taskstate.Queued, taskstate.Running, "claim")
err = s.Transition(t.Context(), task.ID, taskstate.Queued, taskstate.Merged, "why")
if !errors.Is(err, storecontract.ErrStaleTransition) {
t.Fatalf("untruthful from = %v, want ErrStaleTransition", err)
}
}
// A legal pair still lands: the guard must not reject what the table routes,
// and the audit row is written as before.
func TestLegalTransitionStillWrites(t *testing.T) {
s := storeFor(t)
task, err := s.EnqueueChatTask(t.Context(), "acme", "widgets", "t", "b", "implement", "")
if err != nil || task == nil {
t.Fatalf("EnqueueChatTask: %+v %v", task, err)
}
taskstatetest.Seed(t.Context(), t, s, task.ID, taskstate.Queued, taskstate.Running, "claim")
if err := s.Transition(t.Context(), task.ID, taskstate.Running, taskstate.Completed, "done"); err != nil {
t.Fatalf("legal transition running->completed = %v, want nil", err)
}
got, err := s.TaskByID(t.Context(), task.ID)
if err != nil || got == nil {
t.Fatalf("TaskByID: %+v %v", got, err)
}
if got.Status != taskstate.Completed {
t.Fatalf("status = %q, want %q", got.Status, taskstate.Completed)
}
rows := transitions(t, s, task.ID)
if n := len(rows); n == 0 || rows[n-1] != (transitionRow{taskstate.Running, taskstate.Completed, "done"}) {
t.Fatalf("last audit row = %+v, want the running->completed edge", rows)
}
}
// The status-changing methods whose from/to is pinned in SQL rather than
// supplied by a caller (claim, BeginRemediation, RecoverStale) can only write
// the pairs asserted here. Pinning them to the shared table keeps a query
// edit from moving a write off the table without this file failing.
func TestSQLPinnedTransitionsAreTableLegal(t *testing.T) {
tests := []struct {
name string
from, to string
}{
{name: "claim pins queued->running", from: taskstate.Queued, to: taskstate.Running},
{name: "begin remediation pins pr_open->queued", from: taskstate.PROpen, to: taskstate.Queued},
{name: "crash recovery pins running->queued", from: taskstate.Running, to: taskstate.Queued},
}
for _, tt := range tests {
if !taskstate.CanTransition(tt.from, tt.to) {
t.Errorf("%s: %s -> %s is not in the shared transition table", tt.name, tt.from, tt.to)
}
}
}
@@ -173,6 +173,16 @@ func TestStateStoreConformance(t *testing.T) {
t.Fatalf("Transition stale = %v, want ErrStaleTransition", err)
}
// Illegal transition: the from is truthful, but the pair the move
// would execute is outside the shared transition table
// (docs/prds/execution-tree-state-machine.md). The sentinel must
// survive the hop in both directions, which is what the gRPC mode
// of this battery exists to prove.
err = c.Transition(ctx, task.ID, "running", "merged", "")
if !errors.Is(err, storecontract.ErrIllegalTransition) {
t.Fatalf("Transition illegal = %v, want ErrIllegalTransition", err)
}
// Remediation starter: the guarded transition carries the review
// unit, and both guards are the wire contract (the sentinel must
// survive the hop for the reaction consumer's dedup to work).
@@ -185,6 +195,11 @@ func TestStateStoreConformance(t *testing.T) {
if err := c.BeginRemediation(ctx, task.ID, `{"review_id":7}`); !errors.Is(err, storecontract.ErrStaleTransition) {
t.Fatalf("BeginRemediation stale = %v, want ErrStaleTransition", err)
}
// ParkTask refuses an illegal pair too: a queued task cannot be
// parked, however truthful the from is.
if err := c.ParkTask(ctx, task.ID, "queued", "why", "transient"); !errors.Is(err, storecontract.ErrIllegalTransition) {
t.Fatalf("ParkTask illegal = %v, want ErrIllegalTransition", err)
}
remediated, err := c.TaskByID(ctx, task.ID)
if err != nil || remediated == nil {
t.Fatalf("TaskByID after BeginRemediation: %+v %v", remediated, err)
@@ -283,12 +298,22 @@ func TestStateStoreConformance(t *testing.T) {
if err != nil || claimed == nil {
t.Fatalf("ClaimNext: %+v %v", claimed, err)
}
// Requeue and RetryTask both land on queued through table-routed
// edges, and each guard is the wire contract. The battery used to
// walk queued -> queued through RetryTask, a self-write the shared
// transition table refuses, so the retry runs from running instead.
if err := c.RetryTask(ctx, claimed.ID, "running", ""); err != nil {
t.Fatalf("RetryTask: %v", err)
}
if err := c.RetryTask(ctx, claimed.ID, "queued", ""); !errors.Is(err, storecontract.ErrIllegalTransition) {
t.Fatalf("RetryTask queued->queued = %v, want ErrIllegalTransition", err)
}
if err := c.Transition(ctx, claimed.ID, "queued", "running", "re-claimed"); err != nil {
t.Fatalf("re-claim: %v", err)
}
if err := c.Requeue(ctx, claimed.ID, "running", ""); err != nil {
t.Fatalf("Requeue: %v", err)
}
if err := c.RetryTask(ctx, claimed.ID, "queued", ""); err != nil {
t.Fatalf("RetryTask: %v", err)
}
if _, err := c.RecoverStale(ctx); err != nil {
t.Fatalf("RecoverStale: %v", err)
}
@@ -76,6 +76,7 @@ func TestUnmapErrorSentinelFidelity(t *testing.T) {
rehyd error
}{
{name: "stale transition", store: storecontract.ErrStaleTransition, rehyd: storecontract.ErrStaleTransition},
{name: "illegal transition", store: storecontract.ErrIllegalTransition, rehyd: storecontract.ErrIllegalTransition},
{name: "binding not found", store: storecontract.ErrBindingNotFound, rehyd: storecontract.ErrBindingNotFound},
{name: "mapping not found", store: storecontract.ErrMappingNotFound, rehyd: storecontract.ErrMappingNotFound},
{name: "binding overlap", store: storecontract.ErrBindingOverlap, rehyd: storecontract.ErrBindingOverlap},
@@ -329,6 +329,7 @@ func dayTokensValue(d *pb.DayTokens) storecontract.DayTokens {
// matching (code, message), never by parsing free-form text.
const (
msgStaleTransition = "stale transition"
msgIllegalTransition = "illegal transition"
msgBindingNotFound = "binding not found"
msgMappingNotFound = "mapping not found"
msgEventTypeNotFound = "event type not found"
@@ -441,6 +442,7 @@ var wireErrors = []struct {
message string
}{
{storecontract.ErrStaleTransition, codes.FailedPrecondition, msgStaleTransition},
{storecontract.ErrIllegalTransition, codes.FailedPrecondition, msgIllegalTransition},
{storecontract.ErrBindingOverlap, codes.FailedPrecondition, msgBindingOverlap},
{storecontract.ErrBindingTransition, codes.FailedPrecondition, msgBindingTransition},
{storecontract.ErrSourceSigningStale, codes.FailedPrecondition, msgSourceSigning},