feat(statestore): CancelExecution, and one cancel path for daemon stop and operator actions

Plan item 5 of docs/prds/execution-tree-state-machine.md (archie-core-1217).
Until now a stop was a context cancel plus a caller-side transition, and the
step rows a worker still held running could never be told: the execution moved
while the work carried on, and the old interrupt-first ordering existed only
because the record could not defend itself. The step table can: the PRD's
ordering -- the store is the record, the context cancel is the delivery --
became safe the moment a worker's next step write fails ErrStaleTransition
against a cancelled step.

- storecontract.ExecutionCanceller and the CancelExecution RPC: one
  transaction moves every non-terminal StepExecution of the execution's
  current attempt to cancelled (one event row each, in the same
  transaction), and the execution itself to the status the operator action
  names, under the shared transition table with its audit row. Staleness is
  decided before legality, the way guardTransition decides it. Earlier
  attempts are never rewritten. Deliberately administrative: the
  deny-by-default grant rule covers it, pinned by a test.
- taskactions.Service routes stop, cancel, abandon and reject through it and
  the old Transition/CancelTask combinations are deleted; the daemon's
  in-memory cancel is demoted to the delivery hook the PRD names, called
  after the store record.
- The gateway's chat controller records first and delivers second, the same
  order, and records even when no runtime is wired here (a standalone
  gateway cannot reach the daemon's goroutines): the record is what unsticks
  a lost-cancel task, and the delivery is best effort. StopRunning and
  TaskRuntime.CancelRunningTasks are deleted outright -- no production
  caller remained once the task_action tool owned stopping, and the daemon's
  CancelRunningTasks/runningTasks.stopAll go with them.
- The gateway's chat adapter writes through CancelExecution instead of a
  bare Transition, so chat /cancel and the dashboard's action service land
  on the same path.

PRD decisions recorded here:
- Each cancelled step's event row is stage_finish with {cancelled: true} and
  the reason as its detail -- the exact-one-event-per-step-transition rule
  covers the pending -> cancelled transition too, even for a step that never
  ran.
- A parked cancel carries the needs_human class, matching the Transition
  path this replaces; the transient class stays with parkRunningTask's
  unwind parks, which now meet ErrStaleTransition for cancelled runs and are
  ignored there.
- Cancelling a wait:true callee is deferred to #1216's agent-call children:
  the callee's own tree and the wait flag are what make
  "cancelling a caller cancels each callee it waits on" decidable, and the
  callee writes it cannot fake are this issue's half of the contract.
- RecoverStale's interrupted sweep is plan item 6, untouched here.
This commit is contained in:
sam committed 2026-09-27 06:34:01 +10:00
1 parent 68c634edfe
commit cb2ca07d0b
29 files changed
+1646 -1028

No files matched your search

@@ -383,7 +383,7 @@ func startStateStoreGRPC(t *testing.T, local *pgstore.TaskDB) string {
staterpc.RegisterServer(server, staterpc.Deps{
Tasks: local, Captures: eda, Mappings: eda, Bindings: eda,
BindingDispatcher: eda, BindingTaskCreator: local, WorkflowCalls: local,
Steps: local,
Steps: local, Canceller: local,
})
go func() { _ = server.Serve(listener) }()
t.Cleanup(func() { server.Stop(); _ = listener.Close() })
+8 -3
View File
@@ -8,6 +8,7 @@ import (
"github.com/samcharles93/archie-core/internal/app/controlplane"
"github.com/samcharles93/archie-core/internal/config"
"github.com/samcharles93/archie-core/internal/domain/agent"
"github.com/samcharles93/archie-core/internal/domain/storecontract"
"github.com/samcharles93/archie-core/internal/gateway"
"github.com/samcharles93/archie-core/internal/tools"
)
@@ -57,10 +58,14 @@ func (b *boot) setupChatRuntime(ctx context.Context, cfg config.Config) error {
}
b.chatTasks = chatTasks
b.defaultChatIdentity = defaultChatIdentity
var canceller storecontract.ExecutionCanceller
if ec, ok := b.stateStore.(storecontract.ExecutionCanceller); ok {
canceller = ec
}
b.chatController = gateway.NewStoreTaskController(chatTaskControllerAdapter{
taskByID: b.stateStore.TaskByID,
requeue: b.stateStore.Requeue,
transition: b.stateStore.Transition,
taskByID: b.stateStore.TaskByID,
requeue: b.stateStore.Requeue,
cancelExecution: canceller.CancelExecution,
})
b.updateService = makeUpdateService(chatSetup{Cfg: config.NewHolder(cfg)})
return nil
+16 -9
View File
@@ -558,9 +558,9 @@ func (a chatTaskWriterAdapter) EnqueueChatTask(
}
type chatTaskControllerAdapter struct {
taskByID func(context.Context, int64) (*workflow.Task, error)
requeue func(context.Context, int64, string, string) error
transition func(context.Context, int64, string, string, string) error
taskByID func(context.Context, int64) (*workflow.Task, error)
requeue func(context.Context, int64, string, string) error
cancelExecution func(context.Context, int64, string, string) ([]int64, error)
}
func (a chatTaskControllerAdapter) ChatTaskStatus(ctx context.Context, taskID int64) (gateway.ChatTaskStatus, bool, error) {
@@ -593,12 +593,19 @@ func (a chatTaskControllerAdapter) CancelChatTask(ctx context.Context, taskID in
return fmt.Errorf("task %d is not chat-originated", taskID)
}
// An operator declining work lands in the same state whichever surface
// they used. This used to record StatusRejected while the dashboard
// recorded StatusClosedWontDo, so the same decision showed up as two
// different states -- and StatusRejected, which the PR reconciler uses
// for "the pull request was closed without merging", stopped meaning one
// thing.
return a.transition(ctx, taskID, task.Status, workflow.StatusClosedWontDo, reason)
// they used, through the one cancel path: the store records the
// cancellation -- every non-terminal step of the current attempt plus the
// execution's own move (docs/prds/execution-tree-state-machine.md) -- and
// this write is what a late worker's next step write fails against. This
// used to record StatusRejected while the dashboard recorded
// StatusClosedWontDo, so the same decision showed up as two different
// states -- and StatusRejected, which the PR reconciler uses for "the
// pull request was closed without merging", stopped meaning one thing.
if a.cancelExecution == nil {
return fmt.Errorf("execution cancellation is unavailable")
}
_, err = a.cancelExecution(ctx, taskID, reason, workflow.StatusClosedWontDo)
return err
}
// chatTaskLogReaderAdapter gives the gateway a read view of a task's
+13 -13
View File
@@ -669,16 +669,16 @@ func TestChatTaskControllerAdapterRejectsForgeTask(t *testing.T) {
func TestChatTaskControllerAdapterTransitions(t *testing.T) {
task := &workflow.Task{ID: 42, Source: workflow.SourceChat, Status: workflow.StatusWaitingHuman}
var requeueFrom, requeueWorkflow, transitionFrom, transitionTo, transitionDetail string
var requeueFrom, requeueWorkflow, cancelReason, cancelTo string
adapter := chatTaskControllerAdapter{
taskByID: func(context.Context, int64) (*workflow.Task, error) { return task, nil },
requeue: func(_ context.Context, _ int64, from, workflow string) error {
requeueFrom, requeueWorkflow = from, workflow
return nil
},
transition: func(_ context.Context, _ int64, from, to, detail string) error {
transitionFrom, transitionTo, transitionDetail = from, to, detail
return nil
cancelExecution: func(_ context.Context, _ int64, reason, to string) ([]int64, error) {
cancelReason, cancelTo = reason, to
return nil, nil
},
}
@@ -692,12 +692,12 @@ func TestChatTaskControllerAdapterTransitions(t *testing.T) {
t.Fatalf("CancelChatTask(): %v", err)
}
// Declining from chat lands in the same state as declining from the
// dashboard. It used to record StatusRejected, which the PR reconciler
// also uses for "the pull request was closed without merging", so the
// state could not tell an operator's decision from a forge outcome.
if transitionFrom != workflow.StatusWaitingHuman || transitionTo != workflow.StatusClosedWontDo ||
transitionDetail != "cancelled by test" {
t.Errorf("transition = %q/%q/%q", transitionFrom, transitionTo, transitionDetail)
// dashboard, through the one cancel path. It used to record
// StatusRejected, which the PR reconciler also uses for "the pull
// request was closed without merging", so the state could not tell an
// operator's decision from a forge outcome.
if cancelReason != "cancelled by test" || cancelTo != workflow.StatusClosedWontDo {
t.Errorf("cancel = %q/%q", cancelReason, cancelTo)
}
}
@@ -834,9 +834,9 @@ func TestChatTaskCommandsEndToEnd(t *testing.T) {
profiles,
)
controller := gateway.NewStoreTaskController(chatTaskControllerAdapter{
taskByID: st.TaskByID,
requeue: st.Requeue,
transition: st.Transition,
taskByID: st.TaskByID,
requeue: st.Requeue,
cancelExecution: st.CancelExecution,
})
router := gateway.NewRouter(st, nil, "test")
configureTaskCommands(router, creator, controller, chatTaskListerAdapter{tasks: st.Tasks}, defaultIdentity)
+3
View File
@@ -349,6 +349,9 @@ func (b *boot) stateStoreDeps(grants *staterpc.TaskGrants) staterpc.Deps {
if sr, ok := b.st.(storecontract.StepRecorder); ok {
deps.Steps = sr
}
if ec, ok := b.st.(storecontract.ExecutionCanceller); ok {
deps.Canceller = ec
}
b.accessDeps(&deps)
if css, ok := b.st.(storecontract.ConfigSnapshotStore); ok {
deps.ConfigSnapshots = css
+7 -3
View File
@@ -76,10 +76,14 @@ func TestDashboardAndChatAgreeOnTerminalStates(t *testing.T) {
// controller path and fails the test on error.
func runChatAction(t *testing.T, ctx context.Context, st storecontract.TaskStore, action string, taskID int64) {
t.Helper()
canceller, ok := st.(storecontract.ExecutionCanceller)
if !ok {
t.Fatal("the task store cannot cancel executions")
}
controller := gateway.NewStoreTaskController(chatTaskControllerAdapter{
taskByID: st.TaskByID,
requeue: st.Requeue,
transition: st.Transition,
taskByID: st.TaskByID,
requeue: st.Requeue,
cancelExecution: canceller.CancelExecution,
})
var err error
if action == "approve" {
File diff suppressed because it is too large. Load diff
@@ -42,6 +42,7 @@ const (
StateStoreService_Update_FullMethodName = "/state.v1.StateStoreService/Update"
StateStoreService_StartStep_FullMethodName = "/state.v1.StateStoreService/StartStep"
StateStoreService_FinishStep_FullMethodName = "/state.v1.StateStoreService/FinishStep"
StateStoreService_CancelExecution_FullMethodName = "/state.v1.StateStoreService/CancelExecution"
StateStoreService_Requeue_FullMethodName = "/state.v1.StateStoreService/Requeue"
StateStoreService_ParkTask_FullMethodName = "/state.v1.StateStoreService/ParkTask"
StateStoreService_RecoverStale_FullMethodName = "/state.v1.StateStoreService/RecoverStale"
@@ -155,6 +156,10 @@ type StateStoreServiceClient interface {
// credential authorises both on its own execution only.
StartStep(ctx context.Context, in *StartStepRequest, opts ...grpc.CallOption) (*StartStepResponse, error)
FinishStep(ctx context.Context, in *FinishStepRequest, opts ...grpc.CallOption) (*FinishStepResponse, error)
// Cancellation. Deliberately administrative (deny-by-default for a
// task-scoped grant): a cancel is a dashboard, API and dispatch action
// decided by the Authorizer, not a worker call.
CancelExecution(ctx context.Context, in *CancelExecutionRequest, opts ...grpc.CallOption) (*CancelExecutionResponse, error)
Requeue(ctx context.Context, in *RequeueRequest, opts ...grpc.CallOption) (*RequeueResponse, error)
ParkTask(ctx context.Context, in *ParkTaskRequest, opts ...grpc.CallOption) (*ParkTaskResponse, error)
RecoverStale(ctx context.Context, in *RecoverStaleRequest, opts ...grpc.CallOption) (*RecoverStaleResponse, error)
@@ -518,6 +523,16 @@ func (c *stateStoreServiceClient) FinishStep(ctx context.Context, in *FinishStep
return out, nil
}
func (c *stateStoreServiceClient) CancelExecution(ctx context.Context, in *CancelExecutionRequest, opts ...grpc.CallOption) (*CancelExecutionResponse, error) {
cOpts := append([]grpc.CallOption{grpc.StaticMethod()}, opts...)
out := new(CancelExecutionResponse)
err := c.cc.Invoke(ctx, StateStoreService_CancelExecution_FullMethodName, in, out, cOpts...)
if err != nil {
return nil, err
}
return out, nil
}
func (c *stateStoreServiceClient) Requeue(ctx context.Context, in *RequeueRequest, opts ...grpc.CallOption) (*RequeueResponse, error) {
cOpts := append([]grpc.CallOption{grpc.StaticMethod()}, opts...)
out := new(RequeueResponse)
@@ -1259,6 +1274,10 @@ type StateStoreServiceServer interface {
// credential authorises both on its own execution only.
StartStep(context.Context, *StartStepRequest) (*StartStepResponse, error)
FinishStep(context.Context, *FinishStepRequest) (*FinishStepResponse, error)
// Cancellation. Deliberately administrative (deny-by-default for a
// task-scoped grant): a cancel is a dashboard, API and dispatch action
// decided by the Authorizer, not a worker call.
CancelExecution(context.Context, *CancelExecutionRequest) (*CancelExecutionResponse, error)
Requeue(context.Context, *RequeueRequest) (*RequeueResponse, error)
ParkTask(context.Context, *ParkTaskRequest) (*ParkTaskResponse, error)
RecoverStale(context.Context, *RecoverStaleRequest) (*RecoverStaleResponse, error)
@@ -1461,6 +1480,9 @@ func (UnimplementedStateStoreServiceServer) StartStep(context.Context, *StartSte
func (UnimplementedStateStoreServiceServer) FinishStep(context.Context, *FinishStepRequest) (*FinishStepResponse, error) {
return nil, status.Error(codes.Unimplemented, "method FinishStep not implemented")
}
func (UnimplementedStateStoreServiceServer) CancelExecution(context.Context, *CancelExecutionRequest) (*CancelExecutionResponse, error) {
return nil, status.Error(codes.Unimplemented, "method CancelExecution not implemented")
}
func (UnimplementedStateStoreServiceServer) Requeue(context.Context, *RequeueRequest) (*RequeueResponse, error) {
return nil, status.Error(codes.Unimplemented, "method Requeue not implemented")
}
@@ -2097,6 +2119,24 @@ func _StateStoreService_FinishStep_Handler(srv interface{}, ctx context.Context,
return interceptor(ctx, in, info, handler)
}
func _StateStoreService_CancelExecution_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) {
in := new(CancelExecutionRequest)
if err := dec(in); err != nil {
return nil, err
}
if interceptor == nil {
return srv.(StateStoreServiceServer).CancelExecution(ctx, in)
}
info := &grpc.UnaryServerInfo{
Server: srv,
FullMethod: StateStoreService_CancelExecution_FullMethodName,
}
handler := func(ctx context.Context, req interface{}) (interface{}, error) {
return srv.(StateStoreServiceServer).CancelExecution(ctx, req.(*CancelExecutionRequest))
}
return interceptor(ctx, in, info, handler)
}
func _StateStoreService_Requeue_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) {
in := new(RequeueRequest)
if err := dec(in); err != nil {
@@ -3381,6 +3421,10 @@ var StateStoreService_ServiceDesc = grpc.ServiceDesc{
MethodName: "FinishStep",
Handler: _StateStoreService_FinishStep_Handler,
},
{
MethodName: "CancelExecution",
Handler: _StateStoreService_CancelExecution_Handler,
},
{
MethodName: "Requeue",
Handler: _StateStoreService_Requeue_Handler,
-33
View File
@@ -2,7 +2,6 @@ package daemon
import (
"context"
"slices"
"sort"
"sync"
)
@@ -63,32 +62,6 @@ func (r *runningTasks) stop(id int64) bool {
return true
}
// stopAll cancels every running task belonging to identity, or all of them
// when identity is empty. It returns the IDs it cancelled, in order.
func (r *runningTasks) stopAll(identity string) []int64 {
r.mu.Lock()
var (
ids []int64
cancels []context.CancelFunc
)
for id, entry := range r.entries {
if identity != "" && entry.identity != identity {
continue
}
ids = append(ids, id)
cancels = append(cancels, entry.cancel)
}
r.mu.Unlock()
// Cancel outside the lock: a cancel can synchronously unwind a task
// whose cleanup calls back in here to retire itself.
for _, cancel := range cancels {
cancel()
}
slices.Sort(ids)
return ids
}
// list returns the tasks currently running, for reporting.
func (r *runningTasks) list() []RunningTask {
r.mu.Lock()
@@ -109,12 +82,6 @@ func (d *Daemon) CancelTask(id int64) bool {
return d.running.stop(id)
}
// CancelRunningTasks stops every task running for identity, or all of them
// when identity is empty, and returns the IDs it stopped.
func (d *Daemon) CancelRunningTasks(identity string) []int64 {
return d.running.stopAll(identity)
}
// RunningTasks lists the tasks currently executing.
func (d *Daemon) RunningTasks() []RunningTask {
return d.running.list()
-44
View File
@@ -68,47 +68,3 @@ func TestFinishedTaskIsNotCancellable(t *testing.T) {
t.Errorf("list() = %v, want empty after the task finished", got)
}
}
// TestStopAllIsIdentityScoped keeps one identity's /stop from reaching
// another's work.
func TestStopAllIsIdentityScoped(t *testing.T) {
t.Parallel()
var r runningTasks
base := context.Background()
first, doneFirst := r.begin(base, 1, "archie")
defer doneFirst()
other, doneOther := r.begin(base, 2, "winter")
defer doneOther()
third, doneThird := r.begin(base, 3, "archie")
defer doneThird()
stopped := r.stopAll("archie")
if len(stopped) != 2 || stopped[0] != 1 || stopped[1] != 3 {
t.Fatalf("stopAll = %v, want [1 3] in order", stopped)
}
if other.Err() != nil {
t.Error("another identity's task was cancelled")
}
if first.Err() == nil || third.Err() == nil {
t.Error("a task reported as stopped still has a live context")
}
}
// TestStopAllEmptyIdentityStopsEverything covers the operator-wide brake.
func TestStopAllEmptyIdentityStopsEverything(t *testing.T) {
t.Parallel()
var r runningTasks
base := context.Background()
_, doneArchie := r.begin(base, 1, "archie")
defer doneArchie()
_, doneWinter := r.begin(base, 2, "winter")
defer doneWinter()
if got := r.stopAll(""); len(got) != 2 {
t.Errorf("stopAll(\"\") = %v, want both tasks", got)
}
}
@@ -75,6 +75,21 @@ type StepRecorder interface {
FinishStep(ctx context.Context, s task.StepFinish) (events.Event, error)
}
// ExecutionCanceller is the one cancel path
// (docs/prds/execution-tree-state-machine.md, "Cancellation"): one
// transaction moves every non-terminal StepExecution of the execution's
// current attempt to cancelled and the execution itself to the status the
// operator action names (parked or closed_wont_do, as the transition table
// routes). The daemon then cancels the in-memory context that was delivering
// the run: the store is the record of cancellation, the context cancel is
// only the delivery mechanism, so a lost cancel still leaves the store
// correct and the worker's next step write fails ErrStaleTransition. It is a
// dashboard, API and dispatch action, so a task-scoped grant never reaches
// it; the Authorizer decides.
type ExecutionCanceller interface {
CancelExecution(ctx context.Context, taskID int64, reason, to string) ([]int64, error)
}
// TaskArchiver removes one terminal task's local record with an optimistic
// status guard. It is separate from the already broad lifecycle contract so
// consumers that only run tasks do not acquire an operator-only capability.
@@ -0,0 +1,93 @@
package taskactions
import (
"context"
"slices"
"testing"
"github.com/samcharles93/archie-core/internal/events"
"github.com/samcharles93/archie-core/internal/taskstate"
)
// Every operator action that ends a run's work routes through the one cancel
// path, CancelExecution (docs/prds/execution-tree-state-machine.md,
// "Cancellation"): the store records the cancellation first, then the daemon
// cancels the in-memory context that was delivering the run. The actions'
// old Transition-then-interrupt combinations are gone, not alongside.
type cancelRecordingStore struct {
fakeStore
cancels []cancelCall
}
type cancelCall struct {
taskID int64
reason, to string
}
func (f *cancelRecordingStore) CancelExecution(_ context.Context, taskID int64, reason, to string) ([]int64, error) {
f.cancels = append(f.cancels, cancelCall{taskID: taskID, reason: reason, to: to})
return nil, nil
}
func TestOperatorActionsRouteThroughCancelExecution(t *testing.T) {
tests := []struct {
name string
status string
action taskstate.Action
wantTo string
}{
{name: "stop parks the run", status: taskstate.Running, action: taskstate.ActionStop, wantTo: taskstate.Parked},
{name: "cancel declines a queued run", status: taskstate.Queued, action: taskstate.ActionCancel, wantTo: taskstate.Declined},
{name: "abandon declines a parked run", status: taskstate.Parked, action: taskstate.ActionAbandon, wantTo: taskstate.Declined},
{name: "reject declines a running run", status: taskstate.Running, action: taskstate.ActionReject, wantTo: taskstate.Declined},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
store := &cancelRecordingStore{task: &Task{ID: 7, Owner: "acme", Repo: "widgets", Status: tt.status, IssueNumber: 3}}
var deliveries []int64
service := Service{
Store: store,
CancelTask: func(id int64) bool { deliveries = append(deliveries, id); return true },
}
if err := service.Apply(context.Background(), nil, humanActor(), 7, tt.action); err != nil {
t.Fatalf("Apply(%s): %v", tt.action, err)
}
if len(store.cancels) != 1 {
t.Fatalf("CancelExecution calls = %+v, want exactly the one cancel path", store.cancels)
}
if call := store.cancels[0]; call.taskID != 7 || call.to != tt.wantTo {
t.Errorf("CancelExecution = %+v, want task 7 to %q", call, tt.wantTo)
}
// The context cancel is the delivery mechanism, not a second
// store write: it rides the store record, never beside it.
if tt.status == taskstate.Running && !slices.Contains(deliveries, int64(7)) {
t.Errorf("deliveries = %v, want the running execution's context cancelled", deliveries)
}
// The action's own attribution event is the operator's record;
// the store wrote the mechanical rows.
if got := store.last(); got.Kind == "" {
t.Error("the operator event was not recorded")
}
})
}
}
// A stop's detail names the delivery, as before: a context cancel that found
// nothing executing says so, and the parked execution is recorded either way.
func TestStopRecordsEvenWhenNothingWasExecuting(t *testing.T) {
store := &cancelRecordingStore{task: &Task{ID: 7, Owner: "acme", Repo: "widgets", Status: taskstate.Running}}
service := Service{
Store: store,
CancelTask: func(int64) bool { return false },
}
if err := service.Apply(context.Background(), nil, humanActor(), 7, taskstate.ActionStop); err != nil {
t.Fatalf("Apply(stop): %v", err)
}
if len(store.cancels) != 1 || store.cancels[0].to != taskstate.Parked {
t.Fatalf("CancelExecution = %+v, want task 7 parked", store.cancels)
}
if got := store.last(); got.Kind != events.KindTaskStopped || got.Detail == "" {
t.Fatalf("stop event = %+v, want the stop kind and the no-execution detail", got)
}
}
+42 -7
View File
@@ -124,6 +124,13 @@ type Store interface {
RetryTask(context.Context, int64, string, string) error
ArchiveTask(context.Context, int64, string, events.Event) (int64, error)
InsertEvent(context.Context, events.Event) (int64, error)
// CancelExecution is the one cancel path
// (docs/prds/execution-tree-state-machine.md, "Cancellation"): the store
// records the cancellation -- every non-terminal step of the current
// attempt plus the execution's own move to `to` -- in one transaction.
// The service then cancels the in-memory context that was delivering the
// run; a worker's next step write fails ErrStaleTransition and stops.
CancelExecution(context.Context, int64, string, string) ([]int64, error)
}
// Service runs in the daemon, which owns execution cancellation and events.
@@ -189,11 +196,14 @@ func (s Service) apply(ctx context.Context, task *Task, actor Actor, action task
case taskstate.ActionStop:
return s.applyStop(ctx, task, actor, o)
case taskstate.ActionReject:
o.event.Kind, o.event.Detail, o.verb = rejectedKind(actor), actor.describe("rejected"), "rejected"
if task.Status == "running" && s.CancelTask != nil {
s.CancelTask(task.ID)
// A rejection ends the run's work wherever it sits, so it rides the
// one cancel path like every other operator stop.
o.event.Kind, o.event.Detail, o.verb = rejectedKind(actor), actor.describe("rejected"), "declined"
if _, err := s.Store.CancelExecution(ctx, task.ID, actor.describe("rejected"), "closed_wont_do"); err != nil {
return o, err
}
return o, s.Store.Transition(ctx, task.ID, task.Status, "closed_wont_do", actor.describe("declined"))
s.deliver(task.ID)
return o, nil
case taskstate.ActionCancel, taskstate.ActionAbandon:
return s.applyCancelOrAbandon(ctx, task, action, actor, o)
case taskstate.ActionArchive:
@@ -203,6 +213,17 @@ func (s Service) apply(ctx context.Context, task *Task, actor Actor, action task
}
}
// deliver cancels the in-memory context of a run the store has already
// recorded as cancelled. It is the delivery mechanism, never a second store
// write, and it follows the record -- the PRD's order, which the step rows
// make safe: a worker that keeps writing after the record fails
// ErrStaleTransition on its next step write.
func (s Service) deliver(taskID int64) {
if s.CancelTask != nil {
s.CancelTask(taskID)
}
}
// attributed builds the event skeleton every action carries: the task, the run
// and both identities. Every event this service records passes through here, so
// no action can be recorded without its attribution.
@@ -244,10 +265,17 @@ func (s Service) applyStop(ctx context.Context, task *Task, actor Actor, o outco
return o, ErrUnavailable
}
o.event.Kind, o.event.Detail = events.KindTaskStopped, actor.describe("stopped")+"; recoverable work remains parked"
// The store records the stop first, then the context delivers it. A
// context cancel that found nothing executing says so; the parked
// execution is recorded either way, and the task's own unwind cannot win
// the race any more -- its next step write is already stale.
if _, err := s.Store.CancelExecution(ctx, task.ID, o.event.Detail, "parked"); err != nil {
return o, err
}
if !s.CancelTask(task.ID) {
o.event.Detail = "no active execution was found; recoverable work was parked, " + actor.describe("stopped")
}
return o, s.Store.Transition(ctx, task.ID, "running", "parked", o.event.Detail)
return o, nil
}
func (s Service) applyCancelOrAbandon(ctx context.Context, task *Task, action taskstate.Action, actor Actor, o outcome) (outcome, error) {
@@ -255,10 +283,17 @@ func (s Service) applyCancelOrAbandon(ctx context.Context, task *Task, action ta
// Abandoning ends archie's run, not the issue: no verb, so the issue
// stays open for a human.
o.event.Kind, o.event.Detail = events.KindTaskAbandoned, actor.describe("abandoned")
return o, s.Store.Transition(ctx, task.ID, "parked", "closed_wont_do", o.event.Detail)
if _, err := s.Store.CancelExecution(ctx, task.ID, o.event.Detail, "closed_wont_do"); err != nil {
return o, err
}
return o, nil
}
o.event.Kind, o.event.Detail, o.verb = events.KindTaskCancelled, actor.describe("cancelled"), "cancelled"
return o, s.Store.Transition(ctx, task.ID, "queued", "closed_wont_do", o.event.Detail)
if _, err := s.Store.CancelExecution(ctx, task.ID, o.event.Detail, "closed_wont_do"); err != nil {
return o, err
}
s.deliver(task.ID)
return o, nil
}
func (s Service) applyArchive(ctx context.Context, task *Task, actor Actor, o outcome) (outcome, error) {
@@ -19,6 +19,10 @@ func (f *fakeStore) Transition(context.Context, int64, string, string, string) e
return nil
}
func (f *fakeStore) CancelExecution(context.Context, int64, string, string) ([]int64, error) {
return nil, nil
}
func (f *fakeStore) Requeue(context.Context, int64, string, string) error { return nil }
func (f *fakeStore) RetryTask(context.Context, int64, string, string) error {
return nil
+6 -9
View File
@@ -134,16 +134,13 @@ type TaskController interface {
// error (surfaced to the user, not the LLM) if the task doesn't
// exist, isn't owned by identity, or isn't in waiting_human.
Approve(ctx context.Context, taskID int64, identity string) error
// Cancel moves an active task to a rejected/cancelled terminal
// state, interrupting it first if it is running. Returns an error if
// the task doesn't exist, isn't owned by identity, or is already
// terminal.
// Cancel moves an active task to a terminal state through the one
// cancel path (docs/prds/execution-tree-state-machine.md): the store
// records the cancellation first, and the runtime delivers it by
// cancelling the run's in-memory context where this process owns one.
// Returns an error if the task doesn't exist, isn't owned by identity,
// or is already terminal.
Cancel(ctx context.Context, taskID int64, identity string) error
// StopRunning interrupts every task currently executing for
// identity and returns the IDs it stopped. It is the agent half of
// /stop, so it must not require knowing a task ID: someone reaching
// for the brake is not in a position to look one up.
StopRunning(ctx context.Context, identity string) ([]int64, error)
}
// Router dispatches inbound messages. Gateway-local commands (like
+14 -34
View File
@@ -146,14 +146,12 @@ type chatTaskController interface {
// TaskRuntime reaches tasks that are currently executing. The daemon
// implements it; the store cannot, because a row records what a task is
// meant to be doing, not the goroutine doing it.
// meant to be doing, not the goroutine doing it. It is only the delivery
// mechanism: the store's CancelExecution record precedes it.
type TaskRuntime interface {
// CancelTask interrupts one running task, reporting whether it was
// running.
CancelTask(taskID int64) bool
// CancelRunningTasks interrupts every task running for identity, or
// all of them when identity is empty, returning the IDs stopped.
CancelRunningTasks(identity string) []int64
}
// StoreTaskController implements TaskController backed by a store
@@ -197,38 +195,20 @@ func (c *StoreTaskController) Cancel(ctx context.Context, taskID int64, identity
if err := taskstate.CheckDecline(st.Status); err != nil {
return err
}
switch st.Status {
case taskstate.Running:
// Cancelling a running task used to be refused outright, which
// left the only case worth interrupting as the one case that
// could not be. Interrupt the work first, then record it: the
// order matters, because writing the terminal state while the
// task is still executing invites it to write its own state
// afterwards and win.
if c.runtime == nil {
return fmt.Errorf("task is running and no runtime control is configured")
}
if !c.runtime.CancelTask(taskID) {
// The store says running but nothing is executing -- a
// crashed or migrated daemon. Recording the terminal state
// is still right, and is what unsticks the task.
break
}
// The store records the cancellation first -- the one cancel path -- and
// the context cancel delivers it. The order is the PRD's: a worker that
// keeps writing after the record fails ErrStaleTransition on its next
// step write, so it cannot win the race the old interrupt-first ordering
// guarded against. A task the store says is running but nothing is
// executing for (a crashed or migrated daemon) is unstuck by the record
// alone, which is why the delivery is reported, not required.
if err := c.store.CancelChatTask(ctx, taskID, "declined by "+identity); err != nil {
return err
}
return c.store.CancelChatTask(ctx, taskID, "declined by "+identity)
}
// StopRunning interrupts every task executing for identity.
//
// It deliberately does not write terminal states. A stopped task unwinds
// through its own error paths and lands wherever that leaves it, usually
// parked, which is recoverable. Forcing every one to rejected would
// discard work the operator may well want to resume.
func (c *StoreTaskController) StopRunning(_ context.Context, identity string) ([]int64, error) {
if c.runtime == nil {
return nil, fmt.Errorf("no runtime control is configured")
if st.Status == taskstate.Running && c.runtime != nil {
c.runtime.CancelTask(taskID)
}
return c.runtime.CancelRunningTasks(identity), nil
return nil
}
func (c *StoreTaskController) authorize(ctx context.Context, taskID int64, identity string) (ChatTaskStatus, error) {
+11 -43
View File
@@ -271,8 +271,6 @@ func TestStoreTaskControllerCancelActiveStates(t *testing.T) {
type fakeTaskRuntime struct {
cancelled []int64
running bool
stopped []int64
identity string
}
func (f *fakeTaskRuntime) CancelTask(taskID int64) bool {
@@ -280,14 +278,9 @@ func (f *fakeTaskRuntime) CancelTask(taskID int64) bool {
return f.running
}
func (f *fakeTaskRuntime) CancelRunningTasks(identity string) []int64 {
f.identity = identity
return f.stopped
}
// TestStoreTaskControllerCancelInterruptsRunning covers the case that used
// to be refused. A running task is the only one worth interrupting, so
// cancelling it must reach the work and then record the outcome.
// to be refused. The store records first, then the runtime delivers the
// context cancel; both happen.
func TestStoreTaskControllerCancelInterruptsRunning(t *testing.T) {
store := &fakeChatTaskStore{statuses: map[int64]ChatTaskStatus{1: {Status: taskstate.Running}}}
rt := &fakeTaskRuntime{running: true}
@@ -320,44 +313,19 @@ func TestStoreTaskControllerCancelRecordsStaleRunning(t *testing.T) {
}
}
// TestStoreTaskControllerCancelWithoutRuntimeRefusesRunning keeps the
// refusal honest when nothing can interrupt the work: recording the task
// as cancelled while it carries on would be a lie.
func TestStoreTaskControllerCancelWithoutRuntimeRefusesRunning(t *testing.T) {
// TestStoreTaskControllerCancelWithoutRuntimeStillRecords pins the PRD's
// delivery contract: the store records the cancellation even when nothing in
// this process can deliver it, because a worker's next step write fails
// ErrStaleTransition and stops -- the record is what unsticks the task.
func TestStoreTaskControllerCancelWithoutRuntimeStillRecords(t *testing.T) {
store := &fakeChatTaskStore{statuses: map[int64]ChatTaskStatus{1: {Status: taskstate.Running}}}
c := NewStoreTaskController(store)
err := c.Cancel(context.Background(), 1, "")
if err == nil || !strings.Contains(err.Error(), "no runtime control") {
t.Errorf("err = %v, want a refusal naming the missing runtime", err)
if err := c.Cancel(context.Background(), 1, ""); err != nil {
t.Fatalf("Cancel: %v", err)
}
if len(store.cancelled) != 0 {
t.Error("CancelChatTask must not be called when the task cannot be interrupted")
}
}
// TestStoreTaskControllerStopRunningIsIdentityScoped checks the agent half
// of /stop reaches the runtime and stays within the caller's identity.
func TestStoreTaskControllerStopRunningIsIdentityScoped(t *testing.T) {
rt := &fakeTaskRuntime{stopped: []int64{4, 7}}
c := NewStoreTaskController(&fakeChatTaskStore{statuses: map[int64]ChatTaskStatus{}}).WithRuntime(rt)
stopped, err := c.StopRunning(context.Background(), "archie")
if err != nil {
t.Fatalf("StopRunning: %v", err)
}
if len(stopped) != 2 || stopped[0] != 4 || stopped[1] != 7 {
t.Errorf("stopped = %v, want [4 7]", stopped)
}
if rt.identity != "archie" {
t.Errorf("identity = %q, want the caller's identity", rt.identity)
}
}
func TestStoreTaskControllerStopRunningWithoutRuntime(t *testing.T) {
c := NewStoreTaskController(&fakeChatTaskStore{statuses: map[int64]ChatTaskStatus{}})
if _, err := c.StopRunning(context.Background(), ""); err == nil {
t.Error("StopRunning reported success with no runtime wired")
if len(store.cancelled) != 1 {
t.Error("the cancellation was not recorded without a runtime: the record is what unsticks a lost-cancel task")
}
}
@@ -0,0 +1,194 @@
package postgres
import (
"errors"
"testing"
"github.com/samcharles93/archie-core/internal/domain/storecontract"
task "github.com/samcharles93/archie-core/internal/domain/workflow/task"
"github.com/samcharles93/archie-core/internal/events"
"github.com/samcharles93/archie-core/internal/taskstate"
)
// CancelExecution is the one cancel path (docs/prds/execution-tree-state-machine.md,
// "Cancellation"): one transaction cancels every non-terminal StepExecution of
// the execution's current attempt and moves the execution to the status the
// operator action names, with one event row per step transition and the
// execution's own audit row in the same write.
func TestCancelExecutionCancelsTheAttemptsStepsAndTheExecution(t *testing.T) {
s := storeFor(t)
execution := runningExecution(t, s)
stageID, _, err := s.StartStep(t.Context(), task.StepStart{
ExecutionID: execution.ID, Attempt: execution.Attempt, Kind: task.StepKindStage, Name: "review",
})
if err != nil {
t.Fatalf("StartStep stage: %v", err)
}
childID, _, err := s.StartStep(t.Context(), task.StepStart{
ExecutionID: execution.ID, Attempt: execution.Attempt,
ParentID: stageID, Kind: task.StepKindAgent, Name: "review:1",
})
if err != nil {
t.Fatalf("StartStep child: %v", err)
}
if _, err := s.FinishStep(t.Context(), task.StepFinish{
StepID: stageID, ExecutionID: execution.ID,
From: taskstate.StepRunning, To: taskstate.StepSucceeded, Detail: "done",
}); err != nil {
t.Fatalf("FinishStep: %v", err)
}
steps, err := s.CancelExecution(t.Context(), execution.ID, "stopped by operator", taskstate.Parked)
if err != nil {
t.Fatalf("CancelExecution: %v", err)
}
if len(steps) != 1 || steps[0] != childID {
t.Fatalf("cancelled steps = %v, want only the non-terminal %d", steps, childID)
}
// Every non-terminal step of the current attempt is cancelled; the one
// that already finished is never rewritten.
var statuses map[int64]string
if err := s.pool.QueryRow(t.Context(),
`SELECT jsonb_object_agg(id, status) FROM step_executions WHERE execution_id = $1`,
execution.ID).Scan(&statuses); err != nil {
t.Fatalf("read steps: %v", err)
}
if statuses[stageID] != string(taskstate.StepSucceeded) || statuses[childID] != string(taskstate.StepCancelled) {
t.Fatalf("statuses = %v, want the finished step kept and the child cancelled", statuses)
}
got, err := s.TaskByID(t.Context(), execution.ID)
if err != nil || got == nil {
t.Fatalf("TaskByID: %+v %v", got, err)
}
if got.Status != taskstate.Parked || got.ParkReason != "stopped by operator" {
t.Fatalf("execution after cancel = %s (%q), want parked with the reason", got.Status, got.ParkReason)
}
}
func TestCancelExecutionWritesOneEventPerCancelledStep(t *testing.T) {
s := storeFor(t)
execution := runningExecution(t, s)
stageID, _, err := s.StartStep(t.Context(), task.StepStart{
ExecutionID: execution.ID, Attempt: execution.Attempt, Kind: task.StepKindStage, Name: "implement",
})
if err != nil {
t.Fatalf("StartStep: %v", err)
}
if _, _, err := s.StartStep(t.Context(), task.StepStart{
ExecutionID: execution.ID, Attempt: execution.Attempt,
ParentID: stageID, Kind: task.StepKindAgent, Name: "implement",
}); err != nil {
t.Fatalf("StartStep child: %v", err)
}
if _, err := s.CancelExecution(t.Context(), execution.ID, "stopped by operator", taskstate.Parked); err != nil {
t.Fatalf("CancelExecution: %v", err)
}
evs, err := s.TaskEvents(t.Context(), execution.ID)
if err != nil {
t.Fatalf("TaskEvents: %v", err)
}
var cancelled []events.Event
for _, e := range evs {
if e.Data["cancelled"] == true {
cancelled = append(cancelled, e)
}
}
if len(cancelled) != 2 {
t.Fatalf("cancel events = %d, want one per step the transaction cancelled", len(cancelled))
}
for _, e := range cancelled {
if e.Kind != events.KindStageFinish || e.Attempt != execution.Attempt {
t.Errorf("cancel event = %+v, want a stage_finish row attributed to the attempt", e)
}
}
}
func TestCancelExecutionLeavesEarlierAttemptsAlone(t *testing.T) {
s := storeFor(t)
execution := runningExecution(t, s)
firstAttempt := execution.Attempt
stepID, _, err := s.StartStep(t.Context(), task.StepStart{
ExecutionID: execution.ID, Attempt: firstAttempt, Kind: task.StepKindStage, Name: "implement",
})
if err != nil {
t.Fatalf("StartStep: %v", err)
}
if _, err := s.FinishStep(t.Context(), task.StepFinish{
StepID: stepID, ExecutionID: execution.ID,
From: taskstate.StepRunning, To: taskstate.StepSucceeded,
}); err != nil {
t.Fatalf("FinishStep: %v", err)
}
// A retry requeues and reclaims: attempt 2.
if err := s.Requeue(t.Context(), execution.ID, taskstate.Running, "implement"); err != nil {
t.Fatalf("Requeue: %v", err)
}
retried, err := s.ClaimNext(t.Context())
if err != nil || retried == nil {
t.Fatalf("ClaimNext: %+v %v", retried, err)
}
if _, err := s.CancelExecution(t.Context(), execution.ID, "stopped by operator", taskstate.Parked); err != nil {
t.Fatalf("CancelExecution: %v", err)
}
var firstAttemptStatuses []string
if err := s.pool.QueryRow(t.Context(),
`SELECT jsonb_agg(status) FROM step_executions WHERE execution_id = $1 AND attempt = $2`,
execution.ID, firstAttempt).Scan(&firstAttemptStatuses); err != nil {
t.Fatalf("read first attempt's steps: %v", err)
}
for _, status := range firstAttemptStatuses {
if status != string(taskstate.StepSucceeded) {
t.Errorf("an earlier attempt's step was rewritten to %q; history is never rewritten", status)
}
}
}
func TestCancelExecutionRefusesWhatTheTableForbids(t *testing.T) {
tests := []struct {
name string
target string
want error
}{
{name: "a terminal execution", target: taskstate.Queued, want: storecontract.ErrIllegalTransition},
{name: "an unroutable destination", target: "quantum", want: storecontract.ErrIllegalTransition},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
s := storeFor(t)
execution := runningExecution(t, s)
if tt.name == "a terminal execution" {
// completed is terminal: the table routes nothing out of it.
if err := s.Transition(t.Context(), execution.ID, taskstate.Running, taskstate.Completed, "done"); err != nil {
t.Fatalf("seed completed: %v", err)
}
}
if _, err := s.CancelExecution(t.Context(), execution.ID, "why", tt.target); !errors.Is(err, tt.want) {
t.Fatalf("CancelExecution(to=%q) = %v, want %v", tt.target, err, tt.want)
}
})
}
// A queued execution cancels fine: the table routes queued -> closed_wont_do.
s := storeFor(t)
execution := runningExecution(t, s)
if err := s.Transition(t.Context(), execution.ID, taskstate.Running, taskstate.Queued, "requeue"); err != nil {
t.Fatalf("seed queued: %v", err)
}
if _, err := s.CancelExecution(t.Context(), execution.ID, "cancelled by operator", taskstate.Declined); err != nil {
t.Fatalf("CancelExecution(queued -> closed_wont_do) = %v, want nil", err)
}
}
func TestCancelExecutionOnAMissingExecutionIsStale(t *testing.T) {
s := storeFor(t)
if _, err := s.CancelExecution(t.Context(), 999999, "why", taskstate.Parked); !errors.Is(err, storecontract.ErrStaleTransition) {
t.Fatalf("CancelExecution on a missing execution = %v, want ErrStaleTransition", err)
}
}
@@ -11,6 +11,47 @@ import (
"github.com/jackc/pgx/v5/pgtype"
)
const cancelAttemptSteps = `-- name: CancelAttemptSteps :many
UPDATE step_executions
SET status = 'cancelled', finished_at = now()
WHERE execution_id = $1 AND attempt = $2 AND status IN ('pending', 'running')
RETURNING id, name
`
type CancelAttemptStepsParams struct {
ExecutionID int64
Attempt int64
}
type CancelAttemptStepsRow struct {
ID int64
Name string
}
// The half of CancelExecution that cancels the current attempt's
// non-terminal steps, returning them so the store writes one event row per
// transition in the same transaction. Earlier attempts' rows never match:
// history is never rewritten.
func (q *Queries) CancelAttemptSteps(ctx context.Context, arg CancelAttemptStepsParams) ([]CancelAttemptStepsRow, error) {
rows, err := q.db.Query(ctx, cancelAttemptSteps, arg.ExecutionID, arg.Attempt)
if err != nil {
return nil, err
}
defer rows.Close()
var items []CancelAttemptStepsRow
for rows.Next() {
var i CancelAttemptStepsRow
if err := rows.Scan(&i.ID, &i.Name); err != nil {
return nil, err
}
items = append(items, i)
}
if err := rows.Err(); err != nil {
return nil, err
}
return items, nil
}
const finishStepExecution = `-- name: FinishStepExecution :execrows
UPDATE step_executions
SET status = $1, detail = $2, tokens_used = $3, finished_at = now()
@@ -33,4 +33,13 @@ FROM step_executions WHERE id = $1 FOR UPDATE;
-- name: FinishStepExecution :execrows
UPDATE step_executions
SET status = $1, detail = $2, tokens_used = $3, finished_at = now()
WHERE id = $4 AND status = $5;
WHERE id = $4 AND status = $5;
-- name: CancelAttemptSteps :many
-- The half of CancelExecution that cancels the current attempt's
-- non-terminal steps, returning them so the store writes one event row per
-- transition in the same transaction. Earlier attempts' rows never match:
-- history is never rewritten.
UPDATE step_executions
SET status = 'cancelled', finished_at = now()
WHERE execution_id = $1 AND attempt = $2 AND status IN ('pending', 'running')
RETURNING id, name;
+67
View File
@@ -230,3 +230,70 @@ func stepEventData(to taskstate.StepStatus, durationMS int64, detail string) map
func stepParentID(parentID int64) pgtype.Int8 {
return pgtype.Int8{Int64: parentID, Valid: parentID != 0}
}
// CancelExecution is the one cancel path (docs/prds/execution-tree-state-machine.md,
// "Cancellation"): one transaction moves every non-terminal StepExecution of
// the execution's current attempt to cancelled -- one event row each -- and
// the execution itself to the status the operator action names, under the
// shared execution transition table. Staleness is decided before legality, as
// guardTransition decides it, and the audit row lands in the same write. The
// caller cancels the in-memory context after this commits; the worker's next
// step write then fails ErrStaleTransition and stops.
func (s *Store) CancelExecution(ctx context.Context, taskID int64, reason, to string) ([]int64, error) {
tx, err := s.pool.Begin(ctx)
if err != nil {
return nil, err
}
defer func() { _ = tx.Rollback(ctx) }()
q := postgresdb.New(tx)
execution, err := q.LockExecutionForStep(ctx, taskID)
if errors.Is(err, pgx.ErrNoRows) {
return nil, storecontract.ErrStaleTransition
}
if err != nil {
return nil, err
}
if !taskstate.CanTransition(execution.Status, to) {
return nil, fmt.Errorf("%w: %s -> %s is not a transition",
storecontract.ErrIllegalTransition, execution.Status, to)
}
steps, err := q.CancelAttemptSteps(ctx, postgresdb.CancelAttemptStepsParams{
ExecutionID: taskID, Attempt: execution.Attempt,
})
if err != nil {
return nil, err
}
for _, step := range steps {
if _, err := insertEventQ(ctx, q, stepEvent(events.KindStageFinish, execution.ID,
execution.Owner, execution.Repo, execution.IssueNumber, execution.Workflow,
int(execution.Attempt), step.Name, map[string]any{"cancelled": true})); err != nil {
return nil, err
}
}
n, err := q.TransitionTask(ctx, postgresdb.TransitionTaskParams{
ID: taskID, Status: to, ParkReason: clip(reason, 4000),
ParkClass: taskstate.ParkNeedsHuman, Status_2: execution.Status,
})
if err != nil {
return nil, err
}
if n == 0 {
return nil, storecontract.ErrStaleTransition
}
if err := q.InsertTransition(ctx, postgresdb.InsertTransitionParams{
TaskID: taskID, FromStatus: execution.Status, ToStatus: to, Detail: clip(reason, 4000),
}); err != nil {
return nil, err
}
if err := tx.Commit(ctx); err != nil {
return nil, err
}
ids := make([]int64, 0, len(steps))
for _, step := range steps {
ids = append(ids, step.ID)
}
return ids, nil
}
@@ -125,6 +125,7 @@ func taskFromRow(t postgresdb.Task) *workflow.Task {
var (
_ storecontract.TaskStore = (*Store)(nil)
_ storecontract.StepRecorder = (*Store)(nil)
_ storecontract.ExecutionCanceller = (*Store)(nil)
_ storecontract.BindingTaskCreator = (*Store)(nil)
_ storecontract.ConfigSnapshotStore = (*Store)(nil)
_ storecontract.ChannelStatusStore = (*Store)(nil)
@@ -58,6 +58,7 @@ var (
_ storecontract.ConfigSnapshotStore = (*Client)(nil)
_ storecontract.ChannelStatusStore = (*Client)(nil)
_ storecontract.ApplyStatusStore = (*Client)(nil)
_ storecontract.ExecutionCanceller = (*Client)(nil)
_ identity.Repository = (*Client)(nil)
_ storepkg.Manager = (*Client)(nil)
)
@@ -127,6 +128,18 @@ func (c *Client) StartStep(ctx context.Context, s task.StepStart) (int64, events
return r.StepId, event, nil
}
// CancelExecution is the one cancel path (docs/prds/execution-tree-state-machine.md):
// the State Store records the cancellation -- every non-terminal step of the
// current attempt plus the execution's own move -- and returns the steps it
// cancelled; the caller cancels the run's in-memory context afterwards.
func (c *Client) CancelExecution(ctx context.Context, taskID int64, reason, to string) ([]int64, error) {
r, err := c.client.CancelExecution(ctx, &pb.CancelExecutionRequest{TaskId: taskID, Reason: reason, To: to})
if err != nil {
return nil, unmapError(err)
}
return r.CancelledStepIds, nil
}
func (c *Client) FinishStep(ctx context.Context, s task.StepFinish) (events.Event, error) {
r, err := c.client.FinishStep(ctx, &pb.FinishStepRequest{
ExecutionId: s.ExecutionID, StepId: s.StepID,
@@ -28,6 +28,7 @@ import (
type contract interface {
storecontract.TaskStore
storecontract.StepRecorder
storecontract.ExecutionCanceller
storecontract.BindingTaskCreator
storecontract.ConfigSnapshotStore
storecontract.ApplyStatusStore
@@ -84,6 +85,7 @@ func remoteTaskStore(t *testing.T, local *pgstore.TaskDB, logs storecontract.Tas
server := grpc.NewServer()
deps := Deps{
Tasks: local, BindingTaskCreator: local, WorkflowCalls: local, Steps: local,
Canceller: local,
ConfigSnapshots: local, ApplyStatus: local,
TaskLogs: logs,
}
@@ -400,6 +402,16 @@ func TestStateStoreConformance(t *testing.T) {
t.Fatalf("ClaimByIssue missing: %+v %v", byClaim, err)
}
// The one cancel path, at the battery's end where the task's
// status is whatever the lifecycle section left: both adapters
// cancel the attempt's non-terminal steps and move the execution
// in one write, and the archive below then reads the parked row.
cancelled, err := c.CancelExecution(ctx, task.ID, "declined by operator", "closed_wont_do")
if err != nil {
t.Fatalf("CancelExecution: %v", err)
}
_ = cancelled
// Archive. RecoverStale above may have requeued task if it was
// still running, so re-fetch its current status rather than
// trusting the in-memory snapshot.
@@ -407,6 +419,9 @@ func TestStateStoreConformance(t *testing.T) {
if err != nil || current == nil {
t.Fatalf("TaskByID before archive: %+v %v", current, err)
}
if current.Status != "closed_wont_do" {
t.Fatalf("execution after cancel = %q, want closed_wont_do", current.Status)
}
if _, err := c.ArchiveTask(ctx, task.ID, current.Status, events.Event{Kind: "archived"}); err != nil {
t.Fatalf("ArchiveTask: %v", err)
}
@@ -102,6 +102,10 @@ type Deps struct {
// workflow engine's run credential calls. Optional: nil answers both RPCs
// with codes.Unavailable.
Steps storecontract.StepRecorder
// Canceller is the one cancel path (docs/prds/execution-tree-state-machine.md):
// an operator action's guarded write over the execution and its steps.
// Optional: nil answers the RPC with codes.Unavailable.
Canceller storecontract.ExecutionCanceller
// PlaybookDispatcher is the side-effecting-action idempotency ledger
// (docs/prds/eda-playbook-engine.md gap 2). Optional: nil disables the
// two playbook dispatch RPCs with codes.Unavailable.
@@ -214,6 +218,20 @@ func (s *server) StartStep(ctx context.Context, r *pb.StartStepRequest) (*pb.Sta
return &pb.StartStepResponse{StepId: stepID, Event: eventProto(event)}, nil
}
// CancelExecution is the one cancel path (docs/prds/execution-tree-state-machine.md):
// the store records the cancellation and returns the steps it cancelled; the
// caller delivers it by cancelling the run's in-memory context afterwards.
func (s *server) CancelExecution(ctx context.Context, r *pb.CancelExecutionRequest) (*pb.CancelExecutionResponse, error) {
if s.deps.Canceller == nil {
return nil, status.Error(codes.Unavailable, "execution canceller unavailable")
}
ids, err := s.deps.Canceller.CancelExecution(ctx, r.TaskId, r.Reason, r.To)
if err != nil {
return nil, s.logErr("CancelExecution", err)
}
return &pb.CancelExecutionResponse{CancelledStepIds: ids}, nil
}
func (s *server) FinishStep(ctx context.Context, r *pb.FinishStepRequest) (*pb.FinishStepResponse, error) {
if s.deps.Steps == nil {
return nil, status.Error(codes.Unavailable, "step recorder unavailable")
@@ -44,7 +44,8 @@ func grantsServer(t *testing.T, adminToken string) (grants *TaskGrants, dial fun
Captures: eda, Mappings: eda, Bindings: eda,
BindingDispatcher: eda, PlaybookDispatcher: eda, EventTypes: eda,
BindingTaskCreator: local, WorkflowCalls: local, Steps: local,
Grants: grants,
Canceller: local,
Grants: grants,
})
go func() { _ = server.Serve(listener) }()
t.Cleanup(func() { server.Stop(); _ = listener.Close() })
@@ -457,3 +458,29 @@ func TestRunCredentialRecordsItsOwnSteps(t *testing.T) {
t.Fatalf("FinishStep naming another execution = %v, want PermissionDenied from the grant check", err)
}
}
// A task-scoped grant never reaches CancelExecution: a cancel is a dashboard,
// API and dispatch action decided by the Authorizer
// (docs/prds/execution-tree-state-machine.md), and the deny-by-default rule
// covers it without a widening.
func TestTaskGrantCannotCancelAnExecution(t *testing.T) {
const adminToken = "daemon-admin-token"
grants, dial := grantsServer(t, adminToken)
_ = grants
admin := dial(t, adminToken)
ctx := t.Context()
task, err := admin.EnqueueChatTask(ctx, "acme", "widget", "a", "body", "implement", "")
if err != nil {
t.Fatal(err)
}
workerToken, err := admin.RegisterTaskGrant(ctx, task.ID, time.Hour)
if err != nil {
t.Fatalf("RegisterTaskGrant: %v", err)
}
worker := dial(t, workerToken)
if _, err := worker.CancelExecution(ctx, task.ID, "why", "parked"); status.Code(err) != codes.PermissionDenied {
t.Fatalf("CancelExecution with a task grant = %v, want PermissionDenied", err)
}
}
@@ -42,8 +42,11 @@ type stubStore struct {
func (s stubStore) TaskByID(context.Context, int64) (*domain.Task, error) { return s.task, nil }
func (s stubStore) Transition(context.Context, int64, string, string, string) error { return s.err }
func (s stubStore) Requeue(context.Context, int64, string, string) error { return s.err }
func (s stubStore) RetryTask(context.Context, int64, string, string) error { return s.err }
func (s stubStore) CancelExecution(context.Context, int64, string, string) ([]int64, error) {
return nil, s.err
}
func (s stubStore) Requeue(context.Context, int64, string, string) error { return s.err }
func (s stubStore) RetryTask(context.Context, int64, string, string) error { return s.err }
func (s stubStore) ArchiveTask(context.Context, int64, string, events.Event) (int64, error) {
return 0, s.err
@@ -3,6 +3,7 @@ package taskactions
import (
"context"
"fmt"
"github.com/samcharles93/archie-core/internal/config"
"github.com/samcharles93/archie-core/internal/domain/storecontract"
@@ -19,6 +20,19 @@ func (s Store) TaskByID(ctx context.Context, id int64) (*taskactions.Task, error
return &taskactions.Task{ID: t.ID, Owner: t.Owner, Repo: t.Repo, Identity: t.Identity, Status: t.Status, Stage: t.Stage, ParkReason: t.ParkReason, IssueNumber: t.IssueNumber, RetryCount: t.RetryCount, Attempt: t.Attempt, ForgeBacked: t.IsForgeBacked()}, nil
}
// CancelExecution delegates to the store's one cancel path
// (docs/prds/execution-tree-state-machine.md). Both adapters -- the
// PostgreSQL store and the remote gRPC client -- implement it; the type
// assertion failing would be a composition bug, so it is named rather than
// panicked over.
func (s Store) CancelExecution(ctx context.Context, taskID int64, reason, to string) ([]int64, error) {
canceller, ok := s.TaskStore.(storecontract.ExecutionCanceller)
if !ok {
return nil, fmt.Errorf("task store cannot cancel executions")
}
return canceller.CancelExecution(ctx, taskID, reason, to)
}
func MaxRetries(cfg *config.Holder) func(*taskactions.Task) int {
return func(t *taskactions.Task) int {
if cfg == nil {
+18
View File
@@ -44,6 +44,10 @@ service StateStoreService {
// credential authorises both on its own execution only.
rpc StartStep(StartStepRequest) returns (StartStepResponse);
rpc FinishStep(FinishStepRequest) returns (FinishStepResponse);
// Cancellation. Deliberately administrative (deny-by-default for a
// task-scoped grant): a cancel is a dashboard, API and dispatch action
// decided by the Authorizer, not a worker call.
rpc CancelExecution(CancelExecutionRequest) returns (CancelExecutionResponse);
rpc Requeue(RequeueRequest) returns (RequeueResponse);
rpc ParkTask(ParkTaskRequest) returns (ParkTaskResponse);
rpc RecoverStale(RecoverStaleRequest) returns (RecoverStaleResponse);
@@ -558,6 +562,20 @@ message FinishStepResponse {
Event event = 1;
}
message CancelExecutionRequest {
int64 task_id = 1;
// reason records why the run was stopped, in the execution's park reason
// when the cancel lands there, and on every step event it writes.
string reason = 2;
// to is the execution's destination: the transition table routes it, and
// the operator actions name parked or closed_wont_do.
string to = 3;
}
message CancelExecutionResponse {
// cancelled_step_ids are the StepExecutions the transaction cancelled.
repeated int64 cancelled_step_ids = 1;
}
message UpdateRequest { Task task = 1; }
message UpdateResponse {}