fix(daemon): detach terminal state transitions and cleanup from cancelled context

When an operator stopped a running task via chat /stop or the dashboard
Stop button, the task's context was cancelled. Container teardown
correctly completed (as Pool.Release detached its context), but terminal
state recording in Daemon.runViaAgent and worktree cleanup in
Daemon.cleanupTerminalTaskWorktree executed using the same cancelled
context. Consequently, SQLite transactions in store.Transition failed
immediately and errors were discarded, leaving the task row in 'running'
indefinitely until a daemon restart.

Extract parkRunningTask in internal/daemon/daemon.go to perform guarded
state transitions (store.StatusRunning -> store.StatusParked) using a
detached context (context.WithoutCancel) with a bounded 10-second timeout.
Update cleanupTerminalTaskWorktree to also use a detached 10-second
context for looking up the task status. Check transition errors and log
warnings for unexpected database failures while suppressing expected
ErrStaleTransition returns when workers have already completed. Add
comprehensive table-driven regression tests covering task cancellation,
worktree cleanup, and logging behavior.
This commit is contained in:
sam committed 2026-08-25 01:42:54 +10:00
1 parent bb1ab9682f
commit fd014f6a7f
2 files changed
+583 -12

No files matched your search

+46 -12
View File
@@ -809,7 +809,7 @@ func (d *Daemon) process(ctx context.Context, task *store.Task) {
trees := d.treesFor(task)
repo, ok := d.repoFor(task)
if !ok {
_ = d.Store.Transition(ctx, task.ID, store.StatusRunning, store.StatusParked, "repo no longer in config")
d.parkRunningTask(ctx, task.ID, "repo no longer in config")
return
}
if d.ContainerPool == nil {
@@ -856,7 +856,7 @@ func (d *Daemon) process(ctx context.Context, task *store.Task) {
_, branch, err = trees.Prepare(ctx, task.Owner, task.Repo, repo.Base, task.IssueNumber, task.Title, task.Body, task.Labels)
if err != nil {
d.Log.Error("worktree prepare failed", "err", err)
_ = d.Store.Transition(ctx, task.ID, store.StatusRunning, store.StatusParked, "worktree prepare failed: "+err.Error())
d.parkRunningTask(ctx, task.ID, "worktree prepare failed: "+err.Error())
return
}
task.Branch = branch
@@ -899,8 +899,15 @@ func (d *Daemon) cleanupTerminalTaskWorktree(ctx context.Context, task *store.Ta
if d.Store == nil || trees == nil || task == nil {
return
}
latest, err := d.Store.TaskByID(ctx, task.ID)
if err != nil || latest == nil {
cleanupCtx, cancel := context.WithTimeout(context.WithoutCancel(ctx), 10*time.Second)
defer cancel()
latest, err := d.Store.TaskByID(cleanupCtx, task.ID)
if err != nil {
d.Log.Warn("terminal worktree cleanup task lookup failed", "task", task.ID, "err", err)
return
}
if latest == nil {
return
}
switch latest.Status {
@@ -942,7 +949,7 @@ func (d *Daemon) acquireTaskContainer(
) (*container.Container, bool) {
park := func(reason string, err error) {
d.Log.Error(reason, "err", err)
_ = d.Store.Transition(ctx, task.ID, store.StatusRunning, store.StatusParked, reason+": "+err.Error())
d.parkRunningTask(ctx, task.ID, reason+": "+err.Error())
}
// Write task.json -- the container's boot-time brief.
@@ -960,7 +967,7 @@ func (d *Daemon) acquireTaskContainer(
// normal operation, Storage is always set when ContainerPool is set.
if d.Storage == nil {
d.Log.Error("storage backend is nil -- cannot acquire container")
_ = d.Store.Transition(ctx, task.ID, store.StatusRunning, store.StatusParked, "storage backend not configured")
d.parkRunningTask(ctx, task.ID, "storage backend not configured")
return nil, false
}
@@ -997,6 +1004,33 @@ func (d *Daemon) acquireTaskContainer(
return ctr, true
}
// parkRunningTask transitions a task from running to parked using a context
// detached from caller cancellation, with a bounded timeout.
//
// When a task fails or is cancelled (e.g. via /stop or dashboard Stop), ctx is
// already cancelled. Writing terminal state using the cancelled ctx fails
// immediately in SQLite transactions and silently leaves the task row 'running'
// indefinitely. Like container teardown in Pool.Release, terminal state
// recording is cleanup on the way out of a task and must use a bounded context
// that survives cancellation while preserving values.
//
// The transition remains guarded from StatusRunning: if the worker already
// recorded a terminal state over storerpc, store.ErrStaleTransition is returned
// and ignored. Unexpected store errors are logged as warnings.
func (d *Daemon) parkRunningTask(ctx context.Context, taskID int64, reason string) {
if d.Store == nil {
return
}
writeCtx, cancel := context.WithTimeout(context.WithoutCancel(ctx), 10*time.Second)
defer cancel()
if err := d.Store.Transition(writeCtx, taskID, store.StatusRunning, store.StatusParked, reason); err != nil {
if !errors.Is(err, store.ErrStaleTransition) {
d.Log.Warn("terminal park transition failed", "task", taskID, "reason", reason, "err", err)
}
}
}
// containerEnv returns the environment variables passed to agent containers.
// runViaAgent publishes a full-task handoff to archie-agent over the
// infrastructure-owned task subject and waits for its completion report. Every durable
@@ -1011,13 +1045,13 @@ func (d *Daemon) runViaAgent(ctx context.Context, task *store.Task, repo config.
if d.WorktreeGrants == nil {
const reason = "worktree publication grants are unavailable"
d.Log.Error(reason, "task", task.ID)
_ = d.Store.Transition(ctx, task.ID, store.StatusRunning, store.StatusParked, reason)
d.parkRunningTask(ctx, task.ID, reason)
return
}
grant, revoke, err := d.WorktreeGrants.Issue(task)
if err != nil {
d.Log.Error("worktree publication grant failed", "task", task.ID, "err", err)
_ = d.Store.Transition(ctx, task.ID, store.StatusRunning, store.StatusParked, "worktree publication grant failed: "+err.Error())
d.parkRunningTask(ctx, task.ID, "worktree publication grant failed: "+err.Error())
return
}
defer revoke()
@@ -1031,26 +1065,26 @@ func (d *Daemon) runViaAgent(ctx context.Context, task *store.Task, repo config.
data, err := json.Marshal(req)
if err != nil {
d.Log.Error("taskrun encode failed", "task", task.ID, "err", err)
_ = d.Store.Transition(ctx, task.ID, store.StatusRunning, store.StatusParked, "taskrun encode failed: "+err.Error())
d.parkRunningTask(ctx, task.ID, "taskrun encode failed: "+err.Error())
return
}
reply, err := d.requestTaskRun(ctx, task.ID, data)
if err != nil {
d.Log.Error("taskrun request failed", "task", task.ID, "err", err)
_ = d.Store.Transition(ctx, task.ID, store.StatusRunning, store.StatusParked, "taskrun request failed: "+err.Error())
d.parkRunningTask(ctx, task.ID, "taskrun request failed: "+err.Error())
return
}
var resp taskrun.Response
if err := json.Unmarshal(reply, &resp); err != nil {
d.Log.Error("taskrun decode response failed", "task", task.ID, "err", err)
_ = d.Store.Transition(ctx, task.ID, store.StatusRunning, store.StatusParked, "taskrun decode response failed: "+err.Error())
d.parkRunningTask(ctx, task.ID, "taskrun decode response failed: "+err.Error())
return
}
if resp.Error != "" {
d.Log.Error("taskrun run failed", "task", task.ID, "err", resp.Error)
_ = d.Store.Transition(ctx, task.ID, store.StatusRunning, store.StatusParked, "taskrun run failed: "+resp.Error)
d.parkRunningTask(ctx, task.ID, "taskrun run failed: "+resp.Error)
return
}
+537
View File
@@ -0,0 +1,537 @@
package daemon
import (
"bytes"
"context"
"encoding/json"
"errors"
"log/slog"
"net/http"
"net/http/httptest"
"os"
"strings"
"sync"
"testing"
"time"
dockercontainer "github.com/moby/moby/api/types/container"
"github.com/moby/moby/client"
natsio "github.com/nats-io/nats.go"
"github.com/samcharles93/archie-core/internal/config"
archiecontainer "github.com/samcharles93/archie-core/internal/container"
agentnats "github.com/samcharles93/archie-core/internal/infrastructure/agenttransport/nats"
"github.com/samcharles93/archie-core/internal/storage"
"github.com/samcharles93/archie-core/internal/store"
"github.com/samcharles93/archie-core/internal/taskrun"
"github.com/samcharles93/archie-core/internal/worktree"
)
type logBuffer struct {
mu sync.Mutex
buf bytes.Buffer
}
func (l *logBuffer) Write(p []byte) (n int, err error) {
l.mu.Lock()
defer l.mu.Unlock()
return l.buf.Write(p)
}
func (l *logBuffer) String() string {
l.mu.Lock()
defer l.mu.Unlock()
return l.buf.String()
}
type errGrantIssuer struct{}
func (errGrantIssuer) Issue(*store.Task) (string, func(), error) {
return "", nil, errors.New("simulated grant issue error")
}
type errStorage struct {
setupErr error
}
func (s *errStorage) Setup(context.Context, storage.TaskRef) ([]storage.Mount, error) {
if s.setupErr != nil {
return nil, s.setupErr
}
return nil, nil
}
func (s *errStorage) Teardown(context.Context, storage.TaskRef) error {
return nil
}
func (s *errStorage) CleanupExpired(context.Context, time.Duration) (int, error) {
return 0, nil
}
func TestCancelledTaskTransitionsToParked(t *testing.T) {
tests := []struct {
name string
cancelBefore bool
cancelDuring bool
nilGrants bool
errGrants bool
decodeErr bool
agentErr string
wantStatus string
wantParkSubstr string
}{
{
name: "task cancelled during requestTaskRun",
cancelDuring: true,
wantStatus: store.StatusParked,
wantParkSubstr: "context canceled",
},
{
name: "task cancelled before runViaAgent",
cancelBefore: true,
wantStatus: store.StatusParked,
wantParkSubstr: "context canceled",
},
{
name: "worktree grants unavailable",
nilGrants: true,
wantStatus: store.StatusParked,
wantParkSubstr: "worktree publication grants are unavailable",
},
{
name: "worktree grant issue failed",
errGrants: true,
wantStatus: store.StatusParked,
wantParkSubstr: "worktree publication grant failed",
},
{
name: "taskrun decode response failed",
decodeErr: true,
wantStatus: store.StatusParked,
wantParkSubstr: "taskrun decode response failed",
},
{
name: "taskrun agent returned error",
agentErr: "simulated agent execution failure",
wantStatus: store.StatusParked,
wantParkSubstr: "taskrun run failed: simulated agent execution failure",
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
d, s, busClient := daemonWithNATS(t)
ctx, cancel := context.WithCancel(context.Background())
t.Cleanup(cancel)
if _, err := s.EnqueueIssue(context.Background(), "acme", "widget", 1, "task", "body", "", ""); err != nil {
t.Fatal(err)
}
task, err := s.ClaimNext(context.Background())
if err != nil || task == nil {
t.Fatalf("ClaimNext: (%v, %v)", task, err)
}
if tt.nilGrants {
d.WorktreeGrants = nil
}
if tt.errGrants {
d.WorktreeGrants = errGrantIssuer{}
}
if tt.cancelBefore {
cancel()
}
if tt.cancelDuring {
sub, err := mustCoreConn(t, busClient).Subscribe(agentnats.SubjectForTask(task.ID), func(msg *natsio.Msg) {
cancel()
})
if err != nil {
t.Fatal(err)
}
t.Cleanup(func() { _ = sub.Unsubscribe() })
}
if tt.decodeErr {
sub, err := mustCoreConn(t, busClient).Subscribe(agentnats.SubjectForTask(task.ID), func(msg *natsio.Msg) {
_ = msg.Respond([]byte("invalid json response"))
})
if err != nil {
t.Fatal(err)
}
t.Cleanup(func() { _ = sub.Unsubscribe() })
}
if tt.agentErr != "" {
sub, err := mustCoreConn(t, busClient).Subscribe(agentnats.SubjectForTask(task.ID), func(msg *natsio.Msg) {
resp, _ := json.Marshal(taskrun.Response{Error: tt.agentErr})
_ = msg.Respond(resp)
})
if err != nil {
t.Fatal(err)
}
t.Cleanup(func() { _ = sub.Unsubscribe() })
}
d.TaskRunReadyTimeout = 100 * time.Millisecond
d.TaskRunRetryBackoff = 10 * time.Millisecond
d.runViaAgent(ctx, task, config.Repo{Owner: "acme", Name: "widget"})
got, err := s.TaskByID(context.Background(), task.ID)
if err != nil {
t.Fatalf("TaskByID: %v", err)
}
if got.Status != tt.wantStatus {
t.Fatalf("task status = %q, want %q", got.Status, tt.wantStatus)
}
if !strings.Contains(got.ParkReason, tt.wantParkSubstr) {
t.Errorf("park reason = %q, want substring %q", got.ParkReason, tt.wantParkSubstr)
}
})
}
}
func TestCleanupTerminalTaskWorktreeWithCancelledContext(t *testing.T) {
tests := []struct {
name string
taskStatus string
prNumber int
wantTreeExists bool
}{
{
name: "merged task with cancelled context cleans worktree",
taskStatus: store.StatusMerged,
prNumber: 0,
wantTreeExists: false,
},
{
name: "rejected task with cancelled context cleans worktree",
taskStatus: store.StatusRejected,
prNumber: 0,
wantTreeExists: false,
},
{
name: "closed wont do task with cancelled context cleans worktree",
taskStatus: store.StatusClosedWontDo,
prNumber: 0,
wantTreeExists: false,
},
{
name: "parked task with cancelled context preserves worktree",
taskStatus: store.StatusParked,
prNumber: 0,
wantTreeExists: true,
},
{
name: "open PR task with cancelled context preserves worktree",
taskStatus: store.StatusPROpen,
prNumber: 12,
wantTreeExists: true,
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
ctx, cancel := context.WithCancel(context.Background())
cancel() // cancel context before calling cleanup
st := store.OpenTest(t)
t.Cleanup(func() { _ = st.Close() })
trees := &worktree.Manager{WorkDir: t.TempDir()}
log := slog.New(slog.DiscardHandler)
d := &Daemon{
Store: st,
Trees: trees,
Log: log,
}
if _, err := st.EnqueueIssue(context.Background(), "acme", "widget", 101, "task", "body", "bug", ""); err != nil {
t.Fatalf("EnqueueIssue: %v", err)
}
task, err := st.ClaimNext(context.Background())
if err != nil || task == nil {
t.Fatalf("ClaimNext: (%v, %v)", task, err)
}
if err := st.Transition(context.Background(), task.ID, store.StatusRunning, tt.taskStatus, "setup"); err != nil {
t.Fatalf("Transition: %v", err)
}
if tt.prNumber > 0 {
task.PRNumber = tt.prNumber
if err := st.Update(context.Background(), task); err != nil {
t.Fatalf("Update: %v", err)
}
}
workDir := trees.Dir(task.Owner, task.Repo, task.IssueNumber)
if err := os.MkdirAll(workDir, 0o755); err != nil {
t.Fatalf("create workDir: %v", err)
}
d.cleanupTerminalTaskWorktree(ctx, task, trees)
_, statErr := os.Stat(workDir)
treeExists := statErr == nil
if treeExists != tt.wantTreeExists {
t.Errorf("worktree exists = %v, want %v (path: %s)", treeExists, tt.wantTreeExists, workDir)
}
})
}
}
func TestProcessStopRunningTransitionsToParked(t *testing.T) {
host := newLocalRemote(t, "acme", "widget")
d, s, busClient := daemonWithNATS(t)
ctx := context.Background()
rootTrees := &worktree.Manager{
WorkDir: t.TempDir(),
Token: "unused",
BotUser: "archie-bot",
BotEmail: "archie-bot@example.com",
BaseURL: "file://" + host,
}
d.Trees = rootTrees
repo := config.Repo{Owner: "acme", Name: "widget", Base: "main"}
d.Cfg.Set(config.Config{
Repos: []config.Repo{repo},
})
// Mock Docker API server for ContainerPool
dockerAPI := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
switch {
case strings.HasSuffix(r.URL.Path, "/containers/create"):
writeContainerDockerJSON(t, w, dockercontainer.CreateResponse{ID: "test-container-id"})
case strings.Contains(r.URL.Path, "/containers/test-container-id"):
w.WriteHeader(http.StatusNoContent)
case strings.HasSuffix(r.URL.Path, "/containers/json"):
writeContainerDockerJSON(t, w, []any{})
default:
http.Error(w, "unexpected Docker API path "+r.URL.Path, http.StatusNotFound)
}
}))
t.Cleanup(dockerAPI.Close)
dockerClient, err := client.New(client.WithHost(dockerAPI.URL), client.WithAPIVersion("1.55"))
if err != nil {
t.Fatal(err)
}
t.Cleanup(func() { _ = dockerClient.Close() })
pool, err := archiecontainer.NewPool(ctx, archiecontainer.Config{
Image: "archie-agent:test",
DockerClient: dockerClient,
}, d.Log)
if err != nil {
t.Fatal(err)
}
d.ContainerPool = pool
d.Storage = &noopStorage{}
if _, err := s.EnqueueIssue(ctx, "acme", "widget", 42, "issue to stop", "body", "bug", ""); err != nil {
t.Fatal(err)
}
task, err := s.ClaimNext(ctx)
if err != nil || task == nil {
t.Fatalf("ClaimNext: (%v, %v)", task, err)
}
// When agent request is received, stop the running task via d.CancelTask(task.ID)
sub, err := mustCoreConn(t, busClient).Subscribe(agentnats.SubjectForTask(task.ID), func(msg *natsio.Msg) {
stopped := d.CancelTask(task.ID)
if !stopped {
t.Errorf("CancelTask(%d) = false, want true", task.ID)
}
})
if err != nil {
t.Fatal(err)
}
t.Cleanup(func() { _ = sub.Unsubscribe() })
d.TaskRunReadyTimeout = 100 * time.Millisecond
d.TaskRunRetryBackoff = 10 * time.Millisecond
d.process(ctx, task)
got, err := s.TaskByID(ctx, task.ID)
if err != nil {
t.Fatalf("TaskByID: %v", err)
}
if got.Status != store.StatusParked {
t.Fatalf("stopped task status = %q, want %q", got.Status, store.StatusParked)
}
if !strings.Contains(got.ParkReason, "context canceled") {
t.Errorf("park reason = %q, want substring 'context canceled'", got.ParkReason)
}
}
func TestParkRunningTaskGuardedAndLogging(t *testing.T) {
tests := []struct {
name string
initialStatus string
workerStatus string
closeStore bool
cancelContext bool
wantStatus string
wantWarning bool
wantLogSubstr string
}{
{
name: "successful park on cancelled context transitions row to parked without warning",
initialStatus: store.StatusRunning,
cancelContext: true,
wantStatus: store.StatusParked,
wantWarning: false,
},
{
name: "stale transition preserves worker status and logs no warning",
initialStatus: store.StatusRunning,
workerStatus: store.StatusMerged,
cancelContext: true,
wantStatus: store.StatusMerged,
wantWarning: false,
},
{
name: "store write failure logs warning",
initialStatus: store.StatusRunning,
closeStore: true,
wantWarning: true,
wantLogSubstr: "terminal park transition failed",
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
s := store.OpenTest(t)
t.Cleanup(func() { _ = s.Close() })
var buf logBuffer
log := slog.New(slog.NewTextHandler(&buf, nil))
d := &Daemon{
Store: s,
Log: log,
}
ctx := context.Background()
if _, err := s.EnqueueIssue(ctx, "acme", "widget", 55, "task", "body", "bug", ""); err != nil {
t.Fatal(err)
}
task, err := s.ClaimNext(ctx)
if err != nil || task == nil {
t.Fatalf("ClaimNext: (%v, %v)", task, err)
}
if tt.workerStatus != "" {
if err := s.Transition(ctx, task.ID, store.StatusRunning, tt.workerStatus, "worker completion"); err != nil {
t.Fatalf("Transition to workerStatus: %v", err)
}
}
callCtx := ctx
if tt.cancelContext {
cctx, cancel := context.WithCancel(ctx)
cancel()
callCtx = cctx
}
if tt.closeStore {
_ = s.Close()
}
d.parkRunningTask(callCtx, task.ID, "simulated park reason")
logOutput := buf.String()
if tt.wantWarning {
if !strings.Contains(logOutput, tt.wantLogSubstr) {
t.Errorf("log output = %q, want substring %q", logOutput, tt.wantLogSubstr)
}
if !strings.Contains(logOutput, "level=WARN") {
t.Errorf("log output = %q, want level=WARN", logOutput)
}
} else if strings.Contains(logOutput, "terminal park transition failed") {
t.Errorf("unexpected warning in log output: %q", logOutput)
}
if !tt.closeStore {
got, err := s.TaskByID(ctx, task.ID)
if err != nil {
t.Fatalf("TaskByID: %v", err)
}
if got.Status != tt.wantStatus {
t.Errorf("task status = %q, want %q", got.Status, tt.wantStatus)
}
}
})
}
}
func TestAcquireTaskContainerFailureParksTaskOnCancelledContext(t *testing.T) {
tests := []struct {
name string
nilStorage bool
setupErr error
wantParkSubstr string
}{
{
name: "storage backend not configured",
nilStorage: true,
wantParkSubstr: "storage backend not configured",
},
{
name: "storage setup failed",
setupErr: errors.New("simulated setup error"),
wantParkSubstr: "storage setup failed",
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
s := store.OpenTest(t)
t.Cleanup(func() { _ = s.Close() })
d := &Daemon{
Store: s,
Log: slog.New(slog.DiscardHandler),
}
ctx, cancel := context.WithCancel(context.Background())
cancel() // cancelled context
if _, err := s.EnqueueIssue(context.Background(), "acme", "widget", 88, "task", "body", "bug", ""); err != nil {
t.Fatal(err)
}
task, err := s.ClaimNext(context.Background())
if err != nil || task == nil {
t.Fatalf("ClaimNext: (%v, %v)", task, err)
}
workDir := t.TempDir()
if !tt.nilStorage {
d.Storage = &errStorage{setupErr: tt.setupErr}
}
ctr, ok := d.acquireTaskContainer(ctx, task, config.Repo{Owner: "acme", Name: "widget"}, workDir)
if ok || ctr != nil {
t.Fatalf("acquireTaskContainer ok = %v, ctr = %v; want false, nil", ok, ctr)
}
got, err := s.TaskByID(context.Background(), task.ID)
if err != nil {
t.Fatalf("TaskByID: %v", err)
}
if got.Status != store.StatusParked {
t.Fatalf("task status = %q, want %q", got.Status, store.StatusParked)
}
if !strings.Contains(got.ParkReason, tt.wantParkSubstr) {
t.Errorf("park reason = %q, want substring %q", got.ParkReason, tt.wantParkSubstr)
}
})
}
}