diff --git a/internal/daemon/daemon.go b/internal/daemon/daemon.go index 1c802c058..343d87104 100644 --- a/internal/daemon/daemon.go +++ b/internal/daemon/daemon.go @@ -809,7 +809,7 @@ func (d *Daemon) process(ctx context.Context, task *store.Task) { trees := d.treesFor(task) repo, ok := d.repoFor(task) if !ok { - _ = d.Store.Transition(ctx, task.ID, store.StatusRunning, store.StatusParked, "repo no longer in config") + d.parkRunningTask(ctx, task.ID, "repo no longer in config") return } if d.ContainerPool == nil { @@ -856,7 +856,7 @@ func (d *Daemon) process(ctx context.Context, task *store.Task) { _, branch, err = trees.Prepare(ctx, task.Owner, task.Repo, repo.Base, task.IssueNumber, task.Title, task.Body, task.Labels) if err != nil { d.Log.Error("worktree prepare failed", "err", err) - _ = d.Store.Transition(ctx, task.ID, store.StatusRunning, store.StatusParked, "worktree prepare failed: "+err.Error()) + d.parkRunningTask(ctx, task.ID, "worktree prepare failed: "+err.Error()) return } task.Branch = branch @@ -899,8 +899,15 @@ func (d *Daemon) cleanupTerminalTaskWorktree(ctx context.Context, task *store.Ta if d.Store == nil || trees == nil || task == nil { return } - latest, err := d.Store.TaskByID(ctx, task.ID) - if err != nil || latest == nil { + cleanupCtx, cancel := context.WithTimeout(context.WithoutCancel(ctx), 10*time.Second) + defer cancel() + + latest, err := d.Store.TaskByID(cleanupCtx, task.ID) + if err != nil { + d.Log.Warn("terminal worktree cleanup task lookup failed", "task", task.ID, "err", err) + return + } + if latest == nil { return } switch latest.Status { @@ -942,7 +949,7 @@ func (d *Daemon) acquireTaskContainer( ) (*container.Container, bool) { park := func(reason string, err error) { d.Log.Error(reason, "err", err) - _ = d.Store.Transition(ctx, task.ID, store.StatusRunning, store.StatusParked, reason+": "+err.Error()) + d.parkRunningTask(ctx, task.ID, reason+": "+err.Error()) } // Write task.json -- the container's boot-time brief. @@ -960,7 +967,7 @@ func (d *Daemon) acquireTaskContainer( // normal operation, Storage is always set when ContainerPool is set. if d.Storage == nil { d.Log.Error("storage backend is nil -- cannot acquire container") - _ = d.Store.Transition(ctx, task.ID, store.StatusRunning, store.StatusParked, "storage backend not configured") + d.parkRunningTask(ctx, task.ID, "storage backend not configured") return nil, false } @@ -997,6 +1004,33 @@ func (d *Daemon) acquireTaskContainer( return ctr, true } +// parkRunningTask transitions a task from running to parked using a context +// detached from caller cancellation, with a bounded timeout. +// +// When a task fails or is cancelled (e.g. via /stop or dashboard Stop), ctx is +// already cancelled. Writing terminal state using the cancelled ctx fails +// immediately in SQLite transactions and silently leaves the task row 'running' +// indefinitely. Like container teardown in Pool.Release, terminal state +// recording is cleanup on the way out of a task and must use a bounded context +// that survives cancellation while preserving values. +// +// The transition remains guarded from StatusRunning: if the worker already +// recorded a terminal state over storerpc, store.ErrStaleTransition is returned +// and ignored. Unexpected store errors are logged as warnings. +func (d *Daemon) parkRunningTask(ctx context.Context, taskID int64, reason string) { + if d.Store == nil { + return + } + writeCtx, cancel := context.WithTimeout(context.WithoutCancel(ctx), 10*time.Second) + defer cancel() + + if err := d.Store.Transition(writeCtx, taskID, store.StatusRunning, store.StatusParked, reason); err != nil { + if !errors.Is(err, store.ErrStaleTransition) { + d.Log.Warn("terminal park transition failed", "task", taskID, "reason", reason, "err", err) + } + } +} + // containerEnv returns the environment variables passed to agent containers. // runViaAgent publishes a full-task handoff to archie-agent over the // infrastructure-owned task subject and waits for its completion report. Every durable @@ -1011,13 +1045,13 @@ func (d *Daemon) runViaAgent(ctx context.Context, task *store.Task, repo config. if d.WorktreeGrants == nil { const reason = "worktree publication grants are unavailable" d.Log.Error(reason, "task", task.ID) - _ = d.Store.Transition(ctx, task.ID, store.StatusRunning, store.StatusParked, reason) + d.parkRunningTask(ctx, task.ID, reason) return } grant, revoke, err := d.WorktreeGrants.Issue(task) if err != nil { d.Log.Error("worktree publication grant failed", "task", task.ID, "err", err) - _ = d.Store.Transition(ctx, task.ID, store.StatusRunning, store.StatusParked, "worktree publication grant failed: "+err.Error()) + d.parkRunningTask(ctx, task.ID, "worktree publication grant failed: "+err.Error()) return } defer revoke() @@ -1031,26 +1065,26 @@ func (d *Daemon) runViaAgent(ctx context.Context, task *store.Task, repo config. data, err := json.Marshal(req) if err != nil { d.Log.Error("taskrun encode failed", "task", task.ID, "err", err) - _ = d.Store.Transition(ctx, task.ID, store.StatusRunning, store.StatusParked, "taskrun encode failed: "+err.Error()) + d.parkRunningTask(ctx, task.ID, "taskrun encode failed: "+err.Error()) return } reply, err := d.requestTaskRun(ctx, task.ID, data) if err != nil { d.Log.Error("taskrun request failed", "task", task.ID, "err", err) - _ = d.Store.Transition(ctx, task.ID, store.StatusRunning, store.StatusParked, "taskrun request failed: "+err.Error()) + d.parkRunningTask(ctx, task.ID, "taskrun request failed: "+err.Error()) return } var resp taskrun.Response if err := json.Unmarshal(reply, &resp); err != nil { d.Log.Error("taskrun decode response failed", "task", task.ID, "err", err) - _ = d.Store.Transition(ctx, task.ID, store.StatusRunning, store.StatusParked, "taskrun decode response failed: "+err.Error()) + d.parkRunningTask(ctx, task.ID, "taskrun decode response failed: "+err.Error()) return } if resp.Error != "" { d.Log.Error("taskrun run failed", "task", task.ID, "err", resp.Error) - _ = d.Store.Transition(ctx, task.ID, store.StatusRunning, store.StatusParked, "taskrun run failed: "+resp.Error) + d.parkRunningTask(ctx, task.ID, "taskrun run failed: "+resp.Error) return } diff --git a/internal/daemon/task_cancellation_test.go b/internal/daemon/task_cancellation_test.go new file mode 100644 index 000000000..e196760f9 --- /dev/null +++ b/internal/daemon/task_cancellation_test.go @@ -0,0 +1,537 @@ +package daemon + +import ( + "bytes" + "context" + "encoding/json" + "errors" + "log/slog" + "net/http" + "net/http/httptest" + "os" + "strings" + "sync" + "testing" + "time" + + dockercontainer "github.com/moby/moby/api/types/container" + "github.com/moby/moby/client" + natsio "github.com/nats-io/nats.go" + + "github.com/samcharles93/archie-core/internal/config" + archiecontainer "github.com/samcharles93/archie-core/internal/container" + agentnats "github.com/samcharles93/archie-core/internal/infrastructure/agenttransport/nats" + "github.com/samcharles93/archie-core/internal/storage" + "github.com/samcharles93/archie-core/internal/store" + "github.com/samcharles93/archie-core/internal/taskrun" + "github.com/samcharles93/archie-core/internal/worktree" +) + +type logBuffer struct { + mu sync.Mutex + buf bytes.Buffer +} + +func (l *logBuffer) Write(p []byte) (n int, err error) { + l.mu.Lock() + defer l.mu.Unlock() + return l.buf.Write(p) +} + +func (l *logBuffer) String() string { + l.mu.Lock() + defer l.mu.Unlock() + return l.buf.String() +} + +type errGrantIssuer struct{} + +func (errGrantIssuer) Issue(*store.Task) (string, func(), error) { + return "", nil, errors.New("simulated grant issue error") +} + +type errStorage struct { + setupErr error +} + +func (s *errStorage) Setup(context.Context, storage.TaskRef) ([]storage.Mount, error) { + if s.setupErr != nil { + return nil, s.setupErr + } + return nil, nil +} + +func (s *errStorage) Teardown(context.Context, storage.TaskRef) error { + return nil +} + +func (s *errStorage) CleanupExpired(context.Context, time.Duration) (int, error) { + return 0, nil +} + +func TestCancelledTaskTransitionsToParked(t *testing.T) { + tests := []struct { + name string + cancelBefore bool + cancelDuring bool + nilGrants bool + errGrants bool + decodeErr bool + agentErr string + wantStatus string + wantParkSubstr string + }{ + { + name: "task cancelled during requestTaskRun", + cancelDuring: true, + wantStatus: store.StatusParked, + wantParkSubstr: "context canceled", + }, + { + name: "task cancelled before runViaAgent", + cancelBefore: true, + wantStatus: store.StatusParked, + wantParkSubstr: "context canceled", + }, + { + name: "worktree grants unavailable", + nilGrants: true, + wantStatus: store.StatusParked, + wantParkSubstr: "worktree publication grants are unavailable", + }, + { + name: "worktree grant issue failed", + errGrants: true, + wantStatus: store.StatusParked, + wantParkSubstr: "worktree publication grant failed", + }, + { + name: "taskrun decode response failed", + decodeErr: true, + wantStatus: store.StatusParked, + wantParkSubstr: "taskrun decode response failed", + }, + { + name: "taskrun agent returned error", + agentErr: "simulated agent execution failure", + wantStatus: store.StatusParked, + wantParkSubstr: "taskrun run failed: simulated agent execution failure", + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + d, s, busClient := daemonWithNATS(t) + ctx, cancel := context.WithCancel(context.Background()) + t.Cleanup(cancel) + + if _, err := s.EnqueueIssue(context.Background(), "acme", "widget", 1, "task", "body", "", ""); err != nil { + t.Fatal(err) + } + task, err := s.ClaimNext(context.Background()) + if err != nil || task == nil { + t.Fatalf("ClaimNext: (%v, %v)", task, err) + } + + if tt.nilGrants { + d.WorktreeGrants = nil + } + if tt.errGrants { + d.WorktreeGrants = errGrantIssuer{} + } + + if tt.cancelBefore { + cancel() + } + + if tt.cancelDuring { + sub, err := mustCoreConn(t, busClient).Subscribe(agentnats.SubjectForTask(task.ID), func(msg *natsio.Msg) { + cancel() + }) + if err != nil { + t.Fatal(err) + } + t.Cleanup(func() { _ = sub.Unsubscribe() }) + } + + if tt.decodeErr { + sub, err := mustCoreConn(t, busClient).Subscribe(agentnats.SubjectForTask(task.ID), func(msg *natsio.Msg) { + _ = msg.Respond([]byte("invalid json response")) + }) + if err != nil { + t.Fatal(err) + } + t.Cleanup(func() { _ = sub.Unsubscribe() }) + } + + if tt.agentErr != "" { + sub, err := mustCoreConn(t, busClient).Subscribe(agentnats.SubjectForTask(task.ID), func(msg *natsio.Msg) { + resp, _ := json.Marshal(taskrun.Response{Error: tt.agentErr}) + _ = msg.Respond(resp) + }) + if err != nil { + t.Fatal(err) + } + t.Cleanup(func() { _ = sub.Unsubscribe() }) + } + + d.TaskRunReadyTimeout = 100 * time.Millisecond + d.TaskRunRetryBackoff = 10 * time.Millisecond + + d.runViaAgent(ctx, task, config.Repo{Owner: "acme", Name: "widget"}) + + got, err := s.TaskByID(context.Background(), task.ID) + if err != nil { + t.Fatalf("TaskByID: %v", err) + } + if got.Status != tt.wantStatus { + t.Fatalf("task status = %q, want %q", got.Status, tt.wantStatus) + } + if !strings.Contains(got.ParkReason, tt.wantParkSubstr) { + t.Errorf("park reason = %q, want substring %q", got.ParkReason, tt.wantParkSubstr) + } + }) + } +} + +func TestCleanupTerminalTaskWorktreeWithCancelledContext(t *testing.T) { + tests := []struct { + name string + taskStatus string + prNumber int + wantTreeExists bool + }{ + { + name: "merged task with cancelled context cleans worktree", + taskStatus: store.StatusMerged, + prNumber: 0, + wantTreeExists: false, + }, + { + name: "rejected task with cancelled context cleans worktree", + taskStatus: store.StatusRejected, + prNumber: 0, + wantTreeExists: false, + }, + { + name: "closed wont do task with cancelled context cleans worktree", + taskStatus: store.StatusClosedWontDo, + prNumber: 0, + wantTreeExists: false, + }, + { + name: "parked task with cancelled context preserves worktree", + taskStatus: store.StatusParked, + prNumber: 0, + wantTreeExists: true, + }, + { + name: "open PR task with cancelled context preserves worktree", + taskStatus: store.StatusPROpen, + prNumber: 12, + wantTreeExists: true, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + ctx, cancel := context.WithCancel(context.Background()) + cancel() // cancel context before calling cleanup + + st := store.OpenTest(t) + t.Cleanup(func() { _ = st.Close() }) + + trees := &worktree.Manager{WorkDir: t.TempDir()} + log := slog.New(slog.DiscardHandler) + d := &Daemon{ + Store: st, + Trees: trees, + Log: log, + } + + if _, err := st.EnqueueIssue(context.Background(), "acme", "widget", 101, "task", "body", "bug", ""); err != nil { + t.Fatalf("EnqueueIssue: %v", err) + } + task, err := st.ClaimNext(context.Background()) + if err != nil || task == nil { + t.Fatalf("ClaimNext: (%v, %v)", task, err) + } + + if err := st.Transition(context.Background(), task.ID, store.StatusRunning, tt.taskStatus, "setup"); err != nil { + t.Fatalf("Transition: %v", err) + } + if tt.prNumber > 0 { + task.PRNumber = tt.prNumber + if err := st.Update(context.Background(), task); err != nil { + t.Fatalf("Update: %v", err) + } + } + + workDir := trees.Dir(task.Owner, task.Repo, task.IssueNumber) + if err := os.MkdirAll(workDir, 0o755); err != nil { + t.Fatalf("create workDir: %v", err) + } + + d.cleanupTerminalTaskWorktree(ctx, task, trees) + + _, statErr := os.Stat(workDir) + treeExists := statErr == nil + if treeExists != tt.wantTreeExists { + t.Errorf("worktree exists = %v, want %v (path: %s)", treeExists, tt.wantTreeExists, workDir) + } + }) + } +} + +func TestProcessStopRunningTransitionsToParked(t *testing.T) { + host := newLocalRemote(t, "acme", "widget") + d, s, busClient := daemonWithNATS(t) + ctx := context.Background() + + rootTrees := &worktree.Manager{ + WorkDir: t.TempDir(), + Token: "unused", + BotUser: "archie-bot", + BotEmail: "archie-bot@example.com", + BaseURL: "file://" + host, + } + d.Trees = rootTrees + + repo := config.Repo{Owner: "acme", Name: "widget", Base: "main"} + d.Cfg.Set(config.Config{ + Repos: []config.Repo{repo}, + }) + + // Mock Docker API server for ContainerPool + dockerAPI := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + switch { + case strings.HasSuffix(r.URL.Path, "/containers/create"): + writeContainerDockerJSON(t, w, dockercontainer.CreateResponse{ID: "test-container-id"}) + case strings.Contains(r.URL.Path, "/containers/test-container-id"): + w.WriteHeader(http.StatusNoContent) + case strings.HasSuffix(r.URL.Path, "/containers/json"): + writeContainerDockerJSON(t, w, []any{}) + default: + http.Error(w, "unexpected Docker API path "+r.URL.Path, http.StatusNotFound) + } + })) + t.Cleanup(dockerAPI.Close) + + dockerClient, err := client.New(client.WithHost(dockerAPI.URL), client.WithAPIVersion("1.55")) + if err != nil { + t.Fatal(err) + } + t.Cleanup(func() { _ = dockerClient.Close() }) + + pool, err := archiecontainer.NewPool(ctx, archiecontainer.Config{ + Image: "archie-agent:test", + DockerClient: dockerClient, + }, d.Log) + if err != nil { + t.Fatal(err) + } + d.ContainerPool = pool + d.Storage = &noopStorage{} + + if _, err := s.EnqueueIssue(ctx, "acme", "widget", 42, "issue to stop", "body", "bug", ""); err != nil { + t.Fatal(err) + } + task, err := s.ClaimNext(ctx) + if err != nil || task == nil { + t.Fatalf("ClaimNext: (%v, %v)", task, err) + } + + // When agent request is received, stop the running task via d.CancelTask(task.ID) + sub, err := mustCoreConn(t, busClient).Subscribe(agentnats.SubjectForTask(task.ID), func(msg *natsio.Msg) { + stopped := d.CancelTask(task.ID) + if !stopped { + t.Errorf("CancelTask(%d) = false, want true", task.ID) + } + }) + if err != nil { + t.Fatal(err) + } + t.Cleanup(func() { _ = sub.Unsubscribe() }) + + d.TaskRunReadyTimeout = 100 * time.Millisecond + d.TaskRunRetryBackoff = 10 * time.Millisecond + + d.process(ctx, task) + + got, err := s.TaskByID(ctx, task.ID) + if err != nil { + t.Fatalf("TaskByID: %v", err) + } + if got.Status != store.StatusParked { + t.Fatalf("stopped task status = %q, want %q", got.Status, store.StatusParked) + } + if !strings.Contains(got.ParkReason, "context canceled") { + t.Errorf("park reason = %q, want substring 'context canceled'", got.ParkReason) + } +} + +func TestParkRunningTaskGuardedAndLogging(t *testing.T) { + tests := []struct { + name string + initialStatus string + workerStatus string + closeStore bool + cancelContext bool + wantStatus string + wantWarning bool + wantLogSubstr string + }{ + { + name: "successful park on cancelled context transitions row to parked without warning", + initialStatus: store.StatusRunning, + cancelContext: true, + wantStatus: store.StatusParked, + wantWarning: false, + }, + { + name: "stale transition preserves worker status and logs no warning", + initialStatus: store.StatusRunning, + workerStatus: store.StatusMerged, + cancelContext: true, + wantStatus: store.StatusMerged, + wantWarning: false, + }, + { + name: "store write failure logs warning", + initialStatus: store.StatusRunning, + closeStore: true, + wantWarning: true, + wantLogSubstr: "terminal park transition failed", + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + s := store.OpenTest(t) + t.Cleanup(func() { _ = s.Close() }) + + var buf logBuffer + log := slog.New(slog.NewTextHandler(&buf, nil)) + + d := &Daemon{ + Store: s, + Log: log, + } + + ctx := context.Background() + if _, err := s.EnqueueIssue(ctx, "acme", "widget", 55, "task", "body", "bug", ""); err != nil { + t.Fatal(err) + } + task, err := s.ClaimNext(ctx) + if err != nil || task == nil { + t.Fatalf("ClaimNext: (%v, %v)", task, err) + } + + if tt.workerStatus != "" { + if err := s.Transition(ctx, task.ID, store.StatusRunning, tt.workerStatus, "worker completion"); err != nil { + t.Fatalf("Transition to workerStatus: %v", err) + } + } + + callCtx := ctx + if tt.cancelContext { + cctx, cancel := context.WithCancel(ctx) + cancel() + callCtx = cctx + } + + if tt.closeStore { + _ = s.Close() + } + + d.parkRunningTask(callCtx, task.ID, "simulated park reason") + + logOutput := buf.String() + if tt.wantWarning { + if !strings.Contains(logOutput, tt.wantLogSubstr) { + t.Errorf("log output = %q, want substring %q", logOutput, tt.wantLogSubstr) + } + if !strings.Contains(logOutput, "level=WARN") { + t.Errorf("log output = %q, want level=WARN", logOutput) + } + } else if strings.Contains(logOutput, "terminal park transition failed") { + t.Errorf("unexpected warning in log output: %q", logOutput) + } + + if !tt.closeStore { + got, err := s.TaskByID(ctx, task.ID) + if err != nil { + t.Fatalf("TaskByID: %v", err) + } + if got.Status != tt.wantStatus { + t.Errorf("task status = %q, want %q", got.Status, tt.wantStatus) + } + } + }) + } +} + +func TestAcquireTaskContainerFailureParksTaskOnCancelledContext(t *testing.T) { + tests := []struct { + name string + nilStorage bool + setupErr error + wantParkSubstr string + }{ + { + name: "storage backend not configured", + nilStorage: true, + wantParkSubstr: "storage backend not configured", + }, + { + name: "storage setup failed", + setupErr: errors.New("simulated setup error"), + wantParkSubstr: "storage setup failed", + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + s := store.OpenTest(t) + t.Cleanup(func() { _ = s.Close() }) + + d := &Daemon{ + Store: s, + Log: slog.New(slog.DiscardHandler), + } + + ctx, cancel := context.WithCancel(context.Background()) + cancel() // cancelled context + + if _, err := s.EnqueueIssue(context.Background(), "acme", "widget", 88, "task", "body", "bug", ""); err != nil { + t.Fatal(err) + } + task, err := s.ClaimNext(context.Background()) + if err != nil || task == nil { + t.Fatalf("ClaimNext: (%v, %v)", task, err) + } + + workDir := t.TempDir() + + if !tt.nilStorage { + d.Storage = &errStorage{setupErr: tt.setupErr} + } + + ctr, ok := d.acquireTaskContainer(ctx, task, config.Repo{Owner: "acme", Name: "widget"}, workDir) + if ok || ctr != nil { + t.Fatalf("acquireTaskContainer ok = %v, ctr = %v; want false, nil", ok, ctr) + } + + got, err := s.TaskByID(context.Background(), task.ID) + if err != nil { + t.Fatalf("TaskByID: %v", err) + } + if got.Status != store.StatusParked { + t.Fatalf("task status = %q, want %q", got.Status, store.StatusParked) + } + if !strings.Contains(got.ParkReason, tt.wantParkSubstr) { + t.Errorf("park reason = %q, want substring %q", got.ParkReason, tt.wantParkSubstr) + } + }) + } +}