merge: park stopped tasks on a detached context (archie-core-0dic)

This commit is contained in:
sam committed 2026-08-25 01:45:18 +10:00
commit 18c3088b84
2 files changed
+583 -12

No files matched your search

+46 -12
View File
@@ -809,7 +809,7 @@ func (d *Daemon) process(ctx context.Context, task *store.Task) {
trees := d.treesFor(task)
repo, ok := d.repoFor(task)
if !ok {
_ = d.Store.Transition(ctx, task.ID, store.StatusRunning, store.StatusParked, "repo no longer in config")
d.parkRunningTask(ctx, task.ID, "repo no longer in config")
return
}
if d.ContainerPool == nil {
@@ -856,7 +856,7 @@ func (d *Daemon) process(ctx context.Context, task *store.Task) {
_, branch, err = trees.Prepare(ctx, task.Owner, task.Repo, repo.Base, task.IssueNumber, task.Title, task.Body, task.Labels)
if err != nil {
d.Log.Error("worktree prepare failed", "err", err)
_ = d.Store.Transition(ctx, task.ID, store.StatusRunning, store.StatusParked, "worktree prepare failed: "+err.Error())
d.parkRunningTask(ctx, task.ID, "worktree prepare failed: "+err.Error())
return
}
task.Branch = branch
@@ -899,8 +899,15 @@ func (d *Daemon) cleanupTerminalTaskWorktree(ctx context.Context, task *store.Ta
if d.Store == nil || trees == nil || task == nil {
return
}
latest, err := d.Store.TaskByID(ctx, task.ID)
if err != nil || latest == nil {
cleanupCtx, cancel := context.WithTimeout(context.WithoutCancel(ctx), 10*time.Second)
defer cancel()
latest, err := d.Store.TaskByID(cleanupCtx, task.ID)
if err != nil {
d.Log.Warn("terminal worktree cleanup task lookup failed", "task", task.ID, "err", err)
return
}
if latest == nil {
return
}
switch latest.Status {
@@ -942,7 +949,7 @@ func (d *Daemon) acquireTaskContainer(
) (*container.Container, bool) {
park := func(reason string, err error) {
d.Log.Error(reason, "err", err)
_ = d.Store.Transition(ctx, task.ID, store.StatusRunning, store.StatusParked, reason+": "+err.Error())
d.parkRunningTask(ctx, task.ID, reason+": "+err.Error())
}
// Write task.json -- the container's boot-time brief.
@@ -960,7 +967,7 @@ func (d *Daemon) acquireTaskContainer(
// normal operation, Storage is always set when ContainerPool is set.
if d.Storage == nil {
d.Log.Error("storage backend is nil -- cannot acquire container")
_ = d.Store.Transition(ctx, task.ID, store.StatusRunning, store.StatusParked, "storage backend not configured")
d.parkRunningTask(ctx, task.ID, "storage backend not configured")
return nil, false
}
@@ -997,6 +1004,33 @@ func (d *Daemon) acquireTaskContainer(
return ctr, true
}
// parkRunningTask transitions a task from running to parked using a context
// detached from caller cancellation, with a bounded timeout.
//
// When a task fails or is cancelled (e.g. via /stop or dashboard Stop), ctx is
// already cancelled. Writing terminal state using the cancelled ctx fails
// immediately in SQLite transactions and silently leaves the task row 'running'
// indefinitely. Like container teardown in Pool.Release, terminal state
// recording is cleanup on the way out of a task and must use a bounded context
// that survives cancellation while preserving values.
//
// The transition remains guarded from StatusRunning: if the worker already
// recorded a terminal state over storerpc, store.ErrStaleTransition is returned
// and ignored. Unexpected store errors are logged as warnings.
func (d *Daemon) parkRunningTask(ctx context.Context, taskID int64, reason string) {
if d.Store == nil {
return
}
writeCtx, cancel := context.WithTimeout(context.WithoutCancel(ctx), 10*time.Second)
defer cancel()
if err := d.Store.Transition(writeCtx, taskID, store.StatusRunning, store.StatusParked, reason); err != nil {
if !errors.Is(err, store.ErrStaleTransition) {
d.Log.Warn("terminal park transition failed", "task", taskID, "reason", reason, "err", err)
}
}
}
// containerEnv returns the environment variables passed to agent containers.
// runViaAgent publishes a full-task handoff to archie-agent over the
// infrastructure-owned task subject and waits for its completion report. Every durable
@@ -1011,13 +1045,13 @@ func (d *Daemon) runViaAgent(ctx context.Context, task *store.Task, repo config.
if d.WorktreeGrants == nil {
const reason = "worktree publication grants are unavailable"
d.Log.Error(reason, "task", task.ID)
_ = d.Store.Transition(ctx, task.ID, store.StatusRunning, store.StatusParked, reason)
d.parkRunningTask(ctx, task.ID, reason)
return
}
grant, revoke, err := d.WorktreeGrants.Issue(task)
if err != nil {
d.Log.Error("worktree publication grant failed", "task", task.ID, "err", err)
_ = d.Store.Transition(ctx, task.ID, store.StatusRunning, store.StatusParked, "worktree publication grant failed: "+err.Error())
d.parkRunningTask(ctx, task.ID, "worktree publication grant failed: "+err.Error())
return
}
defer revoke()
@@ -1031,26 +1065,26 @@ func (d *Daemon) runViaAgent(ctx context.Context, task *store.Task, repo config.
data, err := json.Marshal(req)
if err != nil {
d.Log.Error("taskrun encode failed", "task", task.ID, "err", err)
_ = d.Store.Transition(ctx, task.ID, store.StatusRunning, store.StatusParked, "taskrun encode failed: "+err.Error())
d.parkRunningTask(ctx, task.ID, "taskrun encode failed: "+err.Error())
return
}
reply, err := d.requestTaskRun(ctx, task.ID, data)
if err != nil {
d.Log.Error("taskrun request failed", "task", task.ID, "err", err)
_ = d.Store.Transition(ctx, task.ID, store.StatusRunning, store.StatusParked, "taskrun request failed: "+err.Error())
d.parkRunningTask(ctx, task.ID, "taskrun request failed: "+err.Error())
return
}
var resp taskrun.Response
if err := json.Unmarshal(reply, &resp); err != nil {
d.Log.Error("taskrun decode response failed", "task", task.ID, "err", err)
_ = d.Store.Transition(ctx, task.ID, store.StatusRunning, store.StatusParked, "taskrun decode response failed: "+err.Error())
d.parkRunningTask(ctx, task.ID, "taskrun decode response failed: "+err.Error())
return
}
if resp.Error != "" {
d.Log.Error("taskrun run failed", "task", task.ID, "err", resp.Error)
_ = d.Store.Transition(ctx, task.ID, store.StatusRunning, store.StatusParked, "taskrun run failed: "+resp.Error)
d.parkRunningTask(ctx, task.ID, "taskrun run failed: "+resp.Error)
return
}
+537
View File
@@ -0,0 +1,537 @@
package daemon
import (
"bytes"
"context"
"encoding/json"
"errors"
"log/slog"
"net/http"
"net/http/httptest"
"os"
"strings"
"sync"
"testing"
"time"
dockercontainer "github.com/moby/moby/api/types/container"
"github.com/moby/moby/client"
natsio "github.com/nats-io/nats.go"
"github.com/samcharles93/archie-core/internal/config"
archiecontainer "github.com/samcharles93/archie-core/internal/container"
agentnats "github.com/samcharles93/archie-core/internal/infrastructure/agenttransport/nats"
"github.com/samcharles93/archie-core/internal/storage"
"github.com/samcharles93/archie-core/internal/store"
"github.com/samcharles93/archie-core/internal/taskrun"
"github.com/samcharles93/archie-core/internal/worktree"
)
type logBuffer struct {
mu sync.Mutex
buf bytes.Buffer
}
func (l *logBuffer) Write(p []byte) (n int, err error) {
l.mu.Lock()
defer l.mu.Unlock()
return l.buf.Write(p)
}
func (l *logBuffer) String() string {
l.mu.Lock()
defer l.mu.Unlock()
return l.buf.String()
}
type errGrantIssuer struct{}
func (errGrantIssuer) Issue(*store.Task) (string, func(), error) {
return "", nil, errors.New("simulated grant issue error")
}
type errStorage struct {
setupErr error
}
func (s *errStorage) Setup(context.Context, storage.TaskRef) ([]storage.Mount, error) {
if s.setupErr != nil {
return nil, s.setupErr
}
return nil, nil
}
func (s *errStorage) Teardown(context.Context, storage.TaskRef) error {
return nil
}
func (s *errStorage) CleanupExpired(context.Context, time.Duration) (int, error) {
return 0, nil
}
func TestCancelledTaskTransitionsToParked(t *testing.T) {
tests := []struct {
name string
cancelBefore bool
cancelDuring bool
nilGrants bool
errGrants bool
decodeErr bool
agentErr string
wantStatus string
wantParkSubstr string
}{
{
name: "task cancelled during requestTaskRun",
cancelDuring: true,
wantStatus: store.StatusParked,
wantParkSubstr: "context canceled",
},
{
name: "task cancelled before runViaAgent",
cancelBefore: true,
wantStatus: store.StatusParked,
wantParkSubstr: "context canceled",
},
{
name: "worktree grants unavailable",
nilGrants: true,
wantStatus: store.StatusParked,
wantParkSubstr: "worktree publication grants are unavailable",
},
{
name: "worktree grant issue failed",
errGrants: true,
wantStatus: store.StatusParked,
wantParkSubstr: "worktree publication grant failed",
},
{
name: "taskrun decode response failed",
decodeErr: true,
wantStatus: store.StatusParked,
wantParkSubstr: "taskrun decode response failed",
},
{
name: "taskrun agent returned error",
agentErr: "simulated agent execution failure",
wantStatus: store.StatusParked,
wantParkSubstr: "taskrun run failed: simulated agent execution failure",
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
d, s, busClient := daemonWithNATS(t)
ctx, cancel := context.WithCancel(context.Background())
t.Cleanup(cancel)
if _, err := s.EnqueueIssue(context.Background(), "acme", "widget", 1, "task", "body", "", ""); err != nil {
t.Fatal(err)
}
task, err := s.ClaimNext(context.Background())
if err != nil || task == nil {
t.Fatalf("ClaimNext: (%v, %v)", task, err)
}
if tt.nilGrants {
d.WorktreeGrants = nil
}
if tt.errGrants {
d.WorktreeGrants = errGrantIssuer{}
}
if tt.cancelBefore {
cancel()
}
if tt.cancelDuring {
sub, err := mustCoreConn(t, busClient).Subscribe(agentnats.SubjectForTask(task.ID), func(msg *natsio.Msg) {
cancel()
})
if err != nil {
t.Fatal(err)
}
t.Cleanup(func() { _ = sub.Unsubscribe() })
}
if tt.decodeErr {
sub, err := mustCoreConn(t, busClient).Subscribe(agentnats.SubjectForTask(task.ID), func(msg *natsio.Msg) {
_ = msg.Respond([]byte("invalid json response"))
})
if err != nil {
t.Fatal(err)
}
t.Cleanup(func() { _ = sub.Unsubscribe() })
}
if tt.agentErr != "" {
sub, err := mustCoreConn(t, busClient).Subscribe(agentnats.SubjectForTask(task.ID), func(msg *natsio.Msg) {
resp, _ := json.Marshal(taskrun.Response{Error: tt.agentErr})
_ = msg.Respond(resp)
})
if err != nil {
t.Fatal(err)
}
t.Cleanup(func() { _ = sub.Unsubscribe() })
}
d.TaskRunReadyTimeout = 100 * time.Millisecond
d.TaskRunRetryBackoff = 10 * time.Millisecond
d.runViaAgent(ctx, task, config.Repo{Owner: "acme", Name: "widget"})
got, err := s.TaskByID(context.Background(), task.ID)
if err != nil {
t.Fatalf("TaskByID: %v", err)
}
if got.Status != tt.wantStatus {
t.Fatalf("task status = %q, want %q", got.Status, tt.wantStatus)
}
if !strings.Contains(got.ParkReason, tt.wantParkSubstr) {
t.Errorf("park reason = %q, want substring %q", got.ParkReason, tt.wantParkSubstr)
}
})
}
}
func TestCleanupTerminalTaskWorktreeWithCancelledContext(t *testing.T) {
tests := []struct {
name string
taskStatus string
prNumber int
wantTreeExists bool
}{
{
name: "merged task with cancelled context cleans worktree",
taskStatus: store.StatusMerged,
prNumber: 0,
wantTreeExists: false,
},
{
name: "rejected task with cancelled context cleans worktree",
taskStatus: store.StatusRejected,
prNumber: 0,
wantTreeExists: false,
},
{
name: "closed wont do task with cancelled context cleans worktree",
taskStatus: store.StatusClosedWontDo,
prNumber: 0,
wantTreeExists: false,
},
{
name: "parked task with cancelled context preserves worktree",
taskStatus: store.StatusParked,
prNumber: 0,
wantTreeExists: true,
},
{
name: "open PR task with cancelled context preserves worktree",
taskStatus: store.StatusPROpen,
prNumber: 12,
wantTreeExists: true,
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
ctx, cancel := context.WithCancel(context.Background())
cancel() // cancel context before calling cleanup
st := store.OpenTest(t)
t.Cleanup(func() { _ = st.Close() })
trees := &worktree.Manager{WorkDir: t.TempDir()}
log := slog.New(slog.DiscardHandler)
d := &Daemon{
Store: st,
Trees: trees,
Log: log,
}
if _, err := st.EnqueueIssue(context.Background(), "acme", "widget", 101, "task", "body", "bug", ""); err != nil {
t.Fatalf("EnqueueIssue: %v", err)
}
task, err := st.ClaimNext(context.Background())
if err != nil || task == nil {
t.Fatalf("ClaimNext: (%v, %v)", task, err)
}
if err := st.Transition(context.Background(), task.ID, store.StatusRunning, tt.taskStatus, "setup"); err != nil {
t.Fatalf("Transition: %v", err)
}
if tt.prNumber > 0 {
task.PRNumber = tt.prNumber
if err := st.Update(context.Background(), task); err != nil {
t.Fatalf("Update: %v", err)
}
}
workDir := trees.Dir(task.Owner, task.Repo, task.IssueNumber)
if err := os.MkdirAll(workDir, 0o755); err != nil {
t.Fatalf("create workDir: %v", err)
}
d.cleanupTerminalTaskWorktree(ctx, task, trees)
_, statErr := os.Stat(workDir)
treeExists := statErr == nil
if treeExists != tt.wantTreeExists {
t.Errorf("worktree exists = %v, want %v (path: %s)", treeExists, tt.wantTreeExists, workDir)
}
})
}
}
func TestProcessStopRunningTransitionsToParked(t *testing.T) {
host := newLocalRemote(t, "acme", "widget")
d, s, busClient := daemonWithNATS(t)
ctx := context.Background()
rootTrees := &worktree.Manager{
WorkDir: t.TempDir(),
Token: "unused",
BotUser: "archie-bot",
BotEmail: "archie-bot@example.com",
BaseURL: "file://" + host,
}
d.Trees = rootTrees
repo := config.Repo{Owner: "acme", Name: "widget", Base: "main"}
d.Cfg.Set(config.Config{
Repos: []config.Repo{repo},
})
// Mock Docker API server for ContainerPool
dockerAPI := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
switch {
case strings.HasSuffix(r.URL.Path, "/containers/create"):
writeContainerDockerJSON(t, w, dockercontainer.CreateResponse{ID: "test-container-id"})
case strings.Contains(r.URL.Path, "/containers/test-container-id"):
w.WriteHeader(http.StatusNoContent)
case strings.HasSuffix(r.URL.Path, "/containers/json"):
writeContainerDockerJSON(t, w, []any{})
default:
http.Error(w, "unexpected Docker API path "+r.URL.Path, http.StatusNotFound)
}
}))
t.Cleanup(dockerAPI.Close)
dockerClient, err := client.New(client.WithHost(dockerAPI.URL), client.WithAPIVersion("1.55"))
if err != nil {
t.Fatal(err)
}
t.Cleanup(func() { _ = dockerClient.Close() })
pool, err := archiecontainer.NewPool(ctx, archiecontainer.Config{
Image: "archie-agent:test",
DockerClient: dockerClient,
}, d.Log)
if err != nil {
t.Fatal(err)
}
d.ContainerPool = pool
d.Storage = &noopStorage{}
if _, err := s.EnqueueIssue(ctx, "acme", "widget", 42, "issue to stop", "body", "bug", ""); err != nil {
t.Fatal(err)
}
task, err := s.ClaimNext(ctx)
if err != nil || task == nil {
t.Fatalf("ClaimNext: (%v, %v)", task, err)
}
// When agent request is received, stop the running task via d.CancelTask(task.ID)
sub, err := mustCoreConn(t, busClient).Subscribe(agentnats.SubjectForTask(task.ID), func(msg *natsio.Msg) {
stopped := d.CancelTask(task.ID)
if !stopped {
t.Errorf("CancelTask(%d) = false, want true", task.ID)
}
})
if err != nil {
t.Fatal(err)
}
t.Cleanup(func() { _ = sub.Unsubscribe() })
d.TaskRunReadyTimeout = 100 * time.Millisecond
d.TaskRunRetryBackoff = 10 * time.Millisecond
d.process(ctx, task)
got, err := s.TaskByID(ctx, task.ID)
if err != nil {
t.Fatalf("TaskByID: %v", err)
}
if got.Status != store.StatusParked {
t.Fatalf("stopped task status = %q, want %q", got.Status, store.StatusParked)
}
if !strings.Contains(got.ParkReason, "context canceled") {
t.Errorf("park reason = %q, want substring 'context canceled'", got.ParkReason)
}
}
func TestParkRunningTaskGuardedAndLogging(t *testing.T) {
tests := []struct {
name string
initialStatus string
workerStatus string
closeStore bool
cancelContext bool
wantStatus string
wantWarning bool
wantLogSubstr string
}{
{
name: "successful park on cancelled context transitions row to parked without warning",
initialStatus: store.StatusRunning,
cancelContext: true,
wantStatus: store.StatusParked,
wantWarning: false,
},
{
name: "stale transition preserves worker status and logs no warning",
initialStatus: store.StatusRunning,
workerStatus: store.StatusMerged,
cancelContext: true,
wantStatus: store.StatusMerged,
wantWarning: false,
},
{
name: "store write failure logs warning",
initialStatus: store.StatusRunning,
closeStore: true,
wantWarning: true,
wantLogSubstr: "terminal park transition failed",
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
s := store.OpenTest(t)
t.Cleanup(func() { _ = s.Close() })
var buf logBuffer
log := slog.New(slog.NewTextHandler(&buf, nil))
d := &Daemon{
Store: s,
Log: log,
}
ctx := context.Background()
if _, err := s.EnqueueIssue(ctx, "acme", "widget", 55, "task", "body", "bug", ""); err != nil {
t.Fatal(err)
}
task, err := s.ClaimNext(ctx)
if err != nil || task == nil {
t.Fatalf("ClaimNext: (%v, %v)", task, err)
}
if tt.workerStatus != "" {
if err := s.Transition(ctx, task.ID, store.StatusRunning, tt.workerStatus, "worker completion"); err != nil {
t.Fatalf("Transition to workerStatus: %v", err)
}
}
callCtx := ctx
if tt.cancelContext {
cctx, cancel := context.WithCancel(ctx)
cancel()
callCtx = cctx
}
if tt.closeStore {
_ = s.Close()
}
d.parkRunningTask(callCtx, task.ID, "simulated park reason")
logOutput := buf.String()
if tt.wantWarning {
if !strings.Contains(logOutput, tt.wantLogSubstr) {
t.Errorf("log output = %q, want substring %q", logOutput, tt.wantLogSubstr)
}
if !strings.Contains(logOutput, "level=WARN") {
t.Errorf("log output = %q, want level=WARN", logOutput)
}
} else if strings.Contains(logOutput, "terminal park transition failed") {
t.Errorf("unexpected warning in log output: %q", logOutput)
}
if !tt.closeStore {
got, err := s.TaskByID(ctx, task.ID)
if err != nil {
t.Fatalf("TaskByID: %v", err)
}
if got.Status != tt.wantStatus {
t.Errorf("task status = %q, want %q", got.Status, tt.wantStatus)
}
}
})
}
}
func TestAcquireTaskContainerFailureParksTaskOnCancelledContext(t *testing.T) {
tests := []struct {
name string
nilStorage bool
setupErr error
wantParkSubstr string
}{
{
name: "storage backend not configured",
nilStorage: true,
wantParkSubstr: "storage backend not configured",
},
{
name: "storage setup failed",
setupErr: errors.New("simulated setup error"),
wantParkSubstr: "storage setup failed",
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
s := store.OpenTest(t)
t.Cleanup(func() { _ = s.Close() })
d := &Daemon{
Store: s,
Log: slog.New(slog.DiscardHandler),
}
ctx, cancel := context.WithCancel(context.Background())
cancel() // cancelled context
if _, err := s.EnqueueIssue(context.Background(), "acme", "widget", 88, "task", "body", "bug", ""); err != nil {
t.Fatal(err)
}
task, err := s.ClaimNext(context.Background())
if err != nil || task == nil {
t.Fatalf("ClaimNext: (%v, %v)", task, err)
}
workDir := t.TempDir()
if !tt.nilStorage {
d.Storage = &errStorage{setupErr: tt.setupErr}
}
ctr, ok := d.acquireTaskContainer(ctx, task, config.Repo{Owner: "acme", Name: "widget"}, workDir)
if ok || ctr != nil {
t.Fatalf("acquireTaskContainer ok = %v, ctr = %v; want false, nil", ok, ctr)
}
got, err := s.TaskByID(context.Background(), task.ID)
if err != nil {
t.Fatalf("TaskByID: %v", err)
}
if got.Status != store.StatusParked {
t.Fatalf("task status = %q, want %q", got.Status, store.StatusParked)
}
if !strings.Contains(got.ParkReason, tt.wantParkSubstr) {
t.Errorf("park reason = %q, want substring %q", got.ParkReason, tt.wantParkSubstr)
}
})
}
}