diff --git a/CHANGELOG.md b/CHANGELOG.md index 363af1a6a..2b314bff2 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,6 +5,10 @@ release's per-component sections are labelled beneath its heading. ## [Unreleased] +### archied + +- archied no longer gets killed by the systemd watchdog when a restart or update leaves many tasks to resume. Each resumed task used to cost a full queue poll before the next was picked up, so a dozen of them outlasted the 30-second watchdog; an update that hit this was rolled back. + ## [1.61.1] - 2026-11-07 ### archie-gateway diff --git a/internal/daemon/daemon.go b/internal/daemon/daemon.go index 49319d4c5..d391dd607 100644 --- a/internal/daemon/daemon.go +++ b/internal/daemon/daemon.go @@ -887,6 +887,7 @@ func (d *Daemon) drainNATS(ctx context.Context) { } if err == nil { d.submitNATSTask(ctx, dispatcher, msg) + d.markPass("drain") continue } @@ -894,15 +895,26 @@ func (d *Daemon) drainNATS(ctx context.Context) { // the SQLite fallback must run: requeued tasks (waiting_approval // approval, retry-parked) never went through a NATS publish, so // nothing will ever arrive on the stream to trigger them. - task, claimErr := d.Store.ClaimNext(ctx) - if claimErr != nil { - d.Log.ErrorContext(ctx, "sqlite claim failed", "err", claimErr) - break + d.claimRequeued(ctx, dispatcher) + return + } +} + +// claimRequeued submits every task the store's claim fallback holds. It does +// not go back to the stream between claims: each empty fetch waits a full poll +// timeout, so a backlog left by a restart would outlast the systemd watchdog. +func (d *Daemon) claimRequeued(ctx context.Context, dispatcher *taskDispatcher) { + for ctx.Err() == nil { + task, err := d.Store.ClaimNext(ctx) + if err != nil { + d.Log.ErrorContext(ctx, "sqlite claim failed", "err", err) + return } if task == nil { - break + return } dispatcher.Submit(ctx, task, d.process) + d.markPass("drain") } } diff --git a/internal/daemon/dispatcher_test.go b/internal/daemon/dispatcher_test.go index a76c56018..c2b15fa50 100644 --- a/internal/daemon/dispatcher_test.go +++ b/internal/daemon/dispatcher_test.go @@ -5,6 +5,7 @@ import ( "log/slog" "runtime" "sync" + "sync/atomic" "testing" "time" @@ -573,3 +574,55 @@ func TestDrainPassDoesNotWaitForItsTasks(t *testing.T) { t.Fatal("WaitForTasks did not return after the task finished") } } + +// countingTasks is an empty stream that counts how often a pass waits on it. +type countingTasks struct { + stubTasks + fetches atomic.Int32 +} + +func (c *countingTasks) Fetch(context.Context) (eventbus.Message, error) { + c.fetches.Add(1) + return nil, eventbus.ErrNoMessage +} + +// queuedStore hands out its queue of claimed tasks one at a time. +type queuedStore struct { + stubStore + queue []*workflow.Task +} + +func (s *queuedStore) ClaimNext(context.Context) (*workflow.Task, error) { + if len(s.queue) == 0 { + return nil, nil + } + task := s.queue[0] + s.queue = s.queue[1:] + return task, nil +} + +// A backlog of requeued tasks is claimed without waiting on the empty stream +// between claims. Each wait is a full poll timeout, so after a restart left +// fourteen tasks running the first pass outlasted the systemd watchdog and +// archied was killed. +func TestDrainPassClaimsABacklogWithoutWaitingPerTask(t *testing.T) { + store := &queuedStore{park: func() {}} + for id := range int64(5) { + store.queue = append(store.queue, &workflow.Task{ID: id + 1, Owner: "acme", Repo: "app"}) + } + tasks := &countingTasks{} + d := &Daemon{ + Cfg: config.NewHolder(config.Config{}), + Log: slog.New(slog.DiscardHandler), + Store: store, + Tasks: tasks, + } + d.drainNATS(context.Background()) + d.WaitForTasks() + if len(store.queue) != 0 { + t.Fatalf("%d tasks left unclaimed", len(store.queue)) + } + if got := tasks.fetches.Load(); got != 1 { + t.Fatalf("the pass waited on the stream %d times for 5 claims, want 1", got) + } +}