fix(daemon): claim a requeued backlog without a poll wait per task
deploy / gate (push) Successful in 4m56s
Quality / check (push) Canceled after 6m11s
deploy / archied-image (push) Successful in 3m1s
deploy / agent-image (push) Successful in 12s
deploy / dist-zip (push) Skipped

This commit is contained in:
sam committed 2026-11-08 06:24:54 +11:00
1 parent ae53e83ff3
commit 49d7619ba3
3 files changed
+74 -5

No files matched your search

+4
View File
@@ -5,6 +5,10 @@ release's per-component sections are labelled beneath its heading.
## [Unreleased]
### archied
- archied no longer gets killed by the systemd watchdog when a restart or update leaves many tasks to resume. Each resumed task used to cost a full queue poll before the next was picked up, so a dozen of them outlasted the 30-second watchdog; an update that hit this was rolled back.
## [1.61.1] - 2026-11-07
### archie-gateway
+17 -5
View File
@@ -887,6 +887,7 @@ func (d *Daemon) drainNATS(ctx context.Context) {
}
if err == nil {
d.submitNATSTask(ctx, dispatcher, msg)
d.markPass("drain")
continue
}
@@ -894,15 +895,26 @@ func (d *Daemon) drainNATS(ctx context.Context) {
// the SQLite fallback must run: requeued tasks (waiting_approval
// approval, retry-parked) never went through a NATS publish, so
// nothing will ever arrive on the stream to trigger them.
task, claimErr := d.Store.ClaimNext(ctx)
if claimErr != nil {
d.Log.ErrorContext(ctx, "sqlite claim failed", "err", claimErr)
break
d.claimRequeued(ctx, dispatcher)
return
}
}
// claimRequeued submits every task the store's claim fallback holds. It does
// not go back to the stream between claims: each empty fetch waits a full poll
// timeout, so a backlog left by a restart would outlast the systemd watchdog.
func (d *Daemon) claimRequeued(ctx context.Context, dispatcher *taskDispatcher) {
for ctx.Err() == nil {
task, err := d.Store.ClaimNext(ctx)
if err != nil {
d.Log.ErrorContext(ctx, "sqlite claim failed", "err", err)
return
}
if task == nil {
break
return
}
dispatcher.Submit(ctx, task, d.process)
d.markPass("drain")
}
}
+53
View File
@@ -5,6 +5,7 @@ import (
"log/slog"
"runtime"
"sync"
"sync/atomic"
"testing"
"time"
@@ -573,3 +574,55 @@ func TestDrainPassDoesNotWaitForItsTasks(t *testing.T) {
t.Fatal("WaitForTasks did not return after the task finished")
}
}
// countingTasks is an empty stream that counts how often a pass waits on it.
type countingTasks struct {
stubTasks
fetches atomic.Int32
}
func (c *countingTasks) Fetch(context.Context) (eventbus.Message, error) {
c.fetches.Add(1)
return nil, eventbus.ErrNoMessage
}
// queuedStore hands out its queue of claimed tasks one at a time.
type queuedStore struct {
stubStore
queue []*workflow.Task
}
func (s *queuedStore) ClaimNext(context.Context) (*workflow.Task, error) {
if len(s.queue) == 0 {
return nil, nil
}
task := s.queue[0]
s.queue = s.queue[1:]
return task, nil
}
// A backlog of requeued tasks is claimed without waiting on the empty stream
// between claims. Each wait is a full poll timeout, so after a restart left
// fourteen tasks running the first pass outlasted the systemd watchdog and
// archied was killed.
func TestDrainPassClaimsABacklogWithoutWaitingPerTask(t *testing.T) {
store := &queuedStore{park: func() {}}
for id := range int64(5) {
store.queue = append(store.queue, &workflow.Task{ID: id + 1, Owner: "acme", Repo: "app"})
}
tasks := &countingTasks{}
d := &Daemon{
Cfg: config.NewHolder(config.Config{}),
Log: slog.New(slog.DiscardHandler),
Store: store,
Tasks: tasks,
}
d.drainNATS(context.Background())
d.WaitForTasks()
if len(store.queue) != 0 {
t.Fatalf("%d tasks left unclaimed", len(store.queue))
}
if got := tasks.fetches.Load(); got != 1 {
t.Fatalf("the pass waited on the stream %d times for 5 claims, want 1", got)
}
}