fix(daemon): claim a requeued backlog without a poll wait per task
This commit is contained in:
3 files changed
+74
-5
No files matched your search
@@ -5,6 +5,10 @@ release's per-component sections are labelled beneath its heading.
|
||||
|
||||
## [Unreleased]
|
||||
|
||||
### archied
|
||||
|
||||
- archied no longer gets killed by the systemd watchdog when a restart or update leaves many tasks to resume. Each resumed task used to cost a full queue poll before the next was picked up, so a dozen of them outlasted the 30-second watchdog; an update that hit this was rolled back.
|
||||
|
||||
## [1.61.1] - 2026-11-07
|
||||
|
||||
### archie-gateway
|
||||
|
||||
@@ -887,6 +887,7 @@ func (d *Daemon) drainNATS(ctx context.Context) {
|
||||
}
|
||||
if err == nil {
|
||||
d.submitNATSTask(ctx, dispatcher, msg)
|
||||
d.markPass("drain")
|
||||
continue
|
||||
}
|
||||
|
||||
@@ -894,15 +895,26 @@ func (d *Daemon) drainNATS(ctx context.Context) {
|
||||
// the SQLite fallback must run: requeued tasks (waiting_approval
|
||||
// approval, retry-parked) never went through a NATS publish, so
|
||||
// nothing will ever arrive on the stream to trigger them.
|
||||
task, claimErr := d.Store.ClaimNext(ctx)
|
||||
if claimErr != nil {
|
||||
d.Log.ErrorContext(ctx, "sqlite claim failed", "err", claimErr)
|
||||
break
|
||||
d.claimRequeued(ctx, dispatcher)
|
||||
return
|
||||
}
|
||||
}
|
||||
|
||||
// claimRequeued submits every task the store's claim fallback holds. It does
|
||||
// not go back to the stream between claims: each empty fetch waits a full poll
|
||||
// timeout, so a backlog left by a restart would outlast the systemd watchdog.
|
||||
func (d *Daemon) claimRequeued(ctx context.Context, dispatcher *taskDispatcher) {
|
||||
for ctx.Err() == nil {
|
||||
task, err := d.Store.ClaimNext(ctx)
|
||||
if err != nil {
|
||||
d.Log.ErrorContext(ctx, "sqlite claim failed", "err", err)
|
||||
return
|
||||
}
|
||||
if task == nil {
|
||||
break
|
||||
return
|
||||
}
|
||||
dispatcher.Submit(ctx, task, d.process)
|
||||
d.markPass("drain")
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -5,6 +5,7 @@ import (
|
||||
"log/slog"
|
||||
"runtime"
|
||||
"sync"
|
||||
"sync/atomic"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
@@ -573,3 +574,55 @@ func TestDrainPassDoesNotWaitForItsTasks(t *testing.T) {
|
||||
t.Fatal("WaitForTasks did not return after the task finished")
|
||||
}
|
||||
}
|
||||
|
||||
// countingTasks is an empty stream that counts how often a pass waits on it.
|
||||
type countingTasks struct {
|
||||
stubTasks
|
||||
fetches atomic.Int32
|
||||
}
|
||||
|
||||
func (c *countingTasks) Fetch(context.Context) (eventbus.Message, error) {
|
||||
c.fetches.Add(1)
|
||||
return nil, eventbus.ErrNoMessage
|
||||
}
|
||||
|
||||
// queuedStore hands out its queue of claimed tasks one at a time.
|
||||
type queuedStore struct {
|
||||
stubStore
|
||||
queue []*workflow.Task
|
||||
}
|
||||
|
||||
func (s *queuedStore) ClaimNext(context.Context) (*workflow.Task, error) {
|
||||
if len(s.queue) == 0 {
|
||||
return nil, nil
|
||||
}
|
||||
task := s.queue[0]
|
||||
s.queue = s.queue[1:]
|
||||
return task, nil
|
||||
}
|
||||
|
||||
// A backlog of requeued tasks is claimed without waiting on the empty stream
|
||||
// between claims. Each wait is a full poll timeout, so after a restart left
|
||||
// fourteen tasks running the first pass outlasted the systemd watchdog and
|
||||
// archied was killed.
|
||||
func TestDrainPassClaimsABacklogWithoutWaitingPerTask(t *testing.T) {
|
||||
store := &queuedStore{park: func() {}}
|
||||
for id := range int64(5) {
|
||||
store.queue = append(store.queue, &workflow.Task{ID: id + 1, Owner: "acme", Repo: "app"})
|
||||
}
|
||||
tasks := &countingTasks{}
|
||||
d := &Daemon{
|
||||
Cfg: config.NewHolder(config.Config{}),
|
||||
Log: slog.New(slog.DiscardHandler),
|
||||
Store: store,
|
||||
Tasks: tasks,
|
||||
}
|
||||
d.drainNATS(context.Background())
|
||||
d.WaitForTasks()
|
||||
if len(store.queue) != 0 {
|
||||
t.Fatalf("%d tasks left unclaimed", len(store.queue))
|
||||
}
|
||||
if got := tasks.fetches.Load(); got != 1 {
|
||||
t.Fatalf("the pass waited on the stream %d times for 5 claims, want 1", got)
|
||||
}
|
||||
}
|
||||
Reference in new issue
Block a user