fix(daemon): claim a requeued backlog without a poll wait per task
This commit is contained in:
3 files changed
+74
-5
No files matched your search
@@ -5,6 +5,10 @@ release's per-component sections are labelled beneath its heading.
|
|||||||
|
|
||||||
## [Unreleased]
|
## [Unreleased]
|
||||||
|
|
||||||
|
### archied
|
||||||
|
|
||||||
|
- archied no longer gets killed by the systemd watchdog when a restart or update leaves many tasks to resume. Each resumed task used to cost a full queue poll before the next was picked up, so a dozen of them outlasted the 30-second watchdog; an update that hit this was rolled back.
|
||||||
|
|
||||||
## [1.61.1] - 2026-11-07
|
## [1.61.1] - 2026-11-07
|
||||||
|
|
||||||
### archie-gateway
|
### archie-gateway
|
||||||
|
|||||||
@@ -887,6 +887,7 @@ func (d *Daemon) drainNATS(ctx context.Context) {
|
|||||||
}
|
}
|
||||||
if err == nil {
|
if err == nil {
|
||||||
d.submitNATSTask(ctx, dispatcher, msg)
|
d.submitNATSTask(ctx, dispatcher, msg)
|
||||||
|
d.markPass("drain")
|
||||||
continue
|
continue
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -894,15 +895,26 @@ func (d *Daemon) drainNATS(ctx context.Context) {
|
|||||||
// the SQLite fallback must run: requeued tasks (waiting_approval
|
// the SQLite fallback must run: requeued tasks (waiting_approval
|
||||||
// approval, retry-parked) never went through a NATS publish, so
|
// approval, retry-parked) never went through a NATS publish, so
|
||||||
// nothing will ever arrive on the stream to trigger them.
|
// nothing will ever arrive on the stream to trigger them.
|
||||||
task, claimErr := d.Store.ClaimNext(ctx)
|
d.claimRequeued(ctx, dispatcher)
|
||||||
if claimErr != nil {
|
return
|
||||||
d.Log.ErrorContext(ctx, "sqlite claim failed", "err", claimErr)
|
}
|
||||||
break
|
}
|
||||||
|
|
||||||
|
// claimRequeued submits every task the store's claim fallback holds. It does
|
||||||
|
// not go back to the stream between claims: each empty fetch waits a full poll
|
||||||
|
// timeout, so a backlog left by a restart would outlast the systemd watchdog.
|
||||||
|
func (d *Daemon) claimRequeued(ctx context.Context, dispatcher *taskDispatcher) {
|
||||||
|
for ctx.Err() == nil {
|
||||||
|
task, err := d.Store.ClaimNext(ctx)
|
||||||
|
if err != nil {
|
||||||
|
d.Log.ErrorContext(ctx, "sqlite claim failed", "err", err)
|
||||||
|
return
|
||||||
}
|
}
|
||||||
if task == nil {
|
if task == nil {
|
||||||
break
|
return
|
||||||
}
|
}
|
||||||
dispatcher.Submit(ctx, task, d.process)
|
dispatcher.Submit(ctx, task, d.process)
|
||||||
|
d.markPass("drain")
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
@@ -5,6 +5,7 @@ import (
|
|||||||
"log/slog"
|
"log/slog"
|
||||||
"runtime"
|
"runtime"
|
||||||
"sync"
|
"sync"
|
||||||
|
"sync/atomic"
|
||||||
"testing"
|
"testing"
|
||||||
"time"
|
"time"
|
||||||
|
|
||||||
@@ -573,3 +574,55 @@ func TestDrainPassDoesNotWaitForItsTasks(t *testing.T) {
|
|||||||
t.Fatal("WaitForTasks did not return after the task finished")
|
t.Fatal("WaitForTasks did not return after the task finished")
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// countingTasks is an empty stream that counts how often a pass waits on it.
|
||||||
|
type countingTasks struct {
|
||||||
|
stubTasks
|
||||||
|
fetches atomic.Int32
|
||||||
|
}
|
||||||
|
|
||||||
|
func (c *countingTasks) Fetch(context.Context) (eventbus.Message, error) {
|
||||||
|
c.fetches.Add(1)
|
||||||
|
return nil, eventbus.ErrNoMessage
|
||||||
|
}
|
||||||
|
|
||||||
|
// queuedStore hands out its queue of claimed tasks one at a time.
|
||||||
|
type queuedStore struct {
|
||||||
|
stubStore
|
||||||
|
queue []*workflow.Task
|
||||||
|
}
|
||||||
|
|
||||||
|
func (s *queuedStore) ClaimNext(context.Context) (*workflow.Task, error) {
|
||||||
|
if len(s.queue) == 0 {
|
||||||
|
return nil, nil
|
||||||
|
}
|
||||||
|
task := s.queue[0]
|
||||||
|
s.queue = s.queue[1:]
|
||||||
|
return task, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// A backlog of requeued tasks is claimed without waiting on the empty stream
|
||||||
|
// between claims. Each wait is a full poll timeout, so after a restart left
|
||||||
|
// fourteen tasks running the first pass outlasted the systemd watchdog and
|
||||||
|
// archied was killed.
|
||||||
|
func TestDrainPassClaimsABacklogWithoutWaitingPerTask(t *testing.T) {
|
||||||
|
store := &queuedStore{park: func() {}}
|
||||||
|
for id := range int64(5) {
|
||||||
|
store.queue = append(store.queue, &workflow.Task{ID: id + 1, Owner: "acme", Repo: "app"})
|
||||||
|
}
|
||||||
|
tasks := &countingTasks{}
|
||||||
|
d := &Daemon{
|
||||||
|
Cfg: config.NewHolder(config.Config{}),
|
||||||
|
Log: slog.New(slog.DiscardHandler),
|
||||||
|
Store: store,
|
||||||
|
Tasks: tasks,
|
||||||
|
}
|
||||||
|
d.drainNATS(context.Background())
|
||||||
|
d.WaitForTasks()
|
||||||
|
if len(store.queue) != 0 {
|
||||||
|
t.Fatalf("%d tasks left unclaimed", len(store.queue))
|
||||||
|
}
|
||||||
|
if got := tasks.fetches.Load(); got != 1 {
|
||||||
|
t.Fatalf("the pass waited on the stream %d times for 5 claims, want 1", got)
|
||||||
|
}
|
||||||
|
}
|
||||||
Reference in new issue
Block a user