Merge pull request #881 from samcharles93/fix/daemon-sweep-budget-diagnostics

fix(daemon): name an exhausted sweep budget as the cause
This commit is contained in:
sam authored and GitHub committed 2026-09-19 03:38:55 +10:00
commit 3246aec112
2 files changed
+199

No files matched your search

+10
View File
@@ -296,6 +296,16 @@ func (d *Daemon) sweepAccess(ctx context.Context) {
}
for _, r := range repos {
if err := fg.VerifyPush(ctx, r.Owner, r.Name); err != nil {
// A sweep cut short by its own context is not a repo failure: name
// the budget when it expired, stay quiet on a shutdown.
switch {
case errors.Is(ctx.Err(), context.DeadlineExceeded):
// Every later call would return the same error, so say it once.
log.Warn("push check abandoned: sweep budget exhausted", "repo", r.FullName(), "err", err)
return
case ctx.Err() != nil:
return
}
log.Warn("repo not pushable -- tasks from it will fail", "repo", r.FullName(), "err", err)
}
}
+189
View File
@@ -3,6 +3,7 @@ package daemon
import (
"context"
"encoding/json"
"errors"
"fmt"
"log/slog"
"os"
@@ -1466,6 +1467,194 @@ func assertSwept(t *testing.T, name string, fg *sweepForge, wantInvitations int,
}
}
// stallForge stands in for a forge client whose request never returns:
// AcceptInvitations signals that its sweep has started, then blocks until the
// test releases it or the sweep's own deadline fires.
type stallForge struct {
testForge
entered chan struct{}
release chan struct{}
}
func (f *stallForge) AcceptInvitations(ctx context.Context) error {
close(f.entered)
select {
case <-f.release:
return nil
case <-ctx.Done():
return ctx.Err()
}
}
// TestStartupSweepIsolatesAStalledForge pins the concurrency half of the
// stalled-forge fix. Sweeping forges one at a time on the startup goroutine
// left every later identity unswept when one forge request never returned,
// and Startup gates the gateways and the run loop, so it held boot open too.
// Each forge now gets its own goroutine; this asserts the ordering that
// proves it: the sibling identity's sweep completes while the root forge is
// still blocked.
func TestStartupSweepIsolatesAStalledForge(t *testing.T) {
rootFg := &stallForge{entered: make(chan struct{}), release: make(chan struct{})}
siblingFg := &sweepForge{}
d := &Daemon{
Cfg: config.NewHolder(config.Config{}),
Store: store.OpenTest(t),
Forge: rootFg,
Log: slog.New(slog.DiscardHandler),
Identities: []*IdentityRunner{{Name: "sibling", Forge: siblingFg}},
}
errs := make(chan error, 1)
done := make(chan struct{})
go func() {
defer close(done)
errs <- d.Startup(context.Background())
}()
waitFor(t, 5*time.Second, "the stalled root forge to start its sweep", func() bool {
select {
case <-rootFg.entered:
return true
default:
return false
}
})
waitFor(t, 5*time.Second, "the sibling identity's sweep to finish", func() bool {
invitations, _ := siblingFg.snapshot()
return invitations == 1
})
close(rootFg.release)
waitFor(t, 5*time.Second, "Startup to return once the stall is released", func() bool {
select {
case <-done:
return true
default:
return false
}
})
if err := <-errs; err != nil {
t.Errorf("Startup: %v", err)
}
}
// waitFor polls cond until it holds or the timeout expires.
func waitFor(t *testing.T, timeout time.Duration, what string, cond func() bool) {
t.Helper()
deadline := time.Now().Add(timeout)
for time.Now().Before(deadline) {
if cond() {
return
}
time.Sleep(time.Millisecond)
}
t.Fatalf("timed out waiting for %s", what)
}
// verifyForge returns a fixed error from VerifyPush, or blocks until the
// sweep's own deadline expires first, standing in for an unreachable host.
type verifyForge struct {
testForge
mu sync.Mutex
calls int
block bool
}
var errVerifyDenied = errors.New("push access denied")
func (f *verifyForge) VerifyPush(ctx context.Context, _, _ string) error {
f.mu.Lock()
f.calls++
f.mu.Unlock()
if f.block {
<-ctx.Done()
return ctx.Err()
}
return errVerifyDenied
}
func (f *verifyForge) callCount() int {
f.mu.Lock()
defer f.mu.Unlock()
return f.calls
}
// TestSweepNamesTheCauseOfAnAbandonedPushCheck pins what the boot sweep
// reports when a push check does not come back. One budget covers a forge's
// whole sweep, so a repo cut short by the deadline used to warn that it was
// "not pushable" -- asserting access had been denied when nothing about
// access had been learned, once per remaining repo. An operator shutdown
// cancels the same context, and that is neither a spent budget nor a denial.
func TestSweepNamesTheCauseOfAnAbandonedPushCheck(t *testing.T) {
repos := []config.Repo{
{Owner: "acme", Name: "one"},
{Owner: "acme", Name: "two"},
{Owner: "acme", Name: "three"},
}
tests := []struct {
name string
mode string
wantDenied int
wantBudget int
wantCalls int
}{
{
name: "a denied push is reported per repo",
mode: "deny",
wantDenied: len(repos),
wantCalls: len(repos),
},
{
name: "an exhausted budget is reported once, as the cause",
mode: "deadline",
wantBudget: 1,
wantCalls: 1,
},
{
name: "a cancelled startup is reported as neither",
mode: "cancel",
wantCalls: 1,
},
}
for _, tc := range tests {
t.Run(tc.name, func(t *testing.T) {
fg := &verifyForge{block: tc.mode != "deny"}
var logs logBuffer
d := &Daemon{
Cfg: config.NewHolder(config.Config{Repos: repos}),
Forge: fg,
Log: slog.New(slog.NewTextHandler(&logs, nil)),
}
ctx := context.Background()
switch tc.mode {
case "deadline":
// Expire the sweep's budget without waiting out sweepTimeout.
deadline, cancel := context.WithTimeout(ctx, 50*time.Millisecond)
defer cancel()
ctx = deadline
case "cancel":
cancelled, cancel := context.WithCancel(ctx)
cancel()
ctx = cancelled
}
d.sweepAccess(ctx)
if got := fg.callCount(); got != tc.wantCalls {
t.Errorf("VerifyPush calls = %d, want %d", got, tc.wantCalls)
}
if got := strings.Count(logs.String(), "repo not pushable"); got != tc.wantDenied {
t.Errorf("denied-push warnings = %d, want %d\nlogs:\n%s", got, tc.wantDenied, logs.String())
}
if got := strings.Count(logs.String(), "sweep budget exhausted"); got != tc.wantBudget {
t.Errorf("budget warnings = %d, want %d\nlogs:\n%s", got, tc.wantBudget, logs.String())
}
})
}
}
// mustCoreConn returns the raw NATS connection for tests that drive core-NATS
// subscriptions directly, failing the test if the client is not connected.
func mustCoreConn(t *testing.T, c *arnats.Client) *natsio.Conn {