From 05a28512680de1c114b19f880360460669587a88 Mon Sep 17 00:00:00 2001 From: Sam Catlow Date: Sat, 19 Sep 2026 02:10:58 +1000 Subject: [PATCH 1/3] test(daemon): prove a stalled forge cannot block a sibling sweep sweepAccess sweeps each forge in its own goroutine, but nothing pinned that: the table test asserts coverage, not ordering, so the sweep could be made sequential again without a failure. The test stalls the root forge in AcceptInvitations and asserts the sibling identity is swept while it is still blocked, then that Startup returns once the stall is released. Verified RED against a sequential sweepAccess (times out waiting for the sibling) and GREEN against the goroutine-per-forge implementation. --- internal/daemon/daemon_test.go | 84 ++++++++++++++++++++++++++++++++++ 1 file changed, 84 insertions(+) diff --git a/internal/daemon/daemon_test.go b/internal/daemon/daemon_test.go index 2ce250463..5f36af8db 100644 --- a/internal/daemon/daemon_test.go +++ b/internal/daemon/daemon_test.go @@ -1465,6 +1465,90 @@ func assertSwept(t *testing.T, name string, fg *sweepForge, wantInvitations int, } } +// stallForge stands in for a forge client whose request never returns: +// AcceptInvitations signals that its sweep has started, then blocks until the +// test releases it or the sweep's own deadline fires. +type stallForge struct { + testForge + entered chan struct{} + release chan struct{} +} + +func (f *stallForge) AcceptInvitations(ctx context.Context) error { + close(f.entered) + select { + case <-f.release: + return nil + case <-ctx.Done(): + return ctx.Err() + } +} + +// TestStartupSweepIsolatesAStalledForge pins the concurrency half of the +// stalled-forge fix. Sweeping forges one at a time on the startup goroutine +// left every later identity unswept when one forge request never returned, +// and Startup gates the gateways and the run loop, so it held boot open too. +// Each forge now gets its own goroutine; this asserts the ordering that +// proves it: the sibling identity's sweep completes while the root forge is +// still blocked. +func TestStartupSweepIsolatesAStalledForge(t *testing.T) { + rootFg := &stallForge{entered: make(chan struct{}), release: make(chan struct{})} + siblingFg := &sweepForge{} + d := &Daemon{ + Cfg: config.NewHolder(config.Config{}), + Store: store.OpenTest(t), + Forge: rootFg, + Log: slog.New(slog.DiscardHandler), + Identities: []*IdentityRunner{{Name: "sibling", Forge: siblingFg}}, + } + + errs := make(chan error, 1) + done := make(chan struct{}) + go func() { + defer close(done) + errs <- d.Startup(context.Background()) + }() + + waitFor(t, 5*time.Second, "the stalled root forge to start its sweep", func() bool { + select { + case <-rootFg.entered: + return true + default: + return false + } + }) + waitFor(t, 5*time.Second, "the sibling identity's sweep to finish", func() bool { + invitations, _ := siblingFg.snapshot() + return invitations == 1 + }) + + close(rootFg.release) + waitFor(t, 5*time.Second, "Startup to return once the stall is released", func() bool { + select { + case <-done: + return true + default: + return false + } + }) + if err := <-errs; err != nil { + t.Errorf("Startup: %v", err) + } +} + +// waitFor polls cond until it holds or the timeout expires. +func waitFor(t *testing.T, timeout time.Duration, what string, cond func() bool) { + t.Helper() + deadline := time.Now().Add(timeout) + for time.Now().Before(deadline) { + if cond() { + return + } + time.Sleep(time.Millisecond) + } + t.Fatalf("timed out waiting for %s", what) +} + // mustCoreConn returns the raw NATS connection for tests that drive core-NATS // subscriptions directly, failing the test if the client is not connected. func mustCoreConn(t *testing.T, c *arnats.Client) *natsio.Conn { From 056d09cdeb11d0e85379ab6a33f110ef6b007795 Mon Sep 17 00:00:00 2001 From: Sam Catlow Date: Sat, 19 Sep 2026 02:31:28 +1000 Subject: [PATCH 2/3] fix(daemon): name an exhausted sweep budget as the cause sweepAccess spends one sweepTimeout budget on a forge's whole boot sweep. When that budget ran out mid-loop, every remaining repo was warned as "repo not pushable -- tasks from it will fail" -- asserting that access had been denied when all that was known was that the deadline passed, once per repo. Report the budget once, name it as the cause, and stop: later calls return the same deadline error, so continuing only multiplies a misleading warning. RED before (three repos, budget exhausted by the first): VerifyPush calls = 3, want 1 "sweep budget exhausted" logged 0 times, want 1 level=WARN msg="repo not pushable -- tasks from it will fail" repo=acme/one err="context deadline exceeded" level=WARN msg="repo not pushable -- tasks from it will fail" repo=acme/two err="context deadline exceeded" level=WARN msg="repo not pushable -- tasks from it will fail" repo=acme/three err="context deadline exceeded" --- internal/daemon/daemon.go | 6 +++ internal/daemon/daemon_test.go | 98 ++++++++++++++++++++++++++++++++++ 2 files changed, 104 insertions(+) diff --git a/internal/daemon/daemon.go b/internal/daemon/daemon.go index 6ac79c1dc..5825e9cb2 100644 --- a/internal/daemon/daemon.go +++ b/internal/daemon/daemon.go @@ -296,6 +296,12 @@ func (d *Daemon) sweepAccess(ctx context.Context) { } for _, r := range repos { if err := fg.VerifyPush(ctx, r.Owner, r.Name); err != nil { + if ctx.Err() != nil { + // The budget failed, not the repo: every later call would return + // the same error, so name the real cause once and stop. + log.Warn("push check abandoned: sweep budget exhausted", "repo", r.FullName(), "err", err) + break + } log.Warn("repo not pushable -- tasks from it will fail", "repo", r.FullName(), "err", err) } } diff --git a/internal/daemon/daemon_test.go b/internal/daemon/daemon_test.go index 5f36af8db..c65741e46 100644 --- a/internal/daemon/daemon_test.go +++ b/internal/daemon/daemon_test.go @@ -3,6 +3,7 @@ package daemon import ( "context" "encoding/json" + "errors" "fmt" "log/slog" "os" @@ -1549,6 +1550,103 @@ func waitFor(t *testing.T, timeout time.Duration, what string, cond func() bool) t.Fatalf("timed out waiting for %s", what) } +// verifyForge returns a fixed error from VerifyPush, or blocks until the +// sweep's own deadline expires first, standing in for an unreachable host. +type verifyForge struct { + testForge + mu sync.Mutex + calls int + block bool +} + +var errVerifyDenied = errors.New("push access denied") + +func (f *verifyForge) VerifyPush(ctx context.Context, _, _ string) error { + f.mu.Lock() + f.calls++ + f.mu.Unlock() + if f.block { + <-ctx.Done() + return ctx.Err() + } + return errVerifyDenied +} + +func (f *verifyForge) callCount() int { + f.mu.Lock() + defer f.mu.Unlock() + return f.calls +} + +// TestSweepNamesAnExhaustedBudgetAsTheCause pins what the boot sweep reports +// when a push check does not come back. One budget covers a forge's whole +// sweep, so a repo cut short by the deadline used to warn that it was "not +// pushable" -- asserting access had been denied when nothing about access had +// been learned, once per remaining repo. +func TestSweepNamesAnExhaustedBudgetAsTheCause(t *testing.T) { + repos := []config.Repo{ + {Owner: "acme", Name: "one"}, + {Owner: "acme", Name: "two"}, + {Owner: "acme", Name: "three"}, + } + tests := []struct { + name string + block bool + wantLog string + wantLogCount int + absentLog string + wantCalls int + }{ + { + name: "a denied push is reported per repo", + wantLog: "repo not pushable", + wantLogCount: len(repos), + absentLog: "sweep budget exhausted", + wantCalls: len(repos), + }, + { + name: "an exhausted budget is reported once, as the cause", + block: true, + wantLog: "sweep budget exhausted", + wantLogCount: 1, + absentLog: "repo not pushable", + wantCalls: 1, + }, + } + + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + fg := &verifyForge{block: tc.block} + var logs logBuffer + d := &Daemon{ + Cfg: config.NewHolder(config.Config{Repos: repos}), + Forge: fg, + Log: slog.New(slog.NewTextHandler(&logs, nil)), + } + + ctx := context.Background() + if tc.block { + // Expire the sweep's budget without waiting out sweepTimeout. + deadline, cancel := context.WithTimeout(ctx, 50*time.Millisecond) + defer cancel() + ctx = deadline + } + + d.sweepAccess(ctx) + + if got := fg.callCount(); got != tc.wantCalls { + t.Errorf("VerifyPush calls = %d, want %d", got, tc.wantCalls) + } + if got := strings.Count(logs.String(), tc.wantLog); got != tc.wantLogCount { + t.Errorf("%q logged %d times, want %d\nlogs:\n%s", tc.wantLog, got, tc.wantLogCount, logs.String()) + } + if strings.Contains(logs.String(), tc.absentLog) { + t.Errorf("logs mention %q, want none\nlogs:\n%s", tc.absentLog, logs.String()) + } + }) + } +} + // mustCoreConn returns the raw NATS connection for tests that drive core-NATS // subscriptions directly, failing the test if the client is not connected. func mustCoreConn(t *testing.T, c *arnats.Client) *natsio.Conn { From b53d5a14c3e702a0faa2f32f7b4b3fd15fee99f0 Mon Sep 17 00:00:00 2001 From: Sam Catlow Date: Sat, 19 Sep 2026 03:10:46 +1000 Subject: [PATCH 3/3] fix(daemon): don't call a cancelled startup a spent sweep budget ctx.Err() is non-nil for a parent cancellation as well as for the sweep budget, so SIGINT/SIGTERM during the boot sweep logged "sweep budget exhausted" with err="context canceled" -- an expected shutdown reported as a failed sweep (Greptile P2 on #881). Attribute on this sweep's own context instead: DeadlineExceeded is the budget, any other non-nil context error means the startup was cancelled, and neither is a repo access failure. A forge's own request timeout, with the sweep context still live, still reports per repo. RED before (cancelled parent, three repos): budget warnings = 1, want 0 level=WARN msg="push check abandoned: sweep budget exhausted" repo=acme/one err="context canceled" --- internal/daemon/daemon.go | 12 ++++--- internal/daemon/daemon_test.go | 65 +++++++++++++++++++--------------- 2 files changed, 44 insertions(+), 33 deletions(-) diff --git a/internal/daemon/daemon.go b/internal/daemon/daemon.go index 5825e9cb2..a0fe7d259 100644 --- a/internal/daemon/daemon.go +++ b/internal/daemon/daemon.go @@ -296,11 +296,15 @@ func (d *Daemon) sweepAccess(ctx context.Context) { } for _, r := range repos { if err := fg.VerifyPush(ctx, r.Owner, r.Name); err != nil { - if ctx.Err() != nil { - // The budget failed, not the repo: every later call would return - // the same error, so name the real cause once and stop. + // A sweep cut short by its own context is not a repo failure: name + // the budget when it expired, stay quiet on a shutdown. + switch { + case errors.Is(ctx.Err(), context.DeadlineExceeded): + // Every later call would return the same error, so say it once. log.Warn("push check abandoned: sweep budget exhausted", "repo", r.FullName(), "err", err) - break + return + case ctx.Err() != nil: + return } log.Warn("repo not pushable -- tasks from it will fail", "repo", r.FullName(), "err", err) } diff --git a/internal/daemon/daemon_test.go b/internal/daemon/daemon_test.go index c65741e46..8b056ffbb 100644 --- a/internal/daemon/daemon_test.go +++ b/internal/daemon/daemon_test.go @@ -1578,45 +1578,47 @@ func (f *verifyForge) callCount() int { return f.calls } -// TestSweepNamesAnExhaustedBudgetAsTheCause pins what the boot sweep reports -// when a push check does not come back. One budget covers a forge's whole -// sweep, so a repo cut short by the deadline used to warn that it was "not -// pushable" -- asserting access had been denied when nothing about access had -// been learned, once per remaining repo. -func TestSweepNamesAnExhaustedBudgetAsTheCause(t *testing.T) { +// TestSweepNamesTheCauseOfAnAbandonedPushCheck pins what the boot sweep +// reports when a push check does not come back. One budget covers a forge's +// whole sweep, so a repo cut short by the deadline used to warn that it was +// "not pushable" -- asserting access had been denied when nothing about +// access had been learned, once per remaining repo. An operator shutdown +// cancels the same context, and that is neither a spent budget nor a denial. +func TestSweepNamesTheCauseOfAnAbandonedPushCheck(t *testing.T) { repos := []config.Repo{ {Owner: "acme", Name: "one"}, {Owner: "acme", Name: "two"}, {Owner: "acme", Name: "three"}, } tests := []struct { - name string - block bool - wantLog string - wantLogCount int - absentLog string - wantCalls int + name string + mode string + wantDenied int + wantBudget int + wantCalls int }{ { - name: "a denied push is reported per repo", - wantLog: "repo not pushable", - wantLogCount: len(repos), - absentLog: "sweep budget exhausted", - wantCalls: len(repos), + name: "a denied push is reported per repo", + mode: "deny", + wantDenied: len(repos), + wantCalls: len(repos), }, { - name: "an exhausted budget is reported once, as the cause", - block: true, - wantLog: "sweep budget exhausted", - wantLogCount: 1, - absentLog: "repo not pushable", - wantCalls: 1, + name: "an exhausted budget is reported once, as the cause", + mode: "deadline", + wantBudget: 1, + wantCalls: 1, + }, + { + name: "a cancelled startup is reported as neither", + mode: "cancel", + wantCalls: 1, }, } for _, tc := range tests { t.Run(tc.name, func(t *testing.T) { - fg := &verifyForge{block: tc.block} + fg := &verifyForge{block: tc.mode != "deny"} var logs logBuffer d := &Daemon{ Cfg: config.NewHolder(config.Config{Repos: repos}), @@ -1625,11 +1627,16 @@ func TestSweepNamesAnExhaustedBudgetAsTheCause(t *testing.T) { } ctx := context.Background() - if tc.block { + switch tc.mode { + case "deadline": // Expire the sweep's budget without waiting out sweepTimeout. deadline, cancel := context.WithTimeout(ctx, 50*time.Millisecond) defer cancel() ctx = deadline + case "cancel": + cancelled, cancel := context.WithCancel(ctx) + cancel() + ctx = cancelled } d.sweepAccess(ctx) @@ -1637,11 +1644,11 @@ func TestSweepNamesAnExhaustedBudgetAsTheCause(t *testing.T) { if got := fg.callCount(); got != tc.wantCalls { t.Errorf("VerifyPush calls = %d, want %d", got, tc.wantCalls) } - if got := strings.Count(logs.String(), tc.wantLog); got != tc.wantLogCount { - t.Errorf("%q logged %d times, want %d\nlogs:\n%s", tc.wantLog, got, tc.wantLogCount, logs.String()) + if got := strings.Count(logs.String(), "repo not pushable"); got != tc.wantDenied { + t.Errorf("denied-push warnings = %d, want %d\nlogs:\n%s", got, tc.wantDenied, logs.String()) } - if strings.Contains(logs.String(), tc.absentLog) { - t.Errorf("logs mention %q, want none\nlogs:\n%s", tc.absentLog, logs.String()) + if got := strings.Count(logs.String(), "sweep budget exhausted"); got != tc.wantBudget { + t.Errorf("budget warnings = %d, want %d\nlogs:\n%s", got, tc.wantBudget, logs.String()) } }) }