diff --git a/internal/daemon/daemon.go b/internal/daemon/daemon.go index 6cfe23d36..a69d19617 100644 --- a/internal/daemon/daemon.go +++ b/internal/daemon/daemon.go @@ -296,6 +296,16 @@ func (d *Daemon) sweepAccess(ctx context.Context) { } for _, r := range repos { if err := fg.VerifyPush(ctx, r.Owner, r.Name); err != nil { + // A sweep cut short by its own context is not a repo failure: name + // the budget when it expired, stay quiet on a shutdown. + switch { + case errors.Is(ctx.Err(), context.DeadlineExceeded): + // Every later call would return the same error, so say it once. + log.Warn("push check abandoned: sweep budget exhausted", "repo", r.FullName(), "err", err) + return + case ctx.Err() != nil: + return + } log.Warn("repo not pushable -- tasks from it will fail", "repo", r.FullName(), "err", err) } } diff --git a/internal/daemon/daemon_test.go b/internal/daemon/daemon_test.go index d2bbbe13d..95d3959af 100644 --- a/internal/daemon/daemon_test.go +++ b/internal/daemon/daemon_test.go @@ -3,6 +3,7 @@ package daemon import ( "context" "encoding/json" + "errors" "fmt" "log/slog" "os" @@ -1466,6 +1467,194 @@ func assertSwept(t *testing.T, name string, fg *sweepForge, wantInvitations int, } } +// stallForge stands in for a forge client whose request never returns: +// AcceptInvitations signals that its sweep has started, then blocks until the +// test releases it or the sweep's own deadline fires. +type stallForge struct { + testForge + entered chan struct{} + release chan struct{} +} + +func (f *stallForge) AcceptInvitations(ctx context.Context) error { + close(f.entered) + select { + case <-f.release: + return nil + case <-ctx.Done(): + return ctx.Err() + } +} + +// TestStartupSweepIsolatesAStalledForge pins the concurrency half of the +// stalled-forge fix. Sweeping forges one at a time on the startup goroutine +// left every later identity unswept when one forge request never returned, +// and Startup gates the gateways and the run loop, so it held boot open too. +// Each forge now gets its own goroutine; this asserts the ordering that +// proves it: the sibling identity's sweep completes while the root forge is +// still blocked. +func TestStartupSweepIsolatesAStalledForge(t *testing.T) { + rootFg := &stallForge{entered: make(chan struct{}), release: make(chan struct{})} + siblingFg := &sweepForge{} + d := &Daemon{ + Cfg: config.NewHolder(config.Config{}), + Store: store.OpenTest(t), + Forge: rootFg, + Log: slog.New(slog.DiscardHandler), + Identities: []*IdentityRunner{{Name: "sibling", Forge: siblingFg}}, + } + + errs := make(chan error, 1) + done := make(chan struct{}) + go func() { + defer close(done) + errs <- d.Startup(context.Background()) + }() + + waitFor(t, 5*time.Second, "the stalled root forge to start its sweep", func() bool { + select { + case <-rootFg.entered: + return true + default: + return false + } + }) + waitFor(t, 5*time.Second, "the sibling identity's sweep to finish", func() bool { + invitations, _ := siblingFg.snapshot() + return invitations == 1 + }) + + close(rootFg.release) + waitFor(t, 5*time.Second, "Startup to return once the stall is released", func() bool { + select { + case <-done: + return true + default: + return false + } + }) + if err := <-errs; err != nil { + t.Errorf("Startup: %v", err) + } +} + +// waitFor polls cond until it holds or the timeout expires. +func waitFor(t *testing.T, timeout time.Duration, what string, cond func() bool) { + t.Helper() + deadline := time.Now().Add(timeout) + for time.Now().Before(deadline) { + if cond() { + return + } + time.Sleep(time.Millisecond) + } + t.Fatalf("timed out waiting for %s", what) +} + +// verifyForge returns a fixed error from VerifyPush, or blocks until the +// sweep's own deadline expires first, standing in for an unreachable host. +type verifyForge struct { + testForge + mu sync.Mutex + calls int + block bool +} + +var errVerifyDenied = errors.New("push access denied") + +func (f *verifyForge) VerifyPush(ctx context.Context, _, _ string) error { + f.mu.Lock() + f.calls++ + f.mu.Unlock() + if f.block { + <-ctx.Done() + return ctx.Err() + } + return errVerifyDenied +} + +func (f *verifyForge) callCount() int { + f.mu.Lock() + defer f.mu.Unlock() + return f.calls +} + +// TestSweepNamesTheCauseOfAnAbandonedPushCheck pins what the boot sweep +// reports when a push check does not come back. One budget covers a forge's +// whole sweep, so a repo cut short by the deadline used to warn that it was +// "not pushable" -- asserting access had been denied when nothing about +// access had been learned, once per remaining repo. An operator shutdown +// cancels the same context, and that is neither a spent budget nor a denial. +func TestSweepNamesTheCauseOfAnAbandonedPushCheck(t *testing.T) { + repos := []config.Repo{ + {Owner: "acme", Name: "one"}, + {Owner: "acme", Name: "two"}, + {Owner: "acme", Name: "three"}, + } + tests := []struct { + name string + mode string + wantDenied int + wantBudget int + wantCalls int + }{ + { + name: "a denied push is reported per repo", + mode: "deny", + wantDenied: len(repos), + wantCalls: len(repos), + }, + { + name: "an exhausted budget is reported once, as the cause", + mode: "deadline", + wantBudget: 1, + wantCalls: 1, + }, + { + name: "a cancelled startup is reported as neither", + mode: "cancel", + wantCalls: 1, + }, + } + + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + fg := &verifyForge{block: tc.mode != "deny"} + var logs logBuffer + d := &Daemon{ + Cfg: config.NewHolder(config.Config{Repos: repos}), + Forge: fg, + Log: slog.New(slog.NewTextHandler(&logs, nil)), + } + + ctx := context.Background() + switch tc.mode { + case "deadline": + // Expire the sweep's budget without waiting out sweepTimeout. + deadline, cancel := context.WithTimeout(ctx, 50*time.Millisecond) + defer cancel() + ctx = deadline + case "cancel": + cancelled, cancel := context.WithCancel(ctx) + cancel() + ctx = cancelled + } + + d.sweepAccess(ctx) + + if got := fg.callCount(); got != tc.wantCalls { + t.Errorf("VerifyPush calls = %d, want %d", got, tc.wantCalls) + } + if got := strings.Count(logs.String(), "repo not pushable"); got != tc.wantDenied { + t.Errorf("denied-push warnings = %d, want %d\nlogs:\n%s", got, tc.wantDenied, logs.String()) + } + if got := strings.Count(logs.String(), "sweep budget exhausted"); got != tc.wantBudget { + t.Errorf("budget warnings = %d, want %d\nlogs:\n%s", got, tc.wantBudget, logs.String()) + } + }) + } +} + // mustCoreConn returns the raw NATS connection for tests that drive core-NATS // subscriptions directly, failing the test if the client is not connected. func mustCoreConn(t *testing.T, c *arnats.Client) *natsio.Conn {