fix(daemon): name an exhausted sweep budget as the cause
sweepAccess spends one sweepTimeout budget on a forge's whole boot
sweep. When that budget ran out mid-loop, every remaining repo was
warned as "repo not pushable -- tasks from it will fail" -- asserting
that access had been denied when all that was known was that the
deadline passed, once per repo.
Report the budget once, name it as the cause, and stop: later calls
return the same deadline error, so continuing only multiplies a
misleading warning.
RED before (three repos, budget exhausted by the first):
VerifyPush calls = 3, want 1
"sweep budget exhausted" logged 0 times, want 1
level=WARN msg="repo not pushable -- tasks from it will fail" repo=acme/one err="context deadline exceeded"
level=WARN msg="repo not pushable -- tasks from it will fail" repo=acme/two err="context deadline exceeded"
level=WARN msg="repo not pushable -- tasks from it will fail" repo=acme/three err="context deadline exceeded"
This commit is contained in:
1 parent
86bbc468c9
commit
08e16c82be
2 files changed
+104
No files matched your search
@@ -296,6 +296,12 @@ func (d *Daemon) sweepAccess(ctx context.Context) {
|
||||
}
|
||||
for _, r := range repos {
|
||||
if err := fg.VerifyPush(ctx, r.Owner, r.Name); err != nil {
|
||||
if ctx.Err() != nil {
|
||||
// The budget failed, not the repo: every later call would return
|
||||
// the same error, so name the real cause once and stop.
|
||||
log.Warn("push check abandoned: sweep budget exhausted", "repo", r.FullName(), "err", err)
|
||||
break
|
||||
}
|
||||
log.Warn("repo not pushable -- tasks from it will fail", "repo", r.FullName(), "err", err)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -3,6 +3,7 @@ package daemon
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"log/slog"
|
||||
"os"
|
||||
@@ -1549,6 +1550,103 @@ func waitFor(t *testing.T, timeout time.Duration, what string, cond func() bool)
|
||||
t.Fatalf("timed out waiting for %s", what)
|
||||
}
|
||||
|
||||
// verifyForge returns a fixed error from VerifyPush, or blocks until the
|
||||
// sweep's own deadline expires first, standing in for an unreachable host.
|
||||
type verifyForge struct {
|
||||
testForge
|
||||
mu sync.Mutex
|
||||
calls int
|
||||
block bool
|
||||
}
|
||||
|
||||
var errVerifyDenied = errors.New("push access denied")
|
||||
|
||||
func (f *verifyForge) VerifyPush(ctx context.Context, _, _ string) error {
|
||||
f.mu.Lock()
|
||||
f.calls++
|
||||
f.mu.Unlock()
|
||||
if f.block {
|
||||
<-ctx.Done()
|
||||
return ctx.Err()
|
||||
}
|
||||
return errVerifyDenied
|
||||
}
|
||||
|
||||
func (f *verifyForge) callCount() int {
|
||||
f.mu.Lock()
|
||||
defer f.mu.Unlock()
|
||||
return f.calls
|
||||
}
|
||||
|
||||
// TestSweepNamesAnExhaustedBudgetAsTheCause pins what the boot sweep reports
|
||||
// when a push check does not come back. One budget covers a forge's whole
|
||||
// sweep, so a repo cut short by the deadline used to warn that it was "not
|
||||
// pushable" -- asserting access had been denied when nothing about access had
|
||||
// been learned, once per remaining repo.
|
||||
func TestSweepNamesAnExhaustedBudgetAsTheCause(t *testing.T) {
|
||||
repos := []config.Repo{
|
||||
{Owner: "acme", Name: "one"},
|
||||
{Owner: "acme", Name: "two"},
|
||||
{Owner: "acme", Name: "three"},
|
||||
}
|
||||
tests := []struct {
|
||||
name string
|
||||
block bool
|
||||
wantLog string
|
||||
wantLogCount int
|
||||
absentLog string
|
||||
wantCalls int
|
||||
}{
|
||||
{
|
||||
name: "a denied push is reported per repo",
|
||||
wantLog: "repo not pushable",
|
||||
wantLogCount: len(repos),
|
||||
absentLog: "sweep budget exhausted",
|
||||
wantCalls: len(repos),
|
||||
},
|
||||
{
|
||||
name: "an exhausted budget is reported once, as the cause",
|
||||
block: true,
|
||||
wantLog: "sweep budget exhausted",
|
||||
wantLogCount: 1,
|
||||
absentLog: "repo not pushable",
|
||||
wantCalls: 1,
|
||||
},
|
||||
}
|
||||
|
||||
for _, tc := range tests {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
fg := &verifyForge{block: tc.block}
|
||||
var logs logBuffer
|
||||
d := &Daemon{
|
||||
Cfg: config.NewHolder(config.Config{Repos: repos}),
|
||||
Forge: fg,
|
||||
Log: slog.New(slog.NewTextHandler(&logs, nil)),
|
||||
}
|
||||
|
||||
ctx := context.Background()
|
||||
if tc.block {
|
||||
// Expire the sweep's budget without waiting out sweepTimeout.
|
||||
deadline, cancel := context.WithTimeout(ctx, 50*time.Millisecond)
|
||||
defer cancel()
|
||||
ctx = deadline
|
||||
}
|
||||
|
||||
d.sweepAccess(ctx)
|
||||
|
||||
if got := fg.callCount(); got != tc.wantCalls {
|
||||
t.Errorf("VerifyPush calls = %d, want %d", got, tc.wantCalls)
|
||||
}
|
||||
if got := strings.Count(logs.String(), tc.wantLog); got != tc.wantLogCount {
|
||||
t.Errorf("%q logged %d times, want %d\nlogs:\n%s", tc.wantLog, got, tc.wantLogCount, logs.String())
|
||||
}
|
||||
if strings.Contains(logs.String(), tc.absentLog) {
|
||||
t.Errorf("logs mention %q, want none\nlogs:\n%s", tc.absentLog, logs.String())
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// mustCoreConn returns the raw NATS connection for tests that drive core-NATS
|
||||
// subscriptions directly, failing the test if the client is not connected.
|
||||
func mustCoreConn(t *testing.T, c *arnats.Client) *natsio.Conn {
|
||||
|
||||
Reference in new issue
Block a user