fix(daemon): don't call a cancelled startup a spent sweep budget
ctx.Err() is non-nil for a parent cancellation as well as for the sweep budget, so SIGINT/SIGTERM during the boot sweep logged "sweep budget exhausted" with err="context canceled" -- an expected shutdown reported as a failed sweep (Greptile P2 on #881). Attribute on this sweep's own context instead: DeadlineExceeded is the budget, any other non-nil context error means the startup was cancelled, and neither is a repo access failure. A forge's own request timeout, with the sweep context still live, still reports per repo. RED before (cancelled parent, three repos): budget warnings = 1, want 0 level=WARN msg="push check abandoned: sweep budget exhausted" repo=acme/one err="context canceled"
This commit is contained in:
1 parent
9bdf999b85
commit
becc2fc698
2 files changed
+44
-33
No files matched your search
@@ -296,11 +296,15 @@ func (d *Daemon) sweepAccess(ctx context.Context) {
|
||||
}
|
||||
for _, r := range repos {
|
||||
if err := fg.VerifyPush(ctx, r.Owner, r.Name); err != nil {
|
||||
if ctx.Err() != nil {
|
||||
// The budget failed, not the repo: every later call would return
|
||||
// the same error, so name the real cause once and stop.
|
||||
// A sweep cut short by its own context is not a repo failure: name
|
||||
// the budget when it expired, stay quiet on a shutdown.
|
||||
switch {
|
||||
case errors.Is(ctx.Err(), context.DeadlineExceeded):
|
||||
// Every later call would return the same error, so say it once.
|
||||
log.Warn("push check abandoned: sweep budget exhausted", "repo", r.FullName(), "err", err)
|
||||
break
|
||||
return
|
||||
case ctx.Err() != nil:
|
||||
return
|
||||
}
|
||||
log.Warn("repo not pushable -- tasks from it will fail", "repo", r.FullName(), "err", err)
|
||||
}
|
||||
|
||||
@@ -1578,45 +1578,47 @@ func (f *verifyForge) callCount() int {
|
||||
return f.calls
|
||||
}
|
||||
|
||||
// TestSweepNamesAnExhaustedBudgetAsTheCause pins what the boot sweep reports
|
||||
// when a push check does not come back. One budget covers a forge's whole
|
||||
// sweep, so a repo cut short by the deadline used to warn that it was "not
|
||||
// pushable" -- asserting access had been denied when nothing about access had
|
||||
// been learned, once per remaining repo.
|
||||
func TestSweepNamesAnExhaustedBudgetAsTheCause(t *testing.T) {
|
||||
// TestSweepNamesTheCauseOfAnAbandonedPushCheck pins what the boot sweep
|
||||
// reports when a push check does not come back. One budget covers a forge's
|
||||
// whole sweep, so a repo cut short by the deadline used to warn that it was
|
||||
// "not pushable" -- asserting access had been denied when nothing about
|
||||
// access had been learned, once per remaining repo. An operator shutdown
|
||||
// cancels the same context, and that is neither a spent budget nor a denial.
|
||||
func TestSweepNamesTheCauseOfAnAbandonedPushCheck(t *testing.T) {
|
||||
repos := []config.Repo{
|
||||
{Owner: "acme", Name: "one"},
|
||||
{Owner: "acme", Name: "two"},
|
||||
{Owner: "acme", Name: "three"},
|
||||
}
|
||||
tests := []struct {
|
||||
name string
|
||||
block bool
|
||||
wantLog string
|
||||
wantLogCount int
|
||||
absentLog string
|
||||
wantCalls int
|
||||
name string
|
||||
mode string
|
||||
wantDenied int
|
||||
wantBudget int
|
||||
wantCalls int
|
||||
}{
|
||||
{
|
||||
name: "a denied push is reported per repo",
|
||||
wantLog: "repo not pushable",
|
||||
wantLogCount: len(repos),
|
||||
absentLog: "sweep budget exhausted",
|
||||
wantCalls: len(repos),
|
||||
name: "a denied push is reported per repo",
|
||||
mode: "deny",
|
||||
wantDenied: len(repos),
|
||||
wantCalls: len(repos),
|
||||
},
|
||||
{
|
||||
name: "an exhausted budget is reported once, as the cause",
|
||||
block: true,
|
||||
wantLog: "sweep budget exhausted",
|
||||
wantLogCount: 1,
|
||||
absentLog: "repo not pushable",
|
||||
wantCalls: 1,
|
||||
name: "an exhausted budget is reported once, as the cause",
|
||||
mode: "deadline",
|
||||
wantBudget: 1,
|
||||
wantCalls: 1,
|
||||
},
|
||||
{
|
||||
name: "a cancelled startup is reported as neither",
|
||||
mode: "cancel",
|
||||
wantCalls: 1,
|
||||
},
|
||||
}
|
||||
|
||||
for _, tc := range tests {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
fg := &verifyForge{block: tc.block}
|
||||
fg := &verifyForge{block: tc.mode != "deny"}
|
||||
var logs logBuffer
|
||||
d := &Daemon{
|
||||
Cfg: config.NewHolder(config.Config{Repos: repos}),
|
||||
@@ -1625,11 +1627,16 @@ func TestSweepNamesAnExhaustedBudgetAsTheCause(t *testing.T) {
|
||||
}
|
||||
|
||||
ctx := context.Background()
|
||||
if tc.block {
|
||||
switch tc.mode {
|
||||
case "deadline":
|
||||
// Expire the sweep's budget without waiting out sweepTimeout.
|
||||
deadline, cancel := context.WithTimeout(ctx, 50*time.Millisecond)
|
||||
defer cancel()
|
||||
ctx = deadline
|
||||
case "cancel":
|
||||
cancelled, cancel := context.WithCancel(ctx)
|
||||
cancel()
|
||||
ctx = cancelled
|
||||
}
|
||||
|
||||
d.sweepAccess(ctx)
|
||||
@@ -1637,11 +1644,11 @@ func TestSweepNamesAnExhaustedBudgetAsTheCause(t *testing.T) {
|
||||
if got := fg.callCount(); got != tc.wantCalls {
|
||||
t.Errorf("VerifyPush calls = %d, want %d", got, tc.wantCalls)
|
||||
}
|
||||
if got := strings.Count(logs.String(), tc.wantLog); got != tc.wantLogCount {
|
||||
t.Errorf("%q logged %d times, want %d\nlogs:\n%s", tc.wantLog, got, tc.wantLogCount, logs.String())
|
||||
if got := strings.Count(logs.String(), "repo not pushable"); got != tc.wantDenied {
|
||||
t.Errorf("denied-push warnings = %d, want %d\nlogs:\n%s", got, tc.wantDenied, logs.String())
|
||||
}
|
||||
if strings.Contains(logs.String(), tc.absentLog) {
|
||||
t.Errorf("logs mention %q, want none\nlogs:\n%s", tc.absentLog, logs.String())
|
||||
if got := strings.Count(logs.String(), "sweep budget exhausted"); got != tc.wantBudget {
|
||||
t.Errorf("budget warnings = %d, want %d\nlogs:\n%s", got, tc.wantBudget, logs.String())
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
Reference in new issue
Block a user