Files
archie/internal/releaseupdate/canary_test.go
T
sam 5356745968
Quality / check (push) Successful in 3m39s
deploy / gate (push) Successful in 4m28s
deploy / archied-image (push) Successful in 3m11s
deploy / agent-image (push) Successful in 11s
deploy / dist-zip (push) Skipped
fix(releaseupdate): address review findings on the #1725 canary change
- one shared UpdateStateDir() resolver for daemon and dashboard, instead of
  two copies that could drift
- the recovery path for a committed/rolled-back journal writes
  installed.json too, so a crash after the outcome phase cannot lose the
  install record
- JudgeCanaryNow is the explicit signals-only entry point for promoting
  without the window, replacing the zero-install-time sentinel
- canary version comparisons use IssuedAt, matching Promote's replay rule,
  so semver/issued-at divergence cannot wedge check red; queuedBehind
  reuses the judgeable predicate instead of a looser check
- promote prints the channel/pin; promote.yml cancels superseded runs
- canary card: no duplicated elapsed rendering, channel row waits for the
  settings resource instead of showing a default, agent fallback '?'
- rehearsal script bounds its docker build/push/run calls

task check passes on this tree.
2026-11-02 02:45:26 +11:00

90 lines
4.6 KiB
Go

package releaseupdate
import (
"testing"
"time"
)
// A release promotes only after a full window counted from its install with
// no hard failure and no relative signal past its threshold; any failure after
// settling rolls it back at once, even mid-window.
func TestJudgeCanary(t *testing.T) {
installed := time.Date(2026, 1, 2, 12, 0, 0, 0, time.UTC)
healthy := func() CanarySignals {
return CanarySignals{
Installed: "1.1.0", Ready: true,
Baseline: CanaryWindowSignals{Finished: 20, Bad: 2, Errors: 10, Covered: CanaryWindow},
Canary: CanaryWindowSignals{Finished: 20, Bad: 2, Errors: 12, Covered: CanaryWindow},
}
}
with := func(change func(*CanarySignals)) CanarySignals {
s := healthy()
change(&s)
return s
}
cases := []struct {
name string
elapsed time.Duration // now minus the install time
window time.Duration
signals CanarySignals
want CanaryDecision
}{
{"settling counts from the install", 10 * time.Minute, CanaryWindow, with(func(s *CanarySignals) { s.Ready = false }), CanaryWait},
{"healthy mid-window waits", 2 * time.Hour, CanaryWindow, healthy(), CanaryWait},
{"healthy at the window promotes", CanaryWindow, CanaryWindow, healthy(), CanaryPromote},
{"healthy at a custom window promotes", 12 * time.Hour, 12 * time.Hour, healthy(), CanaryPromote},
{"the watchdog rolled the install back", 2 * time.Hour, CanaryWindow, with(func(s *CanarySignals) { s.Installed = "1.0.0" }), CanaryRollBack},
{"readiness degraded", 2 * time.Hour, CanaryWindow, with(func(s *CanarySignals) { s.Ready = false }), CanaryRollBack},
{"a new action alert", 2 * time.Hour, CanaryWindow, with(func(s *CanarySignals) { s.NewActionAlerts = []string{"queue stalled"} }), CanaryRollBack},
{"errors past twice the baseline plus ten", CanaryWindow, CanaryWindow, with(func(s *CanarySignals) { s.Canary.Errors = 31 }), CanaryRollBack},
{"errors at the limit", CanaryWindow, CanaryWindow, with(func(s *CanarySignals) { s.Canary.Errors = 30 }), CanaryPromote},
{"errors scaled from a partly covered log", CanaryWindow, CanaryWindow, with(func(s *CanarySignals) {
s.Canary.Errors, s.Canary.Covered = 8, 6*time.Hour
}), CanaryRollBack},
{"a baseline the log doesn't reach skips errors", CanaryWindow, CanaryWindow, with(func(s *CanarySignals) {
s.Baseline.Covered, s.Canary.Errors = 30*time.Minute, 500
}), CanaryPromote},
{"a mid-window error breach rolls back early", 2 * time.Hour, CanaryWindow, with(func(s *CanarySignals) { s.Canary.Errors = 31 }), CanaryRollBack},
{"park rate up more than ten points", CanaryWindow, CanaryWindow, with(func(s *CanarySignals) { s.Canary.Bad = 5 }), CanaryRollBack},
{"park rate up ten points", CanaryWindow, CanaryWindow, with(func(s *CanarySignals) { s.Canary.Bad = 4 }), CanaryPromote},
{"too few canary tasks skips the park rate", CanaryWindow, CanaryWindow, with(func(s *CanarySignals) { s.Canary = CanaryWindowSignals{Finished: 4, Bad: 4, Covered: CanaryWindow} }), CanaryPromote},
{"a thin baseline skips the park rate", CanaryWindow, CanaryWindow, with(func(s *CanarySignals) {
s.Baseline.Finished, s.Baseline.Bad = 2, 0
s.Canary.Bad = 5
}), CanaryPromote},
{"a mid-window park breach rolls back early", 2 * time.Hour, CanaryWindow, with(func(s *CanarySignals) { s.Canary.Bad = 5 }), CanaryRollBack},
}
for _, tc := range cases {
t.Run(tc.name, func(t *testing.T) {
got := JudgeCanary("1.1.0", installed, installed.Add(tc.elapsed), tc.window, tc.signals)
if got.Decision != tc.want {
t.Fatalf("decision = %s, want %s (%v)", got.Decision, tc.want, got.Reasons)
}
})
}
}
// Judging "now" skips the settle and window waits but still applies every
// hard and relative check.
func TestJudgeCanaryNow(t *testing.T) {
healthy := CanarySignals{
Installed: "1.1.0", Ready: true,
Baseline: CanaryWindowSignals{Finished: 20, Bad: 2, Errors: 10, Covered: CanaryWindow},
Canary: CanaryWindowSignals{Finished: 20, Bad: 2, Errors: 12, Covered: CanaryWindow},
}
now := time.Date(2026, 1, 2, 12, 10, 0, 0, time.UTC)
if got := JudgeCanaryNow("1.1.0", now, CanaryWindow, healthy); got.Decision != CanaryPromote {
t.Fatalf("healthy now = %s, want promote (%v)", got.Decision, got.Reasons)
}
degraded := healthy
degraded.Ready = false
if got := JudgeCanaryNow("1.1.0", now, CanaryWindow, degraded); got.Decision != CanaryRollBack {
t.Fatalf("degraded now = %s, want roll back (%v)", got.Decision, got.Reasons)
}
breached := healthy
breached.Canary.Bad = 10
if got := JudgeCanaryNow("1.1.0", now, CanaryWindow, breached); got.Decision != CanaryRollBack {
t.Fatalf("park breach now = %s, want roll back (%v)", got.Decision, got.Reasons)
}
}