- one shared UpdateStateDir() resolver for daemon and dashboard, instead of two copies that could drift - the recovery path for a committed/rolled-back journal writes installed.json too, so a crash after the outcome phase cannot lose the install record - JudgeCanaryNow is the explicit signals-only entry point for promoting without the window, replacing the zero-install-time sentinel - canary version comparisons use IssuedAt, matching Promote's replay rule, so semver/issued-at divergence cannot wedge check red; queuedBehind reuses the judgeable predicate instead of a looser check - promote prints the channel/pin; promote.yml cancels superseded runs - canary card: no duplicated elapsed rendering, channel row waits for the settings resource instead of showing a default, agent fallback '?' - rehearsal script bounds its docker build/push/run calls task check passes on this tree.
90 lines
4.6 KiB
Go
90 lines
4.6 KiB
Go
package releaseupdate
|
|
|
|
import (
|
|
"testing"
|
|
"time"
|
|
)
|
|
|
|
// A release promotes only after a full window counted from its install with
|
|
// no hard failure and no relative signal past its threshold; any failure after
|
|
// settling rolls it back at once, even mid-window.
|
|
func TestJudgeCanary(t *testing.T) {
|
|
installed := time.Date(2026, 1, 2, 12, 0, 0, 0, time.UTC)
|
|
healthy := func() CanarySignals {
|
|
return CanarySignals{
|
|
Installed: "1.1.0", Ready: true,
|
|
Baseline: CanaryWindowSignals{Finished: 20, Bad: 2, Errors: 10, Covered: CanaryWindow},
|
|
Canary: CanaryWindowSignals{Finished: 20, Bad: 2, Errors: 12, Covered: CanaryWindow},
|
|
}
|
|
}
|
|
with := func(change func(*CanarySignals)) CanarySignals {
|
|
s := healthy()
|
|
change(&s)
|
|
return s
|
|
}
|
|
cases := []struct {
|
|
name string
|
|
elapsed time.Duration // now minus the install time
|
|
window time.Duration
|
|
signals CanarySignals
|
|
want CanaryDecision
|
|
}{
|
|
{"settling counts from the install", 10 * time.Minute, CanaryWindow, with(func(s *CanarySignals) { s.Ready = false }), CanaryWait},
|
|
{"healthy mid-window waits", 2 * time.Hour, CanaryWindow, healthy(), CanaryWait},
|
|
{"healthy at the window promotes", CanaryWindow, CanaryWindow, healthy(), CanaryPromote},
|
|
{"healthy at a custom window promotes", 12 * time.Hour, 12 * time.Hour, healthy(), CanaryPromote},
|
|
{"the watchdog rolled the install back", 2 * time.Hour, CanaryWindow, with(func(s *CanarySignals) { s.Installed = "1.0.0" }), CanaryRollBack},
|
|
{"readiness degraded", 2 * time.Hour, CanaryWindow, with(func(s *CanarySignals) { s.Ready = false }), CanaryRollBack},
|
|
{"a new action alert", 2 * time.Hour, CanaryWindow, with(func(s *CanarySignals) { s.NewActionAlerts = []string{"queue stalled"} }), CanaryRollBack},
|
|
{"errors past twice the baseline plus ten", CanaryWindow, CanaryWindow, with(func(s *CanarySignals) { s.Canary.Errors = 31 }), CanaryRollBack},
|
|
{"errors at the limit", CanaryWindow, CanaryWindow, with(func(s *CanarySignals) { s.Canary.Errors = 30 }), CanaryPromote},
|
|
{"errors scaled from a partly covered log", CanaryWindow, CanaryWindow, with(func(s *CanarySignals) {
|
|
s.Canary.Errors, s.Canary.Covered = 8, 6*time.Hour
|
|
}), CanaryRollBack},
|
|
{"a baseline the log doesn't reach skips errors", CanaryWindow, CanaryWindow, with(func(s *CanarySignals) {
|
|
s.Baseline.Covered, s.Canary.Errors = 30*time.Minute, 500
|
|
}), CanaryPromote},
|
|
{"a mid-window error breach rolls back early", 2 * time.Hour, CanaryWindow, with(func(s *CanarySignals) { s.Canary.Errors = 31 }), CanaryRollBack},
|
|
{"park rate up more than ten points", CanaryWindow, CanaryWindow, with(func(s *CanarySignals) { s.Canary.Bad = 5 }), CanaryRollBack},
|
|
{"park rate up ten points", CanaryWindow, CanaryWindow, with(func(s *CanarySignals) { s.Canary.Bad = 4 }), CanaryPromote},
|
|
{"too few canary tasks skips the park rate", CanaryWindow, CanaryWindow, with(func(s *CanarySignals) { s.Canary = CanaryWindowSignals{Finished: 4, Bad: 4, Covered: CanaryWindow} }), CanaryPromote},
|
|
{"a thin baseline skips the park rate", CanaryWindow, CanaryWindow, with(func(s *CanarySignals) {
|
|
s.Baseline.Finished, s.Baseline.Bad = 2, 0
|
|
s.Canary.Bad = 5
|
|
}), CanaryPromote},
|
|
{"a mid-window park breach rolls back early", 2 * time.Hour, CanaryWindow, with(func(s *CanarySignals) { s.Canary.Bad = 5 }), CanaryRollBack},
|
|
}
|
|
for _, tc := range cases {
|
|
t.Run(tc.name, func(t *testing.T) {
|
|
got := JudgeCanary("1.1.0", installed, installed.Add(tc.elapsed), tc.window, tc.signals)
|
|
if got.Decision != tc.want {
|
|
t.Fatalf("decision = %s, want %s (%v)", got.Decision, tc.want, got.Reasons)
|
|
}
|
|
})
|
|
}
|
|
}
|
|
|
|
// Judging "now" skips the settle and window waits but still applies every
|
|
// hard and relative check.
|
|
func TestJudgeCanaryNow(t *testing.T) {
|
|
healthy := CanarySignals{
|
|
Installed: "1.1.0", Ready: true,
|
|
Baseline: CanaryWindowSignals{Finished: 20, Bad: 2, Errors: 10, Covered: CanaryWindow},
|
|
Canary: CanaryWindowSignals{Finished: 20, Bad: 2, Errors: 12, Covered: CanaryWindow},
|
|
}
|
|
now := time.Date(2026, 1, 2, 12, 10, 0, 0, time.UTC)
|
|
if got := JudgeCanaryNow("1.1.0", now, CanaryWindow, healthy); got.Decision != CanaryPromote {
|
|
t.Fatalf("healthy now = %s, want promote (%v)", got.Decision, got.Reasons)
|
|
}
|
|
degraded := healthy
|
|
degraded.Ready = false
|
|
if got := JudgeCanaryNow("1.1.0", now, CanaryWindow, degraded); got.Decision != CanaryRollBack {
|
|
t.Fatalf("degraded now = %s, want roll back (%v)", got.Decision, got.Reasons)
|
|
}
|
|
breached := healthy
|
|
breached.Canary.Bad = 10
|
|
if got := JudgeCanaryNow("1.1.0", now, CanaryWindow, breached); got.Decision != CanaryRollBack {
|
|
t.Fatalf("park breach now = %s, want roll back (%v)", got.Decision, got.Reasons)
|
|
}
|
|
}
|