Files
archie/internal/releaseupdate/canary.go
T
sam 5356745968
Quality / check (push) Successful in 3m39s
deploy / gate (push) Successful in 4m28s
deploy / archied-image (push) Successful in 3m11s
deploy / agent-image (push) Successful in 11s
deploy / dist-zip (push) Skipped
fix(releaseupdate): address review findings on the #1725 canary change
- one shared UpdateStateDir() resolver for daemon and dashboard, instead of
  two copies that could drift
- the recovery path for a committed/rolled-back journal writes
  installed.json too, so a crash after the outcome phase cannot lose the
  install record
- JudgeCanaryNow is the explicit signals-only entry point for promoting
  without the window, replacing the zero-install-time sentinel
- canary version comparisons use IssuedAt, matching Promote's replay rule,
  so semver/issued-at divergence cannot wedge check red; queuedBehind
  reuses the judgeable predicate instead of a looser check
- promote prints the channel/pin; promote.yml cancels superseded runs
- canary card: no duplicated elapsed rendering, channel row waits for the
  settings resource instead of showing a default, agent fallback '?'
- rehearsal script bounds its docker build/push/run calls

task check passes on this tree.
2026-11-02 02:45:26 +11:00

153 lines
5.8 KiB
Go

package releaseupdate
import (
"fmt"
"time"
)
// The canary host installs each release from the next channel. Its behaviour
// in the window after a release was installed is compared with its own
// behaviour in the window before, because there's only one production host to
// compare with.
const (
// CanaryWindow is the default window a release runs on the canary before
// it can be promoted. A day covers one full cycle of schedules and
// traffic.
CanaryWindow = 24 * time.Hour
// canarySettle is how long the install and restart get, counted from when
// the release was installed on the canary, before any check can fail the
// release.
canarySettle = 30 * time.Minute
// minFinished is the fewest finished tasks needed to judge the park rate.
minFinished = 5
// minLogCoverage is how far back into the baseline window the error log
// must reach before error counts are compared.
minLogCoverage = time.Hour
)
// CanaryDecision is what the evaluation says to do with a release.
type CanaryDecision string
const (
CanaryWait CanaryDecision = "wait"
CanaryPromote CanaryDecision = "promote"
CanaryRollBack CanaryDecision = "roll back"
)
// CanaryWindowSignals is one window's task and error-log counts.
type CanaryWindowSignals struct {
// Finished counts tasks that reached a terminal or review status; Bad
// is those that parked or died.
Finished, Bad int
// Errors counts error-level log lines within Covered, the part of the
// window that the log still reaches.
Errors int
Covered time.Duration
}
// CanarySignals is what the canary reported at one check.
type CanarySignals struct {
Installed string
Ready bool
// NewActionAlerts are open action-severity alerts first raised since
// the release was installed.
NewActionAlerts []string
Baseline CanaryWindowSignals
Canary CanaryWindowSignals
}
// CanaryVerdict is a decision and the signals that decided it.
type CanaryVerdict struct {
Decision CanaryDecision
Reasons []string
}
// JudgeCanaryNow decides from the current signals alone, for an operator
// promoting without waiting out the window: the settle and window waits are
// skipped by construction, every hard and relative check still applies.
func JudgeCanaryNow(version string, now time.Time, window time.Duration, s CanarySignals) CanaryVerdict {
return JudgeCanary(version, now.Add(-(window + canarySettle)), now, window, s)
}
// JudgeCanary decides what to do with a release, given when it was installed
// on the canary host. Once the release has settled, a failed hard check rolls
// it back at any time. Error and park rates are judged against the baseline
// whenever both windows carry enough data, so a breach rolls the release back
// even mid-window; a full window with no failure promotes it.
func JudgeCanary(version string, installedAt, now time.Time, window time.Duration, s CanarySignals) CanaryVerdict {
elapsed := now.Sub(installedAt)
if elapsed < canarySettle {
return CanaryVerdict{Decision: CanaryWait, Reasons: []string{"settling after install"}}
}
var failed []string
if s.Installed != version {
failed = append(failed, fmt.Sprintf("canary runs %q, not %s: the install failed or was rolled back", s.Installed, version))
}
if !s.Ready {
failed = append(failed, "readiness is degraded")
}
for _, alert := range s.NewActionAlerts {
failed = append(failed, "new action alert: "+alert)
}
if len(failed) > 0 {
return CanaryVerdict{Decision: CanaryRollBack, Reasons: failed}
}
var notes []string
if reason, ok := judgeErrors(s.Baseline, s.Canary, window); !ok {
failed = append(failed, reason)
} else {
notes = append(notes, reason)
}
if reason, ok := judgeParks(s.Baseline, s.Canary); !ok {
failed = append(failed, reason)
} else {
notes = append(notes, reason)
}
if len(failed) > 0 {
return CanaryVerdict{Decision: CanaryRollBack, Reasons: failed}
}
if elapsed < window {
reasons := append([]string{fmt.Sprintf("%s of %s elapsed since the install", elapsed.Round(time.Minute), window)}, notes...)
return CanaryVerdict{Decision: CanaryWait, Reasons: reasons}
}
return CanaryVerdict{Decision: CanaryPromote, Reasons: notes}
}
// judgeErrors fails when the canary logged more than twice the baseline's
// error lines plus ten. Each count is scaled up to a full window when the log
// covers only part of its window.
func judgeErrors(baseline, canary CanaryWindowSignals, window time.Duration) (string, bool) {
if baseline.Covered < minLogCoverage || canary.Covered <= 0 {
return "error log doesn't reach back over the baseline; error signal skipped", true
}
scale := func(w CanaryWindowSignals) float64 {
return float64(w.Errors) * float64(window) / float64(w.Covered)
}
before, after := scale(baseline), scale(canary)
limit := 2*before + 10
if after > limit {
return fmt.Sprintf("%.0f error log lines per window against a limit of %.0f (baseline %.0f)", after, limit, before), false
}
return fmt.Sprintf("%.0f error log lines per window, limit %.0f", after, limit), true
}
// judgeParks fails when the share of finished tasks that parked or died rose
// more than ten points over the baseline. Both windows need the minimum
// finished tasks before the rates are compared.
func judgeParks(baseline, canary CanaryWindowSignals) (string, bool) {
if canary.Finished < minFinished || baseline.Finished < minFinished {
return fmt.Sprintf("%d finished tasks on the canary and %d in the baseline, fewer than %d each; park signal skipped", canary.Finished, baseline.Finished, minFinished), true
}
rate := func(w CanaryWindowSignals) float64 {
if w.Finished == 0 {
return 0
}
return float64(w.Bad) / float64(w.Finished)
}
before, after := rate(baseline), rate(canary)
if after > before+0.10 {
return fmt.Sprintf("%.0f%% of tasks parked or died against %.0f%% before", 100*after, 100*before), false
}
return fmt.Sprintf("%.0f%% of tasks parked or died, %.0f%% before", 100*after, 100*before), true
}