- one shared UpdateStateDir() resolver for daemon and dashboard, instead of two copies that could drift - the recovery path for a committed/rolled-back journal writes installed.json too, so a crash after the outcome phase cannot lose the install record - JudgeCanaryNow is the explicit signals-only entry point for promoting without the window, replacing the zero-install-time sentinel - canary version comparisons use IssuedAt, matching Promote's replay rule, so semver/issued-at divergence cannot wedge check red; queuedBehind reuses the judgeable predicate instead of a looser check - promote prints the channel/pin; promote.yml cancels superseded runs - canary card: no duplicated elapsed rendering, channel row waits for the settings resource instead of showing a default, agent fallback '?' - rehearsal script bounds its docker build/push/run calls task check passes on this tree.
153 lines
5.8 KiB
Go
153 lines
5.8 KiB
Go
package releaseupdate
|
|
|
|
import (
|
|
"fmt"
|
|
"time"
|
|
)
|
|
|
|
// The canary host installs each release from the next channel. Its behaviour
|
|
// in the window after a release was installed is compared with its own
|
|
// behaviour in the window before, because there's only one production host to
|
|
// compare with.
|
|
const (
|
|
// CanaryWindow is the default window a release runs on the canary before
|
|
// it can be promoted. A day covers one full cycle of schedules and
|
|
// traffic.
|
|
CanaryWindow = 24 * time.Hour
|
|
// canarySettle is how long the install and restart get, counted from when
|
|
// the release was installed on the canary, before any check can fail the
|
|
// release.
|
|
canarySettle = 30 * time.Minute
|
|
// minFinished is the fewest finished tasks needed to judge the park rate.
|
|
minFinished = 5
|
|
// minLogCoverage is how far back into the baseline window the error log
|
|
// must reach before error counts are compared.
|
|
minLogCoverage = time.Hour
|
|
)
|
|
|
|
// CanaryDecision is what the evaluation says to do with a release.
|
|
type CanaryDecision string
|
|
|
|
const (
|
|
CanaryWait CanaryDecision = "wait"
|
|
CanaryPromote CanaryDecision = "promote"
|
|
CanaryRollBack CanaryDecision = "roll back"
|
|
)
|
|
|
|
// CanaryWindowSignals is one window's task and error-log counts.
|
|
type CanaryWindowSignals struct {
|
|
// Finished counts tasks that reached a terminal or review status; Bad
|
|
// is those that parked or died.
|
|
Finished, Bad int
|
|
// Errors counts error-level log lines within Covered, the part of the
|
|
// window that the log still reaches.
|
|
Errors int
|
|
Covered time.Duration
|
|
}
|
|
|
|
// CanarySignals is what the canary reported at one check.
|
|
type CanarySignals struct {
|
|
Installed string
|
|
Ready bool
|
|
// NewActionAlerts are open action-severity alerts first raised since
|
|
// the release was installed.
|
|
NewActionAlerts []string
|
|
Baseline CanaryWindowSignals
|
|
Canary CanaryWindowSignals
|
|
}
|
|
|
|
// CanaryVerdict is a decision and the signals that decided it.
|
|
type CanaryVerdict struct {
|
|
Decision CanaryDecision
|
|
Reasons []string
|
|
}
|
|
|
|
// JudgeCanaryNow decides from the current signals alone, for an operator
|
|
// promoting without waiting out the window: the settle and window waits are
|
|
// skipped by construction, every hard and relative check still applies.
|
|
func JudgeCanaryNow(version string, now time.Time, window time.Duration, s CanarySignals) CanaryVerdict {
|
|
return JudgeCanary(version, now.Add(-(window + canarySettle)), now, window, s)
|
|
}
|
|
|
|
// JudgeCanary decides what to do with a release, given when it was installed
|
|
// on the canary host. Once the release has settled, a failed hard check rolls
|
|
// it back at any time. Error and park rates are judged against the baseline
|
|
// whenever both windows carry enough data, so a breach rolls the release back
|
|
// even mid-window; a full window with no failure promotes it.
|
|
func JudgeCanary(version string, installedAt, now time.Time, window time.Duration, s CanarySignals) CanaryVerdict {
|
|
elapsed := now.Sub(installedAt)
|
|
if elapsed < canarySettle {
|
|
return CanaryVerdict{Decision: CanaryWait, Reasons: []string{"settling after install"}}
|
|
}
|
|
var failed []string
|
|
if s.Installed != version {
|
|
failed = append(failed, fmt.Sprintf("canary runs %q, not %s: the install failed or was rolled back", s.Installed, version))
|
|
}
|
|
if !s.Ready {
|
|
failed = append(failed, "readiness is degraded")
|
|
}
|
|
for _, alert := range s.NewActionAlerts {
|
|
failed = append(failed, "new action alert: "+alert)
|
|
}
|
|
if len(failed) > 0 {
|
|
return CanaryVerdict{Decision: CanaryRollBack, Reasons: failed}
|
|
}
|
|
var notes []string
|
|
if reason, ok := judgeErrors(s.Baseline, s.Canary, window); !ok {
|
|
failed = append(failed, reason)
|
|
} else {
|
|
notes = append(notes, reason)
|
|
}
|
|
if reason, ok := judgeParks(s.Baseline, s.Canary); !ok {
|
|
failed = append(failed, reason)
|
|
} else {
|
|
notes = append(notes, reason)
|
|
}
|
|
if len(failed) > 0 {
|
|
return CanaryVerdict{Decision: CanaryRollBack, Reasons: failed}
|
|
}
|
|
if elapsed < window {
|
|
reasons := append([]string{fmt.Sprintf("%s of %s elapsed since the install", elapsed.Round(time.Minute), window)}, notes...)
|
|
return CanaryVerdict{Decision: CanaryWait, Reasons: reasons}
|
|
}
|
|
return CanaryVerdict{Decision: CanaryPromote, Reasons: notes}
|
|
}
|
|
|
|
// judgeErrors fails when the canary logged more than twice the baseline's
|
|
// error lines plus ten. Each count is scaled up to a full window when the log
|
|
// covers only part of its window.
|
|
func judgeErrors(baseline, canary CanaryWindowSignals, window time.Duration) (string, bool) {
|
|
if baseline.Covered < minLogCoverage || canary.Covered <= 0 {
|
|
return "error log doesn't reach back over the baseline; error signal skipped", true
|
|
}
|
|
scale := func(w CanaryWindowSignals) float64 {
|
|
return float64(w.Errors) * float64(window) / float64(w.Covered)
|
|
}
|
|
before, after := scale(baseline), scale(canary)
|
|
limit := 2*before + 10
|
|
if after > limit {
|
|
return fmt.Sprintf("%.0f error log lines per window against a limit of %.0f (baseline %.0f)", after, limit, before), false
|
|
}
|
|
return fmt.Sprintf("%.0f error log lines per window, limit %.0f", after, limit), true
|
|
}
|
|
|
|
// judgeParks fails when the share of finished tasks that parked or died rose
|
|
// more than ten points over the baseline. Both windows need the minimum
|
|
// finished tasks before the rates are compared.
|
|
func judgeParks(baseline, canary CanaryWindowSignals) (string, bool) {
|
|
if canary.Finished < minFinished || baseline.Finished < minFinished {
|
|
return fmt.Sprintf("%d finished tasks on the canary and %d in the baseline, fewer than %d each; park signal skipped", canary.Finished, baseline.Finished, minFinished), true
|
|
}
|
|
rate := func(w CanaryWindowSignals) float64 {
|
|
if w.Finished == 0 {
|
|
return 0
|
|
}
|
|
return float64(w.Bad) / float64(w.Finished)
|
|
}
|
|
before, after := rate(baseline), rate(canary)
|
|
if after > before+0.10 {
|
|
return fmt.Sprintf("%.0f%% of tasks parked or died against %.0f%% before", 100*after, 100*before), false
|
|
}
|
|
return fmt.Sprintf("%.0f%% of tasks parked or died, %.0f%% before", 100*after, 100*before), true
|
|
}
|