657 lines
24 KiB
Go
657 lines
24 KiB
Go
package archied
|
|
|
|
import (
|
|
"bytes"
|
|
"context"
|
|
"errors"
|
|
"flag"
|
|
"fmt"
|
|
"io"
|
|
"os"
|
|
"os/exec"
|
|
"strings"
|
|
"time"
|
|
|
|
"oras.land/oras-go/v2/errdef"
|
|
|
|
"github.com/samcharles93/archie-core/internal/domain/distribution"
|
|
"github.com/samcharles93/archie-core/internal/releaseupdate"
|
|
)
|
|
|
|
// canaryInstallWait bounds how long an install on the canary may take,
|
|
// including the watchdog's health wait and a rollback.
|
|
const canaryInstallWait = 20 * time.Minute
|
|
|
|
// migrationsDir holds the schema migrations; a release that changes it can't
|
|
// be rolled back without restoring the database snapshot.
|
|
const migrationsDir = "internal/infrastructure/postgres/migrations"
|
|
|
|
func runReleaseCanary(args []string, out, stderr io.Writer) int {
|
|
if len(args) == 0 {
|
|
fmt.Fprintln(stderr, releaseUsage)
|
|
return 2
|
|
}
|
|
switch args[0] {
|
|
case "status":
|
|
return runCanaryStatus(args[1:], out, stderr)
|
|
case "start":
|
|
return runCanaryStart(args[1:], out, stderr)
|
|
case "check":
|
|
return runCanaryCheck(args[1:], out, stderr)
|
|
case "promote":
|
|
return runCanaryPromote(args[1:], out, stderr)
|
|
case "unpin":
|
|
return runCanaryUnpin(args[1:], out, stderr)
|
|
default:
|
|
fmt.Fprintln(stderr, releaseUsage)
|
|
return 2
|
|
}
|
|
}
|
|
|
|
// addCanaryFlags registers the flags every canary subcommand takes.
|
|
func addCanaryFlags(flags *flag.FlagSet) (dashboardURL, tokenEnv *string) {
|
|
dashboardURL = flags.String("dashboard", "", "canary dashboard URL")
|
|
tokenEnv = flags.String("token-env", "ARCHIE_CANARY_TOKEN", "environment variable holding a dashboard API token for the canary")
|
|
return dashboardURL, tokenEnv
|
|
}
|
|
|
|
// addSourceFlags registers the flags naming the release source.
|
|
func addSourceFlags(flags *flag.FlagSet) (reference, key *string) {
|
|
reference = flags.String("source", "", "OCI repository releases are published to; blank is the official releases")
|
|
key = flags.String("key", "", "base64 ed25519 public key the source signs with")
|
|
return reference, key
|
|
}
|
|
|
|
func canaryDashboard(url, tokenEnv string) (releaseupdate.CanaryDashboard, error) {
|
|
if url == "" {
|
|
return releaseupdate.CanaryDashboard{}, errors.New("-dashboard is required")
|
|
}
|
|
return releaseupdate.CanaryDashboard{URL: url, Token: os.Getenv(tokenEnv)}, nil
|
|
}
|
|
|
|
func canarySource(reference, key string) (distribution.Source, error) {
|
|
return distribution.Parse(reference, key, distribution.DefaultReleases())
|
|
}
|
|
|
|
// fetchChannel fetches a channel tag; found is false when the channel has no
|
|
// release yet.
|
|
func fetchChannel(ctx context.Context, source distribution.Source, tag string) (release releaseupdate.Release, found bool, err error) {
|
|
release, err = releaseupdate.Fetch(ctx, source, tag)
|
|
if errors.Is(err, errdef.ErrNotFound) {
|
|
return releaseupdate.Release{}, false, nil
|
|
}
|
|
return release, err == nil, err
|
|
}
|
|
|
|
// publishedChannels are the releases the source currently publishes.
|
|
type publishedChannels struct {
|
|
next, stable releaseupdate.Release
|
|
hasNext, hasStable bool
|
|
}
|
|
|
|
func fetchChannels(ctx context.Context, source distribution.Source) (publishedChannels, error) {
|
|
var channels publishedChannels
|
|
var err error
|
|
if channels.next, channels.hasNext, err = fetchChannel(ctx, source, "next"); err != nil {
|
|
return channels, err
|
|
}
|
|
if channels.stable, channels.hasStable, err = fetchChannel(ctx, source, "stable"); err != nil {
|
|
return channels, err
|
|
}
|
|
return channels, nil
|
|
}
|
|
|
|
// canaryRun is the release the canary actually runs: its version, when it was
|
|
// installed and its fetched release statement.
|
|
type canaryRun struct {
|
|
version string
|
|
installedAt time.Time
|
|
release releaseupdate.Release
|
|
found bool
|
|
}
|
|
|
|
// readCanaryRun reads what the canary runs. A host that doesn't record install
|
|
// times (it updated before they were tracked) degrades to the release
|
|
// statement's issue time.
|
|
func readCanaryRun(ctx context.Context, dashboard releaseupdate.CanaryDashboard, source distribution.Source) (canaryRun, error) {
|
|
var run canaryRun
|
|
version, installedAt, _, err := dashboard.Installed(ctx)
|
|
if err != nil {
|
|
return run, err
|
|
}
|
|
run.version, run.installedAt = version, installedAt
|
|
release, found, err := fetchChannel(ctx, source, releaseupdate.Tag("exact-pin", version))
|
|
if err != nil {
|
|
return run, err
|
|
}
|
|
run.release, run.found = release, found
|
|
if run.installedAt.IsZero() && found {
|
|
run.installedAt = release.Statement.IssuedAt
|
|
}
|
|
return run, nil
|
|
}
|
|
|
|
// newerIssued reports whether release a was issued after b. IssuedAt, not
|
|
// semver order: Promote refuses a release issued before the one stable names,
|
|
// so the judge must agree with that replay rule.
|
|
func newerIssued(a, b releaseupdate.Release) bool {
|
|
return a.Statement.IssuedAt.After(b.Statement.IssuedAt)
|
|
}
|
|
|
|
func displayVersion(version string) string {
|
|
if version == "" {
|
|
return "unknown"
|
|
}
|
|
return version
|
|
}
|
|
|
|
func displayRelease(release releaseupdate.Release, found bool) string {
|
|
if !found {
|
|
return "none"
|
|
}
|
|
return release.Statement.Version
|
|
}
|
|
|
|
func printVerdict(out io.Writer, version string, verdict releaseupdate.CanaryVerdict) {
|
|
fmt.Fprintf(out, "%s: %s\n", displayVersion(version), verdict.Decision)
|
|
for _, reason := range verdict.Reasons {
|
|
fmt.Fprintf(out, " %s\n", reason)
|
|
}
|
|
}
|
|
|
|
func runCanaryStatus(args []string, out, stderr io.Writer) int {
|
|
flags := flag.NewFlagSet("release canary status", flag.ContinueOnError)
|
|
flags.SetOutput(stderr)
|
|
dashboardURL, tokenEnv := addCanaryFlags(flags)
|
|
reference, key := addSourceFlags(flags)
|
|
window := flags.Duration("window", releaseupdate.CanaryWindow, "window a release is judged over, counted from its install")
|
|
if err := flags.Parse(args); err != nil {
|
|
return 2
|
|
}
|
|
dashboard, err := canaryDashboard(*dashboardURL, *tokenEnv)
|
|
if err != nil {
|
|
fmt.Fprintln(stderr, "release canary status:", err)
|
|
return 2
|
|
}
|
|
source, err := canarySource(*reference, *key)
|
|
if err != nil {
|
|
fmt.Fprintln(stderr, "release canary status:", err)
|
|
return 2
|
|
}
|
|
if err := printCanaryStatus(context.Background(), dashboard, source, *window, out); err != nil {
|
|
fmt.Fprintln(stderr, "release canary status:", err)
|
|
return 1
|
|
}
|
|
return 0
|
|
}
|
|
|
|
// printCanaryStatus reports the canary's channel, installed release, the
|
|
// published channels, window progress and the verdict the current signals
|
|
// would give, without acting on anything.
|
|
func printCanaryStatus(ctx context.Context, dashboard releaseupdate.CanaryDashboard, source distribution.Source, window time.Duration, out io.Writer) error {
|
|
channel, pin, err := dashboard.Channel(ctx)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
channels, err := fetchChannels(ctx, source)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
run, err := readCanaryRun(ctx, dashboard, source)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
printCanaryState(out, channel, pin, run, channels, window)
|
|
return printCanaryVerdict(ctx, dashboard, run, window, out)
|
|
}
|
|
|
|
func printCanaryState(out io.Writer, channel, pin string, run canaryRun, channels publishedChannels, window time.Duration) {
|
|
fmt.Fprintf(out, "channel: %s\n", channelDisplay(channel, pin))
|
|
fmt.Fprintf(out, "installed: %s", displayVersion(run.version))
|
|
if run.installedAt.IsZero() {
|
|
fmt.Fprintln(out, " (installed_at not reported)")
|
|
} else {
|
|
fmt.Fprintf(out, " since %s\n", run.installedAt.Format(time.RFC3339))
|
|
}
|
|
fmt.Fprintf(out, "stable: %s\n", displayRelease(channels.stable, channels.hasStable))
|
|
fmt.Fprintf(out, "next: %s\n", displayRelease(channels.next, channels.hasNext))
|
|
if run.installedAt.IsZero() {
|
|
fmt.Fprintln(out, "window: unknown, no install time to count from")
|
|
} else {
|
|
elapsed := time.Since(run.installedAt)
|
|
fmt.Fprintf(out, "window: %s of %s elapsed since the install\n", elapsed.Round(time.Minute), window)
|
|
}
|
|
}
|
|
|
|
func printCanaryVerdict(ctx context.Context, dashboard releaseupdate.CanaryDashboard, run canaryRun, window time.Duration, out io.Writer) error {
|
|
if run.installedAt.IsZero() {
|
|
fmt.Fprintln(out, "verdict: unknown, no install time to judge from")
|
|
return nil
|
|
}
|
|
now := time.Now().UTC()
|
|
signals, err := dashboard.Signals(ctx, run.installedAt, now, window)
|
|
if err != nil {
|
|
return fmt.Errorf("read canary signals: %w", err)
|
|
}
|
|
printSignals(out, signals)
|
|
verdict := releaseupdate.JudgeCanary(run.version, run.installedAt, now, window, signals)
|
|
fmt.Fprintf(out, "verdict on %s: %s (read-only)\n", displayVersion(run.version), verdict.Decision)
|
|
for _, reason := range verdict.Reasons {
|
|
fmt.Fprintf(out, " %s\n", reason)
|
|
}
|
|
return nil
|
|
}
|
|
|
|
func printSignals(out io.Writer, s releaseupdate.CanarySignals) {
|
|
fmt.Fprintf(out, "signals: runs %q, ready=%t, new action alerts=%d\n", s.Installed, s.Ready, len(s.NewActionAlerts))
|
|
fmt.Fprintf(out, " tasks finished %d, parked or died %d (baseline %d, %d)\n",
|
|
s.Canary.Finished, s.Canary.Bad, s.Baseline.Finished, s.Baseline.Bad)
|
|
fmt.Fprintf(out, " error lines %d, log covers %s (baseline %d, covers %s)\n",
|
|
s.Canary.Errors, s.Canary.Covered.Round(time.Minute), s.Baseline.Errors, s.Baseline.Covered.Round(time.Minute))
|
|
}
|
|
|
|
func channelDisplay(channel, pin string) string {
|
|
if pin != "" {
|
|
return channel + " (pinned to " + pin + ")"
|
|
}
|
|
return channel
|
|
}
|
|
|
|
func runCanaryStart(args []string, out, stderr io.Writer) int {
|
|
flags := flag.NewFlagSet("release canary start", flag.ContinueOnError)
|
|
flags.SetOutput(stderr)
|
|
dashboardURL, tokenEnv := addCanaryFlags(flags)
|
|
reference, key := addSourceFlags(flags)
|
|
version := flags.String("version", "", "release to install on the canary")
|
|
force := flags.Bool("force", false, "install even when the canary is still judging a newer release")
|
|
if err := flags.Parse(args); err != nil {
|
|
return 2
|
|
}
|
|
dashboard, err := canaryDashboard(*dashboardURL, *tokenEnv)
|
|
if err != nil {
|
|
fmt.Fprintln(stderr, "release canary start:", err)
|
|
return 2
|
|
}
|
|
if *version == "" {
|
|
fmt.Fprintln(stderr, "release canary start: -version is required")
|
|
return 2
|
|
}
|
|
source, err := canarySource(*reference, *key)
|
|
if err != nil {
|
|
fmt.Fprintln(stderr, "release canary start:", err)
|
|
return 2
|
|
}
|
|
if err := startCanary(context.Background(), dashboard, source, *version, *force, out); err != nil {
|
|
fmt.Fprintln(stderr, "release canary start:", err)
|
|
return 1
|
|
}
|
|
return 0
|
|
}
|
|
|
|
// startCanary points the canary at next — clearing any rollback pin — and
|
|
// installs version on it. A release published while the canary still judges a
|
|
// newer one queues behind it, so the running window isn't restarted under a
|
|
// different release.
|
|
func startCanary(ctx context.Context, dashboard releaseupdate.CanaryDashboard, source distribution.Source, version string, force bool, out io.Writer) error {
|
|
// The channel is read before following next: a canary already on next is
|
|
// judging the release it runs, and a new release queues behind that one.
|
|
channel, _, err := dashboard.Channel(ctx)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
if err := dashboard.Follow(ctx, "next", ""); err != nil {
|
|
return fmt.Errorf("point the canary at next: %w", err)
|
|
}
|
|
if !force {
|
|
queued, err := queuedBehind(ctx, dashboard, source, channel, version, out)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
if queued {
|
|
return nil
|
|
}
|
|
}
|
|
if err := dashboard.Install(ctx, version, canaryInstallWait); err != nil {
|
|
return err
|
|
}
|
|
fmt.Fprintf(out, "the canary follows next and runs %s\n", version)
|
|
return nil
|
|
}
|
|
|
|
// queuedBehind reports whether version must wait: the canary already follows
|
|
// next and is still judging the release it runs (a judgeable one), which is a
|
|
// different release from version.
|
|
func queuedBehind(ctx context.Context, dashboard releaseupdate.CanaryDashboard, source distribution.Source, channel, version string, out io.Writer) (bool, error) {
|
|
if channel != "next" {
|
|
return false, nil
|
|
}
|
|
run, err := readCanaryRun(ctx, dashboard, source)
|
|
if err != nil {
|
|
return false, err
|
|
}
|
|
if run.version == version {
|
|
return false, nil
|
|
}
|
|
channels, err := fetchChannels(ctx, source)
|
|
if err != nil {
|
|
return false, err
|
|
}
|
|
if judgeable(run, channels) {
|
|
fmt.Fprintf(out, "%s is queued behind %s, which the canary is still judging\n", version, run.version)
|
|
return true, nil
|
|
}
|
|
return false, nil
|
|
}
|
|
|
|
func runCanaryCheck(args []string, out, stderr io.Writer) int {
|
|
flags := flag.NewFlagSet("release canary check", flag.ContinueOnError)
|
|
flags.SetOutput(stderr)
|
|
dashboardURL, tokenEnv := addCanaryFlags(flags)
|
|
reference, key := addSourceFlags(flags)
|
|
latest := flags.String("latest", "", "comma-separated image repositories whose v<version> tag becomes latest on promotion")
|
|
checkout := flags.String("checkout", ".", "repository checkout with release tags, to see whether a release migrated the database")
|
|
window := flags.Duration("window", releaseupdate.CanaryWindow, "window the installed release is judged over, counted from its install")
|
|
if err := flags.Parse(args); err != nil {
|
|
return 2
|
|
}
|
|
dashboard, err := canaryDashboard(*dashboardURL, *tokenEnv)
|
|
if err != nil {
|
|
fmt.Fprintln(stderr, "release canary check:", err)
|
|
return 2
|
|
}
|
|
source, err := canarySource(*reference, *key)
|
|
if err != nil {
|
|
fmt.Fprintln(stderr, "release canary check:", err)
|
|
return 2
|
|
}
|
|
code, err := checkCanary(context.Background(), dashboard, source, splitList(*latest), *checkout, *window, out)
|
|
if err != nil {
|
|
fmt.Fprintln(stderr, "release canary check:", err)
|
|
return 1
|
|
}
|
|
return code
|
|
}
|
|
|
|
// checkCanary judges the release the canary actually runs — not the moving
|
|
// next tag — and acts on the verdict. The exit code is 1 when the run should
|
|
// show red: a rollback happened, or the canary is held on a pin that can no
|
|
// longer be installed.
|
|
func checkCanary(ctx context.Context, dashboard releaseupdate.CanaryDashboard, source distribution.Source, latest []string, checkout string, window time.Duration, out io.Writer) (int, error) {
|
|
channels, err := fetchChannels(ctx, source)
|
|
if err != nil {
|
|
return 1, err
|
|
}
|
|
channel, pin, err := dashboard.Channel(ctx)
|
|
if err != nil {
|
|
return 1, err
|
|
}
|
|
if channel != "next" {
|
|
return heldCanary(ctx, source, channel, pin, out)
|
|
}
|
|
run, err := readCanaryRun(ctx, dashboard, source)
|
|
if err != nil {
|
|
return 1, err
|
|
}
|
|
if !judgeable(run, channels) {
|
|
fmt.Fprintf(out, "nothing to judge: the canary runs %s\n", displayVersion(run.version))
|
|
return pickupNext(ctx, dashboard, channels, run.version, out)
|
|
}
|
|
return judgeInstalled(ctx, dashboard, source, run, channels, latest, checkout, window, out)
|
|
}
|
|
|
|
// judgeable reports whether the installed release is one the canary judges: a
|
|
// published, non-prerelease release issued after stable's.
|
|
func judgeable(run canaryRun, channels publishedChannels) bool {
|
|
if !run.found || run.release.Statement.Prerelease {
|
|
return false
|
|
}
|
|
return !channels.hasStable || newerIssued(run.release, channels.stable)
|
|
}
|
|
|
|
func judgeInstalled(ctx context.Context, dashboard releaseupdate.CanaryDashboard, source distribution.Source, run canaryRun, channels publishedChannels, latest []string, checkout string, window time.Duration, out io.Writer) (int, error) {
|
|
now := time.Now().UTC()
|
|
signals, err := dashboard.Signals(ctx, run.installedAt, now, window)
|
|
if err != nil {
|
|
return 1, fmt.Errorf("read canary signals: %w", err)
|
|
}
|
|
verdict := releaseupdate.JudgeCanary(run.version, run.installedAt, now, window, signals)
|
|
printVerdict(out, run.version, verdict)
|
|
switch verdict.Decision {
|
|
case releaseupdate.CanaryPromote:
|
|
return 0, promoteJudged(ctx, dashboard, source, run, channels, latest, out)
|
|
case releaseupdate.CanaryRollBack:
|
|
return 1, rollBackCanary(ctx, dashboard, source, run, channels, checkout, out)
|
|
default:
|
|
return 0, nil
|
|
}
|
|
}
|
|
|
|
// promoteJudged moves stable to the judged release, then picks up a newer next
|
|
// queued behind it.
|
|
func promoteJudged(ctx context.Context, dashboard releaseupdate.CanaryDashboard, source distribution.Source, run canaryRun, channels publishedChannels, latest []string, out io.Writer) error {
|
|
if _, err := releaseupdate.Promote(ctx, source.Reference, source.Key, run.version, latest); err != nil {
|
|
return fmt.Errorf("promote %s: %w", run.version, err)
|
|
}
|
|
fmt.Fprintf(out, "stable is %s\n", run.version)
|
|
fresh := channels
|
|
fresh.stable, fresh.hasStable = run.release, true
|
|
_, err := pickupNext(ctx, dashboard, fresh, run.version, out)
|
|
return err
|
|
}
|
|
|
|
// pickupNext installs the next release when it is newer than stable and the
|
|
// canary doesn't already run it.
|
|
func pickupNext(ctx context.Context, dashboard releaseupdate.CanaryDashboard, channels publishedChannels, installed string, out io.Writer) (int, error) {
|
|
if !channels.hasNext || channels.next.Statement.Version == installed {
|
|
return 0, nil
|
|
}
|
|
if channels.hasStable && !newerIssued(channels.next, channels.stable) {
|
|
return 0, nil
|
|
}
|
|
if err := dashboard.Install(ctx, channels.next.Statement.Version, canaryInstallWait); err != nil {
|
|
return 1, err
|
|
}
|
|
fmt.Fprintf(out, "the canary picks up %s from next\n", channels.next.Statement.Version)
|
|
return 0, nil
|
|
}
|
|
|
|
// heldCanary reports a canary that isn't following next. Its pin must still be
|
|
// installable: a stranded pin fails the run instead of looking green.
|
|
func heldCanary(ctx context.Context, source distribution.Source, channel, pin string, out io.Writer) (int, error) {
|
|
if pin != "" {
|
|
err := checkPin(ctx, source, pin)
|
|
if errors.Is(err, errStrandedPin) {
|
|
fmt.Fprintf(out, "ERROR: the canary is held on %s %s, which is stranded: %v; unpin it or republish the release\n", channel, pin, err)
|
|
return 1, nil
|
|
}
|
|
if err != nil {
|
|
return 1, err
|
|
}
|
|
fmt.Fprintf(out, "held on %s %s, nothing to judge\n", channel, pin)
|
|
return 0, nil
|
|
}
|
|
fmt.Fprintf(out, "held on %s, nothing to judge\n", channel)
|
|
return 0, nil
|
|
}
|
|
|
|
// errStrandedPin marks a held pin that can no longer be installed.
|
|
var errStrandedPin = errors.New("pin is stranded")
|
|
|
|
// checkPin fails when a held pin can no longer be installed: the release is
|
|
// gone from the source, or its artifacts are.
|
|
func checkPin(ctx context.Context, source distribution.Source, pin string) error {
|
|
release, found, err := fetchChannel(ctx, source, releaseupdate.Tag("exact-pin", pin))
|
|
if err != nil {
|
|
return err
|
|
}
|
|
if !found {
|
|
return fmt.Errorf("%w: no release %s in the source", errStrandedPin, releaseupdate.Tag("exact-pin", pin))
|
|
}
|
|
if err := releaseupdate.Installable(ctx, source, release); err != nil {
|
|
return fmt.Errorf("%w: %w", errStrandedPin, err)
|
|
}
|
|
return nil
|
|
}
|
|
|
|
// rollBackCanary returns the canary to stable after a failed verdict. stable
|
|
// is checked installable before the canary is pinned to it: a pin whose
|
|
// artifacts are gone would strand the canary, so the run fails loudly and the
|
|
// canary stays on next. A release that changed the migrations is held on the
|
|
// failed version instead, because going back past a migration needs the
|
|
// pre-update database snapshot.
|
|
func rollBackCanary(ctx context.Context, dashboard releaseupdate.CanaryDashboard, source distribution.Source, run canaryRun, channels publishedChannels, checkout string, out io.Writer) error {
|
|
failed := run.version
|
|
if !channels.hasStable {
|
|
return holdCanary(ctx, dashboard, failed, "there's no stable release to return to", out)
|
|
}
|
|
stable := channels.stable
|
|
if err := releaseupdate.Installable(ctx, source, stable); err != nil {
|
|
return fmt.Errorf("stable %s is not installable, the canary stays on next: %w", stable.Statement.Version, err)
|
|
}
|
|
changed, err := migrated(ctx, checkout, stable.Statement.Version, failed)
|
|
if err != nil {
|
|
return holdCanary(ctx, dashboard, failed, err.Error(), out)
|
|
}
|
|
if changed {
|
|
return holdCanary(ctx, dashboard, failed, failed+" changed the database schema", out)
|
|
}
|
|
if err := dashboard.Follow(ctx, "exact-pin", stable.Statement.Version); err != nil {
|
|
return fmt.Errorf("pin the canary to %s: %w", stable.Statement.Version, err)
|
|
}
|
|
if err := dashboard.Install(ctx, stable.Statement.Version, canaryInstallWait); err != nil {
|
|
return fmt.Errorf("return the canary to %s: %w", stable.Statement.Version, err)
|
|
}
|
|
fmt.Fprintf(out, "the canary is back on %s and stable is unchanged\n", stable.Statement.Version)
|
|
return nil
|
|
}
|
|
|
|
func holdCanary(ctx context.Context, dashboard releaseupdate.CanaryDashboard, version, why string, out io.Writer) error {
|
|
if err := dashboard.Follow(ctx, "exact-pin", version); err != nil {
|
|
return fmt.Errorf("hold the canary on %s: %w", version, err)
|
|
}
|
|
fmt.Fprintf(out, "%s: the canary is held on %s and stable is unchanged; fix forward\n", why, version)
|
|
return nil
|
|
}
|
|
|
|
// runGitDiff runs git diff --quiet between two release tags and reports its
|
|
// exit code and stderr. It is a variable so tests can drive the three
|
|
// outcomes without a repository.
|
|
var runGitDiff = func(ctx context.Context, checkout, from, to string) (int, string, error) {
|
|
cmd := exec.CommandContext(ctx, "git", "-C", checkout, "diff", "--quiet", "v"+from, "v"+to, "--", migrationsDir)
|
|
var buf bytes.Buffer
|
|
cmd.Stderr = &buf
|
|
err := cmd.Run()
|
|
if err == nil {
|
|
return 0, "", nil
|
|
}
|
|
if exitErr, ok := errors.AsType[*exec.ExitError](err); ok {
|
|
return exitErr.ExitCode(), buf.String(), err
|
|
}
|
|
return -1, buf.String(), err
|
|
}
|
|
|
|
// migrated reports whether the migrations differ between two release tags.
|
|
// It returns true, nil when they differ, false, nil when they are identical,
|
|
// and true, err when git could not answer, with the reason in err. Every
|
|
// non-identical outcome holds the canary rather than rolls it back.
|
|
func migrated(ctx context.Context, checkout, from, to string) (bool, error) {
|
|
code, stderr, runErr := runGitDiff(ctx, checkout, from, to)
|
|
switch code {
|
|
case 0:
|
|
return false, nil
|
|
case 1:
|
|
return true, nil
|
|
}
|
|
reason := strings.TrimSpace(stderr)
|
|
if reason == "" && runErr != nil {
|
|
reason = runErr.Error()
|
|
}
|
|
return true, fmt.Errorf("could not tell whether %s changed the database schema: %s", to, reason)
|
|
}
|
|
|
|
func runCanaryPromote(args []string, out, stderr io.Writer) int {
|
|
flags := flag.NewFlagSet("release canary promote", flag.ContinueOnError)
|
|
flags.SetOutput(stderr)
|
|
dashboardURL, tokenEnv := addCanaryFlags(flags)
|
|
reference, key := addSourceFlags(flags)
|
|
latest := flags.String("latest", "", "comma-separated image repositories whose v<version> tag becomes latest on promotion")
|
|
version := flags.String("version", "", "release expected on the canary; must match what it runs")
|
|
window := flags.Duration("window", releaseupdate.CanaryWindow, "window the signals are read over, counted from the install")
|
|
if err := flags.Parse(args); err != nil {
|
|
return 2
|
|
}
|
|
dashboard, err := canaryDashboard(*dashboardURL, *tokenEnv)
|
|
if err != nil {
|
|
fmt.Fprintln(stderr, "release canary promote:", err)
|
|
return 2
|
|
}
|
|
source, err := canarySource(*reference, *key)
|
|
if err != nil {
|
|
fmt.Fprintln(stderr, "release canary promote:", err)
|
|
return 2
|
|
}
|
|
code, err := promoteCanaryNow(context.Background(), dashboard, source, *version, splitList(*latest), *window, out)
|
|
if err != nil {
|
|
fmt.Fprintln(stderr, "release canary promote:", err)
|
|
return 1
|
|
}
|
|
return code
|
|
}
|
|
|
|
// promoteCanaryNow promotes the release the canary runs right now, without
|
|
// waiting out the window: the signals must show no hard failure and no
|
|
// relative breach. It refuses, with exit 1, when anything fails.
|
|
func promoteCanaryNow(ctx context.Context, dashboard releaseupdate.CanaryDashboard, source distribution.Source, expectVersion string, latest []string, window time.Duration, out io.Writer) (int, error) {
|
|
run, err := readCanaryRun(ctx, dashboard, source)
|
|
if err != nil {
|
|
return 1, err
|
|
}
|
|
channel, pin, err := dashboard.Channel(ctx)
|
|
if err != nil {
|
|
return 1, err
|
|
}
|
|
fmt.Fprintf(out, "channel: %s\n", channelDisplay(channel, pin))
|
|
if expectVersion != "" && expectVersion != run.version {
|
|
return 1, fmt.Errorf("the canary runs %s, not %s", displayVersion(run.version), expectVersion)
|
|
}
|
|
if !run.found {
|
|
return 1, fmt.Errorf("no release %s in the source to promote", displayVersion(run.version))
|
|
}
|
|
now := time.Now().UTC()
|
|
signals, err := dashboard.Signals(ctx, run.installedAt, now, window)
|
|
if err != nil {
|
|
return 1, fmt.Errorf("read canary signals: %w", err)
|
|
}
|
|
verdict := releaseupdate.JudgeCanaryNow(run.version, now, window, signals)
|
|
printVerdict(out, run.version, verdict)
|
|
if verdict.Decision == releaseupdate.CanaryRollBack {
|
|
fmt.Fprintln(out, "refusing to promote: the canary's signals fail")
|
|
return 1, nil
|
|
}
|
|
if _, err := releaseupdate.Promote(ctx, source.Reference, source.Key, run.version, latest); err != nil {
|
|
return 1, fmt.Errorf("promote %s: %w", run.version, err)
|
|
}
|
|
fmt.Fprintf(out, "stable is %s\n", run.version)
|
|
return 0, nil
|
|
}
|
|
|
|
func runCanaryUnpin(args []string, out, stderr io.Writer) int {
|
|
flags := flag.NewFlagSet("release canary unpin", flag.ContinueOnError)
|
|
flags.SetOutput(stderr)
|
|
dashboardURL, tokenEnv := addCanaryFlags(flags)
|
|
if err := flags.Parse(args); err != nil {
|
|
return 2
|
|
}
|
|
dashboard, err := canaryDashboard(*dashboardURL, *tokenEnv)
|
|
if err != nil {
|
|
fmt.Fprintln(stderr, "release canary unpin:", err)
|
|
return 2
|
|
}
|
|
if err := dashboard.Follow(context.Background(), "next", ""); err != nil {
|
|
fmt.Fprintln(stderr, "release canary unpin:", err)
|
|
return 1
|
|
}
|
|
fmt.Fprintln(out, "the canary follows next again, with no pin")
|
|
return 0
|
|
}
|