Files
archie/internal/app/archied/release_canary.go
T
sam a6a0c7c11e
deploy / gate (push) Successful in 4m41s
Quality / check (push) Successful in 6m27s
deploy / archied-image (push) Successful in 2m57s
deploy / agent-image (push) Successful in 11s
deploy / dist-zip (push) Skipped
fix(archied): say why the canary can't compare migrations
2026-11-08 03:05:29 +11:00

657 lines
24 KiB
Go

package archied
import (
"bytes"
"context"
"errors"
"flag"
"fmt"
"io"
"os"
"os/exec"
"strings"
"time"
"oras.land/oras-go/v2/errdef"
"github.com/samcharles93/archie-core/internal/domain/distribution"
"github.com/samcharles93/archie-core/internal/releaseupdate"
)
// canaryInstallWait bounds how long an install on the canary may take,
// including the watchdog's health wait and a rollback.
const canaryInstallWait = 20 * time.Minute
// migrationsDir holds the schema migrations; a release that changes it can't
// be rolled back without restoring the database snapshot.
const migrationsDir = "internal/infrastructure/postgres/migrations"
func runReleaseCanary(args []string, out, stderr io.Writer) int {
if len(args) == 0 {
fmt.Fprintln(stderr, releaseUsage)
return 2
}
switch args[0] {
case "status":
return runCanaryStatus(args[1:], out, stderr)
case "start":
return runCanaryStart(args[1:], out, stderr)
case "check":
return runCanaryCheck(args[1:], out, stderr)
case "promote":
return runCanaryPromote(args[1:], out, stderr)
case "unpin":
return runCanaryUnpin(args[1:], out, stderr)
default:
fmt.Fprintln(stderr, releaseUsage)
return 2
}
}
// addCanaryFlags registers the flags every canary subcommand takes.
func addCanaryFlags(flags *flag.FlagSet) (dashboardURL, tokenEnv *string) {
dashboardURL = flags.String("dashboard", "", "canary dashboard URL")
tokenEnv = flags.String("token-env", "ARCHIE_CANARY_TOKEN", "environment variable holding a dashboard API token for the canary")
return dashboardURL, tokenEnv
}
// addSourceFlags registers the flags naming the release source.
func addSourceFlags(flags *flag.FlagSet) (reference, key *string) {
reference = flags.String("source", "", "OCI repository releases are published to; blank is the official releases")
key = flags.String("key", "", "base64 ed25519 public key the source signs with")
return reference, key
}
func canaryDashboard(url, tokenEnv string) (releaseupdate.CanaryDashboard, error) {
if url == "" {
return releaseupdate.CanaryDashboard{}, errors.New("-dashboard is required")
}
return releaseupdate.CanaryDashboard{URL: url, Token: os.Getenv(tokenEnv)}, nil
}
func canarySource(reference, key string) (distribution.Source, error) {
return distribution.Parse(reference, key, distribution.DefaultReleases())
}
// fetchChannel fetches a channel tag; found is false when the channel has no
// release yet.
func fetchChannel(ctx context.Context, source distribution.Source, tag string) (release releaseupdate.Release, found bool, err error) {
release, err = releaseupdate.Fetch(ctx, source, tag)
if errors.Is(err, errdef.ErrNotFound) {
return releaseupdate.Release{}, false, nil
}
return release, err == nil, err
}
// publishedChannels are the releases the source currently publishes.
type publishedChannels struct {
next, stable releaseupdate.Release
hasNext, hasStable bool
}
func fetchChannels(ctx context.Context, source distribution.Source) (publishedChannels, error) {
var channels publishedChannels
var err error
if channels.next, channels.hasNext, err = fetchChannel(ctx, source, "next"); err != nil {
return channels, err
}
if channels.stable, channels.hasStable, err = fetchChannel(ctx, source, "stable"); err != nil {
return channels, err
}
return channels, nil
}
// canaryRun is the release the canary actually runs: its version, when it was
// installed and its fetched release statement.
type canaryRun struct {
version string
installedAt time.Time
release releaseupdate.Release
found bool
}
// readCanaryRun reads what the canary runs. A host that doesn't record install
// times (it updated before they were tracked) degrades to the release
// statement's issue time.
func readCanaryRun(ctx context.Context, dashboard releaseupdate.CanaryDashboard, source distribution.Source) (canaryRun, error) {
var run canaryRun
version, installedAt, _, err := dashboard.Installed(ctx)
if err != nil {
return run, err
}
run.version, run.installedAt = version, installedAt
release, found, err := fetchChannel(ctx, source, releaseupdate.Tag("exact-pin", version))
if err != nil {
return run, err
}
run.release, run.found = release, found
if run.installedAt.IsZero() && found {
run.installedAt = release.Statement.IssuedAt
}
return run, nil
}
// newerIssued reports whether release a was issued after b. IssuedAt, not
// semver order: Promote refuses a release issued before the one stable names,
// so the judge must agree with that replay rule.
func newerIssued(a, b releaseupdate.Release) bool {
return a.Statement.IssuedAt.After(b.Statement.IssuedAt)
}
func displayVersion(version string) string {
if version == "" {
return "unknown"
}
return version
}
func displayRelease(release releaseupdate.Release, found bool) string {
if !found {
return "none"
}
return release.Statement.Version
}
func printVerdict(out io.Writer, version string, verdict releaseupdate.CanaryVerdict) {
fmt.Fprintf(out, "%s: %s\n", displayVersion(version), verdict.Decision)
for _, reason := range verdict.Reasons {
fmt.Fprintf(out, " %s\n", reason)
}
}
func runCanaryStatus(args []string, out, stderr io.Writer) int {
flags := flag.NewFlagSet("release canary status", flag.ContinueOnError)
flags.SetOutput(stderr)
dashboardURL, tokenEnv := addCanaryFlags(flags)
reference, key := addSourceFlags(flags)
window := flags.Duration("window", releaseupdate.CanaryWindow, "window a release is judged over, counted from its install")
if err := flags.Parse(args); err != nil {
return 2
}
dashboard, err := canaryDashboard(*dashboardURL, *tokenEnv)
if err != nil {
fmt.Fprintln(stderr, "release canary status:", err)
return 2
}
source, err := canarySource(*reference, *key)
if err != nil {
fmt.Fprintln(stderr, "release canary status:", err)
return 2
}
if err := printCanaryStatus(context.Background(), dashboard, source, *window, out); err != nil {
fmt.Fprintln(stderr, "release canary status:", err)
return 1
}
return 0
}
// printCanaryStatus reports the canary's channel, installed release, the
// published channels, window progress and the verdict the current signals
// would give, without acting on anything.
func printCanaryStatus(ctx context.Context, dashboard releaseupdate.CanaryDashboard, source distribution.Source, window time.Duration, out io.Writer) error {
channel, pin, err := dashboard.Channel(ctx)
if err != nil {
return err
}
channels, err := fetchChannels(ctx, source)
if err != nil {
return err
}
run, err := readCanaryRun(ctx, dashboard, source)
if err != nil {
return err
}
printCanaryState(out, channel, pin, run, channels, window)
return printCanaryVerdict(ctx, dashboard, run, window, out)
}
func printCanaryState(out io.Writer, channel, pin string, run canaryRun, channels publishedChannels, window time.Duration) {
fmt.Fprintf(out, "channel: %s\n", channelDisplay(channel, pin))
fmt.Fprintf(out, "installed: %s", displayVersion(run.version))
if run.installedAt.IsZero() {
fmt.Fprintln(out, " (installed_at not reported)")
} else {
fmt.Fprintf(out, " since %s\n", run.installedAt.Format(time.RFC3339))
}
fmt.Fprintf(out, "stable: %s\n", displayRelease(channels.stable, channels.hasStable))
fmt.Fprintf(out, "next: %s\n", displayRelease(channels.next, channels.hasNext))
if run.installedAt.IsZero() {
fmt.Fprintln(out, "window: unknown, no install time to count from")
} else {
elapsed := time.Since(run.installedAt)
fmt.Fprintf(out, "window: %s of %s elapsed since the install\n", elapsed.Round(time.Minute), window)
}
}
func printCanaryVerdict(ctx context.Context, dashboard releaseupdate.CanaryDashboard, run canaryRun, window time.Duration, out io.Writer) error {
if run.installedAt.IsZero() {
fmt.Fprintln(out, "verdict: unknown, no install time to judge from")
return nil
}
now := time.Now().UTC()
signals, err := dashboard.Signals(ctx, run.installedAt, now, window)
if err != nil {
return fmt.Errorf("read canary signals: %w", err)
}
printSignals(out, signals)
verdict := releaseupdate.JudgeCanary(run.version, run.installedAt, now, window, signals)
fmt.Fprintf(out, "verdict on %s: %s (read-only)\n", displayVersion(run.version), verdict.Decision)
for _, reason := range verdict.Reasons {
fmt.Fprintf(out, " %s\n", reason)
}
return nil
}
func printSignals(out io.Writer, s releaseupdate.CanarySignals) {
fmt.Fprintf(out, "signals: runs %q, ready=%t, new action alerts=%d\n", s.Installed, s.Ready, len(s.NewActionAlerts))
fmt.Fprintf(out, " tasks finished %d, parked or died %d (baseline %d, %d)\n",
s.Canary.Finished, s.Canary.Bad, s.Baseline.Finished, s.Baseline.Bad)
fmt.Fprintf(out, " error lines %d, log covers %s (baseline %d, covers %s)\n",
s.Canary.Errors, s.Canary.Covered.Round(time.Minute), s.Baseline.Errors, s.Baseline.Covered.Round(time.Minute))
}
func channelDisplay(channel, pin string) string {
if pin != "" {
return channel + " (pinned to " + pin + ")"
}
return channel
}
func runCanaryStart(args []string, out, stderr io.Writer) int {
flags := flag.NewFlagSet("release canary start", flag.ContinueOnError)
flags.SetOutput(stderr)
dashboardURL, tokenEnv := addCanaryFlags(flags)
reference, key := addSourceFlags(flags)
version := flags.String("version", "", "release to install on the canary")
force := flags.Bool("force", false, "install even when the canary is still judging a newer release")
if err := flags.Parse(args); err != nil {
return 2
}
dashboard, err := canaryDashboard(*dashboardURL, *tokenEnv)
if err != nil {
fmt.Fprintln(stderr, "release canary start:", err)
return 2
}
if *version == "" {
fmt.Fprintln(stderr, "release canary start: -version is required")
return 2
}
source, err := canarySource(*reference, *key)
if err != nil {
fmt.Fprintln(stderr, "release canary start:", err)
return 2
}
if err := startCanary(context.Background(), dashboard, source, *version, *force, out); err != nil {
fmt.Fprintln(stderr, "release canary start:", err)
return 1
}
return 0
}
// startCanary points the canary at next — clearing any rollback pin — and
// installs version on it. A release published while the canary still judges a
// newer one queues behind it, so the running window isn't restarted under a
// different release.
func startCanary(ctx context.Context, dashboard releaseupdate.CanaryDashboard, source distribution.Source, version string, force bool, out io.Writer) error {
// The channel is read before following next: a canary already on next is
// judging the release it runs, and a new release queues behind that one.
channel, _, err := dashboard.Channel(ctx)
if err != nil {
return err
}
if err := dashboard.Follow(ctx, "next", ""); err != nil {
return fmt.Errorf("point the canary at next: %w", err)
}
if !force {
queued, err := queuedBehind(ctx, dashboard, source, channel, version, out)
if err != nil {
return err
}
if queued {
return nil
}
}
if err := dashboard.Install(ctx, version, canaryInstallWait); err != nil {
return err
}
fmt.Fprintf(out, "the canary follows next and runs %s\n", version)
return nil
}
// queuedBehind reports whether version must wait: the canary already follows
// next and is still judging the release it runs (a judgeable one), which is a
// different release from version.
func queuedBehind(ctx context.Context, dashboard releaseupdate.CanaryDashboard, source distribution.Source, channel, version string, out io.Writer) (bool, error) {
if channel != "next" {
return false, nil
}
run, err := readCanaryRun(ctx, dashboard, source)
if err != nil {
return false, err
}
if run.version == version {
return false, nil
}
channels, err := fetchChannels(ctx, source)
if err != nil {
return false, err
}
if judgeable(run, channels) {
fmt.Fprintf(out, "%s is queued behind %s, which the canary is still judging\n", version, run.version)
return true, nil
}
return false, nil
}
func runCanaryCheck(args []string, out, stderr io.Writer) int {
flags := flag.NewFlagSet("release canary check", flag.ContinueOnError)
flags.SetOutput(stderr)
dashboardURL, tokenEnv := addCanaryFlags(flags)
reference, key := addSourceFlags(flags)
latest := flags.String("latest", "", "comma-separated image repositories whose v<version> tag becomes latest on promotion")
checkout := flags.String("checkout", ".", "repository checkout with release tags, to see whether a release migrated the database")
window := flags.Duration("window", releaseupdate.CanaryWindow, "window the installed release is judged over, counted from its install")
if err := flags.Parse(args); err != nil {
return 2
}
dashboard, err := canaryDashboard(*dashboardURL, *tokenEnv)
if err != nil {
fmt.Fprintln(stderr, "release canary check:", err)
return 2
}
source, err := canarySource(*reference, *key)
if err != nil {
fmt.Fprintln(stderr, "release canary check:", err)
return 2
}
code, err := checkCanary(context.Background(), dashboard, source, splitList(*latest), *checkout, *window, out)
if err != nil {
fmt.Fprintln(stderr, "release canary check:", err)
return 1
}
return code
}
// checkCanary judges the release the canary actually runs — not the moving
// next tag — and acts on the verdict. The exit code is 1 when the run should
// show red: a rollback happened, or the canary is held on a pin that can no
// longer be installed.
func checkCanary(ctx context.Context, dashboard releaseupdate.CanaryDashboard, source distribution.Source, latest []string, checkout string, window time.Duration, out io.Writer) (int, error) {
channels, err := fetchChannels(ctx, source)
if err != nil {
return 1, err
}
channel, pin, err := dashboard.Channel(ctx)
if err != nil {
return 1, err
}
if channel != "next" {
return heldCanary(ctx, source, channel, pin, out)
}
run, err := readCanaryRun(ctx, dashboard, source)
if err != nil {
return 1, err
}
if !judgeable(run, channels) {
fmt.Fprintf(out, "nothing to judge: the canary runs %s\n", displayVersion(run.version))
return pickupNext(ctx, dashboard, channels, run.version, out)
}
return judgeInstalled(ctx, dashboard, source, run, channels, latest, checkout, window, out)
}
// judgeable reports whether the installed release is one the canary judges: a
// published, non-prerelease release issued after stable's.
func judgeable(run canaryRun, channels publishedChannels) bool {
if !run.found || run.release.Statement.Prerelease {
return false
}
return !channels.hasStable || newerIssued(run.release, channels.stable)
}
func judgeInstalled(ctx context.Context, dashboard releaseupdate.CanaryDashboard, source distribution.Source, run canaryRun, channels publishedChannels, latest []string, checkout string, window time.Duration, out io.Writer) (int, error) {
now := time.Now().UTC()
signals, err := dashboard.Signals(ctx, run.installedAt, now, window)
if err != nil {
return 1, fmt.Errorf("read canary signals: %w", err)
}
verdict := releaseupdate.JudgeCanary(run.version, run.installedAt, now, window, signals)
printVerdict(out, run.version, verdict)
switch verdict.Decision {
case releaseupdate.CanaryPromote:
return 0, promoteJudged(ctx, dashboard, source, run, channels, latest, out)
case releaseupdate.CanaryRollBack:
return 1, rollBackCanary(ctx, dashboard, source, run, channels, checkout, out)
default:
return 0, nil
}
}
// promoteJudged moves stable to the judged release, then picks up a newer next
// queued behind it.
func promoteJudged(ctx context.Context, dashboard releaseupdate.CanaryDashboard, source distribution.Source, run canaryRun, channels publishedChannels, latest []string, out io.Writer) error {
if _, err := releaseupdate.Promote(ctx, source.Reference, source.Key, run.version, latest); err != nil {
return fmt.Errorf("promote %s: %w", run.version, err)
}
fmt.Fprintf(out, "stable is %s\n", run.version)
fresh := channels
fresh.stable, fresh.hasStable = run.release, true
_, err := pickupNext(ctx, dashboard, fresh, run.version, out)
return err
}
// pickupNext installs the next release when it is newer than stable and the
// canary doesn't already run it.
func pickupNext(ctx context.Context, dashboard releaseupdate.CanaryDashboard, channels publishedChannels, installed string, out io.Writer) (int, error) {
if !channels.hasNext || channels.next.Statement.Version == installed {
return 0, nil
}
if channels.hasStable && !newerIssued(channels.next, channels.stable) {
return 0, nil
}
if err := dashboard.Install(ctx, channels.next.Statement.Version, canaryInstallWait); err != nil {
return 1, err
}
fmt.Fprintf(out, "the canary picks up %s from next\n", channels.next.Statement.Version)
return 0, nil
}
// heldCanary reports a canary that isn't following next. Its pin must still be
// installable: a stranded pin fails the run instead of looking green.
func heldCanary(ctx context.Context, source distribution.Source, channel, pin string, out io.Writer) (int, error) {
if pin != "" {
err := checkPin(ctx, source, pin)
if errors.Is(err, errStrandedPin) {
fmt.Fprintf(out, "ERROR: the canary is held on %s %s, which is stranded: %v; unpin it or republish the release\n", channel, pin, err)
return 1, nil
}
if err != nil {
return 1, err
}
fmt.Fprintf(out, "held on %s %s, nothing to judge\n", channel, pin)
return 0, nil
}
fmt.Fprintf(out, "held on %s, nothing to judge\n", channel)
return 0, nil
}
// errStrandedPin marks a held pin that can no longer be installed.
var errStrandedPin = errors.New("pin is stranded")
// checkPin fails when a held pin can no longer be installed: the release is
// gone from the source, or its artifacts are.
func checkPin(ctx context.Context, source distribution.Source, pin string) error {
release, found, err := fetchChannel(ctx, source, releaseupdate.Tag("exact-pin", pin))
if err != nil {
return err
}
if !found {
return fmt.Errorf("%w: no release %s in the source", errStrandedPin, releaseupdate.Tag("exact-pin", pin))
}
if err := releaseupdate.Installable(ctx, source, release); err != nil {
return fmt.Errorf("%w: %w", errStrandedPin, err)
}
return nil
}
// rollBackCanary returns the canary to stable after a failed verdict. stable
// is checked installable before the canary is pinned to it: a pin whose
// artifacts are gone would strand the canary, so the run fails loudly and the
// canary stays on next. A release that changed the migrations is held on the
// failed version instead, because going back past a migration needs the
// pre-update database snapshot.
func rollBackCanary(ctx context.Context, dashboard releaseupdate.CanaryDashboard, source distribution.Source, run canaryRun, channels publishedChannels, checkout string, out io.Writer) error {
failed := run.version
if !channels.hasStable {
return holdCanary(ctx, dashboard, failed, "there's no stable release to return to", out)
}
stable := channels.stable
if err := releaseupdate.Installable(ctx, source, stable); err != nil {
return fmt.Errorf("stable %s is not installable, the canary stays on next: %w", stable.Statement.Version, err)
}
changed, err := migrated(ctx, checkout, stable.Statement.Version, failed)
if err != nil {
return holdCanary(ctx, dashboard, failed, err.Error(), out)
}
if changed {
return holdCanary(ctx, dashboard, failed, failed+" changed the database schema", out)
}
if err := dashboard.Follow(ctx, "exact-pin", stable.Statement.Version); err != nil {
return fmt.Errorf("pin the canary to %s: %w", stable.Statement.Version, err)
}
if err := dashboard.Install(ctx, stable.Statement.Version, canaryInstallWait); err != nil {
return fmt.Errorf("return the canary to %s: %w", stable.Statement.Version, err)
}
fmt.Fprintf(out, "the canary is back on %s and stable is unchanged\n", stable.Statement.Version)
return nil
}
func holdCanary(ctx context.Context, dashboard releaseupdate.CanaryDashboard, version, why string, out io.Writer) error {
if err := dashboard.Follow(ctx, "exact-pin", version); err != nil {
return fmt.Errorf("hold the canary on %s: %w", version, err)
}
fmt.Fprintf(out, "%s: the canary is held on %s and stable is unchanged; fix forward\n", why, version)
return nil
}
// runGitDiff runs git diff --quiet between two release tags and reports its
// exit code and stderr. It is a variable so tests can drive the three
// outcomes without a repository.
var runGitDiff = func(ctx context.Context, checkout, from, to string) (int, string, error) {
cmd := exec.CommandContext(ctx, "git", "-C", checkout, "diff", "--quiet", "v"+from, "v"+to, "--", migrationsDir)
var buf bytes.Buffer
cmd.Stderr = &buf
err := cmd.Run()
if err == nil {
return 0, "", nil
}
if exitErr, ok := errors.AsType[*exec.ExitError](err); ok {
return exitErr.ExitCode(), buf.String(), err
}
return -1, buf.String(), err
}
// migrated reports whether the migrations differ between two release tags.
// It returns true, nil when they differ, false, nil when they are identical,
// and true, err when git could not answer, with the reason in err. Every
// non-identical outcome holds the canary rather than rolls it back.
func migrated(ctx context.Context, checkout, from, to string) (bool, error) {
code, stderr, runErr := runGitDiff(ctx, checkout, from, to)
switch code {
case 0:
return false, nil
case 1:
return true, nil
}
reason := strings.TrimSpace(stderr)
if reason == "" && runErr != nil {
reason = runErr.Error()
}
return true, fmt.Errorf("could not tell whether %s changed the database schema: %s", to, reason)
}
func runCanaryPromote(args []string, out, stderr io.Writer) int {
flags := flag.NewFlagSet("release canary promote", flag.ContinueOnError)
flags.SetOutput(stderr)
dashboardURL, tokenEnv := addCanaryFlags(flags)
reference, key := addSourceFlags(flags)
latest := flags.String("latest", "", "comma-separated image repositories whose v<version> tag becomes latest on promotion")
version := flags.String("version", "", "release expected on the canary; must match what it runs")
window := flags.Duration("window", releaseupdate.CanaryWindow, "window the signals are read over, counted from the install")
if err := flags.Parse(args); err != nil {
return 2
}
dashboard, err := canaryDashboard(*dashboardURL, *tokenEnv)
if err != nil {
fmt.Fprintln(stderr, "release canary promote:", err)
return 2
}
source, err := canarySource(*reference, *key)
if err != nil {
fmt.Fprintln(stderr, "release canary promote:", err)
return 2
}
code, err := promoteCanaryNow(context.Background(), dashboard, source, *version, splitList(*latest), *window, out)
if err != nil {
fmt.Fprintln(stderr, "release canary promote:", err)
return 1
}
return code
}
// promoteCanaryNow promotes the release the canary runs right now, without
// waiting out the window: the signals must show no hard failure and no
// relative breach. It refuses, with exit 1, when anything fails.
func promoteCanaryNow(ctx context.Context, dashboard releaseupdate.CanaryDashboard, source distribution.Source, expectVersion string, latest []string, window time.Duration, out io.Writer) (int, error) {
run, err := readCanaryRun(ctx, dashboard, source)
if err != nil {
return 1, err
}
channel, pin, err := dashboard.Channel(ctx)
if err != nil {
return 1, err
}
fmt.Fprintf(out, "channel: %s\n", channelDisplay(channel, pin))
if expectVersion != "" && expectVersion != run.version {
return 1, fmt.Errorf("the canary runs %s, not %s", displayVersion(run.version), expectVersion)
}
if !run.found {
return 1, fmt.Errorf("no release %s in the source to promote", displayVersion(run.version))
}
now := time.Now().UTC()
signals, err := dashboard.Signals(ctx, run.installedAt, now, window)
if err != nil {
return 1, fmt.Errorf("read canary signals: %w", err)
}
verdict := releaseupdate.JudgeCanaryNow(run.version, now, window, signals)
printVerdict(out, run.version, verdict)
if verdict.Decision == releaseupdate.CanaryRollBack {
fmt.Fprintln(out, "refusing to promote: the canary's signals fail")
return 1, nil
}
if _, err := releaseupdate.Promote(ctx, source.Reference, source.Key, run.version, latest); err != nil {
return 1, fmt.Errorf("promote %s: %w", run.version, err)
}
fmt.Fprintf(out, "stable is %s\n", run.version)
return 0, nil
}
func runCanaryUnpin(args []string, out, stderr io.Writer) int {
flags := flag.NewFlagSet("release canary unpin", flag.ContinueOnError)
flags.SetOutput(stderr)
dashboardURL, tokenEnv := addCanaryFlags(flags)
if err := flags.Parse(args); err != nil {
return 2
}
dashboard, err := canaryDashboard(*dashboardURL, *tokenEnv)
if err != nil {
fmt.Fprintln(stderr, "release canary unpin:", err)
return 2
}
if err := dashboard.Follow(context.Background(), "next", ""); err != nil {
fmt.Fprintln(stderr, "release canary unpin:", err)
return 1
}
fmt.Fprintln(out, "the canary follows next again, with no pin")
return 0
}