From 5b5059615b0c613edddebf6eb445db0f52b4b10d Mon Sep 17 00:00:00 2001 From: Sam Catlow Date: Mon, 2 Nov 2026 01:15:29 +1100 Subject: [PATCH] feat(releaseupdate): judge the installed release, window from install (#1725) The canary window now counts from when a release is installed on the host (installed.json / installed_at in the update snapshot), not from the statement's issued_at, and 'release canary check' judges the release the canary actually runs via its exact-pin tag instead of the moving next tag. A newer release on next queues behind the one being judged instead of restarting the clock, so stable moves once per release. New subcommands: canary status (read-only state, signals, verdict), canary promote (promote now when hard checks pass), canary unpin (back to next), and start -force. A held/pinned canary is verified installable every check and a stranded pin fails the run red instead of looking green. Rollback pre-flight checks stable's zip and agent image digests before pinning. Relative signals are judged whenever both windows carry minimum data, so a breach rolls back mid-window; promote still needs the full window unless the operator promotes early (new promote.yml workflow_dispatch). The window is configurable via -window / CANARY_WINDOW. Also: dashboard canary status card (ui), and task canary:rehearse, a local end-to-end rehearsal of publish -> canary start -> status -> check -> promote against a throwaway registry and host. task check passes; rehearsal run green (REHEARSE_EXIT=0). --- .github/workflows/canary.yml | 16 +- .github/workflows/deploy.yml | 8 +- .github/workflows/promote.yml | 54 ++ RELEASING.md | 87 ++- Taskfile.yml | 7 + internal/app/archied/release.go | 7 +- internal/app/archied/release_canary.go | 640 ++++++++++++++++++--- internal/releaseupdate/canary.go | 53 +- internal/releaseupdate/canary_dashboard.go | 55 +- internal/releaseupdate/canary_test.go | 45 +- internal/releaseupdate/installable.go | 47 ++ internal/releaseupdate/service.go | 3 + internal/releaseupdate/transaction.go | 43 ++ internal/releaseupdate/transaction_test.go | 47 ++ internal/webui/api_chat.go | 17 + tools/canary-rehearse.sh | 352 ++++++++++++ ui/src/settings/CanaryStatusCard.vue | 184 ++++++ ui/src/settings/SystemStatusPage.vue | 2 + 18 files changed, 1487 insertions(+), 180 deletions(-) create mode 100644 .github/workflows/promote.yml create mode 100644 internal/releaseupdate/installable.go create mode 100755 tools/canary-rehearse.sh create mode 100644 ui/src/settings/CanaryStatusCard.vue diff --git a/.github/workflows/canary.yml b/.github/workflows/canary.yml index 4d973a74b..2f2a0848c 100644 --- a/.github/workflows/canary.yml +++ b/.github/workflows/canary.yml @@ -1,7 +1,9 @@ -# Every 15 minutes, checks how the newest release on the `next` channel is -# running on the canary host. If a hard check fails, the canary goes back to -# the stable release straight away. If the release stays healthy for 24 hours, -# it's promoted to stable. A rollback fails the run, so it shows up red. +# Every 15 minutes, checks how the release installed on the canary host is +# running, judged over the window counted from its install (-window, default +# 24h; CANARY_WINDOW overrides it). If a hard check fails, the canary goes +# back to the stable release straight away. If the release stays healthy for +# the window, it's promoted to stable. A rollback fails the run, so it shows +# up red. name: canary on: @@ -48,6 +50,12 @@ jobs: - name: Judge the canary env: ARCHIE_CANARY_TOKEN: ${{ secrets.CANARY_TOKEN }} + CANARY_WINDOW: ${{ vars.CANARY_WINDOW }} run: | + window_args=() + if [ -n "$CANARY_WINDOW" ]; then + window_args=(-window "$CANARY_WINDOW") + fi "$RUNNER_TEMP/archied" release canary check -dashboard "${{ vars.CANARY_URL }}" \ + "${window_args[@]}" \ -latest git.catlow.cloud/sam/archied,git.catlow.cloud/sam/archie-agent diff --git a/.github/workflows/deploy.yml b/.github/workflows/deploy.yml index c666b8514..87bb66355 100644 --- a/.github/workflows/deploy.yml +++ b/.github/workflows/deploy.yml @@ -418,8 +418,12 @@ jobs: "$api/repos/$repo/releases" >/dev/null echo "published release notes for $tag" - # The canary host follows the `next` channel. canary.yml then decides - # whether this release is promoted to stable or the canary is rolled back. + # The canary host follows the `next` channel. canary.yml then judges the + # release installed on the canary and decides whether it is promoted to + # stable or the canary is rolled back. The start is queue-safe: a release + # published while the canary still judges a newer one is queued behind + # it and picked up by the next check, so the running window isn't + # restarted under a different release. - name: Install the release on the canary if: vars.CANARY_URL != '' env: diff --git a/.github/workflows/promote.yml b/.github/workflows/promote.yml new file mode 100644 index 000000000..93d198493 --- /dev/null +++ b/.github/workflows/promote.yml @@ -0,0 +1,54 @@ +# Promotes the release the canary is running right now, without waiting out +# the judging window: the canary's signals must show no hard failure and no +# relative breach, otherwise the run refuses and fails. Dispatch it with the +# version the canary runs (`archied release canary status` shows it). +name: promote + +on: + workflow_dispatch: + inputs: + version: + description: "release the canary is expected to run; must match what it runs" + required: true + type: string + +concurrency: + group: promote + +permissions: + contents: read + packages: write + +jobs: + promote: + if: vars.CANARY_URL != '' + runs-on: self-hosted + steps: + - name: Checkout + uses: actions/checkout@v4 + with: + ref: main + + - name: Set up Go + uses: actions/setup-go@v5 + with: + go-version-file: go.mod + # The self-hosted runner keeps the Go caches on disk between jobs, and + # archiving them for the Actions cache takes minutes. + cache: false + + - name: Build archied + # The promote never serves the dashboard, so the build leaves it out. + run: go build -tags no_ui -o "$RUNNER_TEMP/archied" ./cmd/archied + + # Promotion retags stable and the images' latest in the registry. + - name: Log in to the Gitea registry + run: echo "${{ secrets.DEPLOY_TOKEN }}" | docker login git.catlow.cloud -u "${{ github.actor }}" --password-stdin + + - name: Promote the canary's release + env: + ARCHIE_CANARY_TOKEN: ${{ secrets.CANARY_TOKEN }} + run: | + "$RUNNER_TEMP/archied" release canary promote -dashboard "${{ vars.CANARY_URL }}" \ + -version "${{ inputs.version }}" \ + -latest git.catlow.cloud/sam/archied,git.catlow.cloud/sam/archie-agent diff --git a/RELEASING.md b/RELEASING.md index 9d15cc44f..538d42653 100644 --- a/RELEASING.md +++ b/RELEASING.md @@ -78,27 +78,63 @@ release rather than committing the fix on top of a tagged one. The canary is the production host, following `next`. Every quarter hour `canary.yml` runs `archied release canary check`, which reads the canary -through its dashboard API and judges the release on `next`: +through its dashboard API and judges **the release the canary actually runs**, +not the moving `next` tag: | Signal | Source | Fails when | | --- | --- | --- | -| Install outcome | `GET /api/chat/update` | the canary doesn't run the release (the watchdog rolled it back) | +| Install outcome | `GET /api/chat/update` | the canary doesn't run the judged release (the watchdog rolled it back) | | Readiness | `GET /health/detailed` | the rollup isn't `ok` | -| Alerts | `GET /api/alerts` | an open `action` alert was first raised after the release was issued | -| Error log | `GET /api/logs?level=error` | error lines per day exceed `2 × baseline + 10` | +| Alerts | `GET /api/alerts` | an open `action` alert was first raised after the release was installed | +| Error log | `GET /api/logs?level=error` | error lines per window exceed `2 × baseline + 10` | | Task outcomes | `GET /api/tasks` | the share of finished tasks that parked or died rose more than 10 points | -- **Control.** The canary's own 24 hours before the release was issued. With +- **Control.** The canary's own window before the release was installed. With one production host there's no simultaneous control. -- **Window.** 24 hours from issue, one cycle of schedules and traffic. The - first 30 minutes are for the install and restart. +- **Window.** 24 hours counted from the **install** on the canary, not from + the release's issue time: the host records it in `installed.json` + (`committed_at`) and reports it as `installed_at` in `GET /api/chat/update`. + One cycle of schedules and traffic. The first 30 minutes are for the install + and restart. `canary.yml` passes `-window` when the `CANARY_WINDOW` Actions + variable is set; the flag defaults to 24h. A host that doesn't report an + install time degrades to the release statement's issue time. - **Hard signals** (install, readiness, alerts) roll back at any check after - that. **Relative signals** (errors, tasks) are judged at the end of the - window. + that. **Relative signals** (errors, tasks) are judged against the baseline + whenever both windows carry enough data, so a breach rolls the release back + even mid-window; a full window with no failure promotes it. - **Too little data skips a signal** rather than failing it: fewer than 5 finished tasks in the window, or an error log that reaches back less than an hour into the baseline. A log read that stops short of a window is scaled to - a full day from what it covers. + a full window from what it covers. + +### Queueing + +`deploy.yml` installs each new release on the canary with +`archied release canary start -version V`, which first points the canary back +at `next` (clearing any rollback pin) and then installs `V` — unless the +canary is already judging a release newer than stable that isn't `V`, in which +case `V` is **queued** behind it and the canary keeps running what it judges. +`-force` installs regardless. After a promotion, the next `check` picks up the +newer `next` itself, so a queue drains one release per window. + +### Subcommands + +``` +archied release canary status read-only: channel/pin, installed release and + installed_at, stable/next, window progress, and + the verdict the current signals would give +archied release canary start point the canary at next and install -version, + queued behind a release being judged (-force to + override) +archied release canary check judge the installed release over -window + (default 24h): promote, pick up a newer next, + or roll back; a held canary is reported, and a + held pin that is no longer installable fails + the run red +archied release canary promote promote the installed release now if its + signals pass; refuses with exit 1 otherwise +archied release canary unpin return the canary to next from any pin +``` ### Promote @@ -106,22 +142,36 @@ A release that passes the window is promoted: `stable` is retagged to the `v` manifest. The signature covers the statement, not the tag, so the retag needs no signing key. The `archied` and `archie-agent` images' `:latest` move to their `:v` first, and the agent image must be the digest the -statement pins. Promotion refuses a prerelease and a release issued -before the one `stable` names, the same replay rule hosts enforce. The decision -and its reasons are in the `canary` run log. +statement pins. Promotion refuses a prerelease and a release issued before the +one `stable` names, the same replay rule hosts enforce. The decision and its +reasons are in the `canary` run log. + +Early promotion is a dispatch away: the **promote** workflow +(`.github/workflows/promote.yml`) takes a `version` input and runs +`archied release canary promote -version V` on the canary. It judges the +release the canary runs — the window doesn't matter for the decision — and +refuses (red run) on any hard failure or relative breach with enough data. ### Roll back -`stable` stays put, and the run turns red. The canary is pinned to the `stable` -version through its update settings and reinstalled through its dashboard. +`stable` stays put, and the run turns red. Before the canary is pinned to +`stable`, `check` verifies `stable` is actually installable — its statement's +zip blob must still be in the release repository and its pinned agent image +must still resolve (`releaseupdate.Installable`). If it isn't, the run fails +loudly and the canary stays on `next` rather than being stranded on a pin. +Otherwise the canary is pinned to the `stable` version through its update +settings and reinstalled through its dashboard. A release whose migrations differ from `stable`'s can't go back without restoring the pre-update database snapshot, which loses every write since. The canary is held on the failed release instead (pinned to it) and the fix goes -forward in the next release. The next release's deploy points the canary back at -`next`. +forward in the next release. The next release's deploy points the canary back +at `next`. -A pinned canary isn't judged, so a rollback is decided once. +A pinned canary isn't judged, so a rollback is decided once — but `check` +verifies the pin is still installable every run, so a stranded pin shows red +instead of green. `archied release canary unpin` returns the canary to `next` +by hand. ### Configuration @@ -129,6 +179,7 @@ A pinned canary isn't judged, so a rollback is decided once. | --- | --- | --- | | `CANARY_URL` | Actions variable | the canary's dashboard URL; unset disables the canary steps | | `CANARY_TOKEN` | Actions secret | a personal API token on the canary with instance admin | +| `CANARY_WINDOW` | Actions variable | optional `-window` override for `canary check`; unset is 24h | | `DEPLOY_TOKEN` | Actions secret | registry and repository write, for publishing, promoting and the release push | | `RELEASE_SIGNING_KEY` | Actions secret | the release signing seed | diff --git a/Taskfile.yml b/Taskfile.yml index 62cdb371f..2f600329e 100644 --- a/Taskfile.yml +++ b/Taskfile.yml @@ -221,6 +221,13 @@ tasks: cmds: - node e2e/run.cjs {{.CLI_ARGS}} + canary:rehearse: + # Local rehearsal only: spins up its own registry, database and host, and + # is deliberately absent from check and CI. + desc: Rehearse the release-canary loop on a throwaway local host (needs docker; never part of the gate) + cmds: + - sh tools/canary-rehearse.sh + build: desc: Build archie services and agent binaries deps: [ui] diff --git a/internal/app/archied/release.go b/internal/app/archied/release.go index a3ac97f01..4e2d82b00 100644 --- a/internal/app/archied/release.go +++ b/internal/app/archied/release.go @@ -19,8 +19,11 @@ const releaseUsage = `usage: archied release keygen -seed FILE write a new signing key's seed to FILE and print its public key publish sign a release and push it to a release source's next channel promote -version V point stable at a published release by hand - canary start point the canary at next and install a release on it - canary check judge the release on next and promote it or roll the canary back` + canary status show the canary's channel, installed release, window and verdict (read-only) + canary start point the canary at next and install a release on it, queued behind one being judged + canary check judge the release the canary runs: promote it, pick up a newer next, or roll back + canary promote promote the release the canary runs now, if its signals pass + canary unpin return the canary to the next channel from any pin` // RunRelease is the publisher's side of a release source: it makes the key // a source signs with and pushes signed releases to it. diff --git a/internal/app/archied/release_canary.go b/internal/app/archied/release_canary.go index 78ba9f0ad..fd12f0add 100644 --- a/internal/app/archied/release_canary.go +++ b/internal/app/archied/release_canary.go @@ -8,8 +8,10 @@ import ( "io" "os" "os/exec" + "strings" "time" + "golang.org/x/mod/semver" "oras.land/oras-go/v2/errdef" "github.com/samcharles93/archie-core/internal/domain/distribution" @@ -25,129 +27,510 @@ const canaryInstallWait = 20 * time.Minute const migrationsDir = "internal/infrastructure/postgres/migrations" func runReleaseCanary(args []string, out, stderr io.Writer) int { - if len(args) == 0 || (args[0] != "start" && args[0] != "check") { + if len(args) == 0 { fmt.Fprintln(stderr, releaseUsage) return 2 } - flags := flag.NewFlagSet("release canary "+args[0], flag.ContinueOnError) + switch args[0] { + case "status": + return runCanaryStatus(args[1:], out, stderr) + case "start": + return runCanaryStart(args[1:], out, stderr) + case "check": + return runCanaryCheck(args[1:], out, stderr) + case "promote": + return runCanaryPromote(args[1:], out, stderr) + case "unpin": + return runCanaryUnpin(args[1:], out, stderr) + default: + fmt.Fprintln(stderr, releaseUsage) + return 2 + } +} + +// addCanaryFlags registers the flags every canary subcommand takes. +func addCanaryFlags(flags *flag.FlagSet) (dashboardURL, tokenEnv *string) { + dashboardURL = flags.String("dashboard", "", "canary dashboard URL") + tokenEnv = flags.String("token-env", "ARCHIE_CANARY_TOKEN", "environment variable holding a dashboard API token for the canary") + return dashboardURL, tokenEnv +} + +// addSourceFlags registers the flags naming the release source. +func addSourceFlags(flags *flag.FlagSet) (reference, key *string) { + reference = flags.String("source", "", "OCI repository releases are published to; blank is the official releases") + key = flags.String("key", "", "base64 ed25519 public key the source signs with") + return reference, key +} + +func canaryDashboard(url, tokenEnv string) (releaseupdate.CanaryDashboard, error) { + if url == "" { + return releaseupdate.CanaryDashboard{}, errors.New("-dashboard is required") + } + return releaseupdate.CanaryDashboard{URL: url, Token: os.Getenv(tokenEnv)}, nil +} + +func canarySource(reference, key string) (distribution.Source, error) { + return distribution.Parse(reference, key, distribution.DefaultReleases()) +} + +// fetchChannel fetches a channel tag; found is false when the channel has no +// release yet. +func fetchChannel(ctx context.Context, source distribution.Source, tag string) (release releaseupdate.Release, found bool, err error) { + release, err = releaseupdate.Fetch(ctx, source, tag) + if errors.Is(err, errdef.ErrNotFound) { + return releaseupdate.Release{}, false, nil + } + return release, err == nil, err +} + +// publishedChannels are the releases the source currently publishes. +type publishedChannels struct { + next, stable releaseupdate.Release + hasNext, hasStable bool +} + +func fetchChannels(ctx context.Context, source distribution.Source) (publishedChannels, error) { + var channels publishedChannels + var err error + if channels.next, channels.hasNext, err = fetchChannel(ctx, source, "next"); err != nil { + return channels, err + } + if channels.stable, channels.hasStable, err = fetchChannel(ctx, source, "stable"); err != nil { + return channels, err + } + return channels, nil +} + +// canaryRun is the release the canary actually runs: its version, when it was +// installed and its fetched release statement. +type canaryRun struct { + version string + installedAt time.Time + release releaseupdate.Release + found bool +} + +// readCanaryRun reads what the canary runs. A host that doesn't record install +// times (it updated before they were tracked) degrades to the release +// statement's issue time. +func readCanaryRun(ctx context.Context, dashboard releaseupdate.CanaryDashboard, source distribution.Source) (canaryRun, error) { + var run canaryRun + version, installedAt, _, err := dashboard.Installed(ctx) + if err != nil { + return run, err + } + run.version, run.installedAt = version, installedAt + if version == "" { + return run, nil + } + release, found, err := fetchChannel(ctx, source, releaseupdate.Tag("exact-pin", version)) + if err != nil { + return run, err + } + run.release, run.found = release, found + if run.installedAt.IsZero() && found { + run.installedAt = release.Statement.IssuedAt + } + return run, nil +} + +// newerRelease reports whether version a is a strictly newer release than b. +func newerRelease(a, b string) bool { + return semver.Compare(canonVersion(a), canonVersion(b)) > 0 +} + +func canonVersion(version string) string { + return "v" + strings.TrimPrefix(version, "v") +} + +func displayVersion(version string) string { + if version == "" { + return "unknown" + } + return version +} + +func displayRelease(release releaseupdate.Release, found bool) string { + if !found { + return "none" + } + return release.Statement.Version +} + +func printVerdict(out io.Writer, version string, verdict releaseupdate.CanaryVerdict) { + fmt.Fprintf(out, "%s: %s\n", displayVersion(version), verdict.Decision) + for _, reason := range verdict.Reasons { + fmt.Fprintf(out, " %s\n", reason) + } +} + +func runCanaryStatus(args []string, out, stderr io.Writer) int { + flags := flag.NewFlagSet("release canary status", flag.ContinueOnError) flags.SetOutput(stderr) - dashboardURL := flags.String("dashboard", "", "canary dashboard URL") - tokenEnv := flags.String("token-env", "ARCHIE_CANARY_TOKEN", "environment variable holding a dashboard API token for the canary") - version := flags.String("version", "", "release to install (start)") - reference := flags.String("source", "", "OCI repository releases are published to; blank is the official releases (check)") - key := flags.String("key", "", "base64 ed25519 public key the source signs with (check)") - latest := flags.String("latest", "", "comma-separated image repositories whose v tag becomes latest on promotion (check)") - checkout := flags.String("checkout", ".", "repository checkout with release tags, to see whether a release migrated the database (check)") - if err := flags.Parse(args[1:]); err != nil { + dashboardURL, tokenEnv := addCanaryFlags(flags) + reference, key := addSourceFlags(flags) + window := flags.Duration("window", releaseupdate.CanaryWindow, "window a release is judged over, counted from its install") + if err := flags.Parse(args); err != nil { return 2 } - dashboard := releaseupdate.CanaryDashboard{URL: *dashboardURL, Token: os.Getenv(*tokenEnv)} - if *dashboardURL == "" { - fmt.Fprintln(stderr, "release canary: -dashboard is required") - return 2 - } - ctx := context.Background() - if args[0] == "start" { - if *version == "" { - fmt.Fprintln(stderr, "release canary start: -version is required") - return 2 - } - if err := startCanary(ctx, dashboard, *version); err != nil { - fmt.Fprintln(stderr, err) - return 1 - } - fmt.Fprintf(out, "canary follows next and runs %s\n", *version) - return 0 - } - source, err := distribution.Parse(*reference, *key, distribution.DefaultReleases()) + dashboard, err := canaryDashboard(*dashboardURL, *tokenEnv) if err != nil { - fmt.Fprintln(stderr, "release canary check:", err) + fmt.Fprintln(stderr, "release canary status:", err) return 2 } - rolledBack, err := checkCanary(ctx, dashboard, source, splitList(*latest), *checkout, out) + source, err := canarySource(*reference, *key) if err != nil { - fmt.Fprintln(stderr, err) - return 1 + fmt.Fprintln(stderr, "release canary status:", err) + return 2 } - if rolledBack { + if err := printCanaryStatus(context.Background(), dashboard, source, *window, out); err != nil { + fmt.Fprintln(stderr, "release canary status:", err) return 1 } return 0 } -func startCanary(ctx context.Context, dashboard releaseupdate.CanaryDashboard, version string) error { - if err := dashboard.Follow(ctx, "next", ""); err != nil { - return fmt.Errorf("point the canary at next: %w", err) - } - return dashboard.Install(ctx, version, canaryInstallWait) -} - -// checkCanary judges the release on the next channel once and acts on the -// result. It reports rolledBack so that the caller can fail the run. -func checkCanary(ctx context.Context, dashboard releaseupdate.CanaryDashboard, source distribution.Source, latest []string, checkout string, out io.Writer) (rolledBack bool, err error) { - next, err := releaseupdate.Fetch(ctx, source, "next") - if err != nil { - return false, err - } - stable, err := releaseupdate.Fetch(ctx, source, "stable") - hasStable := err == nil - if err != nil && !errors.Is(err, errdef.ErrNotFound) { - return false, err - } - release := next.Statement - if release.Prerelease || (hasStable && !release.IssuedAt.After(stable.Statement.IssuedAt)) { - fmt.Fprintf(out, "nothing to judge: next is %s and isn't a newer release than stable\n", release.Version) - return false, nil - } +// printCanaryStatus reports the canary's channel, installed release, the +// published channels, window progress and the verdict the current signals +// would give, without acting on anything. +func printCanaryStatus(ctx context.Context, dashboard releaseupdate.CanaryDashboard, source distribution.Source, window time.Duration, out io.Writer) error { channel, pin, err := dashboard.Channel(ctx) if err != nil { - return false, err + return err } - if channel != "next" { - fmt.Fprintf(out, "nothing to judge: the canary is held on %s %s\n", channel, pin) - return false, nil + channels, err := fetchChannels(ctx, source) + if err != nil { + return err + } + run, err := readCanaryRun(ctx, dashboard, source) + if err != nil { + return err + } + printCanaryState(out, channel, pin, run, channels, window) + return printCanaryVerdict(ctx, dashboard, run, window, out) +} + +func printCanaryState(out io.Writer, channel, pin string, run canaryRun, channels publishedChannels, window time.Duration) { + fmt.Fprintf(out, "channel: %s\n", channelDisplay(channel, pin)) + fmt.Fprintf(out, "installed: %s", displayVersion(run.version)) + if run.installedAt.IsZero() { + fmt.Fprintln(out, " (installed_at not reported)") + } else { + fmt.Fprintf(out, " since %s\n", run.installedAt.Format(time.RFC3339)) + } + fmt.Fprintf(out, "stable: %s\n", displayRelease(channels.stable, channels.hasStable)) + fmt.Fprintf(out, "next: %s\n", displayRelease(channels.next, channels.hasNext)) + if run.installedAt.IsZero() { + fmt.Fprintln(out, "window: unknown, no install time to count from") + } else { + elapsed := time.Since(run.installedAt) + fmt.Fprintf(out, "window: %s of %s elapsed since the install\n", elapsed.Round(time.Minute), window) + } +} + +func printCanaryVerdict(ctx context.Context, dashboard releaseupdate.CanaryDashboard, run canaryRun, window time.Duration, out io.Writer) error { + if run.installedAt.IsZero() { + fmt.Fprintln(out, "verdict: unknown, no install time to judge from") + return nil } now := time.Now().UTC() - signals, err := dashboard.Signals(ctx, release.IssuedAt, now) + signals, err := dashboard.Signals(ctx, run.installedAt, now, window) if err != nil { - return false, fmt.Errorf("read canary signals: %w", err) + return fmt.Errorf("read canary signals: %w", err) } - verdict := releaseupdate.JudgeCanary(release.Version, now.Sub(release.IssuedAt), signals) - fmt.Fprintf(out, "%s: %s\n", release.Version, verdict.Decision) + printSignals(out, signals) + verdict := releaseupdate.JudgeCanary(run.version, run.installedAt, now, window, signals) + fmt.Fprintf(out, "verdict on %s: %s (read-only)\n", displayVersion(run.version), verdict.Decision) for _, reason := range verdict.Reasons { fmt.Fprintf(out, " %s\n", reason) } - switch verdict.Decision { - case releaseupdate.CanaryPromote: - if _, err := releaseupdate.Promote(ctx, source.Reference, source.Key, release.Version, latest); err != nil { - return false, fmt.Errorf("promote %s: %w", release.Version, err) + return nil +} + +func printSignals(out io.Writer, s releaseupdate.CanarySignals) { + fmt.Fprintf(out, "signals: runs %q, ready=%t, new action alerts=%d\n", s.Installed, s.Ready, len(s.NewActionAlerts)) + fmt.Fprintf(out, " tasks finished %d, parked or died %d (baseline %d, %d)\n", + s.Canary.Finished, s.Canary.Bad, s.Baseline.Finished, s.Baseline.Bad) + fmt.Fprintf(out, " error lines %d, log covers %s (baseline %d, covers %s)\n", + s.Canary.Errors, s.Canary.Covered.Round(time.Minute), s.Baseline.Errors, s.Baseline.Covered.Round(time.Minute)) +} + +func channelDisplay(channel, pin string) string { + if pin != "" { + return channel + " (pinned to " + pin + ")" + } + return channel +} + +func runCanaryStart(args []string, out, stderr io.Writer) int { + flags := flag.NewFlagSet("release canary start", flag.ContinueOnError) + flags.SetOutput(stderr) + dashboardURL, tokenEnv := addCanaryFlags(flags) + reference, key := addSourceFlags(flags) + version := flags.String("version", "", "release to install on the canary") + force := flags.Bool("force", false, "install even when the canary is still judging a newer release") + if err := flags.Parse(args); err != nil { + return 2 + } + dashboard, err := canaryDashboard(*dashboardURL, *tokenEnv) + if err != nil { + fmt.Fprintln(stderr, "release canary start:", err) + return 2 + } + if *version == "" { + fmt.Fprintln(stderr, "release canary start: -version is required") + return 2 + } + source, err := canarySource(*reference, *key) + if err != nil { + fmt.Fprintln(stderr, "release canary start:", err) + return 2 + } + if err := startCanary(context.Background(), dashboard, source, *version, *force, out); err != nil { + fmt.Fprintln(stderr, "release canary start:", err) + return 1 + } + return 0 +} + +// startCanary points the canary at next — clearing any rollback pin — and +// installs version on it. A release published while the canary still judges a +// newer one queues behind it, so the running window isn't restarted under a +// different release. +func startCanary(ctx context.Context, dashboard releaseupdate.CanaryDashboard, source distribution.Source, version string, force bool, out io.Writer) error { + // The channel is read before following next: a canary already on next is + // judging the release it runs, and a new release queues behind that one. + channel, _, err := dashboard.Channel(ctx) + if err != nil { + return err + } + if err := dashboard.Follow(ctx, "next", ""); err != nil { + return fmt.Errorf("point the canary at next: %w", err) + } + if !force { + queued, err := queuedBehind(ctx, dashboard, source, channel, version, out) + if err != nil { + return err } - fmt.Fprintf(out, "stable is %s\n", release.Version) + if queued { + return nil + } + } + if err := dashboard.Install(ctx, version, canaryInstallWait); err != nil { + return err + } + fmt.Fprintf(out, "the canary follows next and runs %s\n", version) + return nil +} + +// queuedBehind reports whether version must wait: the canary already follows +// next, runs a release newer than stable (the one being judged) and version is +// a different release. +func queuedBehind(ctx context.Context, dashboard releaseupdate.CanaryDashboard, source distribution.Source, channel, version string, out io.Writer) (bool, error) { + if channel != "next" { return false, nil - case releaseupdate.CanaryRollBack: - return true, rollBackCanary(ctx, dashboard, release.Version, stable.Statement.Version, hasStable, checkout, out) + } + installed, _, _, err := dashboard.Installed(ctx) + if err != nil { + return false, err + } + if installed == "" || installed == version { + return false, nil + } + stable, hasStable, err := fetchChannel(ctx, source, "stable") + if err != nil { + return false, err + } + if hasStable && newerRelease(installed, stable.Statement.Version) { + fmt.Fprintf(out, "%s is queued behind %s, which the canary is still judging\n", version, installed) + return true, nil } return false, nil } -// rollBackCanary pins the canary to the stable release and reinstalls it. -// Going back past a migration needs the pre-update database snapshot, which -// loses every write since, so a release that migrated is held instead. -func rollBackCanary(ctx context.Context, dashboard releaseupdate.CanaryDashboard, failed, stable string, hasStable bool, checkout string, out io.Writer) error { - if !hasStable || migrated(ctx, checkout, stable, failed) { - if err := dashboard.Follow(ctx, "exact-pin", failed); err != nil { - return fmt.Errorf("hold the canary on %s: %w", failed, err) +func runCanaryCheck(args []string, out, stderr io.Writer) int { + flags := flag.NewFlagSet("release canary check", flag.ContinueOnError) + flags.SetOutput(stderr) + dashboardURL, tokenEnv := addCanaryFlags(flags) + reference, key := addSourceFlags(flags) + latest := flags.String("latest", "", "comma-separated image repositories whose v tag becomes latest on promotion") + checkout := flags.String("checkout", ".", "repository checkout with release tags, to see whether a release migrated the database") + window := flags.Duration("window", releaseupdate.CanaryWindow, "window the installed release is judged over, counted from its install") + if err := flags.Parse(args); err != nil { + return 2 + } + dashboard, err := canaryDashboard(*dashboardURL, *tokenEnv) + if err != nil { + fmt.Fprintln(stderr, "release canary check:", err) + return 2 + } + source, err := canarySource(*reference, *key) + if err != nil { + fmt.Fprintln(stderr, "release canary check:", err) + return 2 + } + code, err := checkCanary(context.Background(), dashboard, source, splitList(*latest), *checkout, *window, out) + if err != nil { + fmt.Fprintln(stderr, "release canary check:", err) + return 1 + } + return code +} + +// checkCanary judges the release the canary actually runs — not the moving +// next tag — and acts on the verdict. The exit code is 1 when the run should +// show red: a rollback happened, or the canary is held on a pin that can no +// longer be installed. +func checkCanary(ctx context.Context, dashboard releaseupdate.CanaryDashboard, source distribution.Source, latest []string, checkout string, window time.Duration, out io.Writer) (int, error) { + channels, err := fetchChannels(ctx, source) + if err != nil { + return 1, err + } + channel, pin, err := dashboard.Channel(ctx) + if err != nil { + return 1, err + } + if channel != "next" { + return heldCanary(ctx, source, channel, pin, out) + } + run, err := readCanaryRun(ctx, dashboard, source) + if err != nil { + return 1, err + } + if !judgeable(run, channels) { + fmt.Fprintf(out, "nothing to judge: the canary runs %s\n", displayVersion(run.version)) + return pickupNext(ctx, dashboard, channels, run.version, out) + } + return judgeInstalled(ctx, dashboard, source, run, channels, latest, checkout, window, out) +} + +// judgeable reports whether the installed release is one the canary judges: a +// published, non-prerelease release newer than stable. +func judgeable(run canaryRun, channels publishedChannels) bool { + if !run.found || run.release.Statement.Prerelease { + return false + } + return !channels.hasStable || newerRelease(run.version, channels.stable.Statement.Version) +} + +func judgeInstalled(ctx context.Context, dashboard releaseupdate.CanaryDashboard, source distribution.Source, run canaryRun, channels publishedChannels, latest []string, checkout string, window time.Duration, out io.Writer) (int, error) { + now := time.Now().UTC() + signals, err := dashboard.Signals(ctx, run.installedAt, now, window) + if err != nil { + return 1, fmt.Errorf("read canary signals: %w", err) + } + verdict := releaseupdate.JudgeCanary(run.version, run.installedAt, now, window, signals) + printVerdict(out, run.version, verdict) + switch verdict.Decision { + case releaseupdate.CanaryPromote: + return 0, promoteJudged(ctx, dashboard, source, run, channels, latest, out) + case releaseupdate.CanaryRollBack: + return 1, rollBackCanary(ctx, dashboard, source, run, channels, checkout, out) + default: + return 0, nil + } +} + +// promoteJudged moves stable to the judged release, then picks up a newer next +// queued behind it. +func promoteJudged(ctx context.Context, dashboard releaseupdate.CanaryDashboard, source distribution.Source, run canaryRun, channels publishedChannels, latest []string, out io.Writer) error { + if _, err := releaseupdate.Promote(ctx, source.Reference, source.Key, run.version, latest); err != nil { + return fmt.Errorf("promote %s: %w", run.version, err) + } + fmt.Fprintf(out, "stable is %s\n", run.version) + fresh := channels + fresh.stable, fresh.hasStable = run.release, true + _, err := pickupNext(ctx, dashboard, fresh, run.version, out) + return err +} + +// pickupNext installs the next release when it is newer than stable and the +// canary doesn't already run it. +func pickupNext(ctx context.Context, dashboard releaseupdate.CanaryDashboard, channels publishedChannels, installed string, out io.Writer) (int, error) { + if !channels.hasNext || channels.next.Statement.Version == installed { + return 0, nil + } + if channels.hasStable && !newerRelease(channels.next.Statement.Version, channels.stable.Statement.Version) { + return 0, nil + } + if err := dashboard.Install(ctx, channels.next.Statement.Version, canaryInstallWait); err != nil { + return 1, err + } + fmt.Fprintf(out, "the canary picks up %s from next\n", channels.next.Statement.Version) + return 0, nil +} + +// heldCanary reports a canary that isn't following next. Its pin must still be +// installable: a stranded pin fails the run instead of looking green. +func heldCanary(ctx context.Context, source distribution.Source, channel, pin string, out io.Writer) (int, error) { + if pin != "" { + err := checkPin(ctx, source, pin) + if errors.Is(err, errStrandedPin) { + fmt.Fprintf(out, "ERROR: the canary is held on %s %s, which is stranded: %v; unpin it or republish the release\n", channel, pin, err) + return 1, nil } - fmt.Fprintf(out, "%s changed the database schema or there's no stable release to return to: the canary is held on %s and stable is unchanged; fix forward\n", failed, failed) - return nil + if err != nil { + return 1, err + } + fmt.Fprintf(out, "held on %s %s, nothing to judge\n", channel, pin) + return 0, nil } - if err := dashboard.Follow(ctx, "exact-pin", stable); err != nil { - return fmt.Errorf("pin the canary to %s: %w", stable, err) + fmt.Fprintf(out, "held on %s, nothing to judge\n", channel) + return 0, nil +} + +// errStrandedPin marks a held pin that can no longer be installed. +var errStrandedPin = errors.New("pin is stranded") + +// checkPin fails when a held pin can no longer be installed: the release is +// gone from the source, or its artifacts are. +func checkPin(ctx context.Context, source distribution.Source, pin string) error { + release, found, err := fetchChannel(ctx, source, releaseupdate.Tag("exact-pin", pin)) + if err != nil { + return err } - if err := dashboard.Install(ctx, stable, canaryInstallWait); err != nil { - return fmt.Errorf("return the canary to %s: %w", stable, err) + if !found { + return fmt.Errorf("%w: no release %s in the source", errStrandedPin, releaseupdate.Tag("exact-pin", pin)) } - fmt.Fprintf(out, "the canary is back on %s and stable is unchanged\n", stable) + if err := releaseupdate.Installable(ctx, source, release); err != nil { + return fmt.Errorf("%w: %w", errStrandedPin, err) + } + return nil +} + +// rollBackCanary returns the canary to stable after a failed verdict. stable +// is checked installable before the canary is pinned to it: a pin whose +// artifacts are gone would strand the canary, so the run fails loudly and the +// canary stays on next. A release that changed the migrations is held on the +// failed version instead, because going back past a migration needs the +// pre-update database snapshot. +func rollBackCanary(ctx context.Context, dashboard releaseupdate.CanaryDashboard, source distribution.Source, run canaryRun, channels publishedChannels, checkout string, out io.Writer) error { + failed := run.version + if !channels.hasStable { + return holdCanary(ctx, dashboard, failed, "there's no stable release to return to", out) + } + stable := channels.stable + if err := releaseupdate.Installable(ctx, source, stable); err != nil { + return fmt.Errorf("stable %s is not installable, the canary stays on next: %w", stable.Statement.Version, err) + } + if migrated(ctx, checkout, stable.Statement.Version, failed) { + return holdCanary(ctx, dashboard, failed, failed+" changed the database schema", out) + } + if err := dashboard.Follow(ctx, "exact-pin", stable.Statement.Version); err != nil { + return fmt.Errorf("pin the canary to %s: %w", stable.Statement.Version, err) + } + if err := dashboard.Install(ctx, stable.Statement.Version, canaryInstallWait); err != nil { + return fmt.Errorf("return the canary to %s: %w", stable.Statement.Version, err) + } + fmt.Fprintf(out, "the canary is back on %s and stable is unchanged\n", stable.Statement.Version) + return nil +} + +func holdCanary(ctx context.Context, dashboard releaseupdate.CanaryDashboard, version, why string, out io.Writer) error { + if err := dashboard.Follow(ctx, "exact-pin", version); err != nil { + return fmt.Errorf("hold the canary on %s: %w", version, err) + } + fmt.Fprintf(out, "%s: the canary is held on %s and stable is unchanged; fix forward\n", why, version) return nil } @@ -158,3 +541,86 @@ func migrated(ctx context.Context, checkout, from, to string) bool { err := exec.CommandContext(ctx, "git", "-C", checkout, "diff", "--quiet", "v"+from, "v"+to, "--", migrationsDir).Run() return err != nil } + +func runCanaryPromote(args []string, out, stderr io.Writer) int { + flags := flag.NewFlagSet("release canary promote", flag.ContinueOnError) + flags.SetOutput(stderr) + dashboardURL, tokenEnv := addCanaryFlags(flags) + reference, key := addSourceFlags(flags) + latest := flags.String("latest", "", "comma-separated image repositories whose v tag becomes latest on promotion") + version := flags.String("version", "", "release expected on the canary; must match what it runs") + window := flags.Duration("window", releaseupdate.CanaryWindow, "window the signals are read over, counted from the install") + if err := flags.Parse(args); err != nil { + return 2 + } + dashboard, err := canaryDashboard(*dashboardURL, *tokenEnv) + if err != nil { + fmt.Fprintln(stderr, "release canary promote:", err) + return 2 + } + source, err := canarySource(*reference, *key) + if err != nil { + fmt.Fprintln(stderr, "release canary promote:", err) + return 2 + } + code, err := promoteCanaryNow(context.Background(), dashboard, source, *version, splitList(*latest), *window, out) + if err != nil { + fmt.Fprintln(stderr, "release canary promote:", err) + return 1 + } + return code +} + +// promoteCanaryNow promotes the release the canary runs right now, without +// waiting out the window: the signals must show no hard failure and no +// relative breach. It refuses, with exit 1, when anything fails. +func promoteCanaryNow(ctx context.Context, dashboard releaseupdate.CanaryDashboard, source distribution.Source, expectVersion string, latest []string, window time.Duration, out io.Writer) (int, error) { + run, err := readCanaryRun(ctx, dashboard, source) + if err != nil { + return 1, err + } + if expectVersion != "" && expectVersion != run.version { + return 1, fmt.Errorf("the canary runs %s, not %s", displayVersion(run.version), expectVersion) + } + if !run.found { + return 1, fmt.Errorf("no release %s in the source to promote", displayVersion(run.version)) + } + now := time.Now().UTC() + signals, err := dashboard.Signals(ctx, run.installedAt, now, window) + if err != nil { + return 1, fmt.Errorf("read canary signals: %w", err) + } + // A zero install time makes the judge skip its settle and window waits, + // so only the signals decide — exactly what promoting now asks for. + verdict := releaseupdate.JudgeCanary(run.version, time.Time{}, now, window, signals) + printVerdict(out, run.version, verdict) + if verdict.Decision == releaseupdate.CanaryRollBack { + fmt.Fprintln(out, "refusing to promote: the canary's signals fail") + return 1, nil + } + if _, err := releaseupdate.Promote(ctx, source.Reference, source.Key, run.version, latest); err != nil { + return 1, fmt.Errorf("promote %s: %w", run.version, err) + } + fmt.Fprintf(out, "stable is %s\n", run.version) + return 0, nil +} + +func runCanaryUnpin(args []string, out, stderr io.Writer) int { + flags := flag.NewFlagSet("release canary unpin", flag.ContinueOnError) + flags.SetOutput(stderr) + dashboardURL, tokenEnv := addCanaryFlags(flags) + if err := flags.Parse(args); err != nil { + return 2 + } + dashboard, err := canaryDashboard(*dashboardURL, *tokenEnv) + if err != nil { + fmt.Fprintln(stderr, "release canary unpin:", err) + return 2 + } + if err := dashboard.Follow(context.Background(), "next", ""); err != nil { + fmt.Fprintln(stderr, "release canary unpin:", err) + return 1 + } + fmt.Fprintln(out, "the canary follows next again, with no pin") + return 0 +} diff --git a/internal/releaseupdate/canary.go b/internal/releaseupdate/canary.go index 896da432f..0a337daca 100644 --- a/internal/releaseupdate/canary.go +++ b/internal/releaseupdate/canary.go @@ -6,14 +6,17 @@ import ( ) // The canary host installs each release from the next channel. Its behaviour -// in the 24 hours after a release is compared with its own behaviour in the -// 24 hours before, because there's only one production host to compare with. +// in the window after a release was installed is compared with its own +// behaviour in the window before, because there's only one production host to +// compare with. const ( - // CanaryWindow is how long a release runs on the canary before it can be - // promoted. A day covers one full cycle of schedules and traffic. + // CanaryWindow is the default window a release runs on the canary before + // it can be promoted. A day covers one full cycle of schedules and + // traffic. CanaryWindow = 24 * time.Hour // canarySettle is how long the install and restart get, counted from when - // the release was issued, before any check can fail the release. + // the release was installed on the canary, before any check can fail the + // release. canarySettle = 30 * time.Minute // minFinished is the fewest finished tasks needed to judge the park rate. minFinished = 5 @@ -47,7 +50,7 @@ type CanarySignals struct { Installed string Ready bool // NewActionAlerts are open action-severity alerts first raised since - // the release was issued. + // the release was installed. NewActionAlerts []string Baseline CanaryWindowSignals Canary CanaryWindowSignals @@ -59,11 +62,13 @@ type CanaryVerdict struct { Reasons []string } -// JudgeCanary decides what to do with a release, given how long ago it was -// issued. Once the release has settled, a failed hard check rolls it back at -// any time. Error and park rates are compared with the baseline only after the -// full window has passed. -func JudgeCanary(version string, elapsed time.Duration, s CanarySignals) CanaryVerdict { +// JudgeCanary decides what to do with a release, given when it was installed +// on the canary host. Once the release has settled, a failed hard check rolls +// it back at any time. Error and park rates are judged against the baseline +// whenever both windows carry enough data, so a breach rolls the release back +// even mid-window; a full window with no failure promotes it. +func JudgeCanary(version string, installedAt, now time.Time, window time.Duration, s CanarySignals) CanaryVerdict { + elapsed := now.Sub(installedAt) if elapsed < canarySettle { return CanaryVerdict{Decision: CanaryWait, Reasons: []string{"settling after install"}} } @@ -80,11 +85,8 @@ func JudgeCanary(version string, elapsed time.Duration, s CanarySignals) CanaryV if len(failed) > 0 { return CanaryVerdict{Decision: CanaryRollBack, Reasons: failed} } - if elapsed < CanaryWindow { - return CanaryVerdict{Decision: CanaryWait, Reasons: []string{fmt.Sprintf("%s of %s elapsed", elapsed.Round(time.Minute), CanaryWindow)}} - } var notes []string - if reason, ok := judgeErrors(s.Baseline, s.Canary); !ok { + if reason, ok := judgeErrors(s.Baseline, s.Canary, window); !ok { failed = append(failed, reason) } else { notes = append(notes, reason) @@ -97,32 +99,37 @@ func JudgeCanary(version string, elapsed time.Duration, s CanarySignals) CanaryV if len(failed) > 0 { return CanaryVerdict{Decision: CanaryRollBack, Reasons: failed} } + if elapsed < window { + reasons := append([]string{fmt.Sprintf("%s of %s elapsed since the install", elapsed.Round(time.Minute), window)}, notes...) + return CanaryVerdict{Decision: CanaryWait, Reasons: reasons} + } return CanaryVerdict{Decision: CanaryPromote, Reasons: notes} } // judgeErrors fails when the canary logged more than twice the baseline's -// error lines plus ten. Each count is scaled up to a full day when the log +// error lines plus ten. Each count is scaled up to a full window when the log // covers only part of its window. -func judgeErrors(baseline, canary CanaryWindowSignals) (string, bool) { +func judgeErrors(baseline, canary CanaryWindowSignals, window time.Duration) (string, bool) { if baseline.Covered < minLogCoverage || canary.Covered <= 0 { return "error log doesn't reach back over the baseline; error signal skipped", true } scale := func(w CanaryWindowSignals) float64 { - return float64(w.Errors) * float64(CanaryWindow) / float64(w.Covered) + return float64(w.Errors) * float64(window) / float64(w.Covered) } before, after := scale(baseline), scale(canary) limit := 2*before + 10 if after > limit { - return fmt.Sprintf("%.0f error log lines per day against a limit of %.0f (baseline %.0f)", after, limit, before), false + return fmt.Sprintf("%.0f error log lines per window against a limit of %.0f (baseline %.0f)", after, limit, before), false } - return fmt.Sprintf("%.0f error log lines per day, limit %.0f", after, limit), true + return fmt.Sprintf("%.0f error log lines per window, limit %.0f", after, limit), true } // judgeParks fails when the share of finished tasks that parked or died rose -// more than ten points over the baseline. +// more than ten points over the baseline. Both windows need the minimum +// finished tasks before the rates are compared. func judgeParks(baseline, canary CanaryWindowSignals) (string, bool) { - if canary.Finished < minFinished { - return fmt.Sprintf("%d finished tasks, fewer than %d; park signal skipped", canary.Finished, minFinished), true + if canary.Finished < minFinished || baseline.Finished < minFinished { + return fmt.Sprintf("%d finished tasks on the canary and %d in the baseline, fewer than %d each; park signal skipped", canary.Finished, baseline.Finished, minFinished), true } rate := func(w CanaryWindowSignals) float64 { if w.Finished == 0 { diff --git a/internal/releaseupdate/canary_dashboard.go b/internal/releaseupdate/canary_dashboard.go index fa3fbda7d..eda1ba39b 100644 --- a/internal/releaseupdate/canary_dashboard.go +++ b/internal/releaseupdate/canary_dashboard.go @@ -32,44 +32,47 @@ var ( const updateSettingsPath = "/api/control-plane/resources/update-settings" // Signals reads the canary's current state. Tasks and error lines are split -// into the day before the release was issued and the time since. -func (d CanaryDashboard) Signals(ctx context.Context, issued, now time.Time) (CanarySignals, error) { +// into the window before the release was installed on the canary and the time +// since. +func (d CanaryDashboard) Signals(ctx context.Context, installedAt, now time.Time, window time.Duration) (CanarySignals, error) { var s CanarySignals var err error - if s.Installed, _, err = d.installed(ctx); err != nil { + if s.Installed, _, _, err = d.Installed(ctx); err != nil { return s, err } if s.Ready, err = d.ready(ctx); err != nil { return s, err } - if s.NewActionAlerts, err = d.actionAlerts(ctx, issued); err != nil { + if s.NewActionAlerts, err = d.actionAlerts(ctx, installedAt); err != nil { return s, err } - if err := d.tasks(ctx, issued, &s); err != nil { + if err := d.tasks(ctx, installedAt, window, &s); err != nil { return s, err } - return s, d.errors(ctx, issued, now, &s) + return s, d.errors(ctx, installedAt, now, window, &s) } -// installed returns the daemon version the canary runs, and the raw update -// snapshot it offers, which an install request must send back unchanged. -func (d CanaryDashboard) installed(ctx context.Context) (string, json.RawMessage, error) { +// Installed returns the daemon version the canary runs, when that release was +// installed on the canary host (zero when the host doesn't report it), and +// the raw update snapshot it offers, which an install request must send back +// unchanged. +func (d CanaryDashboard) Installed(ctx context.Context) (version string, installedAt time.Time, snapshot json.RawMessage, err error) { var body struct { Snapshot json.RawMessage `json:"snapshot"` } if _, err := d.call(ctx, http.MethodGet, "/api/chat/update", nil, &body); err != nil { - return "", nil, err + return "", time.Time{}, nil, err } - var snapshot Snapshot - if err := json.Unmarshal(body.Snapshot, &snapshot); err != nil { - return "", nil, fmt.Errorf("decode update snapshot: %w", err) + var snap Snapshot + if err := json.Unmarshal(body.Snapshot, &snap); err != nil { + return "", time.Time{}, nil, fmt.Errorf("decode update snapshot: %w", err) } - for _, c := range snapshot.Components { + for _, c := range snap.Components { if c.ID == "daemon" { - return c.Installed, body.Snapshot, nil + return c.Installed, snap.InstalledAt, body.Snapshot, nil } } - return "", body.Snapshot, errors.New("canary reports no daemon version") + return "", time.Time{}, body.Snapshot, errors.New("canary reports no daemon version") } func (d CanaryDashboard) ready(ctx context.Context) (bool, error) { @@ -104,8 +107,8 @@ func (d CanaryDashboard) actionAlerts(ctx context.Context, since time.Time) ([]s // tasks reads tasks, newest update first, until it reaches tasks older than // the baseline window. -func (d CanaryDashboard) tasks(ctx context.Context, issued time.Time, s *CanarySignals) error { - from := issued.Add(-CanaryWindow) +func (d CanaryDashboard) tasks(ctx context.Context, at time.Time, window time.Duration, s *CanarySignals) error { + from := at.Add(-window) cursor := "" for range 50 { var page []struct { @@ -125,7 +128,7 @@ func (d CanaryDashboard) tasks(ctx context.Context, issued time.Time, s *CanaryS continue } w := &s.Canary - if t.UpdatedAt.Before(issued) { + if t.UpdatedAt.Before(at) { w = &s.Baseline } w.Finished++ @@ -144,7 +147,7 @@ func (d CanaryDashboard) tasks(ctx context.Context, issued time.Time, s *CanaryS // errors counts error-level lines in each window. A log read returns a // limited number of lines, so a truncated read only reaches back as far as // its oldest line. -func (d CanaryDashboard) errors(ctx context.Context, issued, now time.Time, s *CanarySignals) error { +func (d CanaryDashboard) errors(ctx context.Context, at, now time.Time, window time.Duration, s *CanarySignals) error { var body struct { Entries []struct { Time time.Time `json:"time"` @@ -154,24 +157,24 @@ func (d CanaryDashboard) errors(ctx context.Context, issued, now time.Time, s *C if _, err := d.call(ctx, http.MethodGet, "/api/logs?level=error&limit=2000", nil, &body); err != nil { return err } - from := issued.Add(-CanaryWindow) + from := at.Add(-window) if body.Truncated { from = now if len(body.Entries) > 0 { - from = later(body.Entries[0].Time, issued.Add(-CanaryWindow)) + from = later(body.Entries[0].Time, at.Add(-window)) } } for _, e := range body.Entries { switch { case e.Time.Before(from): - case e.Time.Before(issued): + case e.Time.Before(at): s.Baseline.Errors++ default: s.Canary.Errors++ } } - s.Baseline.Covered = max(issued.Sub(from), 0) - s.Canary.Covered = now.Sub(later(from, issued)) + s.Baseline.Covered = max(at.Sub(from), 0) + s.Canary.Covered = now.Sub(later(from, at)) return nil } @@ -232,7 +235,7 @@ func (d CanaryDashboard) Install(ctx context.Context, version string, wait time. deadline := time.Now().Add(wait) requested := false for { - installed, snapshot, err := d.installed(ctx) + installed, _, snapshot, err := d.Installed(ctx) switch { case err != nil: // The dashboard drops off while the install restarts it. diff --git a/internal/releaseupdate/canary_test.go b/internal/releaseupdate/canary_test.go index 5fc8b53b7..ade975d91 100644 --- a/internal/releaseupdate/canary_test.go +++ b/internal/releaseupdate/canary_test.go @@ -5,10 +5,11 @@ import ( "time" ) -// A release promotes only after a full window with no hard failure and no -// relative signal past its threshold; any hard failure after settling rolls -// it back at once. +// A release promotes only after a full window counted from its install with +// no hard failure and no relative signal past its threshold; any failure after +// settling rolls it back at once, even mid-window. func TestJudgeCanary(t *testing.T) { + installed := time.Date(2026, 1, 2, 12, 0, 0, 0, time.UTC) healthy := func() CanarySignals { return CanarySignals{ Installed: "1.1.0", Ready: true, @@ -23,31 +24,39 @@ func TestJudgeCanary(t *testing.T) { } cases := []struct { name string - elapsed time.Duration + elapsed time.Duration // now minus the install time + window time.Duration signals CanarySignals want CanaryDecision }{ - {"settling ignores a failed check", 10 * time.Minute, with(func(s *CanarySignals) { s.Ready = false }), CanaryWait}, - {"healthy mid-window waits", 2 * time.Hour, healthy(), CanaryWait}, - {"healthy at the window promotes", CanaryWindow, healthy(), CanaryPromote}, - {"the watchdog rolled the install back", 2 * time.Hour, with(func(s *CanarySignals) { s.Installed = "1.0.0" }), CanaryRollBack}, - {"readiness degraded", 2 * time.Hour, with(func(s *CanarySignals) { s.Ready = false }), CanaryRollBack}, - {"a new action alert", 2 * time.Hour, with(func(s *CanarySignals) { s.NewActionAlerts = []string{"queue stalled"} }), CanaryRollBack}, - {"errors past twice the baseline plus ten", CanaryWindow, with(func(s *CanarySignals) { s.Canary.Errors = 31 }), CanaryRollBack}, - {"errors at the limit", CanaryWindow, with(func(s *CanarySignals) { s.Canary.Errors = 30 }), CanaryPromote}, - {"errors scaled from a partly covered log", CanaryWindow, with(func(s *CanarySignals) { + {"settling counts from the install", 10 * time.Minute, CanaryWindow, with(func(s *CanarySignals) { s.Ready = false }), CanaryWait}, + {"healthy mid-window waits", 2 * time.Hour, CanaryWindow, healthy(), CanaryWait}, + {"healthy at the window promotes", CanaryWindow, CanaryWindow, healthy(), CanaryPromote}, + {"healthy at a custom window promotes", 12 * time.Hour, 12 * time.Hour, healthy(), CanaryPromote}, + {"the watchdog rolled the install back", 2 * time.Hour, CanaryWindow, with(func(s *CanarySignals) { s.Installed = "1.0.0" }), CanaryRollBack}, + {"readiness degraded", 2 * time.Hour, CanaryWindow, with(func(s *CanarySignals) { s.Ready = false }), CanaryRollBack}, + {"a new action alert", 2 * time.Hour, CanaryWindow, with(func(s *CanarySignals) { s.NewActionAlerts = []string{"queue stalled"} }), CanaryRollBack}, + {"errors past twice the baseline plus ten", CanaryWindow, CanaryWindow, with(func(s *CanarySignals) { s.Canary.Errors = 31 }), CanaryRollBack}, + {"errors at the limit", CanaryWindow, CanaryWindow, with(func(s *CanarySignals) { s.Canary.Errors = 30 }), CanaryPromote}, + {"errors scaled from a partly covered log", CanaryWindow, CanaryWindow, with(func(s *CanarySignals) { s.Canary.Errors, s.Canary.Covered = 8, 6*time.Hour }), CanaryRollBack}, - {"a baseline the log doesn't reach skips errors", CanaryWindow, with(func(s *CanarySignals) { + {"a baseline the log doesn't reach skips errors", CanaryWindow, CanaryWindow, with(func(s *CanarySignals) { s.Baseline.Covered, s.Canary.Errors = 30*time.Minute, 500 }), CanaryPromote}, - {"park rate up more than ten points", CanaryWindow, with(func(s *CanarySignals) { s.Canary.Bad = 5 }), CanaryRollBack}, - {"park rate up ten points", CanaryWindow, with(func(s *CanarySignals) { s.Canary.Bad = 4 }), CanaryPromote}, - {"too few tasks skips the park rate", CanaryWindow, with(func(s *CanarySignals) { s.Canary = CanaryWindowSignals{Finished: 4, Bad: 4, Covered: CanaryWindow} }), CanaryPromote}, + {"a mid-window error breach rolls back early", 2 * time.Hour, CanaryWindow, with(func(s *CanarySignals) { s.Canary.Errors = 31 }), CanaryRollBack}, + {"park rate up more than ten points", CanaryWindow, CanaryWindow, with(func(s *CanarySignals) { s.Canary.Bad = 5 }), CanaryRollBack}, + {"park rate up ten points", CanaryWindow, CanaryWindow, with(func(s *CanarySignals) { s.Canary.Bad = 4 }), CanaryPromote}, + {"too few canary tasks skips the park rate", CanaryWindow, CanaryWindow, with(func(s *CanarySignals) { s.Canary = CanaryWindowSignals{Finished: 4, Bad: 4, Covered: CanaryWindow} }), CanaryPromote}, + {"a thin baseline skips the park rate", CanaryWindow, CanaryWindow, with(func(s *CanarySignals) { + s.Baseline.Finished, s.Baseline.Bad = 2, 0 + s.Canary.Bad = 5 + }), CanaryPromote}, + {"a mid-window park breach rolls back early", 2 * time.Hour, CanaryWindow, with(func(s *CanarySignals) { s.Canary.Bad = 5 }), CanaryRollBack}, } for _, tc := range cases { t.Run(tc.name, func(t *testing.T) { - got := JudgeCanary("1.1.0", tc.elapsed, tc.signals) + got := JudgeCanary("1.1.0", installed, installed.Add(tc.elapsed), tc.window, tc.signals) if got.Decision != tc.want { t.Fatalf("decision = %s, want %s (%v)", got.Decision, tc.want, got.Reasons) } diff --git a/internal/releaseupdate/installable.go b/internal/releaseupdate/installable.go new file mode 100644 index 000000000..526a8a9e0 --- /dev/null +++ b/internal/releaseupdate/installable.go @@ -0,0 +1,47 @@ +package releaseupdate + +import ( + "context" + "fmt" + + digest "github.com/opencontainers/go-digest" + ocispec "github.com/opencontainers/image-spec/specs-go/v1" + "oras.land/oras-go/v2/registry" + + "github.com/samcharles93/archie-core/internal/domain/distribution" +) + +// Installable reports whether a release can still be installed from source: +// the zip blob its statement names must be present in the release repository +// and the agent image it pins must still resolve. A canary must never be +// pinned to a release whose artifacts are gone, or it would be stranded on a +// version it can never install again. +func Installable(ctx context.Context, source distribution.Source, release Release) error { + zipDigest, err := digest.Parse(release.Statement.Zip.Digest) + if err != nil { + return fmt.Errorf("zip digest %q: %w", release.Statement.Zip.Digest, err) + } + repo, err := pushRepository(source.Reference) + if err != nil { + return err + } + exists, err := repo.Exists(ctx, ocispec.Descriptor{Digest: zipDigest}) + if err != nil { + return fmt.Errorf("resolve zip blob %s in %s: %w", release.Statement.Zip.Digest, source.Reference, err) + } + if !exists { + return fmt.Errorf("zip blob %s (%s) is absent from %s", release.Statement.Zip.Digest, release.Statement.Zip.Name, source.Reference) + } + image, err := registry.ParseReference(release.Statement.AgentImage) + if err != nil { + return fmt.Errorf("agent image %q: %w", release.Statement.AgentImage, err) + } + imageRepo, err := pushRepository(image.Registry + "/" + image.Repository) + if err != nil { + return err + } + if _, err := imageRepo.Resolve(ctx, image.Reference); err != nil { + return fmt.Errorf("agent image %s: %w", release.Statement.AgentImage, err) + } + return nil +} diff --git a/internal/releaseupdate/service.go b/internal/releaseupdate/service.go index 23b1b0b97..464f8ae0c 100644 --- a/internal/releaseupdate/service.go +++ b/internal/releaseupdate/service.go @@ -44,6 +44,9 @@ type Component struct { type Snapshot struct { Components []Component Deferred bool + // InstalledAt is when the installed release was committed on this host, + // from its installed.json. Zero when the host doesn't record one. + InstalledAt time.Time `json:"installed_at"` } func (s Snapshot) Available() []Component { diff --git a/internal/releaseupdate/transaction.go b/internal/releaseupdate/transaction.go index 30c52f52f..47f787a9b 100644 --- a/internal/releaseupdate/transaction.go +++ b/internal/releaseupdate/transaction.go @@ -40,6 +40,7 @@ const ( journalDir = "transaction" journalFile = "journal.json" snapshotFile = "db.pre-update.dump" + installedFile = "installed.json" lockWait = 60 * time.Second ) @@ -126,6 +127,39 @@ type journal struct { // JournalPath is where an unfinished transaction lives. func JournalPath(stateDir string) string { return filepath.Join(stateDir, journalDir) } +// installedRecord is the host's durable note of which release it last +// committed (or rolled back to) and when, so a canary can count its window +// from the actual install rather than the release statement. +type installedRecord struct { + Version string `json:"version"` + CommittedAt time.Time `json:"committed_at"` +} + +// recordInstall writes installed.json atomically: the file is the source of +// truth for when the running release was installed. +func recordInstall(stateDir, version string, at time.Time) error { + data, err := json.Marshal(installedRecord{Version: version, CommittedAt: at.UTC()}) + if err != nil { + return err + } + return fsutil.WriteFileAtomic(filepath.Join(stateDir, installedFile), data, 0o600) +} + +// LastInstall reads the host's installed record. ok is false when the file is +// missing or malformed, as on a host that updated before install times were +// recorded. +func LastInstall(stateDir string) (version string, at time.Time, ok bool) { + data, err := os.ReadFile(filepath.Join(stateDir, installedFile)) + if err != nil { + return "", time.Time{}, false + } + var record installedRecord + if err := json.Unmarshal(data, &record); err != nil || record.Version == "" || record.CommittedAt.IsZero() { + return "", time.Time{}, false + } + return record.Version, record.CommittedAt, true +} + // Run drives the transaction in the journal to an outcome. recovering is set // on boot: it never waits for the lock, and a transaction that was prepared // but never started is abandoned rather than started unattended. @@ -182,6 +216,9 @@ func (t *Transaction) promote(ctx context.Context, j journal) error { if err := t.setPhase(&j, phaseCommitted); err != nil { return err } + if err := recordInstall(t.StateDir, j.Version, time.Now().UTC()); err != nil { + return fmt.Errorf("record the install: %w", err) + } return t.remove() } @@ -252,6 +289,12 @@ func (t *Transaction) rollback(ctx context.Context, j journal) error { if err := t.setPhase(&j, phaseRolledBack); err != nil { return t.incomplete(err) } + // The host is back on the previous release; installed.json follows it. + if version := j.Previous[ComponentDaemon]; version != "" { + if err := recordInstall(t.StateDir, version, time.Now().UTC()); err != nil { + return t.incomplete(err) + } + } if err := t.startUnits(ctx, j, started); err != nil { return t.incomplete(err) } diff --git a/internal/releaseupdate/transaction_test.go b/internal/releaseupdate/transaction_test.go index e488e632e..a7b01d5e9 100644 --- a/internal/releaseupdate/transaction_test.go +++ b/internal/releaseupdate/transaction_test.go @@ -21,6 +21,7 @@ import ( digest "github.com/opencontainers/go-digest" ocispec "github.com/opencontainers/image-spec/specs-go/v1" + "github.com/samcharles93/archie-core/internal/buildinfo" "github.com/samcharles93/archie-core/internal/domain/distribution" "github.com/samcharles93/archie-core/internal/infrastructure/signedoci" ) @@ -146,6 +147,11 @@ func TestTransaction(t *testing.T) { // the env file beside its config; pin both to this test's own files so // the result never depends on the machine's real archie config. t.Setenv(DatabasePasswordEnv, "test-password") + // Stamp this binary as a release, as a host installed from one is, so the + // journal's previous daemon version is a real version. + stamped := buildinfo.Version + buildinfo.Version = "1.0.0" + defer func() { buildinfo.Version = stamped }() configPath := filepath.Join(t.TempDir(), "config.toml") if err := os.WriteFile(configPath, []byte("database_url = \"postgres://archie@/archie?host=/tmp/archie-test\"\n"), 0o600); err != nil { t.Fatal(err) @@ -257,10 +263,51 @@ func TestTransaction(t *testing.T) { found && (got.HealthCheck != tt.want.report || got.RolledBack != (tt.want.report == "failed") || got.RestoredDatabase != tt.want.restoredDatabase) { t.Errorf("report = %+v (found %v), want %+v", got, found, tt.want) } + // A transaction that ran to an outcome records which release the + // host now runs and when it was installed. + if tt.prepare == "" && tt.crashAt == "" { + if version, at, ok := LastInstall(tx.StateDir); !ok || version != tt.want.version || at.IsZero() { + t.Errorf("installed.json after the transaction = %q, %v, %v; want %s", version, at, ok, tt.want.version) + } + } }) } } +// installed.json is the host's durable record of which release it last +// committed or rolled back to, and when. +func TestLastInstall(t *testing.T) { + dir := t.TempDir() + if _, _, ok := LastInstall(dir); ok { + t.Fatal("a missing installed.json reports ok") + } + at := time.Date(2026, 1, 2, 3, 4, 5, 0, time.UTC) + if err := recordInstall(dir, "1.2.3", at); err != nil { + t.Fatal(err) + } + var raw map[string]any + data, err := os.ReadFile(filepath.Join(dir, installedFile)) + if err != nil { + t.Fatal(err) + } + if err := json.Unmarshal(data, &raw); err != nil { + t.Fatalf("installed.json is not JSON: %v", err) + } + if raw["version"] != "1.2.3" || raw["committed_at"] != "2026-01-02T03:04:05Z" { + t.Fatalf("installed.json = %v, want version 1.2.3 and committed_at 2026-01-02T03:04:05Z", raw) + } + version, got, ok := LastInstall(dir) + if !ok || version != "1.2.3" || !got.Equal(at) { + t.Fatalf("LastInstall = %q, %v, %v; want 1.2.3, %v, true", version, got, ok, at) + } + if err := os.WriteFile(filepath.Join(dir, installedFile), []byte("{not json"), 0o600); err != nil { + t.Fatal(err) + } + if _, _, ok := LastInstall(dir); ok { + t.Fatal("a malformed installed.json reports ok") + } +} + // releaseHandler serves one release's zip and SHA256SUMS the way GitHub // release downloads are laid out. func releaseHandler(t *testing.T, candidate string, corrupt bool) (http.Handler, ed25519.PublicKey) { diff --git a/internal/webui/api_chat.go b/internal/webui/api_chat.go index 70b2e074f..ae25ce1d9 100644 --- a/internal/webui/api_chat.go +++ b/internal/webui/api_chat.go @@ -6,6 +6,7 @@ import ( "errors" "fmt" "net/http" + "os" "path/filepath" "strings" "sync" @@ -15,6 +16,7 @@ import ( "github.com/samcharles93/archie-core/internal/domain/lifecycle" "github.com/samcharles93/archie-core/internal/domain/messaging" + "github.com/samcharles93/archie-core/internal/infrastructure/configuration" "github.com/samcharles93/archie-core/internal/releaseupdate" ) @@ -513,6 +515,11 @@ func (s *Server) handleChatUpdate(w http.ResponseWriter, r *http.Request) { writeUpdateCheckError(w, err) return } + // The snapshot carries when the installed release was committed on this + // host, so a canary can count its window from the actual install. + if _, at, ok := releaseupdate.LastInstall(updateStateDir()); ok { + snapshot.InstalledAt = at + } writeJSON(w, map[string]any{ "snapshot": snapshot, "available": snapshot.Available(), @@ -520,6 +527,16 @@ func (s *Server) handleChatUpdate(w http.ResponseWriter, r *http.Request) { }) } +// updateStateDir resolves the host's update state directory the same way the +// daemon does, so the dashboard reads the installed.json its transactions +// write. +func updateStateDir() string { + if dir := os.Getenv("ARCHIE_UPDATE_STATE_DIR"); dir != "" { + return dir + } + return configuration.DefaultUpdateStateDir() +} + func (s *Server) handleChatUpdateDefer(w http.ResponseWriter, r *http.Request) { chat, ok := s.chatReady(w) if !ok { diff --git a/tools/canary-rehearse.sh b/tools/canary-rehearse.sh new file mode 100755 index 000000000..50851bd18 --- /dev/null +++ b/tools/canary-rehearse.sh @@ -0,0 +1,352 @@ +#!/bin/sh +# Rehearse the release-canary loop against a throwaway local host. Nothing +# here touches production: a local OCI registry (docker run registry:2 on a +# loopback port), a throwaway signing key, a test release published to that +# registry (a stamped archied binary and a real agent image in a proper +# distribution zip), and a local stack (state store, gateway, archied, +# archie-ui) with its own config, state dir and ports. The loop's commands -- +# release canary start, release canary status, release canary check and +# release canary promote -- are then run against that host, each step printed. +# The local host has no systemd, so once the install has switched the binary +# the script restarts archied itself, the way an unmanaged host's operator +# would. +# +# A step that fails is printed and makes the run exit nonzero, so a rehearsal +# never looks greener than it was. The only hard precondition is docker: +# without it there is no registry, PostgreSQL or agent image. +# +# Never part of the gate: task check and CI do not call this. +set -eu + +repo=$(CDPATH= cd -- "$(dirname -- "$0")/.." && pwd) +work=$(mktemp -d "${TMPDIR:-/tmp}/archie-canary-rehearse.XXXXXX") +name=$(basename "$work") +bin=$work/bin +logs=$work/logs + +pids="" +containers="" +keep=0 # set when a step fails: the workdir is kept for inspection +cleanup() { + trap - EXIT INT TERM + for pid in $pids; do kill -- "-$pid" 2>/dev/null || kill "$pid" 2>/dev/null || true; done + for c in $containers; do docker rm -f "$c" >/dev/null 2>&1 || true; done + if [ "$keep" -gt 0 ]; then + echo "canary-rehearse: keeping $work for inspection" >&2 + else + rm -rf "$work" + fi +} +trap cleanup EXIT INT TERM + +die() { + echo "canary-rehearse: $*" >&2 + exit 1 +} + +# --- preflight --------------------------------------------------------------- + +for tool in go curl python3 zip sha256sum timeout setsid openssl docker; do + command -v "$tool" >/dev/null 2>&1 || die "$tool is required but not on PATH" +done +if ! docker info >/dev/null 2>&1; then + die "docker is unavailable (docker info failed): the rehearsal needs it for the local OCI registry and PostgreSQL. Start the docker daemon or skip task canary:rehearse" +fi + +release_version=${CANARY_REHEARSE_VERSION:-9.9.9-canary.1} +host_version="0.0.0-rehearse" + +# Seven loopback ports, picked by binding port 0. +set -- $(python3 -c ' +import socket +ports = [] +while len(ports) < 7: + s = socket.socket() + s.bind(("127.0.0.1", 0)) + ports.append(s.getsockname()[1]) + s.close() +print(" ".join(map(str, ports))) +') +reg_port=$1 pg_port=$2 web_port=$3 state_port=$4 gw_port=$5 daemon_port=$6 health_port=$7 + +# --- registry and PostgreSQL ------------------------------------------------- + +echo "== starting local OCI registry on 127.0.0.1:$reg_port and PostgreSQL on 127.0.0.1:$pg_port" +docker run -d --name "$name-registry" -p "127.0.0.1:$reg_port:5000" registry:2 >/dev/null || + die "could not start registry:2 (is the image pullable?)" +containers="$containers $name-registry" +docker run -d --name "$name-pg" \ + -e POSTGRES_USER=archie -e POSTGRES_PASSWORD=archie -e POSTGRES_DB=archie \ + -p "127.0.0.1:$pg_port:5432" postgres:18 >/dev/null || + die "could not start postgres:18 (is the image pullable?)" +containers="$containers $name-pg" + +# --- build the host's binaries (no_ui: the rehearsal never serves the SPA) ---- + +echo "== building archie-state-store, archie-gateway, archied, archie-ui (no_ui)" +mkdir -p "$bin" "$logs" +ldflags="-X github.com/samcharles93/archie-core/internal/installtype.buildType=binary -X github.com/samcharles93/archie-core/internal/buildinfo.Version=$host_version" +for svc in archie-state-store archie-gateway archied archie-ui; do + go -C "$repo" build -tags no_ui -trimpath -ldflags "$ldflags" -o "$bin/$svc" "./cmd/$svc" || + die "go build ./cmd/$svc failed" +done + +wait_http() { # url label tries + i=0 + while [ "$i" -lt "${3:-30}" ]; do + if curl -fsS --max-time 2 "$1" >/dev/null 2>&1; then return 0; fi + i=$((i + 1)); sleep 1 + done + die "$2 did not answer at $1 within ${3:-30}s" +} +wait_http "http://127.0.0.1:$reg_port/v2/" "the local registry" 30 + +i=0 +while ! docker exec "$name-pg" pg_isready -h 127.0.0.1 -p 5432 -U archie >/dev/null 2>&1; do + i=$((i + 1)) + [ "$i" -gt 90 ] && die "PostgreSQL did not become ready within 90s" + sleep 1 +done + +# --- throwaway signing key and a test release -------------------------------- + +echo "== generating a throwaway release signing key" +pub=$("$bin/archied" release keygen -seed "$work/seed") || die "release keygen failed" +[ -n "$pub" ] || die "release keygen printed no public key" +seed=$(cat "$work/seed") + +# A real agent image in the local registry, so the install's image pull by +# digest succeeds. The Dockerfile stamps the agent with the release version. +echo "== building and pushing the archie-agent test image (this takes a few minutes)" +docker build -q -f "$repo/Dockerfile" --build-arg RUNTIME_VERSION="$release_version" \ + -t "127.0.0.1:$reg_port/archie-agent:v$release_version" "$repo" >/dev/null || + die "docker build of the agent image failed" +docker push -q "127.0.0.1:$reg_port/archie-agent:v$release_version" >/dev/null || + die "docker push of the agent image failed" +agent_image=$(docker image inspect -f '{{range .RepoDigests}}{{println .}}{{end}}' \ + "127.0.0.1:$reg_port/archie-agent:v$release_version" | grep "^127\.0\.0\.1:$reg_port/" | head -1) +[ -n "$agent_image" ] || die "could not resolve the pushed agent image digest" + +# A proper distribution zip, built the way deploy.yml builds it: every binary +# the repo's release.json names, each stamped with the release version, plus +# the manifest with the agent image pinned in. +dist="archie-core-archied-v$release_version-linux-amd64" +echo "== building the stamped release binaries and zipping $dist.zip" +mkdir -p "$work/dist/$dist" +release_ldflags="-X github.com/samcharles93/archie-core/internal/installtype.buildType=binary -X github.com/samcharles93/archie-core/internal/buildinfo.Version=$release_version -X github.com/samcharles93/archie-core/internal/buildinfo.Runtime=$release_version -X github.com/samcharles93/archie-core/internal/app/agentworker.version=$release_version" +for cmd in $(python3 -c 'import json; print(" ".join(json.load(open("'"$repo"'/release.json"))["required_topology"]["units"] + json.load(open("'"$repo"'/release.json"))["required_topology"].get("binaries", [])))'); do + go -C "$repo" build -tags no_ui -trimpath -ldflags "$release_ldflags" \ + -o "$work/dist/$dist/$cmd" "./cmd/$cmd" || die "go build of $cmd failed" +done +python3 -c 'import json, sys +m = json.load(open(sys.argv[1])); m["agent_image"] = sys.argv[2] +print(json.dumps(m))' "$repo/release.json" "$agent_image" > "$work/dist/$dist/release.json" +(CDPATH= cd -- "$work/dist" && zip -q -r "$work/release.zip" "$dist") + +echo "== publishing test release $release_version to 127.0.0.1:$reg_port/archie" +ARCHIE_RELEASE_SEED=$seed "$bin/archied" release publish \ + -source "127.0.0.1:$reg_port/archie" -version "$release_version" \ + -zip "$work/release.zip" -agent-image "$agent_image" || + die "release publish failed" + +# --- the local host ---------------------------------------------------------- + +export XDG_CONFIG_HOME=$work/config XDG_DATA_HOME=$work/data XDG_STATE_HOME=$work/state +export ARCHIE_UPDATE_STATE_DIR=$work/update-state +# Point XDG_RUNTIME_DIR at a path that doesn't exist, so systemctl --user has +# no bus: the rehearsal host reports unmanaged and the install switches the +# binaries without trying to restart units through systemd. +export XDG_RUNTIME_DIR=$work/no-user-bus +mkdir -p "$XDG_CONFIG_HOME/archie" "$XDG_DATA_HOME" "$XDG_STATE_HOME" "$ARCHIE_UPDATE_STATE_DIR" + +cfg=$XDG_CONFIG_HOME/archie/config.toml +# A throwaway CA and per-service certificates, the way `task dev:tls` issues +# them: archie-ui dials the daemon's harness sessions over mTLS even on +# loopback, so the host cannot come up without [tls]. +tls=$work/tls +mkdir -p "$tls" +openssl req -x509 -newkey ec -pkeyopt ec_paramgen_curve:prime256v1 -nodes -days 3650 \ + -subj "/CN=archie-rehearse-ca" -keyout "$tls/ca-key.pem" -out "$tls/ca.pem" 2>/dev/null || + die "could not create the throwaway CA" +for s in archied archie-gateway archie-messaging archie-state-store archie-ui; do + openssl req -x509 -newkey ec -pkeyopt ec_paramgen_curve:prime256v1 -nodes -days 3650 \ + -subj "/CN=$s" -addext "subjectAltName=DNS:$s" -addext extendedKeyUsage=serverAuth,clientAuth \ + -CA "$tls/ca.pem" -CAkey "$tls/ca-key.pem" -keyout "$tls/$s-key.pem" -out "$tls/$s.pem" 2>/dev/null || + die "could not issue a certificate for $s" +done +sed \ + -e "s|^# database_url = .*|database_url = \"postgres://archie:archie@127.0.0.1:$pg_port/archie?sslmode=disable\"|" \ + -e "s|^listen = \"127.0.0.1:8485\"|listen = \"127.0.0.1:$health_port\"|" \ + -e "s|^listen = \"127.0.0.1:8484\"|listen = \"127.0.0.1:$web_port\"|" \ + -e "s|^target = \"127.0.0.1:8585\"|target = \"127.0.0.1:$gw_port\"\nlisten = \"127.0.0.1:$gw_port\"|" \ + -e "s|^target = \"127.0.0.1:9090\"|target = \"127.0.0.1:$state_port\"\nlisten = \"127.0.0.1:$state_port\"|" \ + "$repo/config.example.toml" > "$cfg" +cat >> "$cfg" <"$logs/$svc.log" 2>&1 & + pids="$pids $!" +} + +wait_port() { # port label tries + i=0 + while [ "$i" -lt "${3:-90}" ]; do + if python3 -c "import socket,sys; s=socket.socket(); s.settimeout(1); sys.exit(0 if s.connect_ex(('127.0.0.1', $1)) == 0 else 1)"; then + return 0 + fi + i=$((i + 1)); sleep 1 + done + die "$2 did not listen on 127.0.0.1:$1; see $logs/$2.log (removed on exit -- rerun to inspect)" +} + +host_pids="" +start_host() { + spawn archie-state-store "$bin/archie-state-store" + wait_port "$state_port" archie-state-store + # The gateway only listens once archied has published its embedded NATS + # endpoint, so the rest start together, as e2e/stack.sh does. + spawn archie-gateway "$bin/archie-gateway" + spawn archied "$bin/archied" + spawn archie-ui "$bin/archie-ui" + host_pids="$pids" + wait_port "$gw_port" archie-gateway + wait_port "$daemon_port" archied + wait_port "$web_port" archie-ui +} + +echo "== starting the local host (state store, gateway, archied, archie-ui)" +start_host + +base="http://127.0.0.1:$web_port" +cookie=$work/cookies.txt + +api() { # method path [body] + method=$1 path=$2 body=${3-} + if [ -n "$body" ]; then + curl -sS --max-time 15 -b "$cookie" -c "$cookie" -X "$method" "$base$path" \ + -H 'Content-Type: application/json' -H 'X-Archie-CSRF: 1' -H "Origin: $base" -d "$body" + else + curl -sS --max-time 15 -b "$cookie" -c "$cookie" "$base$path" + fi +} + +py() { python3 -c "import sys, json; d = json.load(sys.stdin); print($1)"; } + +# One throwaway account on the throwaway host, then a personal API token, +# which is what the canary commands authenticate with. +email="canary-rehearse@example.test" +password="archie-canary-rehearse-password" +account=$(python3 -c 'import json, sys +print(json.dumps(dict(email=sys.argv[1], password=sys.argv[2], display_name="Canary rehearsal", + organisation_name="Canary rehearsal", workspace_name="Rehearsal", public_url=sys.argv[3])))' \ + "$email" "$password" "$base") +setup=$(api GET /api/auth/status) +if printf '%s' "$setup" | py 'd["setup_required"]' | grep -q True; then + api POST /api/auth/setup "$account" >/dev/null || die "account setup failed" +else + api POST /api/auth/login "$account" >/dev/null || die "login failed: $setup" +fi +token=$(api POST /api/tokens '{"name":"canary-rehearse","days":7}' | py 'd["token"]') || + die "could not create a personal API token on the local host" + +# Point the host's update settings at the local registry and key. +doc=$(api GET /api/control-plane/resources/update-settings) +ver=$(printf '%s' "$doc" | py 'd["resource"]["version"]') +value=$(python3 -c 'import json, sys +print(json.dumps({"channel": "stable", "pin": "", "source": sys.argv[1], "key": sys.argv[2]}))' \ + "127.0.0.1:$reg_port/archie" "$pub") +resp=$(api POST /api/control-plane/resources/update-settings/commands/replace \ + "{\"value\":$value,\"expected_version\":$ver}") +printf '%s' "$resp" | grep -q '"resource"' || die "could not set update-settings on the local host: $resp" + +# --- the canary loop, each step printed --------------------------------------- + +failed=0 +run_step() { # description command... + desc=$1; shift + echo + echo "== $desc" + printf '$ %s\n' "$*" + if "$@" 2>&1; then + echo "-- ok" + else + rc=$? + echo "-- FAILED (exit $rc)" + failed=$((failed + 1)) + keep=1 + fi +} + +export ARCHIE_CANARY_TOKEN=$token + +echo +echo "== release canary start -- point the host at next and install $release_version" +printf '$ archied release canary start -dashboard %s -version %s -source 127.0.0.1:%s/archie -key \n' \ + "$base" "$release_version" "$reg_port" +timeout 600 "$bin/archied" release canary start -dashboard "$base" -version "$release_version" \ + -source "127.0.0.1:$reg_port/archie" -key "$pub" >"$work/start.out" 2>&1 & +start_pid=$! +# The local host has no systemd: the install switches the binary and records +# the install time, but nothing restarts the daemon. Restart archied once the +# install has committed, the way an unmanaged host's operator would, so the +# host reports the new version. +i=0 +while [ ! -f "$ARCHIE_UPDATE_STATE_DIR/installed.json" ]; do + kill -0 "$start_pid" 2>/dev/null || break + i=$((i + 1)); [ "$i" -gt 300 ] && break + sleep 1 +done +if [ -f "$ARCHIE_UPDATE_STATE_DIR/installed.json" ]; then + # Restart every service, not just the daemon: the update snapshot reports + # the serving binary's version, so the whole host must come back on the + # new release. + for pid in $host_pids; do kill "$pid" 2>/dev/null || true; done + sleep 1 + pids="" + start_host +fi +start_rc=0 +wait "$start_pid" || start_rc=$? +cat "$work/start.out" +if [ "$start_rc" -eq 0 ]; then + echo "-- ok" +else + echo "-- FAILED (exit $start_rc)" + failed=$((failed + 1)) + keep=1 +fi + +run_step "release canary status -- what the canary runs and how the window is going" \ + timeout 120 "$bin/archied" release canary status -dashboard "$base" \ + -source "127.0.0.1:$reg_port/archie" -key "$pub" + +run_step "release canary check -- judge the installed release (still settling right after the install)" \ + timeout 300 "$bin/archied" release canary check -dashboard "$base" \ + -source "127.0.0.1:$reg_port/archie" -key "$pub" -checkout "$repo" + +run_step "release canary promote -- promote $release_version now that its hard checks pass" \ + timeout 120 "$bin/archied" release canary promote -dashboard "$base" -version "$release_version" \ + -source "127.0.0.1:$reg_port/archie" -key "$pub" + +echo +if [ "$failed" -gt 0 ]; then + echo "canary-rehearse: $failed step(s) failed; the steps above show why." >&2 + if [ "$keep" -gt 0 ] && [ -d "$logs" ]; then + for f in "$logs"/*.log; do + echo "--- $f" >&2 + tail -n 20 "$f" >&2 + done + fi + exit 1 +fi +echo "canary-rehearse: every step ran clean; host was $base, registry 127.0.0.1:$reg_port (both torn down)" diff --git a/ui/src/settings/CanaryStatusCard.vue b/ui/src/settings/CanaryStatusCard.vue new file mode 100644 index 000000000..956333302 --- /dev/null +++ b/ui/src/settings/CanaryStatusCard.vue @@ -0,0 +1,184 @@ + + + diff --git a/ui/src/settings/SystemStatusPage.vue b/ui/src/settings/SystemStatusPage.vue index aa513b08e..7e7688e1d 100644 --- a/ui/src/settings/SystemStatusPage.vue +++ b/ui/src/settings/SystemStatusPage.vue @@ -6,6 +6,7 @@ import PageHeader from "@/base/PageHeader.vue"; import { Alert, AlertDescription, AlertTitle } from "@/components/ui/alert"; import { Section } from "@/components/ui/section"; import { Table, TableBody, TableCell, TableRow } from "@/components/ui/table"; +import CanaryStatusCard from "./CanaryStatusCard.vue"; import ServicesCard from "./ServicesCard.vue"; import UpdateSettingsCard from "./UpdateSettingsCard.vue"; import VersionsCard from "./VersionsCard.vue"; @@ -32,6 +33,7 @@ onUnmounted(() => clearInterval(timer)); +