feat(releaseupdate): judge the installed release, window from install (#1725)
The canary window now counts from when a release is installed on the host (installed.json / installed_at in the update snapshot), not from the statement's issued_at, and 'release canary check' judges the release the canary actually runs via its exact-pin tag instead of the moving next tag. A newer release on next queues behind the one being judged instead of restarting the clock, so stable moves once per release. New subcommands: canary status (read-only state, signals, verdict), canary promote (promote now when hard checks pass), canary unpin (back to next), and start -force. A held/pinned canary is verified installable every check and a stranded pin fails the run red instead of looking green. Rollback pre-flight checks stable's zip and agent image digests before pinning. Relative signals are judged whenever both windows carry minimum data, so a breach rolls back mid-window; promote still needs the full window unless the operator promotes early (new promote.yml workflow_dispatch). The window is configurable via -window / CANARY_WINDOW. Also: dashboard canary status card (ui), and task canary:rehearse, a local end-to-end rehearsal of publish -> canary start -> status -> check -> promote against a throwaway registry and host. task check passes; rehearsal run green (REHEARSE_EXIT=0).
This commit is contained in:
18 files changed
+1487
-180
No files matched your search
@@ -1,7 +1,9 @@
|
||||
# Every 15 minutes, checks how the newest release on the `next` channel is
|
||||
# running on the canary host. If a hard check fails, the canary goes back to
|
||||
# the stable release straight away. If the release stays healthy for 24 hours,
|
||||
# it's promoted to stable. A rollback fails the run, so it shows up red.
|
||||
# Every 15 minutes, checks how the release installed on the canary host is
|
||||
# running, judged over the window counted from its install (-window, default
|
||||
# 24h; CANARY_WINDOW overrides it). If a hard check fails, the canary goes
|
||||
# back to the stable release straight away. If the release stays healthy for
|
||||
# the window, it's promoted to stable. A rollback fails the run, so it shows
|
||||
# up red.
|
||||
name: canary
|
||||
|
||||
on:
|
||||
@@ -48,6 +50,12 @@ jobs:
|
||||
- name: Judge the canary
|
||||
env:
|
||||
ARCHIE_CANARY_TOKEN: ${{ secrets.CANARY_TOKEN }}
|
||||
CANARY_WINDOW: ${{ vars.CANARY_WINDOW }}
|
||||
run: |
|
||||
window_args=()
|
||||
if [ -n "$CANARY_WINDOW" ]; then
|
||||
window_args=(-window "$CANARY_WINDOW")
|
||||
fi
|
||||
"$RUNNER_TEMP/archied" release canary check -dashboard "${{ vars.CANARY_URL }}" \
|
||||
"${window_args[@]}" \
|
||||
-latest git.catlow.cloud/sam/archied,git.catlow.cloud/sam/archie-agent
|
||||
@@ -418,8 +418,12 @@ jobs:
|
||||
"$api/repos/$repo/releases" >/dev/null
|
||||
echo "published release notes for $tag"
|
||||
|
||||
# The canary host follows the `next` channel. canary.yml then decides
|
||||
# whether this release is promoted to stable or the canary is rolled back.
|
||||
# The canary host follows the `next` channel. canary.yml then judges the
|
||||
# release installed on the canary and decides whether it is promoted to
|
||||
# stable or the canary is rolled back. The start is queue-safe: a release
|
||||
# published while the canary still judges a newer one is queued behind
|
||||
# it and picked up by the next check, so the running window isn't
|
||||
# restarted under a different release.
|
||||
- name: Install the release on the canary
|
||||
if: vars.CANARY_URL != ''
|
||||
env:
|
||||
|
||||
@@ -0,0 +1,54 @@
|
||||
# Promotes the release the canary is running right now, without waiting out
|
||||
# the judging window: the canary's signals must show no hard failure and no
|
||||
# relative breach, otherwise the run refuses and fails. Dispatch it with the
|
||||
# version the canary runs (`archied release canary status` shows it).
|
||||
name: promote
|
||||
|
||||
on:
|
||||
workflow_dispatch:
|
||||
inputs:
|
||||
version:
|
||||
description: "release the canary is expected to run; must match what it runs"
|
||||
required: true
|
||||
type: string
|
||||
|
||||
concurrency:
|
||||
group: promote
|
||||
|
||||
permissions:
|
||||
contents: read
|
||||
packages: write
|
||||
|
||||
jobs:
|
||||
promote:
|
||||
if: vars.CANARY_URL != ''
|
||||
runs-on: self-hosted
|
||||
steps:
|
||||
- name: Checkout
|
||||
uses: actions/checkout@v4
|
||||
with:
|
||||
ref: main
|
||||
|
||||
- name: Set up Go
|
||||
uses: actions/setup-go@v5
|
||||
with:
|
||||
go-version-file: go.mod
|
||||
# The self-hosted runner keeps the Go caches on disk between jobs, and
|
||||
# archiving them for the Actions cache takes minutes.
|
||||
cache: false
|
||||
|
||||
- name: Build archied
|
||||
# The promote never serves the dashboard, so the build leaves it out.
|
||||
run: go build -tags no_ui -o "$RUNNER_TEMP/archied" ./cmd/archied
|
||||
|
||||
# Promotion retags stable and the images' latest in the registry.
|
||||
- name: Log in to the Gitea registry
|
||||
run: echo "${{ secrets.DEPLOY_TOKEN }}" | docker login git.catlow.cloud -u "${{ github.actor }}" --password-stdin
|
||||
|
||||
- name: Promote the canary's release
|
||||
env:
|
||||
ARCHIE_CANARY_TOKEN: ${{ secrets.CANARY_TOKEN }}
|
||||
run: |
|
||||
"$RUNNER_TEMP/archied" release canary promote -dashboard "${{ vars.CANARY_URL }}" \
|
||||
-version "${{ inputs.version }}" \
|
||||
-latest git.catlow.cloud/sam/archied,git.catlow.cloud/sam/archie-agent
|
||||
+69
-18
@@ -78,27 +78,63 @@ release rather than committing the fix on top of a tagged one.
|
||||
|
||||
The canary is the production host, following `next`. Every quarter hour
|
||||
`canary.yml` runs `archied release canary check`, which reads the canary
|
||||
through its dashboard API and judges the release on `next`:
|
||||
through its dashboard API and judges **the release the canary actually runs**,
|
||||
not the moving `next` tag:
|
||||
|
||||
| Signal | Source | Fails when |
|
||||
| --- | --- | --- |
|
||||
| Install outcome | `GET /api/chat/update` | the canary doesn't run the release (the watchdog rolled it back) |
|
||||
| Install outcome | `GET /api/chat/update` | the canary doesn't run the judged release (the watchdog rolled it back) |
|
||||
| Readiness | `GET /health/detailed` | the rollup isn't `ok` |
|
||||
| Alerts | `GET /api/alerts` | an open `action` alert was first raised after the release was issued |
|
||||
| Error log | `GET /api/logs?level=error` | error lines per day exceed `2 × baseline + 10` |
|
||||
| Alerts | `GET /api/alerts` | an open `action` alert was first raised after the release was installed |
|
||||
| Error log | `GET /api/logs?level=error` | error lines per window exceed `2 × baseline + 10` |
|
||||
| Task outcomes | `GET /api/tasks` | the share of finished tasks that parked or died rose more than 10 points |
|
||||
|
||||
- **Control.** The canary's own 24 hours before the release was issued. With
|
||||
- **Control.** The canary's own window before the release was installed. With
|
||||
one production host there's no simultaneous control.
|
||||
- **Window.** 24 hours from issue, one cycle of schedules and traffic. The
|
||||
first 30 minutes are for the install and restart.
|
||||
- **Window.** 24 hours counted from the **install** on the canary, not from
|
||||
the release's issue time: the host records it in `installed.json`
|
||||
(`committed_at`) and reports it as `installed_at` in `GET /api/chat/update`.
|
||||
One cycle of schedules and traffic. The first 30 minutes are for the install
|
||||
and restart. `canary.yml` passes `-window` when the `CANARY_WINDOW` Actions
|
||||
variable is set; the flag defaults to 24h. A host that doesn't report an
|
||||
install time degrades to the release statement's issue time.
|
||||
- **Hard signals** (install, readiness, alerts) roll back at any check after
|
||||
that. **Relative signals** (errors, tasks) are judged at the end of the
|
||||
window.
|
||||
that. **Relative signals** (errors, tasks) are judged against the baseline
|
||||
whenever both windows carry enough data, so a breach rolls the release back
|
||||
even mid-window; a full window with no failure promotes it.
|
||||
- **Too little data skips a signal** rather than failing it: fewer than 5
|
||||
finished tasks in the window, or an error log that reaches back less than an
|
||||
hour into the baseline. A log read that stops short of a window is scaled to
|
||||
a full day from what it covers.
|
||||
a full window from what it covers.
|
||||
|
||||
### Queueing
|
||||
|
||||
`deploy.yml` installs each new release on the canary with
|
||||
`archied release canary start -version V`, which first points the canary back
|
||||
at `next` (clearing any rollback pin) and then installs `V` — unless the
|
||||
canary is already judging a release newer than stable that isn't `V`, in which
|
||||
case `V` is **queued** behind it and the canary keeps running what it judges.
|
||||
`-force` installs regardless. After a promotion, the next `check` picks up the
|
||||
newer `next` itself, so a queue drains one release per window.
|
||||
|
||||
### Subcommands
|
||||
|
||||
```
|
||||
archied release canary status read-only: channel/pin, installed release and
|
||||
installed_at, stable/next, window progress, and
|
||||
the verdict the current signals would give
|
||||
archied release canary start point the canary at next and install -version,
|
||||
queued behind a release being judged (-force to
|
||||
override)
|
||||
archied release canary check judge the installed release over -window
|
||||
(default 24h): promote, pick up a newer next,
|
||||
or roll back; a held canary is reported, and a
|
||||
held pin that is no longer installable fails
|
||||
the run red
|
||||
archied release canary promote promote the installed release now if its
|
||||
signals pass; refuses with exit 1 otherwise
|
||||
archied release canary unpin return the canary to next from any pin
|
||||
```
|
||||
|
||||
### Promote
|
||||
|
||||
@@ -106,22 +142,36 @@ A release that passes the window is promoted: `stable` is retagged to the
|
||||
`v<version>` manifest. The signature covers the statement, not the tag, so the
|
||||
retag needs no signing key. The `archied` and `archie-agent` images' `:latest`
|
||||
move to their `:v<version>` first, and the agent image must be the digest the
|
||||
statement pins. Promotion refuses a prerelease and a release issued
|
||||
before the one `stable` names, the same replay rule hosts enforce. The decision
|
||||
and its reasons are in the `canary` run log.
|
||||
statement pins. Promotion refuses a prerelease and a release issued before the
|
||||
one `stable` names, the same replay rule hosts enforce. The decision and its
|
||||
reasons are in the `canary` run log.
|
||||
|
||||
Early promotion is a dispatch away: the **promote** workflow
|
||||
(`.github/workflows/promote.yml`) takes a `version` input and runs
|
||||
`archied release canary promote -version V` on the canary. It judges the
|
||||
release the canary runs — the window doesn't matter for the decision — and
|
||||
refuses (red run) on any hard failure or relative breach with enough data.
|
||||
|
||||
### Roll back
|
||||
|
||||
`stable` stays put, and the run turns red. The canary is pinned to the `stable`
|
||||
version through its update settings and reinstalled through its dashboard.
|
||||
`stable` stays put, and the run turns red. Before the canary is pinned to
|
||||
`stable`, `check` verifies `stable` is actually installable — its statement's
|
||||
zip blob must still be in the release repository and its pinned agent image
|
||||
must still resolve (`releaseupdate.Installable`). If it isn't, the run fails
|
||||
loudly and the canary stays on `next` rather than being stranded on a pin.
|
||||
Otherwise the canary is pinned to the `stable` version through its update
|
||||
settings and reinstalled through its dashboard.
|
||||
|
||||
A release whose migrations differ from `stable`'s can't go back without
|
||||
restoring the pre-update database snapshot, which loses every write since. The
|
||||
canary is held on the failed release instead (pinned to it) and the fix goes
|
||||
forward in the next release. The next release's deploy points the canary back at
|
||||
`next`.
|
||||
forward in the next release. The next release's deploy points the canary back
|
||||
at `next`.
|
||||
|
||||
A pinned canary isn't judged, so a rollback is decided once.
|
||||
A pinned canary isn't judged, so a rollback is decided once — but `check`
|
||||
verifies the pin is still installable every run, so a stranded pin shows red
|
||||
instead of green. `archied release canary unpin` returns the canary to `next`
|
||||
by hand.
|
||||
|
||||
### Configuration
|
||||
|
||||
@@ -129,6 +179,7 @@ A pinned canary isn't judged, so a rollback is decided once.
|
||||
| --- | --- | --- |
|
||||
| `CANARY_URL` | Actions variable | the canary's dashboard URL; unset disables the canary steps |
|
||||
| `CANARY_TOKEN` | Actions secret | a personal API token on the canary with instance admin |
|
||||
| `CANARY_WINDOW` | Actions variable | optional `-window` override for `canary check`; unset is 24h |
|
||||
| `DEPLOY_TOKEN` | Actions secret | registry and repository write, for publishing, promoting and the release push |
|
||||
| `RELEASE_SIGNING_KEY` | Actions secret | the release signing seed |
|
||||
|
||||
|
||||
@@ -221,6 +221,13 @@ tasks:
|
||||
cmds:
|
||||
- node e2e/run.cjs {{.CLI_ARGS}}
|
||||
|
||||
canary:rehearse:
|
||||
# Local rehearsal only: spins up its own registry, database and host, and
|
||||
# is deliberately absent from check and CI.
|
||||
desc: Rehearse the release-canary loop on a throwaway local host (needs docker; never part of the gate)
|
||||
cmds:
|
||||
- sh tools/canary-rehearse.sh
|
||||
|
||||
build:
|
||||
desc: Build archie services and agent binaries
|
||||
deps: [ui]
|
||||
|
||||
@@ -19,8 +19,11 @@ const releaseUsage = `usage: archied release <command>
|
||||
keygen -seed FILE write a new signing key's seed to FILE and print its public key
|
||||
publish sign a release and push it to a release source's next channel
|
||||
promote -version V point stable at a published release by hand
|
||||
canary start point the canary at next and install a release on it
|
||||
canary check judge the release on next and promote it or roll the canary back`
|
||||
canary status show the canary's channel, installed release, window and verdict (read-only)
|
||||
canary start point the canary at next and install a release on it, queued behind one being judged
|
||||
canary check judge the release the canary runs: promote it, pick up a newer next, or roll back
|
||||
canary promote promote the release the canary runs now, if its signals pass
|
||||
canary unpin return the canary to the next channel from any pin`
|
||||
|
||||
// RunRelease is the publisher's side of a release source: it makes the key
|
||||
// a source signs with and pushes signed releases to it.
|
||||
|
||||
@@ -8,8 +8,10 @@ import (
|
||||
"io"
|
||||
"os"
|
||||
"os/exec"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"golang.org/x/mod/semver"
|
||||
"oras.land/oras-go/v2/errdef"
|
||||
|
||||
"github.com/samcharles93/archie-core/internal/domain/distribution"
|
||||
@@ -25,129 +27,510 @@ const canaryInstallWait = 20 * time.Minute
|
||||
const migrationsDir = "internal/infrastructure/postgres/migrations"
|
||||
|
||||
func runReleaseCanary(args []string, out, stderr io.Writer) int {
|
||||
if len(args) == 0 || (args[0] != "start" && args[0] != "check") {
|
||||
if len(args) == 0 {
|
||||
fmt.Fprintln(stderr, releaseUsage)
|
||||
return 2
|
||||
}
|
||||
flags := flag.NewFlagSet("release canary "+args[0], flag.ContinueOnError)
|
||||
switch args[0] {
|
||||
case "status":
|
||||
return runCanaryStatus(args[1:], out, stderr)
|
||||
case "start":
|
||||
return runCanaryStart(args[1:], out, stderr)
|
||||
case "check":
|
||||
return runCanaryCheck(args[1:], out, stderr)
|
||||
case "promote":
|
||||
return runCanaryPromote(args[1:], out, stderr)
|
||||
case "unpin":
|
||||
return runCanaryUnpin(args[1:], out, stderr)
|
||||
default:
|
||||
fmt.Fprintln(stderr, releaseUsage)
|
||||
return 2
|
||||
}
|
||||
}
|
||||
|
||||
// addCanaryFlags registers the flags every canary subcommand takes.
|
||||
func addCanaryFlags(flags *flag.FlagSet) (dashboardURL, tokenEnv *string) {
|
||||
dashboardURL = flags.String("dashboard", "", "canary dashboard URL")
|
||||
tokenEnv = flags.String("token-env", "ARCHIE_CANARY_TOKEN", "environment variable holding a dashboard API token for the canary")
|
||||
return dashboardURL, tokenEnv
|
||||
}
|
||||
|
||||
// addSourceFlags registers the flags naming the release source.
|
||||
func addSourceFlags(flags *flag.FlagSet) (reference, key *string) {
|
||||
reference = flags.String("source", "", "OCI repository releases are published to; blank is the official releases")
|
||||
key = flags.String("key", "", "base64 ed25519 public key the source signs with")
|
||||
return reference, key
|
||||
}
|
||||
|
||||
func canaryDashboard(url, tokenEnv string) (releaseupdate.CanaryDashboard, error) {
|
||||
if url == "" {
|
||||
return releaseupdate.CanaryDashboard{}, errors.New("-dashboard is required")
|
||||
}
|
||||
return releaseupdate.CanaryDashboard{URL: url, Token: os.Getenv(tokenEnv)}, nil
|
||||
}
|
||||
|
||||
func canarySource(reference, key string) (distribution.Source, error) {
|
||||
return distribution.Parse(reference, key, distribution.DefaultReleases())
|
||||
}
|
||||
|
||||
// fetchChannel fetches a channel tag; found is false when the channel has no
|
||||
// release yet.
|
||||
func fetchChannel(ctx context.Context, source distribution.Source, tag string) (release releaseupdate.Release, found bool, err error) {
|
||||
release, err = releaseupdate.Fetch(ctx, source, tag)
|
||||
if errors.Is(err, errdef.ErrNotFound) {
|
||||
return releaseupdate.Release{}, false, nil
|
||||
}
|
||||
return release, err == nil, err
|
||||
}
|
||||
|
||||
// publishedChannels are the releases the source currently publishes.
|
||||
type publishedChannels struct {
|
||||
next, stable releaseupdate.Release
|
||||
hasNext, hasStable bool
|
||||
}
|
||||
|
||||
func fetchChannels(ctx context.Context, source distribution.Source) (publishedChannels, error) {
|
||||
var channels publishedChannels
|
||||
var err error
|
||||
if channels.next, channels.hasNext, err = fetchChannel(ctx, source, "next"); err != nil {
|
||||
return channels, err
|
||||
}
|
||||
if channels.stable, channels.hasStable, err = fetchChannel(ctx, source, "stable"); err != nil {
|
||||
return channels, err
|
||||
}
|
||||
return channels, nil
|
||||
}
|
||||
|
||||
// canaryRun is the release the canary actually runs: its version, when it was
|
||||
// installed and its fetched release statement.
|
||||
type canaryRun struct {
|
||||
version string
|
||||
installedAt time.Time
|
||||
release releaseupdate.Release
|
||||
found bool
|
||||
}
|
||||
|
||||
// readCanaryRun reads what the canary runs. A host that doesn't record install
|
||||
// times (it updated before they were tracked) degrades to the release
|
||||
// statement's issue time.
|
||||
func readCanaryRun(ctx context.Context, dashboard releaseupdate.CanaryDashboard, source distribution.Source) (canaryRun, error) {
|
||||
var run canaryRun
|
||||
version, installedAt, _, err := dashboard.Installed(ctx)
|
||||
if err != nil {
|
||||
return run, err
|
||||
}
|
||||
run.version, run.installedAt = version, installedAt
|
||||
if version == "" {
|
||||
return run, nil
|
||||
}
|
||||
release, found, err := fetchChannel(ctx, source, releaseupdate.Tag("exact-pin", version))
|
||||
if err != nil {
|
||||
return run, err
|
||||
}
|
||||
run.release, run.found = release, found
|
||||
if run.installedAt.IsZero() && found {
|
||||
run.installedAt = release.Statement.IssuedAt
|
||||
}
|
||||
return run, nil
|
||||
}
|
||||
|
||||
// newerRelease reports whether version a is a strictly newer release than b.
|
||||
func newerRelease(a, b string) bool {
|
||||
return semver.Compare(canonVersion(a), canonVersion(b)) > 0
|
||||
}
|
||||
|
||||
func canonVersion(version string) string {
|
||||
return "v" + strings.TrimPrefix(version, "v")
|
||||
}
|
||||
|
||||
func displayVersion(version string) string {
|
||||
if version == "" {
|
||||
return "unknown"
|
||||
}
|
||||
return version
|
||||
}
|
||||
|
||||
func displayRelease(release releaseupdate.Release, found bool) string {
|
||||
if !found {
|
||||
return "none"
|
||||
}
|
||||
return release.Statement.Version
|
||||
}
|
||||
|
||||
func printVerdict(out io.Writer, version string, verdict releaseupdate.CanaryVerdict) {
|
||||
fmt.Fprintf(out, "%s: %s\n", displayVersion(version), verdict.Decision)
|
||||
for _, reason := range verdict.Reasons {
|
||||
fmt.Fprintf(out, " %s\n", reason)
|
||||
}
|
||||
}
|
||||
|
||||
func runCanaryStatus(args []string, out, stderr io.Writer) int {
|
||||
flags := flag.NewFlagSet("release canary status", flag.ContinueOnError)
|
||||
flags.SetOutput(stderr)
|
||||
dashboardURL := flags.String("dashboard", "", "canary dashboard URL")
|
||||
tokenEnv := flags.String("token-env", "ARCHIE_CANARY_TOKEN", "environment variable holding a dashboard API token for the canary")
|
||||
version := flags.String("version", "", "release to install (start)")
|
||||
reference := flags.String("source", "", "OCI repository releases are published to; blank is the official releases (check)")
|
||||
key := flags.String("key", "", "base64 ed25519 public key the source signs with (check)")
|
||||
latest := flags.String("latest", "", "comma-separated image repositories whose v<version> tag becomes latest on promotion (check)")
|
||||
checkout := flags.String("checkout", ".", "repository checkout with release tags, to see whether a release migrated the database (check)")
|
||||
if err := flags.Parse(args[1:]); err != nil {
|
||||
dashboardURL, tokenEnv := addCanaryFlags(flags)
|
||||
reference, key := addSourceFlags(flags)
|
||||
window := flags.Duration("window", releaseupdate.CanaryWindow, "window a release is judged over, counted from its install")
|
||||
if err := flags.Parse(args); err != nil {
|
||||
return 2
|
||||
}
|
||||
dashboard := releaseupdate.CanaryDashboard{URL: *dashboardURL, Token: os.Getenv(*tokenEnv)}
|
||||
if *dashboardURL == "" {
|
||||
fmt.Fprintln(stderr, "release canary: -dashboard is required")
|
||||
return 2
|
||||
}
|
||||
ctx := context.Background()
|
||||
if args[0] == "start" {
|
||||
if *version == "" {
|
||||
fmt.Fprintln(stderr, "release canary start: -version is required")
|
||||
return 2
|
||||
}
|
||||
if err := startCanary(ctx, dashboard, *version); err != nil {
|
||||
fmt.Fprintln(stderr, err)
|
||||
return 1
|
||||
}
|
||||
fmt.Fprintf(out, "canary follows next and runs %s\n", *version)
|
||||
return 0
|
||||
}
|
||||
source, err := distribution.Parse(*reference, *key, distribution.DefaultReleases())
|
||||
dashboard, err := canaryDashboard(*dashboardURL, *tokenEnv)
|
||||
if err != nil {
|
||||
fmt.Fprintln(stderr, "release canary check:", err)
|
||||
fmt.Fprintln(stderr, "release canary status:", err)
|
||||
return 2
|
||||
}
|
||||
rolledBack, err := checkCanary(ctx, dashboard, source, splitList(*latest), *checkout, out)
|
||||
source, err := canarySource(*reference, *key)
|
||||
if err != nil {
|
||||
fmt.Fprintln(stderr, err)
|
||||
return 1
|
||||
fmt.Fprintln(stderr, "release canary status:", err)
|
||||
return 2
|
||||
}
|
||||
if rolledBack {
|
||||
if err := printCanaryStatus(context.Background(), dashboard, source, *window, out); err != nil {
|
||||
fmt.Fprintln(stderr, "release canary status:", err)
|
||||
return 1
|
||||
}
|
||||
return 0
|
||||
}
|
||||
|
||||
func startCanary(ctx context.Context, dashboard releaseupdate.CanaryDashboard, version string) error {
|
||||
if err := dashboard.Follow(ctx, "next", ""); err != nil {
|
||||
return fmt.Errorf("point the canary at next: %w", err)
|
||||
}
|
||||
return dashboard.Install(ctx, version, canaryInstallWait)
|
||||
}
|
||||
|
||||
// checkCanary judges the release on the next channel once and acts on the
|
||||
// result. It reports rolledBack so that the caller can fail the run.
|
||||
func checkCanary(ctx context.Context, dashboard releaseupdate.CanaryDashboard, source distribution.Source, latest []string, checkout string, out io.Writer) (rolledBack bool, err error) {
|
||||
next, err := releaseupdate.Fetch(ctx, source, "next")
|
||||
if err != nil {
|
||||
return false, err
|
||||
}
|
||||
stable, err := releaseupdate.Fetch(ctx, source, "stable")
|
||||
hasStable := err == nil
|
||||
if err != nil && !errors.Is(err, errdef.ErrNotFound) {
|
||||
return false, err
|
||||
}
|
||||
release := next.Statement
|
||||
if release.Prerelease || (hasStable && !release.IssuedAt.After(stable.Statement.IssuedAt)) {
|
||||
fmt.Fprintf(out, "nothing to judge: next is %s and isn't a newer release than stable\n", release.Version)
|
||||
return false, nil
|
||||
}
|
||||
// printCanaryStatus reports the canary's channel, installed release, the
|
||||
// published channels, window progress and the verdict the current signals
|
||||
// would give, without acting on anything.
|
||||
func printCanaryStatus(ctx context.Context, dashboard releaseupdate.CanaryDashboard, source distribution.Source, window time.Duration, out io.Writer) error {
|
||||
channel, pin, err := dashboard.Channel(ctx)
|
||||
if err != nil {
|
||||
return false, err
|
||||
return err
|
||||
}
|
||||
if channel != "next" {
|
||||
fmt.Fprintf(out, "nothing to judge: the canary is held on %s %s\n", channel, pin)
|
||||
return false, nil
|
||||
channels, err := fetchChannels(ctx, source)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
run, err := readCanaryRun(ctx, dashboard, source)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
printCanaryState(out, channel, pin, run, channels, window)
|
||||
return printCanaryVerdict(ctx, dashboard, run, window, out)
|
||||
}
|
||||
|
||||
func printCanaryState(out io.Writer, channel, pin string, run canaryRun, channels publishedChannels, window time.Duration) {
|
||||
fmt.Fprintf(out, "channel: %s\n", channelDisplay(channel, pin))
|
||||
fmt.Fprintf(out, "installed: %s", displayVersion(run.version))
|
||||
if run.installedAt.IsZero() {
|
||||
fmt.Fprintln(out, " (installed_at not reported)")
|
||||
} else {
|
||||
fmt.Fprintf(out, " since %s\n", run.installedAt.Format(time.RFC3339))
|
||||
}
|
||||
fmt.Fprintf(out, "stable: %s\n", displayRelease(channels.stable, channels.hasStable))
|
||||
fmt.Fprintf(out, "next: %s\n", displayRelease(channels.next, channels.hasNext))
|
||||
if run.installedAt.IsZero() {
|
||||
fmt.Fprintln(out, "window: unknown, no install time to count from")
|
||||
} else {
|
||||
elapsed := time.Since(run.installedAt)
|
||||
fmt.Fprintf(out, "window: %s of %s elapsed since the install\n", elapsed.Round(time.Minute), window)
|
||||
}
|
||||
}
|
||||
|
||||
func printCanaryVerdict(ctx context.Context, dashboard releaseupdate.CanaryDashboard, run canaryRun, window time.Duration, out io.Writer) error {
|
||||
if run.installedAt.IsZero() {
|
||||
fmt.Fprintln(out, "verdict: unknown, no install time to judge from")
|
||||
return nil
|
||||
}
|
||||
now := time.Now().UTC()
|
||||
signals, err := dashboard.Signals(ctx, release.IssuedAt, now)
|
||||
signals, err := dashboard.Signals(ctx, run.installedAt, now, window)
|
||||
if err != nil {
|
||||
return false, fmt.Errorf("read canary signals: %w", err)
|
||||
return fmt.Errorf("read canary signals: %w", err)
|
||||
}
|
||||
verdict := releaseupdate.JudgeCanary(release.Version, now.Sub(release.IssuedAt), signals)
|
||||
fmt.Fprintf(out, "%s: %s\n", release.Version, verdict.Decision)
|
||||
printSignals(out, signals)
|
||||
verdict := releaseupdate.JudgeCanary(run.version, run.installedAt, now, window, signals)
|
||||
fmt.Fprintf(out, "verdict on %s: %s (read-only)\n", displayVersion(run.version), verdict.Decision)
|
||||
for _, reason := range verdict.Reasons {
|
||||
fmt.Fprintf(out, " %s\n", reason)
|
||||
}
|
||||
switch verdict.Decision {
|
||||
case releaseupdate.CanaryPromote:
|
||||
if _, err := releaseupdate.Promote(ctx, source.Reference, source.Key, release.Version, latest); err != nil {
|
||||
return false, fmt.Errorf("promote %s: %w", release.Version, err)
|
||||
return nil
|
||||
}
|
||||
|
||||
func printSignals(out io.Writer, s releaseupdate.CanarySignals) {
|
||||
fmt.Fprintf(out, "signals: runs %q, ready=%t, new action alerts=%d\n", s.Installed, s.Ready, len(s.NewActionAlerts))
|
||||
fmt.Fprintf(out, " tasks finished %d, parked or died %d (baseline %d, %d)\n",
|
||||
s.Canary.Finished, s.Canary.Bad, s.Baseline.Finished, s.Baseline.Bad)
|
||||
fmt.Fprintf(out, " error lines %d, log covers %s (baseline %d, covers %s)\n",
|
||||
s.Canary.Errors, s.Canary.Covered.Round(time.Minute), s.Baseline.Errors, s.Baseline.Covered.Round(time.Minute))
|
||||
}
|
||||
|
||||
func channelDisplay(channel, pin string) string {
|
||||
if pin != "" {
|
||||
return channel + " (pinned to " + pin + ")"
|
||||
}
|
||||
return channel
|
||||
}
|
||||
|
||||
func runCanaryStart(args []string, out, stderr io.Writer) int {
|
||||
flags := flag.NewFlagSet("release canary start", flag.ContinueOnError)
|
||||
flags.SetOutput(stderr)
|
||||
dashboardURL, tokenEnv := addCanaryFlags(flags)
|
||||
reference, key := addSourceFlags(flags)
|
||||
version := flags.String("version", "", "release to install on the canary")
|
||||
force := flags.Bool("force", false, "install even when the canary is still judging a newer release")
|
||||
if err := flags.Parse(args); err != nil {
|
||||
return 2
|
||||
}
|
||||
dashboard, err := canaryDashboard(*dashboardURL, *tokenEnv)
|
||||
if err != nil {
|
||||
fmt.Fprintln(stderr, "release canary start:", err)
|
||||
return 2
|
||||
}
|
||||
if *version == "" {
|
||||
fmt.Fprintln(stderr, "release canary start: -version is required")
|
||||
return 2
|
||||
}
|
||||
source, err := canarySource(*reference, *key)
|
||||
if err != nil {
|
||||
fmt.Fprintln(stderr, "release canary start:", err)
|
||||
return 2
|
||||
}
|
||||
if err := startCanary(context.Background(), dashboard, source, *version, *force, out); err != nil {
|
||||
fmt.Fprintln(stderr, "release canary start:", err)
|
||||
return 1
|
||||
}
|
||||
return 0
|
||||
}
|
||||
|
||||
// startCanary points the canary at next — clearing any rollback pin — and
|
||||
// installs version on it. A release published while the canary still judges a
|
||||
// newer one queues behind it, so the running window isn't restarted under a
|
||||
// different release.
|
||||
func startCanary(ctx context.Context, dashboard releaseupdate.CanaryDashboard, source distribution.Source, version string, force bool, out io.Writer) error {
|
||||
// The channel is read before following next: a canary already on next is
|
||||
// judging the release it runs, and a new release queues behind that one.
|
||||
channel, _, err := dashboard.Channel(ctx)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if err := dashboard.Follow(ctx, "next", ""); err != nil {
|
||||
return fmt.Errorf("point the canary at next: %w", err)
|
||||
}
|
||||
if !force {
|
||||
queued, err := queuedBehind(ctx, dashboard, source, channel, version, out)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
fmt.Fprintf(out, "stable is %s\n", release.Version)
|
||||
if queued {
|
||||
return nil
|
||||
}
|
||||
}
|
||||
if err := dashboard.Install(ctx, version, canaryInstallWait); err != nil {
|
||||
return err
|
||||
}
|
||||
fmt.Fprintf(out, "the canary follows next and runs %s\n", version)
|
||||
return nil
|
||||
}
|
||||
|
||||
// queuedBehind reports whether version must wait: the canary already follows
|
||||
// next, runs a release newer than stable (the one being judged) and version is
|
||||
// a different release.
|
||||
func queuedBehind(ctx context.Context, dashboard releaseupdate.CanaryDashboard, source distribution.Source, channel, version string, out io.Writer) (bool, error) {
|
||||
if channel != "next" {
|
||||
return false, nil
|
||||
case releaseupdate.CanaryRollBack:
|
||||
return true, rollBackCanary(ctx, dashboard, release.Version, stable.Statement.Version, hasStable, checkout, out)
|
||||
}
|
||||
installed, _, _, err := dashboard.Installed(ctx)
|
||||
if err != nil {
|
||||
return false, err
|
||||
}
|
||||
if installed == "" || installed == version {
|
||||
return false, nil
|
||||
}
|
||||
stable, hasStable, err := fetchChannel(ctx, source, "stable")
|
||||
if err != nil {
|
||||
return false, err
|
||||
}
|
||||
if hasStable && newerRelease(installed, stable.Statement.Version) {
|
||||
fmt.Fprintf(out, "%s is queued behind %s, which the canary is still judging\n", version, installed)
|
||||
return true, nil
|
||||
}
|
||||
return false, nil
|
||||
}
|
||||
|
||||
// rollBackCanary pins the canary to the stable release and reinstalls it.
|
||||
// Going back past a migration needs the pre-update database snapshot, which
|
||||
// loses every write since, so a release that migrated is held instead.
|
||||
func rollBackCanary(ctx context.Context, dashboard releaseupdate.CanaryDashboard, failed, stable string, hasStable bool, checkout string, out io.Writer) error {
|
||||
if !hasStable || migrated(ctx, checkout, stable, failed) {
|
||||
if err := dashboard.Follow(ctx, "exact-pin", failed); err != nil {
|
||||
return fmt.Errorf("hold the canary on %s: %w", failed, err)
|
||||
func runCanaryCheck(args []string, out, stderr io.Writer) int {
|
||||
flags := flag.NewFlagSet("release canary check", flag.ContinueOnError)
|
||||
flags.SetOutput(stderr)
|
||||
dashboardURL, tokenEnv := addCanaryFlags(flags)
|
||||
reference, key := addSourceFlags(flags)
|
||||
latest := flags.String("latest", "", "comma-separated image repositories whose v<version> tag becomes latest on promotion")
|
||||
checkout := flags.String("checkout", ".", "repository checkout with release tags, to see whether a release migrated the database")
|
||||
window := flags.Duration("window", releaseupdate.CanaryWindow, "window the installed release is judged over, counted from its install")
|
||||
if err := flags.Parse(args); err != nil {
|
||||
return 2
|
||||
}
|
||||
dashboard, err := canaryDashboard(*dashboardURL, *tokenEnv)
|
||||
if err != nil {
|
||||
fmt.Fprintln(stderr, "release canary check:", err)
|
||||
return 2
|
||||
}
|
||||
source, err := canarySource(*reference, *key)
|
||||
if err != nil {
|
||||
fmt.Fprintln(stderr, "release canary check:", err)
|
||||
return 2
|
||||
}
|
||||
code, err := checkCanary(context.Background(), dashboard, source, splitList(*latest), *checkout, *window, out)
|
||||
if err != nil {
|
||||
fmt.Fprintln(stderr, "release canary check:", err)
|
||||
return 1
|
||||
}
|
||||
return code
|
||||
}
|
||||
|
||||
// checkCanary judges the release the canary actually runs — not the moving
|
||||
// next tag — and acts on the verdict. The exit code is 1 when the run should
|
||||
// show red: a rollback happened, or the canary is held on a pin that can no
|
||||
// longer be installed.
|
||||
func checkCanary(ctx context.Context, dashboard releaseupdate.CanaryDashboard, source distribution.Source, latest []string, checkout string, window time.Duration, out io.Writer) (int, error) {
|
||||
channels, err := fetchChannels(ctx, source)
|
||||
if err != nil {
|
||||
return 1, err
|
||||
}
|
||||
channel, pin, err := dashboard.Channel(ctx)
|
||||
if err != nil {
|
||||
return 1, err
|
||||
}
|
||||
if channel != "next" {
|
||||
return heldCanary(ctx, source, channel, pin, out)
|
||||
}
|
||||
run, err := readCanaryRun(ctx, dashboard, source)
|
||||
if err != nil {
|
||||
return 1, err
|
||||
}
|
||||
if !judgeable(run, channels) {
|
||||
fmt.Fprintf(out, "nothing to judge: the canary runs %s\n", displayVersion(run.version))
|
||||
return pickupNext(ctx, dashboard, channels, run.version, out)
|
||||
}
|
||||
return judgeInstalled(ctx, dashboard, source, run, channels, latest, checkout, window, out)
|
||||
}
|
||||
|
||||
// judgeable reports whether the installed release is one the canary judges: a
|
||||
// published, non-prerelease release newer than stable.
|
||||
func judgeable(run canaryRun, channels publishedChannels) bool {
|
||||
if !run.found || run.release.Statement.Prerelease {
|
||||
return false
|
||||
}
|
||||
return !channels.hasStable || newerRelease(run.version, channels.stable.Statement.Version)
|
||||
}
|
||||
|
||||
func judgeInstalled(ctx context.Context, dashboard releaseupdate.CanaryDashboard, source distribution.Source, run canaryRun, channels publishedChannels, latest []string, checkout string, window time.Duration, out io.Writer) (int, error) {
|
||||
now := time.Now().UTC()
|
||||
signals, err := dashboard.Signals(ctx, run.installedAt, now, window)
|
||||
if err != nil {
|
||||
return 1, fmt.Errorf("read canary signals: %w", err)
|
||||
}
|
||||
verdict := releaseupdate.JudgeCanary(run.version, run.installedAt, now, window, signals)
|
||||
printVerdict(out, run.version, verdict)
|
||||
switch verdict.Decision {
|
||||
case releaseupdate.CanaryPromote:
|
||||
return 0, promoteJudged(ctx, dashboard, source, run, channels, latest, out)
|
||||
case releaseupdate.CanaryRollBack:
|
||||
return 1, rollBackCanary(ctx, dashboard, source, run, channels, checkout, out)
|
||||
default:
|
||||
return 0, nil
|
||||
}
|
||||
}
|
||||
|
||||
// promoteJudged moves stable to the judged release, then picks up a newer next
|
||||
// queued behind it.
|
||||
func promoteJudged(ctx context.Context, dashboard releaseupdate.CanaryDashboard, source distribution.Source, run canaryRun, channels publishedChannels, latest []string, out io.Writer) error {
|
||||
if _, err := releaseupdate.Promote(ctx, source.Reference, source.Key, run.version, latest); err != nil {
|
||||
return fmt.Errorf("promote %s: %w", run.version, err)
|
||||
}
|
||||
fmt.Fprintf(out, "stable is %s\n", run.version)
|
||||
fresh := channels
|
||||
fresh.stable, fresh.hasStable = run.release, true
|
||||
_, err := pickupNext(ctx, dashboard, fresh, run.version, out)
|
||||
return err
|
||||
}
|
||||
|
||||
// pickupNext installs the next release when it is newer than stable and the
|
||||
// canary doesn't already run it.
|
||||
func pickupNext(ctx context.Context, dashboard releaseupdate.CanaryDashboard, channels publishedChannels, installed string, out io.Writer) (int, error) {
|
||||
if !channels.hasNext || channels.next.Statement.Version == installed {
|
||||
return 0, nil
|
||||
}
|
||||
if channels.hasStable && !newerRelease(channels.next.Statement.Version, channels.stable.Statement.Version) {
|
||||
return 0, nil
|
||||
}
|
||||
if err := dashboard.Install(ctx, channels.next.Statement.Version, canaryInstallWait); err != nil {
|
||||
return 1, err
|
||||
}
|
||||
fmt.Fprintf(out, "the canary picks up %s from next\n", channels.next.Statement.Version)
|
||||
return 0, nil
|
||||
}
|
||||
|
||||
// heldCanary reports a canary that isn't following next. Its pin must still be
|
||||
// installable: a stranded pin fails the run instead of looking green.
|
||||
func heldCanary(ctx context.Context, source distribution.Source, channel, pin string, out io.Writer) (int, error) {
|
||||
if pin != "" {
|
||||
err := checkPin(ctx, source, pin)
|
||||
if errors.Is(err, errStrandedPin) {
|
||||
fmt.Fprintf(out, "ERROR: the canary is held on %s %s, which is stranded: %v; unpin it or republish the release\n", channel, pin, err)
|
||||
return 1, nil
|
||||
}
|
||||
fmt.Fprintf(out, "%s changed the database schema or there's no stable release to return to: the canary is held on %s and stable is unchanged; fix forward\n", failed, failed)
|
||||
return nil
|
||||
if err != nil {
|
||||
return 1, err
|
||||
}
|
||||
fmt.Fprintf(out, "held on %s %s, nothing to judge\n", channel, pin)
|
||||
return 0, nil
|
||||
}
|
||||
if err := dashboard.Follow(ctx, "exact-pin", stable); err != nil {
|
||||
return fmt.Errorf("pin the canary to %s: %w", stable, err)
|
||||
fmt.Fprintf(out, "held on %s, nothing to judge\n", channel)
|
||||
return 0, nil
|
||||
}
|
||||
|
||||
// errStrandedPin marks a held pin that can no longer be installed.
|
||||
var errStrandedPin = errors.New("pin is stranded")
|
||||
|
||||
// checkPin fails when a held pin can no longer be installed: the release is
|
||||
// gone from the source, or its artifacts are.
|
||||
func checkPin(ctx context.Context, source distribution.Source, pin string) error {
|
||||
release, found, err := fetchChannel(ctx, source, releaseupdate.Tag("exact-pin", pin))
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if err := dashboard.Install(ctx, stable, canaryInstallWait); err != nil {
|
||||
return fmt.Errorf("return the canary to %s: %w", stable, err)
|
||||
if !found {
|
||||
return fmt.Errorf("%w: no release %s in the source", errStrandedPin, releaseupdate.Tag("exact-pin", pin))
|
||||
}
|
||||
fmt.Fprintf(out, "the canary is back on %s and stable is unchanged\n", stable)
|
||||
if err := releaseupdate.Installable(ctx, source, release); err != nil {
|
||||
return fmt.Errorf("%w: %w", errStrandedPin, err)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// rollBackCanary returns the canary to stable after a failed verdict. stable
|
||||
// is checked installable before the canary is pinned to it: a pin whose
|
||||
// artifacts are gone would strand the canary, so the run fails loudly and the
|
||||
// canary stays on next. A release that changed the migrations is held on the
|
||||
// failed version instead, because going back past a migration needs the
|
||||
// pre-update database snapshot.
|
||||
func rollBackCanary(ctx context.Context, dashboard releaseupdate.CanaryDashboard, source distribution.Source, run canaryRun, channels publishedChannels, checkout string, out io.Writer) error {
|
||||
failed := run.version
|
||||
if !channels.hasStable {
|
||||
return holdCanary(ctx, dashboard, failed, "there's no stable release to return to", out)
|
||||
}
|
||||
stable := channels.stable
|
||||
if err := releaseupdate.Installable(ctx, source, stable); err != nil {
|
||||
return fmt.Errorf("stable %s is not installable, the canary stays on next: %w", stable.Statement.Version, err)
|
||||
}
|
||||
if migrated(ctx, checkout, stable.Statement.Version, failed) {
|
||||
return holdCanary(ctx, dashboard, failed, failed+" changed the database schema", out)
|
||||
}
|
||||
if err := dashboard.Follow(ctx, "exact-pin", stable.Statement.Version); err != nil {
|
||||
return fmt.Errorf("pin the canary to %s: %w", stable.Statement.Version, err)
|
||||
}
|
||||
if err := dashboard.Install(ctx, stable.Statement.Version, canaryInstallWait); err != nil {
|
||||
return fmt.Errorf("return the canary to %s: %w", stable.Statement.Version, err)
|
||||
}
|
||||
fmt.Fprintf(out, "the canary is back on %s and stable is unchanged\n", stable.Statement.Version)
|
||||
return nil
|
||||
}
|
||||
|
||||
func holdCanary(ctx context.Context, dashboard releaseupdate.CanaryDashboard, version, why string, out io.Writer) error {
|
||||
if err := dashboard.Follow(ctx, "exact-pin", version); err != nil {
|
||||
return fmt.Errorf("hold the canary on %s: %w", version, err)
|
||||
}
|
||||
fmt.Fprintf(out, "%s: the canary is held on %s and stable is unchanged; fix forward\n", why, version)
|
||||
return nil
|
||||
}
|
||||
|
||||
@@ -158,3 +541,86 @@ func migrated(ctx context.Context, checkout, from, to string) bool {
|
||||
err := exec.CommandContext(ctx, "git", "-C", checkout, "diff", "--quiet", "v"+from, "v"+to, "--", migrationsDir).Run()
|
||||
return err != nil
|
||||
}
|
||||
|
||||
func runCanaryPromote(args []string, out, stderr io.Writer) int {
|
||||
flags := flag.NewFlagSet("release canary promote", flag.ContinueOnError)
|
||||
flags.SetOutput(stderr)
|
||||
dashboardURL, tokenEnv := addCanaryFlags(flags)
|
||||
reference, key := addSourceFlags(flags)
|
||||
latest := flags.String("latest", "", "comma-separated image repositories whose v<version> tag becomes latest on promotion")
|
||||
version := flags.String("version", "", "release expected on the canary; must match what it runs")
|
||||
window := flags.Duration("window", releaseupdate.CanaryWindow, "window the signals are read over, counted from the install")
|
||||
if err := flags.Parse(args); err != nil {
|
||||
return 2
|
||||
}
|
||||
dashboard, err := canaryDashboard(*dashboardURL, *tokenEnv)
|
||||
if err != nil {
|
||||
fmt.Fprintln(stderr, "release canary promote:", err)
|
||||
return 2
|
||||
}
|
||||
source, err := canarySource(*reference, *key)
|
||||
if err != nil {
|
||||
fmt.Fprintln(stderr, "release canary promote:", err)
|
||||
return 2
|
||||
}
|
||||
code, err := promoteCanaryNow(context.Background(), dashboard, source, *version, splitList(*latest), *window, out)
|
||||
if err != nil {
|
||||
fmt.Fprintln(stderr, "release canary promote:", err)
|
||||
return 1
|
||||
}
|
||||
return code
|
||||
}
|
||||
|
||||
// promoteCanaryNow promotes the release the canary runs right now, without
|
||||
// waiting out the window: the signals must show no hard failure and no
|
||||
// relative breach. It refuses, with exit 1, when anything fails.
|
||||
func promoteCanaryNow(ctx context.Context, dashboard releaseupdate.CanaryDashboard, source distribution.Source, expectVersion string, latest []string, window time.Duration, out io.Writer) (int, error) {
|
||||
run, err := readCanaryRun(ctx, dashboard, source)
|
||||
if err != nil {
|
||||
return 1, err
|
||||
}
|
||||
if expectVersion != "" && expectVersion != run.version {
|
||||
return 1, fmt.Errorf("the canary runs %s, not %s", displayVersion(run.version), expectVersion)
|
||||
}
|
||||
if !run.found {
|
||||
return 1, fmt.Errorf("no release %s in the source to promote", displayVersion(run.version))
|
||||
}
|
||||
now := time.Now().UTC()
|
||||
signals, err := dashboard.Signals(ctx, run.installedAt, now, window)
|
||||
if err != nil {
|
||||
return 1, fmt.Errorf("read canary signals: %w", err)
|
||||
}
|
||||
// A zero install time makes the judge skip its settle and window waits,
|
||||
// so only the signals decide — exactly what promoting now asks for.
|
||||
verdict := releaseupdate.JudgeCanary(run.version, time.Time{}, now, window, signals)
|
||||
printVerdict(out, run.version, verdict)
|
||||
if verdict.Decision == releaseupdate.CanaryRollBack {
|
||||
fmt.Fprintln(out, "refusing to promote: the canary's signals fail")
|
||||
return 1, nil
|
||||
}
|
||||
if _, err := releaseupdate.Promote(ctx, source.Reference, source.Key, run.version, latest); err != nil {
|
||||
return 1, fmt.Errorf("promote %s: %w", run.version, err)
|
||||
}
|
||||
fmt.Fprintf(out, "stable is %s\n", run.version)
|
||||
return 0, nil
|
||||
}
|
||||
|
||||
func runCanaryUnpin(args []string, out, stderr io.Writer) int {
|
||||
flags := flag.NewFlagSet("release canary unpin", flag.ContinueOnError)
|
||||
flags.SetOutput(stderr)
|
||||
dashboardURL, tokenEnv := addCanaryFlags(flags)
|
||||
if err := flags.Parse(args); err != nil {
|
||||
return 2
|
||||
}
|
||||
dashboard, err := canaryDashboard(*dashboardURL, *tokenEnv)
|
||||
if err != nil {
|
||||
fmt.Fprintln(stderr, "release canary unpin:", err)
|
||||
return 2
|
||||
}
|
||||
if err := dashboard.Follow(context.Background(), "next", ""); err != nil {
|
||||
fmt.Fprintln(stderr, "release canary unpin:", err)
|
||||
return 1
|
||||
}
|
||||
fmt.Fprintln(out, "the canary follows next again, with no pin")
|
||||
return 0
|
||||
}
|
||||
@@ -6,14 +6,17 @@ import (
|
||||
)
|
||||
|
||||
// The canary host installs each release from the next channel. Its behaviour
|
||||
// in the 24 hours after a release is compared with its own behaviour in the
|
||||
// 24 hours before, because there's only one production host to compare with.
|
||||
// in the window after a release was installed is compared with its own
|
||||
// behaviour in the window before, because there's only one production host to
|
||||
// compare with.
|
||||
const (
|
||||
// CanaryWindow is how long a release runs on the canary before it can be
|
||||
// promoted. A day covers one full cycle of schedules and traffic.
|
||||
// CanaryWindow is the default window a release runs on the canary before
|
||||
// it can be promoted. A day covers one full cycle of schedules and
|
||||
// traffic.
|
||||
CanaryWindow = 24 * time.Hour
|
||||
// canarySettle is how long the install and restart get, counted from when
|
||||
// the release was issued, before any check can fail the release.
|
||||
// the release was installed on the canary, before any check can fail the
|
||||
// release.
|
||||
canarySettle = 30 * time.Minute
|
||||
// minFinished is the fewest finished tasks needed to judge the park rate.
|
||||
minFinished = 5
|
||||
@@ -47,7 +50,7 @@ type CanarySignals struct {
|
||||
Installed string
|
||||
Ready bool
|
||||
// NewActionAlerts are open action-severity alerts first raised since
|
||||
// the release was issued.
|
||||
// the release was installed.
|
||||
NewActionAlerts []string
|
||||
Baseline CanaryWindowSignals
|
||||
Canary CanaryWindowSignals
|
||||
@@ -59,11 +62,13 @@ type CanaryVerdict struct {
|
||||
Reasons []string
|
||||
}
|
||||
|
||||
// JudgeCanary decides what to do with a release, given how long ago it was
|
||||
// issued. Once the release has settled, a failed hard check rolls it back at
|
||||
// any time. Error and park rates are compared with the baseline only after the
|
||||
// full window has passed.
|
||||
func JudgeCanary(version string, elapsed time.Duration, s CanarySignals) CanaryVerdict {
|
||||
// JudgeCanary decides what to do with a release, given when it was installed
|
||||
// on the canary host. Once the release has settled, a failed hard check rolls
|
||||
// it back at any time. Error and park rates are judged against the baseline
|
||||
// whenever both windows carry enough data, so a breach rolls the release back
|
||||
// even mid-window; a full window with no failure promotes it.
|
||||
func JudgeCanary(version string, installedAt, now time.Time, window time.Duration, s CanarySignals) CanaryVerdict {
|
||||
elapsed := now.Sub(installedAt)
|
||||
if elapsed < canarySettle {
|
||||
return CanaryVerdict{Decision: CanaryWait, Reasons: []string{"settling after install"}}
|
||||
}
|
||||
@@ -80,11 +85,8 @@ func JudgeCanary(version string, elapsed time.Duration, s CanarySignals) CanaryV
|
||||
if len(failed) > 0 {
|
||||
return CanaryVerdict{Decision: CanaryRollBack, Reasons: failed}
|
||||
}
|
||||
if elapsed < CanaryWindow {
|
||||
return CanaryVerdict{Decision: CanaryWait, Reasons: []string{fmt.Sprintf("%s of %s elapsed", elapsed.Round(time.Minute), CanaryWindow)}}
|
||||
}
|
||||
var notes []string
|
||||
if reason, ok := judgeErrors(s.Baseline, s.Canary); !ok {
|
||||
if reason, ok := judgeErrors(s.Baseline, s.Canary, window); !ok {
|
||||
failed = append(failed, reason)
|
||||
} else {
|
||||
notes = append(notes, reason)
|
||||
@@ -97,32 +99,37 @@ func JudgeCanary(version string, elapsed time.Duration, s CanarySignals) CanaryV
|
||||
if len(failed) > 0 {
|
||||
return CanaryVerdict{Decision: CanaryRollBack, Reasons: failed}
|
||||
}
|
||||
if elapsed < window {
|
||||
reasons := append([]string{fmt.Sprintf("%s of %s elapsed since the install", elapsed.Round(time.Minute), window)}, notes...)
|
||||
return CanaryVerdict{Decision: CanaryWait, Reasons: reasons}
|
||||
}
|
||||
return CanaryVerdict{Decision: CanaryPromote, Reasons: notes}
|
||||
}
|
||||
|
||||
// judgeErrors fails when the canary logged more than twice the baseline's
|
||||
// error lines plus ten. Each count is scaled up to a full day when the log
|
||||
// error lines plus ten. Each count is scaled up to a full window when the log
|
||||
// covers only part of its window.
|
||||
func judgeErrors(baseline, canary CanaryWindowSignals) (string, bool) {
|
||||
func judgeErrors(baseline, canary CanaryWindowSignals, window time.Duration) (string, bool) {
|
||||
if baseline.Covered < minLogCoverage || canary.Covered <= 0 {
|
||||
return "error log doesn't reach back over the baseline; error signal skipped", true
|
||||
}
|
||||
scale := func(w CanaryWindowSignals) float64 {
|
||||
return float64(w.Errors) * float64(CanaryWindow) / float64(w.Covered)
|
||||
return float64(w.Errors) * float64(window) / float64(w.Covered)
|
||||
}
|
||||
before, after := scale(baseline), scale(canary)
|
||||
limit := 2*before + 10
|
||||
if after > limit {
|
||||
return fmt.Sprintf("%.0f error log lines per day against a limit of %.0f (baseline %.0f)", after, limit, before), false
|
||||
return fmt.Sprintf("%.0f error log lines per window against a limit of %.0f (baseline %.0f)", after, limit, before), false
|
||||
}
|
||||
return fmt.Sprintf("%.0f error log lines per day, limit %.0f", after, limit), true
|
||||
return fmt.Sprintf("%.0f error log lines per window, limit %.0f", after, limit), true
|
||||
}
|
||||
|
||||
// judgeParks fails when the share of finished tasks that parked or died rose
|
||||
// more than ten points over the baseline.
|
||||
// more than ten points over the baseline. Both windows need the minimum
|
||||
// finished tasks before the rates are compared.
|
||||
func judgeParks(baseline, canary CanaryWindowSignals) (string, bool) {
|
||||
if canary.Finished < minFinished {
|
||||
return fmt.Sprintf("%d finished tasks, fewer than %d; park signal skipped", canary.Finished, minFinished), true
|
||||
if canary.Finished < minFinished || baseline.Finished < minFinished {
|
||||
return fmt.Sprintf("%d finished tasks on the canary and %d in the baseline, fewer than %d each; park signal skipped", canary.Finished, baseline.Finished, minFinished), true
|
||||
}
|
||||
rate := func(w CanaryWindowSignals) float64 {
|
||||
if w.Finished == 0 {
|
||||
|
||||
@@ -32,44 +32,47 @@ var (
|
||||
const updateSettingsPath = "/api/control-plane/resources/update-settings"
|
||||
|
||||
// Signals reads the canary's current state. Tasks and error lines are split
|
||||
// into the day before the release was issued and the time since.
|
||||
func (d CanaryDashboard) Signals(ctx context.Context, issued, now time.Time) (CanarySignals, error) {
|
||||
// into the window before the release was installed on the canary and the time
|
||||
// since.
|
||||
func (d CanaryDashboard) Signals(ctx context.Context, installedAt, now time.Time, window time.Duration) (CanarySignals, error) {
|
||||
var s CanarySignals
|
||||
var err error
|
||||
if s.Installed, _, err = d.installed(ctx); err != nil {
|
||||
if s.Installed, _, _, err = d.Installed(ctx); err != nil {
|
||||
return s, err
|
||||
}
|
||||
if s.Ready, err = d.ready(ctx); err != nil {
|
||||
return s, err
|
||||
}
|
||||
if s.NewActionAlerts, err = d.actionAlerts(ctx, issued); err != nil {
|
||||
if s.NewActionAlerts, err = d.actionAlerts(ctx, installedAt); err != nil {
|
||||
return s, err
|
||||
}
|
||||
if err := d.tasks(ctx, issued, &s); err != nil {
|
||||
if err := d.tasks(ctx, installedAt, window, &s); err != nil {
|
||||
return s, err
|
||||
}
|
||||
return s, d.errors(ctx, issued, now, &s)
|
||||
return s, d.errors(ctx, installedAt, now, window, &s)
|
||||
}
|
||||
|
||||
// installed returns the daemon version the canary runs, and the raw update
|
||||
// snapshot it offers, which an install request must send back unchanged.
|
||||
func (d CanaryDashboard) installed(ctx context.Context) (string, json.RawMessage, error) {
|
||||
// Installed returns the daemon version the canary runs, when that release was
|
||||
// installed on the canary host (zero when the host doesn't report it), and
|
||||
// the raw update snapshot it offers, which an install request must send back
|
||||
// unchanged.
|
||||
func (d CanaryDashboard) Installed(ctx context.Context) (version string, installedAt time.Time, snapshot json.RawMessage, err error) {
|
||||
var body struct {
|
||||
Snapshot json.RawMessage `json:"snapshot"`
|
||||
}
|
||||
if _, err := d.call(ctx, http.MethodGet, "/api/chat/update", nil, &body); err != nil {
|
||||
return "", nil, err
|
||||
return "", time.Time{}, nil, err
|
||||
}
|
||||
var snapshot Snapshot
|
||||
if err := json.Unmarshal(body.Snapshot, &snapshot); err != nil {
|
||||
return "", nil, fmt.Errorf("decode update snapshot: %w", err)
|
||||
var snap Snapshot
|
||||
if err := json.Unmarshal(body.Snapshot, &snap); err != nil {
|
||||
return "", time.Time{}, nil, fmt.Errorf("decode update snapshot: %w", err)
|
||||
}
|
||||
for _, c := range snapshot.Components {
|
||||
for _, c := range snap.Components {
|
||||
if c.ID == "daemon" {
|
||||
return c.Installed, body.Snapshot, nil
|
||||
return c.Installed, snap.InstalledAt, body.Snapshot, nil
|
||||
}
|
||||
}
|
||||
return "", body.Snapshot, errors.New("canary reports no daemon version")
|
||||
return "", time.Time{}, body.Snapshot, errors.New("canary reports no daemon version")
|
||||
}
|
||||
|
||||
func (d CanaryDashboard) ready(ctx context.Context) (bool, error) {
|
||||
@@ -104,8 +107,8 @@ func (d CanaryDashboard) actionAlerts(ctx context.Context, since time.Time) ([]s
|
||||
|
||||
// tasks reads tasks, newest update first, until it reaches tasks older than
|
||||
// the baseline window.
|
||||
func (d CanaryDashboard) tasks(ctx context.Context, issued time.Time, s *CanarySignals) error {
|
||||
from := issued.Add(-CanaryWindow)
|
||||
func (d CanaryDashboard) tasks(ctx context.Context, at time.Time, window time.Duration, s *CanarySignals) error {
|
||||
from := at.Add(-window)
|
||||
cursor := ""
|
||||
for range 50 {
|
||||
var page []struct {
|
||||
@@ -125,7 +128,7 @@ func (d CanaryDashboard) tasks(ctx context.Context, issued time.Time, s *CanaryS
|
||||
continue
|
||||
}
|
||||
w := &s.Canary
|
||||
if t.UpdatedAt.Before(issued) {
|
||||
if t.UpdatedAt.Before(at) {
|
||||
w = &s.Baseline
|
||||
}
|
||||
w.Finished++
|
||||
@@ -144,7 +147,7 @@ func (d CanaryDashboard) tasks(ctx context.Context, issued time.Time, s *CanaryS
|
||||
// errors counts error-level lines in each window. A log read returns a
|
||||
// limited number of lines, so a truncated read only reaches back as far as
|
||||
// its oldest line.
|
||||
func (d CanaryDashboard) errors(ctx context.Context, issued, now time.Time, s *CanarySignals) error {
|
||||
func (d CanaryDashboard) errors(ctx context.Context, at, now time.Time, window time.Duration, s *CanarySignals) error {
|
||||
var body struct {
|
||||
Entries []struct {
|
||||
Time time.Time `json:"time"`
|
||||
@@ -154,24 +157,24 @@ func (d CanaryDashboard) errors(ctx context.Context, issued, now time.Time, s *C
|
||||
if _, err := d.call(ctx, http.MethodGet, "/api/logs?level=error&limit=2000", nil, &body); err != nil {
|
||||
return err
|
||||
}
|
||||
from := issued.Add(-CanaryWindow)
|
||||
from := at.Add(-window)
|
||||
if body.Truncated {
|
||||
from = now
|
||||
if len(body.Entries) > 0 {
|
||||
from = later(body.Entries[0].Time, issued.Add(-CanaryWindow))
|
||||
from = later(body.Entries[0].Time, at.Add(-window))
|
||||
}
|
||||
}
|
||||
for _, e := range body.Entries {
|
||||
switch {
|
||||
case e.Time.Before(from):
|
||||
case e.Time.Before(issued):
|
||||
case e.Time.Before(at):
|
||||
s.Baseline.Errors++
|
||||
default:
|
||||
s.Canary.Errors++
|
||||
}
|
||||
}
|
||||
s.Baseline.Covered = max(issued.Sub(from), 0)
|
||||
s.Canary.Covered = now.Sub(later(from, issued))
|
||||
s.Baseline.Covered = max(at.Sub(from), 0)
|
||||
s.Canary.Covered = now.Sub(later(from, at))
|
||||
return nil
|
||||
}
|
||||
|
||||
@@ -232,7 +235,7 @@ func (d CanaryDashboard) Install(ctx context.Context, version string, wait time.
|
||||
deadline := time.Now().Add(wait)
|
||||
requested := false
|
||||
for {
|
||||
installed, snapshot, err := d.installed(ctx)
|
||||
installed, _, snapshot, err := d.Installed(ctx)
|
||||
switch {
|
||||
case err != nil:
|
||||
// The dashboard drops off while the install restarts it.
|
||||
|
||||
@@ -5,10 +5,11 @@ import (
|
||||
"time"
|
||||
)
|
||||
|
||||
// A release promotes only after a full window with no hard failure and no
|
||||
// relative signal past its threshold; any hard failure after settling rolls
|
||||
// it back at once.
|
||||
// A release promotes only after a full window counted from its install with
|
||||
// no hard failure and no relative signal past its threshold; any failure after
|
||||
// settling rolls it back at once, even mid-window.
|
||||
func TestJudgeCanary(t *testing.T) {
|
||||
installed := time.Date(2026, 1, 2, 12, 0, 0, 0, time.UTC)
|
||||
healthy := func() CanarySignals {
|
||||
return CanarySignals{
|
||||
Installed: "1.1.0", Ready: true,
|
||||
@@ -23,31 +24,39 @@ func TestJudgeCanary(t *testing.T) {
|
||||
}
|
||||
cases := []struct {
|
||||
name string
|
||||
elapsed time.Duration
|
||||
elapsed time.Duration // now minus the install time
|
||||
window time.Duration
|
||||
signals CanarySignals
|
||||
want CanaryDecision
|
||||
}{
|
||||
{"settling ignores a failed check", 10 * time.Minute, with(func(s *CanarySignals) { s.Ready = false }), CanaryWait},
|
||||
{"healthy mid-window waits", 2 * time.Hour, healthy(), CanaryWait},
|
||||
{"healthy at the window promotes", CanaryWindow, healthy(), CanaryPromote},
|
||||
{"the watchdog rolled the install back", 2 * time.Hour, with(func(s *CanarySignals) { s.Installed = "1.0.0" }), CanaryRollBack},
|
||||
{"readiness degraded", 2 * time.Hour, with(func(s *CanarySignals) { s.Ready = false }), CanaryRollBack},
|
||||
{"a new action alert", 2 * time.Hour, with(func(s *CanarySignals) { s.NewActionAlerts = []string{"queue stalled"} }), CanaryRollBack},
|
||||
{"errors past twice the baseline plus ten", CanaryWindow, with(func(s *CanarySignals) { s.Canary.Errors = 31 }), CanaryRollBack},
|
||||
{"errors at the limit", CanaryWindow, with(func(s *CanarySignals) { s.Canary.Errors = 30 }), CanaryPromote},
|
||||
{"errors scaled from a partly covered log", CanaryWindow, with(func(s *CanarySignals) {
|
||||
{"settling counts from the install", 10 * time.Minute, CanaryWindow, with(func(s *CanarySignals) { s.Ready = false }), CanaryWait},
|
||||
{"healthy mid-window waits", 2 * time.Hour, CanaryWindow, healthy(), CanaryWait},
|
||||
{"healthy at the window promotes", CanaryWindow, CanaryWindow, healthy(), CanaryPromote},
|
||||
{"healthy at a custom window promotes", 12 * time.Hour, 12 * time.Hour, healthy(), CanaryPromote},
|
||||
{"the watchdog rolled the install back", 2 * time.Hour, CanaryWindow, with(func(s *CanarySignals) { s.Installed = "1.0.0" }), CanaryRollBack},
|
||||
{"readiness degraded", 2 * time.Hour, CanaryWindow, with(func(s *CanarySignals) { s.Ready = false }), CanaryRollBack},
|
||||
{"a new action alert", 2 * time.Hour, CanaryWindow, with(func(s *CanarySignals) { s.NewActionAlerts = []string{"queue stalled"} }), CanaryRollBack},
|
||||
{"errors past twice the baseline plus ten", CanaryWindow, CanaryWindow, with(func(s *CanarySignals) { s.Canary.Errors = 31 }), CanaryRollBack},
|
||||
{"errors at the limit", CanaryWindow, CanaryWindow, with(func(s *CanarySignals) { s.Canary.Errors = 30 }), CanaryPromote},
|
||||
{"errors scaled from a partly covered log", CanaryWindow, CanaryWindow, with(func(s *CanarySignals) {
|
||||
s.Canary.Errors, s.Canary.Covered = 8, 6*time.Hour
|
||||
}), CanaryRollBack},
|
||||
{"a baseline the log doesn't reach skips errors", CanaryWindow, with(func(s *CanarySignals) {
|
||||
{"a baseline the log doesn't reach skips errors", CanaryWindow, CanaryWindow, with(func(s *CanarySignals) {
|
||||
s.Baseline.Covered, s.Canary.Errors = 30*time.Minute, 500
|
||||
}), CanaryPromote},
|
||||
{"park rate up more than ten points", CanaryWindow, with(func(s *CanarySignals) { s.Canary.Bad = 5 }), CanaryRollBack},
|
||||
{"park rate up ten points", CanaryWindow, with(func(s *CanarySignals) { s.Canary.Bad = 4 }), CanaryPromote},
|
||||
{"too few tasks skips the park rate", CanaryWindow, with(func(s *CanarySignals) { s.Canary = CanaryWindowSignals{Finished: 4, Bad: 4, Covered: CanaryWindow} }), CanaryPromote},
|
||||
{"a mid-window error breach rolls back early", 2 * time.Hour, CanaryWindow, with(func(s *CanarySignals) { s.Canary.Errors = 31 }), CanaryRollBack},
|
||||
{"park rate up more than ten points", CanaryWindow, CanaryWindow, with(func(s *CanarySignals) { s.Canary.Bad = 5 }), CanaryRollBack},
|
||||
{"park rate up ten points", CanaryWindow, CanaryWindow, with(func(s *CanarySignals) { s.Canary.Bad = 4 }), CanaryPromote},
|
||||
{"too few canary tasks skips the park rate", CanaryWindow, CanaryWindow, with(func(s *CanarySignals) { s.Canary = CanaryWindowSignals{Finished: 4, Bad: 4, Covered: CanaryWindow} }), CanaryPromote},
|
||||
{"a thin baseline skips the park rate", CanaryWindow, CanaryWindow, with(func(s *CanarySignals) {
|
||||
s.Baseline.Finished, s.Baseline.Bad = 2, 0
|
||||
s.Canary.Bad = 5
|
||||
}), CanaryPromote},
|
||||
{"a mid-window park breach rolls back early", 2 * time.Hour, CanaryWindow, with(func(s *CanarySignals) { s.Canary.Bad = 5 }), CanaryRollBack},
|
||||
}
|
||||
for _, tc := range cases {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
got := JudgeCanary("1.1.0", tc.elapsed, tc.signals)
|
||||
got := JudgeCanary("1.1.0", installed, installed.Add(tc.elapsed), tc.window, tc.signals)
|
||||
if got.Decision != tc.want {
|
||||
t.Fatalf("decision = %s, want %s (%v)", got.Decision, tc.want, got.Reasons)
|
||||
}
|
||||
|
||||
@@ -0,0 +1,47 @@
|
||||
package releaseupdate
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
|
||||
digest "github.com/opencontainers/go-digest"
|
||||
ocispec "github.com/opencontainers/image-spec/specs-go/v1"
|
||||
"oras.land/oras-go/v2/registry"
|
||||
|
||||
"github.com/samcharles93/archie-core/internal/domain/distribution"
|
||||
)
|
||||
|
||||
// Installable reports whether a release can still be installed from source:
|
||||
// the zip blob its statement names must be present in the release repository
|
||||
// and the agent image it pins must still resolve. A canary must never be
|
||||
// pinned to a release whose artifacts are gone, or it would be stranded on a
|
||||
// version it can never install again.
|
||||
func Installable(ctx context.Context, source distribution.Source, release Release) error {
|
||||
zipDigest, err := digest.Parse(release.Statement.Zip.Digest)
|
||||
if err != nil {
|
||||
return fmt.Errorf("zip digest %q: %w", release.Statement.Zip.Digest, err)
|
||||
}
|
||||
repo, err := pushRepository(source.Reference)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
exists, err := repo.Exists(ctx, ocispec.Descriptor{Digest: zipDigest})
|
||||
if err != nil {
|
||||
return fmt.Errorf("resolve zip blob %s in %s: %w", release.Statement.Zip.Digest, source.Reference, err)
|
||||
}
|
||||
if !exists {
|
||||
return fmt.Errorf("zip blob %s (%s) is absent from %s", release.Statement.Zip.Digest, release.Statement.Zip.Name, source.Reference)
|
||||
}
|
||||
image, err := registry.ParseReference(release.Statement.AgentImage)
|
||||
if err != nil {
|
||||
return fmt.Errorf("agent image %q: %w", release.Statement.AgentImage, err)
|
||||
}
|
||||
imageRepo, err := pushRepository(image.Registry + "/" + image.Repository)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if _, err := imageRepo.Resolve(ctx, image.Reference); err != nil {
|
||||
return fmt.Errorf("agent image %s: %w", release.Statement.AgentImage, err)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
@@ -44,6 +44,9 @@ type Component struct {
|
||||
type Snapshot struct {
|
||||
Components []Component
|
||||
Deferred bool
|
||||
// InstalledAt is when the installed release was committed on this host,
|
||||
// from its installed.json. Zero when the host doesn't record one.
|
||||
InstalledAt time.Time `json:"installed_at"`
|
||||
}
|
||||
|
||||
func (s Snapshot) Available() []Component {
|
||||
|
||||
@@ -40,6 +40,7 @@ const (
|
||||
journalDir = "transaction"
|
||||
journalFile = "journal.json"
|
||||
snapshotFile = "db.pre-update.dump"
|
||||
installedFile = "installed.json"
|
||||
lockWait = 60 * time.Second
|
||||
)
|
||||
|
||||
@@ -126,6 +127,39 @@ type journal struct {
|
||||
// JournalPath is where an unfinished transaction lives.
|
||||
func JournalPath(stateDir string) string { return filepath.Join(stateDir, journalDir) }
|
||||
|
||||
// installedRecord is the host's durable note of which release it last
|
||||
// committed (or rolled back to) and when, so a canary can count its window
|
||||
// from the actual install rather than the release statement.
|
||||
type installedRecord struct {
|
||||
Version string `json:"version"`
|
||||
CommittedAt time.Time `json:"committed_at"`
|
||||
}
|
||||
|
||||
// recordInstall writes installed.json atomically: the file is the source of
|
||||
// truth for when the running release was installed.
|
||||
func recordInstall(stateDir, version string, at time.Time) error {
|
||||
data, err := json.Marshal(installedRecord{Version: version, CommittedAt: at.UTC()})
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
return fsutil.WriteFileAtomic(filepath.Join(stateDir, installedFile), data, 0o600)
|
||||
}
|
||||
|
||||
// LastInstall reads the host's installed record. ok is false when the file is
|
||||
// missing or malformed, as on a host that updated before install times were
|
||||
// recorded.
|
||||
func LastInstall(stateDir string) (version string, at time.Time, ok bool) {
|
||||
data, err := os.ReadFile(filepath.Join(stateDir, installedFile))
|
||||
if err != nil {
|
||||
return "", time.Time{}, false
|
||||
}
|
||||
var record installedRecord
|
||||
if err := json.Unmarshal(data, &record); err != nil || record.Version == "" || record.CommittedAt.IsZero() {
|
||||
return "", time.Time{}, false
|
||||
}
|
||||
return record.Version, record.CommittedAt, true
|
||||
}
|
||||
|
||||
// Run drives the transaction in the journal to an outcome. recovering is set
|
||||
// on boot: it never waits for the lock, and a transaction that was prepared
|
||||
// but never started is abandoned rather than started unattended.
|
||||
@@ -182,6 +216,9 @@ func (t *Transaction) promote(ctx context.Context, j journal) error {
|
||||
if err := t.setPhase(&j, phaseCommitted); err != nil {
|
||||
return err
|
||||
}
|
||||
if err := recordInstall(t.StateDir, j.Version, time.Now().UTC()); err != nil {
|
||||
return fmt.Errorf("record the install: %w", err)
|
||||
}
|
||||
return t.remove()
|
||||
}
|
||||
|
||||
@@ -252,6 +289,12 @@ func (t *Transaction) rollback(ctx context.Context, j journal) error {
|
||||
if err := t.setPhase(&j, phaseRolledBack); err != nil {
|
||||
return t.incomplete(err)
|
||||
}
|
||||
// The host is back on the previous release; installed.json follows it.
|
||||
if version := j.Previous[ComponentDaemon]; version != "" {
|
||||
if err := recordInstall(t.StateDir, version, time.Now().UTC()); err != nil {
|
||||
return t.incomplete(err)
|
||||
}
|
||||
}
|
||||
if err := t.startUnits(ctx, j, started); err != nil {
|
||||
return t.incomplete(err)
|
||||
}
|
||||
|
||||
@@ -21,6 +21,7 @@ import (
|
||||
digest "github.com/opencontainers/go-digest"
|
||||
ocispec "github.com/opencontainers/image-spec/specs-go/v1"
|
||||
|
||||
"github.com/samcharles93/archie-core/internal/buildinfo"
|
||||
"github.com/samcharles93/archie-core/internal/domain/distribution"
|
||||
"github.com/samcharles93/archie-core/internal/infrastructure/signedoci"
|
||||
)
|
||||
@@ -146,6 +147,11 @@ func TestTransaction(t *testing.T) {
|
||||
// the env file beside its config; pin both to this test's own files so
|
||||
// the result never depends on the machine's real archie config.
|
||||
t.Setenv(DatabasePasswordEnv, "test-password")
|
||||
// Stamp this binary as a release, as a host installed from one is, so the
|
||||
// journal's previous daemon version is a real version.
|
||||
stamped := buildinfo.Version
|
||||
buildinfo.Version = "1.0.0"
|
||||
defer func() { buildinfo.Version = stamped }()
|
||||
configPath := filepath.Join(t.TempDir(), "config.toml")
|
||||
if err := os.WriteFile(configPath, []byte("database_url = \"postgres://archie@/archie?host=/tmp/archie-test\"\n"), 0o600); err != nil {
|
||||
t.Fatal(err)
|
||||
@@ -257,10 +263,51 @@ func TestTransaction(t *testing.T) {
|
||||
found && (got.HealthCheck != tt.want.report || got.RolledBack != (tt.want.report == "failed") || got.RestoredDatabase != tt.want.restoredDatabase) {
|
||||
t.Errorf("report = %+v (found %v), want %+v", got, found, tt.want)
|
||||
}
|
||||
// A transaction that ran to an outcome records which release the
|
||||
// host now runs and when it was installed.
|
||||
if tt.prepare == "" && tt.crashAt == "" {
|
||||
if version, at, ok := LastInstall(tx.StateDir); !ok || version != tt.want.version || at.IsZero() {
|
||||
t.Errorf("installed.json after the transaction = %q, %v, %v; want %s", version, at, ok, tt.want.version)
|
||||
}
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// installed.json is the host's durable record of which release it last
|
||||
// committed or rolled back to, and when.
|
||||
func TestLastInstall(t *testing.T) {
|
||||
dir := t.TempDir()
|
||||
if _, _, ok := LastInstall(dir); ok {
|
||||
t.Fatal("a missing installed.json reports ok")
|
||||
}
|
||||
at := time.Date(2026, 1, 2, 3, 4, 5, 0, time.UTC)
|
||||
if err := recordInstall(dir, "1.2.3", at); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
var raw map[string]any
|
||||
data, err := os.ReadFile(filepath.Join(dir, installedFile))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := json.Unmarshal(data, &raw); err != nil {
|
||||
t.Fatalf("installed.json is not JSON: %v", err)
|
||||
}
|
||||
if raw["version"] != "1.2.3" || raw["committed_at"] != "2026-01-02T03:04:05Z" {
|
||||
t.Fatalf("installed.json = %v, want version 1.2.3 and committed_at 2026-01-02T03:04:05Z", raw)
|
||||
}
|
||||
version, got, ok := LastInstall(dir)
|
||||
if !ok || version != "1.2.3" || !got.Equal(at) {
|
||||
t.Fatalf("LastInstall = %q, %v, %v; want 1.2.3, %v, true", version, got, ok, at)
|
||||
}
|
||||
if err := os.WriteFile(filepath.Join(dir, installedFile), []byte("{not json"), 0o600); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, _, ok := LastInstall(dir); ok {
|
||||
t.Fatal("a malformed installed.json reports ok")
|
||||
}
|
||||
}
|
||||
|
||||
// releaseHandler serves one release's zip and SHA256SUMS the way GitHub
|
||||
// release downloads are laid out.
|
||||
func releaseHandler(t *testing.T, candidate string, corrupt bool) (http.Handler, ed25519.PublicKey) {
|
||||
|
||||
@@ -6,6 +6,7 @@ import (
|
||||
"errors"
|
||||
"fmt"
|
||||
"net/http"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"sync"
|
||||
@@ -15,6 +16,7 @@ import (
|
||||
|
||||
"github.com/samcharles93/archie-core/internal/domain/lifecycle"
|
||||
"github.com/samcharles93/archie-core/internal/domain/messaging"
|
||||
"github.com/samcharles93/archie-core/internal/infrastructure/configuration"
|
||||
"github.com/samcharles93/archie-core/internal/releaseupdate"
|
||||
)
|
||||
|
||||
@@ -513,6 +515,11 @@ func (s *Server) handleChatUpdate(w http.ResponseWriter, r *http.Request) {
|
||||
writeUpdateCheckError(w, err)
|
||||
return
|
||||
}
|
||||
// The snapshot carries when the installed release was committed on this
|
||||
// host, so a canary can count its window from the actual install.
|
||||
if _, at, ok := releaseupdate.LastInstall(updateStateDir()); ok {
|
||||
snapshot.InstalledAt = at
|
||||
}
|
||||
writeJSON(w, map[string]any{
|
||||
"snapshot": snapshot,
|
||||
"available": snapshot.Available(),
|
||||
@@ -520,6 +527,16 @@ func (s *Server) handleChatUpdate(w http.ResponseWriter, r *http.Request) {
|
||||
})
|
||||
}
|
||||
|
||||
// updateStateDir resolves the host's update state directory the same way the
|
||||
// daemon does, so the dashboard reads the installed.json its transactions
|
||||
// write.
|
||||
func updateStateDir() string {
|
||||
if dir := os.Getenv("ARCHIE_UPDATE_STATE_DIR"); dir != "" {
|
||||
return dir
|
||||
}
|
||||
return configuration.DefaultUpdateStateDir()
|
||||
}
|
||||
|
||||
func (s *Server) handleChatUpdateDefer(w http.ResponseWriter, r *http.Request) {
|
||||
chat, ok := s.chatReady(w)
|
||||
if !ok {
|
||||
|
||||
Executable
+352
@@ -0,0 +1,352 @@
|
||||
#!/bin/sh
|
||||
# Rehearse the release-canary loop against a throwaway local host. Nothing
|
||||
# here touches production: a local OCI registry (docker run registry:2 on a
|
||||
# loopback port), a throwaway signing key, a test release published to that
|
||||
# registry (a stamped archied binary and a real agent image in a proper
|
||||
# distribution zip), and a local stack (state store, gateway, archied,
|
||||
# archie-ui) with its own config, state dir and ports. The loop's commands --
|
||||
# release canary start, release canary status, release canary check and
|
||||
# release canary promote -- are then run against that host, each step printed.
|
||||
# The local host has no systemd, so once the install has switched the binary
|
||||
# the script restarts archied itself, the way an unmanaged host's operator
|
||||
# would.
|
||||
#
|
||||
# A step that fails is printed and makes the run exit nonzero, so a rehearsal
|
||||
# never looks greener than it was. The only hard precondition is docker:
|
||||
# without it there is no registry, PostgreSQL or agent image.
|
||||
#
|
||||
# Never part of the gate: task check and CI do not call this.
|
||||
set -eu
|
||||
|
||||
repo=$(CDPATH= cd -- "$(dirname -- "$0")/.." && pwd)
|
||||
work=$(mktemp -d "${TMPDIR:-/tmp}/archie-canary-rehearse.XXXXXX")
|
||||
name=$(basename "$work")
|
||||
bin=$work/bin
|
||||
logs=$work/logs
|
||||
|
||||
pids=""
|
||||
containers=""
|
||||
keep=0 # set when a step fails: the workdir is kept for inspection
|
||||
cleanup() {
|
||||
trap - EXIT INT TERM
|
||||
for pid in $pids; do kill -- "-$pid" 2>/dev/null || kill "$pid" 2>/dev/null || true; done
|
||||
for c in $containers; do docker rm -f "$c" >/dev/null 2>&1 || true; done
|
||||
if [ "$keep" -gt 0 ]; then
|
||||
echo "canary-rehearse: keeping $work for inspection" >&2
|
||||
else
|
||||
rm -rf "$work"
|
||||
fi
|
||||
}
|
||||
trap cleanup EXIT INT TERM
|
||||
|
||||
die() {
|
||||
echo "canary-rehearse: $*" >&2
|
||||
exit 1
|
||||
}
|
||||
|
||||
# --- preflight ---------------------------------------------------------------
|
||||
|
||||
for tool in go curl python3 zip sha256sum timeout setsid openssl docker; do
|
||||
command -v "$tool" >/dev/null 2>&1 || die "$tool is required but not on PATH"
|
||||
done
|
||||
if ! docker info >/dev/null 2>&1; then
|
||||
die "docker is unavailable (docker info failed): the rehearsal needs it for the local OCI registry and PostgreSQL. Start the docker daemon or skip task canary:rehearse"
|
||||
fi
|
||||
|
||||
release_version=${CANARY_REHEARSE_VERSION:-9.9.9-canary.1}
|
||||
host_version="0.0.0-rehearse"
|
||||
|
||||
# Seven loopback ports, picked by binding port 0.
|
||||
set -- $(python3 -c '
|
||||
import socket
|
||||
ports = []
|
||||
while len(ports) < 7:
|
||||
s = socket.socket()
|
||||
s.bind(("127.0.0.1", 0))
|
||||
ports.append(s.getsockname()[1])
|
||||
s.close()
|
||||
print(" ".join(map(str, ports)))
|
||||
')
|
||||
reg_port=$1 pg_port=$2 web_port=$3 state_port=$4 gw_port=$5 daemon_port=$6 health_port=$7
|
||||
|
||||
# --- registry and PostgreSQL -------------------------------------------------
|
||||
|
||||
echo "== starting local OCI registry on 127.0.0.1:$reg_port and PostgreSQL on 127.0.0.1:$pg_port"
|
||||
docker run -d --name "$name-registry" -p "127.0.0.1:$reg_port:5000" registry:2 >/dev/null ||
|
||||
die "could not start registry:2 (is the image pullable?)"
|
||||
containers="$containers $name-registry"
|
||||
docker run -d --name "$name-pg" \
|
||||
-e POSTGRES_USER=archie -e POSTGRES_PASSWORD=archie -e POSTGRES_DB=archie \
|
||||
-p "127.0.0.1:$pg_port:5432" postgres:18 >/dev/null ||
|
||||
die "could not start postgres:18 (is the image pullable?)"
|
||||
containers="$containers $name-pg"
|
||||
|
||||
# --- build the host's binaries (no_ui: the rehearsal never serves the SPA) ----
|
||||
|
||||
echo "== building archie-state-store, archie-gateway, archied, archie-ui (no_ui)"
|
||||
mkdir -p "$bin" "$logs"
|
||||
ldflags="-X github.com/samcharles93/archie-core/internal/installtype.buildType=binary -X github.com/samcharles93/archie-core/internal/buildinfo.Version=$host_version"
|
||||
for svc in archie-state-store archie-gateway archied archie-ui; do
|
||||
go -C "$repo" build -tags no_ui -trimpath -ldflags "$ldflags" -o "$bin/$svc" "./cmd/$svc" ||
|
||||
die "go build ./cmd/$svc failed"
|
||||
done
|
||||
|
||||
wait_http() { # url label tries
|
||||
i=0
|
||||
while [ "$i" -lt "${3:-30}" ]; do
|
||||
if curl -fsS --max-time 2 "$1" >/dev/null 2>&1; then return 0; fi
|
||||
i=$((i + 1)); sleep 1
|
||||
done
|
||||
die "$2 did not answer at $1 within ${3:-30}s"
|
||||
}
|
||||
wait_http "http://127.0.0.1:$reg_port/v2/" "the local registry" 30
|
||||
|
||||
i=0
|
||||
while ! docker exec "$name-pg" pg_isready -h 127.0.0.1 -p 5432 -U archie >/dev/null 2>&1; do
|
||||
i=$((i + 1))
|
||||
[ "$i" -gt 90 ] && die "PostgreSQL did not become ready within 90s"
|
||||
sleep 1
|
||||
done
|
||||
|
||||
# --- throwaway signing key and a test release --------------------------------
|
||||
|
||||
echo "== generating a throwaway release signing key"
|
||||
pub=$("$bin/archied" release keygen -seed "$work/seed") || die "release keygen failed"
|
||||
[ -n "$pub" ] || die "release keygen printed no public key"
|
||||
seed=$(cat "$work/seed")
|
||||
|
||||
# A real agent image in the local registry, so the install's image pull by
|
||||
# digest succeeds. The Dockerfile stamps the agent with the release version.
|
||||
echo "== building and pushing the archie-agent test image (this takes a few minutes)"
|
||||
docker build -q -f "$repo/Dockerfile" --build-arg RUNTIME_VERSION="$release_version" \
|
||||
-t "127.0.0.1:$reg_port/archie-agent:v$release_version" "$repo" >/dev/null ||
|
||||
die "docker build of the agent image failed"
|
||||
docker push -q "127.0.0.1:$reg_port/archie-agent:v$release_version" >/dev/null ||
|
||||
die "docker push of the agent image failed"
|
||||
agent_image=$(docker image inspect -f '{{range .RepoDigests}}{{println .}}{{end}}' \
|
||||
"127.0.0.1:$reg_port/archie-agent:v$release_version" | grep "^127\.0\.0\.1:$reg_port/" | head -1)
|
||||
[ -n "$agent_image" ] || die "could not resolve the pushed agent image digest"
|
||||
|
||||
# A proper distribution zip, built the way deploy.yml builds it: every binary
|
||||
# the repo's release.json names, each stamped with the release version, plus
|
||||
# the manifest with the agent image pinned in.
|
||||
dist="archie-core-archied-v$release_version-linux-amd64"
|
||||
echo "== building the stamped release binaries and zipping $dist.zip"
|
||||
mkdir -p "$work/dist/$dist"
|
||||
release_ldflags="-X github.com/samcharles93/archie-core/internal/installtype.buildType=binary -X github.com/samcharles93/archie-core/internal/buildinfo.Version=$release_version -X github.com/samcharles93/archie-core/internal/buildinfo.Runtime=$release_version -X github.com/samcharles93/archie-core/internal/app/agentworker.version=$release_version"
|
||||
for cmd in $(python3 -c 'import json; print(" ".join(json.load(open("'"$repo"'/release.json"))["required_topology"]["units"] + json.load(open("'"$repo"'/release.json"))["required_topology"].get("binaries", [])))'); do
|
||||
go -C "$repo" build -tags no_ui -trimpath -ldflags "$release_ldflags" \
|
||||
-o "$work/dist/$dist/$cmd" "./cmd/$cmd" || die "go build of $cmd failed"
|
||||
done
|
||||
python3 -c 'import json, sys
|
||||
m = json.load(open(sys.argv[1])); m["agent_image"] = sys.argv[2]
|
||||
print(json.dumps(m))' "$repo/release.json" "$agent_image" > "$work/dist/$dist/release.json"
|
||||
(CDPATH= cd -- "$work/dist" && zip -q -r "$work/release.zip" "$dist")
|
||||
|
||||
echo "== publishing test release $release_version to 127.0.0.1:$reg_port/archie"
|
||||
ARCHIE_RELEASE_SEED=$seed "$bin/archied" release publish \
|
||||
-source "127.0.0.1:$reg_port/archie" -version "$release_version" \
|
||||
-zip "$work/release.zip" -agent-image "$agent_image" ||
|
||||
die "release publish failed"
|
||||
|
||||
# --- the local host ----------------------------------------------------------
|
||||
|
||||
export XDG_CONFIG_HOME=$work/config XDG_DATA_HOME=$work/data XDG_STATE_HOME=$work/state
|
||||
export ARCHIE_UPDATE_STATE_DIR=$work/update-state
|
||||
# Point XDG_RUNTIME_DIR at a path that doesn't exist, so systemctl --user has
|
||||
# no bus: the rehearsal host reports unmanaged and the install switches the
|
||||
# binaries without trying to restart units through systemd.
|
||||
export XDG_RUNTIME_DIR=$work/no-user-bus
|
||||
mkdir -p "$XDG_CONFIG_HOME/archie" "$XDG_DATA_HOME" "$XDG_STATE_HOME" "$ARCHIE_UPDATE_STATE_DIR"
|
||||
|
||||
cfg=$XDG_CONFIG_HOME/archie/config.toml
|
||||
# A throwaway CA and per-service certificates, the way `task dev:tls` issues
|
||||
# them: archie-ui dials the daemon's harness sessions over mTLS even on
|
||||
# loopback, so the host cannot come up without [tls].
|
||||
tls=$work/tls
|
||||
mkdir -p "$tls"
|
||||
openssl req -x509 -newkey ec -pkeyopt ec_paramgen_curve:prime256v1 -nodes -days 3650 \
|
||||
-subj "/CN=archie-rehearse-ca" -keyout "$tls/ca-key.pem" -out "$tls/ca.pem" 2>/dev/null ||
|
||||
die "could not create the throwaway CA"
|
||||
for s in archied archie-gateway archie-messaging archie-state-store archie-ui; do
|
||||
openssl req -x509 -newkey ec -pkeyopt ec_paramgen_curve:prime256v1 -nodes -days 3650 \
|
||||
-subj "/CN=$s" -addext "subjectAltName=DNS:$s" -addext extendedKeyUsage=serverAuth,clientAuth \
|
||||
-CA "$tls/ca.pem" -CAkey "$tls/ca-key.pem" -keyout "$tls/$s-key.pem" -out "$tls/$s.pem" 2>/dev/null ||
|
||||
die "could not issue a certificate for $s"
|
||||
done
|
||||
sed \
|
||||
-e "s|^# database_url = .*|database_url = \"postgres://archie:archie@127.0.0.1:$pg_port/archie?sslmode=disable\"|" \
|
||||
-e "s|^listen = \"127.0.0.1:8485\"|listen = \"127.0.0.1:$health_port\"|" \
|
||||
-e "s|^listen = \"127.0.0.1:8484\"|listen = \"127.0.0.1:$web_port\"|" \
|
||||
-e "s|^target = \"127.0.0.1:8585\"|target = \"127.0.0.1:$gw_port\"\nlisten = \"127.0.0.1:$gw_port\"|" \
|
||||
-e "s|^target = \"127.0.0.1:9090\"|target = \"127.0.0.1:$state_port\"\nlisten = \"127.0.0.1:$state_port\"|" \
|
||||
"$repo/config.example.toml" > "$cfg"
|
||||
cat >> "$cfg" <<EOF
|
||||
|
||||
[services.daemon]
|
||||
target = "127.0.0.1:$daemon_port"
|
||||
listen = "127.0.0.1:$daemon_port"
|
||||
|
||||
[tls]
|
||||
ca_file = "$tls/ca.pem"
|
||||
cert_dir = "$tls"
|
||||
EOF
|
||||
|
||||
spawn() { # name command...
|
||||
svc=$1; shift
|
||||
setsid "$@" >"$logs/$svc.log" 2>&1 &
|
||||
pids="$pids $!"
|
||||
}
|
||||
|
||||
wait_port() { # port label tries
|
||||
i=0
|
||||
while [ "$i" -lt "${3:-90}" ]; do
|
||||
if python3 -c "import socket,sys; s=socket.socket(); s.settimeout(1); sys.exit(0 if s.connect_ex(('127.0.0.1', $1)) == 0 else 1)"; then
|
||||
return 0
|
||||
fi
|
||||
i=$((i + 1)); sleep 1
|
||||
done
|
||||
die "$2 did not listen on 127.0.0.1:$1; see $logs/$2.log (removed on exit -- rerun to inspect)"
|
||||
}
|
||||
|
||||
host_pids=""
|
||||
start_host() {
|
||||
spawn archie-state-store "$bin/archie-state-store"
|
||||
wait_port "$state_port" archie-state-store
|
||||
# The gateway only listens once archied has published its embedded NATS
|
||||
# endpoint, so the rest start together, as e2e/stack.sh does.
|
||||
spawn archie-gateway "$bin/archie-gateway"
|
||||
spawn archied "$bin/archied"
|
||||
spawn archie-ui "$bin/archie-ui"
|
||||
host_pids="$pids"
|
||||
wait_port "$gw_port" archie-gateway
|
||||
wait_port "$daemon_port" archied
|
||||
wait_port "$web_port" archie-ui
|
||||
}
|
||||
|
||||
echo "== starting the local host (state store, gateway, archied, archie-ui)"
|
||||
start_host
|
||||
|
||||
base="http://127.0.0.1:$web_port"
|
||||
cookie=$work/cookies.txt
|
||||
|
||||
api() { # method path [body]
|
||||
method=$1 path=$2 body=${3-}
|
||||
if [ -n "$body" ]; then
|
||||
curl -sS --max-time 15 -b "$cookie" -c "$cookie" -X "$method" "$base$path" \
|
||||
-H 'Content-Type: application/json' -H 'X-Archie-CSRF: 1' -H "Origin: $base" -d "$body"
|
||||
else
|
||||
curl -sS --max-time 15 -b "$cookie" -c "$cookie" "$base$path"
|
||||
fi
|
||||
}
|
||||
|
||||
py() { python3 -c "import sys, json; d = json.load(sys.stdin); print($1)"; }
|
||||
|
||||
# One throwaway account on the throwaway host, then a personal API token,
|
||||
# which is what the canary commands authenticate with.
|
||||
email="canary-rehearse@example.test"
|
||||
password="archie-canary-rehearse-password"
|
||||
account=$(python3 -c 'import json, sys
|
||||
print(json.dumps(dict(email=sys.argv[1], password=sys.argv[2], display_name="Canary rehearsal",
|
||||
organisation_name="Canary rehearsal", workspace_name="Rehearsal", public_url=sys.argv[3])))' \
|
||||
"$email" "$password" "$base")
|
||||
setup=$(api GET /api/auth/status)
|
||||
if printf '%s' "$setup" | py 'd["setup_required"]' | grep -q True; then
|
||||
api POST /api/auth/setup "$account" >/dev/null || die "account setup failed"
|
||||
else
|
||||
api POST /api/auth/login "$account" >/dev/null || die "login failed: $setup"
|
||||
fi
|
||||
token=$(api POST /api/tokens '{"name":"canary-rehearse","days":7}' | py 'd["token"]') ||
|
||||
die "could not create a personal API token on the local host"
|
||||
|
||||
# Point the host's update settings at the local registry and key.
|
||||
doc=$(api GET /api/control-plane/resources/update-settings)
|
||||
ver=$(printf '%s' "$doc" | py 'd["resource"]["version"]')
|
||||
value=$(python3 -c 'import json, sys
|
||||
print(json.dumps({"channel": "stable", "pin": "", "source": sys.argv[1], "key": sys.argv[2]}))' \
|
||||
"127.0.0.1:$reg_port/archie" "$pub")
|
||||
resp=$(api POST /api/control-plane/resources/update-settings/commands/replace \
|
||||
"{\"value\":$value,\"expected_version\":$ver}")
|
||||
printf '%s' "$resp" | grep -q '"resource"' || die "could not set update-settings on the local host: $resp"
|
||||
|
||||
# --- the canary loop, each step printed ---------------------------------------
|
||||
|
||||
failed=0
|
||||
run_step() { # description command...
|
||||
desc=$1; shift
|
||||
echo
|
||||
echo "== $desc"
|
||||
printf '$ %s\n' "$*"
|
||||
if "$@" 2>&1; then
|
||||
echo "-- ok"
|
||||
else
|
||||
rc=$?
|
||||
echo "-- FAILED (exit $rc)"
|
||||
failed=$((failed + 1))
|
||||
keep=1
|
||||
fi
|
||||
}
|
||||
|
||||
export ARCHIE_CANARY_TOKEN=$token
|
||||
|
||||
echo
|
||||
echo "== release canary start -- point the host at next and install $release_version"
|
||||
printf '$ archied release canary start -dashboard %s -version %s -source 127.0.0.1:%s/archie -key <pub>\n' \
|
||||
"$base" "$release_version" "$reg_port"
|
||||
timeout 600 "$bin/archied" release canary start -dashboard "$base" -version "$release_version" \
|
||||
-source "127.0.0.1:$reg_port/archie" -key "$pub" >"$work/start.out" 2>&1 &
|
||||
start_pid=$!
|
||||
# The local host has no systemd: the install switches the binary and records
|
||||
# the install time, but nothing restarts the daemon. Restart archied once the
|
||||
# install has committed, the way an unmanaged host's operator would, so the
|
||||
# host reports the new version.
|
||||
i=0
|
||||
while [ ! -f "$ARCHIE_UPDATE_STATE_DIR/installed.json" ]; do
|
||||
kill -0 "$start_pid" 2>/dev/null || break
|
||||
i=$((i + 1)); [ "$i" -gt 300 ] && break
|
||||
sleep 1
|
||||
done
|
||||
if [ -f "$ARCHIE_UPDATE_STATE_DIR/installed.json" ]; then
|
||||
# Restart every service, not just the daemon: the update snapshot reports
|
||||
# the serving binary's version, so the whole host must come back on the
|
||||
# new release.
|
||||
for pid in $host_pids; do kill "$pid" 2>/dev/null || true; done
|
||||
sleep 1
|
||||
pids=""
|
||||
start_host
|
||||
fi
|
||||
start_rc=0
|
||||
wait "$start_pid" || start_rc=$?
|
||||
cat "$work/start.out"
|
||||
if [ "$start_rc" -eq 0 ]; then
|
||||
echo "-- ok"
|
||||
else
|
||||
echo "-- FAILED (exit $start_rc)"
|
||||
failed=$((failed + 1))
|
||||
keep=1
|
||||
fi
|
||||
|
||||
run_step "release canary status -- what the canary runs and how the window is going" \
|
||||
timeout 120 "$bin/archied" release canary status -dashboard "$base" \
|
||||
-source "127.0.0.1:$reg_port/archie" -key "$pub"
|
||||
|
||||
run_step "release canary check -- judge the installed release (still settling right after the install)" \
|
||||
timeout 300 "$bin/archied" release canary check -dashboard "$base" \
|
||||
-source "127.0.0.1:$reg_port/archie" -key "$pub" -checkout "$repo"
|
||||
|
||||
run_step "release canary promote -- promote $release_version now that its hard checks pass" \
|
||||
timeout 120 "$bin/archied" release canary promote -dashboard "$base" -version "$release_version" \
|
||||
-source "127.0.0.1:$reg_port/archie" -key "$pub"
|
||||
|
||||
echo
|
||||
if [ "$failed" -gt 0 ]; then
|
||||
echo "canary-rehearse: $failed step(s) failed; the steps above show why." >&2
|
||||
if [ "$keep" -gt 0 ] && [ -d "$logs" ]; then
|
||||
for f in "$logs"/*.log; do
|
||||
echo "--- $f" >&2
|
||||
tail -n 20 "$f" >&2
|
||||
done
|
||||
fi
|
||||
exit 1
|
||||
fi
|
||||
echo "canary-rehearse: every step ran clean; host was $base, registry 127.0.0.1:$reg_port (both torn down)"
|
||||
@@ -0,0 +1,184 @@
|
||||
<script setup lang="ts">
|
||||
import { computed, onMounted, onUnmounted, ref } from "vue";
|
||||
|
||||
import { Badge } from "@/components/ui/badge";
|
||||
import { MetadataList } from "@/components/ui/metadata-list";
|
||||
import { Progress } from "@/components/ui/progress";
|
||||
import { Section } from "@/components/ui/section";
|
||||
import { Timestamp } from "@/components/ui/timestamp";
|
||||
import { ApiError, api } from "@/lib/api";
|
||||
import { useSettingsStore } from "@/stores/settings";
|
||||
|
||||
/**
|
||||
* The release canary on this host, read-only: the update settings in force,
|
||||
* the installed release, how long it has been installed and how far through
|
||||
* the default canary window that is. Installing and promoting releases run
|
||||
* in the update UI and CI, not here.
|
||||
*/
|
||||
|
||||
// The wire shape of GET /api/chat/update: Go's default field names, except
|
||||
// installed_at, which the server tags. Zero time means no install recorded.
|
||||
interface UpdateComponent {
|
||||
ID: string;
|
||||
Label: string;
|
||||
Installed: string;
|
||||
Available: string;
|
||||
Reference?: string;
|
||||
Changelog?: string;
|
||||
}
|
||||
|
||||
interface UpdateSnapshot {
|
||||
Components: UpdateComponent[];
|
||||
Deferred?: boolean;
|
||||
installed_at?: string;
|
||||
}
|
||||
|
||||
interface ChatUpdate {
|
||||
snapshot: UpdateSnapshot;
|
||||
available: UpdateComponent[] | null;
|
||||
can_install: boolean;
|
||||
}
|
||||
|
||||
interface UpdateSettings {
|
||||
channel?: string;
|
||||
pin?: string;
|
||||
}
|
||||
|
||||
// The default window a release runs on the canary before it can be promoted,
|
||||
// mirroring releaseupdate.CanaryWindow on the server.
|
||||
const CANARY_WINDOW_MS = 24 * 60 * 60 * 1000;
|
||||
|
||||
const KIND = "update-settings";
|
||||
|
||||
const store = useSettingsStore();
|
||||
const update = ref<ChatUpdate | null>(null);
|
||||
const notConfigured = ref(false);
|
||||
const loading = ref(true);
|
||||
const error = ref<string | null>(null);
|
||||
const now = ref(Date.now());
|
||||
let timer: ReturnType<typeof setInterval> | undefined;
|
||||
let alive = true;
|
||||
|
||||
const settings = computed(() => (store.stateFor(KIND).resource?.value ?? {}) as UpdateSettings);
|
||||
const channel = computed(() => settings.value.channel || "stable");
|
||||
const pin = computed(() => settings.value.pin || "");
|
||||
const pinned = computed(() => channel.value === "exact-pin");
|
||||
|
||||
const channelLabel: Record<string, string> = {
|
||||
stable: "Stable",
|
||||
next: "Prereleases",
|
||||
"exact-pin": "Pinned",
|
||||
};
|
||||
|
||||
const daemon = computed(() => update.value?.snapshot.Components.find((c) => c.ID === "daemon"));
|
||||
const available = computed(() => update.value?.available ?? []);
|
||||
|
||||
const installedAt = computed(() => {
|
||||
const raw = update.value?.snapshot.installed_at;
|
||||
if (!raw) return null;
|
||||
const at = new Date(raw);
|
||||
// The zero time serialises as year 1: no install was recorded.
|
||||
if (Number.isNaN(at.getTime()) || at.getFullYear() <= 1) return null;
|
||||
return at;
|
||||
});
|
||||
const elapsedMs = computed(() =>
|
||||
installedAt.value ? Math.max(0, now.value - installedAt.value.getTime()) : null,
|
||||
);
|
||||
const windowProgress = computed(() =>
|
||||
elapsedMs.value == null ? null : Math.min(100, (elapsedMs.value / CANARY_WINDOW_MS) * 100),
|
||||
);
|
||||
const elapsedText = computed(() => (elapsedMs.value == null ? "" : elapsed(elapsedMs.value)));
|
||||
|
||||
function elapsed(ms: number): string {
|
||||
const totalMinutes = Math.floor(ms / 60_000);
|
||||
const days = Math.floor(totalMinutes / 1440);
|
||||
const hours = Math.floor((totalMinutes % 1440) / 60);
|
||||
const minutes = totalMinutes % 60;
|
||||
if (days) return `${days}d ${hours}h`;
|
||||
if (hours) return `${hours}h ${minutes}m`;
|
||||
return `${minutes}m`;
|
||||
}
|
||||
|
||||
const rows = computed(() => {
|
||||
const list: { label: string; value?: string; mono?: boolean }[] = [
|
||||
{ label: "Channel", value: channelLabel[channel.value] ?? channel.value },
|
||||
{ label: "Installed", value: daemon.value?.Installed || "unknown", mono: true },
|
||||
{ label: "Installed at", value: installedAt.value ? "" : "not recorded" },
|
||||
{
|
||||
label: "Available",
|
||||
value: available.value.length
|
||||
? available.value.map((c) => `${c.Label || c.ID} ${c.Installed} → ${c.Available}`).join(", ")
|
||||
: "none",
|
||||
mono: true,
|
||||
},
|
||||
];
|
||||
return list;
|
||||
});
|
||||
|
||||
async function load(): Promise<void> {
|
||||
loading.value = true;
|
||||
error.value = null;
|
||||
try {
|
||||
const fresh = await api.updateSnapshot<ChatUpdate>();
|
||||
if (!alive) return;
|
||||
update.value = fresh;
|
||||
notConfigured.value = false;
|
||||
} catch (err) {
|
||||
if (!alive) return;
|
||||
if (err instanceof ApiError && err.status === 501) {
|
||||
notConfigured.value = true;
|
||||
update.value = null;
|
||||
} else {
|
||||
error.value = err instanceof Error ? err.message : String(err);
|
||||
}
|
||||
} finally {
|
||||
if (alive) loading.value = false;
|
||||
}
|
||||
}
|
||||
|
||||
onMounted(() => {
|
||||
void store.load();
|
||||
void load();
|
||||
timer = setInterval(() => {
|
||||
now.value = Date.now();
|
||||
void load();
|
||||
}, 15_000);
|
||||
});
|
||||
onUnmounted(() => {
|
||||
alive = false;
|
||||
if (timer) clearInterval(timer);
|
||||
});
|
||||
</script>
|
||||
|
||||
<template>
|
||||
<Section title="Release canary">
|
||||
<p v-if="notConfigured" class="text-sm text-fg-muted">Update checks not configured.</p>
|
||||
<template v-else>
|
||||
<p v-if="error" class="text-sm text-danger">{{ error }}</p>
|
||||
<p v-else-if="loading && !update" class="text-sm text-fg-muted">Checking…</p>
|
||||
<template v-else>
|
||||
<MetadataList :rows="rows">
|
||||
<template #value="{ row }">
|
||||
<template v-if="row.label === 'Channel' && pinned">
|
||||
<Badge variant="danger" class="mr-2">held</Badge>
|
||||
<span class="font-mono">{{ pin }}</span>
|
||||
</template>
|
||||
<template v-else-if="row.label === 'Installed at' && installedAt">
|
||||
<Timestamp :at="installedAt" /> · {{ elapsedText }} ago
|
||||
</template>
|
||||
<template v-else>{{ row.value }}</template>
|
||||
</template>
|
||||
</MetadataList>
|
||||
<div v-if="windowProgress != null" class="mt-3">
|
||||
<Progress :model-value="windowProgress" aria-label="Canary window elapsed" />
|
||||
<p class="mt-1 text-caption text-fg-muted">
|
||||
{{ elapsedText }} of 24h elapsed (default window)
|
||||
</p>
|
||||
</div>
|
||||
<p v-else class="mt-3 text-caption text-fg-muted">
|
||||
Install time not recorded, so the window cannot be measured.
|
||||
</p>
|
||||
</template>
|
||||
</template>
|
||||
</Section>
|
||||
</template>
|
||||
@@ -6,6 +6,7 @@ import PageHeader from "@/base/PageHeader.vue";
|
||||
import { Alert, AlertDescription, AlertTitle } from "@/components/ui/alert";
|
||||
import { Section } from "@/components/ui/section";
|
||||
import { Table, TableBody, TableCell, TableRow } from "@/components/ui/table";
|
||||
import CanaryStatusCard from "./CanaryStatusCard.vue";
|
||||
import ServicesCard from "./ServicesCard.vue";
|
||||
import UpdateSettingsCard from "./UpdateSettingsCard.vue";
|
||||
import VersionsCard from "./VersionsCard.vue";
|
||||
@@ -32,6 +33,7 @@ onUnmounted(() => clearInterval(timer));
|
||||
<ServicesCard />
|
||||
<VersionsCard />
|
||||
<UpdateSettingsCard />
|
||||
<CanaryStatusCard />
|
||||
<Section v-if="config?.bootstrap && Object.keys(config.bootstrap).length" title="Startup settings">
|
||||
<Table>
|
||||
<TableBody>
|
||||
|
||||
Reference in new issue
Block a user