feat(releaseupdate): publish to next and promote to stable only after a canary evaluation (#1716)
This commit is contained in:
11 files changed
+1106
-119
No files matched your search
@@ -0,0 +1,47 @@
|
||||
# Judges the release on next against the canary every quarter hour: a hard
|
||||
# failure rolls the canary back at once, and a release that passes a full
|
||||
# window is promoted to stable. A rollback turns the run red.
|
||||
name: canary
|
||||
|
||||
on:
|
||||
schedule:
|
||||
- cron: "*/15 * * * *"
|
||||
workflow_dispatch:
|
||||
|
||||
concurrency:
|
||||
group: canary
|
||||
|
||||
permissions:
|
||||
contents: read
|
||||
packages: write
|
||||
|
||||
jobs:
|
||||
judge:
|
||||
if: vars.CANARY_URL != ''
|
||||
runs-on: self-hosted
|
||||
steps:
|
||||
# Full history with tags: a rollback compares the migrations between
|
||||
# the stable and failed release tags.
|
||||
- name: Checkout
|
||||
uses: actions/checkout@v4
|
||||
with:
|
||||
ref: main
|
||||
fetch-depth: 0
|
||||
|
||||
- name: Set up Go
|
||||
uses: actions/setup-go@v5
|
||||
with:
|
||||
go-version-file: go.mod
|
||||
|
||||
- name: Build archied
|
||||
run: go build -o "$RUNNER_TEMP/archied" ./cmd/archied
|
||||
|
||||
# Promotion retags stable in the registry.
|
||||
- name: Log in to the Gitea registry
|
||||
run: echo "${{ secrets.DEPLOY_TOKEN }}" | docker login git.catlow.cloud -u "${{ github.actor }}" --password-stdin
|
||||
|
||||
- name: Judge the canary
|
||||
env:
|
||||
ARCHIE_CANARY_TOKEN: ${{ secrets.CANARY_TOKEN }}
|
||||
run: |
|
||||
"$RUNNER_TEMP/archied" release canary check -dashboard "${{ vars.CANARY_URL }}"
|
||||
@@ -353,8 +353,8 @@ jobs:
|
||||
|
||||
# Installs and updates take a release from the signed artifact in the
|
||||
# registry, so this is what publishes it: the statement signed with
|
||||
# RELEASE_SIGNING_KEY and the zip, tagged v<version>, next, and stable
|
||||
# unless it is a prerelease.
|
||||
# RELEASE_SIGNING_KEY and the zip, tagged v<version> and next. stable
|
||||
# moves only when canary.yml promotes it.
|
||||
- name: Log in to the Gitea registry
|
||||
run: echo "${{ secrets.DEPLOY_TOKEN }}" | docker login git.catlow.cloud -u "${{ github.actor }}" --password-stdin
|
||||
|
||||
@@ -402,6 +402,14 @@ jobs:
|
||||
"$api/repos/$repo/releases" >/dev/null
|
||||
echo "published release notes for $tag"
|
||||
|
||||
# The canary follows next; canary.yml judges the release on it and
|
||||
# promotes it to stable or rolls the canary back.
|
||||
- name: Install the release on the canary
|
||||
if: vars.CANARY_URL != ''
|
||||
env:
|
||||
ARCHIE_CANARY_TOKEN: ${{ secrets.CANARY_TOKEN }}
|
||||
run: ./dist/archied release canary start -dashboard "${{ vars.CANARY_URL }}" -version "${{ steps.version.outputs.version }}"
|
||||
|
||||
# A Gitea package belongs to its owner, not a repository, so the ones this
|
||||
# workflow publishes are linked here. Linking is per package and a no-op
|
||||
# once done, so an already linked package is not an error.
|
||||
|
||||
@@ -5,6 +5,10 @@ release's per-component sections are labelled beneath its heading.
|
||||
|
||||
## [Unreleased]
|
||||
|
||||
### archied
|
||||
|
||||
- A release reaches `stable` only after it has run on a canary for a day and passed: `archied release publish` now publishes to `next` alone, `archied release canary start|check` installs a release on a canary and judges it from the canary's dashboard, promoting it or rolling the canary back, and `archied release promote` moves `stable` by hand. Promotion refuses a prerelease and a release older than the one `stable` names.
|
||||
|
||||
## [1.56.0] - 2026-10-31
|
||||
|
||||
### archied
|
||||
|
||||
+109
-112
@@ -1,141 +1,138 @@
|
||||
# Releasing
|
||||
|
||||
One release stream. `CHANGELOG.md` carries every release section, one generated
|
||||
page per version describes it, and one tag names the release. A release is one
|
||||
version covering whatever changed; the components' entries are labelled sections
|
||||
inside it (`### archied`, `### archie-agent`), not separate numbers or separate
|
||||
pages.
|
||||
page per version describes it, and one `v*` tag names the release. One version
|
||||
covers every process in the host bundle (`archied`, `archie-gateway`,
|
||||
`archie-state-store`, `archie-ui`, `archie-messaging`) and the `archie-agent`
|
||||
image; per-component detail is a labelled section inside the version
|
||||
(`### archied`, `### archie-agent`).
|
||||
|
||||
The host bundle ships five processes (`archied`, `archie-gateway`,
|
||||
`archie-state-store`, `archie-ui`, `archie-messaging`), and
|
||||
one version covers all of them. `archie-agent`, the per-task sandboxed runtime,
|
||||
ships as a container image and its changes are the `## [version]` release's
|
||||
`### archie-agent` section.
|
||||
A release is cut by CI, published to `next`, run on the canary for a day, and
|
||||
promoted to `stable` only if the canary's signals pass. Nothing is run locally.
|
||||
|
||||
`tools/release.sh` decides which commits a release carries from the package
|
||||
closure of **every shipped binary**, not from commit-message guessing: a change
|
||||
to a package only the messaging service, state store, playbook runner or UI
|
||||
links still lands in the changelog. Read its header comment for how it
|
||||
partitions commits and enforces `main`-only, clean-tree releases.
|
||||
## Notes are written with the change
|
||||
|
||||
## What justifies a version
|
||||
|
||||
`task release:preview VERSION=X.Y.Z` shows exactly what would land, before
|
||||
anything is written. A commit set that justifies no version produces no version:
|
||||
`release.sh` exits **3** with "nothing to release" rather than inventing one, so
|
||||
a pipeline can finish green without tagging.
|
||||
|
||||
Most sessions touch the gateway side (daemon, gateway, web UI, chat, messaging)
|
||||
and leave the agent runtime alone. That is no longer a per-component `skip`
|
||||
decision — one version covers the release — but a runtime code change being
|
||||
merged to `main` is still not the same as it being in production: the
|
||||
`archie-agent` image is rebuilt by the release, and a host that runs the agent
|
||||
must pull the new image (`docker compose pull agent`) to get it.
|
||||
|
||||
## What the notes are for
|
||||
A commit that changes what a reader experiences adds its bullet under
|
||||
`## [Unreleased]` in `CHANGELOG.md`, in the same commit. The release takes that
|
||||
section as it stands, so write the bullet as the release note, not as a commit
|
||||
subject.
|
||||
|
||||
**Release notes describe the reader's experience of what shipped.** The test is
|
||||
the reader, not whether a sentence is negative:
|
||||
|
||||
- A capability that **never existed** leaves no expectation to correct, so a
|
||||
note about it reads as a feature to someone who never had the thing — out.
|
||||
- **The implementation's journey** — what was rebuilt, what a guard finally
|
||||
covers, what a browser API refuses to surface — is not the reader's experience
|
||||
either — out.
|
||||
note about it reads as a feature to someone who never had the thing: out.
|
||||
- **The implementation's journey** (what was rebuilt, what a guard finally
|
||||
covers, what a browser API refuses to surface) is not the reader's experience
|
||||
either: out.
|
||||
- A capability that **shipped but is not yet wired** _is_ the reader's
|
||||
experience, so it belongs, worded as a caveat on the capability and paired
|
||||
with the behaviour that compensates for it.
|
||||
|
||||
Design constraints and withheld capabilities are not release news.
|
||||
Do not apply this mechanically: deleting a caveat a reader needs misleads them,
|
||||
which is the failure this rule exists to prevent.
|
||||
Design constraints and withheld capabilities are not release news. Deleting a
|
||||
caveat a reader needs misleads them, which is the failure this rule exists to
|
||||
prevent.
|
||||
|
||||
## Cutting a release
|
||||
|
||||
```bash
|
||||
task release:preview VERSION=1.3.0 # preview what would land
|
||||
task release:prepare VERSION=1.3.0 # write the section into CHANGELOG.md, uncommitted
|
||||
# edit CHANGELOG.md -- generated notes are a starting point, not the release
|
||||
task release VERSION=1.3.0 # commit + annotated tag
|
||||
git push --atomic origin main v1.3.0 # both refs or neither
|
||||
```
|
||||
Dispatch the **release** workflow (`.github/workflows/release.yml`) with
|
||||
`major`, `minor`, `patch` or an exact version. It runs `task release` on `main`:
|
||||
|
||||
Pushing the tag is the release. The deploy workflow builds the images and the
|
||||
zip, then `archied release publish` signs the release with the
|
||||
`RELEASE_SIGNING_KEY` Actions secret and pushes it to
|
||||
`git.catlow.cloud/sam/archie` as `v1.3.0`, `next`, and `stable` unless the
|
||||
version is a prerelease. `install.sh` and the updater install only what that
|
||||
key signed.
|
||||
1. refuses a dirty tree, an existing tag, or an `[Unreleased]` with no entries;
|
||||
2. turns `[Unreleased]` into `## [x.y.z] - date` under a new empty
|
||||
`[Unreleased]`, and runs `task news`;
|
||||
3. commits `chore(release): vx.y.z` and tags `vx.y.z`;
|
||||
|
||||
`release:prepare` emits bare commit subjects; they are a starting point. Rewrite
|
||||
them to say what a reader gets, per "What the notes are for" above.
|
||||
then pushes `main` and the tag atomically with `DEPLOY_TOKEN`. A push made with
|
||||
the job token would not start other workflows, so the user token is what lets
|
||||
the tag start the deploy.
|
||||
|
||||
Pushing is a separate, explicit step — never push without confirming first (see
|
||||
the repo's general "check before doing anything hard to reverse" rule). The
|
||||
deploy workflow reads tags pointing at `HEAD`; pushing the commit without its
|
||||
tag gets images stamped `dev`, which is why the push names the tag and is
|
||||
`--atomic`.
|
||||
|
||||
### After a release, `main` and the working branch have diverged
|
||||
|
||||
The release commit lands on `main` — `tools/release.sh` refuses to run anywhere
|
||||
else — while the working branch keeps advancing. So after every release, `main`
|
||||
is ahead by exactly that commit and the branch is ahead by however far it has
|
||||
moved: **neither tip is an ancestor of the other**, and `--ff-only` fails in
|
||||
both directions. That refusal reads as "the tree is not ready" when in fact the
|
||||
tree is fine.
|
||||
|
||||
Merge the release commit into the branch first, so any conflict is resolved on
|
||||
the branch, where it is reversible and nobody is reading the tip:
|
||||
|
||||
```bash
|
||||
git merge main # on the branch
|
||||
git merge --ff-only <branch> # then, in whichever worktree holds main
|
||||
```
|
||||
|
||||
Confirm which worktree holds `main` with `git worktree list` before the
|
||||
`--ff-only`. A detached worktree sitting at the same commit will accept it and
|
||||
move nothing, which looks like success.
|
||||
After a release, pull `main` before committing on it again: CI has added the
|
||||
release commit.
|
||||
|
||||
## What a release publishes
|
||||
|
||||
Tagging one version produces, from a single `deploy` run:
|
||||
The tag push runs `deploy.yml`:
|
||||
|
||||
- **Container images** — `git.catlow.cloud/sam/archied:latest` and, on a
|
||||
release tag, `git.catlow.cloud/sam/archie-agent:latest`. The runtime image is
|
||||
rebuilt by the release, so runtime commits merged to `main` do not reach a
|
||||
running sandbox until the host pulls the new image. The hand-run
|
||||
`tools/publish.sh` publishes the same two images to
|
||||
`git.catlow.cloud/sam` instead, which is the registry the binary's default
|
||||
`[containers].image` names.
|
||||
- **`:latest` names the newest release, not the newest commit.** A push to
|
||||
`main` that is not a release publishes `archied:edge` (mutable, tracking
|
||||
`main`) and `archied:sha-<commit>` (immutable) instead, so a release's
|
||||
`:latest` stamping is not overwritten minutes later by the next ordinary
|
||||
commit. A host that runs releases follows `:latest`; a host that tracks `main`
|
||||
follows `:edge`.
|
||||
- **A GitHub Release with a distribution zip** — `dist-zip` builds every process
|
||||
the reference deployment runs (`archied`, `archie-gateway`,
|
||||
`archie-state-store`, `archie-ui`, `archie-messaging`) for
|
||||
linux/amd64, packs them with `deployments/INSTRUCTIONS.md` and `CHANGELOG.md`,
|
||||
and attaches the zip to the Release. The Release body is that version's
|
||||
`## [version]` section of `CHANGELOG.md`.
|
||||
- **Gate.** `task check TEST_FLAGS=-count=1` (uncached, including the
|
||||
Docker-backed tests `-short` skips) and `govulncheck`, then a check that the
|
||||
tag has a `[version]` section in `CHANGELOG.md`.
|
||||
- **Images.** `archied:latest` and `archie-agent:latest` on a release tag;
|
||||
`archied:edge` and `archied:sha-<commit>` on any other push to `main`, so
|
||||
`:latest` always names the newest release.
|
||||
- **The signed release.** `archied release publish` signs a statement
|
||||
(version, issued_at, zip digest, agent image by digest) with
|
||||
`RELEASE_SIGNING_KEY` and pushes it with the distribution zip to
|
||||
`git.catlow.cloud/sam/archie` as `v<version>` and `next`. It never moves
|
||||
`stable`.
|
||||
- **The release-notes page** on Gitea, from that version's `CHANGELOG.md`
|
||||
section, which the statement links to.
|
||||
- **The canary install**: `archied release canary start` points the canary at
|
||||
`next` and installs the release through its dashboard.
|
||||
|
||||
The zip is also uploaded as an Actions artifact, but that is a backup copy:
|
||||
artifacts expire after 90 days and need an Actions login to download. The
|
||||
Release asset is the durable, operator-reachable one.
|
||||
GitHub-style Actions run the workflow from the tagged commit, so a fix to
|
||||
`deploy.yml` reaches a release only if it is an ancestor of the tag. Re-cut the
|
||||
release rather than committing the fix on top of a tagged one.
|
||||
|
||||
## Gate
|
||||
## The canary
|
||||
|
||||
CI (`deploy.yml`) runs `task check TEST_FLAGS=-count=1` before building images,
|
||||
then verifies any release tag at `HEAD` has a matching `[version]` section in
|
||||
`CHANGELOG.md`. The gate has one definition, in the Taskfile: the release runs
|
||||
it with the uncached suite, including the Docker-backed tests `-short` skips.
|
||||
A tag with no changelog entry is a hard failure. No tags at `HEAD` is a
|
||||
warning, not a failure -- images get stamped `dev`.
|
||||
The canary is the production host, following `next`. Every quarter hour
|
||||
`canary.yml` runs `archied release canary check`, which reads the canary
|
||||
through its dashboard API and judges the release on `next`:
|
||||
|
||||
One consequence worth knowing when a release commit needs amending: GitHub
|
||||
Actions runs the workflow **from the commit that triggered it**, so a fix to
|
||||
`deploy.yml` only takes effect for a release if it is an ancestor of the tagged
|
||||
commit. Delete and re-cut the tag (`git tag -d`, `task release`) rather than
|
||||
committing the fix on top of an already-tagged release.
|
||||
| Signal | Source | Fails when |
|
||||
| --- | --- | --- |
|
||||
| Install outcome | `GET /api/chat/update` | the canary does not run the release (the watchdog rolled it back) |
|
||||
| Readiness | `GET /health/detailed` | the rollup is not `ok` |
|
||||
| Alerts | `GET /api/alerts` | an open `action` alert was first raised after the release was issued |
|
||||
| Error log | `GET /api/logs?level=error` | error lines per day exceed `2 × baseline + 10` |
|
||||
| Task outcomes | `GET /api/tasks` | the share of finished tasks that parked or died rose more than 10 points |
|
||||
|
||||
- **Control.** The canary's own 24 hours before the release was issued. With
|
||||
one production host there is no simultaneous control.
|
||||
- **Window.** 24 hours from issue, one cycle of schedules and traffic. The
|
||||
first 30 minutes are for the install and restart.
|
||||
- **Hard signals** (install, readiness, alerts) roll back at any check after
|
||||
that. **Relative signals** (errors, tasks) are judged at the end of the
|
||||
window.
|
||||
- **Too little data skips a signal** rather than failing it: fewer than 5
|
||||
finished tasks in the window, or an error log that reaches back less than an
|
||||
hour into the baseline. A log read that stops short of a window is scaled to
|
||||
a full day from what it covers.
|
||||
|
||||
### Promote
|
||||
|
||||
A release that passes the window is promoted: `stable` is retagged to the
|
||||
`v<version>` manifest. The signature covers the statement, not the tag, so the
|
||||
retag needs no signing key. Promotion refuses a prerelease and a release issued
|
||||
before the one `stable` names, the same replay rule hosts enforce. The decision
|
||||
and its reasons are in the `canary` run log.
|
||||
|
||||
### Roll back
|
||||
|
||||
`stable` stays put, and the run turns red. The canary is pinned to the `stable`
|
||||
version through its update settings and reinstalled through its dashboard.
|
||||
|
||||
A release whose migrations differ from `stable`'s cannot go back without
|
||||
restoring the pre-update database snapshot, which loses every write since. The
|
||||
canary is held on the failed release instead (pinned to it) and the fix goes
|
||||
forward in the next release. The next release's deploy points the canary back at
|
||||
`next`.
|
||||
|
||||
A pinned canary is not judged, so a rollback is decided once.
|
||||
|
||||
### Configuration
|
||||
|
||||
| Name | Kind | Holds |
|
||||
| --- | --- | --- |
|
||||
| `CANARY_URL` | Actions variable | the canary's dashboard URL; unset disables the canary steps |
|
||||
| `CANARY_TOKEN` | Actions secret | a personal API token on the canary with instance admin |
|
||||
| `DEPLOY_TOKEN` | Actions secret | registry and repository write, for publishing, promoting and the release push |
|
||||
| `RELEASE_SIGNING_KEY` | Actions secret | the release signing seed |
|
||||
|
||||
The runner must reach `CANARY_URL`. Without a canary a release stays on `next`
|
||||
until promoted by hand, under the same refusals:
|
||||
|
||||
```bash
|
||||
archied release promote -version 1.57.0
|
||||
```
|
||||
@@ -17,7 +17,10 @@ import (
|
||||
const releaseUsage = `usage: archied release <command>
|
||||
|
||||
keygen -seed FILE write a new signing key's seed to FILE and print its public key
|
||||
publish sign a release and push it to a release source`
|
||||
publish sign a release and push it to a release source's next channel
|
||||
promote -version V point stable at a published release by hand
|
||||
canary start point the canary at next and install a release on it
|
||||
canary check judge the release on next and promote it or roll the canary back`
|
||||
|
||||
// RunRelease is the publisher's side of a release source: it makes the key
|
||||
// a source signs with and pushes signed releases to it.
|
||||
@@ -31,6 +34,10 @@ func RunRelease(args []string, out, stderr io.Writer) int {
|
||||
return runReleaseKeygen(args[1:], out, stderr)
|
||||
case "publish":
|
||||
return runReleasePublish(args[1:], out, stderr)
|
||||
case "promote":
|
||||
return runReleasePromote(args[1:], out, stderr)
|
||||
case "canary":
|
||||
return runReleaseCanary(args[1:], out, stderr)
|
||||
default:
|
||||
fmt.Fprintln(stderr, releaseUsage)
|
||||
return 2
|
||||
@@ -79,7 +86,7 @@ func runReleasePublish(args []string, out, stderr io.Writer) int {
|
||||
flags.StringVar(&req.Repository, "source", "", "OCI repository to publish to")
|
||||
seedEnv := flags.String("seed-env", "ARCHIE_RELEASE_SEED", "environment variable holding the base64 private key seed")
|
||||
flags.StringVar(&req.Version, "version", "", "release version")
|
||||
flags.BoolVar(&req.Prerelease, "prerelease", false, "publish to next only")
|
||||
flags.BoolVar(&req.Prerelease, "prerelease", false, "mark the release a prerelease, which never reaches stable")
|
||||
flags.StringVar(&req.ZipPath, "zip", "", "distribution zip")
|
||||
flags.StringVar(&req.AgentImage, "agent-image", "", "agent image pinned by digest")
|
||||
flags.StringVar(&req.Changelog, "changelog", "", "URL of the release notes")
|
||||
@@ -97,10 +104,8 @@ func runReleasePublish(args []string, out, stderr io.Writer) int {
|
||||
}
|
||||
req.Signer = ed25519.NewKeyFromSeed(seed)
|
||||
public := distribution.EncodeKey(ed25519.PublicKey(req.Signer[ed25519.SeedSize:]))
|
||||
// stable moves only when the canary promotes the release.
|
||||
req.Channels = []string{"next"}
|
||||
if !req.Prerelease {
|
||||
req.Channels = append(req.Channels, "stable")
|
||||
}
|
||||
statement, err := releaseupdate.Publish(context.Background(), req)
|
||||
if err != nil {
|
||||
fmt.Fprintln(stderr, err)
|
||||
@@ -110,3 +115,28 @@ func runReleasePublish(args []string, out, stderr io.Writer) int {
|
||||
strings.Join(req.Channels, ", "), public)
|
||||
return 0
|
||||
}
|
||||
|
||||
// runReleasePromote moves stable without a canary verdict, under the same
|
||||
// rules the canary promotes by.
|
||||
func runReleasePromote(args []string, out, stderr io.Writer) int {
|
||||
flags := flag.NewFlagSet("release promote", flag.ContinueOnError)
|
||||
flags.SetOutput(stderr)
|
||||
reference := flags.String("source", "", "OCI repository releases are published to; blank is the official releases")
|
||||
key := flags.String("key", "", "base64 ed25519 public key the source signs with")
|
||||
version := flags.String("version", "", "release to promote")
|
||||
if err := flags.Parse(args); err != nil {
|
||||
return 2
|
||||
}
|
||||
source, err := distribution.Parse(*reference, *key, distribution.DefaultReleases())
|
||||
if err != nil || *version == "" {
|
||||
fmt.Fprintln(stderr, "release promote: -version is required, and -source with -key name another source")
|
||||
return 2
|
||||
}
|
||||
statement, err := releaseupdate.Promote(context.Background(), source.Reference, source.Key, *version)
|
||||
if err != nil {
|
||||
fmt.Fprintln(stderr, err)
|
||||
return 1
|
||||
}
|
||||
fmt.Fprintf(out, "stable is %s\n", statement.Version)
|
||||
return 0
|
||||
}
|
||||
@@ -0,0 +1,159 @@
|
||||
package archied
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"flag"
|
||||
"fmt"
|
||||
"io"
|
||||
"os"
|
||||
"os/exec"
|
||||
"time"
|
||||
|
||||
"oras.land/oras-go/v2/errdef"
|
||||
|
||||
"github.com/samcharles93/archie-core/internal/domain/distribution"
|
||||
"github.com/samcharles93/archie-core/internal/releaseupdate"
|
||||
)
|
||||
|
||||
// canaryInstallWait bounds how long an install on the canary may take,
|
||||
// including the watchdog's health wait and a rollback.
|
||||
const canaryInstallWait = 20 * time.Minute
|
||||
|
||||
// migrationsDir holds the schema migrations; a release that changes it cannot
|
||||
// be rolled back without restoring the database snapshot.
|
||||
const migrationsDir = "internal/infrastructure/postgres/migrations"
|
||||
|
||||
func runReleaseCanary(args []string, out, stderr io.Writer) int {
|
||||
if len(args) == 0 || (args[0] != "start" && args[0] != "check") {
|
||||
fmt.Fprintln(stderr, releaseUsage)
|
||||
return 2
|
||||
}
|
||||
flags := flag.NewFlagSet("release canary "+args[0], flag.ContinueOnError)
|
||||
flags.SetOutput(stderr)
|
||||
dashboardURL := flags.String("dashboard", "", "canary dashboard URL")
|
||||
tokenEnv := flags.String("token-env", "ARCHIE_CANARY_TOKEN", "environment variable holding a dashboard API token for the canary")
|
||||
version := flags.String("version", "", "release to install (start)")
|
||||
reference := flags.String("source", "", "OCI repository releases are published to; blank is the official releases (check)")
|
||||
key := flags.String("key", "", "base64 ed25519 public key the source signs with (check)")
|
||||
checkout := flags.String("checkout", ".", "repository checkout with release tags, to see whether a release migrated the database (check)")
|
||||
if err := flags.Parse(args[1:]); err != nil {
|
||||
return 2
|
||||
}
|
||||
dashboard := releaseupdate.CanaryDashboard{URL: *dashboardURL, Token: os.Getenv(*tokenEnv)}
|
||||
if *dashboardURL == "" {
|
||||
fmt.Fprintln(stderr, "release canary: -dashboard is required")
|
||||
return 2
|
||||
}
|
||||
ctx := context.Background()
|
||||
if args[0] == "start" {
|
||||
if *version == "" {
|
||||
fmt.Fprintln(stderr, "release canary start: -version is required")
|
||||
return 2
|
||||
}
|
||||
if err := startCanary(ctx, dashboard, *version); err != nil {
|
||||
fmt.Fprintln(stderr, err)
|
||||
return 1
|
||||
}
|
||||
fmt.Fprintf(out, "canary follows next and runs %s\n", *version)
|
||||
return 0
|
||||
}
|
||||
source, err := distribution.Parse(*reference, *key, distribution.DefaultReleases())
|
||||
if err != nil {
|
||||
fmt.Fprintln(stderr, "release canary check:", err)
|
||||
return 2
|
||||
}
|
||||
rolledBack, err := checkCanary(ctx, dashboard, source, *checkout, out)
|
||||
if err != nil {
|
||||
fmt.Fprintln(stderr, err)
|
||||
return 1
|
||||
}
|
||||
if rolledBack {
|
||||
return 1
|
||||
}
|
||||
return 0
|
||||
}
|
||||
|
||||
func startCanary(ctx context.Context, dashboard releaseupdate.CanaryDashboard, version string) error {
|
||||
if err := dashboard.Follow(ctx, "next", ""); err != nil {
|
||||
return fmt.Errorf("point the canary at next: %w", err)
|
||||
}
|
||||
return dashboard.Install(ctx, version, canaryInstallWait)
|
||||
}
|
||||
|
||||
// checkCanary judges the release on next once and acts on the verdict. It
|
||||
// reports rolledBack so a failed canary turns the run red.
|
||||
func checkCanary(ctx context.Context, dashboard releaseupdate.CanaryDashboard, source distribution.Source, checkout string, out io.Writer) (rolledBack bool, err error) {
|
||||
next, err := releaseupdate.Fetch(ctx, source, "next")
|
||||
if err != nil {
|
||||
return false, err
|
||||
}
|
||||
stable, err := releaseupdate.Fetch(ctx, source, "stable")
|
||||
hasStable := err == nil
|
||||
if err != nil && !errors.Is(err, errdef.ErrNotFound) {
|
||||
return false, err
|
||||
}
|
||||
release := next.Statement
|
||||
if release.Prerelease || (hasStable && !release.IssuedAt.After(stable.Statement.IssuedAt)) {
|
||||
fmt.Fprintf(out, "nothing to judge: next is %s and is not a newer release than stable\n", release.Version)
|
||||
return false, nil
|
||||
}
|
||||
channel, pin, err := dashboard.Channel(ctx)
|
||||
if err != nil {
|
||||
return false, err
|
||||
}
|
||||
if channel != "next" {
|
||||
fmt.Fprintf(out, "nothing to judge: the canary is held on %s %s\n", channel, pin)
|
||||
return false, nil
|
||||
}
|
||||
now := time.Now().UTC()
|
||||
signals, err := dashboard.Signals(ctx, release.IssuedAt, now)
|
||||
if err != nil {
|
||||
return false, fmt.Errorf("read canary signals: %w", err)
|
||||
}
|
||||
verdict := releaseupdate.JudgeCanary(release.Version, now.Sub(release.IssuedAt), signals)
|
||||
fmt.Fprintf(out, "%s: %s\n", release.Version, verdict.Decision)
|
||||
for _, reason := range verdict.Reasons {
|
||||
fmt.Fprintf(out, " %s\n", reason)
|
||||
}
|
||||
switch verdict.Decision {
|
||||
case releaseupdate.CanaryPromote:
|
||||
if _, err := releaseupdate.Promote(ctx, source.Reference, source.Key, release.Version); err != nil {
|
||||
return false, fmt.Errorf("promote %s: %w", release.Version, err)
|
||||
}
|
||||
fmt.Fprintf(out, "stable is %s\n", release.Version)
|
||||
return false, nil
|
||||
case releaseupdate.CanaryRollBack:
|
||||
return true, rollBackCanary(ctx, dashboard, release.Version, stable.Statement.Version, hasStable, checkout, out)
|
||||
}
|
||||
return false, nil
|
||||
}
|
||||
|
||||
// rollBackCanary pins the canary to the stable release and reinstalls it.
|
||||
// Going back past a migration needs the pre-update database snapshot, which
|
||||
// loses every write since, so a release that migrated is held instead.
|
||||
func rollBackCanary(ctx context.Context, dashboard releaseupdate.CanaryDashboard, failed, stable string, hasStable bool, checkout string, out io.Writer) error {
|
||||
if !hasStable || migrated(ctx, checkout, stable, failed) {
|
||||
if err := dashboard.Follow(ctx, "exact-pin", failed); err != nil {
|
||||
return fmt.Errorf("hold the canary on %s: %w", failed, err)
|
||||
}
|
||||
fmt.Fprintf(out, "%s changed the database schema or there is no stable release to return to: the canary is held on %s and stable is unchanged; fix forward\n", failed, failed)
|
||||
return nil
|
||||
}
|
||||
if err := dashboard.Follow(ctx, "exact-pin", stable); err != nil {
|
||||
return fmt.Errorf("pin the canary to %s: %w", stable, err)
|
||||
}
|
||||
if err := dashboard.Install(ctx, stable, canaryInstallWait); err != nil {
|
||||
return fmt.Errorf("return the canary to %s: %w", stable, err)
|
||||
}
|
||||
fmt.Fprintf(out, "the canary is back on %s and stable is unchanged\n", stable)
|
||||
return nil
|
||||
}
|
||||
|
||||
// migrated reports whether the migrations differ between two release tags.
|
||||
// Anything it cannot tell counts as migrated, which holds rather than
|
||||
// restoring a snapshot.
|
||||
func migrated(ctx context.Context, checkout, from, to string) bool {
|
||||
err := exec.CommandContext(ctx, "git", "-C", checkout, "diff", "--quiet", "v"+from, "v"+to, "--", migrationsDir).Run()
|
||||
return err != nil
|
||||
}
|
||||
@@ -0,0 +1,136 @@
|
||||
package releaseupdate
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"time"
|
||||
)
|
||||
|
||||
// Canary evaluation. The canary host takes a release from next; its window
|
||||
// after the release was issued is compared with its own window of the same
|
||||
// length before, since a single production host has no simultaneous control.
|
||||
const (
|
||||
// CanaryWindow is how long a release runs on the canary before the
|
||||
// relative signals are judged: one daily cycle of schedules and traffic.
|
||||
CanaryWindow = 24 * time.Hour
|
||||
// canarySettle is how long after issue the install and restart are given
|
||||
// before the hard signals count.
|
||||
canarySettle = 30 * time.Minute
|
||||
// minFinished is the fewest finished tasks the park rate is judged on.
|
||||
minFinished = 5
|
||||
// minLogCoverage is the least of the baseline window the error log must
|
||||
// cover for the error signal to be judged.
|
||||
minLogCoverage = time.Hour
|
||||
)
|
||||
|
||||
// CanaryDecision is what the evaluation says to do with a release.
|
||||
type CanaryDecision string
|
||||
|
||||
const (
|
||||
CanaryWait CanaryDecision = "wait"
|
||||
CanaryPromote CanaryDecision = "promote"
|
||||
CanaryRollBack CanaryDecision = "roll back"
|
||||
)
|
||||
|
||||
// CanaryWindowSignals is one window's task and error-log counts.
|
||||
type CanaryWindowSignals struct {
|
||||
// Finished counts tasks that reached a terminal or review status; Bad
|
||||
// is those that parked or died.
|
||||
Finished, Bad int
|
||||
// Errors counts error-level log lines over Covered, the part of the
|
||||
// window the log reaches back over.
|
||||
Errors int
|
||||
Covered time.Duration
|
||||
}
|
||||
|
||||
// CanarySignals is what the canary reported at one check.
|
||||
type CanarySignals struct {
|
||||
Installed string
|
||||
Ready bool
|
||||
// NewActionAlerts are open action-severity alerts first raised since
|
||||
// the release was issued.
|
||||
NewActionAlerts []string
|
||||
Baseline CanaryWindowSignals
|
||||
Canary CanaryWindowSignals
|
||||
}
|
||||
|
||||
// CanaryVerdict is a decision and the signals that decided it.
|
||||
type CanaryVerdict struct {
|
||||
Decision CanaryDecision
|
||||
Reasons []string
|
||||
}
|
||||
|
||||
// JudgeCanary decides on version elapsed after it was issued. A hard signal
|
||||
// rolls back at any check once the release has settled; the relative signals
|
||||
// are judged only when the window has passed, against the baseline window.
|
||||
func JudgeCanary(version string, elapsed time.Duration, s CanarySignals) CanaryVerdict {
|
||||
if elapsed < canarySettle {
|
||||
return CanaryVerdict{Decision: CanaryWait, Reasons: []string{"settling after install"}}
|
||||
}
|
||||
var failed []string
|
||||
if s.Installed != version {
|
||||
failed = append(failed, fmt.Sprintf("canary runs %q, not %s: the install failed or was rolled back", s.Installed, version))
|
||||
}
|
||||
if !s.Ready {
|
||||
failed = append(failed, "readiness is degraded")
|
||||
}
|
||||
for _, alert := range s.NewActionAlerts {
|
||||
failed = append(failed, "new action alert: "+alert)
|
||||
}
|
||||
if len(failed) > 0 {
|
||||
return CanaryVerdict{Decision: CanaryRollBack, Reasons: failed}
|
||||
}
|
||||
if elapsed < CanaryWindow {
|
||||
return CanaryVerdict{Decision: CanaryWait, Reasons: []string{fmt.Sprintf("%s of %s elapsed", elapsed.Round(time.Minute), CanaryWindow)}}
|
||||
}
|
||||
var notes []string
|
||||
if reason, ok := judgeErrors(s.Baseline, s.Canary); !ok {
|
||||
failed = append(failed, reason)
|
||||
} else {
|
||||
notes = append(notes, reason)
|
||||
}
|
||||
if reason, ok := judgeParks(s.Baseline, s.Canary); !ok {
|
||||
failed = append(failed, reason)
|
||||
} else {
|
||||
notes = append(notes, reason)
|
||||
}
|
||||
if len(failed) > 0 {
|
||||
return CanaryVerdict{Decision: CanaryRollBack, Reasons: failed}
|
||||
}
|
||||
return CanaryVerdict{Decision: CanaryPromote, Reasons: notes}
|
||||
}
|
||||
|
||||
// judgeErrors fails when the canary logged more than twice the baseline's
|
||||
// error lines plus ten, each scaled to a full window from what the log covers.
|
||||
func judgeErrors(baseline, canary CanaryWindowSignals) (string, bool) {
|
||||
if baseline.Covered < minLogCoverage || canary.Covered <= 0 {
|
||||
return "error log does not reach back over the baseline; error signal skipped", true
|
||||
}
|
||||
scale := func(w CanaryWindowSignals) float64 {
|
||||
return float64(w.Errors) * float64(CanaryWindow) / float64(w.Covered)
|
||||
}
|
||||
before, after := scale(baseline), scale(canary)
|
||||
limit := 2*before + 10
|
||||
if after > limit {
|
||||
return fmt.Sprintf("%.0f error log lines per day against a limit of %.0f (baseline %.0f)", after, limit, before), false
|
||||
}
|
||||
return fmt.Sprintf("%.0f error log lines per day, limit %.0f", after, limit), true
|
||||
}
|
||||
|
||||
// judgeParks fails when the share of finished tasks that parked or died rose
|
||||
// more than ten points over the baseline.
|
||||
func judgeParks(baseline, canary CanaryWindowSignals) (string, bool) {
|
||||
if canary.Finished < minFinished {
|
||||
return fmt.Sprintf("%d finished tasks, fewer than %d; park signal skipped", canary.Finished, minFinished), true
|
||||
}
|
||||
rate := func(w CanaryWindowSignals) float64 {
|
||||
if w.Finished == 0 {
|
||||
return 0
|
||||
}
|
||||
return float64(w.Bad) / float64(w.Finished)
|
||||
}
|
||||
before, after := rate(baseline), rate(canary)
|
||||
if after > before+0.10 {
|
||||
return fmt.Sprintf("%.0f%% of tasks parked or died against %.0f%% before", 100*after, 100*before), false
|
||||
}
|
||||
return fmt.Sprintf("%.0f%% of tasks parked or died, %.0f%% before", 100*after, 100*before), true
|
||||
}
|
||||
@@ -0,0 +1,309 @@
|
||||
package releaseupdate
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"net/http"
|
||||
"net/url"
|
||||
"strings"
|
||||
"time"
|
||||
)
|
||||
|
||||
// CanaryDashboard reaches the canary through its dashboard API, the same
|
||||
// contract an operator uses: it reads the signals, edits the update settings
|
||||
// through the control plane and installs through the dashboard updater.
|
||||
type CanaryDashboard struct {
|
||||
URL string
|
||||
Token string
|
||||
HTTP *http.Client
|
||||
}
|
||||
|
||||
// Statuses a finished task can be in, and the subset that counts against a
|
||||
// release.
|
||||
var (
|
||||
finishedStatuses = map[string]bool{"completed": true, "merged": true, "pr_open": true, "rejected": true, "closed_wont_do": true, "parked": true, "dead": true}
|
||||
badStatuses = map[string]bool{"parked": true, "dead": true}
|
||||
)
|
||||
|
||||
const updateSettingsPath = "/api/control-plane/resources/update-settings"
|
||||
|
||||
// Signals reads what the canary reports, splitting tasks and error lines into
|
||||
// the window before issued and the window from issued to now.
|
||||
func (d CanaryDashboard) Signals(ctx context.Context, issued, now time.Time) (CanarySignals, error) {
|
||||
var s CanarySignals
|
||||
var err error
|
||||
if s.Installed, _, err = d.installed(ctx); err != nil {
|
||||
return s, err
|
||||
}
|
||||
if s.Ready, err = d.ready(ctx); err != nil {
|
||||
return s, err
|
||||
}
|
||||
if s.NewActionAlerts, err = d.actionAlerts(ctx, issued); err != nil {
|
||||
return s, err
|
||||
}
|
||||
if err := d.tasks(ctx, issued, &s); err != nil {
|
||||
return s, err
|
||||
}
|
||||
return s, d.errors(ctx, issued, now, &s)
|
||||
}
|
||||
|
||||
// installed is the daemon version the canary runs and the raw snapshot it
|
||||
// offers, which an install must echo back.
|
||||
func (d CanaryDashboard) installed(ctx context.Context) (string, json.RawMessage, error) {
|
||||
var body struct {
|
||||
Snapshot json.RawMessage `json:"snapshot"`
|
||||
}
|
||||
if _, err := d.call(ctx, http.MethodGet, "/api/chat/update", nil, &body); err != nil {
|
||||
return "", nil, err
|
||||
}
|
||||
var snapshot Snapshot
|
||||
if err := json.Unmarshal(body.Snapshot, &snapshot); err != nil {
|
||||
return "", nil, fmt.Errorf("decode update snapshot: %w", err)
|
||||
}
|
||||
for _, c := range snapshot.Components {
|
||||
if c.ID == "daemon" {
|
||||
return c.Installed, body.Snapshot, nil
|
||||
}
|
||||
}
|
||||
return "", body.Snapshot, errors.New("canary reports no daemon version")
|
||||
}
|
||||
|
||||
func (d CanaryDashboard) ready(ctx context.Context) (bool, error) {
|
||||
var report struct {
|
||||
Status string `json:"status"`
|
||||
}
|
||||
if _, err := d.call(ctx, http.MethodGet, "/health/detailed", nil, &report); err != nil {
|
||||
return false, err
|
||||
}
|
||||
return report.Status == "ok", nil
|
||||
}
|
||||
|
||||
func (d CanaryDashboard) actionAlerts(ctx context.Context, since time.Time) ([]string, error) {
|
||||
var body struct {
|
||||
Alerts []struct {
|
||||
Severity string `json:"severity"`
|
||||
Title string `json:"title"`
|
||||
FirstRaised time.Time `json:"first_raised"`
|
||||
} `json:"alerts"`
|
||||
}
|
||||
if _, err := d.call(ctx, http.MethodGet, "/api/alerts", nil, &body); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
var titles []string
|
||||
for _, a := range body.Alerts {
|
||||
if a.Severity == "action" && !a.FirstRaised.Before(since) {
|
||||
titles = append(titles, a.Title)
|
||||
}
|
||||
}
|
||||
return titles, nil
|
||||
}
|
||||
|
||||
// tasks pages back through tasks by last update until the baseline window
|
||||
// has been read.
|
||||
func (d CanaryDashboard) tasks(ctx context.Context, issued time.Time, s *CanarySignals) error {
|
||||
from := issued.Add(-CanaryWindow)
|
||||
cursor := ""
|
||||
for range 50 {
|
||||
var page []struct {
|
||||
Status string `json:"status"`
|
||||
UpdatedAt time.Time `json:"updated_at"`
|
||||
}
|
||||
path := "/api/tasks?limit=500"
|
||||
if cursor != "" {
|
||||
path += "&cursor=" + url.QueryEscape(cursor)
|
||||
}
|
||||
header, err := d.call(ctx, http.MethodGet, path, nil, &page)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
for _, t := range page {
|
||||
if t.UpdatedAt.Before(from) || !finishedStatuses[t.Status] {
|
||||
continue
|
||||
}
|
||||
w := &s.Canary
|
||||
if t.UpdatedAt.Before(issued) {
|
||||
w = &s.Baseline
|
||||
}
|
||||
w.Finished++
|
||||
if badStatuses[t.Status] {
|
||||
w.Bad++
|
||||
}
|
||||
}
|
||||
cursor = header.Get("X-Next-Cursor")
|
||||
if cursor == "" || len(page) == 0 || page[len(page)-1].UpdatedAt.Before(from) {
|
||||
return nil
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// errors counts error-level lines in each window. The log read is bounded, so
|
||||
// a truncated read covers only back to its oldest line.
|
||||
func (d CanaryDashboard) errors(ctx context.Context, issued, now time.Time, s *CanarySignals) error {
|
||||
var body struct {
|
||||
Entries []struct {
|
||||
Time time.Time `json:"time"`
|
||||
} `json:"entries"`
|
||||
Truncated bool `json:"truncated"`
|
||||
}
|
||||
if _, err := d.call(ctx, http.MethodGet, "/api/logs?level=error&limit=2000", nil, &body); err != nil {
|
||||
return err
|
||||
}
|
||||
from := issued.Add(-CanaryWindow)
|
||||
if body.Truncated {
|
||||
from = now
|
||||
if len(body.Entries) > 0 {
|
||||
from = later(body.Entries[0].Time, issued.Add(-CanaryWindow))
|
||||
}
|
||||
}
|
||||
for _, e := range body.Entries {
|
||||
switch {
|
||||
case e.Time.Before(from):
|
||||
case e.Time.Before(issued):
|
||||
s.Baseline.Errors++
|
||||
default:
|
||||
s.Canary.Errors++
|
||||
}
|
||||
}
|
||||
s.Baseline.Covered = max(issued.Sub(from), 0)
|
||||
s.Canary.Covered = now.Sub(later(from, issued))
|
||||
return nil
|
||||
}
|
||||
|
||||
func later(a, b time.Time) time.Time {
|
||||
if a.After(b) {
|
||||
return a
|
||||
}
|
||||
return b
|
||||
}
|
||||
|
||||
// Follow sets the canary's update channel, keeping its release source and
|
||||
// key. A pin is used only by exact-pin.
|
||||
func (d CanaryDashboard) Follow(ctx context.Context, channel, pin string) error {
|
||||
var current struct {
|
||||
Resource struct {
|
||||
Version int64 `json:"version"`
|
||||
Value json.RawMessage `json:"value"`
|
||||
} `json:"resource"`
|
||||
}
|
||||
if _, err := d.call(ctx, http.MethodGet, updateSettingsPath, nil, ¤t); err != nil {
|
||||
return err
|
||||
}
|
||||
value := map[string]any{}
|
||||
if len(current.Resource.Value) > 0 && string(current.Resource.Value) != "null" {
|
||||
if err := json.Unmarshal(current.Resource.Value, &value); err != nil {
|
||||
return fmt.Errorf("decode update settings: %w", err)
|
||||
}
|
||||
}
|
||||
value["channel"], value["pin"] = channel, pin
|
||||
_, err := d.call(ctx, http.MethodPost, updateSettingsPath+"/commands/replace",
|
||||
map[string]any{"value": value, "expected_version": current.Resource.Version}, nil)
|
||||
return err
|
||||
}
|
||||
|
||||
// Channel is the canary's update channel and pin.
|
||||
func (d CanaryDashboard) Channel(ctx context.Context) (channel, pin string, err error) {
|
||||
var current struct {
|
||||
Resource struct {
|
||||
Value struct {
|
||||
Channel string `json:"channel"`
|
||||
Pin string `json:"pin"`
|
||||
} `json:"value"`
|
||||
} `json:"resource"`
|
||||
}
|
||||
if _, err := d.call(ctx, http.MethodGet, updateSettingsPath, nil, ¤t); err != nil {
|
||||
return "", "", err
|
||||
}
|
||||
if current.Resource.Value.Channel == "" {
|
||||
return "stable", "", nil
|
||||
}
|
||||
return current.Resource.Value.Channel, current.Resource.Value.Pin, nil
|
||||
}
|
||||
|
||||
// Install has the canary install version through its updater and waits until
|
||||
// it reports running it. The install restarts the dashboard, so a dropped
|
||||
// connection after the request is sent is expected.
|
||||
func (d CanaryDashboard) Install(ctx context.Context, version string, wait time.Duration) error {
|
||||
installed, snapshot, err := d.installed(ctx)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if installed == version {
|
||||
return nil
|
||||
}
|
||||
var offered Snapshot
|
||||
if err := json.Unmarshal(snapshot, &offered); err != nil {
|
||||
return fmt.Errorf("decode update snapshot: %w", err)
|
||||
}
|
||||
if available := offered.Available(); len(available) == 0 || available[0].Available != version {
|
||||
return fmt.Errorf("canary does not offer %s yet", version)
|
||||
}
|
||||
_, err = d.call(ctx, http.MethodPost, "/api/chat/update/install", map[string]json.RawMessage{"snapshot": snapshot}, nil)
|
||||
if _, ok := errors.AsType[statusError](err); ok {
|
||||
return err
|
||||
}
|
||||
deadline := time.Now().Add(wait)
|
||||
for time.Now().Before(deadline) {
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return ctx.Err()
|
||||
case <-time.After(15 * time.Second):
|
||||
}
|
||||
if installed, _, err := d.installed(ctx); err == nil && installed == version {
|
||||
return nil
|
||||
}
|
||||
}
|
||||
return fmt.Errorf("canary did not report %s within %s", version, wait)
|
||||
}
|
||||
|
||||
type statusError struct {
|
||||
code int
|
||||
body string
|
||||
}
|
||||
|
||||
func (e statusError) Error() string { return fmt.Sprintf("dashboard answered %d: %s", e.code, e.body) }
|
||||
|
||||
func (d CanaryDashboard) call(ctx context.Context, method, path string, in, out any) (http.Header, error) {
|
||||
var body io.Reader
|
||||
if in != nil {
|
||||
data, err := json.Marshal(in)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
body = bytes.NewReader(data)
|
||||
}
|
||||
ctx, cancel := context.WithTimeout(ctx, 2*time.Minute)
|
||||
defer cancel()
|
||||
req, err := http.NewRequestWithContext(ctx, method, strings.TrimRight(d.URL, "/")+path, body)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
req.Header.Set("Authorization", "Bearer "+d.Token)
|
||||
if in != nil {
|
||||
req.Header.Set("Content-Type", "application/json")
|
||||
}
|
||||
client := d.HTTP
|
||||
if client == nil {
|
||||
client = http.DefaultClient
|
||||
}
|
||||
resp, err := client.Do(req)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("%s %s: %w", method, path, err)
|
||||
}
|
||||
defer func() { _ = resp.Body.Close() }()
|
||||
if resp.StatusCode/100 != 2 {
|
||||
text, _ := io.ReadAll(io.LimitReader(resp.Body, 512))
|
||||
return nil, fmt.Errorf("%s %s: %w", method, path, statusError{resp.StatusCode, strings.TrimSpace(string(text))})
|
||||
}
|
||||
if out == nil {
|
||||
return resp.Header, nil
|
||||
}
|
||||
if err := json.NewDecoder(resp.Body).Decode(out); err != nil {
|
||||
return nil, fmt.Errorf("%s %s: decode: %w", method, path, err)
|
||||
}
|
||||
return resp.Header, nil
|
||||
}
|
||||
@@ -0,0 +1,56 @@
|
||||
package releaseupdate
|
||||
|
||||
import (
|
||||
"testing"
|
||||
"time"
|
||||
)
|
||||
|
||||
// A release promotes only after a full window with no hard failure and no
|
||||
// relative signal past its threshold; any hard failure after settling rolls
|
||||
// it back at once.
|
||||
func TestJudgeCanary(t *testing.T) {
|
||||
healthy := func() CanarySignals {
|
||||
return CanarySignals{
|
||||
Installed: "1.1.0", Ready: true,
|
||||
Baseline: CanaryWindowSignals{Finished: 20, Bad: 2, Errors: 10, Covered: CanaryWindow},
|
||||
Canary: CanaryWindowSignals{Finished: 20, Bad: 2, Errors: 12, Covered: CanaryWindow},
|
||||
}
|
||||
}
|
||||
with := func(change func(*CanarySignals)) CanarySignals {
|
||||
s := healthy()
|
||||
change(&s)
|
||||
return s
|
||||
}
|
||||
cases := []struct {
|
||||
name string
|
||||
elapsed time.Duration
|
||||
signals CanarySignals
|
||||
want CanaryDecision
|
||||
}{
|
||||
{"settling ignores a failed check", 10 * time.Minute, with(func(s *CanarySignals) { s.Ready = false }), CanaryWait},
|
||||
{"healthy mid-window waits", 2 * time.Hour, healthy(), CanaryWait},
|
||||
{"healthy at the window promotes", CanaryWindow, healthy(), CanaryPromote},
|
||||
{"the watchdog rolled the install back", 2 * time.Hour, with(func(s *CanarySignals) { s.Installed = "1.0.0" }), CanaryRollBack},
|
||||
{"readiness degraded", 2 * time.Hour, with(func(s *CanarySignals) { s.Ready = false }), CanaryRollBack},
|
||||
{"a new action alert", 2 * time.Hour, with(func(s *CanarySignals) { s.NewActionAlerts = []string{"queue stalled"} }), CanaryRollBack},
|
||||
{"errors past twice the baseline plus ten", CanaryWindow, with(func(s *CanarySignals) { s.Canary.Errors = 31 }), CanaryRollBack},
|
||||
{"errors at the limit", CanaryWindow, with(func(s *CanarySignals) { s.Canary.Errors = 30 }), CanaryPromote},
|
||||
{"errors scaled from a partly covered log", CanaryWindow, with(func(s *CanarySignals) {
|
||||
s.Canary.Errors, s.Canary.Covered = 8, 6*time.Hour
|
||||
}), CanaryRollBack},
|
||||
{"a baseline the log does not reach skips errors", CanaryWindow, with(func(s *CanarySignals) {
|
||||
s.Baseline.Covered, s.Canary.Errors = 30*time.Minute, 500
|
||||
}), CanaryPromote},
|
||||
{"park rate up more than ten points", CanaryWindow, with(func(s *CanarySignals) { s.Canary.Bad = 5 }), CanaryRollBack},
|
||||
{"park rate up ten points", CanaryWindow, with(func(s *CanarySignals) { s.Canary.Bad = 4 }), CanaryPromote},
|
||||
{"too few tasks skips the park rate", CanaryWindow, with(func(s *CanarySignals) { s.Canary = CanaryWindowSignals{Finished: 4, Bad: 4, Covered: CanaryWindow} }), CanaryPromote},
|
||||
}
|
||||
for _, tc := range cases {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
got := JudgeCanary("1.1.0", tc.elapsed, tc.signals)
|
||||
if got.Decision != tc.want {
|
||||
t.Fatalf("decision = %s, want %s (%v)", got.Decision, tc.want, got.Reasons)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,74 @@
|
||||
package releaseupdate
|
||||
|
||||
import (
|
||||
"context"
|
||||
"crypto/ed25519"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
|
||||
digest "github.com/opencontainers/go-digest"
|
||||
ocispec "github.com/opencontainers/image-spec/specs-go/v1"
|
||||
"oras.land/oras-go/v2/errdef"
|
||||
|
||||
"github.com/samcharles93/archie-core/internal/domain/distribution"
|
||||
"github.com/samcharles93/archie-core/internal/infrastructure/signedoci"
|
||||
)
|
||||
|
||||
// Promote points stable at the release published as version. The signature
|
||||
// covers the statement, not the tag, so stable is the same signed manifest
|
||||
// and needs no re-signing. A prerelease, or a release issued before the one
|
||||
// stable names, is refused: hosts would refuse the latter as a replay.
|
||||
func Promote(ctx context.Context, repository string, key ed25519.PublicKey, version string) (Statement, error) {
|
||||
source := distribution.Source{Reference: repository, Key: key}
|
||||
release, err := Fetch(ctx, source, Tag("exact-pin", version))
|
||||
if err != nil {
|
||||
return Statement{}, err
|
||||
}
|
||||
if release.Statement.Prerelease {
|
||||
return Statement{}, fmt.Errorf("%s is a prerelease and never reaches stable", version)
|
||||
}
|
||||
current, err := Fetch(ctx, source, "stable")
|
||||
switch {
|
||||
case errors.Is(err, errdef.ErrNotFound):
|
||||
case err != nil:
|
||||
return Statement{}, err
|
||||
case release.Statement.IssuedAt.Before(current.Statement.IssuedAt):
|
||||
return Statement{}, fmt.Errorf("%s was issued before %s, which stable names", version, current.Statement.Version)
|
||||
}
|
||||
repo, err := pushRepository(repository)
|
||||
if err != nil {
|
||||
return Statement{}, err
|
||||
}
|
||||
// Tag the manifest the version tag resolves to now, after checking it
|
||||
// carries the statement just verified, so a tag moved in between cannot
|
||||
// slip an unverified manifest onto stable.
|
||||
desc, err := repo.Resolve(ctx, Tag("exact-pin", version))
|
||||
if err != nil {
|
||||
return Statement{}, fmt.Errorf("resolve %s: %w", version, err)
|
||||
}
|
||||
stream, err := repo.Fetch(ctx, desc)
|
||||
if err != nil {
|
||||
return Statement{}, fmt.Errorf("fetch %s manifest: %w", version, err)
|
||||
}
|
||||
raw, err := io.ReadAll(io.LimitReader(stream, desc.Size))
|
||||
_ = stream.Close()
|
||||
if err != nil {
|
||||
return Statement{}, fmt.Errorf("read %s manifest: %w", version, err)
|
||||
}
|
||||
if digest.FromBytes(raw) != desc.Digest {
|
||||
return Statement{}, fmt.Errorf("%s manifest does not match its digest", version)
|
||||
}
|
||||
var manifest ocispec.Manifest
|
||||
if err := json.Unmarshal(raw, &manifest); err != nil {
|
||||
return Statement{}, fmt.Errorf("%s manifest: %w", version, err)
|
||||
}
|
||||
if err := signedoci.Verify(manifest, release.Artifact.Statement, key, Annotation); err != nil {
|
||||
return Statement{}, fmt.Errorf("%s changed while promoting: %w", version, err)
|
||||
}
|
||||
if err := repo.Tag(ctx, desc, "stable"); err != nil {
|
||||
return Statement{}, fmt.Errorf("tag stable: %w", err)
|
||||
}
|
||||
return release.Statement, nil
|
||||
}
|
||||
@@ -0,0 +1,167 @@
|
||||
package releaseupdate
|
||||
|
||||
import (
|
||||
"crypto/ed25519"
|
||||
"encoding/base64"
|
||||
"encoding/json"
|
||||
"io"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"strconv"
|
||||
"strings"
|
||||
"sync"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
digest "github.com/opencontainers/go-digest"
|
||||
ocispec "github.com/opencontainers/image-spec/specs-go/v1"
|
||||
|
||||
"github.com/samcharles93/archie-core/internal/domain/distribution"
|
||||
"github.com/samcharles93/archie-core/internal/infrastructure/signedoci"
|
||||
)
|
||||
|
||||
// stable moves only to a signed, non-prerelease release issued no earlier
|
||||
// than the one it names, and it moves to the very manifest that was verified.
|
||||
func TestPromote(t *testing.T) {
|
||||
trusted, signer, _ := ed25519.GenerateKey(nil)
|
||||
_, stranger, _ := ed25519.GenerateKey(nil)
|
||||
jan1, jan2 := time.Date(2026, 1, 1, 0, 0, 0, 0, time.UTC), time.Date(2026, 1, 2, 0, 0, 0, 0, time.UTC)
|
||||
type published struct {
|
||||
version string
|
||||
issued time.Time
|
||||
prerelease bool
|
||||
signer ed25519.PrivateKey
|
||||
tags []string
|
||||
}
|
||||
cases := []struct {
|
||||
name string
|
||||
release []published
|
||||
promote string
|
||||
want string // error substring, or "" when stable must name promote
|
||||
}{
|
||||
{name: "a newer release", release: []published{{"1.0.0", jan1, false, signer, []string{"stable"}}, {"1.1.0", jan2, false, signer, []string{"next"}}}, promote: "1.1.0"},
|
||||
{name: "the first release", release: []published{{"1.0.0", jan1, false, signer, []string{"next"}}}, promote: "1.0.0"},
|
||||
{name: "a release issued before stable", release: []published{{"1.1.0", jan2, false, signer, []string{"stable"}}, {"1.0.0", jan1, false, signer, nil}}, promote: "1.0.0", want: "issued before"},
|
||||
{name: "a prerelease", release: []published{{"1.1.0-rc.1", jan2, true, signer, []string{"next"}}}, promote: "1.1.0-rc.1", want: "prerelease"},
|
||||
{name: "a release another key signed", release: []published{{"1.1.0", jan2, false, stranger, []string{"next"}}}, promote: "1.1.0", want: "signature"},
|
||||
}
|
||||
for _, tc := range cases {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
t.Setenv("XDG_STATE_HOME", t.TempDir())
|
||||
t.Setenv("DOCKER_CONFIG", t.TempDir())
|
||||
registry := newMemoryRegistry()
|
||||
server := httptest.NewServer(registry)
|
||||
defer server.Close()
|
||||
for _, r := range tc.release {
|
||||
registry.publish(t, r.version, r.issued, r.prerelease, r.signer, r.tags...)
|
||||
}
|
||||
repository := strings.TrimPrefix(server.URL, "http://") + "/archie"
|
||||
_, err := Promote(t.Context(), repository, trusted, tc.promote)
|
||||
switch {
|
||||
case tc.want == "" && err != nil:
|
||||
t.Fatalf("Promote: %v", err)
|
||||
case tc.want != "" && (err == nil || !strings.Contains(err.Error(), tc.want)):
|
||||
t.Fatalf("Promote error = %v, want %q", err, tc.want)
|
||||
}
|
||||
stable, ok := registry.statement("stable")
|
||||
promoted := ok && stable.Version == tc.promote
|
||||
if promoted != (tc.want == "") {
|
||||
t.Fatalf("stable names %q (found %v) after promoting %s", stable.Version, ok, tc.promote)
|
||||
}
|
||||
if tc.want == "" {
|
||||
if _, err := Fetch(t.Context(), distribution.Source{Reference: repository, Key: trusted}, "stable"); err != nil {
|
||||
t.Fatalf("hosts refuse the promoted stable: %v", err)
|
||||
}
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// memoryRegistry serves manifests by tag or digest and accepts manifest
|
||||
// pushes, which is all fetching and retagging a release needs.
|
||||
type memoryRegistry struct {
|
||||
mu sync.Mutex
|
||||
manifests map[string][]byte
|
||||
blobs map[string][]byte
|
||||
}
|
||||
|
||||
func newMemoryRegistry() *memoryRegistry {
|
||||
return &memoryRegistry{manifests: map[string][]byte{}, blobs: map[string][]byte{}}
|
||||
}
|
||||
|
||||
func (m *memoryRegistry) publish(t *testing.T, version string, issued time.Time, prerelease bool, signer ed25519.PrivateKey, tags ...string) {
|
||||
t.Helper()
|
||||
archive := []byte("zip " + version)
|
||||
statement, err := json.Marshal(Statement{
|
||||
Version: version, Prerelease: prerelease, IssuedAt: issued, Platform: Platform, AgentImage: newImage,
|
||||
Zip: Blob{Name: ZipName(version), Digest: signedoci.Digest(archive), Size: int64(len(archive))},
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
manifest, err := json.Marshal(ocispec.Manifest{
|
||||
SchemaVersion: 2, MediaType: ocispec.MediaTypeImageManifest, ArtifactType: ArtifactType, Config: ocispec.DescriptorEmptyJSON,
|
||||
Layers: []ocispec.Descriptor{
|
||||
{MediaType: StatementMediaType, Digest: digest.FromBytes(statement), Size: int64(len(statement))},
|
||||
{MediaType: "application/zip", Digest: digest.FromBytes(archive), Size: int64(len(archive))},
|
||||
},
|
||||
Annotations: map[string]string{Annotation: base64.StdEncoding.EncodeToString(ed25519.Sign(signer, statement))},
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
m.mu.Lock()
|
||||
defer m.mu.Unlock()
|
||||
m.blobs[signedoci.Digest(statement)], m.blobs[signedoci.Digest(archive)] = statement, archive
|
||||
m.manifests[signedoci.Digest(manifest)] = manifest
|
||||
for _, tag := range append(tags, "v"+version) {
|
||||
m.manifests[tag] = manifest
|
||||
}
|
||||
}
|
||||
|
||||
func (m *memoryRegistry) statement(tag string) (Statement, bool) {
|
||||
m.mu.Lock()
|
||||
defer m.mu.Unlock()
|
||||
var manifest ocispec.Manifest
|
||||
if json.Unmarshal(m.manifests[tag], &manifest) != nil || len(manifest.Layers) == 0 {
|
||||
return Statement{}, false
|
||||
}
|
||||
var s Statement
|
||||
return s, json.Unmarshal(m.blobs[manifest.Layers[0].Digest.String()], &s) == nil
|
||||
}
|
||||
|
||||
func (m *memoryRegistry) ServeHTTP(w http.ResponseWriter, r *http.Request) {
|
||||
m.mu.Lock()
|
||||
defer m.mu.Unlock()
|
||||
if ref, ok := strings.CutPrefix(r.URL.Path, "/v2/archie/blobs/"); ok {
|
||||
blob, found := m.blobs[ref]
|
||||
if !found {
|
||||
http.NotFound(w, r)
|
||||
return
|
||||
}
|
||||
_, _ = w.Write(blob)
|
||||
return
|
||||
}
|
||||
ref, ok := strings.CutPrefix(r.URL.Path, "/v2/archie/manifests/")
|
||||
if !ok {
|
||||
http.NotFound(w, r)
|
||||
return
|
||||
}
|
||||
if r.Method == http.MethodPut {
|
||||
body, _ := io.ReadAll(r.Body)
|
||||
m.manifests[ref], m.manifests[signedoci.Digest(body)] = body, body
|
||||
w.WriteHeader(http.StatusCreated)
|
||||
return
|
||||
}
|
||||
manifest, found := m.manifests[ref]
|
||||
if !found {
|
||||
http.NotFound(w, r)
|
||||
return
|
||||
}
|
||||
w.Header().Set("Content-Type", ocispec.MediaTypeImageManifest)
|
||||
w.Header().Set("Docker-Content-Digest", signedoci.Digest(manifest))
|
||||
w.Header().Set("Content-Length", strconv.Itoa(len(manifest)))
|
||||
if r.Method != http.MethodHead {
|
||||
_, _ = w.Write(manifest)
|
||||
}
|
||||
}
|
||||
Reference in new issue
Block a user