From a4a7c96da6507f907421c433c9c94388baf3fdad Mon Sep 17 00:00:00 2001 From: Sam Catlow Date: Sat, 31 Oct 2026 21:39:32 +1100 Subject: [PATCH] feat(releaseupdate): publish to next and promote to stable only after a canary evaluation (#1716) --- .github/workflows/canary.yml | 47 ++++ .github/workflows/deploy.yml | 12 +- CHANGELOG.md | 4 + RELEASING.md | 221 ++++++++------- internal/app/archied/release.go | 40 ++- internal/app/archied/release_canary.go | 159 +++++++++++ internal/releaseupdate/canary.go | 136 +++++++++ internal/releaseupdate/canary_dashboard.go | 309 +++++++++++++++++++++ internal/releaseupdate/canary_test.go | 56 ++++ internal/releaseupdate/promote.go | 74 +++++ internal/releaseupdate/promote_test.go | 167 +++++++++++ 11 files changed, 1106 insertions(+), 119 deletions(-) create mode 100644 .github/workflows/canary.yml create mode 100644 internal/app/archied/release_canary.go create mode 100644 internal/releaseupdate/canary.go create mode 100644 internal/releaseupdate/canary_dashboard.go create mode 100644 internal/releaseupdate/canary_test.go create mode 100644 internal/releaseupdate/promote.go create mode 100644 internal/releaseupdate/promote_test.go diff --git a/.github/workflows/canary.yml b/.github/workflows/canary.yml new file mode 100644 index 000000000..7fdf001e1 --- /dev/null +++ b/.github/workflows/canary.yml @@ -0,0 +1,47 @@ +# Judges the release on next against the canary every quarter hour: a hard +# failure rolls the canary back at once, and a release that passes a full +# window is promoted to stable. A rollback turns the run red. +name: canary + +on: + schedule: + - cron: "*/15 * * * *" + workflow_dispatch: + +concurrency: + group: canary + +permissions: + contents: read + packages: write + +jobs: + judge: + if: vars.CANARY_URL != '' + runs-on: self-hosted + steps: + # Full history with tags: a rollback compares the migrations between + # the stable and failed release tags. + - name: Checkout + uses: actions/checkout@v4 + with: + ref: main + fetch-depth: 0 + + - name: Set up Go + uses: actions/setup-go@v5 + with: + go-version-file: go.mod + + - name: Build archied + run: go build -o "$RUNNER_TEMP/archied" ./cmd/archied + + # Promotion retags stable in the registry. + - name: Log in to the Gitea registry + run: echo "${{ secrets.DEPLOY_TOKEN }}" | docker login git.catlow.cloud -u "${{ github.actor }}" --password-stdin + + - name: Judge the canary + env: + ARCHIE_CANARY_TOKEN: ${{ secrets.CANARY_TOKEN }} + run: | + "$RUNNER_TEMP/archied" release canary check -dashboard "${{ vars.CANARY_URL }}" diff --git a/.github/workflows/deploy.yml b/.github/workflows/deploy.yml index 9a186a04c..c11949776 100644 --- a/.github/workflows/deploy.yml +++ b/.github/workflows/deploy.yml @@ -353,8 +353,8 @@ jobs: # Installs and updates take a release from the signed artifact in the # registry, so this is what publishes it: the statement signed with - # RELEASE_SIGNING_KEY and the zip, tagged v, next, and stable - # unless it is a prerelease. + # RELEASE_SIGNING_KEY and the zip, tagged v and next. stable + # moves only when canary.yml promotes it. - name: Log in to the Gitea registry run: echo "${{ secrets.DEPLOY_TOKEN }}" | docker login git.catlow.cloud -u "${{ github.actor }}" --password-stdin @@ -402,6 +402,14 @@ jobs: "$api/repos/$repo/releases" >/dev/null echo "published release notes for $tag" + # The canary follows next; canary.yml judges the release on it and + # promotes it to stable or rolls the canary back. + - name: Install the release on the canary + if: vars.CANARY_URL != '' + env: + ARCHIE_CANARY_TOKEN: ${{ secrets.CANARY_TOKEN }} + run: ./dist/archied release canary start -dashboard "${{ vars.CANARY_URL }}" -version "${{ steps.version.outputs.version }}" + # A Gitea package belongs to its owner, not a repository, so the ones this # workflow publishes are linked here. Linking is per package and a no-op # once done, so an already linked package is not an error. diff --git a/CHANGELOG.md b/CHANGELOG.md index 28d9d0e8b..6f28895c0 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,6 +5,10 @@ release's per-component sections are labelled beneath its heading. ## [Unreleased] +### archied + +- A release reaches `stable` only after it has run on a canary for a day and passed: `archied release publish` now publishes to `next` alone, `archied release canary start|check` installs a release on a canary and judges it from the canary's dashboard, promoting it or rolling the canary back, and `archied release promote` moves `stable` by hand. Promotion refuses a prerelease and a release older than the one `stable` names. + ## [1.56.0] - 2026-10-31 ### archied diff --git a/RELEASING.md b/RELEASING.md index e06b5a3e8..dee4e2c22 100644 --- a/RELEASING.md +++ b/RELEASING.md @@ -1,141 +1,138 @@ # Releasing One release stream. `CHANGELOG.md` carries every release section, one generated -page per version describes it, and one tag names the release. A release is one -version covering whatever changed; the components' entries are labelled sections -inside it (`### archied`, `### archie-agent`), not separate numbers or separate -pages. +page per version describes it, and one `v*` tag names the release. One version +covers every process in the host bundle (`archied`, `archie-gateway`, +`archie-state-store`, `archie-ui`, `archie-messaging`) and the `archie-agent` +image; per-component detail is a labelled section inside the version +(`### archied`, `### archie-agent`). -The host bundle ships five processes (`archied`, `archie-gateway`, -`archie-state-store`, `archie-ui`, `archie-messaging`), and -one version covers all of them. `archie-agent`, the per-task sandboxed runtime, -ships as a container image and its changes are the `## [version]` release's -`### archie-agent` section. +A release is cut by CI, published to `next`, run on the canary for a day, and +promoted to `stable` only if the canary's signals pass. Nothing is run locally. -`tools/release.sh` decides which commits a release carries from the package -closure of **every shipped binary**, not from commit-message guessing: a change -to a package only the messaging service, state store, playbook runner or UI -links still lands in the changelog. Read its header comment for how it -partitions commits and enforces `main`-only, clean-tree releases. +## Notes are written with the change -## What justifies a version - -`task release:preview VERSION=X.Y.Z` shows exactly what would land, before -anything is written. A commit set that justifies no version produces no version: -`release.sh` exits **3** with "nothing to release" rather than inventing one, so -a pipeline can finish green without tagging. - -Most sessions touch the gateway side (daemon, gateway, web UI, chat, messaging) -and leave the agent runtime alone. That is no longer a per-component `skip` -decision — one version covers the release — but a runtime code change being -merged to `main` is still not the same as it being in production: the -`archie-agent` image is rebuilt by the release, and a host that runs the agent -must pull the new image (`docker compose pull agent`) to get it. - -## What the notes are for +A commit that changes what a reader experiences adds its bullet under +`## [Unreleased]` in `CHANGELOG.md`, in the same commit. The release takes that +section as it stands, so write the bullet as the release note, not as a commit +subject. **Release notes describe the reader's experience of what shipped.** The test is the reader, not whether a sentence is negative: - A capability that **never existed** leaves no expectation to correct, so a - note about it reads as a feature to someone who never had the thing — out. -- **The implementation's journey** — what was rebuilt, what a guard finally - covers, what a browser API refuses to surface — is not the reader's experience - either — out. + note about it reads as a feature to someone who never had the thing: out. +- **The implementation's journey** (what was rebuilt, what a guard finally + covers, what a browser API refuses to surface) is not the reader's experience + either: out. - A capability that **shipped but is not yet wired** _is_ the reader's experience, so it belongs, worded as a caveat on the capability and paired with the behaviour that compensates for it. -Design constraints and withheld capabilities are not release news. -Do not apply this mechanically: deleting a caveat a reader needs misleads them, -which is the failure this rule exists to prevent. +Design constraints and withheld capabilities are not release news. Deleting a +caveat a reader needs misleads them, which is the failure this rule exists to +prevent. ## Cutting a release -```bash -task release:preview VERSION=1.3.0 # preview what would land -task release:prepare VERSION=1.3.0 # write the section into CHANGELOG.md, uncommitted -# edit CHANGELOG.md -- generated notes are a starting point, not the release -task release VERSION=1.3.0 # commit + annotated tag -git push --atomic origin main v1.3.0 # both refs or neither -``` +Dispatch the **release** workflow (`.github/workflows/release.yml`) with +`major`, `minor`, `patch` or an exact version. It runs `task release` on `main`: -Pushing the tag is the release. The deploy workflow builds the images and the -zip, then `archied release publish` signs the release with the -`RELEASE_SIGNING_KEY` Actions secret and pushes it to -`git.catlow.cloud/sam/archie` as `v1.3.0`, `next`, and `stable` unless the -version is a prerelease. `install.sh` and the updater install only what that -key signed. +1. refuses a dirty tree, an existing tag, or an `[Unreleased]` with no entries; +2. turns `[Unreleased]` into `## [x.y.z] - date` under a new empty + `[Unreleased]`, and runs `task news`; +3. commits `chore(release): vx.y.z` and tags `vx.y.z`; -`release:prepare` emits bare commit subjects; they are a starting point. Rewrite -them to say what a reader gets, per "What the notes are for" above. +then pushes `main` and the tag atomically with `DEPLOY_TOKEN`. A push made with +the job token would not start other workflows, so the user token is what lets +the tag start the deploy. -Pushing is a separate, explicit step — never push without confirming first (see -the repo's general "check before doing anything hard to reverse" rule). The -deploy workflow reads tags pointing at `HEAD`; pushing the commit without its -tag gets images stamped `dev`, which is why the push names the tag and is -`--atomic`. - -### After a release, `main` and the working branch have diverged - -The release commit lands on `main` — `tools/release.sh` refuses to run anywhere -else — while the working branch keeps advancing. So after every release, `main` -is ahead by exactly that commit and the branch is ahead by however far it has -moved: **neither tip is an ancestor of the other**, and `--ff-only` fails in -both directions. That refusal reads as "the tree is not ready" when in fact the -tree is fine. - -Merge the release commit into the branch first, so any conflict is resolved on -the branch, where it is reversible and nobody is reading the tip: - -```bash -git merge main # on the branch -git merge --ff-only # then, in whichever worktree holds main -``` - -Confirm which worktree holds `main` with `git worktree list` before the -`--ff-only`. A detached worktree sitting at the same commit will accept it and -move nothing, which looks like success. +After a release, pull `main` before committing on it again: CI has added the +release commit. ## What a release publishes -Tagging one version produces, from a single `deploy` run: +The tag push runs `deploy.yml`: -- **Container images** — `git.catlow.cloud/sam/archied:latest` and, on a - release tag, `git.catlow.cloud/sam/archie-agent:latest`. The runtime image is - rebuilt by the release, so runtime commits merged to `main` do not reach a - running sandbox until the host pulls the new image. The hand-run - `tools/publish.sh` publishes the same two images to - `git.catlow.cloud/sam` instead, which is the registry the binary's default - `[containers].image` names. -- **`:latest` names the newest release, not the newest commit.** A push to - `main` that is not a release publishes `archied:edge` (mutable, tracking - `main`) and `archied:sha-` (immutable) instead, so a release's - `:latest` stamping is not overwritten minutes later by the next ordinary - commit. A host that runs releases follows `:latest`; a host that tracks `main` - follows `:edge`. -- **A GitHub Release with a distribution zip** — `dist-zip` builds every process - the reference deployment runs (`archied`, `archie-gateway`, - `archie-state-store`, `archie-ui`, `archie-messaging`) for - linux/amd64, packs them with `deployments/INSTRUCTIONS.md` and `CHANGELOG.md`, - and attaches the zip to the Release. The Release body is that version's - `## [version]` section of `CHANGELOG.md`. +- **Gate.** `task check TEST_FLAGS=-count=1` (uncached, including the + Docker-backed tests `-short` skips) and `govulncheck`, then a check that the + tag has a `[version]` section in `CHANGELOG.md`. +- **Images.** `archied:latest` and `archie-agent:latest` on a release tag; + `archied:edge` and `archied:sha-` on any other push to `main`, so + `:latest` always names the newest release. +- **The signed release.** `archied release publish` signs a statement + (version, issued_at, zip digest, agent image by digest) with + `RELEASE_SIGNING_KEY` and pushes it with the distribution zip to + `git.catlow.cloud/sam/archie` as `v` and `next`. It never moves + `stable`. +- **The release-notes page** on Gitea, from that version's `CHANGELOG.md` + section, which the statement links to. +- **The canary install**: `archied release canary start` points the canary at + `next` and installs the release through its dashboard. -The zip is also uploaded as an Actions artifact, but that is a backup copy: -artifacts expire after 90 days and need an Actions login to download. The -Release asset is the durable, operator-reachable one. +GitHub-style Actions run the workflow from the tagged commit, so a fix to +`deploy.yml` reaches a release only if it is an ancestor of the tag. Re-cut the +release rather than committing the fix on top of a tagged one. -## Gate +## The canary -CI (`deploy.yml`) runs `task check TEST_FLAGS=-count=1` before building images, -then verifies any release tag at `HEAD` has a matching `[version]` section in -`CHANGELOG.md`. The gate has one definition, in the Taskfile: the release runs -it with the uncached suite, including the Docker-backed tests `-short` skips. -A tag with no changelog entry is a hard failure. No tags at `HEAD` is a -warning, not a failure -- images get stamped `dev`. +The canary is the production host, following `next`. Every quarter hour +`canary.yml` runs `archied release canary check`, which reads the canary +through its dashboard API and judges the release on `next`: -One consequence worth knowing when a release commit needs amending: GitHub -Actions runs the workflow **from the commit that triggered it**, so a fix to -`deploy.yml` only takes effect for a release if it is an ancestor of the tagged -commit. Delete and re-cut the tag (`git tag -d`, `task release`) rather than -committing the fix on top of an already-tagged release. +| Signal | Source | Fails when | +| --- | --- | --- | +| Install outcome | `GET /api/chat/update` | the canary does not run the release (the watchdog rolled it back) | +| Readiness | `GET /health/detailed` | the rollup is not `ok` | +| Alerts | `GET /api/alerts` | an open `action` alert was first raised after the release was issued | +| Error log | `GET /api/logs?level=error` | error lines per day exceed `2 × baseline + 10` | +| Task outcomes | `GET /api/tasks` | the share of finished tasks that parked or died rose more than 10 points | + +- **Control.** The canary's own 24 hours before the release was issued. With + one production host there is no simultaneous control. +- **Window.** 24 hours from issue, one cycle of schedules and traffic. The + first 30 minutes are for the install and restart. +- **Hard signals** (install, readiness, alerts) roll back at any check after + that. **Relative signals** (errors, tasks) are judged at the end of the + window. +- **Too little data skips a signal** rather than failing it: fewer than 5 + finished tasks in the window, or an error log that reaches back less than an + hour into the baseline. A log read that stops short of a window is scaled to + a full day from what it covers. + +### Promote + +A release that passes the window is promoted: `stable` is retagged to the +`v` manifest. The signature covers the statement, not the tag, so the +retag needs no signing key. Promotion refuses a prerelease and a release issued +before the one `stable` names, the same replay rule hosts enforce. The decision +and its reasons are in the `canary` run log. + +### Roll back + +`stable` stays put, and the run turns red. The canary is pinned to the `stable` +version through its update settings and reinstalled through its dashboard. + +A release whose migrations differ from `stable`'s cannot go back without +restoring the pre-update database snapshot, which loses every write since. The +canary is held on the failed release instead (pinned to it) and the fix goes +forward in the next release. The next release's deploy points the canary back at +`next`. + +A pinned canary is not judged, so a rollback is decided once. + +### Configuration + +| Name | Kind | Holds | +| --- | --- | --- | +| `CANARY_URL` | Actions variable | the canary's dashboard URL; unset disables the canary steps | +| `CANARY_TOKEN` | Actions secret | a personal API token on the canary with instance admin | +| `DEPLOY_TOKEN` | Actions secret | registry and repository write, for publishing, promoting and the release push | +| `RELEASE_SIGNING_KEY` | Actions secret | the release signing seed | + +The runner must reach `CANARY_URL`. Without a canary a release stays on `next` +until promoted by hand, under the same refusals: + +```bash +archied release promote -version 1.57.0 +``` diff --git a/internal/app/archied/release.go b/internal/app/archied/release.go index 5f28ea064..01e997bf7 100644 --- a/internal/app/archied/release.go +++ b/internal/app/archied/release.go @@ -17,7 +17,10 @@ import ( const releaseUsage = `usage: archied release keygen -seed FILE write a new signing key's seed to FILE and print its public key - publish sign a release and push it to a release source` + publish sign a release and push it to a release source's next channel + promote -version V point stable at a published release by hand + canary start point the canary at next and install a release on it + canary check judge the release on next and promote it or roll the canary back` // RunRelease is the publisher's side of a release source: it makes the key // a source signs with and pushes signed releases to it. @@ -31,6 +34,10 @@ func RunRelease(args []string, out, stderr io.Writer) int { return runReleaseKeygen(args[1:], out, stderr) case "publish": return runReleasePublish(args[1:], out, stderr) + case "promote": + return runReleasePromote(args[1:], out, stderr) + case "canary": + return runReleaseCanary(args[1:], out, stderr) default: fmt.Fprintln(stderr, releaseUsage) return 2 @@ -79,7 +86,7 @@ func runReleasePublish(args []string, out, stderr io.Writer) int { flags.StringVar(&req.Repository, "source", "", "OCI repository to publish to") seedEnv := flags.String("seed-env", "ARCHIE_RELEASE_SEED", "environment variable holding the base64 private key seed") flags.StringVar(&req.Version, "version", "", "release version") - flags.BoolVar(&req.Prerelease, "prerelease", false, "publish to next only") + flags.BoolVar(&req.Prerelease, "prerelease", false, "mark the release a prerelease, which never reaches stable") flags.StringVar(&req.ZipPath, "zip", "", "distribution zip") flags.StringVar(&req.AgentImage, "agent-image", "", "agent image pinned by digest") flags.StringVar(&req.Changelog, "changelog", "", "URL of the release notes") @@ -97,10 +104,8 @@ func runReleasePublish(args []string, out, stderr io.Writer) int { } req.Signer = ed25519.NewKeyFromSeed(seed) public := distribution.EncodeKey(ed25519.PublicKey(req.Signer[ed25519.SeedSize:])) + // stable moves only when the canary promotes the release. req.Channels = []string{"next"} - if !req.Prerelease { - req.Channels = append(req.Channels, "stable") - } statement, err := releaseupdate.Publish(context.Background(), req) if err != nil { fmt.Fprintln(stderr, err) @@ -110,3 +115,28 @@ func runReleasePublish(args []string, out, stderr io.Writer) int { strings.Join(req.Channels, ", "), public) return 0 } + +// runReleasePromote moves stable without a canary verdict, under the same +// rules the canary promotes by. +func runReleasePromote(args []string, out, stderr io.Writer) int { + flags := flag.NewFlagSet("release promote", flag.ContinueOnError) + flags.SetOutput(stderr) + reference := flags.String("source", "", "OCI repository releases are published to; blank is the official releases") + key := flags.String("key", "", "base64 ed25519 public key the source signs with") + version := flags.String("version", "", "release to promote") + if err := flags.Parse(args); err != nil { + return 2 + } + source, err := distribution.Parse(*reference, *key, distribution.DefaultReleases()) + if err != nil || *version == "" { + fmt.Fprintln(stderr, "release promote: -version is required, and -source with -key name another source") + return 2 + } + statement, err := releaseupdate.Promote(context.Background(), source.Reference, source.Key, *version) + if err != nil { + fmt.Fprintln(stderr, err) + return 1 + } + fmt.Fprintf(out, "stable is %s\n", statement.Version) + return 0 +} diff --git a/internal/app/archied/release_canary.go b/internal/app/archied/release_canary.go new file mode 100644 index 000000000..5412f9b93 --- /dev/null +++ b/internal/app/archied/release_canary.go @@ -0,0 +1,159 @@ +package archied + +import ( + "context" + "errors" + "flag" + "fmt" + "io" + "os" + "os/exec" + "time" + + "oras.land/oras-go/v2/errdef" + + "github.com/samcharles93/archie-core/internal/domain/distribution" + "github.com/samcharles93/archie-core/internal/releaseupdate" +) + +// canaryInstallWait bounds how long an install on the canary may take, +// including the watchdog's health wait and a rollback. +const canaryInstallWait = 20 * time.Minute + +// migrationsDir holds the schema migrations; a release that changes it cannot +// be rolled back without restoring the database snapshot. +const migrationsDir = "internal/infrastructure/postgres/migrations" + +func runReleaseCanary(args []string, out, stderr io.Writer) int { + if len(args) == 0 || (args[0] != "start" && args[0] != "check") { + fmt.Fprintln(stderr, releaseUsage) + return 2 + } + flags := flag.NewFlagSet("release canary "+args[0], flag.ContinueOnError) + flags.SetOutput(stderr) + dashboardURL := flags.String("dashboard", "", "canary dashboard URL") + tokenEnv := flags.String("token-env", "ARCHIE_CANARY_TOKEN", "environment variable holding a dashboard API token for the canary") + version := flags.String("version", "", "release to install (start)") + reference := flags.String("source", "", "OCI repository releases are published to; blank is the official releases (check)") + key := flags.String("key", "", "base64 ed25519 public key the source signs with (check)") + checkout := flags.String("checkout", ".", "repository checkout with release tags, to see whether a release migrated the database (check)") + if err := flags.Parse(args[1:]); err != nil { + return 2 + } + dashboard := releaseupdate.CanaryDashboard{URL: *dashboardURL, Token: os.Getenv(*tokenEnv)} + if *dashboardURL == "" { + fmt.Fprintln(stderr, "release canary: -dashboard is required") + return 2 + } + ctx := context.Background() + if args[0] == "start" { + if *version == "" { + fmt.Fprintln(stderr, "release canary start: -version is required") + return 2 + } + if err := startCanary(ctx, dashboard, *version); err != nil { + fmt.Fprintln(stderr, err) + return 1 + } + fmt.Fprintf(out, "canary follows next and runs %s\n", *version) + return 0 + } + source, err := distribution.Parse(*reference, *key, distribution.DefaultReleases()) + if err != nil { + fmt.Fprintln(stderr, "release canary check:", err) + return 2 + } + rolledBack, err := checkCanary(ctx, dashboard, source, *checkout, out) + if err != nil { + fmt.Fprintln(stderr, err) + return 1 + } + if rolledBack { + return 1 + } + return 0 +} + +func startCanary(ctx context.Context, dashboard releaseupdate.CanaryDashboard, version string) error { + if err := dashboard.Follow(ctx, "next", ""); err != nil { + return fmt.Errorf("point the canary at next: %w", err) + } + return dashboard.Install(ctx, version, canaryInstallWait) +} + +// checkCanary judges the release on next once and acts on the verdict. It +// reports rolledBack so a failed canary turns the run red. +func checkCanary(ctx context.Context, dashboard releaseupdate.CanaryDashboard, source distribution.Source, checkout string, out io.Writer) (rolledBack bool, err error) { + next, err := releaseupdate.Fetch(ctx, source, "next") + if err != nil { + return false, err + } + stable, err := releaseupdate.Fetch(ctx, source, "stable") + hasStable := err == nil + if err != nil && !errors.Is(err, errdef.ErrNotFound) { + return false, err + } + release := next.Statement + if release.Prerelease || (hasStable && !release.IssuedAt.After(stable.Statement.IssuedAt)) { + fmt.Fprintf(out, "nothing to judge: next is %s and is not a newer release than stable\n", release.Version) + return false, nil + } + channel, pin, err := dashboard.Channel(ctx) + if err != nil { + return false, err + } + if channel != "next" { + fmt.Fprintf(out, "nothing to judge: the canary is held on %s %s\n", channel, pin) + return false, nil + } + now := time.Now().UTC() + signals, err := dashboard.Signals(ctx, release.IssuedAt, now) + if err != nil { + return false, fmt.Errorf("read canary signals: %w", err) + } + verdict := releaseupdate.JudgeCanary(release.Version, now.Sub(release.IssuedAt), signals) + fmt.Fprintf(out, "%s: %s\n", release.Version, verdict.Decision) + for _, reason := range verdict.Reasons { + fmt.Fprintf(out, " %s\n", reason) + } + switch verdict.Decision { + case releaseupdate.CanaryPromote: + if _, err := releaseupdate.Promote(ctx, source.Reference, source.Key, release.Version); err != nil { + return false, fmt.Errorf("promote %s: %w", release.Version, err) + } + fmt.Fprintf(out, "stable is %s\n", release.Version) + return false, nil + case releaseupdate.CanaryRollBack: + return true, rollBackCanary(ctx, dashboard, release.Version, stable.Statement.Version, hasStable, checkout, out) + } + return false, nil +} + +// rollBackCanary pins the canary to the stable release and reinstalls it. +// Going back past a migration needs the pre-update database snapshot, which +// loses every write since, so a release that migrated is held instead. +func rollBackCanary(ctx context.Context, dashboard releaseupdate.CanaryDashboard, failed, stable string, hasStable bool, checkout string, out io.Writer) error { + if !hasStable || migrated(ctx, checkout, stable, failed) { + if err := dashboard.Follow(ctx, "exact-pin", failed); err != nil { + return fmt.Errorf("hold the canary on %s: %w", failed, err) + } + fmt.Fprintf(out, "%s changed the database schema or there is no stable release to return to: the canary is held on %s and stable is unchanged; fix forward\n", failed, failed) + return nil + } + if err := dashboard.Follow(ctx, "exact-pin", stable); err != nil { + return fmt.Errorf("pin the canary to %s: %w", stable, err) + } + if err := dashboard.Install(ctx, stable, canaryInstallWait); err != nil { + return fmt.Errorf("return the canary to %s: %w", stable, err) + } + fmt.Fprintf(out, "the canary is back on %s and stable is unchanged\n", stable) + return nil +} + +// migrated reports whether the migrations differ between two release tags. +// Anything it cannot tell counts as migrated, which holds rather than +// restoring a snapshot. +func migrated(ctx context.Context, checkout, from, to string) bool { + err := exec.CommandContext(ctx, "git", "-C", checkout, "diff", "--quiet", "v"+from, "v"+to, "--", migrationsDir).Run() + return err != nil +} diff --git a/internal/releaseupdate/canary.go b/internal/releaseupdate/canary.go new file mode 100644 index 000000000..7a20a1cc4 --- /dev/null +++ b/internal/releaseupdate/canary.go @@ -0,0 +1,136 @@ +package releaseupdate + +import ( + "fmt" + "time" +) + +// Canary evaluation. The canary host takes a release from next; its window +// after the release was issued is compared with its own window of the same +// length before, since a single production host has no simultaneous control. +const ( + // CanaryWindow is how long a release runs on the canary before the + // relative signals are judged: one daily cycle of schedules and traffic. + CanaryWindow = 24 * time.Hour + // canarySettle is how long after issue the install and restart are given + // before the hard signals count. + canarySettle = 30 * time.Minute + // minFinished is the fewest finished tasks the park rate is judged on. + minFinished = 5 + // minLogCoverage is the least of the baseline window the error log must + // cover for the error signal to be judged. + minLogCoverage = time.Hour +) + +// CanaryDecision is what the evaluation says to do with a release. +type CanaryDecision string + +const ( + CanaryWait CanaryDecision = "wait" + CanaryPromote CanaryDecision = "promote" + CanaryRollBack CanaryDecision = "roll back" +) + +// CanaryWindowSignals is one window's task and error-log counts. +type CanaryWindowSignals struct { + // Finished counts tasks that reached a terminal or review status; Bad + // is those that parked or died. + Finished, Bad int + // Errors counts error-level log lines over Covered, the part of the + // window the log reaches back over. + Errors int + Covered time.Duration +} + +// CanarySignals is what the canary reported at one check. +type CanarySignals struct { + Installed string + Ready bool + // NewActionAlerts are open action-severity alerts first raised since + // the release was issued. + NewActionAlerts []string + Baseline CanaryWindowSignals + Canary CanaryWindowSignals +} + +// CanaryVerdict is a decision and the signals that decided it. +type CanaryVerdict struct { + Decision CanaryDecision + Reasons []string +} + +// JudgeCanary decides on version elapsed after it was issued. A hard signal +// rolls back at any check once the release has settled; the relative signals +// are judged only when the window has passed, against the baseline window. +func JudgeCanary(version string, elapsed time.Duration, s CanarySignals) CanaryVerdict { + if elapsed < canarySettle { + return CanaryVerdict{Decision: CanaryWait, Reasons: []string{"settling after install"}} + } + var failed []string + if s.Installed != version { + failed = append(failed, fmt.Sprintf("canary runs %q, not %s: the install failed or was rolled back", s.Installed, version)) + } + if !s.Ready { + failed = append(failed, "readiness is degraded") + } + for _, alert := range s.NewActionAlerts { + failed = append(failed, "new action alert: "+alert) + } + if len(failed) > 0 { + return CanaryVerdict{Decision: CanaryRollBack, Reasons: failed} + } + if elapsed < CanaryWindow { + return CanaryVerdict{Decision: CanaryWait, Reasons: []string{fmt.Sprintf("%s of %s elapsed", elapsed.Round(time.Minute), CanaryWindow)}} + } + var notes []string + if reason, ok := judgeErrors(s.Baseline, s.Canary); !ok { + failed = append(failed, reason) + } else { + notes = append(notes, reason) + } + if reason, ok := judgeParks(s.Baseline, s.Canary); !ok { + failed = append(failed, reason) + } else { + notes = append(notes, reason) + } + if len(failed) > 0 { + return CanaryVerdict{Decision: CanaryRollBack, Reasons: failed} + } + return CanaryVerdict{Decision: CanaryPromote, Reasons: notes} +} + +// judgeErrors fails when the canary logged more than twice the baseline's +// error lines plus ten, each scaled to a full window from what the log covers. +func judgeErrors(baseline, canary CanaryWindowSignals) (string, bool) { + if baseline.Covered < minLogCoverage || canary.Covered <= 0 { + return "error log does not reach back over the baseline; error signal skipped", true + } + scale := func(w CanaryWindowSignals) float64 { + return float64(w.Errors) * float64(CanaryWindow) / float64(w.Covered) + } + before, after := scale(baseline), scale(canary) + limit := 2*before + 10 + if after > limit { + return fmt.Sprintf("%.0f error log lines per day against a limit of %.0f (baseline %.0f)", after, limit, before), false + } + return fmt.Sprintf("%.0f error log lines per day, limit %.0f", after, limit), true +} + +// judgeParks fails when the share of finished tasks that parked or died rose +// more than ten points over the baseline. +func judgeParks(baseline, canary CanaryWindowSignals) (string, bool) { + if canary.Finished < minFinished { + return fmt.Sprintf("%d finished tasks, fewer than %d; park signal skipped", canary.Finished, minFinished), true + } + rate := func(w CanaryWindowSignals) float64 { + if w.Finished == 0 { + return 0 + } + return float64(w.Bad) / float64(w.Finished) + } + before, after := rate(baseline), rate(canary) + if after > before+0.10 { + return fmt.Sprintf("%.0f%% of tasks parked or died against %.0f%% before", 100*after, 100*before), false + } + return fmt.Sprintf("%.0f%% of tasks parked or died, %.0f%% before", 100*after, 100*before), true +} diff --git a/internal/releaseupdate/canary_dashboard.go b/internal/releaseupdate/canary_dashboard.go new file mode 100644 index 000000000..2d2d61fcf --- /dev/null +++ b/internal/releaseupdate/canary_dashboard.go @@ -0,0 +1,309 @@ +package releaseupdate + +import ( + "bytes" + "context" + "encoding/json" + "errors" + "fmt" + "io" + "net/http" + "net/url" + "strings" + "time" +) + +// CanaryDashboard reaches the canary through its dashboard API, the same +// contract an operator uses: it reads the signals, edits the update settings +// through the control plane and installs through the dashboard updater. +type CanaryDashboard struct { + URL string + Token string + HTTP *http.Client +} + +// Statuses a finished task can be in, and the subset that counts against a +// release. +var ( + finishedStatuses = map[string]bool{"completed": true, "merged": true, "pr_open": true, "rejected": true, "closed_wont_do": true, "parked": true, "dead": true} + badStatuses = map[string]bool{"parked": true, "dead": true} +) + +const updateSettingsPath = "/api/control-plane/resources/update-settings" + +// Signals reads what the canary reports, splitting tasks and error lines into +// the window before issued and the window from issued to now. +func (d CanaryDashboard) Signals(ctx context.Context, issued, now time.Time) (CanarySignals, error) { + var s CanarySignals + var err error + if s.Installed, _, err = d.installed(ctx); err != nil { + return s, err + } + if s.Ready, err = d.ready(ctx); err != nil { + return s, err + } + if s.NewActionAlerts, err = d.actionAlerts(ctx, issued); err != nil { + return s, err + } + if err := d.tasks(ctx, issued, &s); err != nil { + return s, err + } + return s, d.errors(ctx, issued, now, &s) +} + +// installed is the daemon version the canary runs and the raw snapshot it +// offers, which an install must echo back. +func (d CanaryDashboard) installed(ctx context.Context) (string, json.RawMessage, error) { + var body struct { + Snapshot json.RawMessage `json:"snapshot"` + } + if _, err := d.call(ctx, http.MethodGet, "/api/chat/update", nil, &body); err != nil { + return "", nil, err + } + var snapshot Snapshot + if err := json.Unmarshal(body.Snapshot, &snapshot); err != nil { + return "", nil, fmt.Errorf("decode update snapshot: %w", err) + } + for _, c := range snapshot.Components { + if c.ID == "daemon" { + return c.Installed, body.Snapshot, nil + } + } + return "", body.Snapshot, errors.New("canary reports no daemon version") +} + +func (d CanaryDashboard) ready(ctx context.Context) (bool, error) { + var report struct { + Status string `json:"status"` + } + if _, err := d.call(ctx, http.MethodGet, "/health/detailed", nil, &report); err != nil { + return false, err + } + return report.Status == "ok", nil +} + +func (d CanaryDashboard) actionAlerts(ctx context.Context, since time.Time) ([]string, error) { + var body struct { + Alerts []struct { + Severity string `json:"severity"` + Title string `json:"title"` + FirstRaised time.Time `json:"first_raised"` + } `json:"alerts"` + } + if _, err := d.call(ctx, http.MethodGet, "/api/alerts", nil, &body); err != nil { + return nil, err + } + var titles []string + for _, a := range body.Alerts { + if a.Severity == "action" && !a.FirstRaised.Before(since) { + titles = append(titles, a.Title) + } + } + return titles, nil +} + +// tasks pages back through tasks by last update until the baseline window +// has been read. +func (d CanaryDashboard) tasks(ctx context.Context, issued time.Time, s *CanarySignals) error { + from := issued.Add(-CanaryWindow) + cursor := "" + for range 50 { + var page []struct { + Status string `json:"status"` + UpdatedAt time.Time `json:"updated_at"` + } + path := "/api/tasks?limit=500" + if cursor != "" { + path += "&cursor=" + url.QueryEscape(cursor) + } + header, err := d.call(ctx, http.MethodGet, path, nil, &page) + if err != nil { + return err + } + for _, t := range page { + if t.UpdatedAt.Before(from) || !finishedStatuses[t.Status] { + continue + } + w := &s.Canary + if t.UpdatedAt.Before(issued) { + w = &s.Baseline + } + w.Finished++ + if badStatuses[t.Status] { + w.Bad++ + } + } + cursor = header.Get("X-Next-Cursor") + if cursor == "" || len(page) == 0 || page[len(page)-1].UpdatedAt.Before(from) { + return nil + } + } + return nil +} + +// errors counts error-level lines in each window. The log read is bounded, so +// a truncated read covers only back to its oldest line. +func (d CanaryDashboard) errors(ctx context.Context, issued, now time.Time, s *CanarySignals) error { + var body struct { + Entries []struct { + Time time.Time `json:"time"` + } `json:"entries"` + Truncated bool `json:"truncated"` + } + if _, err := d.call(ctx, http.MethodGet, "/api/logs?level=error&limit=2000", nil, &body); err != nil { + return err + } + from := issued.Add(-CanaryWindow) + if body.Truncated { + from = now + if len(body.Entries) > 0 { + from = later(body.Entries[0].Time, issued.Add(-CanaryWindow)) + } + } + for _, e := range body.Entries { + switch { + case e.Time.Before(from): + case e.Time.Before(issued): + s.Baseline.Errors++ + default: + s.Canary.Errors++ + } + } + s.Baseline.Covered = max(issued.Sub(from), 0) + s.Canary.Covered = now.Sub(later(from, issued)) + return nil +} + +func later(a, b time.Time) time.Time { + if a.After(b) { + return a + } + return b +} + +// Follow sets the canary's update channel, keeping its release source and +// key. A pin is used only by exact-pin. +func (d CanaryDashboard) Follow(ctx context.Context, channel, pin string) error { + var current struct { + Resource struct { + Version int64 `json:"version"` + Value json.RawMessage `json:"value"` + } `json:"resource"` + } + if _, err := d.call(ctx, http.MethodGet, updateSettingsPath, nil, ¤t); err != nil { + return err + } + value := map[string]any{} + if len(current.Resource.Value) > 0 && string(current.Resource.Value) != "null" { + if err := json.Unmarshal(current.Resource.Value, &value); err != nil { + return fmt.Errorf("decode update settings: %w", err) + } + } + value["channel"], value["pin"] = channel, pin + _, err := d.call(ctx, http.MethodPost, updateSettingsPath+"/commands/replace", + map[string]any{"value": value, "expected_version": current.Resource.Version}, nil) + return err +} + +// Channel is the canary's update channel and pin. +func (d CanaryDashboard) Channel(ctx context.Context) (channel, pin string, err error) { + var current struct { + Resource struct { + Value struct { + Channel string `json:"channel"` + Pin string `json:"pin"` + } `json:"value"` + } `json:"resource"` + } + if _, err := d.call(ctx, http.MethodGet, updateSettingsPath, nil, ¤t); err != nil { + return "", "", err + } + if current.Resource.Value.Channel == "" { + return "stable", "", nil + } + return current.Resource.Value.Channel, current.Resource.Value.Pin, nil +} + +// Install has the canary install version through its updater and waits until +// it reports running it. The install restarts the dashboard, so a dropped +// connection after the request is sent is expected. +func (d CanaryDashboard) Install(ctx context.Context, version string, wait time.Duration) error { + installed, snapshot, err := d.installed(ctx) + if err != nil { + return err + } + if installed == version { + return nil + } + var offered Snapshot + if err := json.Unmarshal(snapshot, &offered); err != nil { + return fmt.Errorf("decode update snapshot: %w", err) + } + if available := offered.Available(); len(available) == 0 || available[0].Available != version { + return fmt.Errorf("canary does not offer %s yet", version) + } + _, err = d.call(ctx, http.MethodPost, "/api/chat/update/install", map[string]json.RawMessage{"snapshot": snapshot}, nil) + if _, ok := errors.AsType[statusError](err); ok { + return err + } + deadline := time.Now().Add(wait) + for time.Now().Before(deadline) { + select { + case <-ctx.Done(): + return ctx.Err() + case <-time.After(15 * time.Second): + } + if installed, _, err := d.installed(ctx); err == nil && installed == version { + return nil + } + } + return fmt.Errorf("canary did not report %s within %s", version, wait) +} + +type statusError struct { + code int + body string +} + +func (e statusError) Error() string { return fmt.Sprintf("dashboard answered %d: %s", e.code, e.body) } + +func (d CanaryDashboard) call(ctx context.Context, method, path string, in, out any) (http.Header, error) { + var body io.Reader + if in != nil { + data, err := json.Marshal(in) + if err != nil { + return nil, err + } + body = bytes.NewReader(data) + } + ctx, cancel := context.WithTimeout(ctx, 2*time.Minute) + defer cancel() + req, err := http.NewRequestWithContext(ctx, method, strings.TrimRight(d.URL, "/")+path, body) + if err != nil { + return nil, err + } + req.Header.Set("Authorization", "Bearer "+d.Token) + if in != nil { + req.Header.Set("Content-Type", "application/json") + } + client := d.HTTP + if client == nil { + client = http.DefaultClient + } + resp, err := client.Do(req) + if err != nil { + return nil, fmt.Errorf("%s %s: %w", method, path, err) + } + defer func() { _ = resp.Body.Close() }() + if resp.StatusCode/100 != 2 { + text, _ := io.ReadAll(io.LimitReader(resp.Body, 512)) + return nil, fmt.Errorf("%s %s: %w", method, path, statusError{resp.StatusCode, strings.TrimSpace(string(text))}) + } + if out == nil { + return resp.Header, nil + } + if err := json.NewDecoder(resp.Body).Decode(out); err != nil { + return nil, fmt.Errorf("%s %s: decode: %w", method, path, err) + } + return resp.Header, nil +} diff --git a/internal/releaseupdate/canary_test.go b/internal/releaseupdate/canary_test.go new file mode 100644 index 000000000..a6028b85e --- /dev/null +++ b/internal/releaseupdate/canary_test.go @@ -0,0 +1,56 @@ +package releaseupdate + +import ( + "testing" + "time" +) + +// A release promotes only after a full window with no hard failure and no +// relative signal past its threshold; any hard failure after settling rolls +// it back at once. +func TestJudgeCanary(t *testing.T) { + healthy := func() CanarySignals { + return CanarySignals{ + Installed: "1.1.0", Ready: true, + Baseline: CanaryWindowSignals{Finished: 20, Bad: 2, Errors: 10, Covered: CanaryWindow}, + Canary: CanaryWindowSignals{Finished: 20, Bad: 2, Errors: 12, Covered: CanaryWindow}, + } + } + with := func(change func(*CanarySignals)) CanarySignals { + s := healthy() + change(&s) + return s + } + cases := []struct { + name string + elapsed time.Duration + signals CanarySignals + want CanaryDecision + }{ + {"settling ignores a failed check", 10 * time.Minute, with(func(s *CanarySignals) { s.Ready = false }), CanaryWait}, + {"healthy mid-window waits", 2 * time.Hour, healthy(), CanaryWait}, + {"healthy at the window promotes", CanaryWindow, healthy(), CanaryPromote}, + {"the watchdog rolled the install back", 2 * time.Hour, with(func(s *CanarySignals) { s.Installed = "1.0.0" }), CanaryRollBack}, + {"readiness degraded", 2 * time.Hour, with(func(s *CanarySignals) { s.Ready = false }), CanaryRollBack}, + {"a new action alert", 2 * time.Hour, with(func(s *CanarySignals) { s.NewActionAlerts = []string{"queue stalled"} }), CanaryRollBack}, + {"errors past twice the baseline plus ten", CanaryWindow, with(func(s *CanarySignals) { s.Canary.Errors = 31 }), CanaryRollBack}, + {"errors at the limit", CanaryWindow, with(func(s *CanarySignals) { s.Canary.Errors = 30 }), CanaryPromote}, + {"errors scaled from a partly covered log", CanaryWindow, with(func(s *CanarySignals) { + s.Canary.Errors, s.Canary.Covered = 8, 6*time.Hour + }), CanaryRollBack}, + {"a baseline the log does not reach skips errors", CanaryWindow, with(func(s *CanarySignals) { + s.Baseline.Covered, s.Canary.Errors = 30*time.Minute, 500 + }), CanaryPromote}, + {"park rate up more than ten points", CanaryWindow, with(func(s *CanarySignals) { s.Canary.Bad = 5 }), CanaryRollBack}, + {"park rate up ten points", CanaryWindow, with(func(s *CanarySignals) { s.Canary.Bad = 4 }), CanaryPromote}, + {"too few tasks skips the park rate", CanaryWindow, with(func(s *CanarySignals) { s.Canary = CanaryWindowSignals{Finished: 4, Bad: 4, Covered: CanaryWindow} }), CanaryPromote}, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + got := JudgeCanary("1.1.0", tc.elapsed, tc.signals) + if got.Decision != tc.want { + t.Fatalf("decision = %s, want %s (%v)", got.Decision, tc.want, got.Reasons) + } + }) + } +} diff --git a/internal/releaseupdate/promote.go b/internal/releaseupdate/promote.go new file mode 100644 index 000000000..58120db4e --- /dev/null +++ b/internal/releaseupdate/promote.go @@ -0,0 +1,74 @@ +package releaseupdate + +import ( + "context" + "crypto/ed25519" + "encoding/json" + "errors" + "fmt" + "io" + + digest "github.com/opencontainers/go-digest" + ocispec "github.com/opencontainers/image-spec/specs-go/v1" + "oras.land/oras-go/v2/errdef" + + "github.com/samcharles93/archie-core/internal/domain/distribution" + "github.com/samcharles93/archie-core/internal/infrastructure/signedoci" +) + +// Promote points stable at the release published as version. The signature +// covers the statement, not the tag, so stable is the same signed manifest +// and needs no re-signing. A prerelease, or a release issued before the one +// stable names, is refused: hosts would refuse the latter as a replay. +func Promote(ctx context.Context, repository string, key ed25519.PublicKey, version string) (Statement, error) { + source := distribution.Source{Reference: repository, Key: key} + release, err := Fetch(ctx, source, Tag("exact-pin", version)) + if err != nil { + return Statement{}, err + } + if release.Statement.Prerelease { + return Statement{}, fmt.Errorf("%s is a prerelease and never reaches stable", version) + } + current, err := Fetch(ctx, source, "stable") + switch { + case errors.Is(err, errdef.ErrNotFound): + case err != nil: + return Statement{}, err + case release.Statement.IssuedAt.Before(current.Statement.IssuedAt): + return Statement{}, fmt.Errorf("%s was issued before %s, which stable names", version, current.Statement.Version) + } + repo, err := pushRepository(repository) + if err != nil { + return Statement{}, err + } + // Tag the manifest the version tag resolves to now, after checking it + // carries the statement just verified, so a tag moved in between cannot + // slip an unverified manifest onto stable. + desc, err := repo.Resolve(ctx, Tag("exact-pin", version)) + if err != nil { + return Statement{}, fmt.Errorf("resolve %s: %w", version, err) + } + stream, err := repo.Fetch(ctx, desc) + if err != nil { + return Statement{}, fmt.Errorf("fetch %s manifest: %w", version, err) + } + raw, err := io.ReadAll(io.LimitReader(stream, desc.Size)) + _ = stream.Close() + if err != nil { + return Statement{}, fmt.Errorf("read %s manifest: %w", version, err) + } + if digest.FromBytes(raw) != desc.Digest { + return Statement{}, fmt.Errorf("%s manifest does not match its digest", version) + } + var manifest ocispec.Manifest + if err := json.Unmarshal(raw, &manifest); err != nil { + return Statement{}, fmt.Errorf("%s manifest: %w", version, err) + } + if err := signedoci.Verify(manifest, release.Artifact.Statement, key, Annotation); err != nil { + return Statement{}, fmt.Errorf("%s changed while promoting: %w", version, err) + } + if err := repo.Tag(ctx, desc, "stable"); err != nil { + return Statement{}, fmt.Errorf("tag stable: %w", err) + } + return release.Statement, nil +} diff --git a/internal/releaseupdate/promote_test.go b/internal/releaseupdate/promote_test.go new file mode 100644 index 000000000..a534e9904 --- /dev/null +++ b/internal/releaseupdate/promote_test.go @@ -0,0 +1,167 @@ +package releaseupdate + +import ( + "crypto/ed25519" + "encoding/base64" + "encoding/json" + "io" + "net/http" + "net/http/httptest" + "strconv" + "strings" + "sync" + "testing" + "time" + + digest "github.com/opencontainers/go-digest" + ocispec "github.com/opencontainers/image-spec/specs-go/v1" + + "github.com/samcharles93/archie-core/internal/domain/distribution" + "github.com/samcharles93/archie-core/internal/infrastructure/signedoci" +) + +// stable moves only to a signed, non-prerelease release issued no earlier +// than the one it names, and it moves to the very manifest that was verified. +func TestPromote(t *testing.T) { + trusted, signer, _ := ed25519.GenerateKey(nil) + _, stranger, _ := ed25519.GenerateKey(nil) + jan1, jan2 := time.Date(2026, 1, 1, 0, 0, 0, 0, time.UTC), time.Date(2026, 1, 2, 0, 0, 0, 0, time.UTC) + type published struct { + version string + issued time.Time + prerelease bool + signer ed25519.PrivateKey + tags []string + } + cases := []struct { + name string + release []published + promote string + want string // error substring, or "" when stable must name promote + }{ + {name: "a newer release", release: []published{{"1.0.0", jan1, false, signer, []string{"stable"}}, {"1.1.0", jan2, false, signer, []string{"next"}}}, promote: "1.1.0"}, + {name: "the first release", release: []published{{"1.0.0", jan1, false, signer, []string{"next"}}}, promote: "1.0.0"}, + {name: "a release issued before stable", release: []published{{"1.1.0", jan2, false, signer, []string{"stable"}}, {"1.0.0", jan1, false, signer, nil}}, promote: "1.0.0", want: "issued before"}, + {name: "a prerelease", release: []published{{"1.1.0-rc.1", jan2, true, signer, []string{"next"}}}, promote: "1.1.0-rc.1", want: "prerelease"}, + {name: "a release another key signed", release: []published{{"1.1.0", jan2, false, stranger, []string{"next"}}}, promote: "1.1.0", want: "signature"}, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + t.Setenv("XDG_STATE_HOME", t.TempDir()) + t.Setenv("DOCKER_CONFIG", t.TempDir()) + registry := newMemoryRegistry() + server := httptest.NewServer(registry) + defer server.Close() + for _, r := range tc.release { + registry.publish(t, r.version, r.issued, r.prerelease, r.signer, r.tags...) + } + repository := strings.TrimPrefix(server.URL, "http://") + "/archie" + _, err := Promote(t.Context(), repository, trusted, tc.promote) + switch { + case tc.want == "" && err != nil: + t.Fatalf("Promote: %v", err) + case tc.want != "" && (err == nil || !strings.Contains(err.Error(), tc.want)): + t.Fatalf("Promote error = %v, want %q", err, tc.want) + } + stable, ok := registry.statement("stable") + promoted := ok && stable.Version == tc.promote + if promoted != (tc.want == "") { + t.Fatalf("stable names %q (found %v) after promoting %s", stable.Version, ok, tc.promote) + } + if tc.want == "" { + if _, err := Fetch(t.Context(), distribution.Source{Reference: repository, Key: trusted}, "stable"); err != nil { + t.Fatalf("hosts refuse the promoted stable: %v", err) + } + } + }) + } +} + +// memoryRegistry serves manifests by tag or digest and accepts manifest +// pushes, which is all fetching and retagging a release needs. +type memoryRegistry struct { + mu sync.Mutex + manifests map[string][]byte + blobs map[string][]byte +} + +func newMemoryRegistry() *memoryRegistry { + return &memoryRegistry{manifests: map[string][]byte{}, blobs: map[string][]byte{}} +} + +func (m *memoryRegistry) publish(t *testing.T, version string, issued time.Time, prerelease bool, signer ed25519.PrivateKey, tags ...string) { + t.Helper() + archive := []byte("zip " + version) + statement, err := json.Marshal(Statement{ + Version: version, Prerelease: prerelease, IssuedAt: issued, Platform: Platform, AgentImage: newImage, + Zip: Blob{Name: ZipName(version), Digest: signedoci.Digest(archive), Size: int64(len(archive))}, + }) + if err != nil { + t.Fatal(err) + } + manifest, err := json.Marshal(ocispec.Manifest{ + SchemaVersion: 2, MediaType: ocispec.MediaTypeImageManifest, ArtifactType: ArtifactType, Config: ocispec.DescriptorEmptyJSON, + Layers: []ocispec.Descriptor{ + {MediaType: StatementMediaType, Digest: digest.FromBytes(statement), Size: int64(len(statement))}, + {MediaType: "application/zip", Digest: digest.FromBytes(archive), Size: int64(len(archive))}, + }, + Annotations: map[string]string{Annotation: base64.StdEncoding.EncodeToString(ed25519.Sign(signer, statement))}, + }) + if err != nil { + t.Fatal(err) + } + m.mu.Lock() + defer m.mu.Unlock() + m.blobs[signedoci.Digest(statement)], m.blobs[signedoci.Digest(archive)] = statement, archive + m.manifests[signedoci.Digest(manifest)] = manifest + for _, tag := range append(tags, "v"+version) { + m.manifests[tag] = manifest + } +} + +func (m *memoryRegistry) statement(tag string) (Statement, bool) { + m.mu.Lock() + defer m.mu.Unlock() + var manifest ocispec.Manifest + if json.Unmarshal(m.manifests[tag], &manifest) != nil || len(manifest.Layers) == 0 { + return Statement{}, false + } + var s Statement + return s, json.Unmarshal(m.blobs[manifest.Layers[0].Digest.String()], &s) == nil +} + +func (m *memoryRegistry) ServeHTTP(w http.ResponseWriter, r *http.Request) { + m.mu.Lock() + defer m.mu.Unlock() + if ref, ok := strings.CutPrefix(r.URL.Path, "/v2/archie/blobs/"); ok { + blob, found := m.blobs[ref] + if !found { + http.NotFound(w, r) + return + } + _, _ = w.Write(blob) + return + } + ref, ok := strings.CutPrefix(r.URL.Path, "/v2/archie/manifests/") + if !ok { + http.NotFound(w, r) + return + } + if r.Method == http.MethodPut { + body, _ := io.ReadAll(r.Body) + m.manifests[ref], m.manifests[signedoci.Digest(body)] = body, body + w.WriteHeader(http.StatusCreated) + return + } + manifest, found := m.manifests[ref] + if !found { + http.NotFound(w, r) + return + } + w.Header().Set("Content-Type", ocispec.MediaTypeImageManifest) + w.Header().Set("Docker-Content-Digest", signedoci.Digest(manifest)) + w.Header().Set("Content-Length", strconv.Itoa(len(manifest))) + if r.Method != http.MethodHead { + _, _ = w.Write(manifest) + } +}