diff --git a/.github/workflows/deploy.yml b/.github/workflows/deploy.yml index 2c92a3cc4..dc7716316 100644 --- a/.github/workflows/deploy.yml +++ b/.github/workflows/deploy.yml @@ -373,45 +373,45 @@ jobs: # rather than fail the release on its absence. [ -f README.md ] && cp README.md "$name/" zip -r "$name.zip" "$name" - sha256sum "$name.zip" > SHA256SUMS echo "name=$name" >> "$GITHUB_OUTPUT" - name: Upload distribution zip uses: actions/upload-artifact@v4 with: name: ${{ steps.zip.outputs.name }} - path: | - ${{ steps.zip.outputs.name }}.zip - SHA256SUMS + path: ${{ steps.zip.outputs.name }}.zip retention-days: 90 - # A release is what an operator can actually download from; the - # artifact above is reachable only from the run's page, behind a - # login, and expires after 90 days. Until this step existed the - # zip was produced and then stranded. - # - # The release lives on this repository, attached to the release's v* - # tag, or the archied tag for a release cut before the single release - # stream. The body is the changelog section -- the release's own - # reviewed prose -- not generated notes: the tag/changelog gate above - # already guarantees it describes this version. - - name: Publish Gitea release with the distribution zip + # Installs and updates take a release from the signed artifact in the + # registry, so this is what publishes it: the statement signed with + # RELEASE_SIGNING_KEY and the zip, tagged v, next, and stable + # unless it is a prerelease. + - name: Log in to the Gitea registry + run: echo "${{ secrets.DEPLOY_TOKEN }}" | docker login git.catlow.cloud -u "${{ github.actor }}" --password-stdin + + - name: Publish the signed release + env: + ARCHIE_RELEASE_SEED: ${{ secrets.RELEASE_SIGNING_KEY }} + run: | + set -e + version="${{ steps.version.outputs.version }}" + prerelease=false + case "$version" in *-*) prerelease=true ;; esac + ./dist/archied release publish -source git.catlow.cloud/sam/archie \ + -version "$version" -prerelease="$prerelease" \ + -zip "${{ steps.zip.outputs.name }}.zip" \ + -agent-image "git.catlow.cloud/sam/archie-agent@${{ needs.agent-image.outputs.agent_digest }}" \ + -changelog "https://git.catlow.cloud/sam/archie/releases/tag/v$version" + + # The release page carries this version's changelog section, the + # release's own reviewed prose; the signed statement links to it. + - name: Publish the Gitea release notes env: - # One PAT covers this step and the image jobs' registry login. It - # needs write:packages and write:repository scopes. DEPLOY_TOKEN: ${{ secrets.DEPLOY_TOKEN }} run: | set -e tag="v${{ steps.version.outputs.version }}" - if ! git ls-remote --exit-code --tags origin "refs/tags/$tag" >/dev/null; then - tag="archied/v${{ steps.version.outputs.version }}" - fi version="${{ steps.version.outputs.version }}" - # The release body is this version's changelog section, not the whole - # file: from its "## [x.y.z]" heading to the next "## [" heading. - # Extracting rather than inlining keeps the changelog the single - # source of truth, and the tag/changelog gate above has already - # proven that section exists. notes="$(mktemp)" awk -v hdr="## [$version]" ' index($0, hdr) == 1 { printing = 1 } @@ -425,19 +425,10 @@ jobs: api="https://git.catlow.cloud/api/v1" repo="sam/archie" auth="Authorization: token $DEPLOY_TOKEN" - # Idempotent: drop any release already on the tag and recreate it, - # so a re-run of this job (a retried or superseded deploy) cannot - # fail on an existing release or duplicate the zip. Deleting a - # release leaves the tag itself alone. + # Idempotent: a re-run replaces the release on the tag. curl -sf -X DELETE -H "$auth" "$api/repos/$repo/releases/tags/$tag" >/dev/null || true - id="$(curl -sf -X POST -H "$auth" -H "Content-Type: application/json" \ - -d "$(jq -n --arg tag "$tag" --arg name "archied v$version" --arg body "$(cat "$notes")" \ + curl -sf -X POST -H "$auth" -H "Content-Type: application/json" \ + -d "$(jq -n --arg tag "$tag" --arg name "Archie v$version" --arg body "$(cat "$notes")" \ '{tag_name: $tag, name: $name, body: $body, draft: false, prerelease: false}')" \ - "$api/repos/$repo/releases" | jq -r '.id')" - test -n "$id" - for asset in "${{ steps.zip.outputs.name }}.zip" SHA256SUMS install.sh; do - curl -sf -X POST -H "$auth" -H "Content-Type: application/octet-stream" \ - --data-binary "@$asset" \ - "$api/repos/$repo/releases/$id/assets?name=$(basename "$asset")" >/dev/null - done - echo "published $tag with ${{ steps.zip.outputs.name }}.zip" + "$api/repos/$repo/releases" >/dev/null + echo "published release notes for $tag" diff --git a/CHANGELOG.md b/CHANGELOG.md index 3c4de0292..34f44b812 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,8 @@ release's per-component sections are labelled beneath its heading. ### archied +- Releases are signed. A release is an OCI artifact whose statement is signed with the release key; updates and `install.sh` refuse a release the key did not sign, a zip that is not the one signed, and a channel moved back to an older release. Releases now come from `git.catlow.cloud/sam/archie`, so a host on 1.55.0 or earlier finds this release only by running the new installer once: `curl -fsSL https://offloaded.dev/install.sh | bash`. +- Where releases and the extension catalogue come from is a setting: an OCI repository and the key it signs with, in the update settings and the extension settings, or `[distribution]` in `config.toml` for a new instance. `archied release keygen` and `archied release publish` make a key and publish signed releases to your own registry. - Chat saves the workflow draft it writes with `workflow_draft` instead of handing over YAML to paste: the draft lands in the workflow editor for review, attributed to the signed-in account. ## [1.55.0] - 2026-10-16 diff --git a/README.md b/README.md index f7675341a..c076ad41e 100644 --- a/README.md +++ b/README.md @@ -101,7 +101,7 @@ or a hosted provider API key. 1. **Install.** ```bash - curl -fsSL https://raw.githubusercontent.com/samcharles93/archie-core/main/install.sh | bash + curl -fsSL https://offloaded.dev/install.sh | bash ``` It downloads the latest release, verifies its checksum, installs the five diff --git a/RELEASING.md b/RELEASING.md index 4bd723c3e..e06b5a3e8 100644 --- a/RELEASING.md +++ b/RELEASING.md @@ -58,14 +58,14 @@ task release:prepare VERSION=1.3.0 # write the section into CHANGELOG.md, un # edit CHANGELOG.md -- generated notes are a starting point, not the release task release VERSION=1.3.0 # commit + annotated tag git push --atomic origin main v1.3.0 # both refs or neither -task release:publish VERSION=1.3.0 # images to GHCR, zip to archie-releases ``` -The source repository is private, so releases are published to the public -`samcharles93/archie-releases` repository, where `install.sh` and the -updater find them. `release:publish` builds from a clean worktree of the tag -and needs `gh` and `docker` logged in with write access to that repository -and to GHCR. +Pushing the tag is the release. The deploy workflow builds the images and the +zip, then `archied release publish` signs the release with the +`RELEASE_SIGNING_KEY` Actions secret and pushes it to +`git.catlow.cloud/sam/archie` as `v1.3.0`, `next`, and `stable` unless the +version is a prerelease. `install.sh` and the updater install only what that +key signed. `release:prepare` emits bare commit subjects; they are a starting point. Rewrite them to say what a reader gets, per "What the notes are for" above. @@ -106,7 +106,7 @@ Tagging one version produces, from a single `deploy` run: rebuilt by the release, so runtime commits merged to `main` do not reach a running sandbox until the host pulls the new image. The hand-run `tools/publish.sh` publishes the same two images to - `ghcr.io/samcharles93` instead, which is the registry the binary's default + `git.catlow.cloud/sam` instead, which is the registry the binary's default `[containers].image` names. - **`:latest` names the newest release, not the newest commit.** A push to `main` that is not a release publishes `archied:edge` (mutable, tracking diff --git a/Taskfile.yml b/Taskfile.yml index f29f80554..e0483a71a 100644 --- a/Taskfile.yml +++ b/Taskfile.yml @@ -319,10 +319,3 @@ tasks: - tools/release.sh --version {{.VERSION}} --tag requires: vars: [VERSION] - - release:publish: - desc: Push the images and publish the pushed tag to the public releases repository (VERSION=1.2.0) - cmds: - - tools/publish.sh {{.VERSION}} - requires: - vars: [VERSION] diff --git a/cmd/archied/main.go b/cmd/archied/main.go index 66f66cb0e..06f0a9ed4 100644 --- a/cmd/archied/main.go +++ b/cmd/archied/main.go @@ -12,7 +12,7 @@ import ( ) const commandUsage = "usage: archied [flags] # run the daemon; archied -h lists the flags\n" + - "subcommands: setup, update, status, prbench, playbooks" + "subcommands: setup, update, release, status, prbench, playbooks" func main() { os.Exit(run(os.Args[1:])) @@ -29,6 +29,8 @@ func run(args []string) int { switch args[0] { case "update": return archied.RunUpdate(args[1:], os.Stdout, os.Stderr) + case "release": + return archied.RunRelease(args[1:], os.Stdout, os.Stderr) case "status": return archied.RunStatus(args[1:], os.Stdout, os.Stderr) case "prbench": diff --git a/config.example.toml b/config.example.toml index 0b1499065..3b15e178e 100644 --- a/config.example.toml +++ b/config.example.toml @@ -85,3 +85,12 @@ target = "127.0.0.1:9090" # Store's sweep deletes it. Zero means 90 days. # [access] # denial_retention = "2160h" + +# Where releases and the extension catalogue come from on a new instance: an +# OCI repository and the base64 ed25519 key it signs with. Blank is the +# official source. Once stored, change them in the dashboard settings. +# [distribution] +# releases = "registry.example/owner/archie" +# releases_key = "base64 public key" +# catalogue = "registry.example/owner/archipelago/catalogue" +# catalogue_key = "base64 public key" diff --git a/deployments/INSTRUCTIONS.md b/deployments/INSTRUCTIONS.md index e511966fc..ce1b4fc85 100644 --- a/deployments/INSTRUCTIONS.md +++ b/deployments/INSTRUCTIONS.md @@ -23,7 +23,7 @@ of its own (`deployments/systemd-user-service.md`). ships beside `archied` because a Kit run mounts it from that directory. The agent image is a separate thing, pulled for the containers `archied` spawns per task: `[containers].image` defaults to -`ghcr.io/samcharles93/archie-agent:latest`, while this archive's `release.json` +`git.catlow.cloud/sam/archie-agent:latest`, while this archive's `release.json` pins the digest the release published. Set `[containers].registry_auth` when that reference is on a registry that needs a login -- see `deployments/systemd-user-service.md#6-pinning-the-archie-agent-image`. @@ -32,7 +32,7 @@ that reference is on a registry that needs a login -- see 1. Pick a starting configuration. The full set of annotated templates lives in this repository's `deployments/` directory on GitHub - () — + () — `standalone.toml` (no forge, the simplest start), `single-forge-github.toml` and `docker-nats-stack.toml` cover the common cases. Every one needs PostgreSQL 18 at `database_url`. Copy whichever fits to: @@ -65,7 +65,7 @@ that reference is on a registry that needs a login -- see 4. For a persistent, 24/7 deployment instead of a foreground run, follow the systemd user-unit runbook: - . + . ## Verifying an install @@ -89,10 +89,11 @@ an install (or an upgrade across v1.30.0) needs its unit as well -- see ### Native self-update -Install on Settings → Status runs `archied update install -version X`, which -downloads the release's Linux/amd64 zip and verifies it against `SHA256SUMS` -before unpacking it. Every host binary in the zip must report `X`. It then pulls -the agent image `release.json` pins by digest, backs up the database to +Install on Settings → Status runs `archied update install -version X` against +the release source in the update settings. It fetches release `vX`, refuses it +unless the source's key signed its statement, and downloads the Linux/amd64 zip +only if it hashes to the digest that statement names. Every host binary in the +zip must report `X`. It then pulls the agent image the statement pins by digest, backs up the database to `db.pre-update.dump` in the state directory, stages the new binaries beside the live ones and hands the transaction to `archied update watchdog` in a process of its own (a transient systemd user unit). Nothing that runs changes until the diff --git a/deployments/single-forge-github.toml b/deployments/single-forge-github.toml index 9bdf27649..95c112723 100644 --- a/deployments/single-forge-github.toml +++ b/deployments/single-forge-github.toml @@ -55,7 +55,7 @@ mode = "embedded" # Pinned to a verified digest -- archie-agent has no published semver tag # yet (a release publishes :latest and :sha-). See # deployments/systemd-user-service.md#6-pinning-the-archie-agent-image. -image = "ghcr.io/samcharles93/archie-agent@sha256:870f1b1357979cfea23fccacb24166ebeb9ab321708ef2c2b218a763ed99b090" +image = "git.catlow.cloud/sam/archie-agent@sha256:" pull_policy = "missing" [budgets] diff --git a/deployments/systemd-user-service.md b/deployments/systemd-user-service.md index 50300d1e4..ef17173b5 100644 --- a/deployments/systemd-user-service.md +++ b/deployments/systemd-user-service.md @@ -134,7 +134,7 @@ Compose service or host `archie-agent` binary is required. Docker must be installed and reachable by the service user. With no explicit broker or container settings, Archie defaults `containers.image` to -`ghcr.io/samcharles93/archie-agent:latest`. `single-forge-github.toml` and +`git.catlow.cloud/sam/archie-agent:latest`. `single-forge-github.toml` and `docker-nats-stack.toml` override that default with a pinned reference (`standalone.toml` and `dev.toml` leave it) -- see [Pinning the archie-agent image](#6-pinning-the-archie-agent-image) below. An @@ -145,7 +145,7 @@ explicit equivalent, pinned the same way, is: mode = "embedded" [containers] -image = "ghcr.io/samcharles93/archie-agent@sha256:870f1b1357979cfea23fccacb24166ebeb9ab321708ef2c2b218a763ed99b090" +image = "git.catlow.cloud/sam/archie-agent@sha256:" pull_policy = "missing" ``` @@ -177,7 +177,7 @@ prior build. Two paths publish it, to different references: as `:latest` and `:sha-` when the commit carries a release tag; an ordinary push to `main` publishes nothing for the agent image. - `tools/publish.sh`, run by hand for a release, publishes - `ghcr.io/samcharles93/archie-agent` as `:latest` and `:sha-`, which is + `git.catlow.cloud/sam/archie-agent` as `:latest` and `:sha-`, which is the registry the shipped default above names. A distribution zip pins whichever one its own release published, by digest, in @@ -192,19 +192,19 @@ explicitly: ```toml [containers] -image = "ghcr.io/samcharles93/archie-agent@sha256:870f1b1357979cfea23fccacb24166ebeb9ab321708ef2c2b218a763ed99b090" +image = "git.catlow.cloud/sam/archie-agent@sha256:" ``` This is the digest `latest` resolved to on 2026-09-05. Digest pinning is the strongest option for production: it is immutable and independently -verifiable (`docker manifest inspect ghcr.io/samcharles93/archie-agent@sha256:...`), +verifiable (`docker manifest inspect git.catlow.cloud/sam/archie-agent@sha256:...`), at the cost of being opaque in a config file a human is meant to edit -- resolve a fresh digest deliberately (`docker pull ...:latest` followed by `docker inspect --format '{{index .RepoDigests 0}}'`) rather than editing this one in place expecting it to track new releases. If/when CI starts publishing a semantic-version tag for `archie-agent` -(e.g. `ghcr.io/samcharles93/archie-agent:v1.21.0`), prefer that tag for +(e.g. `git.catlow.cloud/sam/archie-agent:v1.21.0`), prefer that tag for day-to-day config readability, and reserve digest pinning for deployments that need byte-for-byte reproducibility guarantees a tag alone cannot give (tags are mutable pointers; a compromised or accidentally-overwritten tag diff --git a/docker-compose.yml b/docker-compose.yml index 6bbd159d4..de4026b40 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -95,7 +95,7 @@ services: # name by digest. The digest a task container actually runs is pinned in # the daemon's own [containers] image, so nothing here decides it. See # deployments/systemd-user-service.md#6-pinning-the-archie-agent-image. - image: ghcr.io/samcharles93/archie-agent:latest + image: git.catlow.cloud/sam/archie-agent:latest # ── example agent profile image ───────────────────────────────────── # The agent image plus network lookup tools, for a workflow that names diff --git a/examples/profiles/Dockerfile.agent-net b/examples/profiles/Dockerfile.agent-net index fb3a5df57..3ffd3c5e3 100644 --- a/examples/profiles/Dockerfile.agent-net +++ b/examples/profiles/Dockerfile.agent-net @@ -3,7 +3,7 @@ # the network-investigator profile package (archipelago: profiles/network-investigator). # # Build: docker compose build agent-net -ARG AGENT_IMAGE=ghcr.io/samcharles93/archie-agent:latest +ARG AGENT_IMAGE=git.catlow.cloud/sam/archie-agent:latest FROM ${AGENT_IMAGE} RUN apt-get update && apt-get install -y --no-install-recommends \ diff --git a/install.sh b/install.sh index 587da8d38..d92847496 100755 --- a/install.sh +++ b/install.sh @@ -1,16 +1,25 @@ #!/usr/bin/env bash # install.sh - installs Archie Core for the current user. # -# curl -fsSL https://raw.githubusercontent.com/samcharles93/archie-core/main/install.sh | bash +# curl -fsSL https://offloaded.dev/install.sh | bash # -# Downloads the latest release, verifies it against SHA256SUMS, installs the +# Downloads a release from its OCI registry, verifies the release key signed +# it and that the zip is the one signed, installs the # five services to ${XDG_BIN_HOME:-~/.local/bin}, runs `archied setup`, starts # the bundled PostgreSQL and installs systemd user units. # Config: ${XDG_CONFIG_HOME:-~/.config}/archie Data: ${XDG_DATA_HOME:-~/.local/share}/archie set -euo pipefail -REPO="samcharles93/archie-releases" +MODULE="github.com/samcharles93/archie-core" +SOURCE_GIT="https://git.catlow.cloud/sam/archie.git" +# Where releases come from: an OCI repository and the base64 ed25519 key that +# must have signed them. A self-hosted source sets both. +DEFAULT_RELEASES="git.catlow.cloud/sam/archie" +DEFAULT_RELEASE_KEY="gIBqoQMwy+8WXmGfiuHtnUa50W0IqfzkTR6iF5q3dSM=" +ARCHIE_RELEASES="${ARCHIE_RELEASES:-${DEFAULT_RELEASES}}" +ARCHIE_RELEASE_KEY="${ARCHIE_RELEASE_KEY:-${DEFAULT_RELEASE_KEY}}" +ARCHIE_CHANNEL="${ARCHIE_CHANNEL:-stable}" XDG_CONFIG_HOME="${XDG_CONFIG_HOME:-${HOME}/.config}" XDG_DATA_HOME="${XDG_DATA_HOME:-${HOME}/.local/share}" @@ -32,7 +41,12 @@ if (: /dev/null; then INTERACTIVE=true; fi usage() { cat </dev/null | tr -d '\r' | sed -n 's/^[Ww][Ww][Ww]-[Aa]uthenticate: *Bearer *//p')" || true + if [ -n "${challenge}" ]; then + local realm service token + realm="$(sed -n 's/.*realm="\([^"]*\)".*/\1/p' <<<"${challenge}")" + service="$(sed -n 's/.*service="\([^"]*\)".*/\1/p' <<<"${challenge}")" + token="$(curl -fsSL -G "${realm}" --data-urlencode "service=${service}" --data-urlencode "scope=repository:${repo}:pull" | jq -r '.token // .access_token')" + auth=(-H "Authorization: Bearer ${token}") + fi + echo "==> Fetching ${ARCHIE_RELEASES}:${tag}..." + curl -fsSL "${auth[@]}" -H "Accept: application/vnd.oci.image.manifest.v1+json" "${base}/manifests/${tag}" -o "${TMP}/manifest.json" + blob() { curl -fsSL "${auth[@]}" "${base}/blobs/$1" -o "$2" && [ "sha256:$(sha256sum "$2" | cut -d' ' -f1)" = "$1" ]; } + local statement_digest + statement_digest="$(jq -er '.layers[0].digest' "${TMP}/manifest.json")" + blob "${statement_digest}" "${TMP}/statement.json" || { echo "Error: the release statement does not match its digest." >&2; exit 1; } + jq -er '.annotations["dev.archie.release.signature.v1"]' "${TMP}/manifest.json" | base64 -d >"${TMP}/statement.sig" + # An ed25519 public key is its 32 bytes behind a fixed SubjectPublicKeyInfo header. + { printf '\x30\x2a\x30\x05\x06\x03\x2b\x65\x70\x03\x21\x00'; base64 -d <<<"${ARCHIE_RELEASE_KEY}"; } >"${TMP}/key.der" + openssl pkeyutl -verify -pubin -inkey "${TMP}/key.der" -keyform DER -rawin \ + -in "${TMP}/statement.json" -sigfile "${TMP}/statement.sig" >/dev/null 2>&1 || + { echo "Error: ${ARCHIE_RELEASES}:${tag} is not signed by the release key." >&2; exit 1; } + local zip_name zip_digest + VERSION="$(jq -er .version "${TMP}/statement.json")" + [ "$(jq -r .platform "${TMP}/statement.json")" = linux-amd64 ] || { echo "Error: the release is not built for linux-amd64." >&2; exit 1; } + [ -z "${VERSION}" ] || [ "${tag}" = "${ARCHIE_CHANNEL}" ] || [ "v${VERSION}" = "${tag}" ] || + { echo "Error: ${tag} carries the statement for ${VERSION}." >&2; exit 1; } + zip_name="$(jq -er .zip.name "${TMP}/statement.json")" + zip_digest="$(jq -er .zip.digest "${TMP}/statement.json")" + RELEASE_NAME="${zip_name%.zip}" + case "${RELEASE_NAME}" in archie-core-archied-v*-linux-amd64) ;; *) echo "Error: unexpected release zip ${zip_name}." >&2; exit 1 ;; esac + echo "==> Downloading Archie ${VERSION}..." + blob "${zip_digest}" "${TMP}/${zip_name}" || { echo "Error: ${zip_name} does not match the signed digest." >&2; exit 1; } + unzip -q "${TMP}/${zip_name}" -d "${TMP}" +} + # 1. Prerequisites echo "==> Checking prerequisites..." # The installer depends on systemd user units and GNU tools. @@ -114,7 +172,7 @@ if [ "${FROM_SOURCE}" = true ]; then need git go jq else [ "$(uname -m)" = x86_64 ] || { echo "Error: releases are built for x86_64 only; use --from-source." >&2; exit 1; } - need curl unzip sha256sum + need curl unzip sha256sum openssl jq base64 fi if command -v docker &>/dev/null; then echo " [OK] $(docker --version)" @@ -131,11 +189,11 @@ trap 'rm -rf "${TMP}"' EXIT if [ "${FROM_SOURCE}" = true ]; then SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]:-$0}")" && pwd)" - if [ -f "${SCRIPT_DIR}/go.mod" ] && grep -q "github.com/${REPO}" "${SCRIPT_DIR}/go.mod"; then + if [ -f "${SCRIPT_DIR}/go.mod" ] && grep -q "${MODULE}" "${SCRIPT_DIR}/go.mod"; then SRC_DIR="${SCRIPT_DIR}" else SRC_DIR="${TMP}/src" - git clone --quiet --depth=1 "https://github.com/${REPO}.git" "${SRC_DIR}" + git clone --quiet --depth=1 "${SOURCE_GIT}" "${SRC_DIR}" fi echo "==> Building from ${SRC_DIR}..." # Selects (and under the default GOTOOLCHAIN=auto, downloads) the toolchain @@ -150,30 +208,14 @@ if [ "${FROM_SOURCE}" = true ]; then mkdir -p "${STAGE}" # A source build is not a release: it says `dev`, and buildType=binary lets # it self-update. The agentworker stamp only lands in archie-agent. - LDFLAGS="-X github.com/${REPO}/internal/buildinfo.Version=dev -X github.com/${REPO}/internal/buildinfo.Runtime=dev -X github.com/${REPO}/internal/installtype.buildType=binary -X github.com/${REPO}/internal/app/agentworker.version=dev" + LDFLAGS="-X ${MODULE}/internal/buildinfo.Version=dev -X ${MODULE}/internal/buildinfo.Runtime=dev -X ${MODULE}/internal/installtype.buildType=binary -X ${MODULE}/internal/app/agentworker.version=dev" for cmd in $(jq -er '[.required_topology.units[], .required_topology.binaries[]?] | join(" ")' "${SRC_DIR}/release.json"); do (cd "${SRC_DIR}" && go build -ldflags "${LDFLAGS}" -o "${STAGE}/${cmd}" "./cmd/${cmd}") done cp "${SRC_DIR}/release.json" "${STAGE}/" else - if [ -z "${VERSION}" ]; then - tag="$(curl -fsSL "https://api.github.com/repos/${REPO}/releases/latest" | sed -n 's/.*"tag_name": *"\([^"]*\)".*/\1/p')" - [ -n "${tag}" ] || { echo "Error: could not find the latest release." >&2; exit 1; } - else - tag="v${VERSION}" - fi - VERSION="${tag##*v}" - name="archie-core-archied-v${VERSION}-linux-amd64" - echo "==> Downloading Archie ${VERSION}..." - base="https://github.com/${REPO}/releases/download/${tag}" - curl -fsSL "${base}/${name}.zip" -o "${TMP}/${name}.zip" - curl -fsSL "${base}/SHA256SUMS" -o "${TMP}/SHA256SUMS" - expected="$(awk -v n="${name}.zip" '$2 == n {print $1}' "${TMP}/SHA256SUMS")" - [[ "${expected}" =~ ^[a-f0-9]{64}$ ]] || { echo "Error: ${name}.zip is not in SHA256SUMS." >&2; exit 1; } - printf '%s %s\n' "${expected}" "${TMP}/${name}.zip" | sha256sum --check --status || - { echo "Error: checksum mismatch for ${name}.zip." >&2; exit 1; } - unzip -q "${TMP}/${name}.zip" -d "${TMP}" - STAGE="${TMP}/${name}" + install_release + STAGE="${TMP}/${RELEASE_NAME}" fi TOPOLOGY_UNITS="$(tr -d '\n' <"${STAGE}/release.json" | sed -n 's/.*"units": *\[\([^]]*\)\].*/\1/p' | tr -d '" ' | tr , ' ')" @@ -202,6 +244,10 @@ if [ ! -f "${ARCHIE_CONFIG_DIR}/config.toml" ]; then echo "ERROR: archied setup could not generate ${ARCHIE_CONFIG_DIR}/config.toml" >&2 exit 1 fi + if [ "${ARCHIE_RELEASES}" != "${DEFAULT_RELEASES}" ] || [ "${ARCHIE_RELEASE_KEY}" != "${DEFAULT_RELEASE_KEY}" ]; then + # The instance keeps updating from the source it was installed from. + printf '\n[distribution]\nreleases = "%s"\nreleases_key = "%s"\n' "${ARCHIE_RELEASES}" "${ARCHIE_RELEASE_KEY}" >>"${ARCHIE_CONFIG_DIR}/config.toml" + fi if [ -n "${PORT_BASE}" ]; then # The generated config names the default addresses; move them to this # instance's ports and pin the services the template leaves commented. @@ -465,7 +511,7 @@ if [ "${SERVICE_INSTALLED}" = true ] && [ "${AUTO_START}" = true ]; then echo " - Service status : systemctl --user status ${INSTANCE_UNITS}" echo " - Restart daemon : systemctl --user restart ${U_DAEMON}" else - echo "Manual Startup: start the five services as in https://github.com/${REPO}/blob/main/deployments/README.md" + echo "Manual Startup: start the five services as in https://offloaded.dev" fi echo "" SETUP_HINT="archied setup" diff --git a/internal/app/archied/release.go b/internal/app/archied/release.go new file mode 100644 index 000000000..5f28ea064 --- /dev/null +++ b/internal/app/archied/release.go @@ -0,0 +1,112 @@ +package archied + +import ( + "context" + "crypto/ed25519" + "encoding/base64" + "flag" + "fmt" + "io" + "os" + "strings" + + "github.com/samcharles93/archie-core/internal/domain/distribution" + "github.com/samcharles93/archie-core/internal/releaseupdate" +) + +const releaseUsage = `usage: archied release + + keygen -seed FILE write a new signing key's seed to FILE and print its public key + publish sign a release and push it to a release source` + +// RunRelease is the publisher's side of a release source: it makes the key +// a source signs with and pushes signed releases to it. +func RunRelease(args []string, out, stderr io.Writer) int { + if len(args) == 0 { + fmt.Fprintln(stderr, releaseUsage) + return 2 + } + switch args[0] { + case "keygen": + return runReleaseKeygen(args[1:], out, stderr) + case "publish": + return runReleasePublish(args[1:], out, stderr) + default: + fmt.Fprintln(stderr, releaseUsage) + return 2 + } +} + +// runReleaseKeygen writes the seed to a new file only, never to output, so +// the private half is not left in a terminal or CI log. +func runReleaseKeygen(args []string, out, stderr io.Writer) int { + flags := flag.NewFlagSet("release keygen", flag.ContinueOnError) + flags.SetOutput(stderr) + seedPath := flags.String("seed", "", "file to create with the base64 private key seed") + if err := flags.Parse(args); err != nil { + return 2 + } + if *seedPath == "" { + fmt.Fprintln(stderr, "release keygen: -seed is required") + return 2 + } + public, private, err := ed25519.GenerateKey(nil) + if err != nil { + fmt.Fprintln(stderr, err) + return 1 + } + file, err := os.OpenFile(*seedPath, os.O_CREATE|os.O_EXCL|os.O_WRONLY, 0o600) + if err != nil { + fmt.Fprintln(stderr, err) + return 1 + } + _, err = fmt.Fprintln(file, base64.StdEncoding.EncodeToString(private.Seed())) + if closeErr := file.Close(); err == nil { + err = closeErr + } + if err != nil { + fmt.Fprintln(stderr, err) + return 1 + } + fmt.Fprintln(out, distribution.EncodeKey(public)) + return 0 +} + +func runReleasePublish(args []string, out, stderr io.Writer) int { + flags := flag.NewFlagSet("release publish", flag.ContinueOnError) + flags.SetOutput(stderr) + var req releaseupdate.PublishRequest + flags.StringVar(&req.Repository, "source", "", "OCI repository to publish to") + seedEnv := flags.String("seed-env", "ARCHIE_RELEASE_SEED", "environment variable holding the base64 private key seed") + flags.StringVar(&req.Version, "version", "", "release version") + flags.BoolVar(&req.Prerelease, "prerelease", false, "publish to next only") + flags.StringVar(&req.ZipPath, "zip", "", "distribution zip") + flags.StringVar(&req.AgentImage, "agent-image", "", "agent image pinned by digest") + flags.StringVar(&req.Changelog, "changelog", "", "URL of the release notes") + if err := flags.Parse(args); err != nil { + return 2 + } + if req.Repository == "" || req.Version == "" || req.ZipPath == "" || req.AgentImage == "" { + fmt.Fprintln(stderr, "release publish: -source, -version, -zip and -agent-image are required") + return 2 + } + seed, err := base64.StdEncoding.DecodeString(strings.TrimSpace(os.Getenv(*seedEnv))) + if err != nil || len(seed) != ed25519.SeedSize { + fmt.Fprintf(stderr, "release publish: %s must hold a base64 ed25519 seed\n", *seedEnv) + return 2 + } + req.Signer = ed25519.NewKeyFromSeed(seed) + public := distribution.EncodeKey(ed25519.PublicKey(req.Signer[ed25519.SeedSize:])) + req.Channels = []string{"next"} + if !req.Prerelease { + req.Channels = append(req.Channels, "stable") + } + statement, err := releaseupdate.Publish(context.Background(), req) + if err != nil { + fmt.Fprintln(stderr, err) + return 1 + } + fmt.Fprintf(out, "published %s to %s (%s) with key %s\n", statement.Version, req.Repository, + strings.Join(req.Channels, ", "), public) + return 0 +} diff --git a/internal/app/archied/update.go b/internal/app/archied/update.go index 66b920f14..9b1aece8a 100644 --- a/internal/app/archied/update.go +++ b/internal/app/archied/update.go @@ -20,6 +20,7 @@ import ( "golang.org/x/sys/unix" "github.com/samcharles93/archie-core/internal/app/servicekit" + "github.com/samcharles93/archie-core/internal/domain/distribution" "github.com/samcharles93/archie-core/internal/infrastructure/configuration" "github.com/samcharles93/archie-core/internal/releaseupdate" ) @@ -95,7 +96,7 @@ func runUpdateInstall(args []string, out, stderr io.Writer) int { flags.SetOutput(stderr) configPath := flags.String("config", configuration.DefaultConfigPath(), "bootstrap config file") version := flags.String("version", "", "release version to install") - releaseURL := flags.String("release-url", releaseupdate.ReleaseURL, "base URL of the release downloads") + releases := sourceFlags(flags) var meta releaseupdate.InstallMeta flags.StringVar(&meta.ReportPath, "report", "", "where the watchdog leaves the outcome") flags.StringVar(&meta.Channel, "channel", "", "channel the outcome is reported on") @@ -104,6 +105,11 @@ func runUpdateInstall(args []string, out, stderr io.Writer) int { if err := flags.Parse(args); err != nil { return 2 } + source, err := releases() + if err != nil { + fmt.Fprintln(stderr, err) + return 2 + } self, err := os.Executable() if err != nil { fmt.Fprintln(stderr, err) @@ -116,7 +122,7 @@ func runUpdateInstall(args []string, out, stderr io.Writer) int { return 1 } result, err := tx.Prepare(ctx, releaseupdate.PrepareRequest{ - Version: *version, ReleaseURL: *releaseURL, Meta: meta, + Version: *version, Source: source, Meta: meta, CheckTopology: func(ctx context.Context, required releaseupdate.Topology, managed bool) error { return checkUpdateTopology(ctx, *configPath, required, managed, out) }, @@ -380,3 +386,13 @@ func observedUnits(output []byte, err error) map[string]bool { } return units } + +// sourceFlags registers -source and -key, the release distribution source; a +// blank pair is the official releases. +func sourceFlags(flags *flag.FlagSet) func() (distribution.Source, error) { + reference := flags.String("source", "", "OCI repository releases are published to (default: the official releases)") + key := flags.String("key", "", "base64 ed25519 public key the source signs releases with") + return func() (distribution.Source, error) { + return distribution.Parse(*reference, *key, distribution.DefaultReleases()) + } +} diff --git a/internal/app/archied/update_check.go b/internal/app/archied/update_check.go index f89ff2b7b..0e290c5ca 100644 --- a/internal/app/archied/update_check.go +++ b/internal/app/archied/update_check.go @@ -8,6 +8,7 @@ import ( "io" "os" + "github.com/samcharles93/archie-core/internal/domain/distribution" "github.com/samcharles93/archie-core/internal/releaseupdate" ) @@ -18,10 +19,18 @@ func runUpdateCheck(args []string, out, stderr io.Writer) int { flags.SetOutput(stderr) channel := flags.String("channel", envOr("ARCHIE_UPDATE_RELEASE_CHANNEL", "stable"), "stable, next, or exact-pin") pin := flags.String("pin", os.Getenv("ARCHIE_UPDATE_PIN"), "exact release version") + releases := sourceFlags(flags) if err := flags.Parse(args); err != nil { return 2 } - catalog := &releaseupdate.GitHubCatalog{Channel: func(context.Context) (string, string, error) { return *channel, *pin, nil }} + source, err := releases() + if err != nil { + fmt.Fprintln(stderr, err) + return 2 + } + catalog := &releaseupdate.RegistryCatalog{Settings: func(context.Context) (distribution.Source, string, string, error) { + return source, *channel, *pin, nil + }} snapshot, err := catalog.Check(context.Background()) if err != nil { fmt.Fprintln(stderr, err) diff --git a/internal/app/statestore/catalogue.go b/internal/app/statestore/catalogue.go new file mode 100644 index 000000000..600a97939 --- /dev/null +++ b/internal/app/statestore/catalogue.go @@ -0,0 +1,51 @@ +package statestore + +import ( + "context" + "encoding/json" + "errors" + "fmt" + + "github.com/samcharles93/archie-core/internal/domain/distribution" + "github.com/samcharles93/archie-core/internal/domain/storecontract" + domain "github.com/samcharles93/archie-core/internal/domain/storepkg" + "github.com/samcharles93/archie-core/internal/infrastructure/controlplanerpc" + "github.com/samcharles93/archie-core/internal/infrastructure/controlplanestore" + registry "github.com/samcharles93/archie-core/internal/infrastructure/storepkg" +) + +// settingsCatalogue fetches the catalogue from the source the extension +// settings name, read on every fetch so an edit applies without a restart. +type settingsCatalogue struct { + resources auditRetentionResources +} + +// liveCatalogue is the catalogue the dashboard lists; a store that cannot +// read resources lists the official catalogue. +func (b *server) liveCatalogue() domain.CatalogueSource { + resources, ok := b.st.(auditRetentionResources) + if !ok { + return registry.Catalogue(distribution.DefaultCatalogue()) + } + return settingsCatalogue{resources: resources} +} + +func (c settingsCatalogue) Fetch(ctx context.Context) (domain.Catalogue, error) { + var settings controlplanerpc.ExtensionSettings + resource, err := c.resources.Resource(ctx, storecontract.DefaultOrgID, controlplanestore.ExtensionSettingsKind) + switch { + case errors.Is(err, storecontract.ErrResourceNotFound): + case err != nil: + return domain.Catalogue{}, fmt.Errorf("read extension settings: %w", err) + } + if len(resource.Value) > 0 { + if err := json.Unmarshal(resource.Value, &settings); err != nil { + return domain.Catalogue{}, fmt.Errorf("decode extension settings: %w", err) + } + } + source, err := settings.Catalogue() + if err != nil { + return domain.Catalogue{}, err + } + return registry.Catalogue(source).Fetch(ctx) +} diff --git a/internal/app/statestore/server.go b/internal/app/statestore/server.go index ed58aa3f0..d1a0818c7 100644 --- a/internal/app/statestore/server.go +++ b/internal/app/statestore/server.go @@ -269,7 +269,7 @@ func (b *server) stateStoreDeps(grants *staterpc.TaskGrants) staterpc.Deps { packages := storepkg.Service{ Registry: registry.OCIRegistry{}, Store: postgres.NewInstalledPackages(b.pg), - Catalogue: registry.DefaultCatalogue(), + Catalogue: b.liveCatalogue(), } // A projection failure degrades the org resources a package would have // contributed to, the way a group above degrades: the State Store boots diff --git a/internal/config/config.go b/internal/config/config.go index 46e9cf809..f2d4763b4 100644 --- a/internal/config/config.go +++ b/internal/config/config.go @@ -414,8 +414,11 @@ type Config struct { Containers ContainerConfig `toml:"containers" yaml:"containers"` Chat ChatConfig `toml:"chat" yaml:"chat"` Capture CaptureConfig `toml:"capture" yaml:"capture"` - Access AccessConfig `toml:"access" yaml:"access"` - Bindings BindingsConfig `toml:"bindings" yaml:"bindings"` + // Distribution seeds where releases and the extension catalogue come + // from; once stored, the dashboard's settings own them. + Distribution Distribution `toml:"distribution" yaml:"distribution"` + Access AccessConfig `toml:"access" yaml:"access"` + Bindings BindingsConfig `toml:"bindings" yaml:"bindings"` // Memory holds memory provider configuration (from config.memory.yaml). Memory MemoryConfig `toml:"memory" yaml:"memory"` @@ -690,7 +693,7 @@ type BindingsConfig struct { // ContainerConfig configures Docker sandbox execution of archie-agent. type ContainerConfig struct { - // Image is the Docker image to run (e.g. "ghcr.io/sam/archie-agent:latest"). + // Image is the Docker image to run (e.g. "git.catlow.cloud/sam/archie-agent:latest"). Image string `toml:"image" yaml:"image"` // MaxConcurrency limits simultaneous daemon tasks and containers. // Tasks from the same repository remain serialized. 0 = no limit. @@ -1008,6 +1011,16 @@ func (h Health) URL() string { return "http://" + net.JoinHostPort(host, port) } +// Distribution names the signed artifact sources a new instance starts with. +// Each source is an OCI repository with the base64 ed25519 key it signs +// with; blank pairs are the official sources. +type Distribution struct { + Releases string `toml:"releases" yaml:"releases"` + ReleasesKey string `toml:"releases_key" yaml:"releases_key"` + Catalogue string `toml:"catalogue" yaml:"catalogue"` + CatalogueKey string `toml:"catalogue_key" yaml:"catalogue_key"` +} + type Web struct { // Listen is the dashboard address; "off" disables the web UI. // Bind localhost (or a LAN/tailnet address) -- there is no auth. diff --git a/internal/domain/distribution/source.go b/internal/domain/distribution/source.go new file mode 100644 index 000000000..73bbb5a23 --- /dev/null +++ b/internal/domain/distribution/source.go @@ -0,0 +1,89 @@ +// Package distribution names where an instance takes signed artifacts from: +// its releases and its extension catalogue. +package distribution + +import ( + "crypto/ed25519" + "encoding/base64" + "errors" + "fmt" + "strings" +) + +// Source is an OCI repository and the public key that must have signed what +// it serves. The key, not the registry, is what makes an artifact trusted, so +// a source is never valid without one. +type Source struct { + Reference string + Key ed25519.PublicKey +} + +// Parse builds a source from its stored form. A blank reference and key mean +// fallback; setting only one of them is refused, since a reference without +// its key, or a key for the default reference, is a misconfiguration. +func Parse(reference, key string, fallback Source) (Source, error) { + reference, key = strings.TrimSpace(reference), strings.TrimSpace(key) + if reference == "" && key == "" { + return fallback, nil + } + if reference == "" || key == "" { + return Source{}, errors.New("a distribution source needs both a reference and a key") + } + if !repository(reference) { + return Source{}, fmt.Errorf("distribution source %q must be an OCI repository such as registry.example/owner/name", reference) + } + decoded, err := ParseKey(key) + if err != nil { + return Source{}, err + } + return Source{Reference: reference, Key: decoded}, nil +} + +// ParseKey decodes a base64 ed25519 public key. +func ParseKey(key string) (ed25519.PublicKey, error) { + decoded, err := base64.StdEncoding.DecodeString(strings.TrimSpace(key)) + if err != nil || len(decoded) != ed25519.PublicKeySize { + return nil, errors.New("distribution key must be a base64 ed25519 public key") + } + return decoded, nil +} + +// EncodeKey is the stored form of a public key. +func EncodeKey(key ed25519.PublicKey) string { + return base64.StdEncoding.EncodeToString(key) +} + +// repository accepts registry/path with an optional registry port and no tag +// or digest: a source names a repository, and its tags are the channels. +func repository(reference string) bool { + host, path, ok := strings.Cut(reference, "/") + return ok && host != "" && path != "" && + strings.Count(host, ":") <= 1 && !strings.ContainsAny(path, ":@") && + !strings.ContainsAny(reference, " \t\n") && !strings.Contains(reference, "//") +} + +// mustKey decodes a built-in key; a malformed one is a build defect. +func mustKey(encoded string) ed25519.PublicKey { + key, err := ParseKey(encoded) + if err != nil { + panic("distribution: invalid built-in key") + } + return key +} + +// DefaultCatalogue is the archipelago marketplace an instance trusts until an +// operator names another. +func DefaultCatalogue() Source { + return Source{ + Reference: "git.catlow.cloud/sam/archipelago/catalogue", + Key: mustKey("YL4nvn07uq/5mXZOt9cEmLYu/kB6C2+MLTnbHyoX4Pw="), + } +} + +// DefaultReleases is where Archie's own releases are published. +func DefaultReleases() Source { + return Source{ + Reference: "git.catlow.cloud/sam/archie", + Key: mustKey("gIBqoQMwy+8WXmGfiuHtnUa50W0IqfzkTR6iF5q3dSM="), + } +} diff --git a/internal/infrastructure/configuration/defaults.go b/internal/infrastructure/configuration/defaults.go index 889a07a7c..9e76ed84c 100644 --- a/internal/infrastructure/configuration/defaults.go +++ b/internal/infrastructure/configuration/defaults.go @@ -24,7 +24,7 @@ const ( defaultContainerUptime = 5 * time.Hour defaultVolumeTTL = 72 * time.Hour defaultPullPolicy = "missing" - defaultContainerImage = "ghcr.io/samcharles93/archie-agent:latest" + defaultContainerImage = "git.catlow.cloud/sam/archie-agent:latest" // defaultMaxResultChars bounds one tool result. defaultMaxResultChars = 50_000 diff --git a/internal/infrastructure/controlplanerpc/extensions.go b/internal/infrastructure/controlplanerpc/extensions.go index c4908e311..69f103c6d 100644 --- a/internal/infrastructure/controlplanerpc/extensions.go +++ b/internal/infrastructure/controlplanerpc/extensions.go @@ -1,12 +1,21 @@ package controlplanerpc +import "github.com/samcharles93/archie-core/internal/domain/distribution" + // ExtensionSettingsKind is the control-plane resource kind holding which // installed extensions are enabled and how each is configured. const ExtensionSettingsKind = "extension-settings" // ExtensionSettings is the extension-settings resource document. type ExtensionSettings struct { - Extensions []ExtensionSetting `json:"extensions"` + Extensions []ExtensionSetting `json:"extensions"` + CatalogueSource string `json:"catalogue_source,omitempty" title:"Catalogue source" doc:"OCI repository the package catalogue is published to, read at its latest tag. Blank uses the official catalogue."` + CatalogueKey string `json:"catalogue_key,omitempty" title:"Catalogue signing key" doc:"Base64 ed25519 public key the catalogue source signs with. Required with a source."` +} + +// Catalogue is the distribution source the settings name. +func (s ExtensionSettings) Catalogue() (distribution.Source, error) { + return distribution.Parse(s.CatalogueSource, s.CatalogueKey, distribution.DefaultCatalogue()) } // ExtensionSetting enables or disables one installed extension by package name. diff --git a/internal/infrastructure/controlplanerpc/vocabulary.go b/internal/infrastructure/controlplanerpc/vocabulary.go index 77d93b7c8..d4c1dc0c3 100644 --- a/internal/infrastructure/controlplanerpc/vocabulary.go +++ b/internal/infrastructure/controlplanerpc/vocabulary.go @@ -8,6 +8,7 @@ import ( "time" "github.com/samcharles93/archie-core/internal/config" + "github.com/samcharles93/archie-core/internal/domain/distribution" "github.com/samcharles93/archie-core/internal/installtype" "github.com/samcharles93/archie-core/internal/releaseupdate" ) @@ -118,28 +119,49 @@ const UpdateSettingsKind = "update-settings" type UpdateSettings struct { Channel string `json:"channel" title:"Release channel" doc:"stable, next (includes prereleases) or exact-pin."` Pin string `json:"pin" title:"Pinned version" doc:"The version exact-pin installs."` + Source string `json:"source,omitempty" title:"Release source" doc:"OCI repository releases are published to, such as registry.example/owner/archie. Blank uses the official releases."` + Key string `json:"key,omitempty" title:"Release signing key" doc:"Base64 ed25519 public key the release source signs with. Required with a source."` } -// UpdateService checks GitHub on the stored release channel and installs -// through the archied beside this binary. statePath keeps each recipient's -// deferrals. +// Releases is the distribution source the settings name. +func (s UpdateSettings) Releases() (distribution.Source, error) { + return distribution.Parse(s.Source, s.Key, distribution.DefaultReleases()) +} + +// UpdateService checks the stored release source on the stored channel and +// installs through the archied beside this binary from the same source. +// statePath keeps each recipient's deferrals. func UpdateService(reader ResourceReader, statePath string) *releaseupdate.Service { + settings := updateSettings(reader) return &releaseupdate.Service{ - Catalog: &releaseupdate.GitHubCatalog{Channel: updateChannel(reader)}, - Installer: releaseupdate.ArchiedInstaller{}, + Catalog: &releaseupdate.RegistryCatalog{Settings: func(ctx context.Context) (distribution.Source, string, string, error) { + doc, err := settings(ctx) + if err != nil { + return distribution.Source{}, "", "", err + } + source, err := doc.Releases() + return source, doc.Channel, doc.Pin, err + }}, + Installer: releaseupdate.ArchiedInstaller{Source: func(ctx context.Context) (distribution.Source, error) { + doc, err := settings(ctx) + if err != nil { + return distribution.Source{}, err + } + return doc.Releases() + }}, StatePath: statePath, InstallType: installtype.Type(), } } -// updateChannel reads the update-settings resource. An unstored kind is the -// stable channel. -func updateChannel(reader ResourceReader) func(context.Context) (string, string, error) { - return func(ctx context.Context) (string, string, error) { +// updateSettings reads the update-settings resource. An unstored kind is the +// stable channel from the official releases. +func updateSettings(reader ResourceReader) func(context.Context) (UpdateSettings, error) { + return func(ctx context.Context) (UpdateSettings, error) { settings := UpdateSettings{Channel: "stable"} _, _, err := reader.Query(ctx, UpdateSettingsKind, func(value []byte) error { return json.Unmarshal(value, &settings) }) - return settings.Channel, settings.Pin, err + return settings, err } } diff --git a/internal/infrastructure/controlplanestore/operational_settings.go b/internal/infrastructure/controlplanestore/operational_settings.go index b38f59546..fabfd431e 100644 --- a/internal/infrastructure/controlplanestore/operational_settings.go +++ b/internal/infrastructure/controlplanestore/operational_settings.go @@ -60,11 +60,15 @@ func operationalDefinitions() []Definition { }, Validate: validatePluginSettings}, // Live: each process that supervises extensions re-reads this on its sync // tick, then starts, restarts or stops what changed. - {Kind: ExtensionSettingsKind, Title: "Extension settings", ApplyMode: "live", Document: controlplanerpc.ExtensionSettings{}, Seed: func(config.Config) any { return controlplanerpc.ExtensionSettings{} }, Validate: validateExtensions}, + {Kind: ExtensionSettingsKind, Title: "Extension settings", ApplyMode: "live", Document: controlplanerpc.ExtensionSettings{}, Seed: func(cfg config.Config) any { + return controlplanerpc.ExtensionSettings{CatalogueSource: cfg.Distribution.Catalogue, CatalogueKey: cfg.Distribution.CatalogueKey} + }, Validate: validateExtensions}, // Live: the container pool reads these on every acquire and the dispatcher // is resized on publish. // Live: every check and install reads the channel afresh. - {Kind: UpdateSettingsKind, Title: "Update settings", ApplyMode: "live", Document: controlplanerpc.UpdateSettings{}, Seed: func(config.Config) any { return controlplanerpc.UpdateSettings{Channel: "stable"} }, Validate: validateUpdateSettings}, + {Kind: UpdateSettingsKind, Title: "Update settings", ApplyMode: "live", Document: controlplanerpc.UpdateSettings{}, Seed: func(cfg config.Config) any { + return controlplanerpc.UpdateSettings{Channel: "stable", Source: cfg.Distribution.Releases, Key: cfg.Distribution.ReleasesKey} + }, Validate: validateUpdateSettings}, {Kind: ContainerRuntimePoliciesKind, Title: "Container runtime policies", ApplyMode: "live", Document: containerRuntimePolicies{}, Seed: seedContainerPolicies, Validate: validateContainers, Normalize: normalizeContainerPolicies}, // Live: the retention sweep reads the document on every tick, so a // change applies to the next sweep without a restart. @@ -89,7 +93,8 @@ func validateExtensions(input []byte) error { } seen[setting.Name] = struct{}{} } - return nil + _, err := doc.Catalogue() + return err }) } @@ -176,6 +181,9 @@ func ApplyOrgScheduling(cfg config.Config, stored []byte) (config.Config, error) // validateUpdateSettings refuses a channel the update scripts cannot select. func validateUpdateSettings(input []byte) error { return validateAs(input, func(doc controlplanerpc.UpdateSettings) error { + if _, err := doc.Releases(); err != nil { + return err + } return releaseupdate.ValidateChannel(doc.Channel, doc.Pin) }) } diff --git a/internal/infrastructure/signedoci/freshness.go b/internal/infrastructure/signedoci/freshness.go new file mode 100644 index 000000000..a83231126 --- /dev/null +++ b/internal/infrastructure/signedoci/freshness.go @@ -0,0 +1,86 @@ +package signedoci + +import ( + "context" + "crypto/sha256" + "encoding/hex" + "encoding/json" + "errors" + "fmt" + "io/fs" + "os" + "path/filepath" + "sync" + "time" + + "github.com/samcharles93/archie-core/internal/fsutil" + "github.com/samcharles93/archie-core/internal/infrastructure/configuration" +) + +// FileFreshness is the durable freshness mark of accepted issue times, kept as one small JSON file +// written atomically so a crash never leaves a half-written mark. A missing +// file is no mark yet; a file that exists but does not parse is an error, so +// a corrupt mark fails closed instead of opening a replay window. +type FileFreshness struct { + Path string + + mu sync.Mutex +} + +func (f *FileFreshness) Newest(context.Context) (time.Time, error) { + f.mu.Lock() + defer f.mu.Unlock() + return f.read() +} + +func (f *FileFreshness) Accept(_ context.Context, issuedAt time.Time) error { + f.mu.Lock() + defer f.mu.Unlock() + current, err := f.read() + if err != nil { + return err + } + if !issuedAt.After(current) { + return nil + } + encoded, err := json.Marshal(struct { + IssuedAt time.Time `json:"issued_at"` + }{IssuedAt: issuedAt.UTC()}) + if err != nil { + return err + } + return fsutil.WriteFileAtomic(f.Path, encoded, 0o600) +} + +// read returns the mark, or the zero time when nothing has been +// accepted yet. +func (f *FileFreshness) read() (time.Time, error) { + raw, err := os.ReadFile(f.Path) + if errors.Is(err, fs.ErrNotExist) { + return time.Time{}, nil + } + if err != nil { + return time.Time{}, err + } + var mark struct { + IssuedAt time.Time `json:"issued_at"` + } + if err := json.Unmarshal(raw, &mark); err != nil { + return time.Time{}, fmt.Errorf("freshness mark %s: %w", f.Path, err) + } + return mark.IssuedAt, nil +} + +// MarkPath is where the freshness mark for one kind of artifact from one +// source lives: $XDG_STATE_HOME///.json, the state home +// the update journal uses too. The key names the reference, so two sources +// never share a mark. +func MarkPath(kind, reference string) string { + dir := os.Getenv("XDG_STATE_HOME") + if dir == "" { + home, _ := os.UserHomeDir() + dir = filepath.Join(home, ".local", "state") + } + sum := sha256.Sum256([]byte(reference)) + return filepath.Join(dir, configuration.InstanceName(), kind, hex.EncodeToString(sum[:8])+".json") +} diff --git a/internal/infrastructure/signedoci/freshness_test.go b/internal/infrastructure/signedoci/freshness_test.go new file mode 100644 index 000000000..525f81d4e --- /dev/null +++ b/internal/infrastructure/signedoci/freshness_test.go @@ -0,0 +1,47 @@ +package signedoci + +import ( + "os" + "path/filepath" + "testing" + "time" +) + +// The freshness mark never goes backwards, and a mark that exists but does +// not parse fails closed rather than opening a replay window. +func TestFileFreshness(t *testing.T) { + path := filepath.Join(t.TempDir(), "freshness.json") + mark := &FileFreshness{Path: path} + jan1 := time.Date(2026, 1, 1, 0, 0, 0, 0, time.UTC) + jan2 := time.Date(2026, 1, 2, 0, 0, 0, 0, time.UTC) + + newest, err := mark.Newest(t.Context()) + if err != nil { + t.Fatalf("Newest with no file: %v", err) + } + if !newest.IsZero() { + t.Fatalf("Newest with no file = %s; want the zero time", newest.Format(time.RFC3339)) + } + if err := mark.Accept(t.Context(), jan1); err != nil { + t.Fatalf("Accept jan1: %v", err) + } + if err := mark.Accept(t.Context(), jan2); err != nil { + t.Fatalf("Accept jan2: %v", err) + } + if err := mark.Accept(t.Context(), jan1); err != nil { + t.Fatalf("Accept jan1 again: %v", err) + } + if newest, err = mark.Newest(t.Context()); err != nil { + t.Fatalf("Newest: %v", err) + } + if !newest.Equal(jan2) { + t.Fatalf("Newest = %s; want %s", newest.Format(time.RFC3339), jan2.Format(time.RFC3339)) + } + + if err := os.WriteFile(path, []byte("{"), 0o600); err != nil { + t.Fatal(err) + } + if _, err := mark.Newest(t.Context()); err == nil { + t.Fatal("Newest over a corrupt mark: err = nil, want an error") + } +} diff --git a/internal/infrastructure/signedoci/signedoci.go b/internal/infrastructure/signedoci/signedoci.go new file mode 100644 index 000000000..c02247847 --- /dev/null +++ b/internal/infrastructure/signedoci/signedoci.go @@ -0,0 +1,154 @@ +// Package signedoci fetches OCI artifacts whose first layer is a statement +// signed by a distribution source's key. The extension catalogue and releases +// are both published this way. +package signedoci + +import ( + "context" + "crypto/ed25519" + "crypto/sha256" + "encoding/base64" + "encoding/hex" + "encoding/json" + "errors" + "fmt" + "io" + "net" + "strings" + + ocispec "github.com/opencontainers/image-spec/specs-go/v1" + "oras.land/oras-go/v2/registry/remote" + + "github.com/samcharles93/archie-core/internal/domain/distribution" +) + +const maxManifestBytes = 256 << 10 + +// Artifact is a fetched manifest whose statement layer verified against the +// source's key. Its other layers are read through Blob, which checks each +// against the digest the manifest names. +type Artifact struct { + Manifest ocispec.Manifest + Statement []byte + + repository *remote.Repository +} + +// Fetch resolves tag in the source's repository and verifies the statement +// layer: annotation on the manifest must be the key's signature over the +// statement bytes. maxStatement bounds the statement read. +func Fetch(ctx context.Context, source distribution.Source, tag, annotation string, maxStatement int) (Artifact, error) { + if len(source.Key) != ed25519.PublicKeySize { + return Artifact{}, errors.New("no valid signing key") + } + reference := source.Reference + ":" + tag + repository, err := remote.NewRepository(reference) + if err != nil { + return Artifact{}, err + } + repository.PlainHTTP = Loopback(reference) + _, stream, err := repository.FetchReference(ctx, reference) + if err != nil { + return Artifact{}, fmt.Errorf("%s: %w", reference, err) + } + raw, err := readBounded(stream, maxManifestBytes) + if err != nil { + return Artifact{}, err + } + var manifest ocispec.Manifest + if err := json.Unmarshal(raw, &manifest); err != nil { + return Artifact{}, fmt.Errorf("%s manifest: %w", reference, err) + } + if len(manifest.Layers) == 0 { + return Artifact{}, fmt.Errorf("%s has no statement layer", reference) + } + layer, err := repository.Fetch(ctx, manifest.Layers[0]) + if err != nil { + return Artifact{}, fmt.Errorf("%s statement: %w", reference, err) + } + statement, err := readBounded(layer, maxStatement) + if err != nil { + return Artifact{}, err + } + if err := Verify(manifest, statement, source.Key, annotation); err != nil { + return Artifact{}, fmt.Errorf("%s: %w", reference, err) + } + return Artifact{Manifest: manifest, Statement: statement, repository: repository}, nil +} + +// Verify checks statement is the first layer the manifest names and that key +// signed exactly those bytes under annotation. +func Verify(manifest ocispec.Manifest, statement []byte, key ed25519.PublicKey, annotation string) error { + if len(manifest.Layers) == 0 || Digest(statement) != manifest.Layers[0].Digest.String() { + return errors.New("statement does not match its manifest digest") + } + signature, err := base64.StdEncoding.DecodeString(manifest.Annotations[annotation]) + if err != nil || !ed25519.Verify(key, statement, signature) { + return errors.New("signature does not verify") + } + return nil +} + +// Blob copies the layer named digest to out, refusing one the manifest does +// not list or whose bytes do not hash to it. +func (a Artifact) Blob(ctx context.Context, digest string, out io.Writer, limit int64) error { + for _, layer := range a.Manifest.Layers { + if layer.Digest.String() != digest { + continue + } + if layer.Size > limit { + return fmt.Errorf("layer %s exceeds the size limit", digest) + } + stream, err := a.repository.Fetch(ctx, layer) + if err != nil { + return err + } + defer func() { _ = stream.Close() }() + hash := sha256.New() + n, err := io.Copy(io.MultiWriter(out, hash), io.LimitReader(stream, limit+1)) + if err != nil { + return err + } + if n > limit { + return fmt.Errorf("layer %s exceeds the size limit", digest) + } + if "sha256:"+hex.EncodeToString(hash.Sum(nil)) != digest { + return fmt.Errorf("layer %s does not match its digest", digest) + } + return nil + } + return fmt.Errorf("artifact has no layer %s", digest) +} + +// Digest is the OCI content digest of data. +func Digest(data []byte) string { + sum := sha256.Sum256(data) + return "sha256:" + hex.EncodeToString(sum[:]) +} + +// Loopback reports whether reference names a registry on this host, the +// only one reached over plain HTTP. The signature, not the transport, is +// what makes an artifact trustworthy. +func Loopback(reference string) bool { + host, _, ok := strings.Cut(reference, "/") + if !ok { + return false + } + if name, _, err := net.SplitHostPort(host); err == nil { + host = name + } + host = strings.Trim(host, "[]") + return host == "localhost" || net.ParseIP(host).IsLoopback() +} + +func readBounded(stream io.ReadCloser, limit int) ([]byte, error) { + defer func() { _ = stream.Close() }() + data, err := io.ReadAll(io.LimitReader(stream, int64(limit)+1)) + if err != nil { + return nil, err + } + if len(data) > limit { + return nil, errors.New("artifact content exceeds the size limit") + } + return data, nil +} diff --git a/internal/infrastructure/storepkg/catalogue.go b/internal/infrastructure/storepkg/catalogue.go index 164168737..92162e421 100644 --- a/internal/infrastructure/storepkg/catalogue.go +++ b/internal/infrastructure/storepkg/catalogue.go @@ -4,24 +4,14 @@ import ( "bytes" "context" "crypto/ed25519" - "crypto/sha256" - "encoding/base64" - "encoding/hex" "encoding/json" - "errors" "fmt" - "io/fs" - "os" - "path/filepath" - "sync" - "time" ocispec "github.com/opencontainers/image-spec/specs-go/v1" - "oras.land/oras-go/v2/registry/remote" + "github.com/samcharles93/archie-core/internal/domain/distribution" domain "github.com/samcharles93/archie-core/internal/domain/storepkg" - "github.com/samcharles93/archie-core/internal/fsutil" - "github.com/samcharles93/archie-core/internal/infrastructure/configuration" + "github.com/samcharles93/archie-core/internal/infrastructure/signedoci" ) // SignatureAnnotation carries the base64 ed25519 signature over the @@ -30,84 +20,48 @@ const SignatureAnnotation = "dev.archie.catalogue.signature.v1" const maxCatalogueBytes = 1 << 20 -// DefaultCatalogue is the archipelago marketplace every instance trusts: its -// reference, the public half of the key that signs it, and the durable -// freshness mark that refuses a replayed older catalogue. -func DefaultCatalogue() domain.CatalogueSource { - source := OCICatalogue{ - Reference: "ghcr.io/samcharles93/archipelago/catalogue:latest", - Key: mustKey("YL4nvn07uq/5mXZOt9cEmLYu/kB6C2+MLTnbHyoX4Pw="), - } +// catalogueTag is the tag a catalogue source publishes its current catalogue +// under. +const catalogueTag = "latest" + +// Catalogue fetches the catalogue a source publishes and refuses a replayed +// older one, keeping the freshness mark per source. +func Catalogue(source distribution.Source) domain.CatalogueSource { return domain.FreshCatalogueSource{ - Source: source, - Freshness: &FileFreshness{Path: defaultFreshnessPath(source.Reference)}, + Source: OCICatalogue{Source: source}, + Freshness: &signedoci.FileFreshness{Path: signedoci.MarkPath("catalogue", source.Reference)}, } } -func mustKey(encoded string) ed25519.PublicKey { - key, err := base64.StdEncoding.DecodeString(encoded) - if err != nil || len(key) != ed25519.PublicKeySize { - panic("storepkg: invalid catalogue key") - } - return key -} - -// OCICatalogue fetches a catalogue artifact and verifies it against Key: a -// one-layer OCI manifest whose layer is the catalogue JSON and whose -// annotation signs that layer. The reference may be a tag; the signature, -// not the tag, is what makes the list trustworthy. +// OCICatalogue fetches a catalogue artifact and verifies it against the +// source's key: an OCI manifest whose first layer is the catalogue JSON and +// whose annotation signs that layer. The tag is mutable; the signature is +// what makes the list trustworthy. type OCICatalogue struct { - Reference string - Key ed25519.PublicKey + Source distribution.Source } var _ domain.CatalogueSource = OCICatalogue{} func (c OCICatalogue) Fetch(ctx context.Context) (domain.Catalogue, error) { - if len(c.Key) != ed25519.PublicKeySize { - return domain.Catalogue{}, errors.New("catalogue: no valid signing key") - } - repository, err := remote.NewRepository(c.Reference) - if err != nil { - return domain.Catalogue{}, err - } - repository.PlainHTTP = loopbackReference(c.Reference) - _, stream, err := repository.FetchReference(ctx, c.Reference) + artifact, err := signedoci.Fetch(ctx, c.Source, catalogueTag, SignatureAnnotation, maxCatalogueBytes) if err != nil { return domain.Catalogue{}, fmt.Errorf("catalogue: %w", err) } - manifest, err := readBounded(stream, maxManifestBytes) - if err != nil { - return domain.Catalogue{}, err - } - var image ocispec.Manifest - if err := json.Unmarshal(manifest, &image); err != nil { - return domain.Catalogue{}, fmt.Errorf("catalogue manifest: %w", err) - } - if len(image.Layers) != 1 { - return domain.Catalogue{}, errors.New("catalogue must have one layer") - } - layerStream, err := repository.Fetch(ctx, image.Layers[0]) - if err != nil { - return domain.Catalogue{}, fmt.Errorf("catalogue layer: %w", err) - } - layer, err := readBounded(layerStream, maxCatalogueBytes) - if err != nil { - return domain.Catalogue{}, err - } - return verifyCatalogue(image, layer, c.Key) + return decodeCatalogue(artifact.Statement) } // verifyCatalogue checks the layer is the one the manifest names and that the // key signed it, then decodes it. func verifyCatalogue(image ocispec.Manifest, layer []byte, key ed25519.PublicKey) (domain.Catalogue, error) { - if contentDigest(layer) != image.Layers[0].Digest.String() { - return domain.Catalogue{}, errors.New("catalogue layer does not match its manifest digest") - } - signature, err := base64.StdEncoding.DecodeString(image.Annotations[SignatureAnnotation]) - if err != nil || !ed25519.Verify(key, layer, signature) { - return domain.Catalogue{}, errors.New("catalogue signature does not verify") + if err := signedoci.Verify(image, layer, key, SignatureAnnotation); err != nil { + return domain.Catalogue{}, fmt.Errorf("catalogue: %w", err) } + return decodeCatalogue(layer) +} + +// decodeCatalogue decodes a verified catalogue layer. +func decodeCatalogue(layer []byte) (domain.Catalogue, error) { decoder := json.NewDecoder(bytes.NewReader(layer)) decoder.DisallowUnknownFields() var catalogue domain.Catalogue @@ -119,73 +73,3 @@ func verifyCatalogue(image ocispec.Manifest, layer []byte, key ed25519.PublicKey } return catalogue, nil } - -// FileFreshness is the durable Freshness mark, kept as one small JSON file -// written atomically so a crash never leaves a half-written mark. A missing -// file is no mark yet; a file that exists but does not parse is an error, so -// a corrupt mark fails closed instead of opening a replay window. -type FileFreshness struct { - Path string - - mu sync.Mutex -} - -var _ domain.Freshness = (*FileFreshness)(nil) - -func (f *FileFreshness) Newest(context.Context) (time.Time, error) { - f.mu.Lock() - defer f.mu.Unlock() - return f.read() -} - -func (f *FileFreshness) Accept(_ context.Context, issuedAt time.Time) error { - f.mu.Lock() - defer f.mu.Unlock() - current, err := f.read() - if err != nil { - return err - } - if !issuedAt.After(current) { - return nil - } - encoded, err := json.Marshal(struct { - IssuedAt time.Time `json:"issued_at"` - }{IssuedAt: issuedAt.UTC()}) - if err != nil { - return err - } - return fsutil.WriteFileAtomic(f.Path, encoded, 0o600) -} - -// read returns the mark, or the zero time when no catalogue has been -// accepted yet. -func (f *FileFreshness) read() (time.Time, error) { - raw, err := os.ReadFile(f.Path) - if errors.Is(err, fs.ErrNotExist) { - return time.Time{}, nil - } - if err != nil { - return time.Time{}, err - } - var mark struct { - IssuedAt time.Time `json:"issued_at"` - } - if err := json.Unmarshal(raw, &mark); err != nil { - return time.Time{}, fmt.Errorf("catalogue freshness mark %s: %w", f.Path, err) - } - return mark.IssuedAt, nil -} - -// defaultFreshnessPath is where the default catalogue's freshness mark -// lives: $XDG_STATE_HOME//catalogue/.json, the state home the -// update journal uses too. The key names the catalogue reference, so two -// catalogues never share a mark. -func defaultFreshnessPath(reference string) string { - dir := os.Getenv("XDG_STATE_HOME") - if dir == "" { - home, _ := os.UserHomeDir() - dir = filepath.Join(home, ".local", "state") - } - sum := sha256.Sum256([]byte(reference)) - return filepath.Join(dir, configuration.InstanceName(), "catalogue", hex.EncodeToString(sum[:8])+".json") -} diff --git a/internal/infrastructure/storepkg/catalogue_test.go b/internal/infrastructure/storepkg/catalogue_test.go index dddc7b1b6..cfa069b18 100644 --- a/internal/infrastructure/storepkg/catalogue_test.go +++ b/internal/infrastructure/storepkg/catalogue_test.go @@ -5,15 +5,14 @@ import ( "crypto/ed25519" "encoding/base64" "errors" - "os" "path/filepath" "testing" - "time" digest "github.com/opencontainers/go-digest" ocispec "github.com/opencontainers/image-spec/specs-go/v1" domain "github.com/samcharles93/archie-core/internal/domain/storepkg" + "github.com/samcharles93/archie-core/internal/infrastructure/signedoci" ) // A catalogue is trusted only when its layer is the one the manifest names @@ -87,7 +86,7 @@ func verifiedCatalogue(t *testing.T, key ed25519.PublicKey, signer ed25519.Priva // refuses it survives a restart. func TestFetchRefusesReplayedCatalogue(t *testing.T) { trusted, signer, _ := ed25519.GenerateKey(nil) - mark := &FileFreshness{Path: filepath.Join(t.TempDir(), "catalogue", "freshness.json")} + mark := &signedoci.FileFreshness{Path: filepath.Join(t.TempDir(), "catalogue", "freshness.json")} newer := verifiedCatalogue(t, trusted, signer, "2026-01-02T00:00:00Z") older := verifiedCatalogue(t, trusted, signer, "2026-01-01T00:00:00Z") @@ -104,52 +103,13 @@ func TestFetchRefusesReplayedCatalogue(t *testing.T) { } // The mark is durable: a fresh reader over the same file, as after a // restart, still refuses the older catalogue and still accepts the new. - restarted := domain.FreshCatalogueSource{Source: staticCatalogue{older}, Freshness: &FileFreshness{Path: mark.Path}} + restarted := domain.FreshCatalogueSource{Source: staticCatalogue{older}, Freshness: &signedoci.FileFreshness{Path: mark.Path}} if _, err := restarted.Fetch(t.Context()); !errors.Is(err, domain.ErrCatalogueStale) { t.Fatalf("after restart: err = %v, want ErrCatalogueStale", err) } - if _, err := (domain.FreshCatalogueSource{Source: staticCatalogue{newer}, Freshness: &FileFreshness{Path: mark.Path}}).Fetch(t.Context()); err != nil { + if _, err := (domain.FreshCatalogueSource{Source: staticCatalogue{newer}, Freshness: &signedoci.FileFreshness{Path: mark.Path}}).Fetch(t.Context()); err != nil { t.Fatalf("after restart, newest catalogue: %v", err) } } -// The freshness mark never goes backwards, and a mark that exists but does -// not parse fails closed rather than opening a replay window. -func TestFileFreshness(t *testing.T) { - path := filepath.Join(t.TempDir(), "freshness.json") - mark := &FileFreshness{Path: path} - jan1 := time.Date(2026, 1, 1, 0, 0, 0, 0, time.UTC) - jan2 := time.Date(2026, 1, 2, 0, 0, 0, 0, time.UTC) - - newest, err := mark.Newest(t.Context()) - if err != nil { - t.Fatalf("Newest with no file: %v", err) - } - if !newest.IsZero() { - t.Fatalf("Newest with no file = %s; want the zero time", newest.Format(time.RFC3339)) - } - if err := mark.Accept(t.Context(), jan1); err != nil { - t.Fatalf("Accept jan1: %v", err) - } - if err := mark.Accept(t.Context(), jan2); err != nil { - t.Fatalf("Accept jan2: %v", err) - } - if err := mark.Accept(t.Context(), jan1); err != nil { - t.Fatalf("Accept jan1 again: %v", err) - } - if newest, err = mark.Newest(t.Context()); err != nil { - t.Fatalf("Newest: %v", err) - } - if !newest.Equal(jan2) { - t.Fatalf("Newest = %s; want %s", newest.Format(time.RFC3339), jan2.Format(time.RFC3339)) - } - - if err := os.WriteFile(path, []byte("{"), 0o600); err != nil { - t.Fatal(err) - } - if _, err := mark.Newest(t.Context()); err == nil { - t.Fatal("Newest over a corrupt mark: err = nil, want an error") - } -} - const zeros = "0000000000000000000000000000000000000000000000000000000000000000" diff --git a/internal/releaseupdate/github.go b/internal/releaseupdate/github.go deleted file mode 100644 index 9135d4ab4..000000000 --- a/internal/releaseupdate/github.go +++ /dev/null @@ -1,227 +0,0 @@ -package releaseupdate - -import ( - "context" - "encoding/json" - "errors" - "fmt" - "net/http" - "os" - "path/filepath" - "regexp" - "strings" - "sync" - "time" - - "golang.org/x/mod/semver" - - "github.com/samcharles93/archie-core/internal/buildinfo" -) - -// ReleaseSlug is the public repository releases are published to. The source -// repository is private, so its releases are unreachable without credentials. -const ReleaseSlug = "samcharles93/archie-releases" - -// changelogURL lists every release with its changelog section as notes. -const changelogURL = "https://github.com/" + ReleaseSlug + "/releases" - -// buildSuffix is the git-describe tail of an untagged build (1.7.0-4-g0736708). -var buildSuffix = regexp.MustCompile(`-[0-9]+-g[0-9a-f]+$`) - -// GitHubCatalog finds the newest release on the selected channel that ships a -// linux-amd64 zip, and reports it against this build. Only one version covers -// every component, so the daemon and agent are offered the same release. -type GitHubCatalog struct { - // Channel returns the release channel and pin in force. - Channel func(context.Context) (channel, pin string, err error) - - // A found release is reused for checkTTL: unauthenticated GitHub API - // calls are limited to 60 an hour per address. - mu sync.Mutex - key string - at time.Time - release *release -} - -const checkTTL = 10 * time.Minute - -func (c *GitHubCatalog) Check(ctx context.Context) (Snapshot, error) { - channel, pin, err := c.Channel(ctx) - if err != nil { - return Snapshot{}, err - } - if channel == "" { - channel = "stable" - } - release, err := c.detect(ctx, channel, pin) - if err != nil { - return Snapshot{}, err - } - component := func(id, label, installed string) Component { - return Component{ - ID: id, Label: label, - Installed: installed, Available: release.version, - Reference: release.asset, Changelog: changelogURL, - } - } - return Snapshot{Components: []Component{ - component("daemon", "Gateway", ReleaseVersion(buildinfo.Version)), - component("agent", "Runtime", installedAgent(executableDir())), - }}, nil -} - -func (c *GitHubCatalog) detect(ctx context.Context, channel, pin string) (*release, error) { - c.mu.Lock() - defer c.mu.Unlock() - key := channel + "\x00" + pin - if c.release != nil && c.key == key && time.Since(c.at) < checkTTL { - return c.release, nil - } - release, err := detectRelease(ctx, channel, pin) - if err != nil { - return nil, err - } - c.key, c.at, c.release = key, time.Now(), release - return release, nil -} - -// release is one published version and the zip that installs it here. -type release struct { - version string - asset string -} - -const releasesURL = "https://api.github.com/repos/" + ReleaseSlug + "/releases?per_page=100" - -type githubRelease struct { - TagName string `json:"tag_name"` - Draft bool `json:"draft"` - Prerelease bool `json:"prerelease"` - Assets []struct { - Name string `json:"name"` - } `json:"assets"` -} - -// detectRelease resolves channel to one published release. A release with no -// zip for this platform, or no SHA256SUMS to verify it against, is not -// installable, so it is never offered. -func detectRelease(ctx context.Context, channel, pin string) (*release, error) { - if err := ValidateChannel(channel, pin); err != nil { - return nil, err - } - releases, err := listReleases(ctx) - if err != nil { - return nil, fmt.Errorf("find releases: %w", err) - } - if found := pick(releases, channel, pin); found != nil { - return found, nil - } - return nil, errors.New("no published release with a linux-amd64 zip matches the channel") -} - -func listReleases(ctx context.Context) ([]githubRelease, error) { - req, err := http.NewRequestWithContext(ctx, http.MethodGet, releasesURL, nil) - if err != nil { - return nil, err - } - req.Header.Set("Accept", "application/vnd.github+json") - resp, err := http.DefaultClient.Do(req) - if err != nil { - return nil, err - } - defer func() { _ = resp.Body.Close() }() - if resp.StatusCode != http.StatusOK { - return nil, fmt.Errorf("GitHub answered %s", resp.Status) - } - var releases []githubRelease - if err := json.NewDecoder(resp.Body).Decode(&releases); err != nil { - return nil, fmt.Errorf("decode releases: %w", err) - } - return releases, nil -} - -// pick returns the newest installable release on channel. -func pick(releases []githubRelease, channel, pin string) *release { - var best *release - for _, r := range releases { - if !onChannel(r, channel, pin) { - continue - } - asset := installable(r) - if asset == "" { - continue - } - if best == nil || semver.Compare(r.TagName, "v"+best.version) > 0 { - best = &release{version: strings.TrimPrefix(r.TagName, "v"), asset: asset} - } - } - return best -} - -// onChannel: stable skips prereleases, next includes them, and exact-pin takes -// only the pinned tag. -func onChannel(r githubRelease, channel, pin string) bool { - if r.Draft || !semver.IsValid(r.TagName) { - return false - } - switch channel { - case "exact-pin": - return r.TagName == "v"+strings.TrimPrefix(pin, "v") - case "next": - return true - default: - return !r.Prerelease - } -} - -// installable names the release's linux-amd64 zip, or "" when it has none or -// no SHA256SUMS to verify it against. -func installable(r githubRelease) string { - asset, sums := "", false - for _, a := range r.Assets { - switch { - case a.Name == "SHA256SUMS": - sums = true - case strings.HasSuffix(a.Name, "-linux-amd64.zip"): - asset = a.Name - } - } - if !sums { - return "" - } - return asset -} - -// ReleaseVersion is the release a build was cut from: a git-describe build -// ("1.55.0-218-gabc123") is the same release as its tag. A build that names no -// release ("dev") has none, so nothing can be judged newer than it. -func ReleaseVersion(version string) string { - version = buildSuffix.ReplaceAllString(version, "") - if !semver.IsValid("v" + strings.TrimPrefix(version, "v")) { - return "" - } - return version -} - -// installedAgent is the image the installer last switched to, recorded in a -// sidecar beside the binaries; without one it is the runtime this release was -// cut with. A malformed sidecar is unknown, not guessed. -func installedAgent(binDir string) string { - data, err := os.ReadFile(filepath.Join(binDir, agentVersionFile)) - if binDir == "" || errors.Is(err, os.ErrNotExist) { - return ReleaseVersion(buildinfo.Runtime) - } - version := strings.TrimSpace(string(data)) - if err != nil || strings.ContainsAny(version, " \n") || ValidateChannel("exact-pin", version) != nil { - return "" - } - return ReleaseVersion(version) -} - -func executableDir() string { - exe, err := os.Executable() - if err != nil { - return "" - } - return filepath.Dir(exe) -} diff --git a/internal/releaseupdate/installer.go b/internal/releaseupdate/installer.go index 0ffa3056e..33b12d764 100644 --- a/internal/releaseupdate/installer.go +++ b/internal/releaseupdate/installer.go @@ -12,6 +12,8 @@ import ( "path/filepath" "strconv" "strings" + + "github.com/samcharles93/archie-core/internal/domain/distribution" ) // ResultSentinel prefixes the one line `archied update install` prints to @@ -23,8 +25,11 @@ const ResultSentinel = "ARCHIE_UPDATE_RESULT " var ErrNotConfigured = errors.New("updates are not configured") // ArchiedInstaller runs `archied update install` from beside this binary, so -// every service installs through the daemon's one transaction. -type ArchiedInstaller struct{} +// every service installs through the daemon's one transaction. Source is the +// distribution source the release was found in, so it installs from the same. +type ArchiedInstaller struct { + Source func(context.Context) (distribution.Source, error) +} func (ArchiedInstaller) path() string { path := filepath.Join(executableDir(), WatchdogBinary) @@ -47,8 +52,14 @@ func (i ArchiedInstaller) Install(ctx context.Context, snapshot Snapshot, meta I if len(available) == 0 { return Result{}, errors.New("nothing to install") } + source, err := i.Source(ctx) + if err != nil { + return Result{}, err + } cmd := exec.CommandContext(ctx, path, "update", "install", "-version", available[0].Available, + "-source", source.Reference, + "-key", distribution.EncodeKey(source.Key), "-report", meta.ReportPath, "-channel", meta.Channel, "-chat", strconv.FormatInt(meta.ChatID, 10), diff --git a/internal/releaseupdate/prepare.go b/internal/releaseupdate/prepare.go index 3cfa26a2c..a8cfbe54a 100644 --- a/internal/releaseupdate/prepare.go +++ b/internal/releaseupdate/prepare.go @@ -2,16 +2,11 @@ package releaseupdate import ( "archive/zip" - "bufio" - "bytes" "context" - "crypto/sha256" - "encoding/hex" "encoding/json" "errors" "fmt" "io" - "net/http" "os" "os/exec" "path" @@ -23,11 +18,9 @@ import ( "time" "github.com/samcharles93/archie-core/internal/buildinfo" + "github.com/samcharles93/archie-core/internal/domain/distribution" ) -// ReleaseURL is where published release assets are downloaded from. -const ReleaseURL = "https://github.com/" + ReleaseSlug + "/releases/download" - var ( releaseVersion = regexp.MustCompile(`^[0-9]+\.[0-9]+\.[0-9]+([+-][0-9A-Za-z.-]+)?$`) releaseUnit = regexp.MustCompile(`^archie[a-z0-9-]*$`) @@ -37,22 +30,25 @@ var ( // PrepareRequest names the release to stage and who hears the outcome. type PrepareRequest struct { Version string - // ReleaseURL is the base the release's zip and SHA256SUMS are under. - ReleaseURL string - Meta InstallMeta + // Source is the distribution source the release is fetched and verified + // from. + Source distribution.Source + Meta InstallMeta // CheckTopology refuses a host missing a unit or config section the // release requires. managed reports whether units are observable. CheckTopology func(ctx context.Context, required Topology, managed bool) error } -// Manifest is the release.json a release zip carries. +// Manifest is the release.json a release zip carries, with the agent image +// taken from the signed statement rather than from the zip. type Manifest struct { Required Topology `json:"required_topology"` AgentImage string `json:"agent_image"` } // Prepare verifies and stages a release without changing anything that runs: -// the zip is checked against SHA256SUMS, every binary must report the release, +// the statement must verify against the source's key and the zip against the +// statement, every binary must report the release, // the agent image is pulled by digest and the database is backed up before a // journal is published. The watchdog (Run) takes it from there. func (t *Transaction) Prepare(ctx context.Context, req PrepareRequest) (Result, error) { @@ -117,11 +113,15 @@ func (t *Transaction) fetch(ctx context.Context, req PrepareRequest, work string if runtime.GOOS != "linux" || runtime.GOARCH != "amd64" { return "", Manifest{}, fmt.Errorf("no published release for %s/%s", runtime.GOOS, runtime.GOARCH) } - name := fmt.Sprintf("archie-core-archied-v%s-linux-amd64", req.Version) - base := strings.TrimSuffix(req.ReleaseURL, "/") + "/v" + req.Version + "/" - t.say("downloading %s", name+".zip") - archive := filepath.Join(work, name+".zip") - if err := downloadVerified(ctx, base, name+".zip", archive); err != nil { + verified, err := Fetch(ctx, req.Source, Tag("exact-pin", req.Version)) + if err != nil { + return "", Manifest{}, err + } + zipName := verified.Statement.Zip.Name + name := strings.TrimSuffix(zipName, ".zip") + t.say("downloading %s", zipName) + archive := filepath.Join(work, zipName) + if err := writeBlob(ctx, verified, archive); err != nil { return "", Manifest{}, err } if err := unpack(archive, name, work); err != nil { @@ -132,6 +132,7 @@ func (t *Transaction) fetch(ctx context.Context, req PrepareRequest, work string if err != nil { return "", Manifest{}, err } + manifest.AgentImage = verified.Statement.AgentImage for _, binary := range manifest.Required.apps() { reported, err := reportedVersion(ctx, filepath.Join(release, binary), binary) if err != nil { @@ -226,61 +227,25 @@ func (t *Transaction) stage(req PrepareRequest, release string, j journal) error return syncDir(t.StateDir) } -func downloadVerified(ctx context.Context, base, name, dst string) error { - var sums bytes.Buffer - if err := download(ctx, base+"SHA256SUMS", &sums, 1<<20); err != nil { - return err - } - expected := "" - scanner := bufio.NewScanner(&sums) - for scanner.Scan() { - if fields := strings.Fields(scanner.Text()); len(fields) == 2 && fields[1] == name { - if expected != "" { - return fmt.Errorf("SHA256SUMS lists %s twice", name) - } - expected = strings.ToLower(fields[0]) - } - } - if len(expected) != sha256.Size*2 { - return fmt.Errorf("SHA256SUMS has no checksum for %s", name) - } +// writeBlob writes the release's zip to dst, checked against the digest the +// signed statement names. +func writeBlob(ctx context.Context, release Release, dst string) error { file, err := os.OpenFile(dst, os.O_CREATE|os.O_EXCL|os.O_WRONLY, 0o600) if err != nil { return err } - hash := sha256.New() - err = download(ctx, base+name, io.MultiWriter(file, hash), 1<<30) + ctx, cancel := context.WithTimeout(ctx, 5*time.Minute) + defer cancel() + err = release.Artifact.Blob(ctx, release.Statement.Zip.Digest, file, 1<<30) if closeErr := file.Close(); err == nil { err = closeErr } if err != nil { - return err - } - if hex.EncodeToString(hash.Sum(nil)) != expected { - return fmt.Errorf("release checksum mismatch for %s", name) + return fmt.Errorf("download %s: %w", release.Statement.Zip.Name, err) } return nil } -func download(ctx context.Context, url string, out io.Writer, limit int64) error { - ctx, cancel := context.WithTimeout(ctx, 5*time.Minute) - defer cancel() - req, err := http.NewRequestWithContext(ctx, http.MethodGet, url, nil) - if err != nil { - return err - } - resp, err := http.DefaultClient.Do(req) - if err != nil { - return fmt.Errorf("download %s: %w", url, err) - } - defer func() { _ = resp.Body.Close() }() - if resp.StatusCode != http.StatusOK { - return fmt.Errorf("download %s: %s", url, resp.Status) - } - _, err = io.Copy(out, io.LimitReader(resp.Body, limit)) - return err -} - // unpack extracts the zip's single distribution directory, refusing any // entry outside it and anything but regular files and directories. func unpack(archive, name, dst string) error { @@ -350,9 +315,6 @@ func readManifest(path string) (Manifest, error) { return Manifest{}, fmt.Errorf("invalid release unit %q", unit) } } - if !pinnedImage.MatchString(manifest.AgentImage) { - return Manifest{}, errors.New("release has no digest-pinned agent image") - } return manifest, nil } diff --git a/internal/releaseupdate/publish.go b/internal/releaseupdate/publish.go new file mode 100644 index 000000000..1128e6d01 --- /dev/null +++ b/internal/releaseupdate/publish.go @@ -0,0 +1,133 @@ +package releaseupdate + +import ( + "bytes" + "context" + "crypto/ed25519" + "encoding/base64" + "encoding/json" + "fmt" + "os" + "time" + + digest "github.com/opencontainers/go-digest" + ocispec "github.com/opencontainers/image-spec/specs-go/v1" + "oras.land/oras-go/v2/registry/remote" + "oras.land/oras-go/v2/registry/remote/auth" + "oras.land/oras-go/v2/registry/remote/credentials" + + "github.com/samcharles93/archie-core/internal/infrastructure/signedoci" +) + +// PublishRequest is one release to sign and push. +type PublishRequest struct { + // Repository is the OCI repository the release is pushed to. + Repository string + Signer ed25519.PrivateKey + Version string + Prerelease bool + // ZipPath is the distribution zip, named ZipName(Version). + ZipPath string + AgentImage string + Changelog string + // Channels are the moving tags pointed at this release besides its own + // version tag. + Channels []string +} + +// Publish signs a statement for the release and pushes it with the zip as +// one artifact, tagged with its version and each channel. Registry +// credentials come from the Docker config, as `docker login` left them. +func Publish(ctx context.Context, req PublishRequest) (Statement, error) { + archive, err := os.ReadFile(req.ZipPath) + if err != nil { + return Statement{}, err + } + statement := Statement{ + Version: req.Version, Prerelease: req.Prerelease, IssuedAt: time.Now().UTC(), Platform: Platform, + Zip: Blob{Name: ZipName(req.Version), Digest: signedoci.Digest(archive), Size: int64(len(archive))}, + AgentImage: req.AgentImage, Changelog: req.Changelog, + } + if err := statement.Validate(); err != nil { + return Statement{}, err + } + for _, channel := range req.Channels { + if channel != "stable" && channel != "next" { + return Statement{}, fmt.Errorf("unknown channel %q", channel) + } + } + encoded, err := json.Marshal(statement) + if err != nil { + return Statement{}, err + } + repository, err := pushRepository(req.Repository) + if err != nil { + return Statement{}, err + } + layers := []ocispec.Descriptor{ + {MediaType: StatementMediaType, Digest: digest.FromBytes(encoded), Size: int64(len(encoded))}, + { + MediaType: "application/zip", Digest: digest.FromBytes(archive), Size: int64(len(archive)), + Annotations: map[string]string{ocispec.AnnotationTitle: statement.Zip.Name}, + }, + } + for i, content := range [][]byte{ocispec.DescriptorEmptyJSON.Data, encoded, archive} { + desc := ocispec.DescriptorEmptyJSON + if i > 0 { + desc = layers[i-1] + } + if err := pushBlob(ctx, repository, desc, content); err != nil { + return Statement{}, err + } + } + manifest, err := json.Marshal(ocispec.Manifest{ + SchemaVersion: 2, MediaType: ocispec.MediaTypeImageManifest, + ArtifactType: ArtifactType, Config: ocispec.DescriptorEmptyJSON, Layers: layers, + Annotations: map[string]string{ + Annotation: base64.StdEncoding.EncodeToString(ed25519.Sign(req.Signer, encoded)), + ocispec.AnnotationCreated: statement.IssuedAt.Format(time.RFC3339), + }, + }) + if err != nil { + return Statement{}, err + } + desc := ocispec.Descriptor{MediaType: ocispec.MediaTypeImageManifest, Digest: digest.FromBytes(manifest), Size: int64(len(manifest))} + if err := repository.PushReference(ctx, desc, bytes.NewReader(manifest), Tag("exact-pin", req.Version)); err != nil { + return Statement{}, fmt.Errorf("push release manifest: %w", err) + } + for _, channel := range req.Channels { + if err := repository.Tag(ctx, desc, channel); err != nil { + return Statement{}, fmt.Errorf("tag %s: %w", channel, err) + } + } + return statement, nil +} + +func pushRepository(reference string) (*remote.Repository, error) { + repository, err := remote.NewRepository(reference) + if err != nil { + return nil, err + } + repository.PlainHTTP = signedoci.Loopback(reference) + store, err := credentials.NewStoreFromDocker(credentials.StoreOptions{}) + if err != nil { + return nil, fmt.Errorf("read registry credentials: %w", err) + } + repository.Client = &auth.Client{Credential: credentials.Credential(store)} + return repository, nil +} + +// pushBlob uploads content unless the registry already holds it. +func pushBlob(ctx context.Context, repository *remote.Repository, desc ocispec.Descriptor, content []byte) error { + exists, err := repository.Exists(ctx, desc) + if err != nil { + return fmt.Errorf("check blob %s: %w", desc.Digest, err) + } + if exists { + return nil + } + if err := repository.Push(ctx, desc, bytes.NewReader(content)); err != nil { + return fmt.Errorf("push blob %s: %w", desc.Digest, err) + } + return nil +} diff --git a/internal/releaseupdate/release.go b/internal/releaseupdate/release.go new file mode 100644 index 000000000..dac26ca74 --- /dev/null +++ b/internal/releaseupdate/release.go @@ -0,0 +1,226 @@ +package releaseupdate + +import ( + "context" + "encoding/json" + "errors" + "fmt" + "os" + "path/filepath" + "regexp" + "strings" + "sync" + "time" + + "golang.org/x/mod/semver" + + "github.com/samcharles93/archie-core/internal/buildinfo" + "github.com/samcharles93/archie-core/internal/domain/distribution" + "github.com/samcharles93/archie-core/internal/infrastructure/signedoci" +) + +// Annotation carries the source key's signature over a release statement. +const Annotation = "dev.archie.release.signature.v1" + +// ArtifactType marks a release manifest. +const ArtifactType = "application/vnd.archie.release.v1" + +// StatementMediaType is the release statement layer's media type. +const StatementMediaType = "application/vnd.archie.release.statement.v1+json" + +// Platform is the only platform releases are built for. +const Platform = "linux-amd64" + +const maxStatementBytes = 64 << 10 + +// buildSuffix is the git-describe tail of an untagged build (1.7.0-4-g0736708). +var buildSuffix = regexp.MustCompile(`-[0-9]+-g[0-9a-f]+$`) + +// Statement is what a release's signature covers: the version, when it was +// issued, the zip that installs it and the agent image it runs. Everything an +// install trusts comes from here, never from the registry tag or the zip. +type Statement struct { + Version string `json:"version"` + Prerelease bool `json:"prerelease"` + IssuedAt time.Time `json:"issued_at"` + Platform string `json:"platform"` + Zip Blob `json:"zip"` + AgentImage string `json:"agent_image"` + Changelog string `json:"changelog,omitempty"` +} + +// Blob names a release layer by its content digest. +type Blob struct { + Name string `json:"name"` + Digest string `json:"digest"` + Size int64 `json:"size"` +} + +// Validate refuses a statement an install could not act on safely. +func (s Statement) Validate() error { + switch { + case !releaseVersion.MatchString(s.Version): + return fmt.Errorf("release statement version %q is not a release version", s.Version) + case s.IssuedAt.IsZero(): + return errors.New("release statement has no issue time") + case s.Platform != Platform: + return fmt.Errorf("release statement is for %q, not %s", s.Platform, Platform) + case s.Zip.Name != ZipName(s.Version) || !strings.HasPrefix(s.Zip.Digest, "sha256:") || s.Zip.Size <= 0: + return errors.New("release statement does not name its zip by name, digest and size") + case !pinnedImage.MatchString(s.AgentImage): + return fmt.Errorf("release statement agent image %q is not pinned by digest", s.AgentImage) + } + return nil +} + +// ZipName is the distribution zip a version is published as. +func ZipName(version string) string { + return fmt.Sprintf("archie-core-archied-v%s-%s.zip", version, Platform) +} + +// Tag is the registry tag a channel resolves through: stable and next are +// moving tags, an exact pin is the version's own. +func Tag(channel, pin string) string { + if channel == "exact-pin" { + return "v" + strings.TrimPrefix(pin, "v") + } + return channel +} + +// Release is a verified release and the artifact its zip is read from. +type Release struct { + Statement Statement + Artifact signedoci.Artifact +} + +// Fetch resolves tag at source and verifies its statement. A statement on a +// moving channel tag issued before the newest one accepted from that tag is a +// replay and is refused; an exact pin may name an older release on purpose. +func Fetch(ctx context.Context, source distribution.Source, tag string) (Release, error) { + artifact, err := signedoci.Fetch(ctx, source, tag, Annotation, maxStatementBytes) + if err != nil { + return Release{}, fmt.Errorf("release: %w", err) + } + decoder := json.NewDecoder(strings.NewReader(string(artifact.Statement))) + decoder.DisallowUnknownFields() + var statement Statement + if err := decoder.Decode(&statement); err != nil { + return Release{}, fmt.Errorf("release statement: %w", err) + } + if err := statement.Validate(); err != nil { + return Release{}, err + } + if strings.HasPrefix(tag, "v") { + if "v"+statement.Version != tag { + return Release{}, fmt.Errorf("tag %s carries the statement for %s", tag, statement.Version) + } + return Release{Statement: statement, Artifact: artifact}, nil + } + mark := &signedoci.FileFreshness{Path: signedoci.MarkPath("releases", source.Reference+":"+tag)} + newest, err := mark.Newest(ctx) + if err != nil { + return Release{}, err + } + if !newest.IsZero() && statement.IssuedAt.Before(newest) { + return Release{}, fmt.Errorf("release on %s was issued %s, before the newest accepted %s", + tag, statement.IssuedAt.Format(time.RFC3339), newest.Format(time.RFC3339)) + } + if err := mark.Accept(ctx, statement.IssuedAt); err != nil { + return Release{}, fmt.Errorf("record release freshness: %w", err) + } + return Release{Statement: statement, Artifact: artifact}, nil +} + +// RegistryCatalog reports the release on the selected channel against this build. +// Only one version covers every component, so the daemon and agent are +// offered the same release. +type RegistryCatalog struct { + // Settings returns the source, channel and pin in force. + Settings func(context.Context) (source distribution.Source, channel, pin string, err error) + + // A found release is reused for checkTTL so a dashboard that polls does + // not hit the registry on every view. + mu sync.Mutex + key string + at time.Time + statement Statement +} + +const checkTTL = 10 * time.Minute + +func (c *RegistryCatalog) Check(ctx context.Context) (Snapshot, error) { + source, channel, pin, err := c.Settings(ctx) + if err != nil { + return Snapshot{}, err + } + if channel == "" { + channel = "stable" + } + if err := ValidateChannel(channel, pin); err != nil { + return Snapshot{}, err + } + statement, err := c.detect(ctx, source, Tag(channel, pin)) + if err != nil { + return Snapshot{}, err + } + component := func(id, label, installed string) Component { + return Component{ + ID: id, Label: label, + Installed: installed, Available: statement.Version, + Reference: statement.Zip.Name, Changelog: statement.Changelog, + } + } + return Snapshot{Components: []Component{ + component("daemon", "Gateway", ReleaseVersion(buildinfo.Version)), + component("agent", "Runtime", installedAgent(executableDir())), + }}, nil +} + +func (c *RegistryCatalog) detect(ctx context.Context, source distribution.Source, tag string) (Statement, error) { + c.mu.Lock() + defer c.mu.Unlock() + key := source.Reference + "\x00" + distribution.EncodeKey(source.Key) + "\x00" + tag + if c.key == key && time.Since(c.at) < checkTTL { + return c.statement, nil + } + release, err := Fetch(ctx, source, tag) + if err != nil { + return Statement{}, err + } + c.key, c.at, c.statement = key, time.Now(), release.Statement + return release.Statement, nil +} + +// ReleaseVersion is the release a build was cut from: a git-describe build +// ("1.55.0-218-gabc123") is the same release as its tag. A build that names no +// release ("dev") has none, so nothing can be judged newer than it. +func ReleaseVersion(version string) string { + version = buildSuffix.ReplaceAllString(version, "") + if !semver.IsValid("v" + strings.TrimPrefix(version, "v")) { + return "" + } + return version +} + +// installedAgent is the image the installer last switched to, recorded in a +// sidecar beside the binaries; without one it is the runtime this release was +// cut with. A malformed sidecar is unknown, not guessed. +func installedAgent(binDir string) string { + data, err := os.ReadFile(filepath.Join(binDir, agentVersionFile)) + if binDir == "" || errors.Is(err, os.ErrNotExist) { + return ReleaseVersion(buildinfo.Runtime) + } + version := strings.TrimSpace(string(data)) + if err != nil || strings.ContainsAny(version, " \n") || ValidateChannel("exact-pin", version) != nil { + return "" + } + return ReleaseVersion(version) +} + +func executableDir() string { + exe, err := os.Executable() + if err != nil { + return "" + } + return filepath.Dir(exe) +} diff --git a/internal/releaseupdate/release_test.go b/internal/releaseupdate/release_test.go new file mode 100644 index 000000000..40e67db33 --- /dev/null +++ b/internal/releaseupdate/release_test.go @@ -0,0 +1,74 @@ +package releaseupdate + +import ( + "crypto/ed25519" + "encoding/json" + "net/http" + "net/http/httptest" + "strings" + "testing" + "time" + + "github.com/samcharles93/archie-core/internal/domain/distribution" + "github.com/samcharles93/archie-core/internal/infrastructure/signedoci" +) + +// A release is trusted only when the source's key signed its statement, the +// tag it was found under agrees with it, and a moving channel tag never goes +// back to a release issued before one already accepted from it. +func TestFetchRelease(t *testing.T) { + trusted, signer, _ := ed25519.GenerateKey(nil) + stranger, _, _ := ed25519.GenerateKey(nil) + archive := []byte("zip") + statement := func(version string, issued time.Time) []byte { + data, err := json.Marshal(Statement{ + Version: version, IssuedAt: issued, Platform: Platform, AgentImage: newImage, + Zip: Blob{Name: ZipName(version), Digest: signedoci.Digest(archive), Size: int64(len(archive))}, + }) + if err != nil { + t.Fatal(err) + } + return data + } + jan1, jan2 := time.Date(2026, 1, 1, 0, 0, 0, 0, time.UTC), time.Date(2026, 1, 2, 0, 0, 0, 0, time.UTC) + + cases := []struct { + name string + accepted []byte // a statement already accepted from the same tag + statement []byte + tag string + key ed25519.PublicKey + want string // error substring, or "" to accept + }{ + {name: "signed by the source key", statement: statement("2.0.0", jan1), tag: "stable", key: trusted}, + {name: "signed by another key", statement: statement("2.0.0", jan1), tag: "stable", key: stranger, want: "signature"}, + {name: "a version tag carrying another version", statement: statement("2.0.0", jan1), tag: "v3.0.0", key: trusted, want: "carries"}, + {name: "a channel moved back to an older release", accepted: statement("2.1.0", jan2), statement: statement("2.0.0", jan1), tag: "stable", key: trusted, want: "before the newest"}, + {name: "an exact pin to an older release", accepted: statement("2.1.0", jan2), statement: statement("2.0.0", jan1), tag: "v2.0.0", key: trusted}, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + t.Setenv("XDG_STATE_HOME", t.TempDir()) + // One registry, so the freshness mark for the tag is shared; it + // serves the accepted statement first, then the one under test. + var current http.Handler + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { current.ServeHTTP(w, r) })) + defer server.Close() + source := distribution.Source{Reference: strings.TrimPrefix(server.URL, "http://") + "/archie", Key: tc.key} + if tc.accepted != nil { + current = fakeRegistry(t, "archie", "stable", signer, tc.accepted, archive, archive) + if _, err := Fetch(t.Context(), distribution.Source{Reference: source.Reference, Key: trusted}, "stable"); err != nil { + t.Fatalf("accept the newer release: %v", err) + } + } + current = fakeRegistry(t, "archie", tc.tag, signer, tc.statement, archive, archive) + _, err := Fetch(t.Context(), source, tc.tag) + switch { + case tc.want == "" && err != nil: + t.Fatalf("Fetch: %v", err) + case tc.want != "" && (err == nil || !strings.Contains(err.Error(), tc.want)): + t.Fatalf("Fetch error = %v, want %q", err, tc.want) + } + }) + } +} diff --git a/internal/releaseupdate/transaction_test.go b/internal/releaseupdate/transaction_test.go index 6a51f175e..e488e632e 100644 --- a/internal/releaseupdate/transaction_test.go +++ b/internal/releaseupdate/transaction_test.go @@ -4,7 +4,9 @@ import ( "archive/zip" "bytes" "context" - "crypto/sha256" + "crypto/ed25519" + "encoding/base64" + "encoding/json" "errors" "fmt" "net/http" @@ -15,6 +17,12 @@ import ( "strings" "testing" "time" + + digest "github.com/opencontainers/go-digest" + ocispec "github.com/opencontainers/image-spec/specs-go/v1" + + "github.com/samcharles93/archie-core/internal/domain/distribution" + "github.com/samcharles93/archie-core/internal/infrastructure/signedoci" ) var testUnits = []string{"archie-state-store", "archie-gateway", "archied"} @@ -152,7 +160,7 @@ func TestTransaction(t *testing.T) { name string host fakeHost candidate string // version the release's archied reports - corrupt bool // SHA256SUMS does not match the zip + corrupt bool // the registry serves a zip other than the one signed crashAt string // phase the watchdog dies in, then boot recovers prepare string // Prepare refusal want outcome @@ -162,7 +170,7 @@ func TestTransaction(t *testing.T) { {name: "unhealthy migrated release restores the database", host: fakeHost{managed: true, unhealthy: true, migrates: true}, want: outcome{version: "1.0.0", image: oldImage, report: "failed", restoredDatabase: true}}, {name: "stuck image switch rolls back", host: fakeHost{managed: true, hangs: true}, want: old}, {name: "without systemd it installs without restarting", host: fakeHost{}, want: outcome{version: "2.0.0", image: newImage}}, - {name: "checksum mismatch is refused", host: fakeHost{managed: true}, corrupt: true, prepare: "checksum mismatch", want: outcome{version: "1.0.0", image: oldImage}}, + {name: "a tampered zip is refused", host: fakeHost{managed: true}, corrupt: true, prepare: "digest", want: outcome{version: "1.0.0", image: oldImage}}, {name: "misreported candidate is refused", host: fakeHost{managed: true}, candidate: "1.9.0", prepare: "reports", want: outcome{version: "1.0.0", image: oldImage}}, {name: "crash before the watchdog started is abandoned", host: fakeHost{managed: true}, crashAt: phasePrepared, want: old}, {name: "crash while switching rolls back", host: fakeHost{managed: true}, crashAt: phaseSwitching, want: old}, @@ -180,7 +188,8 @@ func TestTransaction(t *testing.T) { if candidate == "" { candidate = "2.0.0" } - server := httptest.NewServer(releaseHandler(t, candidate, tt.corrupt)) + handler, key := releaseHandler(t, candidate, tt.corrupt) + server := httptest.NewServer(handler) defer server.Close() for _, unit := range testUnits { writeTestFile(t, filepath.Join(host.bin, unit), versionScript(unit, "1.0.0")) @@ -192,7 +201,7 @@ func TestTransaction(t *testing.T) { HealthTimeout: 20 * time.Millisecond, Observe: 5 * time.Millisecond, Poll: time.Millisecond, } - _, err := tx.Prepare(t.Context(), PrepareRequest{Version: "2.0.0", ReleaseURL: server.URL, Meta: InstallMeta{Channel: "webui", ReportPath: report}}) + _, err := tx.Prepare(t.Context(), PrepareRequest{Version: "2.0.0", Source: distribution.Source{Reference: strings.TrimPrefix(server.URL, "http://") + "/archie", Key: key}, Meta: InstallMeta{Channel: "webui", ReportPath: report}}) switch { case tt.prepare != "": if err == nil || !strings.Contains(err.Error(), tt.prepare) { @@ -254,7 +263,7 @@ func TestTransaction(t *testing.T) { // releaseHandler serves one release's zip and SHA256SUMS the way GitHub // release downloads are laid out. -func releaseHandler(t *testing.T, candidate string, corrupt bool) http.Handler { +func releaseHandler(t *testing.T, candidate string, corrupt bool) (http.Handler, ed25519.PublicKey) { t.Helper() name := "archie-core-archied-v2.0.0-linux-amd64" var archive bytes.Buffer @@ -277,17 +286,65 @@ func releaseHandler(t *testing.T, candidate string, corrupt bool) http.Handler { } add(unit, versionScript(unit, version)) } - add("release.json", fmt.Sprintf(`{"required_topology":{"units":["%s"]},"agent_image":%q}`, strings.Join(testUnits, `","`), newImage)) + add("release.json", fmt.Sprintf(`{"required_topology":{"units":["%s"]}}`, strings.Join(testUnits, `","`))) if err := writer.Close(); err != nil { t.Fatal(err) } - sum := sha256.Sum256(archive.Bytes()) + zipDigest := signedoci.Digest(archive.Bytes()) + statement, err := json.Marshal(Statement{ + Version: "2.0.0", IssuedAt: time.Date(2026, 1, 1, 0, 0, 0, 0, time.UTC), Platform: Platform, + Zip: Blob{Name: name + ".zip", Digest: zipDigest, Size: int64(archive.Len())}, + AgentImage: newImage, + }) + if err != nil { + t.Fatal(err) + } + public, private, err := ed25519.GenerateKey(nil) + if err != nil { + t.Fatal(err) + } + served := archive.Bytes() if corrupt { - sum[0] ^= 0xff + served = append([]byte(nil), served...) + served[len(served)/2] ^= 0xff + } + return fakeRegistry(t, "archie", "v2.0.0", private, statement, archive.Bytes(), served), public +} + +// fakeRegistry serves one signed release artifact at repository:tag, the way +// an OCI registry answers oras: the manifest by tag, each layer by digest. +// served is the zip the registry returns, which a tampering registry swaps. +func fakeRegistry(t *testing.T, repository, tag string, signer ed25519.PrivateKey, statement, archive, served []byte) http.Handler { + t.Helper() + blobs := map[string][]byte{signedoci.Digest(statement): statement, signedoci.Digest(archive): served} + manifest, err := json.Marshal(ocispec.Manifest{ + SchemaVersion: 2, + MediaType: ocispec.MediaTypeImageManifest, + ArtifactType: ArtifactType, + Config: ocispec.DescriptorEmptyJSON, + Layers: []ocispec.Descriptor{ + {MediaType: StatementMediaType, Digest: digest.FromBytes(statement), Size: int64(len(statement))}, + {MediaType: "application/zip", Digest: digest.FromBytes(archive), Size: int64(len(archive))}, + }, + Annotations: map[string]string{Annotation: base64.StdEncoding.EncodeToString(ed25519.Sign(signer, statement))}, + }) + if err != nil { + t.Fatal(err) } mux := http.NewServeMux() - mux.HandleFunc("/v2.0.0/"+name+".zip", func(w http.ResponseWriter, _ *http.Request) { _, _ = w.Write(archive.Bytes()) }) - mux.HandleFunc("/v2.0.0/SHA256SUMS", func(w http.ResponseWriter, _ *http.Request) { fmt.Fprintf(w, "%x %s.zip\n", sum, name) }) + mux.HandleFunc("/v2/"+repository+"/manifests/"+tag, func(w http.ResponseWriter, _ *http.Request) { + w.Header().Set("Content-Type", ocispec.MediaTypeImageManifest) + w.Header().Set("Docker-Content-Digest", signedoci.Digest(manifest)) + _, _ = w.Write(manifest) + }) + mux.HandleFunc("/v2/"+repository+"/blobs/", func(w http.ResponseWriter, r *http.Request) { + blob, ok := blobs[strings.TrimPrefix(r.URL.Path, "/v2/"+repository+"/blobs/")] + if !ok { + http.NotFound(w, r) + return + } + _, _ = w.Write(blob) + }) return mux } diff --git a/tools/publish.sh b/tools/publish.sh deleted file mode 100755 index 1cfec1c54..000000000 --- a/tools/publish.sh +++ /dev/null @@ -1,47 +0,0 @@ -#!/usr/bin/env bash -# Publishes a tagged release by hand: the agent and archied images to GHCR, -# then the linux-amd64 zip, SHA256SUMS and install.sh to the public releases -# repository, whose release notes are the version's changelog section. -# Builds from a clean worktree of the tag, so local edits never ship. -set -euo pipefail - -version="${1:?usage: tools/publish.sh X.Y.Z}" -tag="v$version" -releases="samcharles93/archie-releases" -registry="ghcr.io/samcharles93" -root="$(git rev-parse --show-toplevel)" -work="$(mktemp -d)" -trap 'git -C "$root" worktree remove --force "$work/src" >/dev/null 2>&1 || true; rm -rf "$work"' EXIT - -git -C "$root" worktree add -q --detach "$work/src" "$tag" -cd "$work/src" -sha="$(git rev-parse "$tag^{commit}")" - -docker build -q -f Dockerfile --build-arg RUNTIME_VERSION="$version" \ - -t "$registry/archie-agent:latest" -t "$registry/archie-agent:sha-$sha" . >/dev/null -docker push -q "$registry/archie-agent:latest" -docker push -q "$registry/archie-agent:sha-$sha" -digest="$(docker inspect --format '{{range .RepoDigests}}{{println .}}{{end}}' "$registry/archie-agent:latest" | - grep "^$registry/archie-agent@" | head -1 | cut -d@ -f2)" -test -n "$digest" - -docker build -q -f Dockerfile.archied --build-arg GATEWAY_VERSION="$version" --build-arg RUNTIME_VERSION="$version" \ - -t "$registry/archied:latest" -t "$registry/archied:edge" -t "$registry/archied:sha-$sha" . >/dev/null -for t in latest edge "sha-$sha"; do docker push -q "$registry/archied:$t"; done - -ldflags="-X github.com/samcharles93/archie-core/internal/installtype.buildType=binary -X github.com/samcharles93/archie-core/internal/buildinfo.Version=$version -X github.com/samcharles93/archie-core/internal/buildinfo.Runtime=$version" -name="archie-core-archied-$tag-linux-amd64" -mkdir -p "$name" -for cmd in $(jq -r '.required_topology.units[]' release.json); do - GOOS=linux GOARCH=amd64 CGO_ENABLED=0 go build -ldflags "$ldflags" -o "$name/$cmd" "./cmd/$cmd" -done -cp deployments/INSTRUCTIONS.md CHANGELOG.md "$name/" -jq --arg image "$registry/archie-agent@$digest" '. + {agent_image: $image}' release.json >"$name/release.json" -if [ -f README.md ]; then cp README.md "$name/"; fi -zip -qr "$name.zip" "$name" -sha256sum "$name.zip" >SHA256SUMS - -awk -v hdr="## [$version]" 'index($0,hdr)==1{p=1} p&&/^## \[/&&index($0,hdr)!=1{exit} p{print}' CHANGELOG.md >notes.md -test -s notes.md -gh release create "$tag" -R "$releases" --title "archie $tag" --notes-file notes.md "$name.zip" SHA256SUMS install.sh -echo "published $tag to $releases (agent $digest)"