feat(releaseupdate): signed OCI distribution sources for releases and the catalogue (#1715)
Quality / check (push) Successful in 5m53s
deploy / gate (push) Canceled after 0s
deploy / archied-image (push) Canceled after 0s
deploy / agent-image (push) Canceled after 0s
deploy / dist-zip (push) Canceled after 0s

This commit is contained in:
sam committed 2026-10-31 18:18:43 +11:00
1 parent c87b0c31e1
commit 9d8f364af1
37 files changed
+1344 -651

No files matched your search

+30 -39
View File
@@ -373,45 +373,45 @@ jobs:
# rather than fail the release on its absence.
[ -f README.md ] && cp README.md "$name/"
zip -r "$name.zip" "$name"
sha256sum "$name.zip" > SHA256SUMS
echo "name=$name" >> "$GITHUB_OUTPUT"
- name: Upload distribution zip
uses: actions/upload-artifact@v4
with:
name: ${{ steps.zip.outputs.name }}
path: |
${{ steps.zip.outputs.name }}.zip
SHA256SUMS
path: ${{ steps.zip.outputs.name }}.zip
retention-days: 90
# A release is what an operator can actually download from; the
# artifact above is reachable only from the run's page, behind a
# login, and expires after 90 days. Until this step existed the
# zip was produced and then stranded.
#
# The release lives on this repository, attached to the release's v*
# tag, or the archied tag for a release cut before the single release
# stream. The body is the changelog section -- the release's own
# reviewed prose -- not generated notes: the tag/changelog gate above
# already guarantees it describes this version.
- name: Publish Gitea release with the distribution zip
# Installs and updates take a release from the signed artifact in the
# registry, so this is what publishes it: the statement signed with
# RELEASE_SIGNING_KEY and the zip, tagged v<version>, next, and stable
# unless it is a prerelease.
- name: Log in to the Gitea registry
run: echo "${{ secrets.DEPLOY_TOKEN }}" | docker login git.catlow.cloud -u "${{ github.actor }}" --password-stdin
- name: Publish the signed release
env:
ARCHIE_RELEASE_SEED: ${{ secrets.RELEASE_SIGNING_KEY }}
run: |
set -e
version="${{ steps.version.outputs.version }}"
prerelease=false
case "$version" in *-*) prerelease=true ;; esac
./dist/archied release publish -source git.catlow.cloud/sam/archie \
-version "$version" -prerelease="$prerelease" \
-zip "${{ steps.zip.outputs.name }}.zip" \
-agent-image "git.catlow.cloud/sam/archie-agent@${{ needs.agent-image.outputs.agent_digest }}" \
-changelog "https://git.catlow.cloud/sam/archie/releases/tag/v$version"
# The release page carries this version's changelog section, the
# release's own reviewed prose; the signed statement links to it.
- name: Publish the Gitea release notes
env:
# One PAT covers this step and the image jobs' registry login. It
# needs write:packages and write:repository scopes.
DEPLOY_TOKEN: ${{ secrets.DEPLOY_TOKEN }}
run: |
set -e
tag="v${{ steps.version.outputs.version }}"
if ! git ls-remote --exit-code --tags origin "refs/tags/$tag" >/dev/null; then
tag="archied/v${{ steps.version.outputs.version }}"
fi
version="${{ steps.version.outputs.version }}"
# The release body is this version's changelog section, not the whole
# file: from its "## [x.y.z]" heading to the next "## [" heading.
# Extracting rather than inlining keeps the changelog the single
# source of truth, and the tag/changelog gate above has already
# proven that section exists.
notes="$(mktemp)"
awk -v hdr="## [$version]" '
index($0, hdr) == 1 { printing = 1 }
@@ -425,19 +425,10 @@ jobs:
api="https://git.catlow.cloud/api/v1"
repo="sam/archie"
auth="Authorization: token $DEPLOY_TOKEN"
# Idempotent: drop any release already on the tag and recreate it,
# so a re-run of this job (a retried or superseded deploy) cannot
# fail on an existing release or duplicate the zip. Deleting a
# release leaves the tag itself alone.
# Idempotent: a re-run replaces the release on the tag.
curl -sf -X DELETE -H "$auth" "$api/repos/$repo/releases/tags/$tag" >/dev/null || true
id="$(curl -sf -X POST -H "$auth" -H "Content-Type: application/json" \
-d "$(jq -n --arg tag "$tag" --arg name "archied v$version" --arg body "$(cat "$notes")" \
curl -sf -X POST -H "$auth" -H "Content-Type: application/json" \
-d "$(jq -n --arg tag "$tag" --arg name "Archie v$version" --arg body "$(cat "$notes")" \
'{tag_name: $tag, name: $name, body: $body, draft: false, prerelease: false}')" \
"$api/repos/$repo/releases" | jq -r '.id')"
test -n "$id"
for asset in "${{ steps.zip.outputs.name }}.zip" SHA256SUMS install.sh; do
curl -sf -X POST -H "$auth" -H "Content-Type: application/octet-stream" \
--data-binary "@$asset" \
"$api/repos/$repo/releases/$id/assets?name=$(basename "$asset")" >/dev/null
done
echo "published $tag with ${{ steps.zip.outputs.name }}.zip"
"$api/repos/$repo/releases" >/dev/null
echo "published release notes for $tag"
+2
View File
@@ -7,6 +7,8 @@ release's per-component sections are labelled beneath its heading.
### archied
- Releases are signed. A release is an OCI artifact whose statement is signed with the release key; updates and `install.sh` refuse a release the key did not sign, a zip that is not the one signed, and a channel moved back to an older release. Releases now come from `git.catlow.cloud/sam/archie`, so a host on 1.55.0 or earlier finds this release only by running the new installer once: `curl -fsSL https://offloaded.dev/install.sh | bash`.
- Where releases and the extension catalogue come from is a setting: an OCI repository and the key it signs with, in the update settings and the extension settings, or `[distribution]` in `config.toml` for a new instance. `archied release keygen` and `archied release publish` make a key and publish signed releases to your own registry.
- Chat saves the workflow draft it writes with `workflow_draft` instead of handing over YAML to paste: the draft lands in the workflow editor for review, attributed to the signed-in account.
## [1.55.0] - 2026-10-16
+1 -1
View File
@@ -101,7 +101,7 @@ or a hosted provider API key.
1. **Install.**
```bash
curl -fsSL https://raw.githubusercontent.com/samcharles93/archie-core/main/install.sh | bash
curl -fsSL https://offloaded.dev/install.sh | bash
```
It downloads the latest release, verifies its checksum, installs the five
+7 -7
View File
@@ -58,14 +58,14 @@ task release:prepare VERSION=1.3.0 # write the section into CHANGELOG.md, un
# edit CHANGELOG.md -- generated notes are a starting point, not the release
task release VERSION=1.3.0 # commit + annotated tag
git push --atomic origin main v1.3.0 # both refs or neither
task release:publish VERSION=1.3.0 # images to GHCR, zip to archie-releases
```
The source repository is private, so releases are published to the public
`samcharles93/archie-releases` repository, where `install.sh` and the
updater find them. `release:publish` builds from a clean worktree of the tag
and needs `gh` and `docker` logged in with write access to that repository
and to GHCR.
Pushing the tag is the release. The deploy workflow builds the images and the
zip, then `archied release publish` signs the release with the
`RELEASE_SIGNING_KEY` Actions secret and pushes it to
`git.catlow.cloud/sam/archie` as `v1.3.0`, `next`, and `stable` unless the
version is a prerelease. `install.sh` and the updater install only what that
key signed.
`release:prepare` emits bare commit subjects; they are a starting point. Rewrite
them to say what a reader gets, per "What the notes are for" above.
@@ -106,7 +106,7 @@ Tagging one version produces, from a single `deploy` run:
rebuilt by the release, so runtime commits merged to `main` do not reach a
running sandbox until the host pulls the new image. The hand-run
`tools/publish.sh` publishes the same two images to
`ghcr.io/samcharles93` instead, which is the registry the binary's default
`git.catlow.cloud/sam` instead, which is the registry the binary's default
`[containers].image` names.
- **`:latest` names the newest release, not the newest commit.** A push to
`main` that is not a release publishes `archied:edge` (mutable, tracking
-7
View File
@@ -319,10 +319,3 @@ tasks:
- tools/release.sh --version {{.VERSION}} --tag
requires:
vars: [VERSION]
release:publish:
desc: Push the images and publish the pushed tag to the public releases repository (VERSION=1.2.0)
cmds:
- tools/publish.sh {{.VERSION}}
requires:
vars: [VERSION]
+3 -1
View File
@@ -12,7 +12,7 @@ import (
)
const commandUsage = "usage: archied [flags] # run the daemon; archied -h lists the flags\n" +
"subcommands: setup, update, status, prbench, playbooks"
"subcommands: setup, update, release, status, prbench, playbooks"
func main() {
os.Exit(run(os.Args[1:]))
@@ -29,6 +29,8 @@ func run(args []string) int {
switch args[0] {
case "update":
return archied.RunUpdate(args[1:], os.Stdout, os.Stderr)
case "release":
return archied.RunRelease(args[1:], os.Stdout, os.Stderr)
case "status":
return archied.RunStatus(args[1:], os.Stdout, os.Stderr)
case "prbench":
+9
View File
@@ -85,3 +85,12 @@ target = "127.0.0.1:9090"
# Store's sweep deletes it. Zero means 90 days.
# [access]
# denial_retention = "2160h"
# Where releases and the extension catalogue come from on a new instance: an
# OCI repository and the base64 ed25519 key it signs with. Blank is the
# official source. Once stored, change them in the dashboard settings.
# [distribution]
# releases = "registry.example/owner/archie"
# releases_key = "base64 public key"
# catalogue = "registry.example/owner/archipelago/catalogue"
# catalogue_key = "base64 public key"
+8 -7
View File
@@ -23,7 +23,7 @@ of its own (`deployments/systemd-user-service.md`).
ships beside `archied` because a Kit run mounts it from that directory. The
agent image is a separate thing, pulled for the containers `archied` spawns
per task: `[containers].image` defaults to
`ghcr.io/samcharles93/archie-agent:latest`, while this archive's `release.json`
`git.catlow.cloud/sam/archie-agent:latest`, while this archive's `release.json`
pins the digest the release published. Set `[containers].registry_auth` when
that reference is on a registry that needs a login -- see
`deployments/systemd-user-service.md#6-pinning-the-archie-agent-image`.
@@ -32,7 +32,7 @@ that reference is on a registry that needs a login -- see
1. Pick a starting configuration. The full set of annotated templates lives
in this repository's `deployments/` directory on GitHub
(<https://github.com/samcharles93/archie-core/tree/main/deployments>) —
(<https://offloaded.dev>) —
`standalone.toml` (no forge, the simplest start),
`single-forge-github.toml` and `docker-nats-stack.toml` cover the common cases. Every one needs
PostgreSQL 18 at `database_url`. Copy whichever fits to:
@@ -65,7 +65,7 @@ that reference is on a registry that needs a login -- see
4. For a persistent, 24/7 deployment instead of a foreground run, follow the
systemd user-unit runbook:
<https://github.com/samcharles93/archie-core/blob/main/deployments/systemd-user-service.md>.
<https://offloaded.dev>.
## Verifying an install
@@ -89,10 +89,11 @@ an install (or an upgrade across v1.30.0) needs its unit as well -- see
### Native self-update
Install on Settings → Status runs `archied update install -version X`, which
downloads the release's Linux/amd64 zip and verifies it against `SHA256SUMS`
before unpacking it. Every host binary in the zip must report `X`. It then pulls
the agent image `release.json` pins by digest, backs up the database to
Install on Settings → Status runs `archied update install -version X` against
the release source in the update settings. It fetches release `vX`, refuses it
unless the source's key signed its statement, and downloads the Linux/amd64 zip
only if it hashes to the digest that statement names. Every host binary in the
zip must report `X`. It then pulls the agent image the statement pins by digest, backs up the database to
`db.pre-update.dump` in the state directory, stages the new binaries beside the
live ones and hands the transaction to `archied update watchdog` in a process of
its own (a transient systemd user unit). Nothing that runs changes until the
+1 -1
View File
@@ -55,7 +55,7 @@ mode = "embedded"
# Pinned to a verified digest -- archie-agent has no published semver tag
# yet (a release publishes :latest and :sha-<commit>). See
# deployments/systemd-user-service.md#6-pinning-the-archie-agent-image.
image = "ghcr.io/samcharles93/archie-agent@sha256:870f1b1357979cfea23fccacb24166ebeb9ab321708ef2c2b218a763ed99b090"
image = "git.catlow.cloud/sam/archie-agent@sha256:<digest>"
pull_policy = "missing"
[budgets]
+6 -6
View File
@@ -134,7 +134,7 @@ Compose service or host `archie-agent` binary is required.
Docker must be installed and reachable by the service user. With no explicit
broker or container settings, Archie defaults `containers.image` to
`ghcr.io/samcharles93/archie-agent:latest`. `single-forge-github.toml` and
`git.catlow.cloud/sam/archie-agent:latest`. `single-forge-github.toml` and
`docker-nats-stack.toml` override that default with a pinned reference
(`standalone.toml` and `dev.toml` leave it) -- see
[Pinning the archie-agent image](#6-pinning-the-archie-agent-image) below. An
@@ -145,7 +145,7 @@ explicit equivalent, pinned the same way, is:
mode = "embedded"
[containers]
image = "ghcr.io/samcharles93/archie-agent@sha256:870f1b1357979cfea23fccacb24166ebeb9ab321708ef2c2b218a763ed99b090"
image = "git.catlow.cloud/sam/archie-agent@sha256:<digest>"
pull_policy = "missing"
```
@@ -177,7 +177,7 @@ prior build. Two paths publish it, to different references:
as `:latest` and `:sha-<commit>` when the commit carries a release tag; an
ordinary push to `main` publishes nothing for the agent image.
- `tools/publish.sh`, run by hand for a release, publishes
`ghcr.io/samcharles93/archie-agent` as `:latest` and `:sha-<commit>`, which is
`git.catlow.cloud/sam/archie-agent` as `:latest` and `:sha-<commit>`, which is
the registry the shipped default above names.
A distribution zip pins whichever one its own release published, by digest, in
@@ -192,19 +192,19 @@ explicitly:
```toml
[containers]
image = "ghcr.io/samcharles93/archie-agent@sha256:870f1b1357979cfea23fccacb24166ebeb9ab321708ef2c2b218a763ed99b090"
image = "git.catlow.cloud/sam/archie-agent@sha256:<digest>"
```
This is the digest `latest` resolved to on 2026-09-05. Digest pinning is the
strongest option for production: it is immutable and independently
verifiable (`docker manifest inspect ghcr.io/samcharles93/archie-agent@sha256:...`),
verifiable (`docker manifest inspect git.catlow.cloud/sam/archie-agent@sha256:...`),
at the cost of being opaque in a config file a human is meant to edit --
resolve a fresh digest deliberately (`docker pull ...:latest` followed by
`docker inspect --format '{{index .RepoDigests 0}}'`) rather than editing this
one in place expecting it to track new releases.
If/when CI starts publishing a semantic-version tag for `archie-agent`
(e.g. `ghcr.io/samcharles93/archie-agent:v1.21.0`), prefer that tag for
(e.g. `git.catlow.cloud/sam/archie-agent:v1.21.0`), prefer that tag for
day-to-day config readability, and reserve digest pinning for deployments
that need byte-for-byte reproducibility guarantees a tag alone cannot give
(tags are mutable pointers; a compromised or accidentally-overwritten tag
+1 -1
View File
@@ -95,7 +95,7 @@ services:
# name by digest. The digest a task container actually runs is pinned in
# the daemon's own [containers] image, so nothing here decides it. See
# deployments/systemd-user-service.md#6-pinning-the-archie-agent-image.
image: ghcr.io/samcharles93/archie-agent:latest
image: git.catlow.cloud/sam/archie-agent:latest
# ── example agent profile image ─────────────────────────────────────
# The agent image plus network lookup tools, for a workflow that names
+1 -1
View File
@@ -3,7 +3,7 @@
# the network-investigator profile package (archipelago: profiles/network-investigator).
#
# Build: docker compose build agent-net
ARG AGENT_IMAGE=ghcr.io/samcharles93/archie-agent:latest
ARG AGENT_IMAGE=git.catlow.cloud/sam/archie-agent:latest
FROM ${AGENT_IMAGE}
RUN apt-get update && apt-get install -y --no-install-recommends \
+73 -27
View File
@@ -1,16 +1,25 @@
#!/usr/bin/env bash
# install.sh - installs Archie Core for the current user.
#
# curl -fsSL https://raw.githubusercontent.com/samcharles93/archie-core/main/install.sh | bash
# curl -fsSL https://offloaded.dev/install.sh | bash
#
# Downloads the latest release, verifies it against SHA256SUMS, installs the
# Downloads a release from its OCI registry, verifies the release key signed
# it and that the zip is the one signed, installs the
# five services to ${XDG_BIN_HOME:-~/.local/bin}, runs `archied setup`, starts
# the bundled PostgreSQL and installs systemd user units.
# Config: ${XDG_CONFIG_HOME:-~/.config}/archie Data: ${XDG_DATA_HOME:-~/.local/share}/archie
set -euo pipefail
REPO="samcharles93/archie-releases"
MODULE="github.com/samcharles93/archie-core"
SOURCE_GIT="https://git.catlow.cloud/sam/archie.git"
# Where releases come from: an OCI repository and the base64 ed25519 key that
# must have signed them. A self-hosted source sets both.
DEFAULT_RELEASES="git.catlow.cloud/sam/archie"
DEFAULT_RELEASE_KEY="gIBqoQMwy+8WXmGfiuHtnUa50W0IqfzkTR6iF5q3dSM="
ARCHIE_RELEASES="${ARCHIE_RELEASES:-${DEFAULT_RELEASES}}"
ARCHIE_RELEASE_KEY="${ARCHIE_RELEASE_KEY:-${DEFAULT_RELEASE_KEY}}"
ARCHIE_CHANNEL="${ARCHIE_CHANNEL:-stable}"
XDG_CONFIG_HOME="${XDG_CONFIG_HOME:-${HOME}/.config}"
XDG_DATA_HOME="${XDG_DATA_HOME:-${HOME}/.local/share}"
@@ -32,7 +41,12 @@ if (: </dev/tty) 2>/dev/null; then INTERACTIVE=true; fi
usage() {
cat <<EOF
Usage: install.sh [options]
or: curl -fsSL https://raw.githubusercontent.com/${REPO}/main/install.sh | bash -s -- [options]
or: curl -fsSL https://offloaded.dev/install.sh | bash -s -- [options]
Environment:
ARCHIE_RELEASES OCI repository to install from (default ${DEFAULT_RELEASES})
ARCHIE_RELEASE_KEY base64 ed25519 key that source signs with
ARCHIE_CHANNEL stable (default) or next
Options:
--version X.Y.Z Install this release instead of the latest
@@ -103,6 +117,50 @@ need() {
done
}
# install_release fetches the release on ARCHIE_CHANNEL (or VERSION) from
# ARCHIE_RELEASES and unpacks it into TMP: the statement must verify against
# ARCHIE_RELEASE_KEY and the zip must hash to the digest the statement names.
install_release() {
local registry="${ARCHIE_RELEASES%%/*}" repo="${ARCHIE_RELEASES#*/}" scheme=https tag="${ARCHIE_CHANNEL}"
case "${registry}" in localhost | localhost:* | 127.* | "[::1]"*) scheme=http ;; esac
[ -n "${VERSION}" ] && tag="v${VERSION#v}"
local base="${scheme}://${registry}/v2/${repo}" auth=()
# Registries hand anonymous readers a pull token named by the challenge.
local challenge
challenge="$(curl -fsS -o /dev/null -D - "${base}/manifests/${tag}" 2>/dev/null | tr -d '\r' | sed -n 's/^[Ww][Ww][Ww]-[Aa]uthenticate: *Bearer *//p')" || true
if [ -n "${challenge}" ]; then
local realm service token
realm="$(sed -n 's/.*realm="\([^"]*\)".*/\1/p' <<<"${challenge}")"
service="$(sed -n 's/.*service="\([^"]*\)".*/\1/p' <<<"${challenge}")"
token="$(curl -fsSL -G "${realm}" --data-urlencode "service=${service}" --data-urlencode "scope=repository:${repo}:pull" | jq -r '.token // .access_token')"
auth=(-H "Authorization: Bearer ${token}")
fi
echo "==> Fetching ${ARCHIE_RELEASES}:${tag}..."
curl -fsSL "${auth[@]}" -H "Accept: application/vnd.oci.image.manifest.v1+json" "${base}/manifests/${tag}" -o "${TMP}/manifest.json"
blob() { curl -fsSL "${auth[@]}" "${base}/blobs/$1" -o "$2" && [ "sha256:$(sha256sum "$2" | cut -d' ' -f1)" = "$1" ]; }
local statement_digest
statement_digest="$(jq -er '.layers[0].digest' "${TMP}/manifest.json")"
blob "${statement_digest}" "${TMP}/statement.json" || { echo "Error: the release statement does not match its digest." >&2; exit 1; }
jq -er '.annotations["dev.archie.release.signature.v1"]' "${TMP}/manifest.json" | base64 -d >"${TMP}/statement.sig"
# An ed25519 public key is its 32 bytes behind a fixed SubjectPublicKeyInfo header.
{ printf '\x30\x2a\x30\x05\x06\x03\x2b\x65\x70\x03\x21\x00'; base64 -d <<<"${ARCHIE_RELEASE_KEY}"; } >"${TMP}/key.der"
openssl pkeyutl -verify -pubin -inkey "${TMP}/key.der" -keyform DER -rawin \
-in "${TMP}/statement.json" -sigfile "${TMP}/statement.sig" >/dev/null 2>&1 ||
{ echo "Error: ${ARCHIE_RELEASES}:${tag} is not signed by the release key." >&2; exit 1; }
local zip_name zip_digest
VERSION="$(jq -er .version "${TMP}/statement.json")"
[ "$(jq -r .platform "${TMP}/statement.json")" = linux-amd64 ] || { echo "Error: the release is not built for linux-amd64." >&2; exit 1; }
[ -z "${VERSION}" ] || [ "${tag}" = "${ARCHIE_CHANNEL}" ] || [ "v${VERSION}" = "${tag}" ] ||
{ echo "Error: ${tag} carries the statement for ${VERSION}." >&2; exit 1; }
zip_name="$(jq -er .zip.name "${TMP}/statement.json")"
zip_digest="$(jq -er .zip.digest "${TMP}/statement.json")"
RELEASE_NAME="${zip_name%.zip}"
case "${RELEASE_NAME}" in archie-core-archied-v*-linux-amd64) ;; *) echo "Error: unexpected release zip ${zip_name}." >&2; exit 1 ;; esac
echo "==> Downloading Archie ${VERSION}..."
blob "${zip_digest}" "${TMP}/${zip_name}" || { echo "Error: ${zip_name} does not match the signed digest." >&2; exit 1; }
unzip -q "${TMP}/${zip_name}" -d "${TMP}"
}
# 1. Prerequisites
echo "==> Checking prerequisites..."
# The installer depends on systemd user units and GNU tools.
@@ -114,7 +172,7 @@ if [ "${FROM_SOURCE}" = true ]; then
need git go jq
else
[ "$(uname -m)" = x86_64 ] || { echo "Error: releases are built for x86_64 only; use --from-source." >&2; exit 1; }
need curl unzip sha256sum
need curl unzip sha256sum openssl jq base64
fi
if command -v docker &>/dev/null; then
echo " [OK] $(docker --version)"
@@ -131,11 +189,11 @@ trap 'rm -rf "${TMP}"' EXIT
if [ "${FROM_SOURCE}" = true ]; then
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]:-$0}")" && pwd)"
if [ -f "${SCRIPT_DIR}/go.mod" ] && grep -q "github.com/${REPO}" "${SCRIPT_DIR}/go.mod"; then
if [ -f "${SCRIPT_DIR}/go.mod" ] && grep -q "${MODULE}" "${SCRIPT_DIR}/go.mod"; then
SRC_DIR="${SCRIPT_DIR}"
else
SRC_DIR="${TMP}/src"
git clone --quiet --depth=1 "https://github.com/${REPO}.git" "${SRC_DIR}"
git clone --quiet --depth=1 "${SOURCE_GIT}" "${SRC_DIR}"
fi
echo "==> Building from ${SRC_DIR}..."
# Selects (and under the default GOTOOLCHAIN=auto, downloads) the toolchain
@@ -150,30 +208,14 @@ if [ "${FROM_SOURCE}" = true ]; then
mkdir -p "${STAGE}"
# A source build is not a release: it says `dev`, and buildType=binary lets
# it self-update. The agentworker stamp only lands in archie-agent.
LDFLAGS="-X github.com/${REPO}/internal/buildinfo.Version=dev -X github.com/${REPO}/internal/buildinfo.Runtime=dev -X github.com/${REPO}/internal/installtype.buildType=binary -X github.com/${REPO}/internal/app/agentworker.version=dev"
LDFLAGS="-X ${MODULE}/internal/buildinfo.Version=dev -X ${MODULE}/internal/buildinfo.Runtime=dev -X ${MODULE}/internal/installtype.buildType=binary -X ${MODULE}/internal/app/agentworker.version=dev"
for cmd in $(jq -er '[.required_topology.units[], .required_topology.binaries[]?] | join(" ")' "${SRC_DIR}/release.json"); do
(cd "${SRC_DIR}" && go build -ldflags "${LDFLAGS}" -o "${STAGE}/${cmd}" "./cmd/${cmd}")
done
cp "${SRC_DIR}/release.json" "${STAGE}/"
else
if [ -z "${VERSION}" ]; then
tag="$(curl -fsSL "https://api.github.com/repos/${REPO}/releases/latest" | sed -n 's/.*"tag_name": *"\([^"]*\)".*/\1/p')"
[ -n "${tag}" ] || { echo "Error: could not find the latest release." >&2; exit 1; }
else
tag="v${VERSION}"
fi
VERSION="${tag##*v}"
name="archie-core-archied-v${VERSION}-linux-amd64"
echo "==> Downloading Archie ${VERSION}..."
base="https://github.com/${REPO}/releases/download/${tag}"
curl -fsSL "${base}/${name}.zip" -o "${TMP}/${name}.zip"
curl -fsSL "${base}/SHA256SUMS" -o "${TMP}/SHA256SUMS"
expected="$(awk -v n="${name}.zip" '$2 == n {print $1}' "${TMP}/SHA256SUMS")"
[[ "${expected}" =~ ^[a-f0-9]{64}$ ]] || { echo "Error: ${name}.zip is not in SHA256SUMS." >&2; exit 1; }
printf '%s %s\n' "${expected}" "${TMP}/${name}.zip" | sha256sum --check --status ||
{ echo "Error: checksum mismatch for ${name}.zip." >&2; exit 1; }
unzip -q "${TMP}/${name}.zip" -d "${TMP}"
STAGE="${TMP}/${name}"
install_release
STAGE="${TMP}/${RELEASE_NAME}"
fi
TOPOLOGY_UNITS="$(tr -d '\n' <"${STAGE}/release.json" | sed -n 's/.*"units": *\[\([^]]*\)\].*/\1/p' | tr -d '" ' | tr , ' ')"
@@ -202,6 +244,10 @@ if [ ! -f "${ARCHIE_CONFIG_DIR}/config.toml" ]; then
echo "ERROR: archied setup could not generate ${ARCHIE_CONFIG_DIR}/config.toml" >&2
exit 1
fi
if [ "${ARCHIE_RELEASES}" != "${DEFAULT_RELEASES}" ] || [ "${ARCHIE_RELEASE_KEY}" != "${DEFAULT_RELEASE_KEY}" ]; then
# The instance keeps updating from the source it was installed from.
printf '\n[distribution]\nreleases = "%s"\nreleases_key = "%s"\n' "${ARCHIE_RELEASES}" "${ARCHIE_RELEASE_KEY}" >>"${ARCHIE_CONFIG_DIR}/config.toml"
fi
if [ -n "${PORT_BASE}" ]; then
# The generated config names the default addresses; move them to this
# instance's ports and pin the services the template leaves commented.
@@ -465,7 +511,7 @@ if [ "${SERVICE_INSTALLED}" = true ] && [ "${AUTO_START}" = true ]; then
echo " - Service status : systemctl --user status ${INSTANCE_UNITS}"
echo " - Restart daemon : systemctl --user restart ${U_DAEMON}"
else
echo "Manual Startup: start the five services as in https://github.com/${REPO}/blob/main/deployments/README.md"
echo "Manual Startup: start the five services as in https://offloaded.dev"
fi
echo ""
SETUP_HINT="archied setup"
+112
View File
@@ -0,0 +1,112 @@
package archied
import (
"context"
"crypto/ed25519"
"encoding/base64"
"flag"
"fmt"
"io"
"os"
"strings"
"github.com/samcharles93/archie-core/internal/domain/distribution"
"github.com/samcharles93/archie-core/internal/releaseupdate"
)
const releaseUsage = `usage: archied release <command>
keygen -seed FILE write a new signing key's seed to FILE and print its public key
publish sign a release and push it to a release source`
// RunRelease is the publisher's side of a release source: it makes the key
// a source signs with and pushes signed releases to it.
func RunRelease(args []string, out, stderr io.Writer) int {
if len(args) == 0 {
fmt.Fprintln(stderr, releaseUsage)
return 2
}
switch args[0] {
case "keygen":
return runReleaseKeygen(args[1:], out, stderr)
case "publish":
return runReleasePublish(args[1:], out, stderr)
default:
fmt.Fprintln(stderr, releaseUsage)
return 2
}
}
// runReleaseKeygen writes the seed to a new file only, never to output, so
// the private half is not left in a terminal or CI log.
func runReleaseKeygen(args []string, out, stderr io.Writer) int {
flags := flag.NewFlagSet("release keygen", flag.ContinueOnError)
flags.SetOutput(stderr)
seedPath := flags.String("seed", "", "file to create with the base64 private key seed")
if err := flags.Parse(args); err != nil {
return 2
}
if *seedPath == "" {
fmt.Fprintln(stderr, "release keygen: -seed is required")
return 2
}
public, private, err := ed25519.GenerateKey(nil)
if err != nil {
fmt.Fprintln(stderr, err)
return 1
}
file, err := os.OpenFile(*seedPath, os.O_CREATE|os.O_EXCL|os.O_WRONLY, 0o600)
if err != nil {
fmt.Fprintln(stderr, err)
return 1
}
_, err = fmt.Fprintln(file, base64.StdEncoding.EncodeToString(private.Seed()))
if closeErr := file.Close(); err == nil {
err = closeErr
}
if err != nil {
fmt.Fprintln(stderr, err)
return 1
}
fmt.Fprintln(out, distribution.EncodeKey(public))
return 0
}
func runReleasePublish(args []string, out, stderr io.Writer) int {
flags := flag.NewFlagSet("release publish", flag.ContinueOnError)
flags.SetOutput(stderr)
var req releaseupdate.PublishRequest
flags.StringVar(&req.Repository, "source", "", "OCI repository to publish to")
seedEnv := flags.String("seed-env", "ARCHIE_RELEASE_SEED", "environment variable holding the base64 private key seed")
flags.StringVar(&req.Version, "version", "", "release version")
flags.BoolVar(&req.Prerelease, "prerelease", false, "publish to next only")
flags.StringVar(&req.ZipPath, "zip", "", "distribution zip")
flags.StringVar(&req.AgentImage, "agent-image", "", "agent image pinned by digest")
flags.StringVar(&req.Changelog, "changelog", "", "URL of the release notes")
if err := flags.Parse(args); err != nil {
return 2
}
if req.Repository == "" || req.Version == "" || req.ZipPath == "" || req.AgentImage == "" {
fmt.Fprintln(stderr, "release publish: -source, -version, -zip and -agent-image are required")
return 2
}
seed, err := base64.StdEncoding.DecodeString(strings.TrimSpace(os.Getenv(*seedEnv)))
if err != nil || len(seed) != ed25519.SeedSize {
fmt.Fprintf(stderr, "release publish: %s must hold a base64 ed25519 seed\n", *seedEnv)
return 2
}
req.Signer = ed25519.NewKeyFromSeed(seed)
public := distribution.EncodeKey(ed25519.PublicKey(req.Signer[ed25519.SeedSize:]))
req.Channels = []string{"next"}
if !req.Prerelease {
req.Channels = append(req.Channels, "stable")
}
statement, err := releaseupdate.Publish(context.Background(), req)
if err != nil {
fmt.Fprintln(stderr, err)
return 1
}
fmt.Fprintf(out, "published %s to %s (%s) with key %s\n", statement.Version, req.Repository,
strings.Join(req.Channels, ", "), public)
return 0
}
+18 -2
View File
@@ -20,6 +20,7 @@ import (
"golang.org/x/sys/unix"
"github.com/samcharles93/archie-core/internal/app/servicekit"
"github.com/samcharles93/archie-core/internal/domain/distribution"
"github.com/samcharles93/archie-core/internal/infrastructure/configuration"
"github.com/samcharles93/archie-core/internal/releaseupdate"
)
@@ -95,7 +96,7 @@ func runUpdateInstall(args []string, out, stderr io.Writer) int {
flags.SetOutput(stderr)
configPath := flags.String("config", configuration.DefaultConfigPath(), "bootstrap config file")
version := flags.String("version", "", "release version to install")
releaseURL := flags.String("release-url", releaseupdate.ReleaseURL, "base URL of the release downloads")
releases := sourceFlags(flags)
var meta releaseupdate.InstallMeta
flags.StringVar(&meta.ReportPath, "report", "", "where the watchdog leaves the outcome")
flags.StringVar(&meta.Channel, "channel", "", "channel the outcome is reported on")
@@ -104,6 +105,11 @@ func runUpdateInstall(args []string, out, stderr io.Writer) int {
if err := flags.Parse(args); err != nil {
return 2
}
source, err := releases()
if err != nil {
fmt.Fprintln(stderr, err)
return 2
}
self, err := os.Executable()
if err != nil {
fmt.Fprintln(stderr, err)
@@ -116,7 +122,7 @@ func runUpdateInstall(args []string, out, stderr io.Writer) int {
return 1
}
result, err := tx.Prepare(ctx, releaseupdate.PrepareRequest{
Version: *version, ReleaseURL: *releaseURL, Meta: meta,
Version: *version, Source: source, Meta: meta,
CheckTopology: func(ctx context.Context, required releaseupdate.Topology, managed bool) error {
return checkUpdateTopology(ctx, *configPath, required, managed, out)
},
@@ -380,3 +386,13 @@ func observedUnits(output []byte, err error) map[string]bool {
}
return units
}
// sourceFlags registers -source and -key, the release distribution source; a
// blank pair is the official releases.
func sourceFlags(flags *flag.FlagSet) func() (distribution.Source, error) {
reference := flags.String("source", "", "OCI repository releases are published to (default: the official releases)")
key := flags.String("key", "", "base64 ed25519 public key the source signs releases with")
return func() (distribution.Source, error) {
return distribution.Parse(*reference, *key, distribution.DefaultReleases())
}
}
+10 -1
View File
@@ -8,6 +8,7 @@ import (
"io"
"os"
"github.com/samcharles93/archie-core/internal/domain/distribution"
"github.com/samcharles93/archie-core/internal/releaseupdate"
)
@@ -18,10 +19,18 @@ func runUpdateCheck(args []string, out, stderr io.Writer) int {
flags.SetOutput(stderr)
channel := flags.String("channel", envOr("ARCHIE_UPDATE_RELEASE_CHANNEL", "stable"), "stable, next, or exact-pin")
pin := flags.String("pin", os.Getenv("ARCHIE_UPDATE_PIN"), "exact release version")
releases := sourceFlags(flags)
if err := flags.Parse(args); err != nil {
return 2
}
catalog := &releaseupdate.GitHubCatalog{Channel: func(context.Context) (string, string, error) { return *channel, *pin, nil }}
source, err := releases()
if err != nil {
fmt.Fprintln(stderr, err)
return 2
}
catalog := &releaseupdate.RegistryCatalog{Settings: func(context.Context) (distribution.Source, string, string, error) {
return source, *channel, *pin, nil
}}
snapshot, err := catalog.Check(context.Background())
if err != nil {
fmt.Fprintln(stderr, err)
+51
View File
@@ -0,0 +1,51 @@
package statestore
import (
"context"
"encoding/json"
"errors"
"fmt"
"github.com/samcharles93/archie-core/internal/domain/distribution"
"github.com/samcharles93/archie-core/internal/domain/storecontract"
domain "github.com/samcharles93/archie-core/internal/domain/storepkg"
"github.com/samcharles93/archie-core/internal/infrastructure/controlplanerpc"
"github.com/samcharles93/archie-core/internal/infrastructure/controlplanestore"
registry "github.com/samcharles93/archie-core/internal/infrastructure/storepkg"
)
// settingsCatalogue fetches the catalogue from the source the extension
// settings name, read on every fetch so an edit applies without a restart.
type settingsCatalogue struct {
resources auditRetentionResources
}
// liveCatalogue is the catalogue the dashboard lists; a store that cannot
// read resources lists the official catalogue.
func (b *server) liveCatalogue() domain.CatalogueSource {
resources, ok := b.st.(auditRetentionResources)
if !ok {
return registry.Catalogue(distribution.DefaultCatalogue())
}
return settingsCatalogue{resources: resources}
}
func (c settingsCatalogue) Fetch(ctx context.Context) (domain.Catalogue, error) {
var settings controlplanerpc.ExtensionSettings
resource, err := c.resources.Resource(ctx, storecontract.DefaultOrgID, controlplanestore.ExtensionSettingsKind)
switch {
case errors.Is(err, storecontract.ErrResourceNotFound):
case err != nil:
return domain.Catalogue{}, fmt.Errorf("read extension settings: %w", err)
}
if len(resource.Value) > 0 {
if err := json.Unmarshal(resource.Value, &settings); err != nil {
return domain.Catalogue{}, fmt.Errorf("decode extension settings: %w", err)
}
}
source, err := settings.Catalogue()
if err != nil {
return domain.Catalogue{}, err
}
return registry.Catalogue(source).Fetch(ctx)
}
+1 -1
View File
@@ -269,7 +269,7 @@ func (b *server) stateStoreDeps(grants *staterpc.TaskGrants) staterpc.Deps {
packages := storepkg.Service{
Registry: registry.OCIRegistry{},
Store: postgres.NewInstalledPackages(b.pg),
Catalogue: registry.DefaultCatalogue(),
Catalogue: b.liveCatalogue(),
}
// A projection failure degrades the org resources a package would have
// contributed to, the way a group above degrades: the State Store boots
+14 -1
View File
@@ -414,6 +414,9 @@ type Config struct {
Containers ContainerConfig `toml:"containers" yaml:"containers"`
Chat ChatConfig `toml:"chat" yaml:"chat"`
Capture CaptureConfig `toml:"capture" yaml:"capture"`
// Distribution seeds where releases and the extension catalogue come
// from; once stored, the dashboard's settings own them.
Distribution Distribution `toml:"distribution" yaml:"distribution"`
Access AccessConfig `toml:"access" yaml:"access"`
Bindings BindingsConfig `toml:"bindings" yaml:"bindings"`
@@ -690,7 +693,7 @@ type BindingsConfig struct {
// ContainerConfig configures Docker sandbox execution of archie-agent.
type ContainerConfig struct {
// Image is the Docker image to run (e.g. "ghcr.io/sam/archie-agent:latest").
// Image is the Docker image to run (e.g. "git.catlow.cloud/sam/archie-agent:latest").
Image string `toml:"image" yaml:"image"`
// MaxConcurrency limits simultaneous daemon tasks and containers.
// Tasks from the same repository remain serialized. 0 = no limit.
@@ -1008,6 +1011,16 @@ func (h Health) URL() string {
return "http://" + net.JoinHostPort(host, port)
}
// Distribution names the signed artifact sources a new instance starts with.
// Each source is an OCI repository with the base64 ed25519 key it signs
// with; blank pairs are the official sources.
type Distribution struct {
Releases string `toml:"releases" yaml:"releases"`
ReleasesKey string `toml:"releases_key" yaml:"releases_key"`
Catalogue string `toml:"catalogue" yaml:"catalogue"`
CatalogueKey string `toml:"catalogue_key" yaml:"catalogue_key"`
}
type Web struct {
// Listen is the dashboard address; "off" disables the web UI.
// Bind localhost (or a LAN/tailnet address) -- there is no auth.
+89
View File
@@ -0,0 +1,89 @@
// Package distribution names where an instance takes signed artifacts from:
// its releases and its extension catalogue.
package distribution
import (
"crypto/ed25519"
"encoding/base64"
"errors"
"fmt"
"strings"
)
// Source is an OCI repository and the public key that must have signed what
// it serves. The key, not the registry, is what makes an artifact trusted, so
// a source is never valid without one.
type Source struct {
Reference string
Key ed25519.PublicKey
}
// Parse builds a source from its stored form. A blank reference and key mean
// fallback; setting only one of them is refused, since a reference without
// its key, or a key for the default reference, is a misconfiguration.
func Parse(reference, key string, fallback Source) (Source, error) {
reference, key = strings.TrimSpace(reference), strings.TrimSpace(key)
if reference == "" && key == "" {
return fallback, nil
}
if reference == "" || key == "" {
return Source{}, errors.New("a distribution source needs both a reference and a key")
}
if !repository(reference) {
return Source{}, fmt.Errorf("distribution source %q must be an OCI repository such as registry.example/owner/name", reference)
}
decoded, err := ParseKey(key)
if err != nil {
return Source{}, err
}
return Source{Reference: reference, Key: decoded}, nil
}
// ParseKey decodes a base64 ed25519 public key.
func ParseKey(key string) (ed25519.PublicKey, error) {
decoded, err := base64.StdEncoding.DecodeString(strings.TrimSpace(key))
if err != nil || len(decoded) != ed25519.PublicKeySize {
return nil, errors.New("distribution key must be a base64 ed25519 public key")
}
return decoded, nil
}
// EncodeKey is the stored form of a public key.
func EncodeKey(key ed25519.PublicKey) string {
return base64.StdEncoding.EncodeToString(key)
}
// repository accepts registry/path with an optional registry port and no tag
// or digest: a source names a repository, and its tags are the channels.
func repository(reference string) bool {
host, path, ok := strings.Cut(reference, "/")
return ok && host != "" && path != "" &&
strings.Count(host, ":") <= 1 && !strings.ContainsAny(path, ":@") &&
!strings.ContainsAny(reference, " \t\n") && !strings.Contains(reference, "//")
}
// mustKey decodes a built-in key; a malformed one is a build defect.
func mustKey(encoded string) ed25519.PublicKey {
key, err := ParseKey(encoded)
if err != nil {
panic("distribution: invalid built-in key")
}
return key
}
// DefaultCatalogue is the archipelago marketplace an instance trusts until an
// operator names another.
func DefaultCatalogue() Source {
return Source{
Reference: "git.catlow.cloud/sam/archipelago/catalogue",
Key: mustKey("YL4nvn07uq/5mXZOt9cEmLYu/kB6C2+MLTnbHyoX4Pw="),
}
}
// DefaultReleases is where Archie's own releases are published.
func DefaultReleases() Source {
return Source{
Reference: "git.catlow.cloud/sam/archie",
Key: mustKey("gIBqoQMwy+8WXmGfiuHtnUa50W0IqfzkTR6iF5q3dSM="),
}
}
@@ -24,7 +24,7 @@ const (
defaultContainerUptime = 5 * time.Hour
defaultVolumeTTL = 72 * time.Hour
defaultPullPolicy = "missing"
defaultContainerImage = "ghcr.io/samcharles93/archie-agent:latest"
defaultContainerImage = "git.catlow.cloud/sam/archie-agent:latest"
// defaultMaxResultChars bounds one tool result.
defaultMaxResultChars = 50_000
@@ -1,5 +1,7 @@
package controlplanerpc
import "github.com/samcharles93/archie-core/internal/domain/distribution"
// ExtensionSettingsKind is the control-plane resource kind holding which
// installed extensions are enabled and how each is configured.
const ExtensionSettingsKind = "extension-settings"
@@ -7,6 +9,13 @@ const ExtensionSettingsKind = "extension-settings"
// ExtensionSettings is the extension-settings resource document.
type ExtensionSettings struct {
Extensions []ExtensionSetting `json:"extensions"`
CatalogueSource string `json:"catalogue_source,omitempty" title:"Catalogue source" doc:"OCI repository the package catalogue is published to, read at its latest tag. Blank uses the official catalogue."`
CatalogueKey string `json:"catalogue_key,omitempty" title:"Catalogue signing key" doc:"Base64 ed25519 public key the catalogue source signs with. Required with a source."`
}
// Catalogue is the distribution source the settings name.
func (s ExtensionSettings) Catalogue() (distribution.Source, error) {
return distribution.Parse(s.CatalogueSource, s.CatalogueKey, distribution.DefaultCatalogue())
}
// ExtensionSetting enables or disables one installed extension by package name.
@@ -8,6 +8,7 @@ import (
"time"
"github.com/samcharles93/archie-core/internal/config"
"github.com/samcharles93/archie-core/internal/domain/distribution"
"github.com/samcharles93/archie-core/internal/installtype"
"github.com/samcharles93/archie-core/internal/releaseupdate"
)
@@ -118,28 +119,49 @@ const UpdateSettingsKind = "update-settings"
type UpdateSettings struct {
Channel string `json:"channel" title:"Release channel" doc:"stable, next (includes prereleases) or exact-pin."`
Pin string `json:"pin" title:"Pinned version" doc:"The version exact-pin installs."`
Source string `json:"source,omitempty" title:"Release source" doc:"OCI repository releases are published to, such as registry.example/owner/archie. Blank uses the official releases."`
Key string `json:"key,omitempty" title:"Release signing key" doc:"Base64 ed25519 public key the release source signs with. Required with a source."`
}
// UpdateService checks GitHub on the stored release channel and installs
// through the archied beside this binary. statePath keeps each recipient's
// deferrals.
// Releases is the distribution source the settings name.
func (s UpdateSettings) Releases() (distribution.Source, error) {
return distribution.Parse(s.Source, s.Key, distribution.DefaultReleases())
}
// UpdateService checks the stored release source on the stored channel and
// installs through the archied beside this binary from the same source.
// statePath keeps each recipient's deferrals.
func UpdateService(reader ResourceReader, statePath string) *releaseupdate.Service {
settings := updateSettings(reader)
return &releaseupdate.Service{
Catalog: &releaseupdate.GitHubCatalog{Channel: updateChannel(reader)},
Installer: releaseupdate.ArchiedInstaller{},
Catalog: &releaseupdate.RegistryCatalog{Settings: func(ctx context.Context) (distribution.Source, string, string, error) {
doc, err := settings(ctx)
if err != nil {
return distribution.Source{}, "", "", err
}
source, err := doc.Releases()
return source, doc.Channel, doc.Pin, err
}},
Installer: releaseupdate.ArchiedInstaller{Source: func(ctx context.Context) (distribution.Source, error) {
doc, err := settings(ctx)
if err != nil {
return distribution.Source{}, err
}
return doc.Releases()
}},
StatePath: statePath,
InstallType: installtype.Type(),
}
}
// updateChannel reads the update-settings resource. An unstored kind is the
// stable channel.
func updateChannel(reader ResourceReader) func(context.Context) (string, string, error) {
return func(ctx context.Context) (string, string, error) {
// updateSettings reads the update-settings resource. An unstored kind is the
// stable channel from the official releases.
func updateSettings(reader ResourceReader) func(context.Context) (UpdateSettings, error) {
return func(ctx context.Context) (UpdateSettings, error) {
settings := UpdateSettings{Channel: "stable"}
_, _, err := reader.Query(ctx, UpdateSettingsKind, func(value []byte) error {
return json.Unmarshal(value, &settings)
})
return settings.Channel, settings.Pin, err
return settings, err
}
}
@@ -60,11 +60,15 @@ func operationalDefinitions() []Definition {
}, Validate: validatePluginSettings},
// Live: each process that supervises extensions re-reads this on its sync
// tick, then starts, restarts or stops what changed.
{Kind: ExtensionSettingsKind, Title: "Extension settings", ApplyMode: "live", Document: controlplanerpc.ExtensionSettings{}, Seed: func(config.Config) any { return controlplanerpc.ExtensionSettings{} }, Validate: validateExtensions},
{Kind: ExtensionSettingsKind, Title: "Extension settings", ApplyMode: "live", Document: controlplanerpc.ExtensionSettings{}, Seed: func(cfg config.Config) any {
return controlplanerpc.ExtensionSettings{CatalogueSource: cfg.Distribution.Catalogue, CatalogueKey: cfg.Distribution.CatalogueKey}
}, Validate: validateExtensions},
// Live: the container pool reads these on every acquire and the dispatcher
// is resized on publish.
// Live: every check and install reads the channel afresh.
{Kind: UpdateSettingsKind, Title: "Update settings", ApplyMode: "live", Document: controlplanerpc.UpdateSettings{}, Seed: func(config.Config) any { return controlplanerpc.UpdateSettings{Channel: "stable"} }, Validate: validateUpdateSettings},
{Kind: UpdateSettingsKind, Title: "Update settings", ApplyMode: "live", Document: controlplanerpc.UpdateSettings{}, Seed: func(cfg config.Config) any {
return controlplanerpc.UpdateSettings{Channel: "stable", Source: cfg.Distribution.Releases, Key: cfg.Distribution.ReleasesKey}
}, Validate: validateUpdateSettings},
{Kind: ContainerRuntimePoliciesKind, Title: "Container runtime policies", ApplyMode: "live", Document: containerRuntimePolicies{}, Seed: seedContainerPolicies, Validate: validateContainers, Normalize: normalizeContainerPolicies},
// Live: the retention sweep reads the document on every tick, so a
// change applies to the next sweep without a restart.
@@ -89,7 +93,8 @@ func validateExtensions(input []byte) error {
}
seen[setting.Name] = struct{}{}
}
return nil
_, err := doc.Catalogue()
return err
})
}
@@ -176,6 +181,9 @@ func ApplyOrgScheduling(cfg config.Config, stored []byte) (config.Config, error)
// validateUpdateSettings refuses a channel the update scripts cannot select.
func validateUpdateSettings(input []byte) error {
return validateAs(input, func(doc controlplanerpc.UpdateSettings) error {
if _, err := doc.Releases(); err != nil {
return err
}
return releaseupdate.ValidateChannel(doc.Channel, doc.Pin)
})
}
@@ -0,0 +1,86 @@
package signedoci
import (
"context"
"crypto/sha256"
"encoding/hex"
"encoding/json"
"errors"
"fmt"
"io/fs"
"os"
"path/filepath"
"sync"
"time"
"github.com/samcharles93/archie-core/internal/fsutil"
"github.com/samcharles93/archie-core/internal/infrastructure/configuration"
)
// FileFreshness is the durable freshness mark of accepted issue times, kept as one small JSON file
// written atomically so a crash never leaves a half-written mark. A missing
// file is no mark yet; a file that exists but does not parse is an error, so
// a corrupt mark fails closed instead of opening a replay window.
type FileFreshness struct {
Path string
mu sync.Mutex
}
func (f *FileFreshness) Newest(context.Context) (time.Time, error) {
f.mu.Lock()
defer f.mu.Unlock()
return f.read()
}
func (f *FileFreshness) Accept(_ context.Context, issuedAt time.Time) error {
f.mu.Lock()
defer f.mu.Unlock()
current, err := f.read()
if err != nil {
return err
}
if !issuedAt.After(current) {
return nil
}
encoded, err := json.Marshal(struct {
IssuedAt time.Time `json:"issued_at"`
}{IssuedAt: issuedAt.UTC()})
if err != nil {
return err
}
return fsutil.WriteFileAtomic(f.Path, encoded, 0o600)
}
// read returns the mark, or the zero time when nothing has been
// accepted yet.
func (f *FileFreshness) read() (time.Time, error) {
raw, err := os.ReadFile(f.Path)
if errors.Is(err, fs.ErrNotExist) {
return time.Time{}, nil
}
if err != nil {
return time.Time{}, err
}
var mark struct {
IssuedAt time.Time `json:"issued_at"`
}
if err := json.Unmarshal(raw, &mark); err != nil {
return time.Time{}, fmt.Errorf("freshness mark %s: %w", f.Path, err)
}
return mark.IssuedAt, nil
}
// MarkPath is where the freshness mark for one kind of artifact from one
// source lives: $XDG_STATE_HOME/<instance>/<kind>/<key>.json, the state home
// the update journal uses too. The key names the reference, so two sources
// never share a mark.
func MarkPath(kind, reference string) string {
dir := os.Getenv("XDG_STATE_HOME")
if dir == "" {
home, _ := os.UserHomeDir()
dir = filepath.Join(home, ".local", "state")
}
sum := sha256.Sum256([]byte(reference))
return filepath.Join(dir, configuration.InstanceName(), kind, hex.EncodeToString(sum[:8])+".json")
}
@@ -0,0 +1,47 @@
package signedoci
import (
"os"
"path/filepath"
"testing"
"time"
)
// The freshness mark never goes backwards, and a mark that exists but does
// not parse fails closed rather than opening a replay window.
func TestFileFreshness(t *testing.T) {
path := filepath.Join(t.TempDir(), "freshness.json")
mark := &FileFreshness{Path: path}
jan1 := time.Date(2026, 1, 1, 0, 0, 0, 0, time.UTC)
jan2 := time.Date(2026, 1, 2, 0, 0, 0, 0, time.UTC)
newest, err := mark.Newest(t.Context())
if err != nil {
t.Fatalf("Newest with no file: %v", err)
}
if !newest.IsZero() {
t.Fatalf("Newest with no file = %s; want the zero time", newest.Format(time.RFC3339))
}
if err := mark.Accept(t.Context(), jan1); err != nil {
t.Fatalf("Accept jan1: %v", err)
}
if err := mark.Accept(t.Context(), jan2); err != nil {
t.Fatalf("Accept jan2: %v", err)
}
if err := mark.Accept(t.Context(), jan1); err != nil {
t.Fatalf("Accept jan1 again: %v", err)
}
if newest, err = mark.Newest(t.Context()); err != nil {
t.Fatalf("Newest: %v", err)
}
if !newest.Equal(jan2) {
t.Fatalf("Newest = %s; want %s", newest.Format(time.RFC3339), jan2.Format(time.RFC3339))
}
if err := os.WriteFile(path, []byte("{"), 0o600); err != nil {
t.Fatal(err)
}
if _, err := mark.Newest(t.Context()); err == nil {
t.Fatal("Newest over a corrupt mark: err = nil, want an error")
}
}
@@ -0,0 +1,154 @@
// Package signedoci fetches OCI artifacts whose first layer is a statement
// signed by a distribution source's key. The extension catalogue and releases
// are both published this way.
package signedoci
import (
"context"
"crypto/ed25519"
"crypto/sha256"
"encoding/base64"
"encoding/hex"
"encoding/json"
"errors"
"fmt"
"io"
"net"
"strings"
ocispec "github.com/opencontainers/image-spec/specs-go/v1"
"oras.land/oras-go/v2/registry/remote"
"github.com/samcharles93/archie-core/internal/domain/distribution"
)
const maxManifestBytes = 256 << 10
// Artifact is a fetched manifest whose statement layer verified against the
// source's key. Its other layers are read through Blob, which checks each
// against the digest the manifest names.
type Artifact struct {
Manifest ocispec.Manifest
Statement []byte
repository *remote.Repository
}
// Fetch resolves tag in the source's repository and verifies the statement
// layer: annotation on the manifest must be the key's signature over the
// statement bytes. maxStatement bounds the statement read.
func Fetch(ctx context.Context, source distribution.Source, tag, annotation string, maxStatement int) (Artifact, error) {
if len(source.Key) != ed25519.PublicKeySize {
return Artifact{}, errors.New("no valid signing key")
}
reference := source.Reference + ":" + tag
repository, err := remote.NewRepository(reference)
if err != nil {
return Artifact{}, err
}
repository.PlainHTTP = Loopback(reference)
_, stream, err := repository.FetchReference(ctx, reference)
if err != nil {
return Artifact{}, fmt.Errorf("%s: %w", reference, err)
}
raw, err := readBounded(stream, maxManifestBytes)
if err != nil {
return Artifact{}, err
}
var manifest ocispec.Manifest
if err := json.Unmarshal(raw, &manifest); err != nil {
return Artifact{}, fmt.Errorf("%s manifest: %w", reference, err)
}
if len(manifest.Layers) == 0 {
return Artifact{}, fmt.Errorf("%s has no statement layer", reference)
}
layer, err := repository.Fetch(ctx, manifest.Layers[0])
if err != nil {
return Artifact{}, fmt.Errorf("%s statement: %w", reference, err)
}
statement, err := readBounded(layer, maxStatement)
if err != nil {
return Artifact{}, err
}
if err := Verify(manifest, statement, source.Key, annotation); err != nil {
return Artifact{}, fmt.Errorf("%s: %w", reference, err)
}
return Artifact{Manifest: manifest, Statement: statement, repository: repository}, nil
}
// Verify checks statement is the first layer the manifest names and that key
// signed exactly those bytes under annotation.
func Verify(manifest ocispec.Manifest, statement []byte, key ed25519.PublicKey, annotation string) error {
if len(manifest.Layers) == 0 || Digest(statement) != manifest.Layers[0].Digest.String() {
return errors.New("statement does not match its manifest digest")
}
signature, err := base64.StdEncoding.DecodeString(manifest.Annotations[annotation])
if err != nil || !ed25519.Verify(key, statement, signature) {
return errors.New("signature does not verify")
}
return nil
}
// Blob copies the layer named digest to out, refusing one the manifest does
// not list or whose bytes do not hash to it.
func (a Artifact) Blob(ctx context.Context, digest string, out io.Writer, limit int64) error {
for _, layer := range a.Manifest.Layers {
if layer.Digest.String() != digest {
continue
}
if layer.Size > limit {
return fmt.Errorf("layer %s exceeds the size limit", digest)
}
stream, err := a.repository.Fetch(ctx, layer)
if err != nil {
return err
}
defer func() { _ = stream.Close() }()
hash := sha256.New()
n, err := io.Copy(io.MultiWriter(out, hash), io.LimitReader(stream, limit+1))
if err != nil {
return err
}
if n > limit {
return fmt.Errorf("layer %s exceeds the size limit", digest)
}
if "sha256:"+hex.EncodeToString(hash.Sum(nil)) != digest {
return fmt.Errorf("layer %s does not match its digest", digest)
}
return nil
}
return fmt.Errorf("artifact has no layer %s", digest)
}
// Digest is the OCI content digest of data.
func Digest(data []byte) string {
sum := sha256.Sum256(data)
return "sha256:" + hex.EncodeToString(sum[:])
}
// Loopback reports whether reference names a registry on this host, the
// only one reached over plain HTTP. The signature, not the transport, is
// what makes an artifact trustworthy.
func Loopback(reference string) bool {
host, _, ok := strings.Cut(reference, "/")
if !ok {
return false
}
if name, _, err := net.SplitHostPort(host); err == nil {
host = name
}
host = strings.Trim(host, "[]")
return host == "localhost" || net.ParseIP(host).IsLoopback()
}
func readBounded(stream io.ReadCloser, limit int) ([]byte, error) {
defer func() { _ = stream.Close() }()
data, err := io.ReadAll(io.LimitReader(stream, int64(limit)+1))
if err != nil {
return nil, err
}
if len(data) > limit {
return nil, errors.New("artifact content exceeds the size limit")
}
return data, nil
}
+24 -140
View File
@@ -4,24 +4,14 @@ import (
"bytes"
"context"
"crypto/ed25519"
"crypto/sha256"
"encoding/base64"
"encoding/hex"
"encoding/json"
"errors"
"fmt"
"io/fs"
"os"
"path/filepath"
"sync"
"time"
ocispec "github.com/opencontainers/image-spec/specs-go/v1"
"oras.land/oras-go/v2/registry/remote"
"github.com/samcharles93/archie-core/internal/domain/distribution"
domain "github.com/samcharles93/archie-core/internal/domain/storepkg"
"github.com/samcharles93/archie-core/internal/fsutil"
"github.com/samcharles93/archie-core/internal/infrastructure/configuration"
"github.com/samcharles93/archie-core/internal/infrastructure/signedoci"
)
// SignatureAnnotation carries the base64 ed25519 signature over the
@@ -30,84 +20,48 @@ const SignatureAnnotation = "dev.archie.catalogue.signature.v1"
const maxCatalogueBytes = 1 << 20
// DefaultCatalogue is the archipelago marketplace every instance trusts: its
// reference, the public half of the key that signs it, and the durable
// freshness mark that refuses a replayed older catalogue.
func DefaultCatalogue() domain.CatalogueSource {
source := OCICatalogue{
Reference: "ghcr.io/samcharles93/archipelago/catalogue:latest",
Key: mustKey("YL4nvn07uq/5mXZOt9cEmLYu/kB6C2+MLTnbHyoX4Pw="),
}
// catalogueTag is the tag a catalogue source publishes its current catalogue
// under.
const catalogueTag = "latest"
// Catalogue fetches the catalogue a source publishes and refuses a replayed
// older one, keeping the freshness mark per source.
func Catalogue(source distribution.Source) domain.CatalogueSource {
return domain.FreshCatalogueSource{
Source: source,
Freshness: &FileFreshness{Path: defaultFreshnessPath(source.Reference)},
Source: OCICatalogue{Source: source},
Freshness: &signedoci.FileFreshness{Path: signedoci.MarkPath("catalogue", source.Reference)},
}
}
func mustKey(encoded string) ed25519.PublicKey {
key, err := base64.StdEncoding.DecodeString(encoded)
if err != nil || len(key) != ed25519.PublicKeySize {
panic("storepkg: invalid catalogue key")
}
return key
}
// OCICatalogue fetches a catalogue artifact and verifies it against Key: a
// one-layer OCI manifest whose layer is the catalogue JSON and whose
// annotation signs that layer. The reference may be a tag; the signature,
// not the tag, is what makes the list trustworthy.
// OCICatalogue fetches a catalogue artifact and verifies it against the
// source's key: an OCI manifest whose first layer is the catalogue JSON and
// whose annotation signs that layer. The tag is mutable; the signature is
// what makes the list trustworthy.
type OCICatalogue struct {
Reference string
Key ed25519.PublicKey
Source distribution.Source
}
var _ domain.CatalogueSource = OCICatalogue{}
func (c OCICatalogue) Fetch(ctx context.Context) (domain.Catalogue, error) {
if len(c.Key) != ed25519.PublicKeySize {
return domain.Catalogue{}, errors.New("catalogue: no valid signing key")
}
repository, err := remote.NewRepository(c.Reference)
if err != nil {
return domain.Catalogue{}, err
}
repository.PlainHTTP = loopbackReference(c.Reference)
_, stream, err := repository.FetchReference(ctx, c.Reference)
artifact, err := signedoci.Fetch(ctx, c.Source, catalogueTag, SignatureAnnotation, maxCatalogueBytes)
if err != nil {
return domain.Catalogue{}, fmt.Errorf("catalogue: %w", err)
}
manifest, err := readBounded(stream, maxManifestBytes)
if err != nil {
return domain.Catalogue{}, err
}
var image ocispec.Manifest
if err := json.Unmarshal(manifest, &image); err != nil {
return domain.Catalogue{}, fmt.Errorf("catalogue manifest: %w", err)
}
if len(image.Layers) != 1 {
return domain.Catalogue{}, errors.New("catalogue must have one layer")
}
layerStream, err := repository.Fetch(ctx, image.Layers[0])
if err != nil {
return domain.Catalogue{}, fmt.Errorf("catalogue layer: %w", err)
}
layer, err := readBounded(layerStream, maxCatalogueBytes)
if err != nil {
return domain.Catalogue{}, err
}
return verifyCatalogue(image, layer, c.Key)
return decodeCatalogue(artifact.Statement)
}
// verifyCatalogue checks the layer is the one the manifest names and that the
// key signed it, then decodes it.
func verifyCatalogue(image ocispec.Manifest, layer []byte, key ed25519.PublicKey) (domain.Catalogue, error) {
if contentDigest(layer) != image.Layers[0].Digest.String() {
return domain.Catalogue{}, errors.New("catalogue layer does not match its manifest digest")
if err := signedoci.Verify(image, layer, key, SignatureAnnotation); err != nil {
return domain.Catalogue{}, fmt.Errorf("catalogue: %w", err)
}
signature, err := base64.StdEncoding.DecodeString(image.Annotations[SignatureAnnotation])
if err != nil || !ed25519.Verify(key, layer, signature) {
return domain.Catalogue{}, errors.New("catalogue signature does not verify")
return decodeCatalogue(layer)
}
// decodeCatalogue decodes a verified catalogue layer.
func decodeCatalogue(layer []byte) (domain.Catalogue, error) {
decoder := json.NewDecoder(bytes.NewReader(layer))
decoder.DisallowUnknownFields()
var catalogue domain.Catalogue
@@ -119,73 +73,3 @@ func verifyCatalogue(image ocispec.Manifest, layer []byte, key ed25519.PublicKey
}
return catalogue, nil
}
// FileFreshness is the durable Freshness mark, kept as one small JSON file
// written atomically so a crash never leaves a half-written mark. A missing
// file is no mark yet; a file that exists but does not parse is an error, so
// a corrupt mark fails closed instead of opening a replay window.
type FileFreshness struct {
Path string
mu sync.Mutex
}
var _ domain.Freshness = (*FileFreshness)(nil)
func (f *FileFreshness) Newest(context.Context) (time.Time, error) {
f.mu.Lock()
defer f.mu.Unlock()
return f.read()
}
func (f *FileFreshness) Accept(_ context.Context, issuedAt time.Time) error {
f.mu.Lock()
defer f.mu.Unlock()
current, err := f.read()
if err != nil {
return err
}
if !issuedAt.After(current) {
return nil
}
encoded, err := json.Marshal(struct {
IssuedAt time.Time `json:"issued_at"`
}{IssuedAt: issuedAt.UTC()})
if err != nil {
return err
}
return fsutil.WriteFileAtomic(f.Path, encoded, 0o600)
}
// read returns the mark, or the zero time when no catalogue has been
// accepted yet.
func (f *FileFreshness) read() (time.Time, error) {
raw, err := os.ReadFile(f.Path)
if errors.Is(err, fs.ErrNotExist) {
return time.Time{}, nil
}
if err != nil {
return time.Time{}, err
}
var mark struct {
IssuedAt time.Time `json:"issued_at"`
}
if err := json.Unmarshal(raw, &mark); err != nil {
return time.Time{}, fmt.Errorf("catalogue freshness mark %s: %w", f.Path, err)
}
return mark.IssuedAt, nil
}
// defaultFreshnessPath is where the default catalogue's freshness mark
// lives: $XDG_STATE_HOME/<instance>/catalogue/<key>.json, the state home the
// update journal uses too. The key names the catalogue reference, so two
// catalogues never share a mark.
func defaultFreshnessPath(reference string) string {
dir := os.Getenv("XDG_STATE_HOME")
if dir == "" {
home, _ := os.UserHomeDir()
dir = filepath.Join(home, ".local", "state")
}
sum := sha256.Sum256([]byte(reference))
return filepath.Join(dir, configuration.InstanceName(), "catalogue", hex.EncodeToString(sum[:8])+".json")
}
@@ -5,15 +5,14 @@ import (
"crypto/ed25519"
"encoding/base64"
"errors"
"os"
"path/filepath"
"testing"
"time"
digest "github.com/opencontainers/go-digest"
ocispec "github.com/opencontainers/image-spec/specs-go/v1"
domain "github.com/samcharles93/archie-core/internal/domain/storepkg"
"github.com/samcharles93/archie-core/internal/infrastructure/signedoci"
)
// A catalogue is trusted only when its layer is the one the manifest names
@@ -87,7 +86,7 @@ func verifiedCatalogue(t *testing.T, key ed25519.PublicKey, signer ed25519.Priva
// refuses it survives a restart.
func TestFetchRefusesReplayedCatalogue(t *testing.T) {
trusted, signer, _ := ed25519.GenerateKey(nil)
mark := &FileFreshness{Path: filepath.Join(t.TempDir(), "catalogue", "freshness.json")}
mark := &signedoci.FileFreshness{Path: filepath.Join(t.TempDir(), "catalogue", "freshness.json")}
newer := verifiedCatalogue(t, trusted, signer, "2026-01-02T00:00:00Z")
older := verifiedCatalogue(t, trusted, signer, "2026-01-01T00:00:00Z")
@@ -104,52 +103,13 @@ func TestFetchRefusesReplayedCatalogue(t *testing.T) {
}
// The mark is durable: a fresh reader over the same file, as after a
// restart, still refuses the older catalogue and still accepts the new.
restarted := domain.FreshCatalogueSource{Source: staticCatalogue{older}, Freshness: &FileFreshness{Path: mark.Path}}
restarted := domain.FreshCatalogueSource{Source: staticCatalogue{older}, Freshness: &signedoci.FileFreshness{Path: mark.Path}}
if _, err := restarted.Fetch(t.Context()); !errors.Is(err, domain.ErrCatalogueStale) {
t.Fatalf("after restart: err = %v, want ErrCatalogueStale", err)
}
if _, err := (domain.FreshCatalogueSource{Source: staticCatalogue{newer}, Freshness: &FileFreshness{Path: mark.Path}}).Fetch(t.Context()); err != nil {
if _, err := (domain.FreshCatalogueSource{Source: staticCatalogue{newer}, Freshness: &signedoci.FileFreshness{Path: mark.Path}}).Fetch(t.Context()); err != nil {
t.Fatalf("after restart, newest catalogue: %v", err)
}
}
// The freshness mark never goes backwards, and a mark that exists but does
// not parse fails closed rather than opening a replay window.
func TestFileFreshness(t *testing.T) {
path := filepath.Join(t.TempDir(), "freshness.json")
mark := &FileFreshness{Path: path}
jan1 := time.Date(2026, 1, 1, 0, 0, 0, 0, time.UTC)
jan2 := time.Date(2026, 1, 2, 0, 0, 0, 0, time.UTC)
newest, err := mark.Newest(t.Context())
if err != nil {
t.Fatalf("Newest with no file: %v", err)
}
if !newest.IsZero() {
t.Fatalf("Newest with no file = %s; want the zero time", newest.Format(time.RFC3339))
}
if err := mark.Accept(t.Context(), jan1); err != nil {
t.Fatalf("Accept jan1: %v", err)
}
if err := mark.Accept(t.Context(), jan2); err != nil {
t.Fatalf("Accept jan2: %v", err)
}
if err := mark.Accept(t.Context(), jan1); err != nil {
t.Fatalf("Accept jan1 again: %v", err)
}
if newest, err = mark.Newest(t.Context()); err != nil {
t.Fatalf("Newest: %v", err)
}
if !newest.Equal(jan2) {
t.Fatalf("Newest = %s; want %s", newest.Format(time.RFC3339), jan2.Format(time.RFC3339))
}
if err := os.WriteFile(path, []byte("{"), 0o600); err != nil {
t.Fatal(err)
}
if _, err := mark.Newest(t.Context()); err == nil {
t.Fatal("Newest over a corrupt mark: err = nil, want an error")
}
}
const zeros = "0000000000000000000000000000000000000000000000000000000000000000"
-227
View File
@@ -1,227 +0,0 @@
package releaseupdate
import (
"context"
"encoding/json"
"errors"
"fmt"
"net/http"
"os"
"path/filepath"
"regexp"
"strings"
"sync"
"time"
"golang.org/x/mod/semver"
"github.com/samcharles93/archie-core/internal/buildinfo"
)
// ReleaseSlug is the public repository releases are published to. The source
// repository is private, so its releases are unreachable without credentials.
const ReleaseSlug = "samcharles93/archie-releases"
// changelogURL lists every release with its changelog section as notes.
const changelogURL = "https://github.com/" + ReleaseSlug + "/releases"
// buildSuffix is the git-describe tail of an untagged build (1.7.0-4-g0736708).
var buildSuffix = regexp.MustCompile(`-[0-9]+-g[0-9a-f]+$`)
// GitHubCatalog finds the newest release on the selected channel that ships a
// linux-amd64 zip, and reports it against this build. Only one version covers
// every component, so the daemon and agent are offered the same release.
type GitHubCatalog struct {
// Channel returns the release channel and pin in force.
Channel func(context.Context) (channel, pin string, err error)
// A found release is reused for checkTTL: unauthenticated GitHub API
// calls are limited to 60 an hour per address.
mu sync.Mutex
key string
at time.Time
release *release
}
const checkTTL = 10 * time.Minute
func (c *GitHubCatalog) Check(ctx context.Context) (Snapshot, error) {
channel, pin, err := c.Channel(ctx)
if err != nil {
return Snapshot{}, err
}
if channel == "" {
channel = "stable"
}
release, err := c.detect(ctx, channel, pin)
if err != nil {
return Snapshot{}, err
}
component := func(id, label, installed string) Component {
return Component{
ID: id, Label: label,
Installed: installed, Available: release.version,
Reference: release.asset, Changelog: changelogURL,
}
}
return Snapshot{Components: []Component{
component("daemon", "Gateway", ReleaseVersion(buildinfo.Version)),
component("agent", "Runtime", installedAgent(executableDir())),
}}, nil
}
func (c *GitHubCatalog) detect(ctx context.Context, channel, pin string) (*release, error) {
c.mu.Lock()
defer c.mu.Unlock()
key := channel + "\x00" + pin
if c.release != nil && c.key == key && time.Since(c.at) < checkTTL {
return c.release, nil
}
release, err := detectRelease(ctx, channel, pin)
if err != nil {
return nil, err
}
c.key, c.at, c.release = key, time.Now(), release
return release, nil
}
// release is one published version and the zip that installs it here.
type release struct {
version string
asset string
}
const releasesURL = "https://api.github.com/repos/" + ReleaseSlug + "/releases?per_page=100"
type githubRelease struct {
TagName string `json:"tag_name"`
Draft bool `json:"draft"`
Prerelease bool `json:"prerelease"`
Assets []struct {
Name string `json:"name"`
} `json:"assets"`
}
// detectRelease resolves channel to one published release. A release with no
// zip for this platform, or no SHA256SUMS to verify it against, is not
// installable, so it is never offered.
func detectRelease(ctx context.Context, channel, pin string) (*release, error) {
if err := ValidateChannel(channel, pin); err != nil {
return nil, err
}
releases, err := listReleases(ctx)
if err != nil {
return nil, fmt.Errorf("find releases: %w", err)
}
if found := pick(releases, channel, pin); found != nil {
return found, nil
}
return nil, errors.New("no published release with a linux-amd64 zip matches the channel")
}
func listReleases(ctx context.Context) ([]githubRelease, error) {
req, err := http.NewRequestWithContext(ctx, http.MethodGet, releasesURL, nil)
if err != nil {
return nil, err
}
req.Header.Set("Accept", "application/vnd.github+json")
resp, err := http.DefaultClient.Do(req)
if err != nil {
return nil, err
}
defer func() { _ = resp.Body.Close() }()
if resp.StatusCode != http.StatusOK {
return nil, fmt.Errorf("GitHub answered %s", resp.Status)
}
var releases []githubRelease
if err := json.NewDecoder(resp.Body).Decode(&releases); err != nil {
return nil, fmt.Errorf("decode releases: %w", err)
}
return releases, nil
}
// pick returns the newest installable release on channel.
func pick(releases []githubRelease, channel, pin string) *release {
var best *release
for _, r := range releases {
if !onChannel(r, channel, pin) {
continue
}
asset := installable(r)
if asset == "" {
continue
}
if best == nil || semver.Compare(r.TagName, "v"+best.version) > 0 {
best = &release{version: strings.TrimPrefix(r.TagName, "v"), asset: asset}
}
}
return best
}
// onChannel: stable skips prereleases, next includes them, and exact-pin takes
// only the pinned tag.
func onChannel(r githubRelease, channel, pin string) bool {
if r.Draft || !semver.IsValid(r.TagName) {
return false
}
switch channel {
case "exact-pin":
return r.TagName == "v"+strings.TrimPrefix(pin, "v")
case "next":
return true
default:
return !r.Prerelease
}
}
// installable names the release's linux-amd64 zip, or "" when it has none or
// no SHA256SUMS to verify it against.
func installable(r githubRelease) string {
asset, sums := "", false
for _, a := range r.Assets {
switch {
case a.Name == "SHA256SUMS":
sums = true
case strings.HasSuffix(a.Name, "-linux-amd64.zip"):
asset = a.Name
}
}
if !sums {
return ""
}
return asset
}
// ReleaseVersion is the release a build was cut from: a git-describe build
// ("1.55.0-218-gabc123") is the same release as its tag. A build that names no
// release ("dev") has none, so nothing can be judged newer than it.
func ReleaseVersion(version string) string {
version = buildSuffix.ReplaceAllString(version, "")
if !semver.IsValid("v" + strings.TrimPrefix(version, "v")) {
return ""
}
return version
}
// installedAgent is the image the installer last switched to, recorded in a
// sidecar beside the binaries; without one it is the runtime this release was
// cut with. A malformed sidecar is unknown, not guessed.
func installedAgent(binDir string) string {
data, err := os.ReadFile(filepath.Join(binDir, agentVersionFile))
if binDir == "" || errors.Is(err, os.ErrNotExist) {
return ReleaseVersion(buildinfo.Runtime)
}
version := strings.TrimSpace(string(data))
if err != nil || strings.ContainsAny(version, " \n") || ValidateChannel("exact-pin", version) != nil {
return ""
}
return ReleaseVersion(version)
}
func executableDir() string {
exe, err := os.Executable()
if err != nil {
return ""
}
return filepath.Dir(exe)
}
+13 -2
View File
@@ -12,6 +12,8 @@ import (
"path/filepath"
"strconv"
"strings"
"github.com/samcharles93/archie-core/internal/domain/distribution"
)
// ResultSentinel prefixes the one line `archied update install` prints to
@@ -23,8 +25,11 @@ const ResultSentinel = "ARCHIE_UPDATE_RESULT "
var ErrNotConfigured = errors.New("updates are not configured")
// ArchiedInstaller runs `archied update install` from beside this binary, so
// every service installs through the daemon's one transaction.
type ArchiedInstaller struct{}
// every service installs through the daemon's one transaction. Source is the
// distribution source the release was found in, so it installs from the same.
type ArchiedInstaller struct {
Source func(context.Context) (distribution.Source, error)
}
func (ArchiedInstaller) path() string {
path := filepath.Join(executableDir(), WatchdogBinary)
@@ -47,8 +52,14 @@ func (i ArchiedInstaller) Install(ctx context.Context, snapshot Snapshot, meta I
if len(available) == 0 {
return Result{}, errors.New("nothing to install")
}
source, err := i.Source(ctx)
if err != nil {
return Result{}, err
}
cmd := exec.CommandContext(ctx, path, "update", "install",
"-version", available[0].Available,
"-source", source.Reference,
"-key", distribution.EncodeKey(source.Key),
"-report", meta.ReportPath,
"-channel", meta.Channel,
"-chat", strconv.FormatInt(meta.ChatID, 10),
+25 -63
View File
@@ -2,16 +2,11 @@ package releaseupdate
import (
"archive/zip"
"bufio"
"bytes"
"context"
"crypto/sha256"
"encoding/hex"
"encoding/json"
"errors"
"fmt"
"io"
"net/http"
"os"
"os/exec"
"path"
@@ -23,11 +18,9 @@ import (
"time"
"github.com/samcharles93/archie-core/internal/buildinfo"
"github.com/samcharles93/archie-core/internal/domain/distribution"
)
// ReleaseURL is where published release assets are downloaded from.
const ReleaseURL = "https://github.com/" + ReleaseSlug + "/releases/download"
var (
releaseVersion = regexp.MustCompile(`^[0-9]+\.[0-9]+\.[0-9]+([+-][0-9A-Za-z.-]+)?$`)
releaseUnit = regexp.MustCompile(`^archie[a-z0-9-]*$`)
@@ -37,22 +30,25 @@ var (
// PrepareRequest names the release to stage and who hears the outcome.
type PrepareRequest struct {
Version string
// ReleaseURL is the base the release's zip and SHA256SUMS are under.
ReleaseURL string
// Source is the distribution source the release is fetched and verified
// from.
Source distribution.Source
Meta InstallMeta
// CheckTopology refuses a host missing a unit or config section the
// release requires. managed reports whether units are observable.
CheckTopology func(ctx context.Context, required Topology, managed bool) error
}
// Manifest is the release.json a release zip carries.
// Manifest is the release.json a release zip carries, with the agent image
// taken from the signed statement rather than from the zip.
type Manifest struct {
Required Topology `json:"required_topology"`
AgentImage string `json:"agent_image"`
}
// Prepare verifies and stages a release without changing anything that runs:
// the zip is checked against SHA256SUMS, every binary must report the release,
// the statement must verify against the source's key and the zip against the
// statement, every binary must report the release,
// the agent image is pulled by digest and the database is backed up before a
// journal is published. The watchdog (Run) takes it from there.
func (t *Transaction) Prepare(ctx context.Context, req PrepareRequest) (Result, error) {
@@ -117,11 +113,15 @@ func (t *Transaction) fetch(ctx context.Context, req PrepareRequest, work string
if runtime.GOOS != "linux" || runtime.GOARCH != "amd64" {
return "", Manifest{}, fmt.Errorf("no published release for %s/%s", runtime.GOOS, runtime.GOARCH)
}
name := fmt.Sprintf("archie-core-archied-v%s-linux-amd64", req.Version)
base := strings.TrimSuffix(req.ReleaseURL, "/") + "/v" + req.Version + "/"
t.say("downloading %s", name+".zip")
archive := filepath.Join(work, name+".zip")
if err := downloadVerified(ctx, base, name+".zip", archive); err != nil {
verified, err := Fetch(ctx, req.Source, Tag("exact-pin", req.Version))
if err != nil {
return "", Manifest{}, err
}
zipName := verified.Statement.Zip.Name
name := strings.TrimSuffix(zipName, ".zip")
t.say("downloading %s", zipName)
archive := filepath.Join(work, zipName)
if err := writeBlob(ctx, verified, archive); err != nil {
return "", Manifest{}, err
}
if err := unpack(archive, name, work); err != nil {
@@ -132,6 +132,7 @@ func (t *Transaction) fetch(ctx context.Context, req PrepareRequest, work string
if err != nil {
return "", Manifest{}, err
}
manifest.AgentImage = verified.Statement.AgentImage
for _, binary := range manifest.Required.apps() {
reported, err := reportedVersion(ctx, filepath.Join(release, binary), binary)
if err != nil {
@@ -226,61 +227,25 @@ func (t *Transaction) stage(req PrepareRequest, release string, j journal) error
return syncDir(t.StateDir)
}
func downloadVerified(ctx context.Context, base, name, dst string) error {
var sums bytes.Buffer
if err := download(ctx, base+"SHA256SUMS", &sums, 1<<20); err != nil {
return err
}
expected := ""
scanner := bufio.NewScanner(&sums)
for scanner.Scan() {
if fields := strings.Fields(scanner.Text()); len(fields) == 2 && fields[1] == name {
if expected != "" {
return fmt.Errorf("SHA256SUMS lists %s twice", name)
}
expected = strings.ToLower(fields[0])
}
}
if len(expected) != sha256.Size*2 {
return fmt.Errorf("SHA256SUMS has no checksum for %s", name)
}
// writeBlob writes the release's zip to dst, checked against the digest the
// signed statement names.
func writeBlob(ctx context.Context, release Release, dst string) error {
file, err := os.OpenFile(dst, os.O_CREATE|os.O_EXCL|os.O_WRONLY, 0o600)
if err != nil {
return err
}
hash := sha256.New()
err = download(ctx, base+name, io.MultiWriter(file, hash), 1<<30)
ctx, cancel := context.WithTimeout(ctx, 5*time.Minute)
defer cancel()
err = release.Artifact.Blob(ctx, release.Statement.Zip.Digest, file, 1<<30)
if closeErr := file.Close(); err == nil {
err = closeErr
}
if err != nil {
return err
}
if hex.EncodeToString(hash.Sum(nil)) != expected {
return fmt.Errorf("release checksum mismatch for %s", name)
return fmt.Errorf("download %s: %w", release.Statement.Zip.Name, err)
}
return nil
}
func download(ctx context.Context, url string, out io.Writer, limit int64) error {
ctx, cancel := context.WithTimeout(ctx, 5*time.Minute)
defer cancel()
req, err := http.NewRequestWithContext(ctx, http.MethodGet, url, nil)
if err != nil {
return err
}
resp, err := http.DefaultClient.Do(req)
if err != nil {
return fmt.Errorf("download %s: %w", url, err)
}
defer func() { _ = resp.Body.Close() }()
if resp.StatusCode != http.StatusOK {
return fmt.Errorf("download %s: %s", url, resp.Status)
}
_, err = io.Copy(out, io.LimitReader(resp.Body, limit))
return err
}
// unpack extracts the zip's single distribution directory, refusing any
// entry outside it and anything but regular files and directories.
func unpack(archive, name, dst string) error {
@@ -350,9 +315,6 @@ func readManifest(path string) (Manifest, error) {
return Manifest{}, fmt.Errorf("invalid release unit %q", unit)
}
}
if !pinnedImage.MatchString(manifest.AgentImage) {
return Manifest{}, errors.New("release has no digest-pinned agent image")
}
return manifest, nil
}
+133
View File
@@ -0,0 +1,133 @@
package releaseupdate
import (
"bytes"
"context"
"crypto/ed25519"
"encoding/base64"
"encoding/json"
"fmt"
"os"
"time"
digest "github.com/opencontainers/go-digest"
ocispec "github.com/opencontainers/image-spec/specs-go/v1"
"oras.land/oras-go/v2/registry/remote"
"oras.land/oras-go/v2/registry/remote/auth"
"oras.land/oras-go/v2/registry/remote/credentials"
"github.com/samcharles93/archie-core/internal/infrastructure/signedoci"
)
// PublishRequest is one release to sign and push.
type PublishRequest struct {
// Repository is the OCI repository the release is pushed to.
Repository string
Signer ed25519.PrivateKey
Version string
Prerelease bool
// ZipPath is the distribution zip, named ZipName(Version).
ZipPath string
AgentImage string
Changelog string
// Channels are the moving tags pointed at this release besides its own
// version tag.
Channels []string
}
// Publish signs a statement for the release and pushes it with the zip as
// one artifact, tagged with its version and each channel. Registry
// credentials come from the Docker config, as `docker login` left them.
func Publish(ctx context.Context, req PublishRequest) (Statement, error) {
archive, err := os.ReadFile(req.ZipPath)
if err != nil {
return Statement{}, err
}
statement := Statement{
Version: req.Version, Prerelease: req.Prerelease, IssuedAt: time.Now().UTC(), Platform: Platform,
Zip: Blob{Name: ZipName(req.Version), Digest: signedoci.Digest(archive), Size: int64(len(archive))},
AgentImage: req.AgentImage, Changelog: req.Changelog,
}
if err := statement.Validate(); err != nil {
return Statement{}, err
}
for _, channel := range req.Channels {
if channel != "stable" && channel != "next" {
return Statement{}, fmt.Errorf("unknown channel %q", channel)
}
}
encoded, err := json.Marshal(statement)
if err != nil {
return Statement{}, err
}
repository, err := pushRepository(req.Repository)
if err != nil {
return Statement{}, err
}
layers := []ocispec.Descriptor{
{MediaType: StatementMediaType, Digest: digest.FromBytes(encoded), Size: int64(len(encoded))},
{
MediaType: "application/zip", Digest: digest.FromBytes(archive), Size: int64(len(archive)),
Annotations: map[string]string{ocispec.AnnotationTitle: statement.Zip.Name},
},
}
for i, content := range [][]byte{ocispec.DescriptorEmptyJSON.Data, encoded, archive} {
desc := ocispec.DescriptorEmptyJSON
if i > 0 {
desc = layers[i-1]
}
if err := pushBlob(ctx, repository, desc, content); err != nil {
return Statement{}, err
}
}
manifest, err := json.Marshal(ocispec.Manifest{
SchemaVersion: 2, MediaType: ocispec.MediaTypeImageManifest,
ArtifactType: ArtifactType, Config: ocispec.DescriptorEmptyJSON, Layers: layers,
Annotations: map[string]string{
Annotation: base64.StdEncoding.EncodeToString(ed25519.Sign(req.Signer, encoded)),
ocispec.AnnotationCreated: statement.IssuedAt.Format(time.RFC3339),
},
})
if err != nil {
return Statement{}, err
}
desc := ocispec.Descriptor{MediaType: ocispec.MediaTypeImageManifest, Digest: digest.FromBytes(manifest), Size: int64(len(manifest))}
if err := repository.PushReference(ctx, desc, bytes.NewReader(manifest), Tag("exact-pin", req.Version)); err != nil {
return Statement{}, fmt.Errorf("push release manifest: %w", err)
}
for _, channel := range req.Channels {
if err := repository.Tag(ctx, desc, channel); err != nil {
return Statement{}, fmt.Errorf("tag %s: %w", channel, err)
}
}
return statement, nil
}
func pushRepository(reference string) (*remote.Repository, error) {
repository, err := remote.NewRepository(reference)
if err != nil {
return nil, err
}
repository.PlainHTTP = signedoci.Loopback(reference)
store, err := credentials.NewStoreFromDocker(credentials.StoreOptions{})
if err != nil {
return nil, fmt.Errorf("read registry credentials: %w", err)
}
repository.Client = &auth.Client{Credential: credentials.Credential(store)}
return repository, nil
}
// pushBlob uploads content unless the registry already holds it.
func pushBlob(ctx context.Context, repository *remote.Repository, desc ocispec.Descriptor, content []byte) error {
exists, err := repository.Exists(ctx, desc)
if err != nil {
return fmt.Errorf("check blob %s: %w", desc.Digest, err)
}
if exists {
return nil
}
if err := repository.Push(ctx, desc, bytes.NewReader(content)); err != nil {
return fmt.Errorf("push blob %s: %w", desc.Digest, err)
}
return nil
}
+226
View File
@@ -0,0 +1,226 @@
package releaseupdate
import (
"context"
"encoding/json"
"errors"
"fmt"
"os"
"path/filepath"
"regexp"
"strings"
"sync"
"time"
"golang.org/x/mod/semver"
"github.com/samcharles93/archie-core/internal/buildinfo"
"github.com/samcharles93/archie-core/internal/domain/distribution"
"github.com/samcharles93/archie-core/internal/infrastructure/signedoci"
)
// Annotation carries the source key's signature over a release statement.
const Annotation = "dev.archie.release.signature.v1"
// ArtifactType marks a release manifest.
const ArtifactType = "application/vnd.archie.release.v1"
// StatementMediaType is the release statement layer's media type.
const StatementMediaType = "application/vnd.archie.release.statement.v1+json"
// Platform is the only platform releases are built for.
const Platform = "linux-amd64"
const maxStatementBytes = 64 << 10
// buildSuffix is the git-describe tail of an untagged build (1.7.0-4-g0736708).
var buildSuffix = regexp.MustCompile(`-[0-9]+-g[0-9a-f]+$`)
// Statement is what a release's signature covers: the version, when it was
// issued, the zip that installs it and the agent image it runs. Everything an
// install trusts comes from here, never from the registry tag or the zip.
type Statement struct {
Version string `json:"version"`
Prerelease bool `json:"prerelease"`
IssuedAt time.Time `json:"issued_at"`
Platform string `json:"platform"`
Zip Blob `json:"zip"`
AgentImage string `json:"agent_image"`
Changelog string `json:"changelog,omitempty"`
}
// Blob names a release layer by its content digest.
type Blob struct {
Name string `json:"name"`
Digest string `json:"digest"`
Size int64 `json:"size"`
}
// Validate refuses a statement an install could not act on safely.
func (s Statement) Validate() error {
switch {
case !releaseVersion.MatchString(s.Version):
return fmt.Errorf("release statement version %q is not a release version", s.Version)
case s.IssuedAt.IsZero():
return errors.New("release statement has no issue time")
case s.Platform != Platform:
return fmt.Errorf("release statement is for %q, not %s", s.Platform, Platform)
case s.Zip.Name != ZipName(s.Version) || !strings.HasPrefix(s.Zip.Digest, "sha256:") || s.Zip.Size <= 0:
return errors.New("release statement does not name its zip by name, digest and size")
case !pinnedImage.MatchString(s.AgentImage):
return fmt.Errorf("release statement agent image %q is not pinned by digest", s.AgentImage)
}
return nil
}
// ZipName is the distribution zip a version is published as.
func ZipName(version string) string {
return fmt.Sprintf("archie-core-archied-v%s-%s.zip", version, Platform)
}
// Tag is the registry tag a channel resolves through: stable and next are
// moving tags, an exact pin is the version's own.
func Tag(channel, pin string) string {
if channel == "exact-pin" {
return "v" + strings.TrimPrefix(pin, "v")
}
return channel
}
// Release is a verified release and the artifact its zip is read from.
type Release struct {
Statement Statement
Artifact signedoci.Artifact
}
// Fetch resolves tag at source and verifies its statement. A statement on a
// moving channel tag issued before the newest one accepted from that tag is a
// replay and is refused; an exact pin may name an older release on purpose.
func Fetch(ctx context.Context, source distribution.Source, tag string) (Release, error) {
artifact, err := signedoci.Fetch(ctx, source, tag, Annotation, maxStatementBytes)
if err != nil {
return Release{}, fmt.Errorf("release: %w", err)
}
decoder := json.NewDecoder(strings.NewReader(string(artifact.Statement)))
decoder.DisallowUnknownFields()
var statement Statement
if err := decoder.Decode(&statement); err != nil {
return Release{}, fmt.Errorf("release statement: %w", err)
}
if err := statement.Validate(); err != nil {
return Release{}, err
}
if strings.HasPrefix(tag, "v") {
if "v"+statement.Version != tag {
return Release{}, fmt.Errorf("tag %s carries the statement for %s", tag, statement.Version)
}
return Release{Statement: statement, Artifact: artifact}, nil
}
mark := &signedoci.FileFreshness{Path: signedoci.MarkPath("releases", source.Reference+":"+tag)}
newest, err := mark.Newest(ctx)
if err != nil {
return Release{}, err
}
if !newest.IsZero() && statement.IssuedAt.Before(newest) {
return Release{}, fmt.Errorf("release on %s was issued %s, before the newest accepted %s",
tag, statement.IssuedAt.Format(time.RFC3339), newest.Format(time.RFC3339))
}
if err := mark.Accept(ctx, statement.IssuedAt); err != nil {
return Release{}, fmt.Errorf("record release freshness: %w", err)
}
return Release{Statement: statement, Artifact: artifact}, nil
}
// RegistryCatalog reports the release on the selected channel against this build.
// Only one version covers every component, so the daemon and agent are
// offered the same release.
type RegistryCatalog struct {
// Settings returns the source, channel and pin in force.
Settings func(context.Context) (source distribution.Source, channel, pin string, err error)
// A found release is reused for checkTTL so a dashboard that polls does
// not hit the registry on every view.
mu sync.Mutex
key string
at time.Time
statement Statement
}
const checkTTL = 10 * time.Minute
func (c *RegistryCatalog) Check(ctx context.Context) (Snapshot, error) {
source, channel, pin, err := c.Settings(ctx)
if err != nil {
return Snapshot{}, err
}
if channel == "" {
channel = "stable"
}
if err := ValidateChannel(channel, pin); err != nil {
return Snapshot{}, err
}
statement, err := c.detect(ctx, source, Tag(channel, pin))
if err != nil {
return Snapshot{}, err
}
component := func(id, label, installed string) Component {
return Component{
ID: id, Label: label,
Installed: installed, Available: statement.Version,
Reference: statement.Zip.Name, Changelog: statement.Changelog,
}
}
return Snapshot{Components: []Component{
component("daemon", "Gateway", ReleaseVersion(buildinfo.Version)),
component("agent", "Runtime", installedAgent(executableDir())),
}}, nil
}
func (c *RegistryCatalog) detect(ctx context.Context, source distribution.Source, tag string) (Statement, error) {
c.mu.Lock()
defer c.mu.Unlock()
key := source.Reference + "\x00" + distribution.EncodeKey(source.Key) + "\x00" + tag
if c.key == key && time.Since(c.at) < checkTTL {
return c.statement, nil
}
release, err := Fetch(ctx, source, tag)
if err != nil {
return Statement{}, err
}
c.key, c.at, c.statement = key, time.Now(), release.Statement
return release.Statement, nil
}
// ReleaseVersion is the release a build was cut from: a git-describe build
// ("1.55.0-218-gabc123") is the same release as its tag. A build that names no
// release ("dev") has none, so nothing can be judged newer than it.
func ReleaseVersion(version string) string {
version = buildSuffix.ReplaceAllString(version, "")
if !semver.IsValid("v" + strings.TrimPrefix(version, "v")) {
return ""
}
return version
}
// installedAgent is the image the installer last switched to, recorded in a
// sidecar beside the binaries; without one it is the runtime this release was
// cut with. A malformed sidecar is unknown, not guessed.
func installedAgent(binDir string) string {
data, err := os.ReadFile(filepath.Join(binDir, agentVersionFile))
if binDir == "" || errors.Is(err, os.ErrNotExist) {
return ReleaseVersion(buildinfo.Runtime)
}
version := strings.TrimSpace(string(data))
if err != nil || strings.ContainsAny(version, " \n") || ValidateChannel("exact-pin", version) != nil {
return ""
}
return ReleaseVersion(version)
}
func executableDir() string {
exe, err := os.Executable()
if err != nil {
return ""
}
return filepath.Dir(exe)
}
+74
View File
@@ -0,0 +1,74 @@
package releaseupdate
import (
"crypto/ed25519"
"encoding/json"
"net/http"
"net/http/httptest"
"strings"
"testing"
"time"
"github.com/samcharles93/archie-core/internal/domain/distribution"
"github.com/samcharles93/archie-core/internal/infrastructure/signedoci"
)
// A release is trusted only when the source's key signed its statement, the
// tag it was found under agrees with it, and a moving channel tag never goes
// back to a release issued before one already accepted from it.
func TestFetchRelease(t *testing.T) {
trusted, signer, _ := ed25519.GenerateKey(nil)
stranger, _, _ := ed25519.GenerateKey(nil)
archive := []byte("zip")
statement := func(version string, issued time.Time) []byte {
data, err := json.Marshal(Statement{
Version: version, IssuedAt: issued, Platform: Platform, AgentImage: newImage,
Zip: Blob{Name: ZipName(version), Digest: signedoci.Digest(archive), Size: int64(len(archive))},
})
if err != nil {
t.Fatal(err)
}
return data
}
jan1, jan2 := time.Date(2026, 1, 1, 0, 0, 0, 0, time.UTC), time.Date(2026, 1, 2, 0, 0, 0, 0, time.UTC)
cases := []struct {
name string
accepted []byte // a statement already accepted from the same tag
statement []byte
tag string
key ed25519.PublicKey
want string // error substring, or "" to accept
}{
{name: "signed by the source key", statement: statement("2.0.0", jan1), tag: "stable", key: trusted},
{name: "signed by another key", statement: statement("2.0.0", jan1), tag: "stable", key: stranger, want: "signature"},
{name: "a version tag carrying another version", statement: statement("2.0.0", jan1), tag: "v3.0.0", key: trusted, want: "carries"},
{name: "a channel moved back to an older release", accepted: statement("2.1.0", jan2), statement: statement("2.0.0", jan1), tag: "stable", key: trusted, want: "before the newest"},
{name: "an exact pin to an older release", accepted: statement("2.1.0", jan2), statement: statement("2.0.0", jan1), tag: "v2.0.0", key: trusted},
}
for _, tc := range cases {
t.Run(tc.name, func(t *testing.T) {
t.Setenv("XDG_STATE_HOME", t.TempDir())
// One registry, so the freshness mark for the tag is shared; it
// serves the accepted statement first, then the one under test.
var current http.Handler
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { current.ServeHTTP(w, r) }))
defer server.Close()
source := distribution.Source{Reference: strings.TrimPrefix(server.URL, "http://") + "/archie", Key: tc.key}
if tc.accepted != nil {
current = fakeRegistry(t, "archie", "stable", signer, tc.accepted, archive, archive)
if _, err := Fetch(t.Context(), distribution.Source{Reference: source.Reference, Key: trusted}, "stable"); err != nil {
t.Fatalf("accept the newer release: %v", err)
}
}
current = fakeRegistry(t, "archie", tc.tag, signer, tc.statement, archive, archive)
_, err := Fetch(t.Context(), source, tc.tag)
switch {
case tc.want == "" && err != nil:
t.Fatalf("Fetch: %v", err)
case tc.want != "" && (err == nil || !strings.Contains(err.Error(), tc.want)):
t.Fatalf("Fetch error = %v, want %q", err, tc.want)
}
})
}
}
+68 -11
View File
@@ -4,7 +4,9 @@ import (
"archive/zip"
"bytes"
"context"
"crypto/sha256"
"crypto/ed25519"
"encoding/base64"
"encoding/json"
"errors"
"fmt"
"net/http"
@@ -15,6 +17,12 @@ import (
"strings"
"testing"
"time"
digest "github.com/opencontainers/go-digest"
ocispec "github.com/opencontainers/image-spec/specs-go/v1"
"github.com/samcharles93/archie-core/internal/domain/distribution"
"github.com/samcharles93/archie-core/internal/infrastructure/signedoci"
)
var testUnits = []string{"archie-state-store", "archie-gateway", "archied"}
@@ -152,7 +160,7 @@ func TestTransaction(t *testing.T) {
name string
host fakeHost
candidate string // version the release's archied reports
corrupt bool // SHA256SUMS does not match the zip
corrupt bool // the registry serves a zip other than the one signed
crashAt string // phase the watchdog dies in, then boot recovers
prepare string // Prepare refusal
want outcome
@@ -162,7 +170,7 @@ func TestTransaction(t *testing.T) {
{name: "unhealthy migrated release restores the database", host: fakeHost{managed: true, unhealthy: true, migrates: true}, want: outcome{version: "1.0.0", image: oldImage, report: "failed", restoredDatabase: true}},
{name: "stuck image switch rolls back", host: fakeHost{managed: true, hangs: true}, want: old},
{name: "without systemd it installs without restarting", host: fakeHost{}, want: outcome{version: "2.0.0", image: newImage}},
{name: "checksum mismatch is refused", host: fakeHost{managed: true}, corrupt: true, prepare: "checksum mismatch", want: outcome{version: "1.0.0", image: oldImage}},
{name: "a tampered zip is refused", host: fakeHost{managed: true}, corrupt: true, prepare: "digest", want: outcome{version: "1.0.0", image: oldImage}},
{name: "misreported candidate is refused", host: fakeHost{managed: true}, candidate: "1.9.0", prepare: "reports", want: outcome{version: "1.0.0", image: oldImage}},
{name: "crash before the watchdog started is abandoned", host: fakeHost{managed: true}, crashAt: phasePrepared, want: old},
{name: "crash while switching rolls back", host: fakeHost{managed: true}, crashAt: phaseSwitching, want: old},
@@ -180,7 +188,8 @@ func TestTransaction(t *testing.T) {
if candidate == "" {
candidate = "2.0.0"
}
server := httptest.NewServer(releaseHandler(t, candidate, tt.corrupt))
handler, key := releaseHandler(t, candidate, tt.corrupt)
server := httptest.NewServer(handler)
defer server.Close()
for _, unit := range testUnits {
writeTestFile(t, filepath.Join(host.bin, unit), versionScript(unit, "1.0.0"))
@@ -192,7 +201,7 @@ func TestTransaction(t *testing.T) {
HealthTimeout: 20 * time.Millisecond, Observe: 5 * time.Millisecond, Poll: time.Millisecond,
}
_, err := tx.Prepare(t.Context(), PrepareRequest{Version: "2.0.0", ReleaseURL: server.URL, Meta: InstallMeta{Channel: "webui", ReportPath: report}})
_, err := tx.Prepare(t.Context(), PrepareRequest{Version: "2.0.0", Source: distribution.Source{Reference: strings.TrimPrefix(server.URL, "http://") + "/archie", Key: key}, Meta: InstallMeta{Channel: "webui", ReportPath: report}})
switch {
case tt.prepare != "":
if err == nil || !strings.Contains(err.Error(), tt.prepare) {
@@ -254,7 +263,7 @@ func TestTransaction(t *testing.T) {
// releaseHandler serves one release's zip and SHA256SUMS the way GitHub
// release downloads are laid out.
func releaseHandler(t *testing.T, candidate string, corrupt bool) http.Handler {
func releaseHandler(t *testing.T, candidate string, corrupt bool) (http.Handler, ed25519.PublicKey) {
t.Helper()
name := "archie-core-archied-v2.0.0-linux-amd64"
var archive bytes.Buffer
@@ -277,17 +286,65 @@ func releaseHandler(t *testing.T, candidate string, corrupt bool) http.Handler {
}
add(unit, versionScript(unit, version))
}
add("release.json", fmt.Sprintf(`{"required_topology":{"units":["%s"]},"agent_image":%q}`, strings.Join(testUnits, `","`), newImage))
add("release.json", fmt.Sprintf(`{"required_topology":{"units":["%s"]}}`, strings.Join(testUnits, `","`)))
if err := writer.Close(); err != nil {
t.Fatal(err)
}
sum := sha256.Sum256(archive.Bytes())
zipDigest := signedoci.Digest(archive.Bytes())
statement, err := json.Marshal(Statement{
Version: "2.0.0", IssuedAt: time.Date(2026, 1, 1, 0, 0, 0, 0, time.UTC), Platform: Platform,
Zip: Blob{Name: name + ".zip", Digest: zipDigest, Size: int64(archive.Len())},
AgentImage: newImage,
})
if err != nil {
t.Fatal(err)
}
public, private, err := ed25519.GenerateKey(nil)
if err != nil {
t.Fatal(err)
}
served := archive.Bytes()
if corrupt {
sum[0] ^= 0xff
served = append([]byte(nil), served...)
served[len(served)/2] ^= 0xff
}
return fakeRegistry(t, "archie", "v2.0.0", private, statement, archive.Bytes(), served), public
}
// fakeRegistry serves one signed release artifact at repository:tag, the way
// an OCI registry answers oras: the manifest by tag, each layer by digest.
// served is the zip the registry returns, which a tampering registry swaps.
func fakeRegistry(t *testing.T, repository, tag string, signer ed25519.PrivateKey, statement, archive, served []byte) http.Handler {
t.Helper()
blobs := map[string][]byte{signedoci.Digest(statement): statement, signedoci.Digest(archive): served}
manifest, err := json.Marshal(ocispec.Manifest{
SchemaVersion: 2,
MediaType: ocispec.MediaTypeImageManifest,
ArtifactType: ArtifactType,
Config: ocispec.DescriptorEmptyJSON,
Layers: []ocispec.Descriptor{
{MediaType: StatementMediaType, Digest: digest.FromBytes(statement), Size: int64(len(statement))},
{MediaType: "application/zip", Digest: digest.FromBytes(archive), Size: int64(len(archive))},
},
Annotations: map[string]string{Annotation: base64.StdEncoding.EncodeToString(ed25519.Sign(signer, statement))},
})
if err != nil {
t.Fatal(err)
}
mux := http.NewServeMux()
mux.HandleFunc("/v2.0.0/"+name+".zip", func(w http.ResponseWriter, _ *http.Request) { _, _ = w.Write(archive.Bytes()) })
mux.HandleFunc("/v2.0.0/SHA256SUMS", func(w http.ResponseWriter, _ *http.Request) { fmt.Fprintf(w, "%x %s.zip\n", sum, name) })
mux.HandleFunc("/v2/"+repository+"/manifests/"+tag, func(w http.ResponseWriter, _ *http.Request) {
w.Header().Set("Content-Type", ocispec.MediaTypeImageManifest)
w.Header().Set("Docker-Content-Digest", signedoci.Digest(manifest))
_, _ = w.Write(manifest)
})
mux.HandleFunc("/v2/"+repository+"/blobs/", func(w http.ResponseWriter, r *http.Request) {
blob, ok := blobs[strings.TrimPrefix(r.URL.Path, "/v2/"+repository+"/blobs/")]
if !ok {
http.NotFound(w, r)
return
}
_, _ = w.Write(blob)
})
return mux
}
-47
View File
@@ -1,47 +0,0 @@
#!/usr/bin/env bash
# Publishes a tagged release by hand: the agent and archied images to GHCR,
# then the linux-amd64 zip, SHA256SUMS and install.sh to the public releases
# repository, whose release notes are the version's changelog section.
# Builds from a clean worktree of the tag, so local edits never ship.
set -euo pipefail
version="${1:?usage: tools/publish.sh X.Y.Z}"
tag="v$version"
releases="samcharles93/archie-releases"
registry="ghcr.io/samcharles93"
root="$(git rev-parse --show-toplevel)"
work="$(mktemp -d)"
trap 'git -C "$root" worktree remove --force "$work/src" >/dev/null 2>&1 || true; rm -rf "$work"' EXIT
git -C "$root" worktree add -q --detach "$work/src" "$tag"
cd "$work/src"
sha="$(git rev-parse "$tag^{commit}")"
docker build -q -f Dockerfile --build-arg RUNTIME_VERSION="$version" \
-t "$registry/archie-agent:latest" -t "$registry/archie-agent:sha-$sha" . >/dev/null
docker push -q "$registry/archie-agent:latest"
docker push -q "$registry/archie-agent:sha-$sha"
digest="$(docker inspect --format '{{range .RepoDigests}}{{println .}}{{end}}' "$registry/archie-agent:latest" |
grep "^$registry/archie-agent@" | head -1 | cut -d@ -f2)"
test -n "$digest"
docker build -q -f Dockerfile.archied --build-arg GATEWAY_VERSION="$version" --build-arg RUNTIME_VERSION="$version" \
-t "$registry/archied:latest" -t "$registry/archied:edge" -t "$registry/archied:sha-$sha" . >/dev/null
for t in latest edge "sha-$sha"; do docker push -q "$registry/archied:$t"; done
ldflags="-X github.com/samcharles93/archie-core/internal/installtype.buildType=binary -X github.com/samcharles93/archie-core/internal/buildinfo.Version=$version -X github.com/samcharles93/archie-core/internal/buildinfo.Runtime=$version"
name="archie-core-archied-$tag-linux-amd64"
mkdir -p "$name"
for cmd in $(jq -r '.required_topology.units[]' release.json); do
GOOS=linux GOARCH=amd64 CGO_ENABLED=0 go build -ldflags "$ldflags" -o "$name/$cmd" "./cmd/$cmd"
done
cp deployments/INSTRUCTIONS.md CHANGELOG.md "$name/"
jq --arg image "$registry/archie-agent@$digest" '. + {agent_image: $image}' release.json >"$name/release.json"
if [ -f README.md ]; then cp README.md "$name/"; fi
zip -qr "$name.zip" "$name"
sha256sum "$name.zip" >SHA256SUMS
awk -v hdr="## [$version]" 'index($0,hdr)==1{p=1} p&&/^## \[/&&index($0,hdr)!=1{exit} p{print}' CHANGELOG.md >notes.md
test -s notes.md
gh release create "$tag" -R "$releases" --title "archie $tag" --notes-file notes.md "$name.zip" SHA256SUMS install.sh
echo "published $tag to $releases (agent $digest)"