Files
archie/tools/canary-rehearse.sh
T
sam f92c5b4536
ci/woodpecker/pr/quality Pipeline was successful
Quality / check (pull_request) Canceled after 7m39s
chore(runtime): delete Kit and external agent harness support (#1877)
Archie runs its own agent loop only. Removes Kit resolution and launch,
the harness runner and adapters, the setup terminal and sandbox view,
harness secrets and OAuth capture (table dropped in migration 0075), the
task-tools contract that only Kit containers reached, and the
sandbox-kit-spec dependency. Agent profiles carry image and tools only.
The egress proxy keeps the model-session path; the sandbox relay code
stays for #1867.
2026-11-12 21:29:05 +11:00

425 lines
18 KiB
Bash
Executable File

#!/bin/sh
# Rehearse the release-canary loop against a throwaway local host. Nothing
# here touches production: a local OCI registry (docker run registry:2 on a
# loopback port), a throwaway signing key, a test release published to that
# registry (a stamped archied binary and a real agent image in a proper
# distribution zip), and a local stack (state store, gateway, archied,
# archie-ui) with its own config, state dir and ports. The loop's commands --
# release canary start, release canary status, release canary check and
# release canary promote -- are then run against that host, each step printed.
# The loop authenticates with a scoped service-account token carrying only
# the org-canary-grant policy (#1727), and a step proves that token is denied
# everything outside the grant.
# The local host has no systemd, so once the install has switched the binary
# the script restarts archied itself, the way an unmanaged host's operator
# would.
#
# A step that fails is printed and makes the run exit nonzero, so a rehearsal
# never looks greener than it was. The only hard precondition is docker:
# without it there is no registry, PostgreSQL or agent image.
#
# Never part of the gate: task check and CI do not call this.
set -eu
repo=$(CDPATH= cd -- "$(dirname -- "$0")/.." && pwd)
work=$(mktemp -d "${TMPDIR:-/tmp}/archie-canary-rehearse.XXXXXX")
name=$(basename "$work")
bin=$work/bin
logs=$work/logs
pids=""
containers=""
keep=0 # set when a step fails: the workdir is kept for inspection
cleanup() {
trap - EXIT INT TERM
for pid in $pids; do kill -- "-$pid" 2>/dev/null || kill "$pid" 2>/dev/null || true; done
for c in $containers; do docker rm -f "$c" >/dev/null 2>&1 || true; done
if [ "$keep" -gt 0 ]; then
echo "canary-rehearse: keeping $work for inspection" >&2
else
rm -rf "$work"
fi
}
trap cleanup EXIT INT TERM
die() {
echo "canary-rehearse: $*" >&2
keep=1
exit 1
}
# --- preflight ---------------------------------------------------------------
for tool in go curl python3 zip sha256sum timeout setsid openssl docker; do
command -v "$tool" >/dev/null 2>&1 || die "$tool is required but not on PATH"
done
if ! docker info >/dev/null 2>&1; then
die "docker is unavailable (docker info failed): the rehearsal needs it for the local OCI registry and PostgreSQL. Start the docker daemon or skip task canary:rehearse"
fi
release_version=${CANARY_REHEARSE_VERSION:-9.9.9-canary.1}
host_version="0.0.0-rehearse"
# Seven loopback ports, picked by binding port 0.
set -- $(python3 -c '
import socket
ports = []
while len(ports) < 7:
s = socket.socket()
s.bind(("127.0.0.1", 0))
ports.append(s.getsockname()[1])
s.close()
print(" ".join(map(str, ports)))
')
reg_port=$1 pg_port=$2 web_port=$3 state_port=$4 gw_port=$5 daemon_port=$6 health_port=$7
# --- registry and PostgreSQL -------------------------------------------------
echo "== starting local OCI registry on 127.0.0.1:$reg_port and PostgreSQL on 127.0.0.1:$pg_port"
timeout 300 docker run -d --name "$name-registry" -p "127.0.0.1:$reg_port:5000" registry:2 >/dev/null ||
die "could not start registry:2 (is the image pullable?)"
containers="$containers $name-registry"
timeout 300 docker run -d --name "$name-pg" \
-e POSTGRES_USER=archie -e POSTGRES_PASSWORD=archie -e POSTGRES_DB=archie \
-p "127.0.0.1:$pg_port:5432" postgres:18 >/dev/null ||
die "could not start postgres:18 (is the image pullable?)"
containers="$containers $name-pg"
# --- build the host's binaries (no_ui: the rehearsal never serves the SPA) ----
echo "== building archie-state-store, archie-gateway, archied, archie-ui (no_ui)"
mkdir -p "$bin" "$logs"
ldflags="-X github.com/samcharles93/archie-core/internal/installtype.buildType=binary -X github.com/samcharles93/archie-core/internal/buildinfo.Version=$host_version"
for svc in archie-state-store archie-gateway archied archie-ui; do
go -C "$repo" build -tags no_ui -trimpath -ldflags "$ldflags" -o "$bin/$svc" "./cmd/$svc" ||
die "go build ./cmd/$svc failed"
done
wait_http() { # url label tries
i=0
while [ "$i" -lt "${3:-30}" ]; do
if curl -fsS --max-time 2 "$1" >/dev/null 2>&1; then return 0; fi
i=$((i + 1)); sleep 1
done
die "$2 did not answer at $1 within ${3:-30}s"
}
wait_http "http://127.0.0.1:$reg_port/v2/" "the local registry" 30
i=0
while ! docker exec "$name-pg" pg_isready -h 127.0.0.1 -p 5432 -U archie >/dev/null 2>&1; do
i=$((i + 1))
[ "$i" -gt 90 ] && die "PostgreSQL did not become ready within 90s"
sleep 1
done
# --- throwaway signing key and a test release --------------------------------
echo "== generating a throwaway release signing key"
pub=$("$bin/archied" release keygen -seed "$work/seed") || die "release keygen failed"
[ -n "$pub" ] || die "release keygen printed no public key"
seed=$(cat "$work/seed")
# A real agent image in the local registry, so the install's image pull by
# digest succeeds. The Dockerfile stamps the agent with the release version.
echo "== building and pushing the archie-agent test image (this takes a few minutes)"
timeout 1200 docker build -q -f "$repo/Dockerfile" --build-arg RUNTIME_VERSION="$release_version" \
-t "127.0.0.1:$reg_port/archie-agent:v$release_version" "$repo" >/dev/null ||
die "docker build of the agent image failed"
timeout 600 docker push -q "127.0.0.1:$reg_port/archie-agent:v$release_version" >/dev/null ||
die "docker push of the agent image failed"
agent_image=$(docker image inspect -f '{{range .RepoDigests}}{{println .}}{{end}}' \
"127.0.0.1:$reg_port/archie-agent:v$release_version" | grep "^127\.0\.0\.1:$reg_port/" | head -1)
[ -n "$agent_image" ] || die "could not resolve the pushed agent image digest"
# A proper distribution zip, built the way deploy.yml builds it: every binary
# the repo's release.json names, each stamped with the release version, plus
# the manifest with the agent image pinned in.
dist="archie-core-archied-v$release_version-linux-amd64"
echo "== building the stamped release binaries and zipping $dist.zip"
mkdir -p "$work/dist/$dist"
release_ldflags="-X github.com/samcharles93/archie-core/internal/installtype.buildType=binary -X github.com/samcharles93/archie-core/internal/buildinfo.Version=$release_version -X github.com/samcharles93/archie-core/internal/buildinfo.Runtime=$release_version -X github.com/samcharles93/archie-core/internal/app/agentworker.version=$release_version"
for cmd in $(python3 -c 'import json; print(" ".join(json.load(open("'"$repo"'/release.json"))["required_topology"]["units"] + json.load(open("'"$repo"'/release.json"))["required_topology"].get("binaries", [])))'); do
go -C "$repo" build -tags no_ui -trimpath -ldflags "$release_ldflags" \
-o "$work/dist/$dist/$cmd" "./cmd/$cmd" || die "go build of $cmd failed"
done
python3 -c 'import json, sys
m = json.load(open(sys.argv[1])); m["agent_image"] = sys.argv[2]
print(json.dumps(m))' "$repo/release.json" "$agent_image" > "$work/dist/$dist/release.json"
(CDPATH= cd -- "$work/dist" && zip -q -r "$work/release.zip" "$dist")
echo "== publishing test release $release_version to 127.0.0.1:$reg_port/archie"
ARCHIE_RELEASE_SEED=$seed "$bin/archied" release publish \
-source "127.0.0.1:$reg_port/archie" -version "$release_version" \
-zip "$work/release.zip" -agent-image "$agent_image" ||
die "release publish failed"
# --- the local host ----------------------------------------------------------
export XDG_CONFIG_HOME=$work/config XDG_DATA_HOME=$work/data XDG_STATE_HOME=$work/state
export ARCHIE_UPDATE_STATE_DIR=$work/update-state
# Point XDG_RUNTIME_DIR at a path that doesn't exist, so systemctl --user has
# no bus: the rehearsal host reports unmanaged and the install switches the
# binaries without trying to restart units through systemd.
export XDG_RUNTIME_DIR=$work/no-user-bus
mkdir -p "$XDG_CONFIG_HOME/archie" "$XDG_DATA_HOME" "$XDG_STATE_HOME" "$ARCHIE_UPDATE_STATE_DIR"
cfg=$XDG_CONFIG_HOME/archie/config.toml
# A throwaway CA and per-service certificates, the way `task dev:tls` issues
# them: archie-ui dials the daemon contract over mTLS even on
# loopback, so the host cannot come up without [tls].
tls=$work/tls
mkdir -p "$tls"
openssl req -x509 -newkey ec -pkeyopt ec_paramgen_curve:prime256v1 -nodes -days 3650 \
-subj "/CN=archie-rehearse-ca" -keyout "$tls/ca-key.pem" -out "$tls/ca.pem" 2>/dev/null ||
die "could not create the throwaway CA"
for s in archied archie-gateway archie-messaging archie-state-store archie-ui; do
openssl req -x509 -newkey ec -pkeyopt ec_paramgen_curve:prime256v1 -nodes -days 3650 \
-subj "/CN=$s" -addext "subjectAltName=DNS:$s" -addext extendedKeyUsage=serverAuth,clientAuth \
-CA "$tls/ca.pem" -CAkey "$tls/ca-key.pem" -keyout "$tls/$s-key.pem" -out "$tls/$s.pem" 2>/dev/null ||
die "could not issue a certificate for $s"
done
sed \
-e "s|^# database_url = .*|database_url = \"postgres://archie:archie@127.0.0.1:$pg_port/archie?sslmode=disable\"|" \
-e "s|^listen = \"127.0.0.1:8485\"|listen = \"127.0.0.1:$health_port\"|" \
-e "s|^listen = \"127.0.0.1:8484\"|listen = \"127.0.0.1:$web_port\"|" \
-e "s|^target = \"127.0.0.1:8585\"|target = \"127.0.0.1:$gw_port\"\nlisten = \"127.0.0.1:$gw_port\"|" \
-e "s|^target = \"127.0.0.1:9090\"|target = \"127.0.0.1:$state_port\"\nlisten = \"127.0.0.1:$state_port\"|" \
"$repo/config.example.toml" > "$cfg"
cat >> "$cfg" <<EOF
[services.daemon]
target = "127.0.0.1:$daemon_port"
listen = "127.0.0.1:$daemon_port"
[tls]
ca_file = "$tls/ca.pem"
cert_dir = "$tls"
EOF
spawn() { # name command...
svc=$1; shift
setsid "$@" >"$logs/$svc.log" 2>&1 &
pids="$pids $!"
}
wait_port() { # port label tries
i=0
while [ "$i" -lt "${3:-90}" ]; do
if python3 -c "import socket,sys; s=socket.socket(); s.settimeout(1); sys.exit(0 if s.connect_ex(('127.0.0.1', $1)) == 0 else 1)"; then
return 0
fi
i=$((i + 1)); sleep 1
done
die "$2 did not listen on 127.0.0.1:$1; see $logs/$2.log (removed on exit -- rerun to inspect)"
}
host_pids=""
start_host() {
spawn archie-state-store "$bin/archie-state-store"
wait_port "$state_port" archie-state-store
# The gateway only listens once archied has published its embedded NATS
# endpoint, so the rest start together, as e2e/stack.sh does.
spawn archie-gateway "$bin/archie-gateway"
spawn archied "$bin/archied"
spawn archie-ui "$bin/archie-ui"
host_pids="$pids"
wait_port "$gw_port" archie-gateway
wait_port "$daemon_port" archied
wait_port "$web_port" archie-ui
}
echo "== starting the local host (state store, gateway, archied, archie-ui)"
start_host
base="http://127.0.0.1:$web_port"
cookie=$work/cookies.txt
api() { # method path [body]
method=$1 path=$2 body=${3-}
if [ -n "$body" ]; then
curl -sS --max-time 15 -b "$cookie" -c "$cookie" -X "$method" "$base$path" \
-H 'Content-Type: application/json' -H 'X-Archie-CSRF: 1' -H "Origin: $base" -d "$body"
else
curl -sS --max-time 15 -b "$cookie" -c "$cookie" "$base$path"
fi
}
py() { python3 -c "import sys, json; d = json.load(sys.stdin); print($1)"; }
# One throwaway account on the throwaway host, then a personal API token,
# which is what the canary commands authenticate with.
email="canary-rehearse@example.test"
password="archie-canary-rehearse-password"
account=$(python3 -c 'import json, sys
print(json.dumps(dict(email=sys.argv[1], password=sys.argv[2], display_name="Canary rehearsal",
organisation_name="Canary rehearsal", workspace_name="Rehearsal", public_url=sys.argv[3])))' \
"$email" "$password" "$base")
setup=$(api GET /api/auth/status)
if printf '%s' "$setup" | py 'd["setup_required"]' | grep -q True; then
api POST /api/auth/setup "$account" >/dev/null || die "account setup failed"
else
api POST /api/auth/login "$account" >/dev/null || die "login failed: $setup"
fi
token=$(api POST /api/tokens '{"name":"canary-rehearse","days":7}' | py 'd["token"]') ||
die "could not create a personal API token on the local host"
# Point the host's update settings at the local registry and key.
doc=$(api GET /api/control-plane/resources/update-settings)
ver=$(printf '%s' "$doc" | py 'd["resource"]["version"]')
value=$(python3 -c 'import json, sys
print(json.dumps({"channel": "stable", "pin": "", "source": sys.argv[1], "key": sys.argv[2]}))' \
"127.0.0.1:$reg_port/archie" "$pub")
resp=$(api POST /api/control-plane/resources/update-settings/commands/replace \
"{\"value\":$value,\"expected_version\":$ver}")
printf '%s' "$resp" | grep -q '"resource"' || die "could not set update-settings on the local host: $resp"
# --- the scoped canary grant (#1727): a service account, not instance admin --
# Create a service_account identity, mint a token for it, install the
# org-canary-grant policy, then prove the token can read the signals but
# nothing else.
echo
echo "== creating the canary service account and installing the scoped grant"
canary_id=$(api POST /api/identities '{"display_name":"canary","kind":"service_account"}' | py 'd["id"]') ||
die "could not create the canary service account"
[ -n "$canary_id" ] || die "identity create printed no id"
scoped=$(api POST "/api/identities/$canary_id/tokens" '{"name":"canary","days":7}' | py 'd["token"]') ||
die "could not mint a token for the canary identity"
[ -n "$scoped" ] || die "token create printed no token"
grant_policy() { # id prefix permit|forbid -> policy JSON on stdout
python3 -c 'import json, sys
who, effect, identity = sys.argv[1], sys.argv[2], sys.argv[3]
actions = """(
(action == Archie::Action::"read" && ["dashboard", "task"].contains(resource.kind)) ||
(action == Archie::Action::"read" && resource == Archie::Object::"update-settings") ||
(action == Archie::Action::"update" && resource == Archie::Object::"update") ||
(action == Archie::Action::"manage_identities" && resource == Archie::Object::"update-settings")
)"""
body = """ resource.org == principal.org &&
principal == Archie::Identity::"%s" &&
%s%s""" % (identity, "!" if effect == "forbid" else "", actions)
print(json.dumps({"id": "org-canary-grant" + ("-forbid" if effect == "forbid" else ""),
"level": "org", "org_id": "org-sys",
"text": "%s(principal, action, resource) when {\n%s\n};" % (effect, body)}))' "$1" "$2" "$canary_id"
}
for effect in permit forbid; do
resp=$(api PUT /api/access/policies "$(grant_policy grant "$effect")")
printf '%s' "$resp" | grep -q '"version"' || die "could not install the canary grant $effect policy: $resp"
done
bearer() { # path -> curl with the scoped token, prints the status code
curl -sS --max-time 15 -o /dev/null -w '%{http_code}' \
-H "Authorization: Bearer $scoped" "$base$1"
}
bearer_body() { # path -> curl with the scoped token, prints the body
curl -sS --max-time 15 -H "Authorization: Bearer $scoped" "$base$1"
}
# The policy chain reloads asynchronously; allow a few seconds for the grant.
# The negative probe can run now; the positive one must wait for the canary
# loop to switch the channel to next (a missing stable tag is a 502 from the
# update check, not an authorization answer).
i=0
code=000
while :; do
code=$(bearer /api/access/policies)
[ "$code" != "200" ] && break
i=$((i + 1)); [ "$i" -gt 45 ] && die "the scoped canary token can read the policies API: the grant is not scoped"
sleep 1
done
echo "scoped token reads /api/access/policies: $code (denied, as intended)"
# --- the canary loop, each step printed ---------------------------------------
failed=0
run_step() { # description command...
desc=$1; shift
echo
echo "== $desc"
printf '$ %s\n' "$*"
if "$@" 2>&1; then
echo "-- ok"
else
rc=$?
echo "-- FAILED (exit $rc)"
failed=$((failed + 1))
keep=1
fi
}
# The canary loop runs with the scoped service-account token, not the admin
# one: that is the point of #1727.
export ARCHIE_CANARY_TOKEN=$scoped
echo
echo "== release canary start -- point the host at next and install $release_version"
printf '$ archied release canary start -dashboard %s -version %s -source 127.0.0.1:%s/archie -key <pub>\n' \
"$base" "$release_version" "$reg_port"
timeout 600 "$bin/archied" release canary start -dashboard "$base" -version "$release_version" \
-source "127.0.0.1:$reg_port/archie" -key "$pub" >"$work/start.out" 2>&1 &
start_pid=$!
# The local host has no systemd: the install switches the binary and records
# the install time, but nothing restarts the daemon. Restart archied once the
# install has committed, the way an unmanaged host's operator would, so the
# host reports the new version.
i=0
while [ ! -f "$ARCHIE_UPDATE_STATE_DIR/installed.json" ]; do
kill -0 "$start_pid" 2>/dev/null || break
i=$((i + 1)); [ "$i" -gt 300 ] && break
sleep 1
done
if [ -f "$ARCHIE_UPDATE_STATE_DIR/installed.json" ]; then
# Restart every service, not just the daemon: the update snapshot reports
# the serving binary's version, so the whole host must come back on the
# new release.
for pid in $host_pids; do kill "$pid" 2>/dev/null || true; done
sleep 1
pids=""
start_host
fi
start_rc=0
wait "$start_pid" || start_rc=$?
cat "$work/start.out"
if [ "$start_rc" -eq 0 ]; then
echo "-- ok"
else
echo "-- FAILED (exit $start_rc)"
failed=$((failed + 1))
keep=1
fi
# Now that the canary follows next, the scoped token's positive probe: the
# policy chain reloads asynchronously, so allow a few seconds.
i=0
code=000
while :; do
code=$(bearer /api/chat/update)
[ "$code" = "200" ] && break
i=$((i + 1)); [ "$i" -gt 45 ] && die "the scoped canary token never got read access to /api/chat/update (last status $code: $(bearer_body /api/chat/update))"
sleep 1
done
echo "scoped token reads /api/chat/update: 200"
run_step "release canary status -- what the canary runs and how the window is going" \
timeout 120 "$bin/archied" release canary status -dashboard "$base" \
-source "127.0.0.1:$reg_port/archie" -key "$pub"
run_step "release canary check -- judge the installed release (still settling right after the install)" \
timeout 300 "$bin/archied" release canary check -dashboard "$base" \
-source "127.0.0.1:$reg_port/archie" -key "$pub" -checkout "$repo"
run_step "release canary promote -- promote $release_version now that its hard checks pass" \
timeout 120 "$bin/archied" release canary promote -dashboard "$base" -version "$release_version" \
-source "127.0.0.1:$reg_port/archie" -key "$pub"
echo
if [ "$failed" -gt 0 ]; then
echo "canary-rehearse: $failed step(s) failed; the steps above show why." >&2
if [ "$keep" -gt 0 ] && [ -d "$logs" ]; then
for f in "$logs"/*.log; do
echo "--- $f" >&2
tail -n 20 "$f" >&2
done
fi
exit 1
fi
echo "canary-rehearse: every step ran clean; host was $base, registry 127.0.0.1:$reg_port (both torn down)"