feat(e2e): local stack, seed and browser checks for release validation

This commit is contained in:
sam committed 2026-10-12 04:28:26 +11:00
1 parent 2df2f72fc2
commit 94d2f98cc0
11 files changed
+432 -13

No files matched your search

+16
View File
@@ -245,6 +245,22 @@ tasks:
- npm install --no-audit --no-fund
- npm run dev
e2e:up:
desc: Start the local stack without Docker (PostgreSQL 18, services, Vite) and seed it
cmds:
- bash e2e/stack.sh start
- bash e2e/seed.sh
e2e:down:
desc: Stop the local e2e stack
cmds:
- bash e2e/stack.sh stop
e2e:check:
desc: Run the browser checks against the running e2e stack (-- --record VERSION for release evidence)
cmds:
- node e2e/run.cjs {{.CLI_ARGS}}
dev:mobile:
desc: Serve the dashboard on your LAN (token-protected) for phone viewing
deps:
+2
View File
@@ -0,0 +1,2 @@
node_modules/
out/
+61
View File
@@ -0,0 +1,61 @@
# e2e
Validation of the whole stack, run by hand or by an agent, never by
`task check` or CI. It needs PostgreSQL 18, five services, Vite and a browser,
which the gate must not. Docker is not needed: without it tasks queue and
park, which is useful data for the dashboard.
| File | What it does |
| --- | --- |
| `stack.sh` | Starts PostgreSQL 18 (port 55432) and the five services as local processes, then Vite on 5173 |
| `seed.sh` | Seeds the stack through the dashboard API; safe to rerun |
| `run.cjs` | Runs `checks/*.cjs` in headless Chromium; screenshots to `out/` |
| `checks/` | One module per behaviour |
| `results/` | One committed report per release |
## Run
```bash
task e2e:up # build, start everything, seed
task e2e:check # all checks; or: node e2e/run.cjs routes
task e2e:down # stop; e2e/stack.sh reset also deletes the data
```
Nothing reads the installed config: `stack.sh` points the XDG directories at
`${ARCHIE_DEV_DIR:-~/.local/state/archie-dev}` and writes its overlay from
`deployments/dev.toml`. Logs are in its `logs/`. `e2e/stack.sh restart archied`
restarts one service.
The first run needs `npm ci --prefix e2e` and `npx --prefix e2e playwright
install chromium`. In the cloud environment, Chromium and Playwright are
preinstalled; run with `NODE_PATH=$(npm root -g)`.
## Release evidence
```bash
node e2e/run.cjs --record v1.54.0
```
writes `results/v1.54.0.md`, the pass or fail of every check for that
release. Commit it with the release. Screenshots stay in `out/`.
## Writing a check
A module in `checks/` exports `async ({ page, go, shot, fail })`. `go(path)`
loads a dashboard route, `shot(label)` saves a screenshot, `fail(msg)` records
a problem without stopping. A throw, a `fail`, or any console or page error
fails the check. Wait for elements, never for `networkidle`: the event stream
never goes idle.
`routes` loads every static route and flags text with stray leading,
trailing or doubled spaces. `seeded` checks the seed is visible.
## Pitfalls
| Symptom | Cause | Fix |
| --- | --- | --- |
| State Store exits: server is older than PostgreSQL 18 | another server answered | `stack.sh` uses port 55432; set `ARCHIE_DEV_PGPORT` if that is taken |
| Work request: repository is not configured | archied reads repository policies only at boot | `seed.sh` restarts it; do the same after changing policies |
| Your own shell is killed (exit 144) | `pkill -f` matched its own command line | `stack.sh stop` or `restart`, which kill by PID |
| `npx shadcn-vue add` cannot fetch the registry | Node's fetch ignores the cloud agent proxy | `curl` the registry JSON and copy the files |
| Postgres aborts in the cloud | `/tmp/claude-0` is reset to mode 700 | `stack.sh` keeps the root cluster in `/var/lib/archie-dev` |
+36
View File
@@ -0,0 +1,36 @@
// Every static route loads without errors, renders, and shows no text with
// stray leading, trailing or doubled spaces.
const fs = require('fs');
const path = require('path');
function routes() {
const src = fs.readFileSync(path.join(__dirname, '../../ui/src/router/index.ts'), 'utf8');
const out = [];
for (const line of src.split('\n')) {
if (/redirect:/.test(line)) continue;
const m = line.match(/path: "([^"]*)"/);
if (!m || m[1].includes(':')) continue;
if (m[1].startsWith('/')) out.push(m[1]);
else if (m[1]) out.push('/org/' + m[1]);
}
return [...new Set(out)];
}
module.exports = async ({ page, go, shot, fail }) => {
const stray = new Map();
for (const r of routes()) {
await go(r);
await shot(r);
// Log lines are data, not copy.
if (r === '/logs') continue;
const bad = await page.evaluate(() => [...document.querySelectorAll('body *')]
.filter((el) => !el.children.length && !el.closest('pre,code,textarea,.xterm') && el.getClientRects().length && !getComputedStyle(el).whiteSpace.startsWith('pre'))
.map((el) => el.textContent)
.filter((t) => t.trim() && (t !== t.trim() || / /.test(t.trim()))));
for (const t of bad) {
const key = JSON.stringify(t.length > 60 ? t.slice(0, 60) + '…' : t);
stray.set(key, (stray.get(key) || new Set()).add(r));
}
}
for (const [text, where] of stray) fail(`stray whitespace ${text} on ${[...where].join(', ')}`);
};
+9
View File
@@ -0,0 +1,9 @@
// The seed (e2e/seed.sh) is visible where an operator would look for it.
module.exports = async ({ page, go, shot }) => {
await go('/tasks');
await page.getByText('Fix flaky login test').first().waitFor({ timeout: 10000 });
await shot('tasks');
await go('/events');
await page.getByText('e2e-hook').first().waitFor({ timeout: 10000 });
await shot('events');
};
+42
View File
@@ -0,0 +1,42 @@
{
"name": "archie-e2e",
"lockfileVersion": 3,
"requires": true,
"packages": {
"": {
"name": "archie-e2e",
"devDependencies": {
"playwright": "1.64.0"
}
},
"node_modules/playwright": {
"version": "1.64.0",
"resolved": "https://registry.npmjs.org/playwright/-/playwright-1.64.0.tgz",
"integrity": "sha512-kVzTnYFYEyQ9RGzBjNCd2+yEqWmVbxl3AHPynv3qQ6JTjv4a+kwRcnqQwIpuP3NnrnIrE/Q6wWAIAvMfIE2/Yg==",
"dev": true,
"license": "Apache-2.0",
"dependencies": {
"playwright-core": "1.64.0"
},
"bin": {
"playwright": "cli.js"
},
"engines": {
"node": ">=20"
}
},
"node_modules/playwright-core": {
"version": "1.64.0",
"resolved": "https://registry.npmjs.org/playwright-core/-/playwright-core-1.64.0.tgz",
"integrity": "sha512-T9r+MZkTECl2+oUcZ26YgZzTrYidTsIhYYkJix+6iDOymOb1FB2RLLSOxRGnZGv5DStt+aduIVbt+E8WygXYpg==",
"dev": true,
"license": "Apache-2.0",
"bin": {
"playwright-core": "cli.js"
},
"engines": {
"node": ">=20"
}
}
}
}
+11
View File
@@ -0,0 +1,11 @@
{
"name": "archie-e2e",
"private": true,
"description": "Validation of a running local stack; never part of the gate",
"scripts": {
"check": "node run.cjs"
},
"devDependencies": {
"playwright": "1.64.0"
}
}
+62
View File
@@ -0,0 +1,62 @@
// Drive the dashboard of a running local stack (e2e/stack.sh) in headless
// Chromium. Each module in checks/ exports async ({ page, go, shot, fail }).
// A check fails on a thrown error or any console or page error.
//
// node e2e/run.cjs [--record VERSION] [check ...]
//
// Screenshots and results.json go to e2e/out/<stamp>/. --record also writes
// e2e/results/VERSION.md, the committed evidence for a release.
const fs = require('fs');
const path = require('path');
const { chromium } = require('playwright');
const base = process.env.ARCHIE_E2E_URL || 'http://localhost:5173';
const args = process.argv.slice(2);
let record = '';
if (args[0] === '--record') { record = args[1]; args.splice(0, 2); }
const names = args.length ? args : fs.readdirSync(path.join(__dirname, 'checks')).filter((f) => f.endsWith('.cjs')).map((f) => f.slice(0, -4)).sort();
const stamp = new Date().toISOString().replace(/[:.]/g, '-');
const out = path.join(__dirname, 'out', stamp);
fs.mkdirSync(out, { recursive: true });
(async () => {
const browser = await chromium.launch();
const results = [];
for (const name of names) {
const check = require(path.join(__dirname, 'checks', name + '.cjs'));
// A fresh page per check: open event streams pile up across navigations.
const page = await browser.newPage({ viewport: { width: 1440, height: 900 } });
const errors = [];
const notes = [];
page.on('pageerror', (e) => errors.push(String(e)));
page.on('console', (m) => m.type() === 'error' && errors.push(m.text()));
// Wait for load, never networkidle: the event stream never goes idle.
const go = async (p) => { await page.goto(base + p, { waitUntil: 'load' }); await page.waitForTimeout(1200); };
const shot = (label) => page.screenshot({ path: path.join(out, `${name}-${label.replace(/[^a-z0-9-]+/gi, '_') || 'root'}.png`) });
const fail = (msg) => notes.push(msg);
const started = Date.now();
let error = '';
try { await check({ page, go, shot, fail }); } catch (e) { error = String(e && e.message || e); }
await page.close();
const problems = [error, ...notes, ...errors].filter(Boolean);
results.push({ check: name, ok: problems.length === 0, ms: Date.now() - started, problems });
console.log(`${problems.length ? 'FAIL' : 'ok '} ${name}${problems.length ? '\n ' + problems.join('\n ') : ''}`);
}
await browser.close();
fs.writeFileSync(path.join(out, 'results.json'), JSON.stringify(results, null, 2));
if (record) writeRecord(record, results);
console.log(`evidence: ${out}`);
process.exit(results.every((r) => r.ok) ? 0 : 1);
})().catch((e) => { console.error(e); process.exit(2); });
function writeRecord(version, results) {
const dir = path.join(__dirname, 'results');
fs.mkdirSync(dir, { recursive: true });
const rows = results.map((r) => `| ${r.check} | ${r.ok ? 'pass' : 'fail'} | ${r.problems.join('<br>').replace(/\|/g, '\\|') || ''} |`);
const passed = results.filter((r) => r.ok).length;
fs.writeFileSync(path.join(dir, `${version}.md`), [
`# ${version}`, '',
`Run ${new Date().toISOString()} against ${base}: ${passed} of ${results.length} checks passed.`, '',
'| Check | Result | Problems |', '| --- | --- | --- |', ...rows, '',
].join('\n'));
}
Executable
+51
View File
@@ -0,0 +1,51 @@
#!/usr/bin/env bash
# Seed a running local stack (e2e/stack.sh) through the dashboard's own HTTP
# API, so the seed exercises real validation. Safe to run again: each step
# skips what already exists.
set -euo pipefail
base=${ARCHIE_E2E_API:-http://localhost:8484}
here=$(cd "$(dirname "$0")" && pwd)
H=(-H 'Content-Type: application/json' -H 'X-Archie-CSRF: 1' -H "Origin: $base")
get() { curl -fsS "$base$1"; }
post() { curl -fsS -X POST "$base$1" "${H[@]}" -d "$2"; }
# jq-free JSON reads: py EXPR, with the parsed body as d.
py() { python3 -c "import sys, json; d = json.load(sys.stdin); print($1)"; }
# Repository policy. archied reads policies only at boot, so restart it.
if ! get /api/control-plane/resources/repository-policies | grep -q '"widgets"'; then
version=$(get /api/control-plane/resources/repository-policies | py 'd["resource"]["version"]')
post /api/control-plane/resources/repository-policies/commands/replace \
"{\"value\":[{\"owner\":\"acme\",\"name\":\"widgets\",\"base\":\"main\"}],\"expected_version\":$version}" >/dev/null
"$here/stack.sh" restart archied
sleep 5
fi
# Work requests. Without Docker they park within a minute, which fills
# Needs you, Quality gates and the task pages.
tasks=$(get /api/tasks)
for t in "Fix flaky login test" "Add retry to webhook sender" "Document the gate" "Speed up the CI cache"; do
grep -qF "\"$t\"" <<<"$tasks" && continue
post /api/work-requests "{\"identity\":\"archie-dev\",\"repository\":\"acme/widgets\",\"workflow\":\"triage\",\"title\":\"$t\",\"instructions\":\"Do it.\"}" >/dev/null
done
# An unsigned capture source with one delivery, an event type, a mapping and
# a binding left pending approval.
src=e2e-hook
if ! get /api/sources | grep -q "\"$src\""; then
post /api/sources "{\"path\":\"$src\"}" >/dev/null
post "/api/sources/$src/signing" '{"signed":false}' >/dev/null
post "/api/sources/$src/approve-unsigned" '{}' >/dev/null
bash "$here/../tools/webhook-send.sh" --url "$base" --source "$src" --kind github-issue >/dev/null
fi
et=$(get /api/event-types | py '"".join(t["id"] for t in (d.get("event_types", []) if isinstance(d, dict) else d) if t["name"] == "issue-opened")')
if [ -z "$et" ]; then
et=$(post /api/event-types "{\"source\":\"$src\",\"name\":\"issue-opened\",\"rule\":{\"headers\":[],\"payload\":[{\"path\":\"action\",\"op\":\"equals\",\"value\":\"opened\"}]},\"example\":{\"headers\":{},\"body\":\"{\\\"action\\\":\\\"opened\\\",\\\"issue\\\":{\\\"title\\\":\\\"x\\\"}}\"}}" | py 'd["id"]')
fi
if ! get /api/bindings | grep -q '"e2e-binding"'; then
mapping=$(post /api/mappings "{\"name\":\"e2e-issue\",\"source_hint\":\"$src\",\"event_type_id\":\"$et\",\"fields\":[{\"name\":\"title\",\"path\":\"issue.title\",\"type\":\"string\",\"required\":true}]}" | py 'd["id"]')
post /api/bindings "{\"name\":\"e2e-binding\",\"matcher\":{\"source\":\"$src\"},\"mapping_id\":\"$mapping\",\"workflow\":\"feasibility\",\"owner\":\"acme\",\"repo\":\"widgets\"}" >/dev/null
fi
echo "seeded: $(get /api/tasks | py 'len(d.get("tasks", d) if isinstance(d, dict) else d)') tasks"
Executable
+140
View File
@@ -0,0 +1,140 @@
#!/usr/bin/env bash
# Run the whole stack as local processes against a throwaway PostgreSQL 18,
# for manual and scripted checks. Never part of the gate. Nothing here reads
# the installed config: XDG paths point into the dev directory.
#
# e2e/stack.sh start | stop | restart <service> | status | reset
set -euo pipefail
repo=$(cd "$(dirname "$0")/.." && pwd)
dir=${ARCHIE_DEV_DIR:-${XDG_STATE_HOME:-$HOME/.local/state}/archie-dev}
pgport=${ARCHIE_DEV_PGPORT:-55432}
services=(archie-state-store archie-gateway archied archie-messaging archie-ui)
# initdb refuses root, and postgres cannot traverse root's home.
if [ "$(id -u)" -eq 0 ]; then
pgdata=/var/lib/archie-dev/pg
as_pg() { runuser -u postgres -- "$@"; }
else
pgdata=$dir/pg
as_pg() { "$@"; }
fi
export XDG_CONFIG_HOME=$dir/config XDG_DATA_HOME=$dir/data XDG_STATE_HOME=$dir/state
pgbin() {
local d
for d in /usr/lib/postgresql/18/bin "$(dirname "$(command -v postgres 2>/dev/null || echo /x/x)")"; do
if [ -x "$d/postgres" ] && "$d/postgres" --version | grep -q ' 18\.'; then
echo "$d"
return
fi
done
echo "PostgreSQL 18 server not found" >&2
exit 1
}
wait_port() {
for _ in $(seq 1 "${3:-60}"); do
(echo >"/dev/tcp/127.0.0.1/$1") 2>/dev/null && return 0
sleep 1
done
echo "$2 did not listen on $1; see $dir/logs/$2.log" >&2
return 1
}
start_pg() {
local bin
bin=$(pgbin)
if [ ! -f "$pgdata/PG_VERSION" ]; then
mkdir -p "$pgdata"
[ "$(id -u)" -ne 0 ] || chown postgres "$pgdata"
as_pg "$bin/initdb" -D "$pgdata" -U archie --auth=trust >/dev/null
fi
if ! as_pg "$bin/pg_ctl" -D "$pgdata" status >/dev/null 2>&1; then
as_pg "$bin/pg_ctl" -D "$pgdata" -w -l "$pgdata/server.log" \
-o "-p $pgport -k $pgdata -c listen_addresses=127.0.0.1" start >/dev/null
fi
"$bin/psql" -h 127.0.0.1 -p "$pgport" -U archie -d postgres -qtAc \
"SELECT 1 FROM pg_database WHERE datname = 'archie'" | grep -q 1 ||
"$bin/createdb" -h 127.0.0.1 -p "$pgport" -U archie archie
}
prepare() {
mkdir -p "$dir"/{bin,logs,pids} "$XDG_CONFIG_HOME/archie" "$XDG_DATA_HOME" "$XDG_STATE_HOME"
[ -f "$XDG_CONFIG_HOME/archie/config.toml" ] || cp "$repo/config.example.toml" "$XDG_CONFIG_HOME/archie/config.toml"
sed "s#^database_url = .*#database_url = \"postgres://archie@127.0.0.1:$pgport/archie?sslmode=disable\"#" \
"$repo/deployments/dev.toml" >"$dir/dev.toml"
local s
for s in "${services[@]}"; do
(cd "$repo" && go build -o "$dir/bin/$s" "./cmd/$s")
done
}
# spawn NAME DIR CMD...: a detached session leader, so stop takes its children.
spawn() {
local name=$1 cwd=$2
shift 2
setsid -f bash -c 'cd "$1" && echo $$ >"$2" && shift 2 && exec "$@"' _ "$cwd" "$dir/pids/$name" "$@" \
</dev/null >"$dir/logs/$name.log" 2>&1
}
start_service() {
spawn "$1" "$repo" "$dir/bin/$1" -config-overlay "$dir/dev.toml"
}
stop_pid() {
local f=$1 pid
[ -f "$f" ] || return 0
pid=$(cat "$f")
if kill -- "-$pid" 2>/dev/null; then
for _ in $(seq 1 20); do kill -0 "$pid" 2>/dev/null || break; sleep 0.5; done
kill -9 -- "-$pid" 2>/dev/null || true
fi
rm -f "$f"
}
start() {
prepare
start_pg
start_service archie-state-store
wait_port 9091 archie-state-store 90
local s
for s in archie-gateway archied archie-messaging archie-ui; do start_service "$s"; done
wait_port 8484 archie-ui
[ -d "$repo/ui/node_modules" ] || (cd "$repo/ui" && npm ci --no-audit --no-fund >/dev/null)
spawn vite "$repo/ui" npx vite --host 127.0.0.1 --port 5173 --strictPort
wait_port 5173 vite
echo "dashboard: http://localhost:5173 logs: $dir/logs"
}
stop() {
local f
for f in "$dir"/pids/*; do stop_pid "$f"; done
if [ -f "$pgdata/PG_VERSION" ]; then
as_pg "$(pgbin)/pg_ctl" -D "$pgdata" -m fast stop >/dev/null 2>&1 || true
fi
}
case ${1:-} in
start) start ;;
stop) stop ;;
restart)
stop_pid "$dir/pids/${2:?service}"
start_service "$2"
;;
status)
for f in "$dir"/pids/*; do
[ -f "$f" ] || continue
if kill -0 "$(cat "$f")" 2>/dev/null; then echo "up $(basename "$f")"; else echo "down $(basename "$f")"; fi
done
;;
reset)
stop
rm -rf "$dir" "$pgdata"
;;
*)
echo "usage: $0 start | stop | restart <service> | status | reset" >&2
exit 2
;;
esac
+2 -13
View File
@@ -47,24 +47,13 @@ go install "google.golang.org/grpc/cmd/protoc-gen-go-grpc@$PROTOC_GEN_GO_GRPC_VE
curl -fsSL https://raw.githubusercontent.com/gastownhall/beads/main/scripts/install.sh | bash
(cd "$HOME" && bd setup claude --global)
# PostgreSQL 18 for running full checks against the real stack
# (deployments/dev.toml). Never for tests: they use fakes.
# PostgreSQL 18 for e2e/stack.sh, which runs its own cluster. Never for
# tests: they use fakes.
if [ ! -x /usr/lib/postgresql/18/bin/postgres ]; then
$SUDO apt-get install -y --no-install-recommends postgresql-common
$SUDO /usr/share/postgresql-common/pgdg/apt.postgresql.org.sh -y
$SUDO apt-get install -y --no-install-recommends postgresql-18
fi
for v in $(pg_lsclusters -h | awk '$1 != 18 {print $1}'); do
$SUDO pg_ctlcluster "$v" main stop || true
$SUDO pg_dropcluster "$v" main
done
pg_lsclusters -h | grep -q '^18 main' || $SUDO pg_createcluster 18 main --port 5432
$SUDO pg_ctlcluster 18 main start || true
as_pg() { $SUDO runuser -u postgres -- "$@"; }
as_pg psql -qtc "SELECT 1 FROM pg_roles WHERE rolname = 'archie'" | grep -q 1 ||
as_pg psql -qc "CREATE USER archie WITH PASSWORD 'archie' SUPERUSER"
as_pg psql -qtc "SELECT 1 FROM pg_database WHERE datname = 'archie'" | grep -q 1 ||
as_pg createdb -O archie archie
# Commit and PR attribution stays off, matching local settings.
mkdir -p "$HOME/.claude"