From 94d2f98cc014f6d80c4e28e61d4995120d214ce8 Mon Sep 17 00:00:00 2001 From: Sam Catlow Date: Mon, 12 Oct 2026 04:28:26 +1100 Subject: [PATCH] feat(e2e): local stack, seed and browser checks for release validation --- Taskfile.yml | 16 +++++ e2e/.gitignore | 2 + e2e/README.md | 61 ++++++++++++++++++ e2e/checks/routes.cjs | 36 +++++++++++ e2e/checks/seeded.cjs | 9 +++ e2e/package-lock.json | 42 +++++++++++++ e2e/package.json | 11 ++++ e2e/run.cjs | 62 +++++++++++++++++++ e2e/seed.sh | 51 +++++++++++++++ e2e/stack.sh | 140 ++++++++++++++++++++++++++++++++++++++++++ tools/cloud-setup.sh | 15 +---- 11 files changed, 432 insertions(+), 13 deletions(-) create mode 100644 e2e/.gitignore create mode 100644 e2e/README.md create mode 100644 e2e/checks/routes.cjs create mode 100644 e2e/checks/seeded.cjs create mode 100644 e2e/package-lock.json create mode 100644 e2e/package.json create mode 100644 e2e/run.cjs create mode 100755 e2e/seed.sh create mode 100755 e2e/stack.sh diff --git a/Taskfile.yml b/Taskfile.yml index 3be922e81..ef57d16ac 100644 --- a/Taskfile.yml +++ b/Taskfile.yml @@ -245,6 +245,22 @@ tasks: - npm install --no-audit --no-fund - npm run dev + e2e:up: + desc: Start the local stack without Docker (PostgreSQL 18, services, Vite) and seed it + cmds: + - bash e2e/stack.sh start + - bash e2e/seed.sh + + e2e:down: + desc: Stop the local e2e stack + cmds: + - bash e2e/stack.sh stop + + e2e:check: + desc: Run the browser checks against the running e2e stack (-- --record VERSION for release evidence) + cmds: + - node e2e/run.cjs {{.CLI_ARGS}} + dev:mobile: desc: Serve the dashboard on your LAN (token-protected) for phone viewing deps: diff --git a/e2e/.gitignore b/e2e/.gitignore new file mode 100644 index 000000000..72aae85fa --- /dev/null +++ b/e2e/.gitignore @@ -0,0 +1,2 @@ +node_modules/ +out/ diff --git a/e2e/README.md b/e2e/README.md new file mode 100644 index 000000000..dd85b07e5 --- /dev/null +++ b/e2e/README.md @@ -0,0 +1,61 @@ +# e2e + +Validation of the whole stack, run by hand or by an agent, never by +`task check` or CI. It needs PostgreSQL 18, five services, Vite and a browser, +which the gate must not. Docker is not needed: without it tasks queue and +park, which is useful data for the dashboard. + +| File | What it does | +| --- | --- | +| `stack.sh` | Starts PostgreSQL 18 (port 55432) and the five services as local processes, then Vite on 5173 | +| `seed.sh` | Seeds the stack through the dashboard API; safe to rerun | +| `run.cjs` | Runs `checks/*.cjs` in headless Chromium; screenshots to `out/` | +| `checks/` | One module per behaviour | +| `results/` | One committed report per release | + +## Run + +```bash +task e2e:up # build, start everything, seed +task e2e:check # all checks; or: node e2e/run.cjs routes +task e2e:down # stop; e2e/stack.sh reset also deletes the data +``` + +Nothing reads the installed config: `stack.sh` points the XDG directories at +`${ARCHIE_DEV_DIR:-~/.local/state/archie-dev}` and writes its overlay from +`deployments/dev.toml`. Logs are in its `logs/`. `e2e/stack.sh restart archied` +restarts one service. + +The first run needs `npm ci --prefix e2e` and `npx --prefix e2e playwright +install chromium`. In the cloud environment, Chromium and Playwright are +preinstalled; run with `NODE_PATH=$(npm root -g)`. + +## Release evidence + +```bash +node e2e/run.cjs --record v1.54.0 +``` + +writes `results/v1.54.0.md`, the pass or fail of every check for that +release. Commit it with the release. Screenshots stay in `out/`. + +## Writing a check + +A module in `checks/` exports `async ({ page, go, shot, fail })`. `go(path)` +loads a dashboard route, `shot(label)` saves a screenshot, `fail(msg)` records +a problem without stopping. A throw, a `fail`, or any console or page error +fails the check. Wait for elements, never for `networkidle`: the event stream +never goes idle. + +`routes` loads every static route and flags text with stray leading, +trailing or doubled spaces. `seeded` checks the seed is visible. + +## Pitfalls + +| Symptom | Cause | Fix | +| --- | --- | --- | +| State Store exits: server is older than PostgreSQL 18 | another server answered | `stack.sh` uses port 55432; set `ARCHIE_DEV_PGPORT` if that is taken | +| Work request: repository is not configured | archied reads repository policies only at boot | `seed.sh` restarts it; do the same after changing policies | +| Your own shell is killed (exit 144) | `pkill -f` matched its own command line | `stack.sh stop` or `restart`, which kill by PID | +| `npx shadcn-vue add` cannot fetch the registry | Node's fetch ignores the cloud agent proxy | `curl` the registry JSON and copy the files | +| Postgres aborts in the cloud | `/tmp/claude-0` is reset to mode 700 | `stack.sh` keeps the root cluster in `/var/lib/archie-dev` | diff --git a/e2e/checks/routes.cjs b/e2e/checks/routes.cjs new file mode 100644 index 000000000..f0ad83002 --- /dev/null +++ b/e2e/checks/routes.cjs @@ -0,0 +1,36 @@ +// Every static route loads without errors, renders, and shows no text with +// stray leading, trailing or doubled spaces. +const fs = require('fs'); +const path = require('path'); + +function routes() { + const src = fs.readFileSync(path.join(__dirname, '../../ui/src/router/index.ts'), 'utf8'); + const out = []; + for (const line of src.split('\n')) { + if (/redirect:/.test(line)) continue; + const m = line.match(/path: "([^"]*)"/); + if (!m || m[1].includes(':')) continue; + if (m[1].startsWith('/')) out.push(m[1]); + else if (m[1]) out.push('/org/' + m[1]); + } + return [...new Set(out)]; +} + +module.exports = async ({ page, go, shot, fail }) => { + const stray = new Map(); + for (const r of routes()) { + await go(r); + await shot(r); + // Log lines are data, not copy. + if (r === '/logs') continue; + const bad = await page.evaluate(() => [...document.querySelectorAll('body *')] + .filter((el) => !el.children.length && !el.closest('pre,code,textarea,.xterm') && el.getClientRects().length && !getComputedStyle(el).whiteSpace.startsWith('pre')) + .map((el) => el.textContent) + .filter((t) => t.trim() && (t !== t.trim() || / /.test(t.trim())))); + for (const t of bad) { + const key = JSON.stringify(t.length > 60 ? t.slice(0, 60) + '…' : t); + stray.set(key, (stray.get(key) || new Set()).add(r)); + } + } + for (const [text, where] of stray) fail(`stray whitespace ${text} on ${[...where].join(', ')}`); +}; diff --git a/e2e/checks/seeded.cjs b/e2e/checks/seeded.cjs new file mode 100644 index 000000000..ba4a702b1 --- /dev/null +++ b/e2e/checks/seeded.cjs @@ -0,0 +1,9 @@ +// The seed (e2e/seed.sh) is visible where an operator would look for it. +module.exports = async ({ page, go, shot }) => { + await go('/tasks'); + await page.getByText('Fix flaky login test').first().waitFor({ timeout: 10000 }); + await shot('tasks'); + await go('/events'); + await page.getByText('e2e-hook').first().waitFor({ timeout: 10000 }); + await shot('events'); +}; diff --git a/e2e/package-lock.json b/e2e/package-lock.json new file mode 100644 index 000000000..c5bce8f7d --- /dev/null +++ b/e2e/package-lock.json @@ -0,0 +1,42 @@ +{ + "name": "archie-e2e", + "lockfileVersion": 3, + "requires": true, + "packages": { + "": { + "name": "archie-e2e", + "devDependencies": { + "playwright": "1.64.0" + } + }, + "node_modules/playwright": { + "version": "1.64.0", + "resolved": "https://registry.npmjs.org/playwright/-/playwright-1.64.0.tgz", + "integrity": "sha512-kVzTnYFYEyQ9RGzBjNCd2+yEqWmVbxl3AHPynv3qQ6JTjv4a+kwRcnqQwIpuP3NnrnIrE/Q6wWAIAvMfIE2/Yg==", + "dev": true, + "license": "Apache-2.0", + "dependencies": { + "playwright-core": "1.64.0" + }, + "bin": { + "playwright": "cli.js" + }, + "engines": { + "node": ">=20" + } + }, + "node_modules/playwright-core": { + "version": "1.64.0", + "resolved": "https://registry.npmjs.org/playwright-core/-/playwright-core-1.64.0.tgz", + "integrity": "sha512-T9r+MZkTECl2+oUcZ26YgZzTrYidTsIhYYkJix+6iDOymOb1FB2RLLSOxRGnZGv5DStt+aduIVbt+E8WygXYpg==", + "dev": true, + "license": "Apache-2.0", + "bin": { + "playwright-core": "cli.js" + }, + "engines": { + "node": ">=20" + } + } + } +} diff --git a/e2e/package.json b/e2e/package.json new file mode 100644 index 000000000..8bf9df7a1 --- /dev/null +++ b/e2e/package.json @@ -0,0 +1,11 @@ +{ + "name": "archie-e2e", + "private": true, + "description": "Validation of a running local stack; never part of the gate", + "scripts": { + "check": "node run.cjs" + }, + "devDependencies": { + "playwright": "1.64.0" + } +} diff --git a/e2e/run.cjs b/e2e/run.cjs new file mode 100644 index 000000000..ed53ebe9c --- /dev/null +++ b/e2e/run.cjs @@ -0,0 +1,62 @@ +// Drive the dashboard of a running local stack (e2e/stack.sh) in headless +// Chromium. Each module in checks/ exports async ({ page, go, shot, fail }). +// A check fails on a thrown error or any console or page error. +// +// node e2e/run.cjs [--record VERSION] [check ...] +// +// Screenshots and results.json go to e2e/out//. --record also writes +// e2e/results/VERSION.md, the committed evidence for a release. +const fs = require('fs'); +const path = require('path'); +const { chromium } = require('playwright'); + +const base = process.env.ARCHIE_E2E_URL || 'http://localhost:5173'; +const args = process.argv.slice(2); +let record = ''; +if (args[0] === '--record') { record = args[1]; args.splice(0, 2); } +const names = args.length ? args : fs.readdirSync(path.join(__dirname, 'checks')).filter((f) => f.endsWith('.cjs')).map((f) => f.slice(0, -4)).sort(); +const stamp = new Date().toISOString().replace(/[:.]/g, '-'); +const out = path.join(__dirname, 'out', stamp); +fs.mkdirSync(out, { recursive: true }); + +(async () => { + const browser = await chromium.launch(); + const results = []; + for (const name of names) { + const check = require(path.join(__dirname, 'checks', name + '.cjs')); + // A fresh page per check: open event streams pile up across navigations. + const page = await browser.newPage({ viewport: { width: 1440, height: 900 } }); + const errors = []; + const notes = []; + page.on('pageerror', (e) => errors.push(String(e))); + page.on('console', (m) => m.type() === 'error' && errors.push(m.text())); + // Wait for load, never networkidle: the event stream never goes idle. + const go = async (p) => { await page.goto(base + p, { waitUntil: 'load' }); await page.waitForTimeout(1200); }; + const shot = (label) => page.screenshot({ path: path.join(out, `${name}-${label.replace(/[^a-z0-9-]+/gi, '_') || 'root'}.png`) }); + const fail = (msg) => notes.push(msg); + const started = Date.now(); + let error = ''; + try { await check({ page, go, shot, fail }); } catch (e) { error = String(e && e.message || e); } + await page.close(); + const problems = [error, ...notes, ...errors].filter(Boolean); + results.push({ check: name, ok: problems.length === 0, ms: Date.now() - started, problems }); + console.log(`${problems.length ? 'FAIL' : 'ok '} ${name}${problems.length ? '\n ' + problems.join('\n ') : ''}`); + } + await browser.close(); + fs.writeFileSync(path.join(out, 'results.json'), JSON.stringify(results, null, 2)); + if (record) writeRecord(record, results); + console.log(`evidence: ${out}`); + process.exit(results.every((r) => r.ok) ? 0 : 1); +})().catch((e) => { console.error(e); process.exit(2); }); + +function writeRecord(version, results) { + const dir = path.join(__dirname, 'results'); + fs.mkdirSync(dir, { recursive: true }); + const rows = results.map((r) => `| ${r.check} | ${r.ok ? 'pass' : 'fail'} | ${r.problems.join('
').replace(/\|/g, '\\|') || ''} |`); + const passed = results.filter((r) => r.ok).length; + fs.writeFileSync(path.join(dir, `${version}.md`), [ + `# ${version}`, '', + `Run ${new Date().toISOString()} against ${base}: ${passed} of ${results.length} checks passed.`, '', + '| Check | Result | Problems |', '| --- | --- | --- |', ...rows, '', + ].join('\n')); +} diff --git a/e2e/seed.sh b/e2e/seed.sh new file mode 100755 index 000000000..d604cbefa --- /dev/null +++ b/e2e/seed.sh @@ -0,0 +1,51 @@ +#!/usr/bin/env bash +# Seed a running local stack (e2e/stack.sh) through the dashboard's own HTTP +# API, so the seed exercises real validation. Safe to run again: each step +# skips what already exists. +set -euo pipefail + +base=${ARCHIE_E2E_API:-http://localhost:8484} +here=$(cd "$(dirname "$0")" && pwd) +H=(-H 'Content-Type: application/json' -H 'X-Archie-CSRF: 1' -H "Origin: $base") + +get() { curl -fsS "$base$1"; } +post() { curl -fsS -X POST "$base$1" "${H[@]}" -d "$2"; } +# jq-free JSON reads: py EXPR, with the parsed body as d. +py() { python3 -c "import sys, json; d = json.load(sys.stdin); print($1)"; } + +# Repository policy. archied reads policies only at boot, so restart it. +if ! get /api/control-plane/resources/repository-policies | grep -q '"widgets"'; then + version=$(get /api/control-plane/resources/repository-policies | py 'd["resource"]["version"]') + post /api/control-plane/resources/repository-policies/commands/replace \ + "{\"value\":[{\"owner\":\"acme\",\"name\":\"widgets\",\"base\":\"main\"}],\"expected_version\":$version}" >/dev/null + "$here/stack.sh" restart archied + sleep 5 +fi + +# Work requests. Without Docker they park within a minute, which fills +# Needs you, Quality gates and the task pages. +tasks=$(get /api/tasks) +for t in "Fix flaky login test" "Add retry to webhook sender" "Document the gate" "Speed up the CI cache"; do + grep -qF "\"$t\"" <<<"$tasks" && continue + post /api/work-requests "{\"identity\":\"archie-dev\",\"repository\":\"acme/widgets\",\"workflow\":\"triage\",\"title\":\"$t\",\"instructions\":\"Do it.\"}" >/dev/null +done + +# An unsigned capture source with one delivery, an event type, a mapping and +# a binding left pending approval. +src=e2e-hook +if ! get /api/sources | grep -q "\"$src\""; then + post /api/sources "{\"path\":\"$src\"}" >/dev/null + post "/api/sources/$src/signing" '{"signed":false}' >/dev/null + post "/api/sources/$src/approve-unsigned" '{}' >/dev/null + bash "$here/../tools/webhook-send.sh" --url "$base" --source "$src" --kind github-issue >/dev/null +fi +et=$(get /api/event-types | py '"".join(t["id"] for t in (d.get("event_types", []) if isinstance(d, dict) else d) if t["name"] == "issue-opened")') +if [ -z "$et" ]; then + et=$(post /api/event-types "{\"source\":\"$src\",\"name\":\"issue-opened\",\"rule\":{\"headers\":[],\"payload\":[{\"path\":\"action\",\"op\":\"equals\",\"value\":\"opened\"}]},\"example\":{\"headers\":{},\"body\":\"{\\\"action\\\":\\\"opened\\\",\\\"issue\\\":{\\\"title\\\":\\\"x\\\"}}\"}}" | py 'd["id"]') +fi +if ! get /api/bindings | grep -q '"e2e-binding"'; then + mapping=$(post /api/mappings "{\"name\":\"e2e-issue\",\"source_hint\":\"$src\",\"event_type_id\":\"$et\",\"fields\":[{\"name\":\"title\",\"path\":\"issue.title\",\"type\":\"string\",\"required\":true}]}" | py 'd["id"]') + post /api/bindings "{\"name\":\"e2e-binding\",\"matcher\":{\"source\":\"$src\"},\"mapping_id\":\"$mapping\",\"workflow\":\"feasibility\",\"owner\":\"acme\",\"repo\":\"widgets\"}" >/dev/null +fi + +echo "seeded: $(get /api/tasks | py 'len(d.get("tasks", d) if isinstance(d, dict) else d)') tasks" diff --git a/e2e/stack.sh b/e2e/stack.sh new file mode 100755 index 000000000..6f4863a52 --- /dev/null +++ b/e2e/stack.sh @@ -0,0 +1,140 @@ +#!/usr/bin/env bash +# Run the whole stack as local processes against a throwaway PostgreSQL 18, +# for manual and scripted checks. Never part of the gate. Nothing here reads +# the installed config: XDG paths point into the dev directory. +# +# e2e/stack.sh start | stop | restart | status | reset +set -euo pipefail + +repo=$(cd "$(dirname "$0")/.." && pwd) +dir=${ARCHIE_DEV_DIR:-${XDG_STATE_HOME:-$HOME/.local/state}/archie-dev} +pgport=${ARCHIE_DEV_PGPORT:-55432} +services=(archie-state-store archie-gateway archied archie-messaging archie-ui) + +# initdb refuses root, and postgres cannot traverse root's home. +if [ "$(id -u)" -eq 0 ]; then + pgdata=/var/lib/archie-dev/pg + as_pg() { runuser -u postgres -- "$@"; } +else + pgdata=$dir/pg + as_pg() { "$@"; } +fi + +export XDG_CONFIG_HOME=$dir/config XDG_DATA_HOME=$dir/data XDG_STATE_HOME=$dir/state + +pgbin() { + local d + for d in /usr/lib/postgresql/18/bin "$(dirname "$(command -v postgres 2>/dev/null || echo /x/x)")"; do + if [ -x "$d/postgres" ] && "$d/postgres" --version | grep -q ' 18\.'; then + echo "$d" + return + fi + done + echo "PostgreSQL 18 server not found" >&2 + exit 1 +} + +wait_port() { + for _ in $(seq 1 "${3:-60}"); do + (echo >"/dev/tcp/127.0.0.1/$1") 2>/dev/null && return 0 + sleep 1 + done + echo "$2 did not listen on $1; see $dir/logs/$2.log" >&2 + return 1 +} + +start_pg() { + local bin + bin=$(pgbin) + if [ ! -f "$pgdata/PG_VERSION" ]; then + mkdir -p "$pgdata" + [ "$(id -u)" -ne 0 ] || chown postgres "$pgdata" + as_pg "$bin/initdb" -D "$pgdata" -U archie --auth=trust >/dev/null + fi + if ! as_pg "$bin/pg_ctl" -D "$pgdata" status >/dev/null 2>&1; then + as_pg "$bin/pg_ctl" -D "$pgdata" -w -l "$pgdata/server.log" \ + -o "-p $pgport -k $pgdata -c listen_addresses=127.0.0.1" start >/dev/null + fi + "$bin/psql" -h 127.0.0.1 -p "$pgport" -U archie -d postgres -qtAc \ + "SELECT 1 FROM pg_database WHERE datname = 'archie'" | grep -q 1 || + "$bin/createdb" -h 127.0.0.1 -p "$pgport" -U archie archie +} + +prepare() { + mkdir -p "$dir"/{bin,logs,pids} "$XDG_CONFIG_HOME/archie" "$XDG_DATA_HOME" "$XDG_STATE_HOME" + [ -f "$XDG_CONFIG_HOME/archie/config.toml" ] || cp "$repo/config.example.toml" "$XDG_CONFIG_HOME/archie/config.toml" + sed "s#^database_url = .*#database_url = \"postgres://archie@127.0.0.1:$pgport/archie?sslmode=disable\"#" \ + "$repo/deployments/dev.toml" >"$dir/dev.toml" + local s + for s in "${services[@]}"; do + (cd "$repo" && go build -o "$dir/bin/$s" "./cmd/$s") + done +} + +# spawn NAME DIR CMD...: a detached session leader, so stop takes its children. +spawn() { + local name=$1 cwd=$2 + shift 2 + setsid -f bash -c 'cd "$1" && echo $$ >"$2" && shift 2 && exec "$@"' _ "$cwd" "$dir/pids/$name" "$@" \ + "$dir/logs/$name.log" 2>&1 +} + +start_service() { + spawn "$1" "$repo" "$dir/bin/$1" -config-overlay "$dir/dev.toml" +} + +stop_pid() { + local f=$1 pid + [ -f "$f" ] || return 0 + pid=$(cat "$f") + if kill -- "-$pid" 2>/dev/null; then + for _ in $(seq 1 20); do kill -0 "$pid" 2>/dev/null || break; sleep 0.5; done + kill -9 -- "-$pid" 2>/dev/null || true + fi + rm -f "$f" +} + +start() { + prepare + start_pg + start_service archie-state-store + wait_port 9091 archie-state-store 90 + local s + for s in archie-gateway archied archie-messaging archie-ui; do start_service "$s"; done + wait_port 8484 archie-ui + [ -d "$repo/ui/node_modules" ] || (cd "$repo/ui" && npm ci --no-audit --no-fund >/dev/null) + spawn vite "$repo/ui" npx vite --host 127.0.0.1 --port 5173 --strictPort + wait_port 5173 vite + echo "dashboard: http://localhost:5173 logs: $dir/logs" +} + +stop() { + local f + for f in "$dir"/pids/*; do stop_pid "$f"; done + if [ -f "$pgdata/PG_VERSION" ]; then + as_pg "$(pgbin)/pg_ctl" -D "$pgdata" -m fast stop >/dev/null 2>&1 || true + fi +} + +case ${1:-} in +start) start ;; +stop) stop ;; +restart) + stop_pid "$dir/pids/${2:?service}" + start_service "$2" + ;; +status) + for f in "$dir"/pids/*; do + [ -f "$f" ] || continue + if kill -0 "$(cat "$f")" 2>/dev/null; then echo "up $(basename "$f")"; else echo "down $(basename "$f")"; fi + done + ;; +reset) + stop + rm -rf "$dir" "$pgdata" + ;; +*) + echo "usage: $0 start | stop | restart | status | reset" >&2 + exit 2 + ;; +esac diff --git a/tools/cloud-setup.sh b/tools/cloud-setup.sh index e573e9739..60a4b636a 100755 --- a/tools/cloud-setup.sh +++ b/tools/cloud-setup.sh @@ -47,24 +47,13 @@ go install "google.golang.org/grpc/cmd/protoc-gen-go-grpc@$PROTOC_GEN_GO_GRPC_VE curl -fsSL https://raw.githubusercontent.com/gastownhall/beads/main/scripts/install.sh | bash (cd "$HOME" && bd setup claude --global) -# PostgreSQL 18 for running full checks against the real stack -# (deployments/dev.toml). Never for tests: they use fakes. +# PostgreSQL 18 for e2e/stack.sh, which runs its own cluster. Never for +# tests: they use fakes. if [ ! -x /usr/lib/postgresql/18/bin/postgres ]; then $SUDO apt-get install -y --no-install-recommends postgresql-common $SUDO /usr/share/postgresql-common/pgdg/apt.postgresql.org.sh -y $SUDO apt-get install -y --no-install-recommends postgresql-18 fi -for v in $(pg_lsclusters -h | awk '$1 != 18 {print $1}'); do - $SUDO pg_ctlcluster "$v" main stop || true - $SUDO pg_dropcluster "$v" main -done -pg_lsclusters -h | grep -q '^18 main' || $SUDO pg_createcluster 18 main --port 5432 -$SUDO pg_ctlcluster 18 main start || true -as_pg() { $SUDO runuser -u postgres -- "$@"; } -as_pg psql -qtc "SELECT 1 FROM pg_roles WHERE rolname = 'archie'" | grep -q 1 || - as_pg psql -qc "CREATE USER archie WITH PASSWORD 'archie' SUPERUSER" -as_pg psql -qtc "SELECT 1 FROM pg_database WHERE datname = 'archie'" | grep -q 1 || - as_pg createdb -O archie archie # Commit and PR attribution stays off, matching local settings. mkdir -p "$HOME/.claude"