test(e2e): per-run report.html, fix stale workflow-logic and audit-runtime, add org selector, org pages, setup, recovery, readiness and schedule-trigger checks

This commit is contained in:
sam committed 2026-10-27 11:44:23 +11:00
1 parent 50530c0c9a
commit 02edc380e7
12 files changed
+265 -41

No files matched your search

+10 -6
View File
@@ -9,7 +9,7 @@ park, which is useful data for the dashboard.
| --- | --- | | --- | --- |
| `stack.sh` | Starts PostgreSQL 18 (port 55432) and the five services as local processes, then Vite on 5173 | | `stack.sh` | Starts PostgreSQL 18 (port 55432) and the five services as local processes, then Vite on 5173 |
| `seed.sh` | Seeds the stack through the dashboard API; safe to rerun | | `seed.sh` | Seeds the stack through the dashboard API; safe to rerun |
| `run.cjs` | Runs `checks/*.cjs` in Chromium; screenshots, manifest and gallery to `out/<stamp>/` | | `run.cjs` | Runs `checks/*.cjs` in Chromium; screenshots, manifest and `report.html` to `out/<stamp>/` |
| `checks/` | One module per behaviour | | `checks/` | One module per behaviour |
| `results/` | One committed report per release | | `results/` | One committed report per release |
@@ -59,7 +59,7 @@ that release. Commit it with the release. Screenshots stay in `out/`.
## Writing a check ## Writing a check
A module in `checks/` exports `async ({ page, go, shot, fail, skip, settled })`. A module in `checks/` exports `async ({ page, go, shot, fail, skip, settled, attach, tolerate })`.
- `go(path)` loads a dashboard route: `waitUntil: 'load'` plus 1200ms. - `go(path)` loads a dashboard route: `waitUntil: 'load'` plus 1200ms.
Wait for elements, never for `networkidle`: the event stream never goes idle. Wait for elements, never for `networkidle`: the event stream never goes idle.
@@ -71,6 +71,8 @@ A module in `checks/` exports `async ({ page, go, shot, fail, skip, settled })`.
- `fail(msg)` records a problem without stopping. - `fail(msg)` records a problem without stopping.
- `skip(reason)` stops the check and records a skip: a missing - `skip(reason)` stops the check and records a skip: a missing
precondition, never a failure. precondition, never a failure.
- `tolerate(re)` ignores console errors matching `re`, for a response the page is designed to survive.
- `attach(name, data)` saves a non-image artifact as `<check>-<name>`.
- `settled(sel?)` waits for a selector or URL, then settles briefly. - `settled(sel?)` waits for a selector or URL, then settles briefly.
It is the harness-owned wait; checks need no capture wrapper or It is the harness-owned wait; checks need no capture wrapper or
ad-hoc sleeps around screenshots. ad-hoc sleeps around screenshots.
@@ -92,10 +94,12 @@ data, not copy.
## Output ## Output
Every run writes `out/<stamp>/` with the screenshots, Every run writes `out/<stamp>/` with the screenshots,
`results.json` (the manifest: `{check, ok, ms, problems, shots[]}`, `results.json` (the manifest: `{check, ok, ms, problems, artifacts[]}`,
plus `skipped`/`skipReason` for skips) and `index.html` (a gallery plus `skipped`/`skipReason` for skips) and `report.html`: a self-contained
grouped by check with pass, fail or skip plus problems, for human test record with the commit, totals and one row per check (status, time,
image review). message and that check's artifacts, screenshots inline). `attach(name, data)`
adds a non-image artifact to the current check; a failed check also gets a
`failure` screenshot.
Exits: 0 when every check passes or skips, 1 when any check fails, Exits: 0 when every check passes or skips, 1 when any check fails,
2 when the harness itself errors. `ARCHIE_E2E_URL` points the run at 2 when the harness itself errors. `ARCHIE_E2E_URL` points the run at
+5 -4
View File
@@ -1,18 +1,19 @@
// A runtime event with a task target expands to its metadata and links to // A runtime event with a task target expands to its metadata and links to
// the task. The event comes from the seed; without it there is nothing to // the task. The seed's work requests park and record the event; the feed is
// exercise, so the check skips instead of failing. // filtered to it so settings noise cannot push it off the page. Without it
// there is nothing to exercise, so the check skips instead of failing.
const assert = require('node:assert/strict'); const assert = require('node:assert/strict');
const { DEFAULT_WINDOW_MS, trackAudit } = require('./helpers/audit-helpers.cjs'); const { DEFAULT_WINDOW_MS, trackAudit } = require('./helpers/audit-helpers.cjs');
module.exports = async ({ page, go, skip }) => { module.exports = async ({ page, go, skip }) => {
trackAudit(page); trackAudit(page);
await go('/org/audit'); await go('/org/audit?kind=parked');
await page.locator('[data-row-key]').first().waitFor(); await page.locator('[data-row-key]').first().waitFor();
const from = new Date(Date.now() - DEFAULT_WINDOW_MS).toISOString(); const from = new Date(Date.now() - DEFAULT_WINDOW_MS).toISOString();
// The feed page size is 100: only rows in this response are rendered. // The feed page size is 100: only rows in this response are rendered.
const feed = await page.evaluate( const feed = await page.evaluate(
async (iso) => (await fetch(`/api/audit?limit=100&from=${encodeURIComponent(iso)}`)).json(), async (iso) => (await fetch(`/api/audit?limit=100&kind=parked&from=${encodeURIComponent(iso)}`)).json(),
from, from,
); );
const records = feed.records || []; const records = feed.records || [];
+18
View File
@@ -0,0 +1,18 @@
// Shared by the org-* and setup checks: acting in a seeded org and the 404s a
// fresh org's pages are built to survive.
const ORG = { home: 'org-sys', beta: 'org-beta', outsider: 'org-gamma' };
// A new org has written no documents, and the instance-only reads answer
// not found outside the system org.
const ABSENT = /status of 404.*\/api\/(control-plane|schedules|identities)/;
// Selects an org the way the topbar does, over the page's own session.
async function selectOrg(page, id) {
const origin = new URL(page.url() === 'about:blank' ? 'http://localhost:5173' : page.url()).origin;
return page.request.put(`${origin}/api/orgs/current`, {
headers: { 'X-Archie-CSRF': '1', Origin: origin },
data: { id },
});
}
module.exports = { ORG, ABSENT, selectOrg };
+30
View File
@@ -0,0 +1,30 @@
// The organisation pages: each loads its form in the selected org, and the old
// settings audit route lands on the organisation's audit page.
const assert = require('node:assert/strict');
const { trackAudit } = require('./helpers/audit-helpers.cjs');
const PAGES = [
{ path: '/org/channels', heading: 'Channels', text: 'Chat session defaults' },
{ path: '/org/scheduling-policy', heading: 'Scheduling policy', text: 'Retries and polling' },
{ path: '/org/alerts', heading: 'Alert webhook', text: 'Webhook severity' },
];
module.exports = async ({ page, go, shot }) => {
for (const { path, heading, text } of PAGES) {
await go(path);
await page.getByRole('heading', { name: heading, exact: true }).waitFor();
await page.getByText(text, { exact: true }).first().waitFor();
assert.equal(new URL(page.url()).pathname, path);
await shot(path);
}
trackAudit(page);
await go('/org/audit');
await page.getByRole('heading', { name: 'Audit', exact: true }).first().waitFor();
await page.locator('[data-row-key]').first().waitFor();
await shot('/org/audit');
await go('/settings/audit?since=7d');
assert.equal(new URL(page.url()).pathname, '/org/audit', '/settings/audit did not redirect');
assert.equal(new URL(page.url()).searchParams.get('since'), '7d', 'redirect dropped the query');
await page.locator('[data-row-key]').first().waitFor();
};
+31
View File
@@ -0,0 +1,31 @@
// A non-system org can arm a schedule trigger: the schedule source's built-in
// mapping is readable there, so a binding on it is accepted. The editor's own
// schedule list is instance-only, so the binding is written over the API.
const assert = require('node:assert/strict');
const { ORG, ABSENT, selectOrg } = require('./helpers/org-helpers.cjs');
module.exports = async ({ page, go, shot, tolerate }) => {
tolerate(ABSENT);
await go('/');
assert.equal((await selectOrg(page, ORG.beta)).status(), 204);
await go('/workflows?workflow=feasibility');
const origin = new URL(page.url()).origin;
const headers = { 'X-Archie-CSRF': '1', Origin: origin };
const created = await page.request.post(`${origin}/api/bindings`, {
headers,
data: {
name: 'e2e schedule trigger', matcher: { source: 'schedule' }, mapping_id: 'schedule-fired',
filter: 'schedule_id == "issue-discovery"', workflow: 'feasibility', repo_param: '', owner: '', repo: '', inputs: {},
},
});
try {
assert.equal(created.status(), 201, `binding on the built-in schedule mapping refused: ${await created.text()}`);
const binding = await created.json();
assert.equal(binding.org_id, ORG.beta);
await go('/workflows?workflow=feasibility');
await page.getByText('Starts when').first().waitFor();
await shot('beta-trigger');
} finally {
if (created.ok()) await page.request.delete(`${origin}/api/bindings/${(await created.json()).id}`, { headers });
}
};
+43
View File
@@ -0,0 +1,43 @@
// The topbar org selector: choosing a member org reloads the dashboard into it,
// Tasks and Workflows then show only that org's work, and the API refuses an
// org the caller is not a member of.
const assert = require('node:assert/strict');
const { ORG, ABSENT, selectOrg } = require('./helpers/org-helpers.cjs');
module.exports = async ({ page, go, shot, settled, tolerate }) => {
tolerate(ABSENT);
await go('/tasks');
await page.getByText('Fix flaky login test').first().waitFor();
const picker = page.getByRole('combobox', { name: 'Organisation' });
await picker.waitFor();
const offered = await (async () => {
await picker.click();
const names = await page.getByRole('option').allInnerTexts();
await page.keyboard.press('Escape');
return names;
})();
assert.deepEqual(offered.sort(), ['Beta', 'E2E'], 'selector must offer exactly the orgs the caller belongs to');
await go('/workflows');
await page.getByText('Parked', { exact: true }).first().waitFor();
await shot('home-org-workflows');
await picker.click();
await Promise.all([page.waitForEvent('load'), page.getByRole('option', { name: 'Beta', exact: true }).click()]);
await settled('/workflows');
await page.getByRole('combobox', { name: 'Organisation' }).waitFor();
assert.equal(await page.getByText('Parked', { exact: true }).count(), 0, 'workflow run counts leaked across orgs');
await shot('beta-workflows');
await go('/tasks');
await page.getByText('No tasks yet').waitFor();
assert.equal(await page.getByText('Fix flaky login test').count(), 0, 'tasks leaked across orgs');
await shot('beta-tasks');
// The selector never writes an org the caller is outside of; the server decides.
const refused = await selectOrg(page, ORG.outsider);
assert.equal(refused.status(), 403, 'selecting a non-member org must be forbidden');
await go('/tasks');
await page.getByText('No tasks yet').waitFor();
assert.equal((await selectOrg(page, ORG.home)).status(), 204);
await go('/tasks');
await page.getByText('Fix flaky login test').first().waitFor();
};
+13
View File
@@ -0,0 +1,13 @@
// Repositories and connections say what stops them serving work: the seeded
// repository has no connection, and the seeded connection's forge is not enabled.
module.exports = async ({ page, go, shot }) => {
await go('/org/repositories');
await page.getByText('acme/widgets', { exact: true }).first().waitFor();
await page.getByText('No connection', { exact: true }).waitFor();
await shot('repositories');
await go('/org/connections');
await page.getByText('github.com', { exact: true }).first().waitFor();
await page.getByText('Forge not enabled', { exact: true }).waitFor();
await shot('connections');
};
+21
View File
@@ -0,0 +1,21 @@
// The setup checklist reads the selected org: an org with no model to run gets a
// Model step that leads to the page that fixes it.
const assert = require('node:assert/strict');
const { ORG, ABSENT, selectOrg } = require('./helpers/org-helpers.cjs');
module.exports = async ({ page, go, shot, settled, tolerate }) => {
tolerate(ABSENT);
await go('/');
await page.getByPlaceholder('Describe a task').waitFor();
assert.equal(await page.getByRole('link', { name: 'Model', exact: true }).count(), 0, 'an org with a model got a Model step');
assert.equal((await selectOrg(page, ORG.beta)).status(), 204);
await go('/');
await page.getByText('Setup', { exact: true }).waitFor();
const model = page.getByRole('link', { name: 'Model', exact: true });
await model.waitFor();
assert.equal(await model.getAttribute('href'), '/org/models');
await shot('beta-dashboard');
await model.click();
await settled('/org/models');
await shot('model-page');
};
+29
View File
@@ -0,0 +1,29 @@
// A parked task whose blocker has a fix links to the page that supplies it, from
// Needs attention on the dashboard and from the Stage Rail on the task page.
// The seed's work requests park for want of a repository connection.
const assert = require('node:assert/strict');
module.exports = async ({ page, go, shot, settled, skip }) => {
await go('/');
const tasks = await page.evaluate(async () => (await fetch('/api/tasks?limit=100')).json());
if (!tasks.some((task) => task.status === 'parked')) skip('no parked task');
const parked = tasks.find((task) => task.status === 'parked' && task.recovery);
assert.ok(parked, 'no parked task in the list carries a recovery link');
const { label, link } = parked.recovery;
const row = page.getByRole('link', { name: parked.title, exact: true }).locator('xpath=ancestor::*[.//a[normalize-space()="' + label + '"]][1]');
const fromDashboard = row.getByRole('link', { name: label, exact: true });
await fromDashboard.waitFor();
assert.equal(await fromDashboard.getAttribute('href'), link);
await shot('needs-attention');
await fromDashboard.click();
await settled(link);
await go(`/tasks/${parked.id}`);
const fromRail = page.getByRole('complementary', { name: 'Task details' }).getByRole('link', { name: label, exact: true });
await fromRail.waitFor();
assert.equal(await fromRail.getAttribute('href'), link);
await shot('stage-rail');
await fromRail.click();
await settled(link);
};
+3 -5
View File
@@ -23,6 +23,7 @@ module.exports = async ({ page, go, shot, settled }) => {
} }
}; };
// Reproduce a previously stored title ID without changing the test server. // Reproduce a previously stored title ID without changing the test server.
await go('/workflows');
const response = await page.request.get(new URL('/api/control-plane/resources/workflow-definitions', page.url()).href); const response = await page.request.get(new URL('/api/control-plane/resources/workflow-definitions', page.url()).href);
const resource = await response.json(); const resource = await response.json();
const definition = resource.resource.value.definitions.find(entry => entry.id === 'feasibility'); const definition = resource.resource.value.definitions.find(entry => entry.id === 'feasibility');
@@ -38,9 +39,7 @@ module.exports = async ({ page, go, shot, settled }) => {
const approval = page.locator('.vue-flow__node').filter({ hasText: 'Approval' }).first(); const approval = page.locator('.vue-flow__node').filter({ hasText: 'Approval' }).first();
await panTo(approval); await panTo(approval);
await approval.click(); await approval.click();
const dialog = page.getByRole('dialog'); const chip = approval.getByRole('button', { name: 'write-a-product-requirement-document', exact: true });
await dialog.waitFor();
const chip = dialog.getByRole('button', { name: 'write-a-product-requirement-document', exact: true });
await chip.waitFor(); await chip.waitFor();
await chip.click(); await chip.click();
await shot('legacy-reference'); await shot('legacy-reference');
@@ -83,7 +82,6 @@ module.exports = async ({ page, go, shot, settled }) => {
const historicalStep = page.locator('.vue-flow__node').filter({ hasText: 'Write a Product Requirement Document' }).first(); const historicalStep = page.locator('.vue-flow__node').filter({ hasText: 'Write a Product Requirement Document' }).first();
await panTo(historicalStep); await panTo(historicalStep);
await historicalStep.click(); await historicalStep.click();
await page.getByRole('dialog').waitFor();
await page.getByText('Historical failure', { exact: true }).first().waitFor(); await page.getByText('Historical failure', { exact: true }).first().waitFor();
await shot('historical-run'); await shot('historical-run');
await page.keyboard.press('Escape'); await page.keyboard.press('Escape');
@@ -100,7 +98,7 @@ module.exports = async ({ page, go, shot, settled }) => {
const handoff = page.locator('.vue-flow__node').filter({ hasText: 'Hand off' }).first(); const handoff = page.locator('.vue-flow__node').filter({ hasText: 'Hand off' }).first();
await panTo(handoff); await panTo(handoff);
await handoff.click(); await handoff.click();
const target = page.getByRole('dialog').getByLabel('Workflow', { exact: true }); const target = handoff.getByLabel('Workflow', { exact: true });
await target.waitFor(); await target.waitFor();
assert.equal(await target.inputValue(), '{{ steps.classify.result.workflow }}'); assert.equal(await target.inputValue(), '{{ steps.classify.result.workflow }}');
await target.fill('{{ steps.classify.result.alternative }}'); await target.fill('{{ steps.classify.result.alternative }}');
+49 -25
View File
@@ -4,7 +4,7 @@
// //
// node e2e/run.cjs [--headed] [--record VERSION] [list | all | check ...] // node e2e/run.cjs [--headed] [--record VERSION] [list | all | check ...]
// //
// Screenshots, results.json (the manifest) and index.html (the gallery) go to // Screenshots, results.json (the manifest) and report.html (the run report) go to
// e2e/out/<stamp>/. --record also writes e2e/results/VERSION.md, the committed // e2e/out/<stamp>/. --record also writes e2e/results/VERSION.md, the committed
// evidence for a release. // evidence for a release.
const fs = require('fs'); const fs = require('fs');
@@ -72,7 +72,11 @@ fs.mkdirSync(out, { recursive: true });
const errors = []; const errors = [];
const notes = []; const notes = [];
page.on('pageerror', (e) => errors.push(String(e))); page.on('pageerror', (e) => errors.push(String(e)));
page.on('console', (m) => m.type() === 'error' && errors.push(m.text())); page.on('console', (m) => {
if (m.type() !== 'error') return;
const url = m.location().url;
errors.push(url ? `${m.text()} (${url})` : m.text());
});
// Wait for load, never networkidle: the event stream never goes idle. // Wait for load, never networkidle: the event stream never goes idle.
const go = async (p) => { await page.goto(base + p, { waitUntil: 'load' }); await page.waitForTimeout(1200); }; const go = async (p) => { await page.goto(base + p, { waitUntil: 'load' }); await page.waitForTimeout(1200); };
// Harness-owned wait: wait for a selector or URL, then settle briefly. // Harness-owned wait: wait for a selector or URL, then settle briefly.
@@ -84,7 +88,7 @@ fs.mkdirSync(out, { recursive: true });
} }
await page.waitForTimeout(150); await page.waitForTimeout(150);
}; };
const shots = []; const artifacts = [];
// <check>-<kebab>[-<w>x<h>].png; the viewport suffix marks non-desktop // <check>-<kebab>[-<w>x<h>].png; the viewport suffix marks non-desktop
// shots. A short settle precedes every capture so checks need no capture // shots. A short settle precedes every capture so checks need no capture
// wrapper of their own. Every capture is kept: a repeated label gets -2, -3... // wrapper of their own. Every capture is kept: a repeated label gets -2, -3...
@@ -96,28 +100,39 @@ fs.mkdirSync(out, { recursive: true });
let file = `${stem}.png`; let file = `${stem}.png`;
for (let n = 2; fs.existsSync(path.join(out, file)); n++) file = `${stem}-${n}.png`; for (let n = 2; fs.existsSync(path.join(out, file)); n++) file = `${stem}-${n}.png`;
await page.screenshot({ path: path.join(out, file) }); await page.screenshot({ path: path.join(out, file) });
shots.push({ file, w: vp.width, h: vp.height, check: name }); artifacts.push({ file });
return file; return file;
}; };
// Keep a non-image artifact for this case: written as <check>-<name>.
const attach = (fileName, data) => {
const file = `${name}-${fileName}`;
fs.writeFileSync(path.join(out, file), data);
artifacts.push({ file });
};
const fail = (msg) => notes.push(msg); const fail = (msg) => notes.push(msg);
// Drop console errors matching re: a response the page is designed to survive.
const tolerated = [];
const tolerate = (re) => tolerated.push(re);
const skip = (reason) => { throw new SkipCheck(reason || 'precondition missing'); }; const skip = (reason) => { throw new SkipCheck(reason || 'precondition missing'); };
const started = Date.now(); const started = Date.now();
let error = ''; let error = '';
let skipped = ''; let skipped = '';
try { await check({ page, go, shot, fail, skip, settled }); } catch (e) { try { await check({ page, go, shot, fail, skip, settled, attach, tolerate }); } catch (e) {
if (e instanceof SkipCheck) skipped = e.message; if (e instanceof SkipCheck) skipped = e.message;
else error = String(e && e.message || e); else error = String(e && e.message || e);
} }
if (error || notes.length) await shot('failure').catch(() => {});
await page.close(); await page.close();
const problems = skipped ? [] : [error, ...notes, ...errors].filter(Boolean); const problems = skipped ? [] : [error, ...notes, ...errors.filter((e) => !tolerated.some((re) => re.test(e)))].filter(Boolean);
const result = { check: name, ok: problems.length === 0, ms: Date.now() - started, problems, shots }; const result = { check: name, ok: problems.length === 0, ms: Date.now() - started, problems, artifacts };
if (skipped) { result.skipped = true; result.skipReason = skipped; } if (skipped) { result.skipped = true; result.skipReason = skipped; }
results.push(result); results.push(result);
console.log(`${skipped ? 'SKIP' : problems.length ? 'FAIL' : 'ok '} ${name}${skipped ? ` (${skipped})` : problems.length ? '\n ' + problems.join('\n ') : ''}`); console.log(`${skipped ? 'SKIP' : problems.length ? 'FAIL' : 'ok '} ${name}${skipped ? ` (${skipped})` : problems.length ? '\n ' + problems.join('\n ') : ''}`);
} }
await browser.close(); await browser.close();
fs.writeFileSync(path.join(out, 'results.json'), JSON.stringify(results, null, 2)); fs.writeFileSync(path.join(out, 'results.json'), JSON.stringify(results, null, 2));
fs.writeFileSync(path.join(out, 'index.html'), gallery(results)); const commit = require('child_process').execFileSync('git', ['rev-parse', '--short', 'HEAD'], { cwd: __dirname, encoding: 'utf8' }).trim();
fs.writeFileSync(path.join(out, 'report.html'), report(results, { time: new Date().toISOString(), commit }));
if (record) writeRecord(record, results); if (record) writeRecord(record, results);
console.log(`evidence: ${out}`); console.log(`evidence: ${out}`);
process.exit(results.every((r) => r.ok) ? 0 : 1); process.exit(results.every((r) => r.ok) ? 0 : 1);
@@ -127,25 +142,34 @@ function esc(s) {
return String(s).replace(/&/g, '&amp;').replace(/</g, '&lt;').replace(/>/g, '&gt;').replace(/"/g, '&quot;'); return String(s).replace(/&/g, '&amp;').replace(/</g, '&lt;').replace(/>/g, '&gt;').replace(/"/g, '&quot;');
} }
// Per-run gallery for human image review, grouped by check. // One self-contained page per run: a test plan and verification record. Each
function gallery(results) { // case carries the artifacts its own run produced; images are inlined.
const sections = results.map((r) => { function report(results, meta) {
const status = r.skipped ? 'skip' : r.ok ? 'pass' : 'fail'; const status = (r) => (r.skipped ? 'SKIP' : r.ok ? 'ok' : 'FAIL');
const head = `<h2>${esc(r.check)} <span class="${status}">${status}</span> <small>${r.ms}ms</small></h2>`; const count = (s) => results.filter((r) => status(r) === s).length;
const skipNote = r.skipped ? `<p class="skip-reason">skipped: ${esc(r.skipReason || '')}</p>` : ''; const artifact = (a) => {
const probs = r.problems.length const data = fs.readFileSync(path.join(out, a.file));
? `<ul class="problems">${r.problems.map((p) => `<li>${esc(p)}</li>`).join('')}</ul>` : ''; if (!a.file.endsWith('.png')) return `<a href="${esc(a.file)}">${esc(a.file)}</a>`;
const shots = r.shots.length const src = `data:image/png;base64,${data.toString('base64')}`;
? `<div class="shots">${r.shots.map((s) => `<figure><a href="${esc(s.file)}"><img src="${esc(s.file)}" loading="lazy"></a><figcaption>${esc(s.file)} ${s.w}x${s.h}</figcaption></figure>`).join('')}</div>` return `<figure><img src="${src}" alt="${esc(a.file)}" title="${esc(a.file)}"></figure>`;
: '<p class="no-shots">no screenshots</p>'; };
return `<section class="${status}">${head}${skipNote}${probs}${shots}</section>`; const rows = results.map((r) => {
const s = status(r);
const msg = r.skipped ? [r.skipReason] : r.problems;
return `<tr class="${s}"><td>${esc(r.check)}</td><td class="st">${s}</td><td>${r.ms}ms</td>`
+ `<td>${msg.map((m) => `<pre>${esc(m)}</pre>`).join('')}</td>`
+ `<td class="art">${r.artifacts.map(artifact).join('')}</td></tr>`;
}).join('\n'); }).join('\n');
const passed = results.filter((r) => r.ok && !r.skipped).length;
const skipped = results.filter((r) => r.skipped).length;
return `<!doctype html> return `<!doctype html>
<html><head><meta charset="utf-8"><title>e2e ${esc(stamp)}</title> <html><head><meta charset="utf-8"><title>e2e ${esc(meta.time)}</title>
<style>body{font-family:system-ui,sans-serif;max-width:1200px;margin:0 auto;padding:1em}h2 small{color:#666;font-weight:normal}.pass>h2 .pass,.fail>h2 .fail,.skip>h2 .skip{padding:.1em .5em;border-radius:.3em;font-size:.8em}.pass>h2 .pass{background:#dfd}.fail>h2 .fail{background:#fdd}.skip>h2 .skip{background:#eef}.problems{color:#a00}.skip-reason{color:#555}.no-shots{color:#888;font-style:italic}.shots{display:flex;flex-wrap:wrap;gap:1em}figure{margin:0;width:320px}img{width:100%;border:1px solid #ccc}figcaption{font-size:.8em;color:#555;word-break:break-all}section{border-top:1px solid #ddd;padding-bottom:1em}</style> <style>body{font:14px system-ui,sans-serif;margin:1em}table{border-collapse:collapse;width:100%}td,th{border-top:1px solid #ccc;padding:.4em;vertical-align:top;text-align:left}pre{margin:0 0 .3em;white-space:pre-wrap;font-size:12px}.st{font-weight:600}.ok .st{color:#060}.FAIL .st{color:#a00}.SKIP .st{color:#668}.FAIL{background:#fff3f3}.art{display:flex;flex-wrap:wrap;gap:.5em;border:0}figure{margin:0}.art img{width:150px;border:1px solid #ccc;cursor:zoom-in}.art img.big{position:fixed;inset:2vh 2vw;width:96vw;height:96vh;object-fit:contain;background:#fff;z-index:9;cursor:zoom-out}</style>
</head><body><h1>e2e run ${esc(stamp)}</h1><p>${passed} passed, ${skipped} skipped, ${results.length - passed - skipped} failed against ${esc(base)}.</p>${sections}</body></html> </head><body><h1>e2e ${esc(meta.time)}</h1>
<p>commit ${esc(meta.commit)} against ${esc(base)}: ${results.length} cases, ${count('ok')} ok, ${count('FAIL')} FAIL, ${count('SKIP')} SKIP</p>
<table><tr><th>Case</th><th>Status</th><th>Time</th><th>Message</th><th>Artifacts</th></tr>
${rows}
</table>
<script>document.addEventListener('click',e=>{if(e.target.matches('.art img'))e.target.classList.toggle('big')})</script>
</body></html>
`; `;
} }
+13 -1
View File
@@ -39,7 +39,7 @@ fi
# Work requests. Without Docker they park within a minute, which fills # Work requests. Without Docker they park within a minute, which fills
# Needs you, Quality gates and the task pages. # Needs you, Quality gates and the task pages.
tasks=$(get /api/tasks) tasks=$(get /api/tasks)
for t in "Fix flaky login test" "Add retry to webhook sender" "Document the gate" "Speed up the CI cache"; do for t in "Fix flaky login test" "Add retry to webhook sender" "Document the gate" "Speed up the CI cache" "Recovery probe"; do
grep -qF "\"$t\"" <<<"$tasks" && continue grep -qF "\"$t\"" <<<"$tasks" && continue
post /api/work-requests "{\"repository\":\"acme/widgets\",\"workflow\":\"triage\",\"title\":\"$t\",\"instructions\":\"Do it.\"}" >/dev/null post /api/work-requests "{\"repository\":\"acme/widgets\",\"workflow\":\"triage\",\"title\":\"$t\",\"instructions\":\"Do it.\"}" >/dev/null
done done
@@ -62,4 +62,16 @@ if ! get /api/bindings | grep -q '"e2e-binding"'; then
post /api/bindings "{\"name\":\"e2e-binding\",\"matcher\":{\"source\":\"$src\"},\"mapping_id\":\"$mapping\",\"workflow\":\"feasibility\",\"owner\":\"acme\",\"repo\":\"widgets\"}" >/dev/null post /api/bindings "{\"name\":\"e2e-binding\",\"matcher\":{\"source\":\"$src\"},\"mapping_id\":\"$mapping\",\"workflow\":\"feasibility\",\"owner\":\"acme\",\"repo\":\"widgets\"}" >/dev/null
fi fi
# Two more orgs: the operator owns Beta, so the org selector offers it; Gamma
# belongs to an identity the operator is not a member of, so selecting it is refused.
orgs=$(get /api/orgs)
if ! grep -q '"org-beta"' <<<"$orgs"; then
operator=$(get /api/identities | py '"".join(i["id"] for i in d["identities"] if i["display_name"] == "E2E operator")')
post /api/orgs "{\"id\":\"org-beta\",\"name\":\"Beta\",\"owner_identity\":\"$operator\"}" >/dev/null
fi
if ! grep -q '"org-gamma"' <<<"$orgs"; then
outsider=$(post /api/identities '{"display_name":"E2E outsider","kind":"user"}' | py 'd["id"]')
post /api/orgs "{\"id\":\"org-gamma\",\"name\":\"Gamma\",\"owner_identity\":\"$outsider\"}" >/dev/null
fi
echo "seeded: $(get /api/tasks | py 'len(d.get("tasks", d) if isinstance(d, dict) else d)') tasks" echo "seeded: $(get /api/tasks | py 'len(d.get("tasks", d) if isinstance(d, dict) else d)') tasks"