Compare commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
cbeab6afd2 | ||
|
|
ed698fced1 | ||
|
|
081ee4a8ba | ||
|
|
5de07fac0e | ||
|
|
03335f49c1 | ||
|
|
d0d759d216 | ||
|
|
67ead19dd9 | ||
|
|
f7c646163b | ||
|
|
8d05f99526 | ||
|
|
625ead95b3 | ||
|
|
0acc922d9a | ||
|
|
595956cc75 | ||
|
|
929784667b | ||
|
|
d4e254b349 | ||
|
|
89c5f8e2ad | ||
|
|
2ffe924a44 | ||
|
|
c9f677f35b | ||
|
|
d60f024e74 | ||
|
|
1fa3acdbe9 |
No files matched your search
@@ -5,6 +5,22 @@ release's per-component sections are labelled beneath its heading.
|
||||
|
||||
## [Unreleased]
|
||||
|
||||
## [1.44.0] - 2026-09-29
|
||||
|
||||
### archied
|
||||
|
||||
Archie can now review pull requests.
|
||||
|
||||
- **Reviews on request and by watch.** Ask in chat or on the dashboard work form, add a repository to the watch list, or just let archie review its own PRs before it opens them. Review intensity follows a depth dial (quick/standard/deep) classified from the size of the change, and every finished review passes a verification layer — coverage-gap and cross-finding consistency checks — before it is produced.
|
||||
- **No forge credential reaches the review sandbox.** The daemon prefetches the pull request's data and rebuilds its source snapshot over the forge's HTTP API, so the agent container works from an isolated copy with no read access to the repository's credentials.
|
||||
- **Reviews stop at the operator gate before posting.** `review.precision_gate` / `approve_before_post` decide whether a finished review posts itself or parks for operator approval. With this release the parked state is a stop sign, not a checkpoint: the approve/reject/re-review responses behind the gate are not wired yet, and the gate's settings are not yet editable in the dashboard (configure them in `config.toml`). A parked review sits until that follow-up lands.
|
||||
- The adversarial self-review pass archie previously ran before opening its own PRs is superseded by this pipeline and has been removed.
|
||||
- Hardened harness input handling: harness identifiers are bounded to integer types.
|
||||
|
||||
### archie-agent
|
||||
|
||||
The runtime image is rebuilt with this release; it links the shared pr-review workflow packages behind the reviews above. Hosts that run agent containers need `docker compose pull agent` after updating to pick it up.
|
||||
|
||||
## [1.43.0] - 2026-09-28
|
||||
|
||||
### archied
|
||||
|
||||
@@ -341,10 +341,12 @@ StateStoreGrantIssuer` (`staterpc.GrantIssuer`) registers a fresh
|
||||
separate fresh-context reviewer pass on every change as a matter of course.
|
||||
Red-green TDD plus `task check` is the gate; a manual adversarial pass is
|
||||
opt-in per request — ask before running one. This is the rule for a human or
|
||||
agent _contributor_. Separately, archied runs the same adversarial review
|
||||
automatically on the PRs it opens (gated per repo by `repo.review_enabled`)
|
||||
— see `docs/architecture/adversarial-review.md`. The two paths are distinct:
|
||||
you run one when asked; archied runs one before it opens a PR.
|
||||
agent _contributor_. Separately, archied runs its own PR review pipeline
|
||||
(`docs/prds/pr-review-agent.md`) before it opens a PR of its own — see that
|
||||
PRD's Triggers section ("archie's own PRs"). It supersedes the older
|
||||
single-reviewer stage `docs/architecture/adversarial-review.md` described
|
||||
(`repo.review_enabled` no longer exists). The two paths are distinct: you
|
||||
run a manual pass when asked; archied runs its own before it opens a PR.
|
||||
6. **Linter Guard:**
|
||||
|
||||
- When using `errorlint` fixes, ensure boolean predicates (e.g.
|
||||
|
||||
+1
-1
@@ -154,7 +154,7 @@ tasks:
|
||||
- bash tools/webhook-send.sh {{.CLI_ARGS}}
|
||||
|
||||
bench:prreview:
|
||||
desc: Score PR review findings on the Martian benchmark (-- -findings DIR; --help for flags)
|
||||
desc: Review or rescore the Martian benchmark (-- -review-model PROVIDER/MODEL -classification-model PROVIDER/MODEL; --help for flags)
|
||||
cmds:
|
||||
- go run ./cmd/archie-prbench {{.CLI_ARGS}}
|
||||
|
||||
|
||||
@@ -20,13 +20,20 @@ func main() { os.Exit(run()) }
|
||||
func run() int {
|
||||
var opts prbench.Options
|
||||
var problems string
|
||||
flag.StringVar(&opts.Config, "config", configuration.DefaultConfigPath(), "configuration file whose providers serve the judge model")
|
||||
flag.StringVar(&opts.Fixtures, "findings", "", "directory of <problem id>.json findings to score")
|
||||
flag.StringVar(&opts.Config, "config", configuration.DefaultConfigPath(), "configuration file whose providers serve the reviewer and judge")
|
||||
flag.StringVar(&opts.Fixtures, "findings", "", "directory of <problem id>.json findings to rescore; omit to review live")
|
||||
flag.StringVar(&opts.ReviewModel, "review-model", "", "provider/model for the review role in live runs")
|
||||
flag.StringVar(&opts.ClassifyModel, "classification-model", "", "provider/model for the classification role in live runs")
|
||||
flag.StringVar(&opts.JudgeModel, "judge-model", "anthropic/claude-sonnet-4.6", "provider/model of the independent judge")
|
||||
flag.StringVar(&opts.Out, "out", "prbench-out", "directory for per-problem results and summary.json")
|
||||
flag.StringVar(&opts.Out, "out", "bin/prbench-results", "directory for per-problem results and summary.json")
|
||||
flag.StringVar(&opts.Resume, "resume", "", "reuse complete results from a previous run with the same models")
|
||||
flag.StringVar(&problems, "problems", "", "comma-separated problem ids (default: all)")
|
||||
flag.IntVar(&opts.Concurrency, "concurrency", 4, "problems judged at once")
|
||||
flag.Parse()
|
||||
opts.GitHubToken = os.Getenv("GH_TOKEN")
|
||||
if opts.GitHubToken == "" {
|
||||
opts.GitHubToken = os.Getenv("GITHUB_TOKEN")
|
||||
}
|
||||
if problems != "" {
|
||||
opts.Problems = strings.Split(problems, ",")
|
||||
}
|
||||
|
||||
@@ -355,7 +355,7 @@ func TestStateStoreRealProcessSmoke(t *testing.T) {
|
||||
// Combined task/event/capture view: a task, an event on that task, and an
|
||||
// unrelated capture written through one consumer, read back across all
|
||||
// three surfaces in a single pass (the dashboard/observability read path).
|
||||
task, err := cl.EnqueueChatTask(ctx, "acme", "widget", "verify state store", "body", "implement", "")
|
||||
task, err := cl.EnqueueChatTask(ctx, "acme", "widget", "verify state store", "body", "implement", "", nil)
|
||||
if err != nil || task == nil {
|
||||
t.Fatalf("EnqueueChatTask = %+v, %v", task, err)
|
||||
}
|
||||
@@ -485,7 +485,7 @@ func TestStateStoreRealProcessRestartRecovery(t *testing.T) {
|
||||
|
||||
first := startStateStoreProcess(t, bin, cfg)
|
||||
s1 := dial(t, first.addr)
|
||||
task, err := s1.EnqueueChatTask(t.Context(), "acme", "widget", "survive restart", "body", "implement", "")
|
||||
task, err := s1.EnqueueChatTask(t.Context(), "acme", "widget", "survive restart", "body", "implement", "", nil)
|
||||
if err != nil || task == nil {
|
||||
t.Fatalf("first EnqueueChatTask = %+v, %v", task, err)
|
||||
}
|
||||
|
||||
@@ -52,7 +52,7 @@ func recoveryFixture(t *testing.T) (string, *pgstore.TaskDB) {
|
||||
// seedTask writes one task so the database holds more than its settings.
|
||||
func seedTask(t *testing.T, st *pgstore.TaskDB, title string) {
|
||||
t.Helper()
|
||||
if _, err := st.EnqueueChatTask(t.Context(), "acme", "widget", title, "body", "implement", ""); err != nil {
|
||||
if _, err := st.EnqueueChatTask(t.Context(), "acme", "widget", title, "body", "implement", "", nil); err != nil {
|
||||
t.Fatalf("seed task %q: %v", title, err)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -496,9 +496,6 @@ target = "127.0.0.1:9090" # REQUIRED. The State Store gRPC address arch
|
||||
# # [containers].max_concurrency globally.
|
||||
# # persistent_storage = true # reuse a per-repo Docker volume and Git object cache until volume_ttl
|
||||
# # max_retries = 5 # per-repo override of the global max_retries above
|
||||
# # review_enabled = true # run a fresh adversarial reviewer (no path to the implementer's
|
||||
# # context) after gates pass and before the PR opens. Default false --
|
||||
# # an uncalibrated reviewer that cries wolf is worse than no review.
|
||||
|
||||
# ── Tools ───────────────────────────────────────────────────────────────────
|
||||
# Limits on what a tool may return. These bound how much of the context window
|
||||
|
||||
@@ -124,7 +124,7 @@
|
||||
"url": "/dev/development/daemon/",
|
||||
"title": "Daemon",
|
||||
"section": "Development",
|
||||
"body": "# Daemon\n\nThe daemon claims and runs tasks, and turns captured events into tasks through\nbindings. This page covers both.\n\n## Task lifecycle\n\nAuthority: `docs/architecture/agent-system.md`. The status vocabulary lives in\n`internal/taskstate`, which imports no internal package.\n\n### Adding a status\n\n1. Add the constant to `internal/taskstate/taskstate.go` and place it in\n `Terminal`, `Actions` and `Statuses`. Its value is stored, so renaming it\n later is a migration.\n2. Alias it in `internal/domain/workflow/task/task.go` and\n `internal/domain/workflow/task_aliases.go`.\n3. Add it to the default catalogue in `ui/src/lib/task-meta.ts`. The server\n catalogue replaces the defaults when it loads, but the defaults render first.\n4. Find every hand-written status list in SQL:\n `grep -rn \"'merged'\" internal/infrastructure/postgres/queries`.\n `ClearTerminalTasks` must match `taskstate.Terminal`, which\n `TestClearTerminalTasksMatchesTaskstate` enforces. The stats queries count\n outcomes by name; decide whether the new status counts.\n5. Extend the table in `taskstate_test.go`.\n\n### Adding a task field\n\nFollow [State Store](/dev/development/state-store/) for the\ncolumn and the wire. Then decide who writes it: `UpdateTask` writes a fixed\ncolumn set, so a field set at enqueue needs its own statement (as\n`StampTaskBinding` does for binding provenance and inputs).\n\n### Changing what the agent receives\n\n- The task brief is `container.TaskPayload`, written to\n `\u003cworkspace\u003e/.git/task.json` by `WriteTaskJSON`. Never write it to the\n workspace root: the commit stage stages everything there.\n- The run request is `taskrun.Request`. `Validate` rejects a request that\n cannot run; a new requirement goes there.\n- Parking always carries a class (`taskstate.ParkTransient`,\n `ParkNeedsHuman`, `ParkTerminal`) chosen by the code that parks. Pick the one\n that says what an operator must do.\n- A task with no repository (`Task.HasRepository`) runs in a scratch workspace\n with no worktree grant. Code that assumes an owner and repo must check first.\n\n## Event automation\n\nAuthority: `docs/prds/event-automation.md` (the model: source, event type,\nmapping, binding, workflow, profile) and `docs/architecture/bindings.md`\n(threat model).\n\n### Where each check lives\n\nA binding is checked twice, and a new rule usually belongs in both places:\n\n- **On save**, in `checkedBinding` (`internal/webui/api_binding.go`), so the\n operator sees the problem while editing. It calls the domain checks\n `binding.Validate`, `binding.CompileFilter` and `Binding.CheckWorkflow`.\n- **At dispatch**, in `dispatchOneBinding` and `resolveBindingTarget`\n (`internal/daemon`), because the workflow or mapping may have changed since\n the save. A refusal is recorded with `recordDispatchFailure` and starts no\n task.\n\nPut the rule itself in `internal/domain/binding` or\n`internal/domain/workflow/task` and call it from both, rather than writing it\ntwice.\n\n### Adding a binding field\n\n1. Domain type and `Validate` in `internal/domain/binding`.\n2. Column, queries, store mapping, wire field, `newColumns`\n ([State Store](/dev/development/state-store/)).\n3. The request type and `checkedBinding` in `api_binding.go`.\n4. The draft, payload and editor in `ui/src/bindings/`\n ([Frontend UI](/dev/development/frontend-ui/)).\n5. Its effect at dispatch, with a case in\n `internal/daemon/dispatch_inputs_test.go` or `dispatch_bindings_test.go`.\n\n### Rules that must hold\n\n- Signing belongs to the source. Intake verifies the HMAC against the source's\n secret before the payload is parsed.\n- An event must be identified as an event type before anything dispatches it,\n and a binding applies only to its mapping's event type. Dispatch test\n fixtures therefore create an event type before inserting captures.\n- The dispatch ledger is claimed before the task is enqueued (at most once).\n- Anything an event supplies that selects what the agent acts on, such as a\n repository, must be checked against configuration. An event never widens\n what an agent can reach.\n"
|
||||
"body": "# Daemon\n\nThe daemon claims and runs tasks, and turns captured events into tasks through\nbindings. This page covers both.\n\n## Task lifecycle\n\nAuthority: `docs/architecture/agent-system.md`. The status vocabulary lives in\n`internal/taskstate`, which imports no internal package.\n\n### Adding a status\n\n1. Add the constant to `internal/taskstate/taskstate.go` and place it in\n `Terminal`, `Actions` and `Statuses`. Its value is stored, so renaming it\n later is a migration.\n2. Alias it in `internal/domain/workflow/task/task.go` and\n `internal/domain/workflow/task_aliases.go`.\n3. Add it to the default catalogue in `ui/src/lib/task-meta.ts`. The server\n catalogue replaces the defaults when it loads, but the defaults render first.\n4. Find every hand-written status list in SQL:\n `grep -rn \"'merged'\" internal/infrastructure/postgres/queries`.\n `ClearTerminalTasks` must match `taskstate.Terminal`, which\n `TestClearTerminalTasksMatchesTaskstate` enforces. The stats queries count\n outcomes by name; decide whether the new status counts.\n5. Extend the table in `taskstate_test.go`.\n\n### Adding a task field\n\nFollow [State Store](/dev/development/state-store/) for the\ncolumn and the wire. Then decide who writes it: `UpdateTask` writes a fixed\ncolumn set, so a field set at enqueue belongs in the insert that creates the\nrow (as `InsertChatTask` does for inputs), or in its own statement when only\nsome producers set it (as `StampTaskBinding` does for binding provenance).\n\n### Changing what the agent receives\n\n- The task brief is `container.TaskPayload`, written to\n `\u003cworkspace\u003e/.git/task.json` by `WriteTaskJSON`. Never write it to the\n workspace root: the commit stage stages everything there.\n- The run request is `taskrun.Request`. `Validate` rejects a request that\n cannot run; a new requirement goes there.\n- Parking always carries a class (`taskstate.ParkTransient`,\n `ParkNeedsHuman`, `ParkTerminal`) chosen by the code that parks. Pick the one\n that says what an operator must do.\n- A task with no repository (`Task.HasRepository`) runs in a scratch workspace\n with no worktree grant. Code that assumes an owner and repo must check first.\n\n## Event automation\n\nAuthority: `docs/prds/event-automation.md` (the model: source, event type,\nmapping, binding, workflow, profile) and `docs/architecture/bindings.md`\n(threat model).\n\n### Where each check lives\n\nA binding is checked twice, and a new rule usually belongs in both places:\n\n- **On save**, in `checkedBinding` (`internal/webui/api_binding.go`), so the\n operator sees the problem while editing. It calls the domain checks\n `binding.Validate`, `binding.CompileFilter` and `Binding.CheckWorkflow`.\n- **At dispatch**, in `dispatchOneBinding` and `resolveBindingTarget`\n (`internal/daemon`), because the workflow or mapping may have changed since\n the save. A refusal is recorded with `recordDispatchFailure` and starts no\n task.\n\nPut the rule itself in `internal/domain/binding` or\n`internal/domain/workflow/task` and call it from both, rather than writing it\ntwice.\n\n### Adding a binding field\n\n1. Domain type and `Validate` in `internal/domain/binding`.\n2. Column, queries, store mapping, wire field, `newColumns`\n ([State Store](/dev/development/state-store/)).\n3. The request type and `checkedBinding` in `api_binding.go`.\n4. The draft, payload and editor in `ui/src/bindings/`\n ([Frontend UI](/dev/development/frontend-ui/)).\n5. Its effect at dispatch, with a case in\n `internal/daemon/dispatch_inputs_test.go` or `dispatch_bindings_test.go`.\n\n### Rules that must hold\n\n- Signing belongs to the source. Intake verifies the HMAC against the source's\n secret before the payload is parsed.\n- An event must be identified as an event type before anything dispatches it,\n and a binding applies only to its mapping's event type. Dispatch test\n fixtures therefore create an event type before inserting captures.\n- The dispatch ledger is claimed before the task is enqueued (at most once).\n- Anything an event supplies that selects what the agent acts on, such as a\n repository, must be checked against configuration. An event never widens\n what an agent can reach.\n"
|
||||
},
|
||||
{
|
||||
"url": "/dev/development/frontend-ui/",
|
||||
@@ -154,7 +154,7 @@
|
||||
"url": "/dev/development/workflows/",
|
||||
"title": "Workflows",
|
||||
"section": "Development",
|
||||
"body": "# Workflows\n\nAuthority: `docs/architecture/agent-system.md` for the lifecycle,\n`docs/prds/event-automation.md` for inputs, repository modes and profiles.\n\nWorkflow definitions are YAML stored as a control-plane resource. The State\nStore validates them and the agent compiles them, each against the step\nvocabulary it registers, so both must see the same step types.\n\n## Adding a step type\n\n1. Write the factory in `internal/domain/workflow` as a `StepType`. It decodes\n and validates its settings and returns an error for anything it cannot\n honour. The factory runs when a definition is saved and again when it is\n compiled, so a bad definition is refused on save.\n2. Register it through a provider in\n `internal/infrastructure/workflowsteps/workflowsteps.go`. That one list is\n what every process registers, which is how the validating and executing\n sides agree.\n3. An agent-driven step builds on `AgentStage`; do not start a second path to\n the agent runner.\n4. A step that needs no repository goes in `repoFreeStepTypes` in\n `definition.go`, or workflows declaring `repository: none` or `optional`\n cannot use it. A step that needs one may assume `tc.Dir` holds a worktree\n only when the workflow's repository is `required`.\n5. A step that ends the workflow sets `tc.Outcome`. A workflow that runs out of\n steps without an outcome is parked as a definition bug.\n6. Add a parse test (settings accepted and refused) and a run test with fake\n `Trees` and agent runner.\n\n## Changing the definition format\n\n- The fields a caller needs without the engine (inputs, repository, profile)\n live in `task.WorkflowInterface`, because the dashboard process must not\n import the workflow engine (`cmd/archie-ui/architecture_test.go`). Anything a\n binding or the dashboard checks goes there.\n- The YAML decoder rejects unknown keys. Existing stored definitions must keep\n parsing, so a new key is optional with a default that preserves today's\n behaviour.\n- Examples in `examples/workflows/` are parsed against the registered\n vocabulary by `TestExampleWorkflowsParse`. Add one for a new capability.\n"
|
||||
"body": "# Workflows\n\nAuthority: `docs/architecture/agent-system.md` for the lifecycle,\n`docs/prds/event-automation.md` for inputs, repository modes and profiles.\n\nWorkflow definitions are YAML stored as a control-plane resource. The State\nStore validates them and the agent compiles them, each against the step\nvocabulary it registers, so both must see the same step types.\n\n## Adding a step type\n\n1. Write the factory in `internal/domain/workflow` as a `StepType`. It decodes\n and validates its settings and returns an error for anything it cannot\n honour. The factory runs when a definition is saved and again when it is\n compiled, so a bad definition is refused on save.\n2. Register it through a provider in\n `internal/infrastructure/workflowsteps/workflowsteps.go`. That one list is\n what every process registers, which is how the validating and executing\n sides agree.\n3. An agent-driven step builds on `AgentStage`; do not start a second path to\n the agent runner.\n4. A step that needs no repository goes in `repoFreeStepTypes` in\n `definition.go`, or workflows declaring `repository: none` or `optional`\n cannot use it. A step that needs one may assume `tc.Dir` holds a worktree\n only when the workflow's repository is `required`.\n5. A step that ends the workflow sets `tc.Outcome`. A workflow that runs out of\n steps without an outcome is parked as a definition bug.\n6. Add a parse test (settings accepted and refused) and a run test with fake\n `Trees` and agent runner.\n\n## Changing the definition format\n\n- The fields a caller needs without the engine (inputs, repository, profile)\n live in `task.WorkflowInterface`, because the dashboard process must not\n import the workflow engine (`cmd/archie-ui/architecture_test.go`). Anything a\n binding or the dashboard checks goes there.\n- The YAML decoder rejects unknown keys. Existing stored definitions must keep\n parsing, so a new key is optional with a default that preserves today's\n behaviour.\n- Examples in `examples/workflows/` are parsed against the registered\n vocabulary by `TestExampleWorkflowsParse`. Add one for a new capability.\n\n## Starting a workflow from chat\n\n`task_spawn` accepts `workflow` and an `inputs` object. The gateway forwards\ninputs through the task creation contract; it does not interpret individual\nworkflows. The daemon checks inputs against the pinned `WorkflowInterface`\nbefore acquiring a container and parks invalid requests with the reason.\n"
|
||||
},
|
||||
{
|
||||
"url": "/dev/prds/01-project-management/",
|
||||
@@ -412,7 +412,7 @@
|
||||
"url": "/dev/prds/pr-review-agent/",
|
||||
"title": "PR review agent",
|
||||
"section": "PRDs",
|
||||
"body": "# PR review agent\n\n**Status:** Approved\n**Authority:** `docs/architecture/agent-system.md`; supersedes the reviewer\nexecutor and blocking rule in `docs/architecture/adversarial-review.md`, and\nthe designs in `docs/prds/operator-pr-review.md` and\n`docs/prds/review-calibration.md`.\n**Compounds with:** `docs/prds/execution-tree-state-machine.md` (every agent\ncall is a StepExecution), `docs/prds/external-agent-harness.md` (any agent\nstep may run on a harness), `docs/prds/inline-review.md` (posting),\n`docs/prds/pr-review-remediation.md` (consumes the findings).\n\n## Decision\n\nArchie ships a pull request reviewer as its first end-to-end product. It\nreviews any pull request on a watched repository, a PR an operator names, and\narchie's own PRs before they open.\n\nThe reviewer is a fixed pipeline, not a single agent. The review strategy is\ngenerated per PR: three lens agents propose review dimensions from the PR's\ncontent, one reviewer definition runs once per dimension, and later agents\nverify, challenge and combine the findings. Everything that can be computed\nis code: diff parsing, clustering, blast radius, evidence extraction,\nscoring, deduplication, line mapping and the review event.\n\nThe pipeline is **recall-first**. It posts every evidenced finding above the\nconfidence threshold, up to the comment cap. A precision gate exists as an\noperator dial and is off by default.\n\nAcceptance is measured, not asserted: see Verification.\n\n## Pipeline\n\nEach numbered phase is a stage of the `pr-review` workflow. Each agent call is\nan `agent` StepExecution; fan-out phases record one child per call.\n\n1. **Intake.** Code reads the PR metadata and diff statistics: type,\n complexity, languages, areas touched, risk signals. Depth comes from lines\n changed: under 100 is `quick` (at most 3 dimensions), under 500 is\n `standard` (6), otherwise `deep` (12). An operator may set depth\n explicitly.\n2. **Anatomy.** Code parses the diff into files and hunks, clusters related\n files, and computes the blast radius: unchanged files that import or are\n imported by changed files. One agent writes the PR narrative, risk\n surfaces, changes that do not fit the narrative, and gaps between the\n description and the diff.\n3. **Dimension selection.** Three lens agents run in parallel over the intake,\n anatomy and diff, each able to read the repository:\n - **behaviour:** where the old and new code diverge for any input; API\n contracts, concurrency and state, security, error handling, data flow;\n - **mechanics:** whether the code runs at the language and framework\n level; types, signatures and every caller, decorators and middleware,\n framework contracts, imports;\n - **fit:** patterns, complexity, abstraction, test adequacy, documentation\n that no longer matches, dependencies, migration completeness.\n\n Each lens returns review dimensions: a name, a reviewer prompt written for\n this PR, target files, context files and a priority. Code merges them,\n drops duplicates and keeps the highest-priority dimensions up to the\n depth's cap.\n4. **Review.** One reviewer definition runs once per dimension, in parallel,\n at most 8 at a time. Each reads its target and context files and may\n follow references. Each finding carries file, line range, severity\n (`critical`, `important`, `suggestion`, `nitpick`), title, body, an\n optional suggested fix, quoted evidence, confidence and tags.\n5. **Verification layer**, over the review findings:\n - code extracts an evidence package per finding: the code at the cited\n lines and snippets from its callers;\n - an evidence verifier checks high-priority findings against their\n packages;\n - an adversary confirms or challenges each finding, looks for issues every\n reviewer missed, and is more sceptical when intake scores the PR as\n likely machine-written;\n - a compound agent looks, within each cluster of confirmed findings, for\n defects that only exist in combination.\n6. **Coverage and consistency**, in parallel:\n - a coverage gate checks every cluster and high-exposure blast-radius file\n was reviewed, and runs gap reviewers for what was not, at most 2 rounds;\n - consistency verification lists every cross-location obligation the\n changed code creates (what must be true elsewhere for this line to be\n correct), then verifies each by reading both ends. A broken obligation\n becomes a finding.\n7. **Synthesis**, code only:\n - score = severity weight (1.0, 0.7, 0.3, 0.1) × confidence × multipliers:\n compound 1.5, adversary-confirmed 1.3, adversary-challenged 0.5, likely\n machine-written PR 1.2, blast radius over 10 files 1.2;\n - drop findings under the severity's confidence floor: critical 0.2,\n important 0.3, suggestion 0.4, nitpick 0.4;\n - merge exact duplicates (file, overlapping lines, category) in code, and\n nearLine truncated
|
||||
"body": "# PR review agent\n\n**Status:** Approved\n**Authority:** `docs/architecture/agent-system.md`; supersedes the reviewer\nexecutor and blocking rule in `docs/architecture/adversarial-review.md`, and\nthe designs in `docs/prds/operator-pr-review.md` and\n`docs/prds/review-calibration.md`.\n**Compounds with:** `docs/prds/execution-tree-state-machine.md` (every agent\ncall is a StepExecution), `docs/prds/external-agent-harness.md` (any agent\nstep may run on a harness), `docs/prds/inline-review.md` (posting),\n`docs/prds/pr-review-remediation.md` (consumes the findings).\n\n## Decision\n\nArchie ships a pull request reviewer as its first end-to-end product. It\nreviews any pull request on a watched repository, a PR an operator names, and\narchie's own PRs before they open.\n\nThe reviewer is a fixed pipeline, not a single agent. The review strategy is\ngenerated per PR: three lens agents propose review dimensions from the PR's\ncontent, one reviewer definition runs once per dimension, and later agents\nverify, challenge and combine the findings. Everything that can be computed\nis code: diff parsing, clustering, blast radius, evidence extraction,\nscoring, deduplication, line mapping and the review event.\n\nThe pipeline is **recall-first**. It posts every evidenced finding above the\nconfidence threshold, up to the comment cap. A precision gate exists as an\noperator dial and is off by default.\n\nAcceptance is measured, not asserted: see Verification.\n\n## Pipeline\n\nEach numbered phase is a stage of the `pr-review` workflow. Each agent call is\nan `agent` StepExecution; fan-out phases record one child per call.\n\n1. **Intake.** Code reads the PR metadata and diff statistics: type,\n complexity, languages, areas touched, risk signals. Depth comes from lines\n changed: under 100 is `quick` (at most 3 dimensions), under 500 is\n `standard` (6), otherwise `deep` (12). An operator may set depth\n explicitly.\n2. **Anatomy.** Code parses the diff into files and hunks, clusters related\n files, and computes the blast radius: unchanged files that import or are\n imported by changed files. One agent writes the PR narrative, risk\n surfaces, changes that do not fit the narrative, and gaps between the\n description and the diff.\n3. **Dimension selection.** Three lens agents run in parallel over the intake\n (including its machine-written score), anatomy and diff, each able to read\n the repository. A PR that intake scores as likely machine-written adds a\n hallucination-check dimension: claims, citations and behaviour the diff\n does not actually implement.\n - **behaviour:** where the old and new code diverge for any input; API\n contracts, concurrency and state, security, error handling, data flow;\n - **mechanics:** whether the code runs at the language and framework\n level; types, signatures and every caller, decorators and middleware,\n framework contracts, imports;\n - **fit:** patterns, complexity, abstraction, test adequacy, documentation\n that no longer matches, dependencies, migration completeness.\n\n Each lens returns review dimensions: a name, a reviewer prompt written for\n this PR, target files, context files and a priority. Code merges them,\n drops duplicates and keeps the highest-priority dimensions up to the\n depth's cap.\n4. **Review.** One reviewer definition runs once per dimension, in parallel,\n at most 8 at a time. Each reads its target and context files and may\n follow references. Each finding carries file, line range, severity\n (`critical`, `important`, `suggestion`, `nitpick`), title, body, an\n optional suggested fix, quoted evidence, confidence and tags. A reviewer\n that exhausts its turn cap or deadline before its terminal tool call\n records a distinct StepExecution status from a reviewer that completed\n and found nothing; synthesis and the posted review treat the two\n differently (the former is unreviewed, the latter is reviewed-clean).\n5. **Verification layer**, over the review findings:\n - code extracts an evidence package per finding: the code at the cited\n lines and snippets from its callers;\n - an evidence verifier checks high-priority findings against their\n packages;\n - an adversary confirms or challenges each finding, looks for issues every\n reviewer missed, and is more sceptical when intake scores the PR as\n likely machine-written;\n - a compound agent looks, within each cluster of confirmed findings, for\n defects that only exist in combination.\n6. **Coverage and consistency**, in parallel:\n - a coverage gate checks every cluster and high-exposure blast-radius file\n was reviewed, and runs gap reviewers for what was not, at most 2 rounds;\n - consistency verification lists every cross-location obligation the\n changed code creates (what must be true elsewhere for this line to be\n correct), then verifies each by reading both endLine truncated
|
||||
},
|
||||
{
|
||||
"url": "/dev/prds/pr-review-remediation/",
|
||||
|
||||
@@ -11,6 +11,12 @@
|
||||
"title": "Building your first playbook binding",
|
||||
"section": "Guides",
|
||||
"body": "# Building your first playbook binding\n\nThis walks through the whole no-code path the event-capture epic promises:\npoint an external app's webhook at archie, see the real payload it sends,\nmap the fields you care about, bind them to a workflow, and watch the next\nmatching event turn into a task -- no code, no redeploy, no restart. See\n`docs/architecture/bindings.md` for the model and threat model behind each\nstep below.\n\n## 1. Send a test event\n\nPoint whatever external app you want archie to react to at:\n\n```\nPOST http://\u003cyour-archied-host\u003e:8484/webhooks/capture/\u003csource\u003e\n```\n\n`\u003csource\u003e` is a name you choose for this sender (e.g. `sentry`, `github-ci`,\n`stripe`) -- it's just a URL path segment, not a preset list. Send one real\nevent however is easiest for a first test, e.g.:\n\n```bash\ncurl -X POST http://localhost:8484/webhooks/capture/sentry \\\n -H \"Content-Type: application/json\" \\\n -d '{\"issue\": {\"title\": \"NullPointerException in checkout\"}, \"level\": \"error\"}'\n```\n\nThis event isn't signed yet, so it's captured but marked **unauthenticated**\n-- visible for you to design against, but incapable of triggering anything\n(`docs/architecture/bindings.md`'s threat-model section explains why: an\nunauthenticated event can never match a binding, no matter how it looks).\n\n## 2. Inspect the captured payload\n\nOpen the dashboard's **Event inspector** (`/captures`). Your test event\nappears there with its real body -- this is schema-by-example: you're\nlooking at an actual payload from the actual sender, not guessing at a\nschema in advance.\n\n## 3. Map the fields you care about\n\nOpen **Field mappings** (`/mappings`) → **New mapping**. Pick your captured\nevent from the dropdown, then click through its payload tree on the right --\nevery key and value is clickable. Clicking a value binds it as a field\n(name, JSON path, and an inferred type you can change). Add whichever fields\nyour workflow needs (e.g. the issue title), mark the ones that must be\npresent as **Required**, hit **Preview** to confirm they resolve correctly\nagainst the real payload, then **Save**.\n\n## 4. Create the binding\n\nOpen **Playbook bindings** (`/bindings`) → **New binding**:\n\n- **Name** -- anything descriptive.\n- **Source** -- the same `\u003csource\u003e` segment from step 1 (e.g. `sentry`).\n- **Field mapping** -- the one you just saved.\n- **Workflow** -- which registered workflow should run on a match.\n- **Repo pin** (optional) -- leave both blank to use the single configured\n repo; set owner/repo explicitly if you run a multi-repo deployment.\n- **Shared secret** -- a real secret (16+ bytes), never `test1234`. The\n sender signs every request body with this via HMAC-SHA256 so archie can\n tell a genuine event from anyone who finds the URL.\n\nSave. The binding starts in **draft** -- nothing fires yet.\n\n## 5. Approve it\n\nA **draft** or freshly-**edited** binding needs an explicit human approval\nbefore it can ever match a live event -- this is deliberate, not a missing\nstep: an operator authoring a binding is a considered act, an event arriving\nis not, and nothing should self-arm (`docs/architecture/bindings.md`'s\nthreat-model section, point 2). Click **Approve**. Status becomes **armed**.\n\n## 6. Fire it for real\n\nSend the same kind of event again, this time **signed** with the binding's\nsecret:\n\n```bash\nBODY='{\"issue\": {\"title\": \"NullPointerException in checkout\"}, \"level\": \"error\"}'\nSECRET=\"the-secret-you-set-in-step-4\"\nSIG=$(printf '%s' \"$BODY\" | openssl dgst -sha256 -hmac \"$SECRET\" | sed 's/^.* //')\n\ncurl -X POST http://localhost:8484/webhooks/capture/sentry \\\n -H \"Content-Type: application/json\" \\\n -H \"X-Hub-Signature-256: sha256=$SIG\" \\\n -d \"$BODY\"\n```\n\nThis event is captured as **authenticated**, matched against your armed\nbinding, and dispatched into a new task through the exact same\ngate/worktree/sandbox pipeline every other task goes through -- see\n`docs/architecture/messaging-and-work-intake.md`'s \"Intake provenance\". Check\nthe **Tasks** page; a new task should appear, its body built from the fields\nyour mapping resolved.\n\n## Editing a live binding\n\nAny edit to an armed binding -- even a typo fix -- drops it back to\n**pending_approval** automatically. This can't be bypassed by editing; it\nhas to be re-approved. When editing, leave the **Shared secret** field blank\nto keep the existing secret; only fill it in if you're deliberately\nrotating it.\n"
|
||||
},
|
||||
{
|
||||
"url": "/docs/guides/pr-review-benchmark/",
|
||||
"title": "PR review benchmark",
|
||||
"section": "Guides",
|
||||
"body": "# PR review benchmark\n\nRun `scripts/pr-review-benchmark.sh` with `PRBENCH_REVIEW_MODEL` and\n`PRBENCH_CLASSIFICATION_MODEL` set to configured `provider/model` references.\nThe judge defaults to `anthropic/claude-sonnet-4.6`; set\n`PRBENCH_JUDGE_MODEL` to another configured model when needed. The script uses\n`~/.config/archie/config.toml` for providers and `GH_TOKEN` or `GITHUB_TOKEN`\nfor read access to the benchmark pull requests. It posts no forge review.\n\nThe default run reviews the 38 runnable Martian pull requests once each at\n`deep`, judges the comments against 102 golden comments, and writes\n`bin/prbench-results/run.json`, `summary.json`, and one JSON result per pull\nrequest. That output directory is ignored by git. A completed full run fails\nwhen any problem fails or golden micro-recall is below 0.70. Golden-only\nprecision is reported alongside recall. Set `PRBENCH_OUT` to keep separate\nruns; `-problems ID` limits a smoke run to one pull request. Completed results\nare saved as each PR finishes. Pass `-resume DIR` to reuse completed results\nfrom a run with the same models; each reused problem must match the current\nbenchmark entry, and failed problems are rerun.\nA smoke run alone is not the acceptance run.\n\nThe benchmark passes only each problem's ID and pull request URL to the\nreviewer. The judge receives the golden comments after the review has ended.\nThe runner uses the production decision phases through the merge gate,\nincluding scoring and the 25-comment cap, then stops before polish and forge\nposting. A reviewer that fails to finish makes that problem fail. The results\nrecord budget-skipped phases and the selected models. Model substitutions do\nnot satisfy the PRD's named-model criterion even if recall clears the threshold.\n"
|
||||
}
|
||||
],
|
||||
"nav": [
|
||||
@@ -24,6 +30,10 @@
|
||||
{
|
||||
"title": "Building your first playbook binding",
|
||||
"url": "/docs/guides/first-playbook/"
|
||||
},
|
||||
{
|
||||
"title": "PR review benchmark",
|
||||
"url": "/docs/guides/pr-review-benchmark/"
|
||||
}
|
||||
]
|
||||
}
|
||||
|
||||
@@ -28,8 +28,9 @@ Authority: `docs/architecture/agent-system.md`. The status vocabulary lives in
|
||||
|
||||
Follow [State Store](state-store.md) for the
|
||||
column and the wire. Then decide who writes it: `UpdateTask` writes a fixed
|
||||
column set, so a field set at enqueue needs its own statement (as
|
||||
`StampTaskBinding` does for binding provenance and inputs).
|
||||
column set, so a field set at enqueue belongs in the insert that creates the
|
||||
row (as `InsertChatTask` does for inputs), or in its own statement when only
|
||||
some producers set it (as `StampTaskBinding` does for binding provenance).
|
||||
|
||||
### Changing what the agent receives
|
||||
|
||||
|
||||
@@ -39,3 +39,10 @@ vocabulary it registers, so both must see the same step types.
|
||||
behaviour.
|
||||
- Examples in `examples/workflows/` are parsed against the registered
|
||||
vocabulary by `TestExampleWorkflowsParse`. Add one for a new capability.
|
||||
|
||||
## Starting a workflow from chat
|
||||
|
||||
`task_spawn` accepts `workflow` and an `inputs` object. The gateway forwards
|
||||
inputs through the task creation contract; it does not interpret individual
|
||||
workflows. The daemon checks inputs against the pinned `WorkflowInterface`
|
||||
before acquiring a container and parks invalid requests with the reason.
|
||||
@@ -0,0 +1,28 @@
|
||||
# PR review benchmark
|
||||
|
||||
Run `scripts/pr-review-benchmark.sh` with `PRBENCH_REVIEW_MODEL` and
|
||||
`PRBENCH_CLASSIFICATION_MODEL` set to configured `provider/model` references.
|
||||
The judge defaults to `anthropic/claude-sonnet-4.6`; set
|
||||
`PRBENCH_JUDGE_MODEL` to another configured model when needed. The script uses
|
||||
`~/.config/archie/config.toml` for providers and `GH_TOKEN` or `GITHUB_TOKEN`
|
||||
for read access to the benchmark pull requests. It posts no forge review.
|
||||
|
||||
The default run reviews the 38 runnable Martian pull requests once each at
|
||||
`deep`, judges the comments against 102 golden comments, and writes
|
||||
`bin/prbench-results/run.json`, `summary.json`, and one JSON result per pull
|
||||
request. That output directory is ignored by git. A completed full run fails
|
||||
when any problem fails or golden micro-recall is below 0.70. Golden-only
|
||||
precision is reported alongside recall. Set `PRBENCH_OUT` to keep separate
|
||||
runs; `-problems ID` limits a smoke run to one pull request. Completed results
|
||||
are saved as each PR finishes. Pass `-resume DIR` to reuse completed results
|
||||
from a run with the same models; each reused problem must match the current
|
||||
benchmark entry, and failed problems are rerun.
|
||||
A smoke run alone is not the acceptance run.
|
||||
|
||||
The benchmark passes only each problem's ID and pull request URL to the
|
||||
reviewer. The judge receives the golden comments after the review has ended.
|
||||
The runner uses the production decision phases through the merge gate,
|
||||
including scoring and the 25-comment cap, then stops before polish and forge
|
||||
posting. A reviewer that fails to finish makes that problem fail. The results
|
||||
record budget-skipped phases and the selected models. Model substitutions do
|
||||
not satisfy the PRD's named-model criterion even if recall clears the threshold.
|
||||
@@ -57,6 +57,7 @@
|
||||
"/news/archie/1.41.2/": "/news/1.41.2/",
|
||||
"/news/archie/1.42.0/": "/news/1.42.0/",
|
||||
"/news/archie/1.43.0/": "/news/1.43.0/",
|
||||
"/news/archie/1.44.0/": "/news/1.44.0/",
|
||||
"/news/archie/1.5.0/": "/news/1.5.0/",
|
||||
"/news/archie/1.6.0/": "/news/1.6.0/",
|
||||
"/news/archie/1.7.0/": "/news/1.7.0/",
|
||||
@@ -131,6 +132,7 @@
|
||||
"/news/archied/1.41.2/": "/news/1.41.2/",
|
||||
"/news/archied/1.42.0/": "/news/1.42.0/",
|
||||
"/news/archied/1.43.0/": "/news/1.43.0/",
|
||||
"/news/archied/1.44.0/": "/news/1.44.0/",
|
||||
"/news/archied/1.5.0/": "/news/1.5.0/",
|
||||
"/news/archied/1.6.0/": "/news/1.6.0/",
|
||||
"/news/archied/1.7.0/": "/news/1.7.0/",
|
||||
|
||||
@@ -1,4 +1,9 @@
|
||||
[
|
||||
{
|
||||
"version": "1.44.0",
|
||||
"date": "2026-09-29",
|
||||
"body": "### archied\n\nArchie can now review pull requests.\n\n- **Reviews on request and by watch.** Ask in chat or on the dashboard work form, add a repository to the watch list, or just let archie review its own PRs before it opens them. Review intensity follows a depth dial (quick/standard/deep) classified from the size of the change, and every finished review passes a verification layer — coverage-gap and cross-finding consistency checks — before it is produced.\n- **No forge credential reaches the review sandbox.** The daemon prefetches the pull request's data and rebuilds its source snapshot over the forge's HTTP API, so the agent container works from an isolated copy with no read access to the repository's credentials.\n- **Reviews stop at the operator gate before posting.** `review.precision_gate` / `approve_before_post` decide whether a finished review posts itself or parks for operator approval. With this release the parked state is a stop sign, not a checkpoint: the approve/reject/re-review responses behind the gate are not wired yet, and the gate's settings are not yet editable in the dashboard (configure them in `config.toml`). A parked review sits until that follow-up lands.\n- The adversarial self-review pass archie previously ran before opening its own PRs is superseded by this pipeline and has been removed.\n- Hardened harness input handling: harness identifiers are bounded to integer types.\n\n### archie-agent\n\nThe runtime image is rebuilt with this release; it links the shared pr-review workflow packages behind the reviews above. Hosts that run agent containers need `docker compose pull agent` after updating to pick it up."
|
||||
},
|
||||
{
|
||||
"version": "1.43.0",
|
||||
"date": "2026-09-28",
|
||||
|
||||
@@ -44,8 +44,11 @@ an `agent` StepExecution; fan-out phases record one child per call.
|
||||
imported by changed files. One agent writes the PR narrative, risk
|
||||
surfaces, changes that do not fit the narrative, and gaps between the
|
||||
description and the diff.
|
||||
3. **Dimension selection.** Three lens agents run in parallel over the intake,
|
||||
anatomy and diff, each able to read the repository:
|
||||
3. **Dimension selection.** Three lens agents run in parallel over the intake
|
||||
(including its machine-written score), anatomy and diff, each able to read
|
||||
the repository. A PR that intake scores as likely machine-written adds a
|
||||
hallucination-check dimension: claims, citations and behaviour the diff
|
||||
does not actually implement.
|
||||
- **behaviour:** where the old and new code diverge for any input; API
|
||||
contracts, concurrency and state, security, error handling, data flow;
|
||||
- **mechanics:** whether the code runs at the language and framework
|
||||
@@ -62,7 +65,11 @@ an `agent` StepExecution; fan-out phases record one child per call.
|
||||
at most 8 at a time. Each reads its target and context files and may
|
||||
follow references. Each finding carries file, line range, severity
|
||||
(`critical`, `important`, `suggestion`, `nitpick`), title, body, an
|
||||
optional suggested fix, quoted evidence, confidence and tags.
|
||||
optional suggested fix, quoted evidence, confidence and tags. A reviewer
|
||||
that exhausts its turn cap or deadline before its terminal tool call
|
||||
records a distinct StepExecution status from a reviewer that completed
|
||||
and found nothing; synthesis and the posted review treat the two
|
||||
differently (the former is unreviewed, the latter is reviewed-clean).
|
||||
5. **Verification layer**, over the review findings:
|
||||
- code extracts an evidence package per finding: the code at the cited
|
||||
lines and snippets from its callers;
|
||||
@@ -94,8 +101,9 @@ an `agent` StepExecution; fan-out phases record one child per call.
|
||||
8. **Merge gate.** One classification call per finding decides blocking or
|
||||
advisory: blocking only for broken builds, security, data loss, contract
|
||||
breaks and regressions. A failed call means advisory.
|
||||
9. **Output.** One call per comment tightens its wording, keeping the
|
||||
original on failure. The review is posted as one forge review with inline
|
||||
9. **Output.** One call per comment tightens its wording, running in a
|
||||
bounded pool at most 8 at a time (matching phase 4), keeping the original
|
||||
on failure. The review is posted as one forge review with inline
|
||||
comments. Any blocking finding requests changes; otherwise the review is a
|
||||
comment. Archie never posts an approval.
|
||||
|
||||
|
||||
@@ -41,10 +41,11 @@ func runAsHarnessUser(cmd *exec.Cmd, name string) error {
|
||||
}
|
||||
|
||||
// harnessCredential resolves a user name or "uid[:gid]" to a credential.
|
||||
// Root is refused whichever way it is named.
|
||||
// Root is refused whichever way it is named. IDs are bounded to 31 bits so
|
||||
// they convert to int for os.Chown on every platform.
|
||||
func harnessCredential(name string) (*syscall.Credential, error) {
|
||||
uidText, gidText, hasGid := strings.Cut(name, ":")
|
||||
uid, uidErr := strconv.ParseUint(uidText, 10, 32)
|
||||
uid, uidErr := strconv.ParseUint(uidText, 10, 31)
|
||||
var cred syscall.Credential
|
||||
if uidErr == nil {
|
||||
cred.Uid, cred.Gid = uint32(uid), uint32(uid)
|
||||
@@ -53,18 +54,18 @@ func harnessCredential(name string) (*syscall.Credential, error) {
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("harness user %q: %w", uidText, err)
|
||||
}
|
||||
id, err := strconv.ParseUint(u.Uid, 10, 32)
|
||||
id, err := strconv.ParseUint(u.Uid, 10, 31)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("harness user %q has non-numeric uid %q", uidText, u.Uid)
|
||||
}
|
||||
group, err := strconv.ParseUint(u.Gid, 10, 32)
|
||||
group, err := strconv.ParseUint(u.Gid, 10, 31)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("harness user %q has non-numeric gid %q", uidText, u.Gid)
|
||||
}
|
||||
cred.Uid, cred.Gid = uint32(id), uint32(group)
|
||||
}
|
||||
if hasGid {
|
||||
gid, err := strconv.ParseUint(gidText, 10, 32)
|
||||
gid, err := strconv.ParseUint(gidText, 10, 31)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("harness group %q is not numeric", gidText)
|
||||
}
|
||||
|
||||
@@ -388,6 +388,8 @@ func TestHarnessUser(t *testing.T) {
|
||||
{"0", 0, 0, true},
|
||||
{"root", 0, 0, true},
|
||||
{"not-a-user-anywhere", 0, 0, true},
|
||||
{"2147483648", 0, 0, true},
|
||||
{"1000:2147483648", 0, 0, true},
|
||||
}
|
||||
for _, tt := range tests {
|
||||
cred, err := harnessCredential(tt.user)
|
||||
|
||||
@@ -1,272 +0,0 @@
|
||||
package agentworker
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"strings"
|
||||
|
||||
"github.com/samcharles93/ai-sdk/agent"
|
||||
"github.com/samcharles93/ai-sdk/core"
|
||||
"github.com/samcharles93/ai-sdk/runtime"
|
||||
"github.com/samcharles93/ai-sdk/toolkit"
|
||||
|
||||
"github.com/samcharles93/archie-core/internal/agentexec"
|
||||
"github.com/samcharles93/archie-core/internal/domain/workflow"
|
||||
"github.com/samcharles93/archie-core/internal/taskrun"
|
||||
)
|
||||
|
||||
// newReviewerFor builds the workflow.Reviewer for one task run, or nil
|
||||
// when the repo has not opted into review (Repo.ReviewEnabled == false)
|
||||
// or no providers are configured. Model resolution mirrors AgentStage's
|
||||
// own role->"builder" fallback (agent.go): Models["reviewer"], falling
|
||||
// back to Models["builder"].
|
||||
func newReviewerFor(req taskrun.Request) workflow.Reviewer {
|
||||
if !req.Repo.ReviewEnabled {
|
||||
return nil
|
||||
}
|
||||
return NewOperatorReviewer(req.Cfg.Models, req.Providers)
|
||||
}
|
||||
|
||||
// NewOperatorReviewer builds the workflow.Reviewer for an operator-triggered
|
||||
// review, resolved from daemon-level models and providers rather than a task
|
||||
// run. It shares newReviewerFor's role->"builder" model fallback but drops
|
||||
// the Repo.ReviewEnabled gate: the operator asked for the review explicitly,
|
||||
// so "not opted in" is not a reason to refuse. It returns nil when no model
|
||||
// resolves (no reviewer/builder model configured) or no provider runtime
|
||||
// builds, so the caller can report a clear "review not configured" result.
|
||||
func NewOperatorReviewer(models map[string]string, providers map[string]agentexec.Provider) workflow.Reviewer {
|
||||
modelRef := ReviewerModel(models)
|
||||
if modelRef == "" {
|
||||
return nil
|
||||
}
|
||||
rt := agentexec.NewRuntime(providers)
|
||||
if rt == nil {
|
||||
return nil
|
||||
}
|
||||
return newSubagentReviewer(rt, modelRef)
|
||||
}
|
||||
|
||||
// ReviewerModel resolves the reviewer role's model with the builder fallback.
|
||||
// It is the single source of truth for the role->"builder" rule, shared by
|
||||
// the workflow stage (via NewOperatorReviewer/newReviewerFor) and the
|
||||
// operator-triggered review's result reporting so the reported model and the
|
||||
// model actually run cannot diverge.
|
||||
func ReviewerModel(models map[string]string) string {
|
||||
if models["reviewer"] != "" {
|
||||
return models["reviewer"]
|
||||
}
|
||||
return models["builder"]
|
||||
}
|
||||
|
||||
// defaultReviewMaxSteps bounds the reviewer's tool loop when neither
|
||||
// ReviewRequest.MaxSteps nor the repo's configured budget set one.
|
||||
const defaultReviewMaxSteps = 10
|
||||
|
||||
// subagentReviewer implements workflow.Reviewer using ai-sdk's
|
||||
// agent.Subagent: a nested, synchronous generation in this same worker
|
||||
// process, with its own model, system prompt, toolset, and step budget.
|
||||
//
|
||||
// Isolation is two-fold. Conversation isolation is structural: Subagent.Run
|
||||
// only ever sends a prompt string, never the implementer's message
|
||||
// history. Workspace isolation is built by the caller (StageReview via
|
||||
// workflow.Trees.Snapshot): reviewerToolSet is rooted at that snapshot
|
||||
// directory and holds only read-only tools -- never the worker's own
|
||||
// toolset, which carries the worktreerpc publication grant a reviewer must
|
||||
// not be able to reach.
|
||||
type subagentReviewer struct {
|
||||
runtime *runtime.Runtime
|
||||
modelRef string
|
||||
}
|
||||
|
||||
// newSubagentReviewer constructs a Reviewer that runs on modelRef, resolved
|
||||
// through rt (the same provider runtime the worker's own agent runner
|
||||
// uses). rt must be non-nil.
|
||||
func newSubagentReviewer(rt *runtime.Runtime, modelRef string) *subagentReviewer {
|
||||
return &subagentReviewer{runtime: rt, modelRef: modelRef}
|
||||
}
|
||||
|
||||
var _ workflow.Reviewer = (*subagentReviewer)(nil)
|
||||
|
||||
// Review runs one adversarial review. It never returns a Go error: every
|
||||
// failure mode (unresolvable model, a Subagent.Run error, a truncated
|
||||
// run) folds into the returned ReviewReport, per
|
||||
// docs/prds/adversarial-self-review.md section 4's fail-closed contract.
|
||||
func (s *subagentReviewer) Review(ctx context.Context, req workflow.ReviewRequest) workflow.ReviewReport {
|
||||
if s.runtime == nil {
|
||||
return workflow.NewNotRunReviewReport("no provider runtime configured for the reviewer")
|
||||
}
|
||||
provider, model, err := s.runtime.ChatProvider(ctx, s.modelRef)
|
||||
if err != nil {
|
||||
return workflow.NewNotRunReviewReport(fmt.Sprintf("resolve reviewer model %q: %v", s.modelRef, err))
|
||||
}
|
||||
|
||||
var findings []workflow.ReviewFinding
|
||||
var checks []workflow.ReviewCheck
|
||||
tools := reviewerToolSet(req.SnapshotDir, &findings, &checks)
|
||||
|
||||
maxSteps := req.MaxSteps
|
||||
if maxSteps <= 0 {
|
||||
maxSteps = defaultReviewMaxSteps
|
||||
}
|
||||
|
||||
sub := agent.Subagent{
|
||||
Provider: provider,
|
||||
Model: model,
|
||||
System: reviewerSystemPrompt,
|
||||
Tools: tools,
|
||||
MaxSteps: maxSteps,
|
||||
}
|
||||
|
||||
summary, runErr := sub.Run(ctx, reviewerPrompt(req))
|
||||
if runErr != nil {
|
||||
// A truncated or errored run still yields whatever the reviewer
|
||||
// captured before it ran out of budget -- those findings are real
|
||||
// and must not be discarded just because the run's own
|
||||
// conclusion never arrived (docs/prds/adversarial-self-review.md
|
||||
// section 1, "findings return through a closure"). Zero captured
|
||||
// findings on error means the reviewer never got anywhere, which
|
||||
// fail-closed treats as not having run at all.
|
||||
if len(findings) == 0 {
|
||||
return workflow.NewNotRunReviewReport(runErr.Error())
|
||||
}
|
||||
return completedReview(findings, checks, "")
|
||||
}
|
||||
return completedReview(findings, checks, summary)
|
||||
}
|
||||
|
||||
// completedReview builds a completed report carrying the captured findings
|
||||
// and the properties the reviewer verified clean.
|
||||
func completedReview(findings []workflow.ReviewFinding, checks []workflow.ReviewCheck, summary string) workflow.ReviewReport {
|
||||
report := workflow.NewCompletedReviewReport(findings, summary)
|
||||
report.Checked = checks
|
||||
return report
|
||||
}
|
||||
|
||||
// reviewerSystemPrompt instructs the reviewer per CLAUDE.md's own
|
||||
// adversarial-review discipline: assume every line is wrong until proven
|
||||
// correct, work the checklist, report only what a fresh reader can verify
|
||||
// from what it was given.
|
||||
const reviewerSystemPrompt = `You are a fresh adversarial code reviewer. You did not write this change and have no access to how or why it was written -- only what is given to you below.
|
||||
|
||||
Assume every line is wrong until you have verified it is correct. Read whole files with your read tool, not just diff hunks, before concluding a call site or interface is satisfied. Use grep/find to check for other call sites a change affects.
|
||||
|
||||
Check for: dead code, unchecked errors, hardcoded values that should be parameters, interface-satisfaction defects, nil-pointer risk, goroutine leaks, races and unsynchronized shared state.
|
||||
|
||||
For every defect you find, call record_finding with: the file and line, a one-sentence defect statement, a concrete failure scenario (specific inputs or state that produce the wrong output or a crash -- not a hypothetical), a verdict, a level, and a category.
|
||||
|
||||
A finding must describe wrong behaviour or a crash. Style, formatting, naming, and preference nits are out of contract -- do not report them; they are exactly the padding that teaches an operator to skim past a findings list.
|
||||
|
||||
As you clear each checklist property, call record_checked with the property and how you verified it (which files or call sites you read). Record every property you actually checked and found correct, even when you also found defects; a review that reports no findings must still record what it checked, because "I checked and it is fine" and "I did not look" must not read the same.
|
||||
|
||||
Verdict is "confirmed" only when you have traced the actual failure -- read the code paths involved and can state exactly how it goes wrong. Verdict is "plausible" for a real worry you have not fully traced. Only a "confirmed" finding at level "error" blocks the pull request; do not inflate a plausible worry to confirmed/error to make it count -- an unjustified block is worse than a missed one, because it teaches the operator to stop trusting your reports.
|
||||
|
||||
Level is "error" for a defect that blocks; "warn" for something worth knowing but not blocking.
|
||||
|
||||
When a confirmed defect's fix is mechanical, and only then, pass suggestion: the exact text that should replace the anchored line, so the author can apply it in one click. It replaces that line exactly, so include the whole line (indentation included) and nothing else, and never guess -- a wrong one-click fix is worse than none. Do not pass a suggestion for a plausible finding, or for a fix you cannot state as a single line.
|
||||
|
||||
When you have finished checking, reply with a short summary of what you checked and what you found (or that you found nothing). If you find nothing, say so plainly -- reporting zero findings on properties you actually checked is a valid and expected outcome, not a failure to find something.`
|
||||
|
||||
// reviewerPrompt renders the reviewer's mission: the diff and issue text.
|
||||
// The snapshot itself is not embedded here -- it reaches the reviewer only
|
||||
// through its read-only toolset, rooted at SnapshotDir, never as prompt
|
||||
// text the reviewer could mistake for something it already checked.
|
||||
func reviewerPrompt(req workflow.ReviewRequest) string {
|
||||
var b strings.Builder
|
||||
b.WriteString("Review the following change. The full file tree at the reviewed commit is available through your tools; ")
|
||||
b.WriteString("start by reading the files the diff touches, not just the diff itself.\n\n")
|
||||
b.WriteString("<issue>\n")
|
||||
b.WriteString(req.IssueText)
|
||||
b.WriteString("\n</issue>\n\n<diff>\n")
|
||||
b.WriteString(req.Diff)
|
||||
b.WriteString("\n</diff>\n")
|
||||
return b.String()
|
||||
}
|
||||
|
||||
// reviewerToolSet builds the reviewer's read-only toolset, rooted at
|
||||
// snapshotDir, plus the structured capture tools that append directly to
|
||||
// findings and checks. This is a distinct toolset built fresh for each
|
||||
// review -- never the worker's own registry, which holds write/edit/shell
|
||||
// tools and the worktreerpc publication grant.
|
||||
func reviewerToolSet(snapshotDir string, findings *[]workflow.ReviewFinding, checks *[]workflow.ReviewCheck) core.ToolSet {
|
||||
reg := toolkit.NewRegistry()
|
||||
readOnly := []toolkit.Tool{
|
||||
toolkit.NewReadTool(snapshotDir, toolkit.NewReadTracker()),
|
||||
toolkit.NewGrepTool(snapshotDir),
|
||||
toolkit.NewFindTool(snapshotDir),
|
||||
}
|
||||
for _, tool := range readOnly {
|
||||
_ = reg.Register(tool) // names are fixed and distinct; Register cannot fail here
|
||||
}
|
||||
set := reg.CoreToolSet(toolkit.NonInteractiveBridge{})
|
||||
set["record_finding"] = recordFindingTool(findings)
|
||||
set["record_checked"] = recordCheckedTool(checks)
|
||||
return set
|
||||
}
|
||||
|
||||
// checkInput is the reviewer-facing shape of workflow.ReviewCheck.
|
||||
type checkInput struct {
|
||||
Property string `json:"property" jsonschema:"description=The property or code path you verified clean, e.g. 'nil-safety of Foo callers'."`
|
||||
Evidence string `json:"evidence" jsonschema:"description=How you verified it, e.g. 'read all 4 callers; each nil-checks before use'."`
|
||||
}
|
||||
|
||||
// recordCheckedTool builds the typed capture tool the reviewer calls once per
|
||||
// property it verified clean. Without it a zero-finding review reads the same
|
||||
// as a review that never looked; with it, "ran and found nothing" arrives
|
||||
// backed by what was actually checked.
|
||||
func recordCheckedTool(checks *[]workflow.ReviewCheck) *core.Tool {
|
||||
return core.NewTypedTool(
|
||||
"record_checked",
|
||||
"Record one property or code path you verified clean. Call this once per property you actually checked and found correct -- a review that reports no findings must still record what it checked.",
|
||||
func(_ context.Context, in checkInput) (string, error) {
|
||||
check := workflow.ReviewCheck{Property: in.Property, Evidence: in.Evidence}
|
||||
if err := check.Validate(); err != nil {
|
||||
return "record_checked rejected: " + err.Error(), nil //nolint:nilerr // rejection feedback lets the model retry with a valid check
|
||||
}
|
||||
*checks = append(*checks, check)
|
||||
return "check recorded", nil
|
||||
},
|
||||
)
|
||||
}
|
||||
|
||||
// findingInput is the reviewer-facing shape of workflow.ReviewFinding: the
|
||||
// model reports plain strings for Verdict/Level/Category, validated and
|
||||
// converted before being appended.
|
||||
type findingInput struct {
|
||||
File string `json:"file" jsonschema:"description=Repo-relative file path where the defect is located."`
|
||||
Line int `json:"line" jsonschema:"description=Line number where the defect is located."`
|
||||
Defect string `json:"defect" jsonschema:"description=One-sentence statement of the defect."`
|
||||
FailureScenario string `json:"failure_scenario" jsonschema:"description=Concrete inputs or state that produce the wrong output or a crash -- not a hypothetical."`
|
||||
Suggestion string `json:"suggestion,omitempty" jsonschema:"description=Optional. The exact replacement for the anchored line, when the fix is mechanical and you are certain of it -- it is offered as a one-click apply, so an approximate or partial line is worse than none. Only a confirmed finding may carry one; omit it for a plausible finding. Single line only."`
|
||||
Verdict string `json:"verdict" jsonschema:"description=Only 'confirmed' may block. Use 'plausible' for a real worry you have not fully traced.,enum=confirmed|plausible"`
|
||||
Level string `json:"level" jsonschema:"description=error blocks when confirmed; warn is advisory only.,enum=error|warn"`
|
||||
Category string `json:"category" jsonschema:"enum=dead-code|unchecked-error|hardcoded-value|interface-satisfaction|nil-risk|goroutine-leak|race|other"`
|
||||
}
|
||||
|
||||
// recordFindingTool builds the typed capture tool the reviewer must call
|
||||
// to report a defect. Rejecting an invalid finding with a plain-text
|
||||
// message (rather than a Go error) lets the model see what was wrong and
|
||||
// retry, the same environmental-enforcement pattern the rest of the
|
||||
// agent-loop tool contract uses.
|
||||
func recordFindingTool(findings *[]workflow.ReviewFinding) *core.Tool {
|
||||
return core.NewTypedTool(
|
||||
"record_finding",
|
||||
"Report one defect found during review. Call this once per defect; do not summarize multiple defects into one call.",
|
||||
func(_ context.Context, in findingInput) (string, error) {
|
||||
finding := workflow.ReviewFinding{
|
||||
File: in.File,
|
||||
Line: in.Line,
|
||||
Defect: in.Defect,
|
||||
FailureScenario: in.FailureScenario,
|
||||
Suggestion: in.Suggestion,
|
||||
Verdict: workflow.ReviewVerdict(in.Verdict),
|
||||
Level: workflow.ReviewLevel(in.Level),
|
||||
Category: workflow.ReviewCategory(in.Category),
|
||||
}
|
||||
if err := finding.Validate(); err != nil {
|
||||
return "record_finding rejected: " + err.Error(), nil //nolint:nilerr // rejection feedback lets the model retry with a valid finding
|
||||
}
|
||||
*findings = append(*findings, finding)
|
||||
return "finding recorded", nil
|
||||
},
|
||||
)
|
||||
}
|
||||
@@ -1,252 +0,0 @@
|
||||
package agentworker
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"slices"
|
||||
"testing"
|
||||
|
||||
"github.com/samcharles93/ai-sdk/agent"
|
||||
"github.com/samcharles93/ai-sdk/chat"
|
||||
|
||||
"github.com/samcharles93/archie-core/internal/domain/workflow"
|
||||
)
|
||||
|
||||
// TestReviewerToolSetIsReadOnlyAndHasNoMutationPath asserts, as a
|
||||
// construction fact rather than a prompt claim, that the reviewer's
|
||||
// toolset carries no tool capable of writing, editing, or running a
|
||||
// shell command -- the class of tool that could reach the worktreerpc
|
||||
// publication grant or otherwise mutate anything. Read/grep/find plus the
|
||||
// findings-capture tool are the entire surface.
|
||||
func TestReviewerToolSetIsReadOnlyAndHasNoMutationPath(t *testing.T) {
|
||||
var findings []workflow.ReviewFinding
|
||||
var checks []workflow.ReviewCheck
|
||||
set := reviewerToolSet(t.TempDir(), &findings, &checks)
|
||||
|
||||
names := make([]string, 0, len(set))
|
||||
for name := range set {
|
||||
names = append(names, name)
|
||||
}
|
||||
slices.Sort(names)
|
||||
|
||||
want := []string{"find", "grep", "read", "record_checked", "record_finding"}
|
||||
slices.Sort(want)
|
||||
if !slices.Equal(names, want) {
|
||||
t.Fatalf("reviewer toolset = %v, want exactly %v (no write/edit/shell)", names, want)
|
||||
}
|
||||
}
|
||||
|
||||
// TestReviewerToolSetIsRootedAtTheGivenSnapshot asserts the read tool
|
||||
// cannot read outside the snapshot directory it was constructed with --
|
||||
// it has no path back to the implementer's actual worktree.
|
||||
func TestReviewerToolSetIsRootedAtTheGivenSnapshot(t *testing.T) {
|
||||
snapshot := t.TempDir()
|
||||
if err := os.WriteFile(filepath.Join(snapshot, "in-snapshot.txt"), []byte("ok\n"), 0o644); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
outside := t.TempDir()
|
||||
if err := os.WriteFile(filepath.Join(outside, "secret.txt"), []byte("nope\n"), 0o644); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
var findings []workflow.ReviewFinding
|
||||
var checks []workflow.ReviewCheck
|
||||
set := reviewerToolSet(snapshot, &findings, &checks)
|
||||
readTool := set["read"]
|
||||
if readTool == nil {
|
||||
t.Fatal("reviewer toolset has no read tool")
|
||||
}
|
||||
|
||||
out, err := readTool.Execute(t.Context(), `{"path":"in-snapshot.txt"}`)
|
||||
if err != nil {
|
||||
t.Fatalf("read in-snapshot file: %v", err)
|
||||
}
|
||||
if !containsSubstr(out, "ok") {
|
||||
t.Errorf("read in-snapshot file = %q, want it to contain the file's content", out)
|
||||
}
|
||||
|
||||
// Tool failures are encoded in-band in the returned content, not as a
|
||||
// Go error (toolkit.Registry.CoreToolSet's own contract) -- an escape
|
||||
// attempt must reject there, not succeed.
|
||||
escapeOut, err := readTool.Execute(t.Context(), `{"path":"`+filepath.Join(outside, "secret.txt")+`"}`)
|
||||
if err != nil {
|
||||
t.Fatalf("read outside-snapshot path: %v", err)
|
||||
}
|
||||
if containsSubstr(escapeOut, "nope") {
|
||||
t.Errorf("read tool returned content from outside the snapshot: %q", escapeOut)
|
||||
}
|
||||
if !containsSubstr(escapeOut, "escapes") {
|
||||
t.Errorf("read tool output = %q, want a path-confinement rejection", escapeOut)
|
||||
}
|
||||
}
|
||||
|
||||
// TestRecordFindingToolAppendsToCapturedSlice asserts the capture-tool
|
||||
// contract: findings are recorded via the closure over the findings
|
||||
// slice, not returned from Subagent.Run's text.
|
||||
func TestRecordFindingToolAppendsToCapturedSlice(t *testing.T) {
|
||||
var findings []workflow.ReviewFinding
|
||||
tool := recordFindingTool(&findings)
|
||||
|
||||
input := `{"file":"main.go","line":10,"defect":"nil deref","failure_scenario":"calling Foo(nil) panics","verdict":"confirmed","level":"error","category":"nil-risk"}`
|
||||
if _, err := tool.Execute(t.Context(), input); err != nil {
|
||||
t.Fatalf("record_finding: %v", err)
|
||||
}
|
||||
if len(findings) != 1 {
|
||||
t.Fatalf("captured %d findings, want 1", len(findings))
|
||||
}
|
||||
if !findings[0].Blocking() {
|
||||
t.Error("captured finding is not blocking, want confirmed+error to block")
|
||||
}
|
||||
}
|
||||
|
||||
// TestRecordFindingToolRejectsInvalidVerdictWithoutError asserts an
|
||||
// invalid finding is rejected in-band (so the model can retry) rather
|
||||
// than surfacing as a hard tool error, and is not captured.
|
||||
func TestRecordFindingToolRejectsInvalidVerdictWithoutError(t *testing.T) {
|
||||
var findings []workflow.ReviewFinding
|
||||
tool := recordFindingTool(&findings)
|
||||
|
||||
input := `{"file":"main.go","line":1,"defect":"x","failure_scenario":"y","verdict":"maybe","level":"error","category":"other"}`
|
||||
out, err := tool.Execute(t.Context(), input)
|
||||
if err != nil {
|
||||
t.Fatalf("record_finding returned a Go error for invalid input: %v", err)
|
||||
}
|
||||
if !containsSubstr(out, "rejected") {
|
||||
t.Errorf("record_finding output = %q, want a rejection message", out)
|
||||
}
|
||||
if len(findings) != 0 {
|
||||
t.Errorf("captured %d findings, want 0 for an invalid finding", len(findings))
|
||||
}
|
||||
}
|
||||
|
||||
// TestRecordCheckedToolAppendsToCapturedSlice asserts the calibration capture
|
||||
// contract: a cleared property is recorded through the closure, so a
|
||||
// zero-finding review can report what it actually checked.
|
||||
func TestRecordCheckedToolAppendsToCapturedSlice(t *testing.T) {
|
||||
var checks []workflow.ReviewCheck
|
||||
tool := recordCheckedTool(&checks)
|
||||
|
||||
input := `{"property":"nil-safety of Foo callers","evidence":"read all 4 callers; each nil-checks"}`
|
||||
if _, err := tool.Execute(t.Context(), input); err != nil {
|
||||
t.Fatalf("record_checked: %v", err)
|
||||
}
|
||||
if len(checks) != 1 || checks[0].Property != "nil-safety of Foo callers" {
|
||||
t.Fatalf("captured checks = %+v, want one check for the recorded property", checks)
|
||||
}
|
||||
}
|
||||
|
||||
// TestRecordCheckedToolRejectsMissingEvidence asserts a check without evidence
|
||||
// is rejected in-band and not captured, so "verified clean" cannot be recorded
|
||||
// as a bare assertion.
|
||||
func TestRecordCheckedToolRejectsMissingEvidence(t *testing.T) {
|
||||
var checks []workflow.ReviewCheck
|
||||
tool := recordCheckedTool(&checks)
|
||||
|
||||
out, err := tool.Execute(t.Context(), `{"property":"looks fine","evidence":""}`)
|
||||
if err != nil {
|
||||
t.Fatalf("record_checked returned a Go error for invalid input: %v", err)
|
||||
}
|
||||
if !containsSubstr(out, "rejected") {
|
||||
t.Errorf("record_checked output = %q, want a rejection message", out)
|
||||
}
|
||||
if len(checks) != 0 {
|
||||
t.Errorf("captured %d checks, want 0 for a check with no evidence", len(checks))
|
||||
}
|
||||
}
|
||||
|
||||
// TestCompletedReviewCarriesChecks asserts the report the reviewer returns
|
||||
// carries the captured checks, not just the findings.
|
||||
func TestCompletedReviewCarriesChecks(t *testing.T) {
|
||||
checks := []workflow.ReviewCheck{{Property: "p", Evidence: "e"}}
|
||||
report := completedReview(nil, checks, "summary")
|
||||
if !report.Ran() {
|
||||
t.Fatal("completedReview report did not run")
|
||||
}
|
||||
if len(report.Checked) != 1 || report.Checked[0].Property != "p" {
|
||||
t.Fatalf("report.Checked = %+v, want the captured check", report.Checked)
|
||||
}
|
||||
}
|
||||
|
||||
// TestReviewSubagentConstructionNeverSendsConversationHistory asserts, at
|
||||
// the exact agent.Subagent construction Review performs, that only a
|
||||
// prompt string reaches the model. This exercises the same Subagent{}
|
||||
// shape subagentReviewer.Review builds (provider, model, system prompt,
|
||||
// reviewerToolSet, MaxSteps) directly against a fake chat.Provider, so it
|
||||
// proves the construction fact without needing a full *runtime.Runtime:
|
||||
// Subagent.Run populates GenerateOptions.Prompt and never Messages, so
|
||||
// there is no code path for the implementer's transcript to travel
|
||||
// through.
|
||||
func TestReviewSubagentConstructionNeverSendsConversationHistory(t *testing.T) {
|
||||
captured := &capturingChatProvider{}
|
||||
var findings []workflow.ReviewFinding
|
||||
var checks []workflow.ReviewCheck
|
||||
req := workflow.ReviewRequest{
|
||||
SnapshotDir: t.TempDir(),
|
||||
Diff: "diff --git a/x.go b/x.go\n",
|
||||
IssueText: "some issue text",
|
||||
}
|
||||
|
||||
sub := agent.Subagent{
|
||||
Provider: captured,
|
||||
Model: "fake-model",
|
||||
System: reviewerSystemPrompt,
|
||||
Tools: reviewerToolSet(req.SnapshotDir, &findings, &checks),
|
||||
MaxSteps: defaultReviewMaxSteps,
|
||||
}
|
||||
prompt := reviewerPrompt(req)
|
||||
if _, err := sub.Run(t.Context(), prompt); err != nil {
|
||||
t.Fatalf("Subagent.Run() error = %v", err)
|
||||
}
|
||||
|
||||
if captured.lastRequest == nil {
|
||||
t.Fatal("chat provider was never called")
|
||||
}
|
||||
// The only non-system message the model ever sees is the one
|
||||
// Subagent.Run derives from the prompt string -- there is no code
|
||||
// path here for a second, implementer-authored message to arrive
|
||||
// alongside it.
|
||||
var userMessages int
|
||||
for _, m := range captured.lastRequest.Messages {
|
||||
if m.Role == chat.RoleUser {
|
||||
userMessages++
|
||||
if m.Content != prompt {
|
||||
t.Errorf("user message content = %q, want the review prompt %q", m.Content, prompt)
|
||||
}
|
||||
}
|
||||
}
|
||||
if userMessages != 1 {
|
||||
t.Errorf("chat.Request carried %d user messages, want exactly 1 (the review prompt, no implementer history)", userMessages)
|
||||
}
|
||||
}
|
||||
|
||||
func containsSubstr(s, substr string) bool {
|
||||
return len(substr) == 0 ||
|
||||
(len(s) >= len(substr) && func() bool {
|
||||
for i := 0; i+len(substr) <= len(s); i++ {
|
||||
if s[i:i+len(substr)] == substr {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}())
|
||||
}
|
||||
|
||||
// capturingChatProvider is a minimal chat.Provider fake that records the
|
||||
// last chat.Request it received and returns a fixed, valid completion.
|
||||
type capturingChatProvider struct {
|
||||
lastRequest *chat.Request
|
||||
}
|
||||
|
||||
func (c *capturingChatProvider) Name() string { return "fake" }
|
||||
|
||||
func (c *capturingChatProvider) Chat(_ context.Context, req chat.Request) (chat.Response, error) {
|
||||
r := req
|
||||
c.lastRequest = &r
|
||||
return chat.Response{Content: "no findings", FinishReason: "stop"}, nil
|
||||
}
|
||||
|
||||
func (c *capturingChatProvider) ChatStream(_ context.Context, req chat.Request) (chat.Stream, error) {
|
||||
return nil, errors.New("not implemented")
|
||||
}
|
||||
@@ -15,6 +15,7 @@ import (
|
||||
"github.com/samcharles93/archie-core/internal/domain/workflow"
|
||||
"github.com/samcharles93/archie-core/internal/domain/workflow/task"
|
||||
"github.com/samcharles93/archie-core/internal/events"
|
||||
"github.com/samcharles93/archie-core/internal/infrastructure/prsource"
|
||||
"github.com/samcharles93/archie-core/internal/installtype"
|
||||
"github.com/samcharles93/archie-core/internal/storage"
|
||||
"github.com/samcharles93/archie-core/internal/taskrun"
|
||||
@@ -217,12 +218,11 @@ func applyToolLimits(agent agentexec.Runner, policy config.ToolPolicy, allow []s
|
||||
runner.AllowTools = allow
|
||||
}
|
||||
|
||||
// stageRunners builds the runner every agent stage uses, and the reviewer.
|
||||
func stageRunners(req taskrun.Request, mcpSet *mcpProviderSet, newRunner runnerFactory, log *slog.Logger) (agentexec.Runner, workflow.Reviewer, error) {
|
||||
// stageRunners builds the runner every agent stage uses.
|
||||
func stageRunners(req taskrun.Request, mcpSet *mcpProviderSet, newRunner runnerFactory, log *slog.Logger) (agentexec.Runner, error) {
|
||||
if req.Harness != nil {
|
||||
// The built-in loop has no route to a model from a Kit container, so
|
||||
// the review stage parks rather than running on it.
|
||||
return agentexec.HarnessStages{Runner: agentexec.NewHarnessRunner(nil), Spec: *req.Harness}, nil, nil
|
||||
// The built-in loop has no route to a model from a Kit container.
|
||||
return agentexec.HarnessStages{Runner: agentexec.NewHarnessRunner(nil), Spec: *req.Harness}, nil
|
||||
}
|
||||
var agent agentexec.Runner
|
||||
if mcpSet != nil && mcpSet.registry != nil {
|
||||
@@ -231,10 +231,10 @@ func stageRunners(req taskrun.Request, mcpSet *mcpProviderSet, newRunner runnerF
|
||||
agent = newRunner(req.Providers, log)
|
||||
}
|
||||
if agent == nil {
|
||||
return nil, nil, fmt.Errorf("no agent runner configured for task %d", req.Task.ID)
|
||||
return nil, fmt.Errorf("no agent runner configured for task %d", req.Task.ID)
|
||||
}
|
||||
applyToolLimits(agent, req.Cfg.ToolPolicy, req.Tools)
|
||||
return agent, newReviewerFor(req), nil
|
||||
return agent, nil
|
||||
}
|
||||
|
||||
// routeTask remains available for routing-only callers. Production execution
|
||||
@@ -286,7 +286,7 @@ func runTask(ctx context.Context, req taskrun.Request, dependencies taskDependen
|
||||
log.Warn("mcp providers had errors, continuing with available tools", "err", mcpErr)
|
||||
}
|
||||
|
||||
agent, reviewer, err := stageRunners(req, mcpSet, newRunner, log)
|
||||
agent, err := stageRunners(req, mcpSet, newRunner, log)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
@@ -310,18 +310,27 @@ func runTask(ctx context.Context, req taskrun.Request, dependencies taskDependen
|
||||
}
|
||||
|
||||
tc := &workflow.TaskContext{
|
||||
Task: req.Task,
|
||||
Repo: req.Repo,
|
||||
Cfg: req.Cfg.ToConfig(),
|
||||
Forge: dependencies.forge,
|
||||
Store: dependencies.store,
|
||||
Calls: dependencies.calls,
|
||||
Trees: trees,
|
||||
Agent: agent,
|
||||
Reviewer: reviewer,
|
||||
Bus: bus,
|
||||
Log: log,
|
||||
Task: req.Task,
|
||||
Repo: req.Repo,
|
||||
Cfg: req.Cfg.ToConfig(),
|
||||
Forge: dependencies.forge,
|
||||
Store: dependencies.store,
|
||||
Calls: dependencies.calls,
|
||||
Trees: trees,
|
||||
Agent: agent,
|
||||
Bus: bus,
|
||||
Log: log,
|
||||
}
|
||||
// pr-review reads an external pull request's data from workDir, which the
|
||||
// daemon already populated before this task started (daemon's
|
||||
// prefetchPRReview, writing under workDir/.archie-pr-review -- the same
|
||||
// host directory Docker bind-mounts at storage.WorktreeMountDir) -- never
|
||||
// over the network, and never with a forge credential in this process.
|
||||
// See docs/development/agent.md, "Giving the agent something new".
|
||||
if req.Task.Workflow == "pr-review" {
|
||||
tc.PRSource = prsource.NewFromMount(filepath.Join(workDir, prsource.PrefetchDirName))
|
||||
}
|
||||
|
||||
if req.Repo.PersistentStorage {
|
||||
memory, readErr := readProjectMemory(storage.MemoryPath)
|
||||
if readErr != nil {
|
||||
|
||||
@@ -60,7 +60,7 @@ func TestExecuteTaskRequestStartsWorkflowCallCallee(t *testing.T) {
|
||||
const callerYAML = "id: caller\nrepository: none\ninputs:\n src_ip: {type: string}\nsteps:\n - type: workflow.call\n settings:\n workflow: callee\n inputs: {src_ip: \"inputs.src_ip\"}\n wait: false\n - type: agent.run\n settings:\n mission: report\n"
|
||||
// A real caller row, claimed and running, so the store's enqueue reads
|
||||
// the same row shape production does.
|
||||
task, err := st.EnqueueChatTask(ctx, "", "", "workflow.call integration", "", "caller", "")
|
||||
task, err := st.EnqueueChatTask(ctx, "", "", "workflow.call integration", "", "caller", "", nil)
|
||||
if err != nil {
|
||||
t.Fatalf("seed caller task: %v", err)
|
||||
}
|
||||
|
||||
@@ -238,15 +238,11 @@ type boot struct {
|
||||
// unrelated call.
|
||||
embeddings domainembedding.Client
|
||||
|
||||
toolReg *tools.Registry
|
||||
chatModels *chatModelManager
|
||||
personas *gateway.PersonaRegistry
|
||||
chatTasks gateway.TaskCreator
|
||||
chatController *gateway.StoreTaskController
|
||||
// chatPRReviewer is the operator-triggered PR review capability, built
|
||||
// once and shared by every channel so a review in flight on one channel
|
||||
// is deduplicated against the same request on another (single-flight).
|
||||
chatPRReviewer gateway.ChatPRReviewer
|
||||
toolReg *tools.Registry
|
||||
chatModels *chatModelManager
|
||||
personas *gateway.PersonaRegistry
|
||||
chatTasks gateway.TaskCreator
|
||||
chatController *gateway.StoreTaskController
|
||||
defaultChatIdentity string
|
||||
updateService *releaseupdate.Service
|
||||
|
||||
@@ -930,10 +926,10 @@ func (b *boot) loadPlugins() error {
|
||||
|
||||
// buildWorktreeManager composes the process-wide worktree manager from the
|
||||
// resolved primary forge credential and the loaded config. Both process roots
|
||||
// own one: the daemon works tasks in it, and the Gateway materialises a pull
|
||||
// request head in it when an operator asks for a review (boot.prReviewer
|
||||
// refuses to build a reviewer without one, so a Gateway that skips this step
|
||||
// advertises no review_pr). It reads only, so it has nothing to report.
|
||||
// own one: the daemon works tasks in it, and the Gateway used it to
|
||||
// materialise a pull request head for the now-removed synchronous
|
||||
// operator-review path (archie-core-afbk.7). It reads only, so it has
|
||||
// nothing to report.
|
||||
//
|
||||
// Identity managers are not built here: buildTreesAndIdentities builds one per
|
||||
// identity, each in its own WorkDir, because the Gateway must not own daemon
|
||||
|
||||
@@ -41,7 +41,6 @@ type chatSetup struct {
|
||||
ChatTaskLister gateway.ChatTaskLister
|
||||
ChatTaskLogs gateway.ChatTaskLogReader
|
||||
ChatTaskActor gateway.ChatTaskActor
|
||||
ChatPRReviewer gateway.ChatPRReviewer
|
||||
DefaultChatIdentity string
|
||||
// Bus carries primary-input events (archie-core-035): a completed
|
||||
// chat turn is published here so input-driven curators can wake. Nil
|
||||
@@ -182,7 +181,6 @@ func newChatTurnRunner(
|
||||
TaskLogs: s.ChatTaskLogs,
|
||||
TaskActor: s.ChatTaskActor,
|
||||
TaskIdentity: s.DefaultChatIdentity,
|
||||
PRReviewer: s.ChatPRReviewer,
|
||||
Bus: s.Bus,
|
||||
BotUser: cfg.BotUser,
|
||||
Channel: channel,
|
||||
|
||||
@@ -32,29 +32,3 @@ func TestStartGatewayRuntimeSetsUpMemoryBeforeGatewayChat(t *testing.T) {
|
||||
t.Fatal("startGatewayRuntime wires setupGatewayChat before setupMemoryEngine; the turn runner would capture a nil memory engine")
|
||||
}
|
||||
}
|
||||
|
||||
// TestStartGatewayRuntimeBuildsWorktreeManagerBeforeGatewayChat pins the
|
||||
// Gateway's own worktree-manager ordering. The Gateway owns PR review, so
|
||||
// startGatewayRuntime must compose the process-wide worktree manager before
|
||||
// setupGatewayChat resolves boot.prReviewer: prReviewer returns nil without a
|
||||
// manager, and a nil reviewer omits review_pr from every channel's tool set
|
||||
// (internal/gateway/review_tools.go). The Gateway deliberately does not call
|
||||
// buildTreesAndIdentities -- that also registers daemon identity runners it
|
||||
// must not own -- so nothing at runtime connects the manager to this
|
||||
// composition phase except the call this asserts on.
|
||||
func TestStartGatewayRuntimeBuildsWorktreeManagerBeforeGatewayChat(t *testing.T) {
|
||||
fileset := token.NewFileSet()
|
||||
file, err := parser.ParseFile(fileset, "gateway.go", nil, 0)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
run := methodBody(t, file, "startGatewayRuntime")
|
||||
trees := methodCallPosition(run, "buildWorktreeManager")
|
||||
chat := methodCallPosition(run, "setupGatewayChat")
|
||||
if trees == token.NoPos || chat == token.NoPos {
|
||||
t.Fatalf("startGatewayRuntime call positions: buildWorktreeManager=%v setupGatewayChat=%v, want both present", trees, chat)
|
||||
}
|
||||
if chat < trees {
|
||||
t.Fatal("startGatewayRuntime wires setupGatewayChat before buildWorktreeManager; the Gateway would resolve a nil reviewer and never advertise review_pr")
|
||||
}
|
||||
}
|
||||
@@ -156,13 +156,14 @@ func (b *boot) startGatewayRuntime(ctx context.Context, actor gateway.ChatTaskAc
|
||||
b.addCleanup(b.bus.Close)
|
||||
b.capabilityHost = plugin.NewHost()
|
||||
b.startRateLimiter(ctx, b.cfg.Chat.RateLimit)
|
||||
// The Gateway owns PR review (docs/prds/operator-pr-review.md §4), so it
|
||||
// composes the process-wide worktree manager before setupGatewayChat
|
||||
// resolves boot.prReviewer: prReviewer returns nil without a manager, and
|
||||
// a nil reviewer omits review_pr from every channel's tool set. The
|
||||
// manager is the only part of the daemon's tree composition the Gateway
|
||||
// takes -- it must not own identity runners, so it does not call
|
||||
// buildTreesAndIdentities.
|
||||
// The Gateway composes the process-wide worktree manager before
|
||||
// setupGatewayChat; nothing in this process reads it today (the old
|
||||
// synchronous operator-review path that did was removed with the
|
||||
// adversarial self-review system, archie-core-afbk.7 -- see that bead's
|
||||
// continuation for the operator trigger's replacement). Kept for the
|
||||
// manager's other consumers; it is the only part of the daemon's tree
|
||||
// composition the Gateway takes -- it must not own identity runners, so
|
||||
// it does not call buildTreesAndIdentities.
|
||||
b.buildWorktreeManager()
|
||||
// setupMemoryEngine must run before setupGatewayChat: setupGatewayChat
|
||||
// constructs the turn runner, which captures b.memEngines at
|
||||
|
||||
@@ -126,9 +126,9 @@ func (b *boot) setupGatewayChat(ctx context.Context, actor gateway.ChatTaskActor
|
||||
setup := chatSetup{
|
||||
Cfg: config.NewHolder(cfg), LLM: b.chatLLM, ChatModels: b.chatModels, ToolReg: b.toolReg,
|
||||
Personas: b.personas, ChatTasks: b.chatTasks,
|
||||
ChatTaskLister: chatTaskListerAdapter{tasks: b.stateStore.Tasks},
|
||||
ChatTaskLogs: chatTaskLogReaderAdapter{tasks: b.stateStore.TaskByID, taskLogs: b.taskLogs},
|
||||
ChatTaskActor: actor, ChatPRReviewer: b.prReviewer(),
|
||||
ChatTaskLister: chatTaskListerAdapter{tasks: b.stateStore.Tasks},
|
||||
ChatTaskLogs: chatTaskLogReaderAdapter{tasks: b.stateStore.TaskByID, taskLogs: b.taskLogs},
|
||||
ChatTaskActor: actor,
|
||||
DefaultChatIdentity: b.defaultChatIdentity,
|
||||
Bus: b.bus, Log: b.log,
|
||||
MemoryEngine: b.memoryStore(),
|
||||
|
||||
@@ -539,14 +539,16 @@ type chatTaskWriterAdapter struct {
|
||||
enqueue func(
|
||||
ctx context.Context,
|
||||
owner, repo, title, body, workflow, identity string,
|
||||
inputs map[string]any,
|
||||
) (*workflow.Task, error)
|
||||
}
|
||||
|
||||
func (a chatTaskWriterAdapter) EnqueueChatTask(
|
||||
ctx context.Context,
|
||||
owner, repo, title, body, workflow, identity string,
|
||||
inputs map[string]any,
|
||||
) (int64, error) {
|
||||
task, err := a.enqueue(ctx, owner, repo, title, body, workflow, identity)
|
||||
task, err := a.enqueue(ctx, owner, repo, title, body, workflow, identity, inputs)
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
|
||||
@@ -567,7 +567,7 @@ func TestManualRequeueTaskRejectsOtherStatuses(t *testing.T) {
|
||||
func TestChatTaskListerAdapterReadsSQLiteIdentity(t *testing.T) {
|
||||
st := pgstore.Open(t)
|
||||
t.Cleanup(func() { _ = st.Close() })
|
||||
want, err := st.EnqueueChatTask(t.Context(), "acme", "widget", "identity task", "", "tdd", "reviewer")
|
||||
want, err := st.EnqueueChatTask(t.Context(), "acme", "widget", "identity task", "", "tdd", "reviewer", nil)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
@@ -610,13 +610,14 @@ func TestChatTaskWriterAdapter(t *testing.T) {
|
||||
enqueue: func(
|
||||
context.Context,
|
||||
string, string, string, string, string, string,
|
||||
map[string]any,
|
||||
) (*workflow.Task, error) {
|
||||
return tt.result, tt.err
|
||||
},
|
||||
}
|
||||
id, err := adapter.EnqueueChatTask(
|
||||
context.Background(),
|
||||
"owner", "repo", "title", "body", "workflow", "identity",
|
||||
"owner", "repo", "title", "body", "workflow", "identity", nil,
|
||||
)
|
||||
if (err != nil) != tt.wantErr {
|
||||
t.Fatalf("EnqueueChatTask() error = %v, wantErr %t", err, tt.wantErr)
|
||||
|
||||
@@ -1,364 +0,0 @@
|
||||
package archied
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"log/slog"
|
||||
"os"
|
||||
"strconv"
|
||||
"strings"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"github.com/samcharles93/archie-core/internal/agentexec"
|
||||
"github.com/samcharles93/archie-core/internal/app/agentworker"
|
||||
"github.com/samcharles93/archie-core/internal/config"
|
||||
"github.com/samcharles93/archie-core/internal/domain/workflow"
|
||||
"github.com/samcharles93/archie-core/internal/forge"
|
||||
"github.com/samcharles93/archie-core/internal/gateway"
|
||||
"github.com/samcharles93/archie-core/internal/infrastructure/artifactsync"
|
||||
"github.com/samcharles93/archie-core/internal/worktree"
|
||||
)
|
||||
|
||||
// prReviewer implements gateway.ChatPRReviewer for the daemon. It fetches
|
||||
// an existing PR's metadata, materialises its head in a fresh clone, and runs
|
||||
// the same adversarial reviewer the workflow stage uses, all synchronously in
|
||||
// the requesting turn. Authorization is identity-scoped (the task_spawn
|
||||
// repository allow-list) and one review per PR runs at a time.
|
||||
type prReviewer struct {
|
||||
forge forge.PullRequestReader // nil = forge cannot read PRs
|
||||
trees *worktree.Manager
|
||||
models map[string]string
|
||||
providers map[string]agentexec.Provider
|
||||
maxSteps int
|
||||
// artifacts publishes the completed review to the collaborative editor.
|
||||
// nil = publishing is disabled (no [artifacts] base_url configured).
|
||||
artifacts artifactsync.Sender
|
||||
// currentMaxSteps lets the composition root apply live control-plane
|
||||
// versions to reviews that have not started yet. Tests and direct
|
||||
// constructions retain maxSteps as the fallback.
|
||||
currentMaxSteps func() int
|
||||
// allowed maps a chat identity to the "owner/name" repos it may review.
|
||||
// A nil identity is an authenticated dashboard operator, who may review
|
||||
// any configured repository.
|
||||
allowed map[string]map[string]bool
|
||||
log *slog.Logger
|
||||
|
||||
mu sync.Mutex
|
||||
inflight map[string]struct{}
|
||||
}
|
||||
|
||||
var _ gateway.ChatPRReviewer = (*prReviewer)(nil)
|
||||
|
||||
// ReviewPR fetches the PR, runs the isolated reviewer against its head, and
|
||||
// returns the structured findings. It never half-reviews: a missing forge
|
||||
// capability, an unresolvable head/base, or an absent reviewer model returns
|
||||
// an error rather than a fabricated result.
|
||||
func (r *prReviewer) ReviewPR(ctx context.Context, identity *string, repo string, number int) (gateway.PRReviewResult, error) {
|
||||
owner, name, err := splitOwnerRepo(repo)
|
||||
if err != nil {
|
||||
return gateway.PRReviewResult{}, err
|
||||
}
|
||||
if err := r.authorize(identity, owner, name); err != nil {
|
||||
return gateway.PRReviewResult{}, err
|
||||
}
|
||||
|
||||
key := owner + "/" + name + "#" + strconv.Itoa(number)
|
||||
if !r.begin(key) {
|
||||
return gateway.PRReviewResult{}, fmt.Errorf("review already in progress for %s", key)
|
||||
}
|
||||
defer r.end(key)
|
||||
|
||||
if r.forge == nil {
|
||||
return gateway.PRReviewResult{}, fmt.Errorf("forge does not support pull request review")
|
||||
}
|
||||
pr, err := r.forge.GetPullRequest(ctx, owner, name, number)
|
||||
if err != nil {
|
||||
return gateway.PRReviewResult{}, err
|
||||
}
|
||||
|
||||
model := agentworker.ReviewerModel(r.models)
|
||||
if model == "" {
|
||||
return gateway.PRReviewResult{}, fmt.Errorf("reviewer model is not configured (set models.reviewer, or models.builder as a fallback)")
|
||||
}
|
||||
reviewer := agentworker.NewOperatorReviewer(r.models, r.providers)
|
||||
if reviewer == nil {
|
||||
return gateway.PRReviewResult{}, fmt.Errorf("no provider runtime configured for the reviewer model %q", model)
|
||||
}
|
||||
if r.trees == nil {
|
||||
return gateway.PRReviewResult{}, fmt.Errorf("worktree manager is not configured")
|
||||
}
|
||||
|
||||
dir, cleanup, err := r.trees.CheckoutPR(ctx, owner, name, pr.HeadRef, pr.BaseRef)
|
||||
if err != nil {
|
||||
return gateway.PRReviewResult{}, err
|
||||
}
|
||||
defer cleanup()
|
||||
|
||||
diff, err := r.trees.Diff(ctx, dir, pr.BaseRef)
|
||||
if err != nil {
|
||||
return gateway.PRReviewResult{}, fmt.Errorf("compute PR diff: %w", err)
|
||||
}
|
||||
|
||||
snapshotDir, err := os.MkdirTemp("", "pr-review-snapshot-*")
|
||||
if err != nil {
|
||||
return gateway.PRReviewResult{}, fmt.Errorf("create review snapshot directory: %w", err)
|
||||
}
|
||||
defer os.RemoveAll(snapshotDir)
|
||||
if err := r.trees.Snapshot(ctx, dir, snapshotDir); err != nil {
|
||||
return gateway.PRReviewResult{}, fmt.Errorf("snapshot PR head: %w", err)
|
||||
}
|
||||
|
||||
maxSteps := r.maxSteps
|
||||
if r.currentMaxSteps != nil {
|
||||
maxSteps = r.currentMaxSteps()
|
||||
}
|
||||
report := reviewer.Review(ctx, workflow.ReviewRequest{
|
||||
SnapshotDir: snapshotDir,
|
||||
Diff: diff,
|
||||
IssueText: prIssueText(pr),
|
||||
MaxSteps: maxSteps,
|
||||
})
|
||||
r.logReview(key, model, report)
|
||||
|
||||
result := mapReviewResult(repo, number, model, report)
|
||||
return r.attachArtifact(ctx, result, identity, owner, name, number, report), nil
|
||||
}
|
||||
|
||||
// authorize enforces the identity-scoped repository allow-list. A nil
|
||||
// identity (dashboard operator) may review any configured repository.
|
||||
func (r *prReviewer) authorize(identity *string, owner, repo string) error {
|
||||
full := owner + "/" + repo
|
||||
if identity == nil {
|
||||
for _, set := range r.allowed {
|
||||
if set[full] {
|
||||
return nil
|
||||
}
|
||||
}
|
||||
return fmt.Errorf("repository %q is not configured", full)
|
||||
}
|
||||
set, ok := r.allowed[*identity]
|
||||
if !ok {
|
||||
return fmt.Errorf("identity %q is not configured for PR review", *identity)
|
||||
}
|
||||
if !set[full] {
|
||||
return fmt.Errorf("repository %q is not configured for this identity", full)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (r *prReviewer) begin(key string) bool {
|
||||
r.mu.Lock()
|
||||
defer r.mu.Unlock()
|
||||
if r.inflight == nil {
|
||||
r.inflight = make(map[string]struct{})
|
||||
}
|
||||
if _, ok := r.inflight[key]; ok {
|
||||
return false
|
||||
}
|
||||
r.inflight[key] = struct{}{}
|
||||
return true
|
||||
}
|
||||
|
||||
func (r *prReviewer) end(key string) {
|
||||
r.mu.Lock()
|
||||
defer r.mu.Unlock()
|
||||
delete(r.inflight, key)
|
||||
}
|
||||
|
||||
func (r *prReviewer) logReview(key, model string, report workflow.ReviewReport) {
|
||||
if r.log == nil {
|
||||
return
|
||||
}
|
||||
blocking := 0
|
||||
for _, f := range report.Findings {
|
||||
if f.Blocking() {
|
||||
blocking++
|
||||
}
|
||||
}
|
||||
r.log.Info("operator PR review complete",
|
||||
"pr", key, "model", model, "status", string(report.Status),
|
||||
"findings", len(report.Findings), "blocking", blocking)
|
||||
}
|
||||
|
||||
// attachArtifact publishes the report when the sender is configured and
|
||||
// records the editor deep link on the result. Publishing never fails the
|
||||
// review: a failure degrades to an empty URL.
|
||||
func (r *prReviewer) attachArtifact(ctx context.Context, result gateway.PRReviewResult, identity *string, owner, name string, number int, report workflow.ReviewReport) gateway.PRReviewResult {
|
||||
result.ArtifactURL = r.publishArtifact(ctx, reviewActor(identity), owner, name, number, report)
|
||||
return result
|
||||
}
|
||||
|
||||
// reviewActor resolves the attribution: the requesting chat identity when one
|
||||
// exists, else the daemon's operator-reviewer actor.
|
||||
func reviewActor(identity *string) string {
|
||||
if identity != nil && *identity != "" {
|
||||
return *identity
|
||||
}
|
||||
return "system:pr-reviewer"
|
||||
}
|
||||
|
||||
// publishArtifact renders the report and hands it to the collaborative
|
||||
// editor when artifact publishing is configured. It returns the editor deep
|
||||
// link, or "" when publishing is disabled or failed: a publish failure is a
|
||||
// logged degradation, never an error that fails the producing review.
|
||||
func (r *prReviewer) publishArtifact(ctx context.Context, actor, owner, name string, number int, report workflow.ReviewReport) string {
|
||||
if r.artifacts == nil {
|
||||
return ""
|
||||
}
|
||||
title := fmt.Sprintf("Review of %s/%s#%d", owner, name, number)
|
||||
artifact := artifactsync.Artifact{
|
||||
Title: title,
|
||||
Source: report.RenderMarkdown(title),
|
||||
Meta: artifactsync.Meta{
|
||||
Format: "markdown",
|
||||
Category: "pr-review",
|
||||
Visibility: "account-only",
|
||||
Egress: "none",
|
||||
Actor: actor,
|
||||
Origin: "pr-review",
|
||||
RequestID: "artifact:pr-review:" + owner + "/" + name + "#" + strconv.Itoa(number),
|
||||
Version: "r1",
|
||||
SentAt: time.Now().UTC(),
|
||||
Task: &artifactsync.Task{Owner: owner, Repo: name, Number: number},
|
||||
},
|
||||
}
|
||||
url, err := r.artifacts.Publish(ctx, artifact)
|
||||
if err != nil {
|
||||
if r.log != nil {
|
||||
r.log.Warn("artifact publish failed; the review result is unaffected",
|
||||
"artifact", artifact.RequestID(), "err", err)
|
||||
}
|
||||
return ""
|
||||
}
|
||||
if r.log != nil {
|
||||
r.log.Info("artifact published", "artifact", artifact.RequestID(), "url", url)
|
||||
}
|
||||
return url
|
||||
}
|
||||
|
||||
// splitOwnerRepo splits "owner/name" with the same validation the gateway's
|
||||
// task_spawn allow-list uses.
|
||||
func splitOwnerRepo(s string) (owner, repo string, err error) {
|
||||
owner, repo, ok := strings.Cut(s, "/")
|
||||
if !ok || owner == "" || repo == "" {
|
||||
return "", "", fmt.Errorf("repository %q must be owner/name", s)
|
||||
}
|
||||
return owner, repo, nil
|
||||
}
|
||||
|
||||
// prIssueText renders the PR title/body as the review's issue text.
|
||||
func prIssueText(pr forge.PullRequest) string {
|
||||
if pr.Body == "" {
|
||||
return pr.Title
|
||||
}
|
||||
return pr.Title + "\n\n" + pr.Body
|
||||
}
|
||||
|
||||
// mapReviewResult adapts a workflow.ReviewReport into the gateway's
|
||||
// channel-neutral result shape.
|
||||
func mapReviewResult(repo string, number int, model string, report workflow.ReviewReport) gateway.PRReviewResult {
|
||||
out := gateway.PRReviewResult{
|
||||
Repo: repo,
|
||||
Number: number,
|
||||
Model: model,
|
||||
Status: string(report.Status),
|
||||
Summary: report.Summary,
|
||||
Reason: report.SkipReason,
|
||||
Findings: make([]gateway.PRReviewFinding, 0, len(report.Findings)),
|
||||
}
|
||||
for _, f := range report.Findings {
|
||||
out.Findings = append(out.Findings, gateway.PRReviewFinding{
|
||||
File: f.File,
|
||||
Line: f.Line,
|
||||
Defect: f.Defect,
|
||||
FailureScenario: f.FailureScenario,
|
||||
Verdict: string(f.Verdict),
|
||||
Level: string(f.Level),
|
||||
Category: string(f.Category),
|
||||
Blocking: f.Blocking(),
|
||||
})
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// buildReviewAllowlist derives the identity-scoped review allow-list from the
|
||||
// same profiles task_spawn uses, so a chat identity and the review command
|
||||
// agree on which repositories the identity may touch.
|
||||
func buildReviewAllowlist(cfg config.Config) map[string]map[string]bool {
|
||||
profiles, _ := chatTaskProfiles(cfg)
|
||||
out := make(map[string]map[string]bool, len(profiles))
|
||||
for _, p := range profiles {
|
||||
set := make(map[string]bool, len(p.Repos))
|
||||
for _, repo := range p.Repos {
|
||||
set[repo] = true
|
||||
}
|
||||
out[p.Identity] = set
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// prReviewer assembles the operator-triggered review capability, or nil when
|
||||
// the forge cannot read PRs (the noop forge, or a forge with no PR read
|
||||
// support). A review tool that can never run must not be advertised, matching
|
||||
// the TaskTools nil-backend rule.
|
||||
//
|
||||
// The instance is built once and memoized on the boot struct so every channel
|
||||
// shares one in-flight set: a review of the same PR requested concurrently
|
||||
// from the Web UI and Telegram deduplicates instead of running twice.
|
||||
func (b *boot) prReviewer() gateway.ChatPRReviewer {
|
||||
if b.chatPRReviewer != nil {
|
||||
return b.chatPRReviewer
|
||||
}
|
||||
if b.trees == nil {
|
||||
return nil
|
||||
}
|
||||
forgeReader, _ := b.forgeClient.(forge.PullRequestReader)
|
||||
if forgeReader == nil {
|
||||
return nil
|
||||
}
|
||||
maxSteps := b.cfg.Budgets.MaxSteps
|
||||
if b.cfgHolder != nil {
|
||||
maxSteps = b.cfgHolder.Get().Budgets.MaxSteps
|
||||
}
|
||||
review := &prReviewer{
|
||||
forge: forgeReader,
|
||||
trees: b.trees,
|
||||
models: b.cfg.Models,
|
||||
providers: executionProviders(b.cfg),
|
||||
maxSteps: maxSteps,
|
||||
artifacts: b.artifactSender(),
|
||||
allowed: buildReviewAllowlist(b.cfg),
|
||||
log: b.log,
|
||||
}
|
||||
if b.cfgHolder != nil {
|
||||
review.currentMaxSteps = func() int { return b.cfgHolder.Get().Budgets.MaxSteps }
|
||||
}
|
||||
b.chatPRReviewer = review
|
||||
return b.chatPRReviewer
|
||||
}
|
||||
|
||||
// artifactSender builds the collaborative-editor sender when the [artifacts]
|
||||
// section configures one. An empty base_url disables publishing entirely; a
|
||||
// base_url with an unresolvable token is a missing credential, which disables
|
||||
// the capability with a loud warning rather than stopping the daemon
|
||||
// (docs/architecture/configuration.md, "A missing credential disables a
|
||||
// capability").
|
||||
func (b *boot) artifactSender() artifactsync.Sender {
|
||||
cfg := b.cfg.Artifacts
|
||||
if cfg.BaseURL == "" {
|
||||
return nil
|
||||
}
|
||||
token, err := b.secrets.Resolve(cfg.Token)
|
||||
if err != nil {
|
||||
b.log.Warn("artifact publishing disabled: the service credential could not be resolved",
|
||||
"base_url", cfg.BaseURL, "token_engine", cfg.Token.Engine, "err", err)
|
||||
return nil
|
||||
}
|
||||
if token == "" {
|
||||
b.log.Warn("artifact publishing disabled: the service token resolved empty",
|
||||
"base_url", cfg.BaseURL, "token_engine", cfg.Token.Engine)
|
||||
return nil
|
||||
}
|
||||
return artifactsync.NewClient(cfg.BaseURL, token)
|
||||
}
|
||||
@@ -1,317 +0,0 @@
|
||||
package archied
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"log/slog"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"github.com/samcharles93/archie-core/internal/config"
|
||||
"github.com/samcharles93/archie-core/internal/domain/workflow"
|
||||
"github.com/samcharles93/archie-core/internal/forge"
|
||||
"github.com/samcharles93/archie-core/internal/gateway"
|
||||
"github.com/samcharles93/archie-core/internal/infrastructure/artifactsync"
|
||||
"github.com/samcharles93/archie-core/internal/tools"
|
||||
"github.com/samcharles93/archie-core/internal/worktree"
|
||||
)
|
||||
|
||||
func TestSplitOwnerRepo(t *testing.T) {
|
||||
tests := []struct {
|
||||
in string
|
||||
owner string
|
||||
repo string
|
||||
wantErr bool
|
||||
}{
|
||||
{"acme/widget", "acme", "widget", false},
|
||||
{"acme/widget/service", "acme", "widget/service", false},
|
||||
{"acme", "", "", true},
|
||||
{"/widget", "", "", true},
|
||||
{"acme/", "", "", true},
|
||||
{"", "", "", true},
|
||||
}
|
||||
for _, tt := range tests {
|
||||
owner, repo, err := splitOwnerRepo(tt.in)
|
||||
if tt.wantErr {
|
||||
if err == nil {
|
||||
t.Errorf("splitOwnerRepo(%q) error = nil, want error", tt.in)
|
||||
}
|
||||
continue
|
||||
}
|
||||
if err != nil {
|
||||
t.Errorf("splitOwnerRepo(%q) error = %v", tt.in, err)
|
||||
continue
|
||||
}
|
||||
if owner != tt.owner || repo != tt.repo {
|
||||
t.Errorf("splitOwnerRepo(%q) = (%q, %q), want (%q, %q)", tt.in, owner, repo, tt.owner, tt.repo)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestPRIssueText(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
pr forge.PullRequest
|
||||
want string
|
||||
}{
|
||||
{"title only", forge.PullRequest{Title: "Fix login"}, "Fix login"},
|
||||
{"title and body", forge.PullRequest{Title: "Fix login", Body: "The form 500s."}, "Fix login\n\nThe form 500s."},
|
||||
}
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
if got := prIssueText(tt.pr); got != tt.want {
|
||||
t.Errorf("prIssueText = %q, want %q", got, tt.want)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestMapReviewResult(t *testing.T) {
|
||||
report := workflow.ReviewReport{
|
||||
Status: workflow.ReviewStatusCompleted,
|
||||
Summary: "checked and cleared",
|
||||
Findings: []workflow.ReviewFinding{
|
||||
{File: "a.go", Line: 3, Defect: "nil deref", FailureScenario: "x", Verdict: workflow.ReviewVerdictConfirmed, Level: workflow.ReviewLevelError, Category: workflow.ReviewCategoryNilRisk},
|
||||
{File: "b.go", Line: 0, Defect: "style", FailureScenario: "y", Verdict: workflow.ReviewVerdictPlausible, Level: workflow.ReviewLevelWarn, Category: workflow.ReviewCategoryOther},
|
||||
},
|
||||
}
|
||||
got := mapReviewResult("acme/widget", 7, "openai/gpt-5", report)
|
||||
|
||||
if got.Repo != "acme/widget" || got.Number != 7 || got.Model != "openai/gpt-5" || got.Status != "completed" || got.Summary != "checked and cleared" {
|
||||
t.Fatalf("result meta = %+v", got)
|
||||
}
|
||||
if len(got.Findings) != 2 {
|
||||
t.Fatalf("findings = %d, want 2", len(got.Findings))
|
||||
}
|
||||
if !got.Findings[0].Blocking {
|
||||
t.Error("confirmed error finding should be blocking")
|
||||
}
|
||||
if got.Findings[1].Blocking {
|
||||
t.Error("plausible warn finding must not be blocking")
|
||||
}
|
||||
if got.Findings[0].Line != 3 {
|
||||
t.Errorf("line = %d, want 3", got.Findings[0].Line)
|
||||
}
|
||||
}
|
||||
|
||||
func TestMapReviewResultNotRunPreservesReason(t *testing.T) {
|
||||
report := workflow.NewNotRunReviewReport("no reviewer model configured")
|
||||
got := mapReviewResult("acme/widget", 7, "", report)
|
||||
if got.Status != "not_run" || got.Reason != "no reviewer model configured" {
|
||||
t.Fatalf("not_run result = %+v, want status not_run and reason preserved", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestPrReviewerAuthorize(t *testing.T) {
|
||||
r := &prReviewer{allowed: map[string]map[string]bool{
|
||||
"": {"acme/widget": true, "acme/core": true},
|
||||
"archie": {"acme/widget": true},
|
||||
}}
|
||||
archie := "archie"
|
||||
other := "other"
|
||||
|
||||
tests := []struct {
|
||||
name string
|
||||
identity *string
|
||||
owner string
|
||||
repo string
|
||||
wantErr bool
|
||||
}{
|
||||
{"identity own repo", &archie, "acme", "widget", false},
|
||||
{"identity foreign repo", &archie, "acme", "core", true},
|
||||
{"unknown identity", &other, "acme", "widget", true},
|
||||
{"operator any configured", nil, "acme", "core", false},
|
||||
{"operator unconfigured repo", nil, "acme", "other", true},
|
||||
}
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
err := r.authorize(tt.identity, tt.owner, tt.repo)
|
||||
if tt.wantErr && err == nil {
|
||||
t.Errorf("authorize(%v, %s/%s) error = nil, want error", tt.identity, tt.owner, tt.repo)
|
||||
}
|
||||
if !tt.wantErr && err != nil {
|
||||
t.Errorf("authorize(%v, %s/%s) error = %v, want nil", tt.identity, tt.owner, tt.repo, err)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestPrReviewerSingleflight(t *testing.T) {
|
||||
r := &prReviewer{}
|
||||
key := "acme/widget#7"
|
||||
if !r.begin(key) {
|
||||
t.Fatal("first begin = false, want true")
|
||||
}
|
||||
if r.begin(key) {
|
||||
t.Fatal("concurrent begin = true, want false (deterministic single-flight)")
|
||||
}
|
||||
r.end(key)
|
||||
if !r.begin(key) {
|
||||
t.Fatal("begin after end = false, want true")
|
||||
}
|
||||
}
|
||||
|
||||
func TestBuildReviewAllowlist(t *testing.T) {
|
||||
cfg := config.Config{
|
||||
Identities: []config.IdentityConfig{
|
||||
{Name: "archie", Repos: []config.Repo{{Owner: "acme", Name: "widget"}}},
|
||||
},
|
||||
}
|
||||
allowed := buildReviewAllowlist(cfg)
|
||||
|
||||
if !allowed["archie"]["acme/widget"] {
|
||||
t.Errorf("allowlist missing acme/widget for identity archie: %+v", allowed)
|
||||
}
|
||||
if allowed["archie"]["acme/core"] {
|
||||
t.Errorf("allowlist leaked acme/core to identity archie: %+v", allowed)
|
||||
}
|
||||
if _, ok := allowed["nobody"]; ok {
|
||||
t.Errorf("allowlist contains unknown identity nobody: %+v", allowed)
|
||||
}
|
||||
}
|
||||
|
||||
// reviewCapableForge satisfies both forge.Forge (via embedding) and
|
||||
// forge.PullRequestReader, so boot.prReviewer can be exercised in isolation.
|
||||
type reviewCapableForge struct {
|
||||
forge.Forge
|
||||
}
|
||||
|
||||
func (reviewCapableForge) GetPullRequest(context.Context, string, string, int) (forge.PullRequest, error) {
|
||||
return forge.PullRequest{}, nil
|
||||
}
|
||||
|
||||
// TestPrReviewerIsMemoized pins the cross-channel single-flight fix: every
|
||||
// gateway must share one prReviewer so a review of the same PR requested from
|
||||
// two channels at once deduplicates instead of running twice. The boot must
|
||||
// own a worktree manager: a reviewer without one can never run and must not be
|
||||
// produced (see TestPrReviewerNilWithoutTrees).
|
||||
func TestPrReviewerIsMemoized(t *testing.T) {
|
||||
b := &boot{
|
||||
forgeClient: reviewCapableForge{Forge: forge.NewNoop(slog.New(slog.DiscardHandler))},
|
||||
trees: &worktree.Manager{},
|
||||
}
|
||||
|
||||
first := b.prReviewer()
|
||||
if first == nil {
|
||||
t.Fatal("prReviewer() = nil, want a reviewer")
|
||||
}
|
||||
if second := b.prReviewer(); first != second {
|
||||
t.Fatal("prReviewer() returned distinct instances; channels would not share the in-flight set")
|
||||
}
|
||||
}
|
||||
|
||||
// TestPrReviewerNilWithoutTrees pins the producer-side invariant: a PR-capable
|
||||
// forge without a worktree manager must yield no reviewer. The standalone
|
||||
// Gateway never builds trees, and the daemon used to capture b.trees before it
|
||||
// was assigned, so both advertised a review_pr tool that always failed.
|
||||
func TestPrReviewerNilWithoutTrees(t *testing.T) {
|
||||
b := &boot{forgeClient: reviewCapableForge{Forge: forge.NewNoop(slog.New(slog.DiscardHandler))}}
|
||||
|
||||
if got := b.prReviewer(); got != nil {
|
||||
t.Fatalf("prReviewer() with no worktree manager = %T, want nil", got)
|
||||
}
|
||||
}
|
||||
|
||||
// TestReviewPRToolRegisteredOnlyWithWorkingReviewer composes the real boot
|
||||
// wiring (boot.prReviewer -> gateway.ReviewTools) and asserts review_pr is
|
||||
// advertised only when the reviewer can actually run.
|
||||
func TestReviewPRToolRegisteredOnlyWithWorkingReviewer(t *testing.T) {
|
||||
logger := slog.New(slog.DiscardHandler)
|
||||
b := &boot{
|
||||
forgeClient: reviewCapableForge{Forge: forge.NewNoop(logger)},
|
||||
log: logger,
|
||||
}
|
||||
|
||||
if hasReviewPRTool(gateway.ReviewTools(b.prReviewer(), "archie")) {
|
||||
t.Fatal("review_pr advertised with no worktree manager: a tool that cannot run must not be offered")
|
||||
}
|
||||
|
||||
b.trees = &worktree.Manager{}
|
||||
if !hasReviewPRTool(gateway.ReviewTools(b.prReviewer(), "archie")) {
|
||||
t.Fatal("review_pr not advertised with a worktree manager")
|
||||
}
|
||||
}
|
||||
|
||||
// hasReviewPRTool reports whether review_pr is offered. Presence is asserted by
|
||||
// name rather than by entry count: the contract is that this one capability is
|
||||
// offered only when it can run, so adding a sibling review tool must not turn
|
||||
// this red.
|
||||
func hasReviewPRTool(entries []tools.ToolEntry) bool {
|
||||
for _, e := range entries {
|
||||
if e.Name == "review_pr" {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
type fakeArtifactSender struct {
|
||||
calls int
|
||||
url string
|
||||
err error
|
||||
last artifactsync.Artifact
|
||||
}
|
||||
|
||||
func (f *fakeArtifactSender) Publish(_ context.Context, a artifactsync.Artifact) (string, error) {
|
||||
f.calls++
|
||||
f.last = a
|
||||
if f.err != nil {
|
||||
return "", f.err
|
||||
}
|
||||
return f.url, nil
|
||||
}
|
||||
|
||||
func TestPrReviewerPublishesTheCompletedReview(t *testing.T) {
|
||||
fake := &fakeArtifactSender{url: "https://offloaded.dev/collab?document=a_1"}
|
||||
r := &prReviewer{artifacts: fake, log: slog.New(slog.DiscardHandler)}
|
||||
report := workflow.NewCompletedReviewReport([]workflow.ReviewFinding{{
|
||||
File: "handler.go", Line: 42, Defect: "nil deref",
|
||||
FailureScenario: "empty payload crashes", Verdict: workflow.ReviewVerdictConfirmed,
|
||||
Level: workflow.ReviewLevelError,
|
||||
}}, "one finding")
|
||||
report.Checked = []workflow.ReviewCheck{{Property: "p", Evidence: "e"}}
|
||||
|
||||
url := r.publishArtifact(context.Background(), "system:pr-reviewer", "acme", "widget", 7, report)
|
||||
if url != fake.url {
|
||||
t.Fatalf("publishArtifact = %q, want the sender's url %q", url, fake.url)
|
||||
}
|
||||
if fake.calls != 1 {
|
||||
t.Fatalf("Publish called %d times, want 1", fake.calls)
|
||||
}
|
||||
got := fake.last
|
||||
if got.Meta.Format != "markdown" || got.Meta.Category != "pr-review" {
|
||||
t.Fatalf("format/category = %s/%s, want markdown/pr-review", got.Meta.Format, got.Meta.Category)
|
||||
}
|
||||
if got.Meta.Actor != "system:pr-reviewer" || got.Meta.Origin != "pr-review" {
|
||||
t.Fatalf("attribution = %s/%s", got.Meta.Actor, got.Meta.Origin)
|
||||
}
|
||||
if got.Meta.Visibility != "account-only" || got.Meta.Egress != "none" {
|
||||
t.Fatalf("visibility/egress = %s/%s", got.Meta.Visibility, got.Meta.Egress)
|
||||
}
|
||||
if got.Title != "Review of acme/widget#7" {
|
||||
t.Fatalf("title = %q", got.Title)
|
||||
}
|
||||
if got.Meta.RequestID != "artifact:pr-review:acme/widget#7" {
|
||||
t.Fatalf("requestId = %q, want the deterministic artifact key", got.Meta.RequestID)
|
||||
}
|
||||
if got.Meta.Task == nil || got.Meta.Task.Owner != "acme" || got.Meta.Task.Number != 7 {
|
||||
t.Fatalf("task coordinates = %+v", got.Meta.Task)
|
||||
}
|
||||
if !strings.Contains(got.Source, "## Findings") {
|
||||
t.Fatalf("source is not the rendered report:\n%s", got.Source)
|
||||
}
|
||||
}
|
||||
|
||||
func TestPrReviewerPublishDegradesOnSenderFailure(t *testing.T) {
|
||||
r := &prReviewer{artifacts: &fakeArtifactSender{err: errors.New("503 exhausted")}, log: slog.New(slog.DiscardHandler)}
|
||||
if url := r.publishArtifact(context.Background(), "system:pr-reviewer", "acme", "widget", 7, workflow.NewCompletedReviewReport(nil, "s")); url != "" {
|
||||
t.Fatalf("url = %q, want empty on a failed publish", url)
|
||||
}
|
||||
}
|
||||
|
||||
func TestPrReviewerPublishIsANoOpWithoutASender(t *testing.T) {
|
||||
r := &prReviewer{}
|
||||
if url := r.publishArtifact(context.Background(), "system:pr-reviewer", "acme", "widget", 7, workflow.NewCompletedReviewReport(nil, "s")); url != "" {
|
||||
t.Fatalf("url = %q, want empty when the sender is disabled", url)
|
||||
}
|
||||
}
|
||||
@@ -117,6 +117,7 @@ var reloadableFields = map[string]bool{
|
||||
"Models": true,
|
||||
"ModelLimits": true,
|
||||
"Notify": true,
|
||||
"Review": true,
|
||||
// BotUser/BotEmail used to be poll-path only; ForTask now carries both
|
||||
// into the task snapshot as well, because the sandboxed worker builds
|
||||
// its own worktree.Manager and signs its own commits (agentworker
|
||||
|
||||
@@ -175,7 +175,7 @@ func TestStateStoreDataSurvivesRestart(t *testing.T) {
|
||||
if err := first.openStateStore(t.Context()); err != nil {
|
||||
t.Fatalf("first openStateStore: %v", err)
|
||||
}
|
||||
task, err := first.st.EnqueueChatTask(t.Context(), "acme", "widget", "restart check", "body", "implement", "")
|
||||
task, err := first.st.EnqueueChatTask(t.Context(), "acme", "widget", "restart check", "body", "implement", "", nil)
|
||||
if err != nil {
|
||||
t.Fatalf("EnqueueChatTask: %v", err)
|
||||
}
|
||||
|
||||
@@ -52,7 +52,7 @@ func TestChatTaskActorAdapterCrossIdentityRefused(t *testing.T) {
|
||||
st := pgstore.Open(t)
|
||||
t.Cleanup(func() { _ = st.Close() })
|
||||
|
||||
task, err := st.EnqueueChatTask(ctx, "acme", "widget", "parked job", "", "", "identity-owner")
|
||||
task, err := st.EnqueueChatTask(ctx, "acme", "widget", "parked job", "", "", "identity-owner", nil)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
@@ -146,7 +146,7 @@ func TestChatTaskActorAdapterRefusesDisallowedStateAction(t *testing.T) {
|
||||
|
||||
for _, tc := range tests {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
task, err := st.EnqueueChatTask(ctx, "acme", "widget", "test task", "", "", "archie")
|
||||
task, err := st.EnqueueChatTask(ctx, "acme", "widget", "test task", "", "", "archie", nil)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
@@ -251,7 +251,7 @@ func TestChatTaskActorAdapterAppliesActionsToStore(t *testing.T) {
|
||||
|
||||
adapter := newChatTaskActorForTest(t, st, config.Config{MaxRetries: 3})
|
||||
|
||||
task, err := st.EnqueueChatTask(ctx, "acme", "widget", "actionable task", "", "", "archie")
|
||||
task, err := st.EnqueueChatTask(ctx, "acme", "widget", "actionable task", "", "", "archie", nil)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
@@ -48,7 +48,7 @@ func TestDashboardAndChatAgreeOnTerminalStates(t *testing.T) {
|
||||
st := pgstore.Open(t)
|
||||
t.Cleanup(func() { _ = st.Close() })
|
||||
|
||||
task, err := st.EnqueueChatTask(ctx, "acme", "widget", "decide on me", "", "", "reviewer")
|
||||
task, err := st.EnqueueChatTask(ctx, "acme", "widget", "decide on me", "", "", "reviewer", nil)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
@@ -6,55 +6,9 @@ import (
|
||||
"testing"
|
||||
|
||||
"github.com/samcharles93/archie-core/internal/config"
|
||||
"github.com/samcharles93/archie-core/internal/forge"
|
||||
"github.com/samcharles93/archie-core/internal/gateway"
|
||||
"github.com/samcharles93/archie-core/internal/secret"
|
||||
)
|
||||
|
||||
// TestGatewayWorktreeManagerAdvertisesReviewPR composes the Gateway's own
|
||||
// tree step with the reviewer and the review tool: whatever
|
||||
// buildWorktreeManager produces must be enough for boot.prReviewer to yield a
|
||||
// reviewer and for ReviewTools to advertise review_pr. The call-order
|
||||
// assertion in composition_order_test.go proves the Gateway makes this step;
|
||||
// this proves the step is sufficient, so a manager built without the forge
|
||||
// credential or the configured work dir cannot pass as wired.
|
||||
func TestGatewayWorktreeManagerAdvertisesReviewPR(t *testing.T) {
|
||||
const token = "gateway-forge-token"
|
||||
cfg := config.Config{
|
||||
WorkDir: t.TempDir(),
|
||||
BotUser: "archie-bot",
|
||||
BotEmail: "bot@example.test",
|
||||
Forge: config.Forge{Host: "https://forge.example.test"},
|
||||
}
|
||||
b := &boot{
|
||||
cfg: cfg,
|
||||
cfgHolder: config.NewHolder(cfg),
|
||||
token: token,
|
||||
log: slog.New(slog.DiscardHandler),
|
||||
forgeClient: reviewCapableForge{Forge: forge.NewNoop(slog.New(slog.DiscardHandler))},
|
||||
}
|
||||
|
||||
b.buildWorktreeManager()
|
||||
|
||||
if b.trees.WorkDir != cfg.WorkDir {
|
||||
t.Errorf("worktree manager WorkDir = %q, want the configured %q", b.trees.WorkDir, cfg.WorkDir)
|
||||
}
|
||||
if b.trees.BaseURL != cfg.Forge.Host {
|
||||
t.Errorf("worktree manager BaseURL = %q, want the configured forge host %q", b.trees.BaseURL, cfg.Forge.Host)
|
||||
}
|
||||
if b.trees.Token != token {
|
||||
t.Errorf("worktree manager Token = %q, want the resolved forge token %q", b.trees.Token, token)
|
||||
}
|
||||
|
||||
reviewer := b.prReviewer()
|
||||
if reviewer == nil {
|
||||
t.Fatal("prReviewer() = nil after the Gateway's worktree-manager step; review_pr would never be advertised")
|
||||
}
|
||||
if !hasReviewPRTool(gateway.ReviewTools(reviewer, "archie")) {
|
||||
t.Fatal("review_pr not advertised after the Gateway's worktree-manager step")
|
||||
}
|
||||
}
|
||||
|
||||
// TestBuildTreesAndIdentitiesBuildsRootManagerAndIdentityRunners pins the
|
||||
// daemon's tree composition, which the shared buildWorktreeManager must not
|
||||
// change: the daemon still owns the process-wide manager *and* one runner per
|
||||
|
||||
@@ -171,7 +171,7 @@ func composeUIProcess(t *testing.T) (*webui.Server, int64) {
|
||||
}
|
||||
t.Cleanup(closeClient)
|
||||
|
||||
task, err := st.EnqueueChatTask(t.Context(), "acme", "widget", "a task", "body", "implement", "")
|
||||
task, err := st.EnqueueChatTask(t.Context(), "acme", "widget", "a task", "body", "implement", "", nil)
|
||||
if err != nil {
|
||||
t.Fatalf("seed task: %v", err)
|
||||
}
|
||||
|
||||
@@ -103,7 +103,7 @@ func serveGRPC(t *testing.T, register func(grpc.ServiceRegistrar)) (target strin
|
||||
func TestUIServesDashboardAgainstRemoteContracts(t *testing.T) {
|
||||
st := pgstore.Open(t)
|
||||
defer st.Close()
|
||||
if _, err := st.EnqueueChatTask(t.Context(), "acme", "widget", "remote summary", "body", "implement", ""); err != nil {
|
||||
if _, err := st.EnqueueChatTask(t.Context(), "acme", "widget", "remote summary", "body", "implement", "", nil); err != nil {
|
||||
t.Fatalf("seed task: %v", err)
|
||||
}
|
||||
// Claimed (not just enqueued): StartStep guards on the execution's own
|
||||
|
||||
@@ -156,7 +156,7 @@ func composeUIProcessWithLog(t *testing.T) (*webui.Server, int64, int) {
|
||||
st := pgstore.Open(t)
|
||||
t.Cleanup(func() { _ = st.Close() })
|
||||
|
||||
task, err := st.EnqueueChatTask(t.Context(), "acme", "widget", "a task", "body", "implement", "")
|
||||
task, err := st.EnqueueChatTask(t.Context(), "acme", "widget", "a task", "body", "implement", "", nil)
|
||||
if err != nil {
|
||||
t.Fatalf("seed task: %v", err)
|
||||
}
|
||||
|
||||
@@ -0,0 +1,106 @@
|
||||
package prbench
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"log/slog"
|
||||
"net/url"
|
||||
"strconv"
|
||||
"strings"
|
||||
"sync/atomic"
|
||||
|
||||
"github.com/samcharles93/ai-sdk/runtime"
|
||||
|
||||
"github.com/samcharles93/archie-core/internal/agentexec"
|
||||
"github.com/samcharles93/archie-core/internal/config"
|
||||
"github.com/samcharles93/archie-core/internal/domain/workflow"
|
||||
"github.com/samcharles93/archie-core/internal/domain/workflow/prreview"
|
||||
"github.com/samcharles93/archie-core/internal/domain/workflow/prreview/bench"
|
||||
"github.com/samcharles93/archie-core/internal/forge"
|
||||
"github.com/samcharles93/archie-core/internal/infrastructure/prsource"
|
||||
)
|
||||
|
||||
func newBenchmarkReviewer(opts Options, cfg config.Config, rt *runtime.Runtime) (bench.Reviewer, error) {
|
||||
if opts.Fixtures != "" {
|
||||
return fixtureReviewer{dir: opts.Fixtures}, nil
|
||||
}
|
||||
if opts.ReviewModel == "" || opts.ClassifyModel == "" {
|
||||
return nil, fmt.Errorf("live review requires both -review-model and -classification-model")
|
||||
}
|
||||
client, err := forge.NewGitHub(opts.GitHubToken, "https://github.com", slog.Default())
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
prReader, prOK := client.(forge.PullRequestReader)
|
||||
diffReader, diffOK := client.(forge.PullRequestDiffReader)
|
||||
archiveReader, archiveOK := client.(forge.RepoArchiveReader)
|
||||
if !prOK || !diffOK || !archiveOK {
|
||||
return nil, fmt.Errorf("GitHub client cannot read pull request snapshots")
|
||||
}
|
||||
cfg.Models = map[string]string{"review": opts.ReviewModel, "classification": opts.ClassifyModel}
|
||||
cfg.Review.ApproveBeforePost = false
|
||||
cfg.Review.PrecisionGate = false
|
||||
return &liveReviewer{
|
||||
cfg: cfg, agent: agentexec.NewLoopRunner(rt, slog.Default()),
|
||||
source: prsource.New(forgeSource{PullRequestReader: prReader, PullRequestDiffReader: diffReader, RepoArchiveReader: archiveReader}),
|
||||
}, nil
|
||||
}
|
||||
|
||||
type liveReviewer struct {
|
||||
cfg config.Config
|
||||
agent agentexec.Runner
|
||||
source workflow.PRSource
|
||||
nextID atomic.Int64
|
||||
}
|
||||
|
||||
func (r *liveReviewer) Review(ctx context.Context, p bench.Problem) (bench.Review, error) {
|
||||
owner, repo, number, err := parsePRURL(p.PRURL)
|
||||
if err != nil {
|
||||
return bench.Review{}, err
|
||||
}
|
||||
slog.Info("benchmark review started", "problem", p.ID)
|
||||
tc := &workflow.TaskContext{
|
||||
Task: &workflow.Task{ID: r.nextID.Add(1), Attempt: 1, Owner: owner, Repo: repo, PRNumber: number, Workflow: "pr-review", Inputs: map[string]any{"depth": "deep"}},
|
||||
Cfg: r.cfg, Agent: r.agent, PRSource: r.source, Log: slog.Default(),
|
||||
}
|
||||
decision, err := workflow.ReviewPullRequest(ctx, tc)
|
||||
if err != nil {
|
||||
return bench.Review{}, err
|
||||
}
|
||||
// Non-nil empty comments means a completed pipeline found nothing.
|
||||
if decision.Comments == nil {
|
||||
decision.Comments = []prreview.ScoredFinding{}
|
||||
}
|
||||
slog.Info("benchmark review completed", "problem", p.ID, "comments", len(decision.Comments), "skipped_phases", decision.SkippedPhases)
|
||||
return bench.Review{Comments: decision.Comments, SkippedPhases: decision.SkippedPhases}, nil
|
||||
}
|
||||
|
||||
func parsePRURL(raw string) (string, string, int, error) {
|
||||
u, err := url.Parse(raw)
|
||||
if err != nil {
|
||||
return "", "", 0, fmt.Errorf("invalid pull request URL: %w", err)
|
||||
}
|
||||
parts := strings.Split(strings.Trim(u.Path, "/"), "/")
|
||||
if u.Scheme != "https" || u.Host != "github.com" || len(parts) != 4 || parts[2] != "pull" || u.RawQuery != "" || u.Fragment != "" || u.User != nil {
|
||||
return "", "", 0, fmt.Errorf("benchmark requires a GitHub pull request URL")
|
||||
}
|
||||
n, err := strconv.Atoi(parts[3])
|
||||
if err != nil || n <= 0 || parts[0] == "" || parts[1] == "" {
|
||||
return "", "", 0, fmt.Errorf("invalid pull request URL")
|
||||
}
|
||||
return parts[0], parts[1], n, nil
|
||||
}
|
||||
|
||||
type forgeSource struct {
|
||||
forge.PullRequestReader
|
||||
forge.PullRequestDiffReader
|
||||
forge.RepoArchiveReader
|
||||
}
|
||||
|
||||
func (s forgeSource) GetPullRequest(ctx context.Context, owner, repo string, number int) (prsource.PullRequest, error) {
|
||||
p, err := s.PullRequestReader.GetPullRequest(ctx, owner, repo, number)
|
||||
if err != nil {
|
||||
return prsource.PullRequest{}, err
|
||||
}
|
||||
return prsource.PullRequest{Title: p.Title, Body: p.Body, HeadSHA: p.HeadSHA}, nil
|
||||
}
|
||||
+136
-15
@@ -6,13 +6,14 @@ import (
|
||||
"context"
|
||||
_ "embed"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"log/slog"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"reflect"
|
||||
"slices"
|
||||
"sync"
|
||||
|
||||
"github.com/samcharles93/archie-core/internal/agentexec"
|
||||
"github.com/samcharles93/archie-core/internal/app/archied"
|
||||
@@ -48,20 +49,21 @@ func Problems(ids []string) ([]bench.Problem, error) {
|
||||
|
||||
// Options configures one benchmark run.
|
||||
type Options struct {
|
||||
Config string
|
||||
Fixtures string
|
||||
JudgeModel string
|
||||
Out string
|
||||
Problems []string
|
||||
Concurrency int
|
||||
Config string
|
||||
Fixtures string
|
||||
JudgeModel string
|
||||
ReviewModel string
|
||||
ClassifyModel string
|
||||
GitHubToken string
|
||||
Resume string
|
||||
Out string
|
||||
Problems []string
|
||||
Concurrency int
|
||||
}
|
||||
|
||||
// Run reviews and judges the selected problems, writes one result file per
|
||||
// problem and a summary to Options.Out, and prints the summary to w.
|
||||
func Run(ctx context.Context, opts Options, w io.Writer) (bench.Summary, error) {
|
||||
if opts.Fixtures == "" {
|
||||
return bench.Summary{}, errors.New("no reviewer: pass a findings fixture directory")
|
||||
}
|
||||
problems, err := Problems(opts.Problems)
|
||||
if err != nil {
|
||||
return bench.Summary{}, err
|
||||
@@ -78,10 +80,17 @@ func Run(ctx context.Context, opts Options, w io.Writer) (bench.Summary, error)
|
||||
return bench.Summary{}, fmt.Errorf("%s configures no providers for the judge", opts.Config)
|
||||
}
|
||||
judge := &modelJudge{runtime: rt, modelRef: opts.JudgeModel}
|
||||
reviewer, err := newBenchmarkReviewer(opts, doc.Config, rt)
|
||||
if err != nil {
|
||||
return bench.Summary{}, err
|
||||
}
|
||||
|
||||
results := bench.Run(ctx, problems, fixtureReviewer{dir: opts.Fixtures}, judge, opts.Concurrency)
|
||||
results, err := runReviews(ctx, problems, reviewer, judge, opts)
|
||||
if err != nil {
|
||||
return bench.Summary{}, err
|
||||
}
|
||||
summary := bench.Summarize(results)
|
||||
if err := writeResults(opts.Out, results, summary); err != nil {
|
||||
if err := writeResults(opts.Out, results, summary, opts); err != nil {
|
||||
return summary, err
|
||||
}
|
||||
for _, r := range results {
|
||||
@@ -92,10 +101,51 @@ func Run(ctx context.Context, opts Options, w io.Writer) (bench.Summary, error)
|
||||
fmt.Fprintf(w, "problems %d (failed %d) recall %.3f (%d/%d) golden-only precision %.3f (%d/%d)\n",
|
||||
summary.Problems, summary.Failed, summary.Recall, summary.Hits, summary.Goldens,
|
||||
summary.Precision, summary.Matched, summary.Comments)
|
||||
if summary.Failed != 0 {
|
||||
return summary, fmt.Errorf("%d benchmark problem(s) failed; results are incomplete", summary.Failed)
|
||||
}
|
||||
if opts.Fixtures == "" && len(opts.Problems) == 0 && summary.Recall < 0.70 {
|
||||
return summary, fmt.Errorf("golden micro-recall %.3f is below 0.70", summary.Recall)
|
||||
}
|
||||
return summary, nil
|
||||
}
|
||||
|
||||
func writeResults(dir string, results []bench.Result, summary bench.Summary) error {
|
||||
func runReviews(ctx context.Context, problems []bench.Problem, reviewer bench.Reviewer, judge bench.Judge, opts Options) ([]bench.Result, error) {
|
||||
results, pending, err := resumeResults(problems, opts)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if err := os.MkdirAll(filepath.Join(opts.Out, "results"), 0o755); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if err := writeJSON(filepath.Join(opts.Out, "run.json"), metadata(opts)); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
var saveMu sync.Mutex
|
||||
var saveErr error
|
||||
newResults := bench.Run(ctx, pending, reviewer, judge, opts.Concurrency, func(r bench.Result) {
|
||||
if err := writeJSON(filepath.Join(opts.Out, "results", r.Problem.ID+".json"), r); err != nil {
|
||||
saveMu.Lock()
|
||||
saveErr = err
|
||||
saveMu.Unlock()
|
||||
}
|
||||
})
|
||||
if saveErr != nil {
|
||||
return nil, fmt.Errorf("save benchmark result: %w", saveErr)
|
||||
}
|
||||
byID := make(map[string]bench.Result, len(newResults))
|
||||
for _, r := range newResults {
|
||||
byID[r.Problem.ID] = r
|
||||
}
|
||||
for i, r := range results {
|
||||
if r.Problem.ID == "" {
|
||||
results[i] = byID[problems[i].ID]
|
||||
}
|
||||
}
|
||||
return results, nil
|
||||
}
|
||||
|
||||
func writeResults(dir string, results []bench.Result, summary bench.Summary, opts Options) error {
|
||||
if err := os.MkdirAll(filepath.Join(dir, "results"), 0o755); err != nil {
|
||||
return err
|
||||
}
|
||||
@@ -104,7 +154,66 @@ func writeResults(dir string, results []bench.Result, summary bench.Summary) err
|
||||
return err
|
||||
}
|
||||
}
|
||||
return writeJSON(filepath.Join(dir, "summary.json"), summary)
|
||||
if err := writeJSON(filepath.Join(dir, "summary.json"), summary); err != nil {
|
||||
return err
|
||||
}
|
||||
return writeJSON(filepath.Join(dir, "run.json"), metadata(opts))
|
||||
}
|
||||
|
||||
type runMetadata struct {
|
||||
ReviewModel string `json:"review_model"`
|
||||
ClassificationModel string `json:"classification_model"`
|
||||
JudgeModel string `json:"judge_model"`
|
||||
Live bool `json:"live"`
|
||||
}
|
||||
|
||||
func metadata(opts Options) runMetadata {
|
||||
return runMetadata{opts.ReviewModel, opts.ClassifyModel, opts.JudgeModel, opts.Fixtures == ""}
|
||||
}
|
||||
|
||||
func resumeResults(problems []bench.Problem, opts Options) ([]bench.Result, []bench.Problem, error) {
|
||||
results := make([]bench.Result, len(problems))
|
||||
if opts.Resume == "" {
|
||||
return results, problems, nil
|
||||
}
|
||||
data, err := os.ReadFile(filepath.Join(opts.Resume, "run.json"))
|
||||
if err != nil {
|
||||
return nil, nil, fmt.Errorf("read benchmark resume metadata: %w", err)
|
||||
}
|
||||
var previous runMetadata
|
||||
if err := json.Unmarshal(data, &previous); err != nil {
|
||||
return nil, nil, fmt.Errorf("decode benchmark resume metadata: %w", err)
|
||||
}
|
||||
if previous != metadata(opts) {
|
||||
return nil, nil, fmt.Errorf("benchmark resume models or mode differ from this run")
|
||||
}
|
||||
var pending []bench.Problem
|
||||
for i, p := range problems {
|
||||
data, err := os.ReadFile(filepath.Join(opts.Resume, "results", p.ID+".json"))
|
||||
if os.IsNotExist(err) {
|
||||
pending = append(pending, p)
|
||||
continue
|
||||
}
|
||||
if err != nil {
|
||||
return nil, nil, fmt.Errorf("read benchmark result %s: %w", p.ID, err)
|
||||
}
|
||||
var r bench.Result
|
||||
if err := json.Unmarshal(data, &r); err != nil {
|
||||
return nil, nil, fmt.Errorf("decode benchmark result %s: %w", p.ID, err)
|
||||
}
|
||||
if !reflect.DeepEqual(r.Problem, p) {
|
||||
return nil, nil, fmt.Errorf("benchmark result %s belongs to a different problem", p.ID)
|
||||
}
|
||||
if r.Err != "" {
|
||||
pending = append(pending, p)
|
||||
continue
|
||||
}
|
||||
if len(r.Verdicts) != len(p.Goldens) {
|
||||
return nil, nil, fmt.Errorf("benchmark result %s is incomplete or belongs to a different problem", p.ID)
|
||||
}
|
||||
results[i] = r
|
||||
}
|
||||
return results, pending, nil
|
||||
}
|
||||
|
||||
func writeJSON(path string, v any) error {
|
||||
@@ -112,5 +221,17 @@ func writeJSON(path string, v any) error {
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
return os.WriteFile(path, append(data, '\n'), 0o644)
|
||||
f, err := os.CreateTemp(filepath.Dir(path), ".prbench-*")
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer os.Remove(f.Name())
|
||||
if _, err := f.Write(append(data, '\n')); err != nil {
|
||||
f.Close()
|
||||
return err
|
||||
}
|
||||
if err := f.Close(); err != nil {
|
||||
return err
|
||||
}
|
||||
return os.Rename(f.Name(), path)
|
||||
}
|
||||
@@ -63,3 +63,21 @@ func TestFixtureReviewer(t *testing.T) {
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestResumeResultsOnlyReusesCompleteMatchingRun(t *testing.T) {
|
||||
dir := t.TempDir()
|
||||
opts := Options{ReviewModel: "openai/review", ClassifyModel: "openai/classify", JudgeModel: "openai/judge", Resume: dir}
|
||||
p := bench.Problem{ID: "first", PRURL: "https://github.com/acme/repo/pull/1", Goldens: []bench.Golden{{Comment: "defect"}}}
|
||||
other := bench.Problem{ID: "second", PRURL: "https://github.com/acme/repo/pull/2", Goldens: []bench.Golden{{Comment: "other"}}}
|
||||
if err := writeResults(dir, []bench.Result{{Problem: p, Verdicts: []bench.Verdict{{Golden: 0, Comment: bench.NoComment}}}}, bench.Summary{}, opts); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
results, pending, err := resumeResults([]bench.Problem{p, other}, opts)
|
||||
if err != nil || len(pending) != 1 || pending[0].ID != other.ID || results[0].Problem.ID != p.ID {
|
||||
t.Fatalf("results = %+v, pending = %+v, err = %v", results, pending, err)
|
||||
}
|
||||
opts.JudgeModel = "openai/different"
|
||||
if _, _, err := resumeResults([]bench.Problem{p}, opts); err == nil {
|
||||
t.Fatal("mixed model results were accepted")
|
||||
}
|
||||
}
|
||||
@@ -95,13 +95,6 @@ type Repo struct {
|
||||
// won't collide (e.g. different base branches or packages per
|
||||
// task). Still bounded by [containers].max_concurrency globally.
|
||||
AllowConcurrent bool `toml:"allow_concurrent" json:"allow_concurrent" yaml:"allow_concurrent"`
|
||||
// ReviewEnabled opts this repo into the adversarial self-review stage
|
||||
// (docs/prds/adversarial-self-review.md): a fresh reviewer agent, with
|
||||
// no path to the implementer's context, runs after gates pass and
|
||||
// before the PR opens. Default false -- an uncalibrated reviewer that
|
||||
// cries wolf is worse than no review, so a repo opts in once it
|
||||
// trusts the stage.
|
||||
ReviewEnabled bool `toml:"review_enabled" json:"review_enabled" yaml:"review_enabled"`
|
||||
}
|
||||
|
||||
// Protected reports whether path matches a protected suffix.
|
||||
@@ -210,6 +203,23 @@ type Forge struct {
|
||||
WebhookAddr string `toml:"webhook_addr" yaml:"webhook_addr"`
|
||||
}
|
||||
|
||||
// Review configures the pr-review workflow (docs/prds/pr-review-agent.md).
|
||||
// Both fields are off by default: the pipeline is recall-first, and posts
|
||||
// automatically, unless an operator opts into either dial.
|
||||
type Review struct {
|
||||
// PrecisionGate runs a post-worthiness pass after phase 4 (review) and
|
||||
// before phase 5 (verification): it keeps every concrete, evidenced
|
||||
// defect, drops nitpicks, style and unverifiable claims, and keeps a
|
||||
// finding when unsure. It trades recall for precision and cuts
|
||||
// verification cost by filtering early.
|
||||
PrecisionGate bool `toml:"precision_gate" yaml:"precision_gate"`
|
||||
// ApproveBeforePost ends the pipeline in waiting_human right after
|
||||
// synthesis, before the merge gate runs, so an operator can choose which
|
||||
// findings to post, reject the review, or ask for a re-review with
|
||||
// instructions. A review with no findings never waits.
|
||||
ApproveBeforePost bool `toml:"approve_before_post" yaml:"approve_before_post"`
|
||||
}
|
||||
|
||||
// Dispatch configures how archied discovers work and reflects task state
|
||||
// onto the forge via labels and reactions.
|
||||
type Dispatch struct {
|
||||
@@ -486,6 +496,7 @@ type Config struct {
|
||||
|
||||
Forge Forge `toml:"forge" yaml:"forge"`
|
||||
Dispatch Dispatch `toml:"dispatch" yaml:"dispatch"`
|
||||
Review Review `toml:"review" yaml:"review"`
|
||||
|
||||
// Models maps a role ("planner", "builder", "triage") to a runtime
|
||||
// model ref ("provider/model").
|
||||
@@ -604,6 +615,7 @@ type TaskConfig struct {
|
||||
Notify Notify `json:"notify"`
|
||||
Forge TaskForge `json:"forge"`
|
||||
ToolPolicy ToolPolicy `json:"tool_policy"`
|
||||
Review Review `json:"review"`
|
||||
}
|
||||
|
||||
// DiffCapOf returns a DiffCapLines value for n. It exists because the field is
|
||||
@@ -644,6 +656,7 @@ func (c Config) ForTask() TaskConfig {
|
||||
Notify: c.Notify,
|
||||
Forge: TaskForge{Host: c.Forge.Host},
|
||||
ToolPolicy: c.Tools.Policy,
|
||||
Review: c.Review,
|
||||
}
|
||||
}
|
||||
|
||||
@@ -663,6 +676,7 @@ func (tc TaskConfig) ToConfig() Config {
|
||||
Notify: tc.Notify,
|
||||
Forge: Forge{Host: tc.Forge.Host},
|
||||
Tools: ToolsConfig{Policy: tc.ToolPolicy},
|
||||
Review: tc.Review,
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -157,8 +157,9 @@ func TestConfigForTaskJSONRoundTrip(t *testing.T) {
|
||||
Providers: map[string]Provider{
|
||||
"secret": {APIKeyEnv: "TOP_SECRET_PROVIDER_TOKEN"},
|
||||
},
|
||||
NATS: NATSConfig{TokenEnv: "TOP_SECRET_NATS_TOKEN"},
|
||||
Tools: ToolsConfig{Policy: ToolPolicy{MaxResultChars: 50_000, SpillDir: "/var/tmp/archie-spill"}},
|
||||
NATS: NATSConfig{TokenEnv: "TOP_SECRET_NATS_TOKEN"},
|
||||
Tools: ToolsConfig{Policy: ToolPolicy{MaxResultChars: 50_000, SpillDir: "/var/tmp/archie-spill"}},
|
||||
Review: Review{PrecisionGate: true, ApproveBeforePost: true},
|
||||
}
|
||||
|
||||
want := TaskConfig{
|
||||
@@ -171,6 +172,7 @@ func TestConfigForTaskJSONRoundTrip(t *testing.T) {
|
||||
Notify: cfg.Notify,
|
||||
Forge: TaskForge{Host: cfg.Forge.Host},
|
||||
ToolPolicy: cfg.Tools.Policy,
|
||||
Review: cfg.Review,
|
||||
}
|
||||
got := cfg.ForTask()
|
||||
if !reflect.DeepEqual(got, want) {
|
||||
@@ -197,7 +199,7 @@ func TestConfigForTaskJSONRoundTrip(t *testing.T) {
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
assertJSONKeys(t, data, "bot_user", "bot_email", "models", "budgets", "dispatch", "diff_cap_lines", "notify", "forge", "tool_policy")
|
||||
assertJSONKeys(t, data, "bot_user", "bot_email", "models", "budgets", "dispatch", "diff_cap_lines", "notify", "forge", "tool_policy", "review")
|
||||
|
||||
var forgePayload map[string]json.RawMessage
|
||||
var payload map[string]json.RawMessage
|
||||
@@ -251,7 +253,6 @@ func TestRepoJSONRoundTrip(t *testing.T) {
|
||||
PersistentStorage: true,
|
||||
MaxRetries: 4,
|
||||
AllowConcurrent: true,
|
||||
ReviewEnabled: true,
|
||||
}
|
||||
|
||||
data, err := json.Marshal(repo)
|
||||
@@ -271,7 +272,6 @@ func TestRepoJSONRoundTrip(t *testing.T) {
|
||||
"persistent_storage",
|
||||
"max_retries",
|
||||
"allow_concurrent",
|
||||
"review_enabled",
|
||||
)
|
||||
|
||||
var got Repo
|
||||
|
||||
@@ -2957,13 +2957,17 @@ func (x *EnqueueIssueResponse) GetInserted() bool {
|
||||
}
|
||||
|
||||
type EnqueueChatTaskRequest struct {
|
||||
state protoimpl.MessageState `protogen:"open.v1"`
|
||||
Owner string `protobuf:"bytes,1,opt,name=owner,proto3" json:"owner,omitempty"`
|
||||
Repo string `protobuf:"bytes,2,opt,name=repo,proto3" json:"repo,omitempty"`
|
||||
Title string `protobuf:"bytes,3,opt,name=title,proto3" json:"title,omitempty"`
|
||||
Body string `protobuf:"bytes,4,opt,name=body,proto3" json:"body,omitempty"`
|
||||
Workflow string `protobuf:"bytes,5,opt,name=workflow,proto3" json:"workflow,omitempty"`
|
||||
Identity string `protobuf:"bytes,6,opt,name=identity,proto3" json:"identity,omitempty"`
|
||||
state protoimpl.MessageState `protogen:"open.v1"`
|
||||
Owner string `protobuf:"bytes,1,opt,name=owner,proto3" json:"owner,omitempty"`
|
||||
Repo string `protobuf:"bytes,2,opt,name=repo,proto3" json:"repo,omitempty"`
|
||||
Title string `protobuf:"bytes,3,opt,name=title,proto3" json:"title,omitempty"`
|
||||
Body string `protobuf:"bytes,4,opt,name=body,proto3" json:"body,omitempty"`
|
||||
Workflow string `protobuf:"bytes,5,opt,name=workflow,proto3" json:"workflow,omitempty"`
|
||||
Identity string `protobuf:"bytes,6,opt,name=identity,proto3" json:"identity,omitempty"`
|
||||
// inputs_json assigns workflow inputs, a JSON object of the same form
|
||||
// EnqueueBindingTaskRequest carries. A chat task sets them at insert
|
||||
// time: it has no binding provenance to stamp afterwards.
|
||||
InputsJson string `protobuf:"bytes,7,opt,name=inputs_json,json=inputsJson,proto3" json:"inputs_json,omitempty"`
|
||||
unknownFields protoimpl.UnknownFields
|
||||
sizeCache protoimpl.SizeCache
|
||||
}
|
||||
@@ -3040,6 +3044,13 @@ func (x *EnqueueChatTaskRequest) GetIdentity() string {
|
||||
return ""
|
||||
}
|
||||
|
||||
func (x *EnqueueChatTaskRequest) GetInputsJson() string {
|
||||
if x != nil {
|
||||
return x.InputsJson
|
||||
}
|
||||
return ""
|
||||
}
|
||||
|
||||
type EnqueueChatTaskResponse struct {
|
||||
state protoimpl.MessageState `protogen:"open.v1"`
|
||||
Task *Task `protobuf:"bytes,1,opt,name=task,proto3" json:"task,omitempty"`
|
||||
@@ -11902,14 +11913,16 @@ const file_state_v1_state_proto_rawDesc = "" +
|
||||
"\x06labels\x18\x06 \x01(\tR\x06labels\x12\x1a\n" +
|
||||
"\bidentity\x18\a \x01(\tR\bidentity\"2\n" +
|
||||
"\x14EnqueueIssueResponse\x12\x1a\n" +
|
||||
"\binserted\x18\x01 \x01(\bR\binserted\"\xa4\x01\n" +
|
||||
"\binserted\x18\x01 \x01(\bR\binserted\"\xc5\x01\n" +
|
||||
"\x16EnqueueChatTaskRequest\x12\x14\n" +
|
||||
"\x05owner\x18\x01 \x01(\tR\x05owner\x12\x12\n" +
|
||||
"\x04repo\x18\x02 \x01(\tR\x04repo\x12\x14\n" +
|
||||
"\x05title\x18\x03 \x01(\tR\x05title\x12\x12\n" +
|
||||
"\x04body\x18\x04 \x01(\tR\x04body\x12\x1a\n" +
|
||||
"\bworkflow\x18\x05 \x01(\tR\bworkflow\x12\x1a\n" +
|
||||
"\bidentity\x18\x06 \x01(\tR\bidentity\"=\n" +
|
||||
"\bidentity\x18\x06 \x01(\tR\bidentity\x12\x1f\n" +
|
||||
"\vinputs_json\x18\a \x01(\tR\n" +
|
||||
"inputsJson\"=\n" +
|
||||
"\x17EnqueueChatTaskResponse\x12\"\n" +
|
||||
"\x04task\x18\x01 \x01(\v2\x0e.state.v1.TaskR\x04task\"\x12\n" +
|
||||
"\x10ClaimNextRequest\"M\n" +
|
||||
|
||||
@@ -1522,11 +1522,36 @@ func (d *Daemon) acquireTaskContainer(
|
||||
d.parkRunningTask(ctx, task.ID, reason+": "+err.Error(), taskstate.ParkTransient)
|
||||
}
|
||||
|
||||
// A watched-repositories binding or the operator chat trigger assigns
|
||||
// the pull request generically through Task.Inputs (its declared
|
||||
// pr_number input); archie's own PRs sets PRNumber directly and never
|
||||
// goes through either path. Resolving here, before the brief is
|
||||
// written, means task.json and the prefetch below always agree with
|
||||
// what every later reader (stagePRIntake, output posting) sees.
|
||||
if task.Workflow == "pr-review" {
|
||||
task.PRNumber = task.EffectivePRNumber()
|
||||
}
|
||||
|
||||
if err := writeTaskBrief(workDir, task); err != nil {
|
||||
park("task.json write failed", err)
|
||||
return nil, nil, false
|
||||
}
|
||||
|
||||
// pr-review needs an external pull request's metadata, diff and head
|
||||
// snapshot inside the container, but the container must never hold a
|
||||
// forge credential (docs/development/agent.md, "Giving the agent
|
||||
// something new"). The daemon fetches it here, before the container
|
||||
// exists, using its own credentialed forge client, and writes the result
|
||||
// into workDir -- the same host directory Docker bind-mounts at
|
||||
// storage.WorktreeMountDir -- so the sandboxed pipeline can read it back
|
||||
// with prsource.MountSource and no network call of its own.
|
||||
if task.Workflow == "pr-review" {
|
||||
if err := prefetchPRReview(ctx, d.forgeFor(task), task.Owner, task.Repo, task.PRNumber, workDir); err != nil {
|
||||
park("pr-review prefetch failed", err)
|
||||
return nil, nil, false
|
||||
}
|
||||
}
|
||||
|
||||
// Guard: Storage may be nil if the daemon was wired incorrectly. In
|
||||
// normal operation, Storage is always set when ContainerPool is set.
|
||||
if d.Storage == nil {
|
||||
@@ -1759,6 +1784,12 @@ func (d *Daemon) pinTaskProfile(ctx context.Context, task *workflow.Task) (confi
|
||||
return config.AgentProfile{}, false
|
||||
}
|
||||
iface, err := workflowtask.ParseWorkflowInterface(task.WorkflowDefinitionYAML)
|
||||
if err == nil {
|
||||
// Every producer meets the declared interface here, including the
|
||||
// ones that cannot read definitions when they enqueue (chat and
|
||||
// dashboard spawns go through the Gateway, which never does).
|
||||
err = iface.CheckInputs(task.Inputs)
|
||||
}
|
||||
if err == nil {
|
||||
var profile config.AgentProfile
|
||||
if profile, err = d.configFor(task).Containers.Profile(iface.Profile); err == nil {
|
||||
|
||||
@@ -1911,7 +1911,7 @@ func TestReconcilePRsSkipsChatTasks(t *testing.T) {
|
||||
d, s, fg := testDaemon(t, 3, 0)
|
||||
ctx := context.Background()
|
||||
|
||||
task, err := s.EnqueueChatTask(ctx, "acme", "widget", "chat task", "", "", "")
|
||||
task, err := s.EnqueueChatTask(ctx, "acme", "widget", "chat task", "", "", "", nil)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
@@ -2098,7 +2098,7 @@ func TestDispatchCapturesTheAttemptsEffectiveConfig(t *testing.T) {
|
||||
// repositories, identities or lock state.
|
||||
wantKeys := []string{
|
||||
"bot_email", "bot_user", "budgets", "diff_cap_lines", "dispatch",
|
||||
"forge", "model_limits", "models", "notify", "tool_policy",
|
||||
"forge", "model_limits", "models", "notify", "review", "tool_policy",
|
||||
}
|
||||
gotKeys := make([]string, 0, len(doc))
|
||||
for key := range doc {
|
||||
@@ -2412,6 +2412,52 @@ func TestPinTaskProfileRefusesCapturesTheHarnessCannotServe(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
// TestPinTaskProfileRefusesInputsTheWorkflowDoesNotAccept holds every
|
||||
// producer to the workflow's declared interface at the one point each task
|
||||
// meets its definition. Bindings check inputs before they enqueue; a chat or
|
||||
// dashboard spawn cannot, because the Gateway never reads workflow
|
||||
// definitions, so without this a spawn missing a required input would run
|
||||
// and fail deep inside the workflow instead of parking with the reason.
|
||||
func TestPinTaskProfileRefusesInputsTheWorkflowDoesNotAccept(t *testing.T) {
|
||||
d, s, _ := daemonWithNATS(t)
|
||||
d.WorkflowDefinitions = &workflowDefinitionsStub{collection: workflow.WorkflowDefinitionCollection{Definitions: []workflow.WorkflowDefinitionEntry{
|
||||
{ID: "investigate", YAML: investigateYAML},
|
||||
}}, version: 1}
|
||||
ctx := context.Background()
|
||||
|
||||
for _, tt := range []struct {
|
||||
name string
|
||||
inputs map[string]any
|
||||
ok bool
|
||||
reason string
|
||||
}{
|
||||
{"declared inputs run", map[string]any{"src_ip": "10.0.0.1"}, true, ""},
|
||||
{"missing required input parks", nil, false, `input "src_ip" is required`},
|
||||
{"undeclared input parks", map[string]any{"src_ip": "10.0.0.1", "extra": "x"}, false, `input "extra" is not declared`},
|
||||
{"mistyped input parks", map[string]any{"src_ip": 7}, false, `input "src_ip" is number, want string`},
|
||||
} {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
if _, err := s.EnqueueChatTask(ctx, "acme", "widget", tt.name, "", "investigate", "", tt.inputs); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
task, err := s.ClaimNext(ctx)
|
||||
if err != nil || task == nil {
|
||||
t.Fatalf("claim: (%v, %v)", task, err)
|
||||
}
|
||||
if _, ok := d.pinTaskProfile(ctx, task); ok != tt.ok {
|
||||
t.Fatalf("pinTaskProfile ok = %v, want %v", ok, tt.ok)
|
||||
}
|
||||
if tt.ok {
|
||||
return
|
||||
}
|
||||
got, err := s.TaskByID(ctx, task.ID)
|
||||
if err != nil || got.Status != workflow.StatusParked || !strings.Contains(got.ParkReason, tt.reason) {
|
||||
t.Fatalf("task = %+v, %v; want parked naming %q", got, err, tt.reason)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// TestWorkflowCallGivesACalleeItsOwnProfile is the harness PRD's "a stage may
|
||||
// name a different profile" (docs/prds/external-agent-harness.md, Selection):
|
||||
// there is no mid-container runner switch, and none is built here. A stage
|
||||
|
||||
@@ -0,0 +1,105 @@
|
||||
package daemon
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"os"
|
||||
"path/filepath"
|
||||
|
||||
"github.com/samcharles93/archie-core/internal/forge"
|
||||
"github.com/samcharles93/archie-core/internal/infrastructure/prsource"
|
||||
)
|
||||
|
||||
// prefetchPRReview fetches a pull request's metadata, diff and head snapshot
|
||||
// over f (the daemon's own forge client, which holds the real credential)
|
||||
// and writes them under workDir/prReviewPrefetchDir, before the task's
|
||||
// container is acquired. This is how pr-review gets its data into the
|
||||
// sandbox with no forge credential ever entering the container: the daemon
|
||||
// does the real HTTP fetch here, unsandboxed, and the container-side
|
||||
// prsource.MountSource only ever reads the result off the bind-mounted disk
|
||||
// (docs/development/agent.md, "Giving the agent something new": "the
|
||||
// daemon's own administrative token never enters a container").
|
||||
func prefetchPRReview(ctx context.Context, f any, owner, repo string, number int, workDir string) error {
|
||||
prReader, ok := f.(forge.PullRequestReader)
|
||||
if !ok {
|
||||
return errors.New("prefetch pr-review: this forge cannot read pull requests")
|
||||
}
|
||||
diffReader, ok := f.(forge.PullRequestDiffReader)
|
||||
if !ok {
|
||||
return errors.New("prefetch pr-review: this forge cannot read pull request diffs")
|
||||
}
|
||||
archiveReader, ok := f.(forge.RepoArchiveReader)
|
||||
if !ok {
|
||||
return errors.New("prefetch pr-review: this forge cannot read repository archives")
|
||||
}
|
||||
|
||||
source := prsource.New(forgeAdapter{pr: prReader, diff: diffReader, archive: archiveReader})
|
||||
|
||||
base := filepath.Join(workDir, prsource.PrefetchDirName)
|
||||
if err := os.MkdirAll(base, 0o755); err != nil {
|
||||
return fmt.Errorf("prefetch pr-review: create prefetch directory: %w", err)
|
||||
}
|
||||
|
||||
meta, err := source.Metadata(ctx, owner, repo, number)
|
||||
if err != nil {
|
||||
return fmt.Errorf("prefetch pr-review: %w", err)
|
||||
}
|
||||
|
||||
snapshotDir := filepath.Join(base, "snapshot")
|
||||
if err := os.MkdirAll(snapshotDir, 0o755); err != nil {
|
||||
return fmt.Errorf("prefetch pr-review: create snapshot directory: %w", err)
|
||||
}
|
||||
headSHA, err := source.Snapshot(ctx, owner, repo, number, snapshotDir)
|
||||
if err != nil {
|
||||
return fmt.Errorf("prefetch pr-review: %w", err)
|
||||
}
|
||||
|
||||
diff, err := source.Diff(ctx, owner, repo, number)
|
||||
if err != nil {
|
||||
return fmt.Errorf("prefetch pr-review: %w", err)
|
||||
}
|
||||
if err := os.WriteFile(filepath.Join(base, "diff.patch"), []byte(diff), 0o644); err != nil {
|
||||
return fmt.Errorf("prefetch pr-review: write diff: %w", err)
|
||||
}
|
||||
|
||||
metaJSON, err := json.Marshal(struct {
|
||||
Title string `json:"title"`
|
||||
Body string `json:"body"`
|
||||
HeadSHA string `json:"head_sha"`
|
||||
}{Title: meta.Title, Body: meta.Body, HeadSHA: headSHA})
|
||||
if err != nil {
|
||||
return fmt.Errorf("prefetch pr-review: encode metadata: %w", err)
|
||||
}
|
||||
if err := os.WriteFile(filepath.Join(base, "metadata.json"), metaJSON, 0o644); err != nil {
|
||||
return fmt.Errorf("prefetch pr-review: write metadata: %w", err)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// forgeAdapter narrows a concrete forge client's optional capabilities into
|
||||
// prsource.Forge, converting forge.PullRequest (the daemon-side type, richer
|
||||
// than prsource needs) into prsource.PullRequest.
|
||||
type forgeAdapter struct {
|
||||
pr forge.PullRequestReader
|
||||
diff forge.PullRequestDiffReader
|
||||
archive forge.RepoArchiveReader
|
||||
}
|
||||
|
||||
func (a forgeAdapter) GetPullRequest(ctx context.Context, owner, repo string, number int) (prsource.PullRequest, error) {
|
||||
pr, err := a.pr.GetPullRequest(ctx, owner, repo, number)
|
||||
if err != nil {
|
||||
return prsource.PullRequest{}, err
|
||||
}
|
||||
return prsource.PullRequest{Title: pr.Title, Body: pr.Body, HeadSHA: pr.HeadSHA}, nil
|
||||
}
|
||||
|
||||
func (a forgeAdapter) GetPullRequestDiff(ctx context.Context, owner, repo string, number int) (string, error) {
|
||||
return a.diff.GetPullRequestDiff(ctx, owner, repo, number)
|
||||
}
|
||||
|
||||
func (a forgeAdapter) GetRepoArchive(ctx context.Context, owner, repo, ref string) (io.ReadCloser, error) {
|
||||
return a.archive.GetRepoArchive(ctx, owner, repo, ref)
|
||||
}
|
||||
@@ -0,0 +1,127 @@
|
||||
package daemon
|
||||
|
||||
import (
|
||||
"archive/tar"
|
||||
"bytes"
|
||||
"compress/gzip"
|
||||
"context"
|
||||
"errors"
|
||||
"io"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"github.com/samcharles93/archie-core/internal/forge"
|
||||
"github.com/samcharles93/archie-core/internal/infrastructure/prsource"
|
||||
)
|
||||
|
||||
// buildTestArchive builds a gzipped tar containing files, each wrapped under
|
||||
// a synthetic top-level directory the way a forge's repo-archive endpoint
|
||||
// does, so prefetchPRReview exercises the same extraction path production
|
||||
// traffic does.
|
||||
func buildTestArchive(t *testing.T, files map[string]string) []byte {
|
||||
t.Helper()
|
||||
var buf bytes.Buffer
|
||||
gz := gzip.NewWriter(&buf)
|
||||
tw := tar.NewWriter(gz)
|
||||
for name, content := range files {
|
||||
hdr := &tar.Header{
|
||||
Name: "repo-abc123/" + name,
|
||||
Mode: 0o644,
|
||||
Size: int64(len(content)),
|
||||
}
|
||||
if err := tw.WriteHeader(hdr); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, err := tw.Write([]byte(content)); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
if err := tw.Close(); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := gz.Close(); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return buf.Bytes()
|
||||
}
|
||||
|
||||
type fakePRForge struct {
|
||||
pr forge.PullRequest
|
||||
diff string
|
||||
archive []byte
|
||||
pullErr error
|
||||
diffErr error
|
||||
archErr error
|
||||
archCalls int
|
||||
}
|
||||
|
||||
func (f *fakePRForge) GetPullRequest(context.Context, string, string, int) (forge.PullRequest, error) {
|
||||
return f.pr, f.pullErr
|
||||
}
|
||||
|
||||
func (f *fakePRForge) GetPullRequestDiff(context.Context, string, string, int) (string, error) {
|
||||
return f.diff, f.diffErr
|
||||
}
|
||||
|
||||
func (f *fakePRForge) GetRepoArchive(context.Context, string, string, string) (io.ReadCloser, error) {
|
||||
f.archCalls++
|
||||
if f.archErr != nil {
|
||||
return nil, f.archErr
|
||||
}
|
||||
return io.NopCloser(strings.NewReader(string(f.archive))), nil
|
||||
}
|
||||
|
||||
func TestPrefetchPRReviewWritesMetadataDiffAndSnapshot(t *testing.T) {
|
||||
archive := buildTestArchive(t, map[string]string{"a.go": "package a\n"})
|
||||
f := &fakePRForge{
|
||||
pr: forge.PullRequest{Title: "t", Body: "b", HeadSHA: "abc123"},
|
||||
diff: "diff --git a/x b/x\n",
|
||||
archive: archive,
|
||||
}
|
||||
workDir := t.TempDir()
|
||||
|
||||
if err := prefetchPRReview(context.Background(), f, "acme", "widget", 7, workDir); err != nil {
|
||||
t.Fatalf("prefetchPRReview: %v", err)
|
||||
}
|
||||
|
||||
base := filepath.Join(workDir, prsource.PrefetchDirName)
|
||||
meta, err := os.ReadFile(filepath.Join(base, "metadata.json"))
|
||||
if err != nil {
|
||||
t.Fatalf("read metadata.json: %v", err)
|
||||
}
|
||||
if !strings.Contains(string(meta), "abc123") || !strings.Contains(string(meta), `"t"`) {
|
||||
t.Fatalf("metadata.json = %s, want title and head_sha", meta)
|
||||
}
|
||||
diff, err := os.ReadFile(filepath.Join(base, "diff.patch"))
|
||||
if err != nil {
|
||||
t.Fatalf("read diff.patch: %v", err)
|
||||
}
|
||||
if string(diff) != "diff --git a/x b/x\n" {
|
||||
t.Fatalf("diff.patch = %q", diff)
|
||||
}
|
||||
if _, err := os.Stat(filepath.Join(base, "snapshot", "a.go")); err != nil {
|
||||
t.Fatalf("snapshot not extracted: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestPrefetchPRReviewFailsWhenForgeCannotReadPullRequests(t *testing.T) {
|
||||
workDir := t.TempDir()
|
||||
if err := prefetchPRReview(context.Background(), noPRCapability{}, "acme", "widget", 7, workDir); err == nil {
|
||||
t.Fatal("want an error when the forge cannot read pull requests")
|
||||
}
|
||||
}
|
||||
|
||||
func TestPrefetchPRReviewPropagatesForgeError(t *testing.T) {
|
||||
f := &fakePRForge{pullErr: errors.New("boom")}
|
||||
workDir := t.TempDir()
|
||||
if err := prefetchPRReview(context.Background(), f, "acme", "widget", 7, workDir); err == nil {
|
||||
t.Fatal("want the forge error surfaced")
|
||||
}
|
||||
}
|
||||
|
||||
// noPRCapability satisfies forge.Forge's method set with none of
|
||||
// PullRequestReader/PullRequestDiffReader/RepoArchiveReader -- like the noop
|
||||
// forge, or a forge configured without PR-reading support.
|
||||
type noPRCapability struct{}
|
||||
@@ -20,7 +20,7 @@ func (s *workflowDefinitionsStub) WorkflowDefinitions(context.Context) (workflow
|
||||
func TestWorkflowDefinitionPinSurvivesActiveOverride(t *testing.T) {
|
||||
resources := pgstore.Open(t)
|
||||
defer resources.Close()
|
||||
task, err := resources.EnqueueChatTask(t.Context(), "acme", "widget", "custom", "", "custom", "operator")
|
||||
task, err := resources.EnqueueChatTask(t.Context(), "acme", "widget", "custom", "", "custom", "operator", nil)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
@@ -52,10 +52,14 @@ type Inbound struct {
|
||||
// Workflow are optional -- empty means "the daemon's configured
|
||||
// default for this identity".
|
||||
type SpawnRequest struct {
|
||||
Title string
|
||||
Body string // operator instructions carried into the admitted task
|
||||
Repo string // "owner/name"; empty = identity's default repo
|
||||
Title string
|
||||
Body string // operator instructions carried into the admitted task
|
||||
Repo string // "owner/name"; empty = identity's default repo
|
||||
// Workflow may name a workflow whose interface declares inputs
|
||||
// (pr-review's pr_number, for instance). Inputs assigns them; a
|
||||
// workflow that declares none ignores it.
|
||||
Workflow string // empty = daemon's default workflow routing
|
||||
Inputs map[string]any
|
||||
Identity string // the identity spawning this task; propagated from Router.Identity
|
||||
}
|
||||
|
||||
|
||||
@@ -47,7 +47,7 @@ type TaskStore interface {
|
||||
// poll, chat spawn, drain loop) do not acquire a binding-specific shape.
|
||||
type TaskLifecycle interface {
|
||||
EnqueueIssue(ctx context.Context, owner, repo string, number int, title, body, labels, identity string) (bool, error)
|
||||
EnqueueChatTask(ctx context.Context, owner, repo, title, body, wf, identity string) (*task.Task, error)
|
||||
EnqueueChatTask(ctx context.Context, owner, repo, title, body, wf, identity string, inputs map[string]any) (*task.Task, error)
|
||||
ClaimNext(ctx context.Context) (*task.Task, error)
|
||||
ClaimByIssue(ctx context.Context, owner, repo string, number int) (*task.Task, error)
|
||||
Transition(ctx context.Context, taskID int64, from, to, detail string) error
|
||||
|
||||
@@ -458,42 +458,6 @@ func TestOpenPRCapturesThePullRequestNumber(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
// TestOpenPRRecordsTheReviewedRevision is the producer half of the q9au
|
||||
// head-drift guard. Line numbers in a review finding describe one revision of
|
||||
// the worktree, and OpenPR is the last point at which the worktree is still
|
||||
// there to name it -- StageReview reads that same worktree and commits
|
||||
// nothing, so the head OpenPR measures is the head the review read. Without
|
||||
// the revision recorded here the posting stage has nothing to check the pull
|
||||
// request's head against.
|
||||
func TestOpenPRRecordsTheReviewedRevision(t *testing.T) {
|
||||
trees := &capturingTrees{stats: sampleStats()}
|
||||
tc := captureTaskContext(t, trees, &recordingStore{})
|
||||
tc.Forge = &fakeForge{}
|
||||
|
||||
if err := OpenPR(context.Background(), tc, "body"); err != nil {
|
||||
t.Fatalf("OpenPR() = %v, want nil", err)
|
||||
}
|
||||
if tc.ReviewedHeadSHA != trees.stats.HeadSHA {
|
||||
t.Errorf("ReviewedHeadSHA = %q, want the measured worktree head %q", tc.ReviewedHeadSHA, trees.stats.HeadSHA)
|
||||
}
|
||||
}
|
||||
|
||||
// TestOpenPRLeavesTheReviewedRevisionUnsetWithoutAMeasurement pins the degrade
|
||||
// path: a worktree implementation that cannot report a diffstat leaves the
|
||||
// revision empty, which the forge reads as "not measured" and posts without the
|
||||
// check. Inventing a revision here would refuse every posting instead.
|
||||
func TestOpenPRLeavesTheReviewedRevisionUnsetWithoutAMeasurement(t *testing.T) {
|
||||
tc := captureTaskContext(t, &fakeTrees{}, &recordingStore{})
|
||||
tc.Forge = &fakeForge{}
|
||||
|
||||
if err := OpenPR(context.Background(), tc, "body"); err != nil {
|
||||
t.Fatalf("OpenPR() = %v, want nil", err)
|
||||
}
|
||||
if tc.ReviewedHeadSHA != "" {
|
||||
t.Errorf("ReviewedHeadSHA = %q, want empty when nothing measured the worktree", tc.ReviewedHeadSHA)
|
||||
}
|
||||
}
|
||||
|
||||
// TestChangeCaptureTruncatesTheFileListButNotTheTotals: one large refactor
|
||||
// must not write an unbounded payload, and a capture that dropped entries must
|
||||
// still report how much the attempt actually changed.
|
||||
|
||||
@@ -193,14 +193,36 @@ func ShippedDefinitions() WorkflowDefinitionCollection {
|
||||
for _, stage := range wf.Stages {
|
||||
fmt.Fprintf(&builder, " - type: %s.%s\n", id, stage.Name)
|
||||
}
|
||||
// Only pr-review declares inputs today; every other builtin's
|
||||
// generated YAML stays byte-identical to before this field existed.
|
||||
if len(wf.Interface.Inputs) > 0 {
|
||||
builder.WriteString("inputs:\n")
|
||||
for _, name := range sortedInputNames(wf.Interface.Inputs) {
|
||||
spec := wf.Interface.Inputs[name]
|
||||
fmt.Fprintf(&builder, " %s:\n type: %s\n required: %t\n", name, spec.Type, spec.Required)
|
||||
}
|
||||
}
|
||||
collection.Definitions = append(collection.Definitions, WorkflowDefinitionEntry{ID: id, YAML: builder.String()})
|
||||
}
|
||||
return collection
|
||||
}
|
||||
|
||||
// sortedInputNames returns a Workflow.Interface's declared input names in a
|
||||
// deterministic order, so the generated YAML (and its digest) never depends
|
||||
// on map iteration order.
|
||||
func sortedInputNames(inputs map[string]task.InputSpec) []string {
|
||||
names := make([]string, 0, len(inputs))
|
||||
for name := range inputs {
|
||||
names = append(names, name)
|
||||
}
|
||||
sort.Strings(names)
|
||||
return names
|
||||
}
|
||||
|
||||
func legacyBuiltinWorkflows() Registry {
|
||||
return Registry{
|
||||
"bootstrap": Bootstrap(), "implement": Implement(), "tdd": TDD(),
|
||||
"feasibility": Feasibility(), "triage": Triage(), "remediate": Remediate(),
|
||||
"pr-review": PRReview(),
|
||||
}
|
||||
}
|
||||
@@ -3,6 +3,8 @@ package workflow
|
||||
import (
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"github.com/samcharles93/archie-core/internal/domain/workflow/task"
|
||||
)
|
||||
|
||||
func TestParseDefinitionRejectsUnsafeDefinitions(t *testing.T) {
|
||||
@@ -31,6 +33,34 @@ func TestParseDefinitionAllowsRepeatingAnOperation(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestShippedDefinitionsEmitDeclaredInputsForPRReview(t *testing.T) {
|
||||
entry, ok := ShippedDefinitions().DefinitionByID("pr-review")
|
||||
if !ok {
|
||||
t.Fatal("missing shipped workflow \"pr-review\"")
|
||||
}
|
||||
iface, err := task.ParseWorkflowInterface(entry.YAML)
|
||||
if err != nil {
|
||||
t.Fatalf("ParseWorkflowInterface: %v", err)
|
||||
}
|
||||
spec, ok := iface.Inputs["pr_number"]
|
||||
if !ok {
|
||||
t.Fatal("pr-review does not declare a pr_number input")
|
||||
}
|
||||
if spec.Type != "number" || !spec.Required {
|
||||
t.Fatalf("pr_number input = %+v, want {Type: number, Required: true}", spec)
|
||||
}
|
||||
}
|
||||
|
||||
func TestShippedDefinitionsOmitInputsBlockForWorkflowsThatDeclareNone(t *testing.T) {
|
||||
entry, ok := ShippedDefinitions().DefinitionByID("implement")
|
||||
if !ok {
|
||||
t.Fatal("missing shipped workflow \"implement\"")
|
||||
}
|
||||
if strings.Contains(entry.YAML, "inputs:") {
|
||||
t.Fatalf("implement's generated YAML unexpectedly carries an inputs: block:\n%s", entry.YAML)
|
||||
}
|
||||
}
|
||||
|
||||
func TestShippedDefinitionsPreserveStageOrder(t *testing.T) {
|
||||
registry := BuiltinStepRegistry()
|
||||
for id, legacy := range legacyBuiltinWorkflows() {
|
||||
|
||||
@@ -70,7 +70,7 @@ const (
|
||||
|
||||
// diffRuleDetailBytes bounds how much of a park Detail the rendered findings
|
||||
// occupy, matching the store's own park-reason cap headroom (see
|
||||
// reviewDetailBytes in review.go).
|
||||
// parkDetailBytes in prreview_own_pr.go).
|
||||
const diffRuleDetailBytes = 4000
|
||||
|
||||
// DiffRule is one declarative rule applied to the added lines of the committed
|
||||
|
||||
@@ -197,9 +197,7 @@ func Implement() Workflow {
|
||||
}),
|
||||
StageYaegiGate(),
|
||||
StageDiffCap(),
|
||||
StageReview(),
|
||||
StageOpenPR(implementPRBody),
|
||||
StagePostReviewComments(),
|
||||
StagePRReviewAndOpenPR(implementPRBody),
|
||||
},
|
||||
}
|
||||
}
|
||||
@@ -208,13 +206,16 @@ func Implement() Workflow {
|
||||
// reports the fresh-vs-cached breakdown of tc.Task.TokensUsed (see
|
||||
// formatTokenUsage) so the operator isn't misled by the raw prompt-token sum
|
||||
// -- most of it is typically prefix-cache hits billed at a steep discount,
|
||||
// not full price. When the adversarial review ran, its findings section is
|
||||
// appended so the PR states what the reviewer checked and found (h019.6).
|
||||
// not full price.
|
||||
//
|
||||
// This used to also append the adversarial self-review's findings section
|
||||
// (docs/prds/adversarial-self-review.md, now superseded by
|
||||
// docs/prds/pr-review-agent.md per that PRD's own header). archie-core-afbk.7
|
||||
// removed the old review stage; hooking the new pr-review pipeline in as
|
||||
// this workflow's "archie's own PRs" trigger is tracked separately (see the
|
||||
// bead's continuation comment) rather than guessed at here, since it needs
|
||||
// the same stage-loop-around-StageOpenPR investigation that bead flagged.
|
||||
func implementPRBody(tc *TaskContext) string {
|
||||
summary := tc.BuildSummary
|
||||
if section := renderPRReviewSection(tc.ReviewReport); section != "" {
|
||||
summary += "\n\n" + section
|
||||
}
|
||||
return fmt.Sprintf("%s\n\n---\n*workflow: implement · %d iterations · %s*",
|
||||
summary, tc.Task.Iterations, formatTokenUsage(tc.Task.TokensUsed, tc.RunUsage))
|
||||
tc.BuildSummary, tc.Task.Iterations, formatTokenUsage(tc.Task.TokensUsed, tc.RunUsage))
|
||||
}
|
||||
@@ -41,10 +41,11 @@ var outcomeSites = []outcomeSite{
|
||||
{"diff_rules.go", "StatusParked", "the diff breaks one of the repo's blocking rules"},
|
||||
{"feasibility.go", "StatusClosedWontDo", "feasibility decided this work should not be done"},
|
||||
{"feasibility.go", "StatusWaitingHuman", "the PRD is delivered and the run waits for a go/no-go"},
|
||||
{"prreview_own_pr.go", "StatusParked", "archie's own PR trigger found an unchallenged blocking finding before opening the PR"},
|
||||
{"prreview_stages.go", "StatusCompleted", "the pr-review pipeline posted its findings (or found none to post)"},
|
||||
{"prreview_stages_verify.go", "StatusWaitingHuman", "review.approve_before_post is set and synthesis produced findings: the run waits for an operator decision before the merge gate"},
|
||||
{"remediate.go", "StatusParked", "the remediation round cap is spent"},
|
||||
{"remediate.go", "StatusPROpen", "a remediation round pushed the task back to its open PR"},
|
||||
{"review.go", "StatusParked", "review is enabled but no reviewer is wired"},
|
||||
{"review.go", "StatusParked", "review confirmed an error-level finding"},
|
||||
{"steps.go", "StatusCompleted", "a no-change build closed the forge issue"},
|
||||
{"steps.go", "StatusCompleted", "a no-change build on a task with no forge issue"},
|
||||
{"steps.go", "StatusPROpen", "the PR opened"},
|
||||
|
||||
@@ -29,11 +29,16 @@ type Problem struct {
|
||||
// Review is what a reviewer returns for one problem: its findings and the run
|
||||
// context that scores them.
|
||||
type Review struct {
|
||||
// Comments is the pipeline's final selection. Non-nil bypasses fixture scoring.
|
||||
Comments []prreview.ScoredFinding
|
||||
SkippedPhases []string
|
||||
|
||||
Findings []prreview.Finding
|
||||
Inputs prreview.ScoreInputs
|
||||
}
|
||||
|
||||
// Reviewer reviews one problem's pull request blind: it never sees the goldens.
|
||||
// Reviewer reviews one problem's pull request blind: Run passes only ID and
|
||||
// PRURL, never the goldens.
|
||||
type Reviewer interface {
|
||||
Review(ctx context.Context, p Problem) (Review, error)
|
||||
}
|
||||
@@ -61,10 +66,11 @@ type Judge interface {
|
||||
// Result is one problem's outcome. Err is set when the reviewer or the judge
|
||||
// failed, and then the problem is not scored at all: a failure is not a miss.
|
||||
type Result struct {
|
||||
Problem Problem `json:"problem"`
|
||||
Comments []prreview.ScoredFinding `json:"comments"`
|
||||
Verdicts []Verdict `json:"verdicts"`
|
||||
Err string `json:"error,omitempty"`
|
||||
Problem Problem `json:"problem"`
|
||||
Comments []prreview.ScoredFinding `json:"comments"`
|
||||
Verdicts []Verdict `json:"verdicts"`
|
||||
SkippedPhases []string `json:"skipped_phases,omitempty"`
|
||||
Err string `json:"error,omitempty"`
|
||||
}
|
||||
|
||||
// Hits is how many goldens a posted comment caught.
|
||||
@@ -90,8 +96,9 @@ func (r Result) MatchedComments() int {
|
||||
}
|
||||
|
||||
// Run reviews and judges every problem, at most concurrency at a time, and
|
||||
// returns one result per problem in the order given.
|
||||
func Run(ctx context.Context, problems []Problem, reviewer Reviewer, judge Judge, concurrency int) []Result {
|
||||
// returns one result per problem in the order given. onResult, when supplied,
|
||||
// runs as each problem finishes and may be called concurrently.
|
||||
func Run(ctx context.Context, problems []Problem, reviewer Reviewer, judge Judge, concurrency int, onResult ...func(Result)) []Result {
|
||||
results := make([]Result, len(problems))
|
||||
slots := make(chan struct{}, max(concurrency, 1))
|
||||
var wg sync.WaitGroup
|
||||
@@ -99,7 +106,11 @@ func Run(ctx context.Context, problems []Problem, reviewer Reviewer, judge Judge
|
||||
wg.Go(func() {
|
||||
slots <- struct{}{}
|
||||
defer func() { <-slots }()
|
||||
results[i] = runOne(ctx, p, reviewer, judge)
|
||||
result := runOne(ctx, p, reviewer, judge)
|
||||
results[i] = result
|
||||
if len(onResult) != 0 {
|
||||
onResult[0](result)
|
||||
}
|
||||
})
|
||||
}
|
||||
wg.Wait()
|
||||
@@ -108,12 +119,16 @@ func Run(ctx context.Context, problems []Problem, reviewer Reviewer, judge Judge
|
||||
|
||||
func runOne(ctx context.Context, p Problem, reviewer Reviewer, judge Judge) Result {
|
||||
res := Result{Problem: p}
|
||||
review, err := reviewer.Review(ctx, p)
|
||||
review, err := reviewer.Review(ctx, Problem{ID: p.ID, PRURL: p.PRURL})
|
||||
if err != nil {
|
||||
res.Err = fmt.Sprintf("review: %v", err)
|
||||
return res
|
||||
}
|
||||
res.Comments = prreview.CapInlineComments(prreview.Score(review.Findings, review.Inputs))
|
||||
res.SkippedPhases = review.SkippedPhases
|
||||
res.Comments = review.Comments
|
||||
if res.Comments == nil {
|
||||
res.Comments = prreview.CapInlineComments(prreview.Score(review.Findings, review.Inputs))
|
||||
}
|
||||
if len(res.Comments) == 0 {
|
||||
res.Verdicts = misses(p)
|
||||
return res
|
||||
|
||||
@@ -139,3 +139,32 @@ func TestSummarize(t *testing.T) {
|
||||
t.Errorf("Summarize = %+v\nwant %+v", got, want)
|
||||
}
|
||||
}
|
||||
|
||||
type blindReviewer struct{ t *testing.T }
|
||||
|
||||
func (r blindReviewer) Review(_ context.Context, p Problem) (Review, error) {
|
||||
if len(p.Goldens) != 0 {
|
||||
r.t.Error("reviewer received golden answers")
|
||||
}
|
||||
if p.ID != "blind" || p.PRURL != "https://example.org/pr/1" {
|
||||
r.t.Error("reviewer lost PR identity")
|
||||
}
|
||||
return Review{}, nil
|
||||
}
|
||||
|
||||
func TestRunKeepsGoldensFromReviewer(t *testing.T) {
|
||||
p := Problem{ID: "blind", PRURL: "https://example.org/pr/1", Goldens: []Golden{{Comment: "secret answer"}}}
|
||||
r := Run(t.Context(), []Problem{p}, blindReviewer{t}, &fakeJudge{}, 1)
|
||||
if len(r[0].Problem.Goldens) != 1 || len(r[0].Verdicts) != 1 {
|
||||
t.Fatal("scoring lost goldens")
|
||||
}
|
||||
}
|
||||
|
||||
func TestRunUsesPipelineCommentsWithoutRescoring(t *testing.T) {
|
||||
p := problem("pipeline", 1)
|
||||
comments := []prreview.ScoredFinding{{File: "final.go", LineStart: 4, Severity: prreview.SeverityImportant, Confidence: 0.1, Score: 7}}
|
||||
r := Run(t.Context(), []Problem{p}, fakeReviewer{p.ID: {Comments: comments, SkippedPhases: []string{"merge-gate"}}}, &fakeJudge{verdicts: map[string][]Verdict{p.ID: {{Golden: 0, Comment: 0}}}}, 1)
|
||||
if r[0].Err != "" || len(r[0].Comments) != 1 || r[0].Comments[0].Score != 7 || len(r[0].SkippedPhases) != 1 || r[0].SkippedPhases[0] != "merge-gate" {
|
||||
t.Fatalf("result = %+v", r[0])
|
||||
}
|
||||
}
|
||||
@@ -51,6 +51,58 @@ func BlastRadius(fsys fs.FS, changed []string) ([]string, error) {
|
||||
return radius, nil
|
||||
}
|
||||
|
||||
// ExposureCounts returns, for every file the change reaches without touching,
|
||||
// how many changed files reach it: a fan-in count computed from the same
|
||||
// import graph BlastRadius walks. It is BlastRadius's detail, not its
|
||||
// replacement -- the coverage gate needs to tell a file three unrelated
|
||||
// changed files each reach apart from one only one of them happens to import.
|
||||
func ExposureCounts(fsys fs.FS, changed []string) (map[string]int, error) {
|
||||
graph, err := buildImportGraph(fsys)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
changedSet := make(map[string]bool, len(changed))
|
||||
for _, file := range changed {
|
||||
changedSet[file] = true
|
||||
}
|
||||
|
||||
counts := map[string]int{}
|
||||
for _, file := range changed {
|
||||
reached := map[string]bool{}
|
||||
for _, imported := range graph.imports[file] {
|
||||
reached[imported] = true
|
||||
}
|
||||
for _, dependent := range graph.dependents[file] {
|
||||
reached[dependent] = true
|
||||
}
|
||||
for target := range reached {
|
||||
if changedSet[target] {
|
||||
continue
|
||||
}
|
||||
counts[target]++
|
||||
}
|
||||
}
|
||||
return counts, nil
|
||||
}
|
||||
|
||||
// HighExposureThreshold is the fan-in above which a blast-radius file counts
|
||||
// as high-exposure for the coverage gate: reached by more than one changed
|
||||
// file, so a regression there is not one dimension's problem to have covered.
|
||||
const HighExposureThreshold = 1
|
||||
|
||||
// HighExposureFiles returns the blast-radius files whose exposure count
|
||||
// exceeds HighExposureThreshold, sorted for a deterministic gap-review order.
|
||||
func HighExposureFiles(counts map[string]int) []string {
|
||||
var files []string
|
||||
for file, count := range counts {
|
||||
if count > HighExposureThreshold {
|
||||
files = append(files, file)
|
||||
}
|
||||
}
|
||||
sort.Strings(files)
|
||||
return files
|
||||
}
|
||||
|
||||
// importGraph is a snapshot's file-level dependency graph, in both directions:
|
||||
// an edge exists when one file's imports resolve to another file.
|
||||
type importGraph struct {
|
||||
|
||||
@@ -174,3 +174,25 @@ func TestBlastRadiusJavaScriptAndTypeScript(t *testing.T) {
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestExposureCountsAndHighExposureFiles(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
fsys := fstest.MapFS{
|
||||
"go.mod": {Data: []byte("module example.com/m\n\ngo 1.27.0\n")},
|
||||
"internal/store/store.go": {Data: []byte("package store\n\nfunc Get() int { return 1 }\n")},
|
||||
"internal/app/app.go": {Data: []byte("package app\n\nimport \"example.com/m/internal/store\"\n\nfunc Run() int { return store.Get() }\n")},
|
||||
"internal/other/other.go": {Data: []byte("package other\n\nimport \"example.com/m/internal/store\"\n\nfunc Run2() int { return store.Get() }\n")},
|
||||
}
|
||||
|
||||
counts, err := ExposureCounts(fsys, []string{"internal/app/app.go", "internal/other/other.go"})
|
||||
if err != nil {
|
||||
t.Fatalf("ExposureCounts() error = %v", err)
|
||||
}
|
||||
if counts["internal/store/store.go"] != 2 {
|
||||
t.Errorf("store.go exposure = %d, want 2 (reached by both changed files)", counts["internal/store/store.go"])
|
||||
}
|
||||
|
||||
high := HighExposureFiles(counts)
|
||||
wantStrings(t, high, []string{"internal/store/store.go"})
|
||||
}
|
||||
@@ -0,0 +1,36 @@
|
||||
package prreview
|
||||
|
||||
import "time"
|
||||
|
||||
// Budget bounds a pr-review run's total cost (tokens, the only spend the
|
||||
// agent runtime accounts for) and wall-clock, split evenly across every
|
||||
// budget-tracked phase (docs/prds/pr-review-agent.md, Budget: "A run has a
|
||||
// cost cap and a wall-clock cap, split into per-phase shares"). Zero
|
||||
// disables that dimension.
|
||||
type Budget struct {
|
||||
MaxTokens int
|
||||
WallClock time.Duration
|
||||
}
|
||||
|
||||
// PhaseExhausted reports whether phase (1-based, counting the phase about to
|
||||
// start) has already spent its share, given how much of the run's total
|
||||
// budget every phase up to and including it is allotted out of phases total.
|
||||
//
|
||||
// The check is against the cumulative allotment through phase, not phase's
|
||||
// own isolated slice: an earlier phase's underspend is not wasted, but its
|
||||
// overspend still eats into a later phase's share. That is what "a phase
|
||||
// whose share is spent is skipped" means for a phase that has not started
|
||||
// yet and so has spent nothing of its own -- the run's spend so far is
|
||||
// compared against how much the whole run should have spent by this point.
|
||||
func (b Budget) PhaseExhausted(phase, phases, tokensSpent int, elapsed time.Duration) bool {
|
||||
if phase <= 0 || phases <= 0 {
|
||||
return false
|
||||
}
|
||||
if b.MaxTokens > 0 && tokensSpent >= b.MaxTokens*phase/phases {
|
||||
return true
|
||||
}
|
||||
if b.WallClock > 0 && elapsed >= b.WallClock*time.Duration(phase)/time.Duration(phases) {
|
||||
return true
|
||||
}
|
||||
return false
|
||||
}
|
||||
@@ -0,0 +1,86 @@
|
||||
package prreview
|
||||
|
||||
import (
|
||||
"testing"
|
||||
"time"
|
||||
)
|
||||
|
||||
func TestBudgetPhaseExhausted(t *testing.T) {
|
||||
cases := []struct {
|
||||
name string
|
||||
budget Budget
|
||||
phase int
|
||||
phases int
|
||||
tokensSpent int
|
||||
elapsed time.Duration
|
||||
want bool
|
||||
}{
|
||||
{
|
||||
name: "zero budget never exhausts",
|
||||
budget: Budget{},
|
||||
phase: 1, phases: 6,
|
||||
tokensSpent: 1_000_000, elapsed: time.Hour,
|
||||
want: false,
|
||||
},
|
||||
{
|
||||
name: "fresh phase under its cumulative token allotment",
|
||||
budget: Budget{MaxTokens: 600},
|
||||
phase: 1, phases: 6, // allotted 100 tokens by phase 1
|
||||
tokensSpent: 50,
|
||||
want: false,
|
||||
},
|
||||
{
|
||||
name: "tokens spent already meets the cumulative allotment",
|
||||
budget: Budget{MaxTokens: 600},
|
||||
phase: 1, phases: 6, // allotted 100 tokens by phase 1
|
||||
tokensSpent: 100,
|
||||
want: true,
|
||||
},
|
||||
{
|
||||
name: "later phase has a larger cumulative allotment",
|
||||
budget: Budget{MaxTokens: 600},
|
||||
phase: 3, phases: 6, // allotted 300 tokens by phase 3
|
||||
tokensSpent: 250,
|
||||
want: false,
|
||||
},
|
||||
{
|
||||
name: "wall clock exhausted independently of tokens",
|
||||
budget: Budget{WallClock: 60 * time.Minute},
|
||||
phase: 2, phases: 6, // allotted 20 minutes by phase 2
|
||||
elapsed: 25 * time.Minute,
|
||||
want: true,
|
||||
},
|
||||
{
|
||||
name: "wall clock not yet exhausted",
|
||||
budget: Budget{WallClock: 60 * time.Minute},
|
||||
phase: 2, phases: 6,
|
||||
elapsed: 10 * time.Minute,
|
||||
want: false,
|
||||
},
|
||||
{
|
||||
name: "phase zero is never exhausted (guard)",
|
||||
budget: Budget{MaxTokens: 1},
|
||||
phase: 0,
|
||||
phases: 6,
|
||||
tokensSpent: 1_000_000,
|
||||
want: false,
|
||||
},
|
||||
{
|
||||
name: "phases zero is never exhausted (guard)",
|
||||
budget: Budget{MaxTokens: 1},
|
||||
phase: 1,
|
||||
phases: 0,
|
||||
tokensSpent: 1_000_000,
|
||||
want: false,
|
||||
},
|
||||
}
|
||||
for _, c := range cases {
|
||||
t.Run(c.name, func(t *testing.T) {
|
||||
got := c.budget.PhaseExhausted(c.phase, c.phases, c.tokensSpent, c.elapsed)
|
||||
if got != c.want {
|
||||
t.Errorf("PhaseExhausted(%d, %d, %d, %v) = %v, want %v",
|
||||
c.phase, c.phases, c.tokensSpent, c.elapsed, got, c.want)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,53 @@
|
||||
package prreview
|
||||
|
||||
// CoverageGaps returns the clusters with no finding pointing at any of their
|
||||
// files, and the high-exposure blast-radius files no finding points at
|
||||
// either: the two things phase 6's coverage gate checks (docs/prds/
|
||||
// pr-review-agent.md, phase 6).
|
||||
func CoverageGaps(clusters []Cluster, highExposure []string, findings []Finding) (uncoveredClusters []Cluster, uncoveredFiles []string) {
|
||||
reviewed := make(map[string]bool, len(findings))
|
||||
for _, f := range findings {
|
||||
reviewed[f.File] = true
|
||||
}
|
||||
for _, cluster := range clusters {
|
||||
covered := false
|
||||
for _, file := range cluster.Files {
|
||||
if reviewed[file] {
|
||||
covered = true
|
||||
break
|
||||
}
|
||||
}
|
||||
if !covered {
|
||||
uncoveredClusters = append(uncoveredClusters, cluster)
|
||||
}
|
||||
}
|
||||
for _, file := range highExposure {
|
||||
if !reviewed[file] {
|
||||
uncoveredFiles = append(uncoveredFiles, file)
|
||||
}
|
||||
}
|
||||
return uncoveredClusters, uncoveredFiles
|
||||
}
|
||||
|
||||
// ClusterFindings groups findings by the cluster whose files contain them,
|
||||
// keyed by cluster ID. Phase 5's compound-defect check runs once per cluster
|
||||
// with more than one confirmed finding, because a compound defect is a
|
||||
// property of findings that share a place, not of the whole PR. A finding
|
||||
// whose file belongs to no cluster contributes to none.
|
||||
func ClusterFindings(clusters []Cluster, findings []Finding) map[string][]Finding {
|
||||
fileToCluster := make(map[string]string, len(findings))
|
||||
for _, cluster := range clusters {
|
||||
for _, file := range cluster.Files {
|
||||
fileToCluster[file] = cluster.ID
|
||||
}
|
||||
}
|
||||
grouped := make(map[string][]Finding)
|
||||
for _, f := range findings {
|
||||
id, ok := fileToCluster[f.File]
|
||||
if !ok {
|
||||
continue
|
||||
}
|
||||
grouped[id] = append(grouped[id], f)
|
||||
}
|
||||
return grouped
|
||||
}
|
||||
@@ -0,0 +1,66 @@
|
||||
package prreview
|
||||
|
||||
import "testing"
|
||||
|
||||
func TestCoverageGapsFindsUncoveredClustersAndFiles(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
clusters := []Cluster{
|
||||
{ID: "cluster_0", Name: "internal/a", Files: []string{"internal/a/a.go"}},
|
||||
{ID: "cluster_1", Name: "internal/b", Files: []string{"internal/b/b.go"}},
|
||||
}
|
||||
highExposure := []string{"internal/c/c.go", "internal/d/d.go"}
|
||||
findings := []Finding{
|
||||
{File: "internal/a/a.go"},
|
||||
{File: "internal/d/d.go"},
|
||||
}
|
||||
|
||||
uncoveredClusters, uncoveredFiles := CoverageGaps(clusters, highExposure, findings)
|
||||
|
||||
if len(uncoveredClusters) != 1 || uncoveredClusters[0].ID != "cluster_1" {
|
||||
t.Errorf("uncoveredClusters = %+v, want only cluster_1", uncoveredClusters)
|
||||
}
|
||||
if len(uncoveredFiles) != 1 || uncoveredFiles[0] != "internal/c/c.go" {
|
||||
t.Errorf("uncoveredFiles = %v, want only internal/c/c.go", uncoveredFiles)
|
||||
}
|
||||
}
|
||||
|
||||
func TestCoverageGapsReportsNothingWhenFullyCovered(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
clusters := []Cluster{{ID: "cluster_0", Files: []string{"a.go"}}}
|
||||
highExposure := []string{"b.go"}
|
||||
findings := []Finding{{File: "a.go"}, {File: "b.go"}}
|
||||
|
||||
uncoveredClusters, uncoveredFiles := CoverageGaps(clusters, highExposure, findings)
|
||||
if uncoveredClusters != nil || uncoveredFiles != nil {
|
||||
t.Errorf("want no gaps, got clusters=%v files=%v", uncoveredClusters, uncoveredFiles)
|
||||
}
|
||||
}
|
||||
|
||||
func TestClusterFindingsGroupsByFileMembership(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
clusters := []Cluster{
|
||||
{ID: "cluster_0", Files: []string{"a.go", "a_test.go"}},
|
||||
{ID: "cluster_1", Files: []string{"b.go"}},
|
||||
}
|
||||
findings := []Finding{
|
||||
{File: "a.go", Title: "f1"},
|
||||
{File: "a_test.go", Title: "f2"},
|
||||
{File: "b.go", Title: "f3"},
|
||||
{File: "unrelated.go", Title: "f4"},
|
||||
}
|
||||
|
||||
grouped := ClusterFindings(clusters, findings)
|
||||
|
||||
if len(grouped["cluster_0"]) != 2 {
|
||||
t.Errorf("cluster_0 findings = %d, want 2", len(grouped["cluster_0"]))
|
||||
}
|
||||
if len(grouped["cluster_1"]) != 1 {
|
||||
t.Errorf("cluster_1 findings = %d, want 1", len(grouped["cluster_1"]))
|
||||
}
|
||||
if _, ok := grouped["cluster_2"]; ok {
|
||||
t.Error("no finding belongs to a cluster_2 that doesn't exist")
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,69 @@
|
||||
package prreview
|
||||
|
||||
import (
|
||||
"cmp"
|
||||
"slices"
|
||||
)
|
||||
|
||||
// Dimension is one review dimension a lens proposed: a name, a reviewer
|
||||
// prompt written for this PR, the files the reviewer must read, and a
|
||||
// priority phase 3 ranks dimensions by when the depth's cap forces a cut.
|
||||
type Dimension struct {
|
||||
Name string
|
||||
Prompt string
|
||||
TargetFiles []string
|
||||
ContextFiles []string
|
||||
Priority float64
|
||||
}
|
||||
|
||||
// MergeDimensions drops duplicate dimension names across the lenses' outputs
|
||||
// (keeping the higher-priority copy), sorts by priority, and cuts the tail at
|
||||
// cap. Order is stable for equal priority: the input order of the first lens
|
||||
// that produced the name wins, so re-running the same lenses in the same
|
||||
// order always keeps the same dimensions.
|
||||
func MergeDimensions(lensOutputs [][]Dimension, limit int) []Dimension {
|
||||
byName := make(map[string]Dimension)
|
||||
order := make([]string, 0)
|
||||
for _, lens := range lensOutputs {
|
||||
for _, d := range lens {
|
||||
existing, ok := byName[d.Name]
|
||||
if !ok {
|
||||
order = append(order, d.Name)
|
||||
byName[d.Name] = d
|
||||
continue
|
||||
}
|
||||
if d.Priority > existing.Priority {
|
||||
byName[d.Name] = d
|
||||
}
|
||||
}
|
||||
}
|
||||
merged := make([]Dimension, 0, len(order))
|
||||
for _, name := range order {
|
||||
merged = append(merged, byName[name])
|
||||
}
|
||||
slices.SortStableFunc(merged, func(a, b Dimension) int {
|
||||
return cmp.Compare(b.Priority, a.Priority)
|
||||
})
|
||||
if len(merged) > limit {
|
||||
merged = merged[:limit]
|
||||
}
|
||||
return merged
|
||||
}
|
||||
|
||||
// HallucinationDimension is the extra review dimension phase 3 adds for a PR
|
||||
// intake scores as likely machine-written: claims, citations and behaviour
|
||||
// the diff does not actually implement. Returns nil at or below
|
||||
// AIGeneratedThreshold -- a coin-flip is not evidence, so it is not added.
|
||||
func HallucinationDimension(aiGenerated float64) *Dimension {
|
||||
if aiGenerated <= AIGeneratedThreshold {
|
||||
return nil
|
||||
}
|
||||
return &Dimension{
|
||||
Name: "hallucination-check",
|
||||
Prompt: "This PR's description or commit messages read as likely machine-written. " +
|
||||
"Check every claim the PR makes -- what it says it does, any cited behaviour, API, or test result -- " +
|
||||
"against what the diff actually implements. Report a finding for any claim, citation, or described " +
|
||||
"behaviour that the diff does not actually carry out.",
|
||||
Priority: 1,
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,68 @@
|
||||
package prreview
|
||||
|
||||
import "testing"
|
||||
|
||||
func TestMergeDimensionsDropsDuplicatesByName(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
lenses := [][]Dimension{
|
||||
{{Name: "security", Priority: 0.9}, {Name: "types", Priority: 0.4}},
|
||||
{{Name: "security", Priority: 0.5}, {Name: "docs", Priority: 0.2}},
|
||||
}
|
||||
got := MergeDimensions(lenses, 10)
|
||||
if len(got) != 3 {
|
||||
t.Fatalf("MergeDimensions() returned %d dimensions, want 3 (security, types, docs merged from duplicates)", len(got))
|
||||
}
|
||||
for _, d := range got {
|
||||
if d.Name == "security" && d.Priority != 0.9 {
|
||||
t.Errorf("duplicate %q kept priority %v, want the higher-priority lens's 0.9", d.Name, d.Priority)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestMergeDimensionsCapsAtTheDepthLimit(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
lenses := [][]Dimension{
|
||||
{
|
||||
{Name: "a", Priority: 0.9},
|
||||
{Name: "b", Priority: 0.8},
|
||||
{Name: "c", Priority: 0.7},
|
||||
{Name: "d", Priority: 0.1},
|
||||
},
|
||||
}
|
||||
got := MergeDimensions(lenses, 2)
|
||||
if len(got) != 2 {
|
||||
t.Fatalf("MergeDimensions() with cap 2 returned %d dimensions, want 2", len(got))
|
||||
}
|
||||
if got[0].Name != "a" || got[1].Name != "b" {
|
||||
t.Errorf("MergeDimensions() with cap 2 kept %+v, want the two highest-priority dimensions (a, b)", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestMergeDimensionsIsDeterministicForTheSameInput(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
lenses := [][]Dimension{{{Name: "x", Priority: 0.5}, {Name: "y", Priority: 0.5}}}
|
||||
|
||||
got1 := MergeDimensions(lenses, 10)
|
||||
got2 := MergeDimensions(lenses, 10)
|
||||
if got1[0].Name != got2[0].Name || got1[1].Name != got2[1].Name {
|
||||
t.Errorf("MergeDimensions() on the same input produced different orders across calls: %+v vs %+v", got1, got2)
|
||||
}
|
||||
}
|
||||
|
||||
func TestHallucinationDimensionOnlyAddedAboveTheAIGeneratedThreshold(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
if got := HallucinationDimension(AIGeneratedThreshold); got != nil {
|
||||
t.Errorf("HallucinationDimension() at the threshold = %+v, want nil (the PRD says above, not at)", got)
|
||||
}
|
||||
got := HallucinationDimension(AIGeneratedThreshold + 0.01)
|
||||
if got == nil {
|
||||
t.Fatal("HallucinationDimension() above the threshold = nil, want a dimension")
|
||||
}
|
||||
if got.Name == "" || got.Prompt == "" {
|
||||
t.Errorf("HallucinationDimension() = %+v, want a named dimension with a reviewer prompt", got)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,62 @@
|
||||
package prreview
|
||||
|
||||
// Depth is how many review dimensions a run affords, driven by the size of
|
||||
// the change (docs/prds/pr-review-agent.md, phase 1).
|
||||
type Depth string
|
||||
|
||||
const (
|
||||
DepthQuick Depth = "quick"
|
||||
DepthStandard Depth = "standard"
|
||||
DepthDeep Depth = "deep"
|
||||
)
|
||||
|
||||
// The line-count boundaries phase 1 classifies depth from: under 100 is
|
||||
// quick, under 500 is standard, otherwise deep.
|
||||
const (
|
||||
quickLinesCeiling = 100
|
||||
standardLinesCeiling = 500
|
||||
)
|
||||
|
||||
// The dimension caps each depth affords.
|
||||
const (
|
||||
maxDimensionsQuick = 3
|
||||
maxDimensionsStandard = 6
|
||||
maxDimensionsDeep = 12
|
||||
)
|
||||
|
||||
// ClassifyDepth derives a review depth from the number of changed lines
|
||||
// (additions plus deletions).
|
||||
func ClassifyDepth(changedLines int) Depth {
|
||||
switch {
|
||||
case changedLines < quickLinesCeiling:
|
||||
return DepthQuick
|
||||
case changedLines < standardLinesCeiling:
|
||||
return DepthStandard
|
||||
default:
|
||||
return DepthDeep
|
||||
}
|
||||
}
|
||||
|
||||
// MaxDimensionsFor returns how many review dimensions a depth affords. An
|
||||
// unrecognised depth gets the cheapest cap, never the most expensive: a typo
|
||||
// or a stale value must not silently buy a deep run's budget.
|
||||
func MaxDimensionsFor(depth Depth) int {
|
||||
switch depth {
|
||||
case DepthStandard:
|
||||
return maxDimensionsStandard
|
||||
case DepthDeep:
|
||||
return maxDimensionsDeep
|
||||
default:
|
||||
return maxDimensionsQuick
|
||||
}
|
||||
}
|
||||
|
||||
// ResolveDepth applies the PRD's override rule: an operator-set depth wins
|
||||
// over the line-count classification. An empty explicit depth means no
|
||||
// override was given.
|
||||
func ResolveDepth(changedLines int, explicit Depth) Depth {
|
||||
if explicit != "" {
|
||||
return explicit
|
||||
}
|
||||
return ClassifyDepth(changedLines)
|
||||
}
|
||||
@@ -0,0 +1,61 @@
|
||||
package prreview
|
||||
|
||||
import "testing"
|
||||
|
||||
func TestClassifyDepth(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
tests := []struct {
|
||||
name string
|
||||
changedLines int
|
||||
want Depth
|
||||
}{
|
||||
{"zero lines is quick", 0, DepthQuick},
|
||||
{"just under the quick boundary", 99, DepthQuick},
|
||||
{"the quick boundary itself is standard", 100, DepthStandard},
|
||||
{"just under the standard boundary", 499, DepthStandard},
|
||||
{"the standard boundary itself is deep", 500, DepthDeep},
|
||||
{"far above the standard boundary is deep", 5000, DepthDeep},
|
||||
}
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
t.Parallel()
|
||||
if got := ClassifyDepth(tt.changedLines); got != tt.want {
|
||||
t.Errorf("ClassifyDepth(%d) = %q, want %q", tt.changedLines, got, tt.want)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestMaxDimensionsFor(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
tests := []struct {
|
||||
depth Depth
|
||||
want int
|
||||
}{
|
||||
{DepthQuick, 3},
|
||||
{DepthStandard, 6},
|
||||
{DepthDeep, 12},
|
||||
{Depth("unknown"), 3}, // an unrecognised depth is treated as the cheapest, never the most expensive
|
||||
}
|
||||
for _, tt := range tests {
|
||||
if got := MaxDimensionsFor(tt.depth); got != tt.want {
|
||||
t.Errorf("MaxDimensionsFor(%q) = %d, want %d", tt.depth, got, tt.want)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestExplicitDepthOverridesClassification(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
got := ResolveDepth(5000, DepthQuick)
|
||||
if got != DepthQuick {
|
||||
t.Errorf("ResolveDepth with an explicit operator depth = %q, want %q (the explicit value must win over the line count)", got, DepthQuick)
|
||||
}
|
||||
|
||||
got = ResolveDepth(5000, "")
|
||||
if got != DepthDeep {
|
||||
t.Errorf("ResolveDepth with no explicit depth = %q, want %q (falls back to classification)", got, DepthDeep)
|
||||
}
|
||||
}
|
||||
@@ -127,6 +127,11 @@ type ScoredFinding struct {
|
||||
Score float64
|
||||
Multipliers []string
|
||||
Blocking bool
|
||||
// Adversary carries phase 5's verdict through synthesis, so a caller
|
||||
// deciding whether a blocking finding was ever challenged (the pr-review
|
||||
// PRD's archie's-own-PR trigger rule) does not have to cross-reference
|
||||
// back into the pre-synthesis Finding slice.
|
||||
Adversary AdversaryVerdict
|
||||
}
|
||||
|
||||
// ScoreInputs is the run context that moves every finding's score.
|
||||
@@ -170,6 +175,7 @@ func Score(findings []Finding, inputs ScoreInputs) []ScoredFinding {
|
||||
Score: score,
|
||||
Multipliers: applied,
|
||||
Blocking: finding.Blocking,
|
||||
Adversary: finding.Adversary,
|
||||
})
|
||||
}
|
||||
|
||||
@@ -414,6 +420,26 @@ func CapInlineComments(findings []ScoredFinding) []ScoredFinding {
|
||||
return findings
|
||||
}
|
||||
|
||||
// IsAIGenerated reports whether intake's machine-written confidence clears
|
||||
// AIGeneratedThreshold: a coin-flip is not evidence, so it takes strictly
|
||||
// more than the threshold to count.
|
||||
func IsAIGenerated(confidence float64) bool {
|
||||
return confidence > AIGeneratedThreshold
|
||||
}
|
||||
|
||||
// IsHighPriority reports whether a finding's severity is high enough for
|
||||
// phase 5's evidence verifier to check it against its evidence package: a
|
||||
// hallucinated nitpick costs a reader little, a hallucinated critical costs
|
||||
// them a debugging session chasing a defect that was never there.
|
||||
func IsHighPriority(f Finding) bool {
|
||||
switch normalizeSeverity(f.Severity) {
|
||||
case SeverityCritical, SeverityImportant:
|
||||
return true
|
||||
default:
|
||||
return false
|
||||
}
|
||||
}
|
||||
|
||||
// ReviewEventFor decides the event a review is submitted with: a blocking
|
||||
// finding requests changes, and anything else -- including a review that found
|
||||
// nothing -- is a comment. A pipeline that found nothing has said nothing about
|
||||
|
||||
@@ -0,0 +1,51 @@
|
||||
package workflow
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"os"
|
||||
|
||||
"github.com/samcharles93/archie-core/internal/domain/workflow/prreview"
|
||||
)
|
||||
|
||||
// ReviewPullRequest runs the same decision phases as PRReview through its
|
||||
// merge gate, without phase 9 or any forge writes. The caller supplies a fresh
|
||||
// context with operator approval disabled.
|
||||
type PRReviewDecision struct {
|
||||
Comments []prreview.ScoredFinding
|
||||
SkippedPhases []string
|
||||
}
|
||||
|
||||
func ReviewPullRequest(ctx context.Context, tc *TaskContext) (PRReviewDecision, error) {
|
||||
if tc.Cfg.Review.ApproveBeforePost {
|
||||
return PRReviewDecision{}, fmt.Errorf("headless review cannot wait for operator approval")
|
||||
}
|
||||
ctx, cancel := context.WithTimeout(ctx, prReviewTotalBudget.WallClock)
|
||||
defer cancel()
|
||||
defer func() {
|
||||
if tc.prReview != nil && tc.prReview.snapshotDir != "" {
|
||||
_ = os.RemoveAll(tc.prReview.snapshotDir)
|
||||
}
|
||||
}()
|
||||
for _, stage := range prReviewDecisionStages() {
|
||||
if err := ctx.Err(); err != nil {
|
||||
return PRReviewDecision{}, err
|
||||
}
|
||||
if err := stage.Run(ctx, tc); err != nil {
|
||||
return PRReviewDecision{}, fmt.Errorf("pr-review %s: %w", stage.Name, err)
|
||||
}
|
||||
if tc.Outcome.Status != "" {
|
||||
return PRReviewDecision{}, fmt.Errorf("pr-review stopped: %s", tc.Outcome.Detail)
|
||||
}
|
||||
}
|
||||
if err := ctx.Err(); err != nil {
|
||||
return PRReviewDecision{}, err
|
||||
}
|
||||
if failed := tc.prReview.reviewerFailures.Load(); failed != 0 {
|
||||
return PRReviewDecision{}, fmt.Errorf("pr-review: %d reviewer(s) did not finish", failed)
|
||||
}
|
||||
return PRReviewDecision{
|
||||
Comments: tc.prReview.scored,
|
||||
SkippedPhases: append([]string(nil), tc.prReview.skippedPhases...),
|
||||
}, nil
|
||||
}
|
||||
@@ -0,0 +1,153 @@
|
||||
package workflow
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"strings"
|
||||
|
||||
"github.com/samcharles93/archie-core/internal/domain/workflow/prreview"
|
||||
)
|
||||
|
||||
// localPRSource implements PRSource against the task's own worktree instead
|
||||
// of an external forge fetch: archie's own PRs trigger (docs/prds/pr-review-
|
||||
// agent.md, Triggers: "the implement workflow runs the pipeline before
|
||||
// opening its PR") reviews a change that has no PR number yet, so there is
|
||||
// nothing for a forge-backed PRSource to fetch by. Trees already holds
|
||||
// everything the pipeline's phase 1/2 need: the diff against base, and a
|
||||
// snapshot of the (as yet unopened) PR's head.
|
||||
type localPRSource struct {
|
||||
tc *TaskContext
|
||||
}
|
||||
|
||||
var _ PRSource = (*localPRSource)(nil)
|
||||
|
||||
// Metadata returns the task's own title and body as the PR's title and body
|
||||
// -- the same text OpenPR uses to open the real PR moments later. Commit
|
||||
// messages are left empty, matching the external PRSource implementation's
|
||||
// precedent (internal/infrastructure/prsource): neither Trees nor Task
|
||||
// carries them structured, and they are only one of three hallucination-check
|
||||
// inputs, not load-bearing alone.
|
||||
func (s *localPRSource) Metadata(context.Context, string, string, int) (PRMetadata, error) {
|
||||
return PRMetadata{Title: s.tc.Task.Title, Body: s.tc.Task.Body}, nil
|
||||
}
|
||||
|
||||
// Diff returns the task's own worktree diff against its base branch.
|
||||
func (s *localPRSource) Diff(ctx context.Context, _, _ string, _ int) (string, error) {
|
||||
return s.tc.Trees.Diff(ctx, s.tc.Dir, s.tc.Repo.BaseBranch())
|
||||
}
|
||||
|
||||
// Snapshot exports the task's own worktree HEAD, no .git present, and
|
||||
// reports the commit SHA it was measured on -- read through the same
|
||||
// changeStatsReader capability captureChanges already uses, since Trees'
|
||||
// core contract has no head-SHA method. A Trees implementation without that
|
||||
// capability (a test fake, most commonly) snapshots successfully but
|
||||
// reports an empty SHA; postPRReview already treats an empty headSHA as "not
|
||||
// measured" rather than fails the run over it, matching captureChanges'
|
||||
// same convention.
|
||||
func (s *localPRSource) Snapshot(ctx context.Context, _, _ string, _ int, destDir string) (string, error) {
|
||||
if err := s.tc.Trees.Snapshot(ctx, s.tc.Dir, destDir); err != nil {
|
||||
return "", err
|
||||
}
|
||||
reader, ok := s.tc.Trees.(changeStatsReader)
|
||||
if !ok {
|
||||
return "", nil
|
||||
}
|
||||
stats, err := reader.ChangedFileStats(ctx, s.tc.Dir, s.tc.Repo.BaseBranch())
|
||||
if err != nil {
|
||||
return "", nil //nolint:nilerr // head SHA is best-effort provenance, not a required result
|
||||
}
|
||||
return stats.HeadSHA, nil
|
||||
}
|
||||
|
||||
// parkDetailBytes bounds how much of a park Detail the rendered blocking
|
||||
// findings occupy, matching the deleted adversarial-self-review's
|
||||
// reviewDetailBytes (diff_rules.go's own park Detail cap headroom comment
|
||||
// still points at that constant by name; it named this cap, not a specific
|
||||
// symbol).
|
||||
const parkDetailBytes = 4000
|
||||
|
||||
// unchallengedBlockingFindings returns the scored findings the merge gate
|
||||
// marked blocking that the adversary did not challenge -- confirmed or never
|
||||
// reviewed by the adversary both count, since only an explicit challenge
|
||||
// argues the finding away (docs/prds/pr-review-agent.md, Triggers: "archie's
|
||||
// own PRs").
|
||||
func unchallengedBlockingFindings(scored []prreview.ScoredFinding) []prreview.ScoredFinding {
|
||||
var out []prreview.ScoredFinding
|
||||
for _, f := range scored {
|
||||
if f.Blocking && f.Adversary != prreview.AdversaryChallenged {
|
||||
out = append(out, f)
|
||||
}
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// renderBlockingFindingsDetail formats the unchallenged blocking findings
|
||||
// that parked the task, in the same shape the deleted adversarial-self-
|
||||
// review's renderReviewDetail used, so a park reason here reads the way an
|
||||
// operator following this repository already expects.
|
||||
func renderBlockingFindingsDetail(findings []prreview.ScoredFinding) string {
|
||||
var b strings.Builder
|
||||
b.WriteString("pr-review found unchallenged blocking defects:\n")
|
||||
for _, f := range findings {
|
||||
fmt.Fprintf(&b, "- %s:%d %s (%s)\n", f.File, f.LineStart, f.Title, f.Body)
|
||||
}
|
||||
return clip(b.String(), parkDetailBytes)
|
||||
}
|
||||
|
||||
// StagePRReviewAndOpenPR runs the pr-review pipeline's decision phases (1-8)
|
||||
// against the task's own uncommitted change, then opens the PR body builds
|
||||
// and posts the pipeline's advisory findings against it -- archie's own PRs
|
||||
// trigger (docs/prds/pr-review-agent.md, Triggers). An unchallenged blocking
|
||||
// finding parks the task with the findings instead of opening a PR; a
|
||||
// pipeline stage failure parks the task the same way any other stage failure
|
||||
// does. Once the PR is open, posting failure is logged and does not revert
|
||||
// the outcome the PR's existence already earned -- the same best-effort
|
||||
// convention the deleted StagePostReviewComments followed.
|
||||
//
|
||||
// This must run every decision stage from inside one Stage.Run body rather
|
||||
// than as separate Stage entries in the workflow's list: the engine ends a
|
||||
// run the instant any stage sets tc.Outcome (workflow.go's Run loop), so a
|
||||
// stage that opens the PR could never be reached if the review phases were
|
||||
// spliced in as their own preceding stages -- OpenPR's own doc comment names
|
||||
// this exact pattern ("call this and then do so in the same stage").
|
||||
func StagePRReviewAndOpenPR(body func(*TaskContext) string) Stage {
|
||||
return stagePRReviewAndOpenPR(prReviewDecisionStages(), body)
|
||||
}
|
||||
|
||||
// stagePRReviewAndOpenPR is StagePRReviewAndOpenPR's implementation, taking
|
||||
// the decision-stage list as a parameter so tests can substitute a small
|
||||
// fake pipeline instead of scripting all nine real phases' agent calls.
|
||||
func stagePRReviewAndOpenPR(decisionStages []Stage, body func(*TaskContext) string) Stage {
|
||||
return Stage{Name: "pr-review-and-open-pr", Run: func(ctx context.Context, tc *TaskContext) error {
|
||||
if tc.PRSource == nil {
|
||||
tc.PRSource = &localPRSource{tc: tc}
|
||||
}
|
||||
for _, stage := range decisionStages {
|
||||
if err := stage.Run(ctx, tc); err != nil {
|
||||
return fmt.Errorf("pr-review (%s): %w", stage.Name, err)
|
||||
}
|
||||
if tc.Outcome.Status != "" {
|
||||
// A decision stage ended the run itself (operator-approval's
|
||||
// waiting_human, or a decision stage's own park). Honor it
|
||||
// rather than proceed to open a PR the operator has not
|
||||
// cleared yet.
|
||||
return nil
|
||||
}
|
||||
}
|
||||
|
||||
if blocking := unchallengedBlockingFindings(tc.prReview.scored); len(blocking) > 0 {
|
||||
tc.Outcome = Outcome{Status: StatusParked, Detail: renderBlockingFindingsDetail(blocking)}
|
||||
return nil
|
||||
}
|
||||
|
||||
if err := OpenPR(ctx, tc, body(tc)); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
if _, err := runPROutputPhase(ctx, tc); err != nil {
|
||||
tc.Log.Error("pr-review findings not posted; the pull request is open regardless", "err", err,
|
||||
"pr", tc.Task.PRNumber)
|
||||
}
|
||||
return nil
|
||||
}}
|
||||
}
|
||||
@@ -0,0 +1,176 @@
|
||||
package workflow
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"log/slog"
|
||||
"testing"
|
||||
|
||||
"github.com/samcharles93/archie-core/internal/config"
|
||||
"github.com/samcharles93/archie-core/internal/domain/workflow/prreview"
|
||||
)
|
||||
|
||||
var errBoom = errors.New("boom")
|
||||
|
||||
// fakeDecisionStage is a minimal decision-stage substitute: it runs fn, so a
|
||||
// test can seed tc.prReview / tc.Outcome / return an error without scripting
|
||||
// a full nine-phase pipeline's agent calls.
|
||||
func fakeDecisionStage(name string, fn func(tc *TaskContext) error) Stage {
|
||||
return Stage{Name: name, Run: func(_ context.Context, tc *TaskContext) error { return fn(tc) }}
|
||||
}
|
||||
|
||||
func ownPRTaskContext(t *testing.T) (*TaskContext, *fakeForge) {
|
||||
t.Helper()
|
||||
forge := &fakeForge{}
|
||||
tc := &TaskContext{
|
||||
Task: &Task{ID: 1, Owner: "acme", Repo: "widgets", Title: "add a thing", Body: "does the thing"},
|
||||
Repo: configRepoWithBase(t),
|
||||
Forge: forge,
|
||||
Trees: &fakeTrees{diff: smallDiff},
|
||||
Store: &threadSafeStore{},
|
||||
Log: slog.New(slog.DiscardHandler),
|
||||
}
|
||||
return tc, forge
|
||||
}
|
||||
|
||||
func TestStagePRReviewAndOpenPRParksOnUnchallengedBlockingFinding(t *testing.T) {
|
||||
tc, forge := ownPRTaskContext(t)
|
||||
seed := fakeDecisionStage("seed", func(tc *TaskContext) error {
|
||||
tc.prReview = &prReviewState{scored: []prreview.ScoredFinding{
|
||||
{Title: "sql injection", File: "a.go", LineStart: 3, Blocking: true},
|
||||
}}
|
||||
return nil
|
||||
})
|
||||
|
||||
stage := stagePRReviewAndOpenPR([]Stage{seed}, func(*TaskContext) string { return "body" })
|
||||
if err := stage.Run(context.Background(), tc); err != nil {
|
||||
t.Fatalf("stage: %v", err)
|
||||
}
|
||||
if tc.Outcome.Status != StatusParked {
|
||||
t.Fatalf("outcome = %q, want %q", tc.Outcome.Status, StatusParked)
|
||||
}
|
||||
if len(forge.calls) != 0 {
|
||||
t.Errorf("forge calls = %v, want none: an unchallenged blocking finding must not open a PR", forge.calls)
|
||||
}
|
||||
}
|
||||
|
||||
func TestStagePRReviewAndOpenPRProceedsWhenBlockingFindingWasChallenged(t *testing.T) {
|
||||
tc, forge := ownPRTaskContext(t)
|
||||
seed := fakeDecisionStage("seed", func(tc *TaskContext) error {
|
||||
tc.prReview = &prReviewState{scored: []prreview.ScoredFinding{
|
||||
{Title: "false alarm", File: "a.go", LineStart: 3, Blocking: true, Adversary: prreview.AdversaryChallenged},
|
||||
}}
|
||||
return nil
|
||||
})
|
||||
|
||||
stage := stagePRReviewAndOpenPR([]Stage{seed}, func(*TaskContext) string { return "body" })
|
||||
if err := stage.Run(context.Background(), tc); err != nil {
|
||||
t.Fatalf("stage: %v", err)
|
||||
}
|
||||
if tc.Outcome.Status != StatusPROpen {
|
||||
t.Fatalf("outcome = %q, want %q (a challenged blocking finding must not park)", tc.Outcome.Status, StatusPROpen)
|
||||
}
|
||||
if forge.prNumber == 0 {
|
||||
t.Error("no PR was opened")
|
||||
}
|
||||
}
|
||||
|
||||
func TestStagePRReviewAndOpenPROpensAndPostsCleanReview(t *testing.T) {
|
||||
tc, forge := ownPRTaskContext(t)
|
||||
seed := fakeDecisionStage("seed", func(tc *TaskContext) error {
|
||||
tc.prReview = &prReviewState{scored: []prreview.ScoredFinding{
|
||||
{Title: "nit", File: "a.go", LineStart: 3, Body: "tidy this up"},
|
||||
}}
|
||||
return nil
|
||||
})
|
||||
|
||||
stage := stagePRReviewAndOpenPR([]Stage{seed}, func(*TaskContext) string { return "the pr body" })
|
||||
if err := stage.Run(context.Background(), tc); err != nil {
|
||||
t.Fatalf("stage: %v", err)
|
||||
}
|
||||
if tc.Outcome.Status != StatusPROpen {
|
||||
t.Fatalf("outcome = %q, want %q", tc.Outcome.Status, StatusPROpen)
|
||||
}
|
||||
if forge.prNumber != 1 {
|
||||
t.Fatalf("PR number = %d, want 1", forge.prNumber)
|
||||
}
|
||||
if forge.reviewCalls != 1 {
|
||||
t.Fatalf("review comment calls = %d, want 1: the advisory finding must post after the PR opens", forge.reviewCalls)
|
||||
}
|
||||
if forge.reviewNumber != forge.prNumber {
|
||||
t.Errorf("review posted against PR %d, want %d", forge.reviewNumber, forge.prNumber)
|
||||
}
|
||||
}
|
||||
|
||||
func TestStagePRReviewAndOpenPRHonorsAnEarlyOutcomeFromADecisionStage(t *testing.T) {
|
||||
tc, forge := ownPRTaskContext(t)
|
||||
seed := fakeDecisionStage("seed", func(tc *TaskContext) error {
|
||||
tc.Outcome = Outcome{Status: StatusWaitingHuman, Detail: "awaiting operator review"}
|
||||
return nil
|
||||
})
|
||||
|
||||
stage := stagePRReviewAndOpenPR([]Stage{seed}, func(*TaskContext) string {
|
||||
t.Fatal("PR body should never be built: a decision stage already ended the run")
|
||||
return ""
|
||||
})
|
||||
if err := stage.Run(context.Background(), tc); err != nil {
|
||||
t.Fatalf("stage: %v", err)
|
||||
}
|
||||
if tc.Outcome.Status != StatusWaitingHuman {
|
||||
t.Fatalf("outcome = %q, want %q (the decision stage's own outcome must survive)", tc.Outcome.Status, StatusWaitingHuman)
|
||||
}
|
||||
if len(forge.calls) != 0 {
|
||||
t.Errorf("forge calls = %v, want none", forge.calls)
|
||||
}
|
||||
}
|
||||
|
||||
func TestStagePRReviewAndOpenPRParksOnADecisionStageFailure(t *testing.T) {
|
||||
tc, forge := ownPRTaskContext(t)
|
||||
seed := fakeDecisionStage("seed", func(*TaskContext) error {
|
||||
return errBoom
|
||||
})
|
||||
|
||||
stage := stagePRReviewAndOpenPR([]Stage{seed}, func(*TaskContext) string { return "body" })
|
||||
if err := stage.Run(context.Background(), tc); err == nil {
|
||||
t.Fatal("want an error: a failed decision stage must fail this stage (the engine parks on a stage error)")
|
||||
}
|
||||
if len(forge.calls) != 0 {
|
||||
t.Errorf("forge calls = %v, want none", forge.calls)
|
||||
}
|
||||
}
|
||||
|
||||
func TestLocalPRSourceReadsFromTheTasksOwnWorktree(t *testing.T) {
|
||||
tc, _ := ownPRTaskContext(t)
|
||||
src := &localPRSource{tc: tc}
|
||||
|
||||
meta, err := src.Metadata(context.Background(), "", "", 0)
|
||||
if err != nil {
|
||||
t.Fatalf("Metadata: %v", err)
|
||||
}
|
||||
if meta.Title != tc.Task.Title || meta.Body != tc.Task.Body {
|
||||
t.Errorf("Metadata = %+v, want the task's own title/body", meta)
|
||||
}
|
||||
|
||||
diff, err := src.Diff(context.Background(), "", "", 0)
|
||||
if err != nil {
|
||||
t.Fatalf("Diff: %v", err)
|
||||
}
|
||||
if diff != smallDiff {
|
||||
t.Errorf("Diff = %q, want the worktree's own diff", diff)
|
||||
}
|
||||
|
||||
headSHA, err := src.Snapshot(context.Background(), "", "", 0, t.TempDir())
|
||||
if err != nil {
|
||||
t.Fatalf("Snapshot: %v", err)
|
||||
}
|
||||
if headSHA != "" {
|
||||
t.Errorf("headSHA = %q, want empty: fakeTrees does not implement changeStatsReader", headSHA)
|
||||
}
|
||||
}
|
||||
|
||||
// configRepoWithBase builds a minimal config.Repo whose BaseBranch()
|
||||
// resolves without a live git call.
|
||||
func configRepoWithBase(t *testing.T) config.Repo {
|
||||
t.Helper()
|
||||
return config.Repo{Base: "main"}
|
||||
}
|
||||
@@ -0,0 +1,849 @@
|
||||
package workflow
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"os"
|
||||
"slices"
|
||||
"strings"
|
||||
"sync"
|
||||
"sync/atomic"
|
||||
"time"
|
||||
|
||||
"github.com/samcharles93/archie-core/internal/agentexec"
|
||||
"github.com/samcharles93/archie-core/internal/domain/workflow/prreview"
|
||||
"github.com/samcharles93/archie-core/internal/domain/workflow/task"
|
||||
"github.com/samcharles93/archie-core/internal/taskstate"
|
||||
)
|
||||
|
||||
// PRSource fetches a pull request under review: its descriptive content, its
|
||||
// diff, and a read-only, .git-free snapshot of its head. It is Trees'
|
||||
// counterpart for the pr-review workflow -- Trees reaches the task's own
|
||||
// worktree, which the pipeline must never read (docs/prds/pr-review-agent.md,
|
||||
// Isolation); PRSource reaches the arbitrary pull request named by
|
||||
// Task.Owner/Repo/PRNumber instead, and carries no push credential.
|
||||
//
|
||||
// The real forge-backed implementation belongs to whichever bead wires
|
||||
// pr-review's triggers (archie-core-afbk.7); this bead builds and tests the
|
||||
// Stage wiring against a fake.
|
||||
type PRSource interface {
|
||||
// Metadata returns the pull request's title, body and commit messages --
|
||||
// phase 1's "PR metadata" and phase 3's hallucination-check dimension's
|
||||
// "what the PR claims".
|
||||
Metadata(ctx context.Context, owner, repo string, number int) (PRMetadata, error)
|
||||
// Diff returns the pull request's unified diff against its base branch.
|
||||
Diff(ctx context.Context, owner, repo string, number int) (string, error)
|
||||
// Snapshot materializes the pull request head's tracked files into
|
||||
// destDir with no .git directory present, and returns the head commit
|
||||
// SHA those files (and the diff's line numbers) were measured on.
|
||||
Snapshot(ctx context.Context, owner, repo string, number int, destDir string) (headSHA string, err error)
|
||||
}
|
||||
|
||||
// PRMetadata is a pull request's descriptive content.
|
||||
type PRMetadata struct {
|
||||
Title string
|
||||
Body string
|
||||
Commits []string // commit messages, oldest first
|
||||
}
|
||||
|
||||
// prReviewState is the pr-review workflow's cross-stage scratch state: each
|
||||
// phase's output for the phases after it to consume. Unexported, following
|
||||
// the pattern the feasibility workflow's decision field set.
|
||||
type prReviewState struct {
|
||||
metadata PRMetadata
|
||||
diff string
|
||||
files []prreview.FileChange
|
||||
stats prreview.DiffStats
|
||||
depth prreview.Depth
|
||||
aiGenerated float64
|
||||
|
||||
snapshotDir string
|
||||
headSHA string
|
||||
blastRadius []string
|
||||
// clusters and highExposure are phase 6's coverage-gate inputs, computed
|
||||
// once in anatomy alongside blastRadius: clusters groups changed files by
|
||||
// directory, highExposure is the blastRadius files more than one changed
|
||||
// file reaches.
|
||||
clusters []prreview.Cluster
|
||||
highExposure []string
|
||||
|
||||
narrative string
|
||||
dimensions []prreview.Dimension
|
||||
findings []prreview.Finding
|
||||
scored []prreview.ScoredFinding
|
||||
reviewerFailures atomic.Int64
|
||||
|
||||
// findingsMu guards findings during phase 6, where the coverage gate and
|
||||
// consistency verification append to it from two goroutines running in
|
||||
// parallel (docs/prds/pr-review-agent.md, phase 6). Every other phase
|
||||
// touches findings from a single goroutine and does not need it.
|
||||
findingsMu sync.Mutex
|
||||
|
||||
// runStart is when phase 1 started, the origin every budget-share check
|
||||
// measures elapsed wall-clock against.
|
||||
runStart time.Time
|
||||
// tokensSpent is the run's cumulative token spend so far, across every
|
||||
// agent call any phase has made. It is an atomic because phases 3, 4 and
|
||||
// 9 add to it from concurrent goroutines.
|
||||
tokensSpent atomic.Int64
|
||||
// skippedPhases names every phase a budget check skipped, in the order
|
||||
// they were skipped, so phase 9's output can report them (docs/prds/
|
||||
// pr-review-agent.md, Budget: "The posted review names every skipped
|
||||
// phase, so an exhausted run never reads as a clean one"). Every phase
|
||||
// runs sequentially in the workflow engine's stage loop, so appends here
|
||||
// need no lock.
|
||||
skippedPhases []string
|
||||
}
|
||||
|
||||
// prReviewTotalBudget bounds the whole pr-review run's cost (tokens, the
|
||||
// only spend the agent runtime accounts for) and wall-clock, split evenly
|
||||
// across prReviewBudgetPhases in pipeline order (docs/prds/pr-review-agent.md,
|
||||
// Budget section). Sized generously: exhausting it is meant to catch a run
|
||||
// that is genuinely stuck or unexpectedly expensive, not a typical one.
|
||||
var prReviewTotalBudget = prreview.Budget{MaxTokens: 2_000_000, WallClock: 30 * time.Minute}
|
||||
|
||||
// prReviewBudgetPhases are the budget-tracked phases, in pipeline order.
|
||||
// Intake is not tracked: it must always run, since the depth, the AI-score
|
||||
// and the budget's own runStart all come from it, and every later phase and
|
||||
// this list itself depend on its output existing.
|
||||
var prReviewBudgetPhases = []string{"anatomy", "lenses", "review", "precision-gate", "verification", "coverage-consistency", "merge-gate", "output"}
|
||||
|
||||
// budgetExhausted reports whether phase has already spent its cumulative
|
||||
// share of the run's total budget, checked once before that phase's
|
||||
// expensive (agent-call) work starts. A phase absent from
|
||||
// prReviewBudgetPhases is a bug in that list, not a runtime condition, and
|
||||
// is never treated as exhausted.
|
||||
func budgetExhausted(tc *TaskContext, phase string) bool {
|
||||
idx := slices.Index(prReviewBudgetPhases, phase)
|
||||
if idx < 0 {
|
||||
return false
|
||||
}
|
||||
// A zero runStart means the run's clock was never started (a stage
|
||||
// invoked directly against a hand-built prReviewState, as the stage unit
|
||||
// tests do) rather than a run that has been going since the epoch.
|
||||
var elapsed time.Duration
|
||||
if !tc.prReview.runStart.IsZero() {
|
||||
elapsed = time.Since(tc.prReview.runStart)
|
||||
}
|
||||
return prReviewTotalBudget.PhaseExhausted(idx+1, len(prReviewBudgetPhases),
|
||||
int(tc.prReview.tokensSpent.Load()), elapsed)
|
||||
}
|
||||
|
||||
// skipPhase records phase as budget-skipped.
|
||||
func skipPhase(tc *TaskContext, phase string) {
|
||||
tc.prReview.skippedPhases = append(tc.prReview.skippedPhases, phase)
|
||||
}
|
||||
|
||||
// prReviewConcurrency bounds phases 3 (lenses), 4 (reviewers) and 9 (polish),
|
||||
// all "in parallel, at most 8 at a time" per the PRD except phase 3, which
|
||||
// never has more than three lenses to begin with.
|
||||
const prReviewConcurrency = 8
|
||||
|
||||
// PRReview is the pull request reviewer: a fixed pipeline, not a single
|
||||
// agent (docs/prds/pr-review-agent.md). archie-core-afbk.3 wired phases 1, 2,
|
||||
// 3, 4, 7 and 9; archie-core-afbk.4 added 5 and 6 (verification,
|
||||
// coverage/consistency); archie-core-afbk.5 added phase 8 (the merge gate)
|
||||
// and the budget shares/skipped-phase reporting that run throughout;
|
||||
// archie-core-afbk.8 adds the precision dial (between review and
|
||||
// verification) and the operator-approval gate (between synthesis and the
|
||||
// merge gate), both off by default and each gated by its own config flag.
|
||||
func PRReview() Workflow {
|
||||
return Workflow{
|
||||
Name: "pr-review",
|
||||
// pr_number is required so a binding (watched-repos) or a chat task
|
||||
// (the operator trigger) can only target pr-review by actually
|
||||
// naming a pull request; stagePRIntake reads it from Task.Inputs
|
||||
// when Task.PRNumber is unset (archie's-own-PRs sets PRNumber
|
||||
// directly and never goes through binding validation, so this
|
||||
// declaration does not affect that path).
|
||||
Interface: task.WorkflowInterface{
|
||||
Inputs: map[string]task.InputSpec{
|
||||
"pr_number": {Type: "number", Required: true},
|
||||
"depth": {Type: "string"},
|
||||
},
|
||||
},
|
||||
Stages: append(prReviewDecisionStages(), stagePROutput()),
|
||||
}
|
||||
}
|
||||
|
||||
// prReviewDecisionStages is phases 1 through 8: everything up to and
|
||||
// including the merge gate's blocking/advisory verdict, but before phase 9
|
||||
// posts anything. Split out from PRReview so the implement workflow can
|
||||
// splice it in before StageOpenPR (archie-core-afbk.7's "archie's own PRs"
|
||||
// trigger): the decision of whether an unchallenged blocking finding parks
|
||||
// the task must happen before a PR exists to post to, while phase 9's
|
||||
// posting must happen after, once a PR number exists to anchor comments to.
|
||||
func prReviewDecisionStages() []Stage {
|
||||
return []Stage{
|
||||
stagePRIntake(),
|
||||
stagePRAnatomy(),
|
||||
stagePRLenses(),
|
||||
stagePRReview(),
|
||||
stagePRPrecisionGate(),
|
||||
stagePRVerification(),
|
||||
stagePRCoverageConsistency(),
|
||||
stagePRSynthesis(),
|
||||
stagePROperatorApproval(),
|
||||
stagePRMergeGate(),
|
||||
}
|
||||
}
|
||||
|
||||
// stagePRIntake is pipeline phase 1: PR metadata and diff statistics, the
|
||||
// depth they classify, and the machine-written confidence phase 3's
|
||||
// hallucination-check dimension and phase 7's scoring both read.
|
||||
func stagePRIntake() Stage {
|
||||
return Stage{Name: "intake", Run: func(ctx context.Context, tc *TaskContext) error {
|
||||
if tc.PRSource == nil {
|
||||
return fmt.Errorf("pr-review: no PRSource configured")
|
||||
}
|
||||
// A watched-repositories binding or the operator chat trigger
|
||||
// assigns the pull request through the declared pr_number input,
|
||||
// not PRNumber directly; the daemon's container-acquisition path
|
||||
// already resolves this before dispatch, but an in-process or
|
||||
// subprocess run that bypasses it still needs it resolved here.
|
||||
// Archie's own PRs (Task.Workflow == "implement", embedding this
|
||||
// stage through StagePRReviewAndOpenPR) legitimately has no PR
|
||||
// number at all -- localPRSource ignores it, since the PR does not
|
||||
// exist yet -- so the missing-number error only applies to the
|
||||
// standalone pr-review workflow, which always needs a real,
|
||||
// externally-fetchable pull request to review.
|
||||
if tc.Task.PRNumber == 0 {
|
||||
tc.Task.PRNumber = tc.Task.EffectivePRNumber()
|
||||
}
|
||||
if tc.Task.PRNumber == 0 && tc.Task.Workflow == "pr-review" {
|
||||
return fmt.Errorf("pr-review: no pull request number (set directly or via the pr_number input)")
|
||||
}
|
||||
var explicit prreview.Depth
|
||||
if value, supplied := tc.Task.Inputs["depth"]; supplied {
|
||||
depth, ok := value.(string)
|
||||
if !ok || (depth != "quick" && depth != "standard" && depth != "deep") {
|
||||
return fmt.Errorf("pr-review: depth must be quick, standard or deep")
|
||||
}
|
||||
explicit = prreview.Depth(depth)
|
||||
}
|
||||
meta, err := tc.PRSource.Metadata(ctx, tc.Task.Owner, tc.Task.Repo, tc.Task.PRNumber)
|
||||
if err != nil {
|
||||
return fmt.Errorf("fetch pull request metadata: %w", err)
|
||||
}
|
||||
diff, err := tc.PRSource.Diff(ctx, tc.Task.Owner, tc.Task.Repo, tc.Task.PRNumber)
|
||||
if err != nil {
|
||||
return fmt.Errorf("fetch pull request diff: %w", err)
|
||||
}
|
||||
files := prreview.ParseDiff(diff)
|
||||
stats := prreview.SummarizeFiles(files)
|
||||
depth := prreview.ResolveDepth(stats.TotalAdditions+stats.TotalDeletions, explicit)
|
||||
|
||||
// tc.prReview must exist before scoreAIGenerated runs: every
|
||||
// pr-review agent call, including this one, records its token spend
|
||||
// against tc.prReview.tokensSpent.
|
||||
tc.prReview = &prReviewState{
|
||||
metadata: meta, diff: diff, files: files, stats: stats,
|
||||
depth: depth, runStart: time.Now(),
|
||||
}
|
||||
|
||||
aiGenerated, err := scoreAIGenerated(ctx, tc, meta)
|
||||
if err != nil {
|
||||
return fmt.Errorf("score likely machine-written PR: %w", err)
|
||||
}
|
||||
tc.prReview.aiGenerated = aiGenerated
|
||||
return nil
|
||||
}}
|
||||
}
|
||||
|
||||
// scoreAIGenerated runs phase 1's classification-role call: how confident an
|
||||
// agent is that the PR's own description reads as machine-written, which
|
||||
// phase 3's hallucination-check dimension and phase 7's scoring both use.
|
||||
func scoreAIGenerated(ctx context.Context, tc *TaskContext, meta PRMetadata) (float64, error) {
|
||||
// This call reads no files -- only the metadata already in its mission --
|
||||
// so it gets its own empty scratch directory rather than the pull
|
||||
// request's snapshot, which does not exist yet at phase 1.
|
||||
scratch, err := os.MkdirTemp("", "pr-review-intake-*")
|
||||
if err != nil {
|
||||
return 0, fmt.Errorf("create intake scratch directory: %w", err)
|
||||
}
|
||||
defer os.RemoveAll(scratch)
|
||||
|
||||
params := json.RawMessage(`{
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"confidence": {"type": "number", "description": "0 = clearly written by the human author, 1 = clearly machine-generated."},
|
||||
"reasons": {"type": "string", "description": "What in the title, body or commit messages drove the confidence."}
|
||||
},
|
||||
"required": ["confidence"]
|
||||
}`)
|
||||
mission := fmt.Sprintf(
|
||||
"Judge whether this pull request's own description reads as written by an AI "+
|
||||
"coding assistant rather than the human who opened it: generic boilerplate, a "+
|
||||
"summary that oversells the diff, or phrasing typical of an LLM's own commit "+
|
||||
"or PR style. This is about the PR's description, not the code.\n\n"+
|
||||
"Title: %s\n\nDescription:\n%s\n\nCommit messages:\n%s\n\n"+
|
||||
"Call score_ai_generated exactly once with your confidence, then call finish "+
|
||||
"with status \"passed\".",
|
||||
meta.Title, meta.Body, strings.Join(meta.Commits, "\n"),
|
||||
)
|
||||
res, err := runPRReviewAgent(ctx, tc, scratch, "intake-ai-score", "classification", mission, 6, []agentexec.CaptureTool{{
|
||||
Name: "score_ai_generated", Description: "Record the machine-written confidence. Call exactly once, before finish.",
|
||||
Parameters: params, RequiredFields: []string{"confidence"}, MaxCalls: 1,
|
||||
}})
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
calls := res.Captures["score_ai_generated"]
|
||||
if len(calls) != 1 {
|
||||
return 0, fmt.Errorf("intake-ai-score called score_ai_generated %d times (want exactly once)", len(calls))
|
||||
}
|
||||
var captured struct {
|
||||
Confidence float64 `json:"confidence"`
|
||||
}
|
||||
if err := json.Unmarshal(calls[0], &captured); err != nil {
|
||||
return 0, fmt.Errorf("decode machine-written confidence: %w", err)
|
||||
}
|
||||
return captured.Confidence, nil
|
||||
}
|
||||
|
||||
// stagePRAnatomy is pipeline phase 2: the read-only snapshot every later
|
||||
// phase reads from, the deterministic clustering and blast radius, and one
|
||||
// agent's narrative of the change.
|
||||
func stagePRAnatomy() Stage {
|
||||
return Stage{Name: "anatomy", Run: func(ctx context.Context, tc *TaskContext) error {
|
||||
dir, err := os.MkdirTemp("", fmt.Sprintf("pr-review-snapshot-task%d-*", tc.Task.ID))
|
||||
if err != nil {
|
||||
return fmt.Errorf("create pull request snapshot directory: %w", err)
|
||||
}
|
||||
tc.prReview.snapshotDir = dir
|
||||
headSHA, err := tc.PRSource.Snapshot(ctx, tc.Task.Owner, tc.Task.Repo, tc.Task.PRNumber, dir)
|
||||
if err != nil {
|
||||
return fmt.Errorf("snapshot pull request head: %w", err)
|
||||
}
|
||||
tc.prReview.headSHA = headSHA
|
||||
|
||||
changed := make([]string, 0, len(tc.prReview.files))
|
||||
for _, f := range tc.prReview.files {
|
||||
changed = append(changed, f.Path)
|
||||
}
|
||||
blast, err := prreview.BlastRadius(os.DirFS(dir), changed)
|
||||
if err != nil {
|
||||
return fmt.Errorf("compute blast radius: %w", err)
|
||||
}
|
||||
tc.prReview.blastRadius = blast
|
||||
tc.prReview.clusters = prreview.ClusterFiles(tc.prReview.files)
|
||||
exposure, err := prreview.ExposureCounts(os.DirFS(dir), changed)
|
||||
if err != nil {
|
||||
return fmt.Errorf("compute exposure counts: %w", err)
|
||||
}
|
||||
tc.prReview.highExposure = prreview.HighExposureFiles(exposure)
|
||||
|
||||
mission := fmt.Sprintf(
|
||||
"Read this pull request's diff and enough of the surrounding repository to "+
|
||||
"write a short narrative of the change: what it does, the riskiest surfaces "+
|
||||
"it touches, any change that does not fit the PR's own description, and any "+
|
||||
"gap between what the description claims and what the diff carries out.\n\n"+
|
||||
"Title: %s\n\nDescription:\n%s\n\nDiff:\n%s\n\n"+
|
||||
"Call finish with status \"passed\" and your narrative as the summary.",
|
||||
tc.prReview.metadata.Title, tc.prReview.metadata.Body, clip(tc.prReview.diff, 60000),
|
||||
)
|
||||
if budgetExhausted(tc, "anatomy") {
|
||||
skipPhase(tc, "anatomy")
|
||||
return nil
|
||||
}
|
||||
res, err := runPRReviewAgent(ctx, tc, tc.prReview.snapshotDir, "anatomy", "review", mission, 15, nil)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
tc.prReview.narrative = res.Summary
|
||||
return nil
|
||||
}}
|
||||
}
|
||||
|
||||
// prReviewLenses are phase 3's three fixed lenses. Each proposes review
|
||||
// dimensions from a different angle; code merges, dedups and caps the result.
|
||||
var prReviewLenses = []struct {
|
||||
name string
|
||||
prompt string
|
||||
}{
|
||||
{name: "behaviour", prompt: "where the old and new code diverge for any input: API contracts, concurrency and state, security, error handling, data flow"},
|
||||
{name: "mechanics", prompt: "whether the code runs at the language and framework level: types, signatures and every caller, decorators and middleware, framework contracts, imports"},
|
||||
{name: "fit", prompt: "patterns, complexity, abstraction, test adequacy, documentation that no longer matches, dependencies, migration completeness"},
|
||||
}
|
||||
|
||||
// stagePRLenses is pipeline phase 3: three lens agents proposing review
|
||||
// dimensions in parallel, merged with the hallucination-check dimension (for
|
||||
// a likely machine-written PR) as one more candidate, deduplicated and
|
||||
// capped by code. The hallucination dimension competes for the same cap as
|
||||
// everything else -- "at most N dimensions" is a total, not a total plus one
|
||||
// -- so it is added before MergeDimensions cuts the tail, never after.
|
||||
func stagePRLenses() Stage {
|
||||
return Stage{Name: "lenses", Run: func(ctx context.Context, tc *TaskContext) error {
|
||||
if budgetExhausted(tc, "lenses") {
|
||||
skipPhase(tc, "lenses")
|
||||
return nil
|
||||
}
|
||||
outputs := make([][]prreview.Dimension, len(prReviewLenses))
|
||||
errs := make([]error, len(prReviewLenses))
|
||||
forEachBounded(len(prReviewLenses), len(prReviewLenses), func(i int) {
|
||||
outputs[i], errs[i] = runLens(ctx, tc, prReviewLenses[i].name, prReviewLenses[i].prompt)
|
||||
})
|
||||
for i, err := range errs {
|
||||
if err != nil {
|
||||
return fmt.Errorf("lens %s: %w", prReviewLenses[i].name, err)
|
||||
}
|
||||
}
|
||||
candidates := outputs
|
||||
if extra := prreview.HallucinationDimension(tc.prReview.aiGenerated); extra != nil {
|
||||
candidates = append(append([][]prreview.Dimension{}, outputs...), []prreview.Dimension{*extra})
|
||||
}
|
||||
tc.prReview.dimensions = prreview.MergeDimensions(candidates, prreview.MaxDimensionsFor(tc.prReview.depth))
|
||||
return nil
|
||||
}}
|
||||
}
|
||||
|
||||
// runLens runs one phase-3 lens call and decodes its proposed dimensions.
|
||||
func runLens(ctx context.Context, tc *TaskContext, name, angle string) ([]prreview.Dimension, error) {
|
||||
params := json.RawMessage(`{
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"dimensions": {
|
||||
"type": "array",
|
||||
"items": {
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"name": {"type": "string"},
|
||||
"prompt": {"type": "string", "description": "The reviewer's mission for this dimension, written for this PR."},
|
||||
"target_files": {"type": "array", "items": {"type": "string"}},
|
||||
"context_files": {"type": "array", "items": {"type": "string"}},
|
||||
"priority": {"type": "number"}
|
||||
},
|
||||
"required": ["name", "prompt", "target_files", "priority"]
|
||||
}
|
||||
}
|
||||
},
|
||||
"required": ["dimensions"]
|
||||
}`)
|
||||
mission := fmt.Sprintf(
|
||||
"You are the %s lens over this pull request: %s.\n\n"+
|
||||
"Title: %s\n\nDescription:\n%s\n\nDiff:\n%s\n\n"+
|
||||
"Propose the review dimensions this angle needs, each with a reviewer prompt "+
|
||||
"written for this specific PR, its target files, any context files a reviewer "+
|
||||
"should also read, and a priority (higher runs first when the depth caps how "+
|
||||
"many dimensions survive). Call propose_dimensions exactly once, then call "+
|
||||
"finish with status \"passed\".",
|
||||
name, angle, tc.prReview.metadata.Title, tc.prReview.metadata.Body, clip(tc.prReview.diff, 60000),
|
||||
)
|
||||
res, err := runPRReviewAgent(ctx, tc, tc.prReview.snapshotDir, "lens-"+name, "review", mission, 15, []agentexec.CaptureTool{{
|
||||
Name: "propose_dimensions", Description: "Record this lens's proposed review dimensions. Call exactly once, before finish.",
|
||||
Parameters: params, RequiredFields: []string{"dimensions"}, MaxCalls: 1,
|
||||
}})
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
calls := res.Captures["propose_dimensions"]
|
||||
if len(calls) != 1 {
|
||||
return nil, fmt.Errorf("lens %s called propose_dimensions %d times (want exactly once)", name, len(calls))
|
||||
}
|
||||
var captured struct {
|
||||
Dimensions []struct {
|
||||
Name string `json:"name"`
|
||||
Prompt string `json:"prompt"`
|
||||
TargetFiles []string `json:"target_files"`
|
||||
ContextFiles []string `json:"context_files"`
|
||||
Priority float64 `json:"priority"`
|
||||
} `json:"dimensions"`
|
||||
}
|
||||
if err := json.Unmarshal(calls[0], &captured); err != nil {
|
||||
return nil, fmt.Errorf("decode lens %s dimensions: %w", name, err)
|
||||
}
|
||||
out := make([]prreview.Dimension, len(captured.Dimensions))
|
||||
for i, d := range captured.Dimensions {
|
||||
out[i] = prreview.Dimension{
|
||||
Name: d.Name, Prompt: d.Prompt, TargetFiles: d.TargetFiles,
|
||||
ContextFiles: d.ContextFiles, Priority: d.Priority,
|
||||
}
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
|
||||
// stagePRReview is pipeline phase 4: one reviewer call per dimension, at most
|
||||
// prReviewConcurrency at a time. A reviewer that exhausts its turn cap or
|
||||
// deadline before its terminal tool call records StepFailed -- distinct from
|
||||
// StepSucceeded for a reviewer that completed and found nothing -- so
|
||||
// synthesis and the posted review can tell "unreviewed" from "reviewed-clean"
|
||||
// (docs/prds/pr-review-agent.md, phase 4).
|
||||
func stagePRReview() Stage {
|
||||
return Stage{Name: "review", Run: func(ctx context.Context, tc *TaskContext) error {
|
||||
if budgetExhausted(tc, "review") {
|
||||
skipPhase(tc, "review")
|
||||
return nil
|
||||
}
|
||||
dimensions := tc.prReview.dimensions
|
||||
findings := make([][]prreview.Finding, len(dimensions))
|
||||
forEachBounded(prReviewConcurrency, len(dimensions), func(i int) {
|
||||
findings[i] = runReviewer(ctx, tc, dimensions[i])
|
||||
})
|
||||
var all []prreview.Finding
|
||||
for _, f := range findings {
|
||||
all = append(all, f...)
|
||||
}
|
||||
tc.prReview.findings = all
|
||||
return nil
|
||||
}}
|
||||
}
|
||||
|
||||
// runReviewer runs one phase-4 reviewer call. It never returns a Go error to
|
||||
// its caller: a reviewer that failed, or did not reach its terminal tool
|
||||
// call, contributes no findings -- the same zero-findings shape a reviewer
|
||||
// that ran cleanly and found nothing produces. The two are not confused in
|
||||
// the record, only in this return value: runReviewerAgent gives the first
|
||||
// case a distinct StepFailed child step, so a reader of the execution tree
|
||||
// (not this slice) can tell "unreviewed" from "reviewed-clean" apart.
|
||||
func runReviewer(ctx context.Context, tc *TaskContext, dim prreview.Dimension) []prreview.Finding {
|
||||
mission := fmt.Sprintf(
|
||||
"%s\n\nTarget files: %s\n\nRead the target files (and, if useful, the context "+
|
||||
"files) in the snapshot and report every finding for this dimension. Quote the "+
|
||||
"exact evidence for each finding; do not report anything you cannot point at in "+
|
||||
"the code. Call report_findings exactly once (an empty array is a valid, "+
|
||||
"complete report), then call finish with status \"passed\".",
|
||||
dim.Prompt, strings.Join(dim.TargetFiles, ", "),
|
||||
)
|
||||
name := "reviewer-" + dim.Name
|
||||
res, runErr := runReviewerAgent(ctx, tc, name, mission, []agentexec.CaptureTool{reportFindingsTool})
|
||||
if runErr != nil || res.Status != agentexec.StatusPassed {
|
||||
tc.prReview.reviewerFailures.Add(1)
|
||||
return nil
|
||||
}
|
||||
findings, err := decodeReportedFindings(res.Captures["report_findings"], dim.Name)
|
||||
if err != nil {
|
||||
tc.prReview.reviewerFailures.Add(1)
|
||||
return nil
|
||||
}
|
||||
return findings
|
||||
}
|
||||
|
||||
// reportFindingsSchema is the finding shape every pr-review agent call that
|
||||
// reports findings captures them in: phase 4's reviewer, phase 6's gap
|
||||
// reviewers (which reuse runReviewer directly) and consistency verification.
|
||||
var reportFindingsSchema = json.RawMessage(`{
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"findings": {
|
||||
"type": "array",
|
||||
"items": {
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"file": {"type": "string"},
|
||||
"line_start": {"type": "integer"},
|
||||
"line_end": {"type": "integer"},
|
||||
"severity": {"type": "string", "enum": ["critical", "important", "suggestion", "nitpick"]},
|
||||
"title": {"type": "string"},
|
||||
"body": {"type": "string"},
|
||||
"suggestion": {"type": "string"},
|
||||
"evidence": {"type": "string"},
|
||||
"confidence": {"type": "number"},
|
||||
"tags": {"type": "array", "items": {"type": "string"}}
|
||||
},
|
||||
"required": ["file", "line_start", "severity", "title", "body", "evidence", "confidence"]
|
||||
}
|
||||
}
|
||||
},
|
||||
"required": ["findings"]
|
||||
}`)
|
||||
|
||||
// reportFindingsTool is the capture tool every findings-reporting call
|
||||
// registers, at the shape reportFindingsSchema names.
|
||||
var reportFindingsTool = agentexec.CaptureTool{
|
||||
Name: "report_findings", Description: "Record findings. Call exactly once, before finish.",
|
||||
Parameters: reportFindingsSchema, RequiredFields: []string{"findings"}, MaxCalls: 1,
|
||||
}
|
||||
|
||||
// decodeReportedFindings decodes one report_findings capture into findings
|
||||
// tagged with dimension, shared by every call site reportFindingsSchema
|
||||
// backs. A call that never reported (no captures) or reported malformed JSON
|
||||
// decodes to no findings and an error the caller treats as "nothing to add",
|
||||
// not as a stage failure -- an agent that could not report is not evidence
|
||||
// the code that follows should stop.
|
||||
func decodeReportedFindings(calls []json.RawMessage, dimension string) ([]prreview.Finding, error) {
|
||||
if len(calls) != 1 {
|
||||
return nil, fmt.Errorf("report_findings called %d times (want exactly once)", len(calls))
|
||||
}
|
||||
var captured struct {
|
||||
Findings []struct {
|
||||
File string `json:"file"`
|
||||
LineStart int `json:"line_start"`
|
||||
LineEnd int `json:"line_end"`
|
||||
Severity string `json:"severity"`
|
||||
Title string `json:"title"`
|
||||
Body string `json:"body"`
|
||||
Suggestion string `json:"suggestion"`
|
||||
Evidence string `json:"evidence"`
|
||||
Confidence float64 `json:"confidence"`
|
||||
Tags []string `json:"tags"`
|
||||
} `json:"findings"`
|
||||
}
|
||||
if err := json.Unmarshal(calls[0], &captured); err != nil {
|
||||
return nil, fmt.Errorf("decode findings: %w", err)
|
||||
}
|
||||
out := make([]prreview.Finding, len(captured.Findings))
|
||||
for i, f := range captured.Findings {
|
||||
lineEnd := f.LineEnd
|
||||
if lineEnd == 0 {
|
||||
lineEnd = f.LineStart
|
||||
}
|
||||
out[i] = prreview.Finding{
|
||||
Dimension: dimension, File: f.File, LineStart: f.LineStart, LineEnd: lineEnd,
|
||||
Severity: prreview.Severity(f.Severity), Title: f.Title, Body: f.Body,
|
||||
Suggestion: f.Suggestion, Evidence: f.Evidence, Confidence: f.Confidence, Tags: f.Tags,
|
||||
}
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
|
||||
// prReviewMaxSteps bounds one reviewer's tool-loop iterations. Exhausting it
|
||||
// (or the wall-clock budget) before the terminal finish call is exactly the
|
||||
// "turn cap or deadline" case the phase-4 status distinction exists for.
|
||||
const prReviewMaxSteps = 25
|
||||
|
||||
// runReviewerAgent runs one phase-4 reviewer call and records its child step
|
||||
// with the phase-4 status distinction (docs/prds/pr-review-agent.md, phase
|
||||
// 4): StepFailed for a reviewer that exhausted its turn cap or deadline
|
||||
// before its terminal tool call, StepSucceeded for one that reached it --
|
||||
// whether or not it reported any findings. tc.RunAgentChild does not serve
|
||||
// this: it maps only the Go error, so a "parked" (turn-cap/deadline) result
|
||||
// with no Go error would record StepSucceeded exactly like a clean,
|
||||
// zero-findings run, and the two would be indistinguishable to anything
|
||||
// reading the execution tree afterwards.
|
||||
func runReviewerAgent(
|
||||
ctx context.Context, tc *TaskContext, name, mission string, captureTools []agentexec.CaptureTool,
|
||||
) (agentexec.Result, error) {
|
||||
modelRef := tc.Cfg.Models["review"]
|
||||
if modelRef == "" {
|
||||
return agentexec.Result{}, fmt.Errorf("no model configured for role %q (set [models] in config)", "review")
|
||||
}
|
||||
req := agentexec.Request{
|
||||
Version: agentexec.ProtocolVersion, TaskID: tc.Task.ID, Attempt: tc.Task.Attempt,
|
||||
Stage: name, Workflow: tc.Task.Workflow, Model: modelRef,
|
||||
ContextWindow: modelContextBudget(tc.Cfg, modelRef),
|
||||
Mission: mission, ReadOnly: true,
|
||||
Budget: agentexec.Budget{MaxSteps: prReviewMaxSteps, WallClock: tc.Cfg.Budgets.WallClock.Std()},
|
||||
CaptureTools: captureTools,
|
||||
}
|
||||
stepID, _, err := tc.startChildStep(ctx, task.StepKindAgent, name)
|
||||
if err != nil {
|
||||
return agentexec.Result{}, err
|
||||
}
|
||||
res, runErr := tc.Agent.Run(ctx, tc.prReview.snapshotDir, req, tc.toolCallReporter(name))
|
||||
tc.prReview.tokensSpent.Add(int64(res.TokensUsed))
|
||||
to, detail := taskstate.StepSucceeded, res.Summary
|
||||
switch {
|
||||
case runErr != nil:
|
||||
to, detail = taskstate.StepFailed, runErr.Error()
|
||||
case res.Status != agentexec.StatusPassed:
|
||||
to, detail = taskstate.StepFailed, fmt.Sprintf("reviewer did not complete (%s: %s)", res.Status, res.StopReason)
|
||||
}
|
||||
if ferr := tc.finishChildStep(ctx, stepID, to, detail, int64(res.TokensUsed)); ferr != nil {
|
||||
return res, ferr
|
||||
}
|
||||
tc.prReview.tokensSpent.Add(int64(res.TokensUsed))
|
||||
return res, runErr
|
||||
}
|
||||
|
||||
// stagePRSynthesis is pipeline phase 7, code only: score, drop findings
|
||||
// under their severity's confidence floor, merge duplicates, rank, and cap
|
||||
// at the inline comment limit. Phase 8 (the merge gate) runs after this
|
||||
// stage, so every surviving finding's Blocking flag is still whatever phase
|
||||
// 5/6 left it as (false, since neither sets it) until that stage decides.
|
||||
func stagePRSynthesis() Stage {
|
||||
return Stage{Name: "synthesis", Run: func(_ context.Context, tc *TaskContext) error {
|
||||
scored := prreview.Score(tc.prReview.findings, prreview.ScoreInputs{
|
||||
AIGenerated: tc.prReview.aiGenerated,
|
||||
BlastRadiusFiles: len(tc.prReview.blastRadius),
|
||||
})
|
||||
tc.prReview.scored = prreview.CapInlineComments(scored)
|
||||
return nil
|
||||
}}
|
||||
}
|
||||
|
||||
// stagePROutput is pipeline phase 9: one polish call per comment, at most
|
||||
// prReviewConcurrency at a time, keeping the original wording on failure,
|
||||
// then posting the surviving inline-anchored findings as one forge review.
|
||||
// The polish pass is what a budget-exhausted run skips; posting itself
|
||||
// always runs, since it is the pipeline's only externally visible act and
|
||||
// must report what happened even when every other phase was skipped.
|
||||
//
|
||||
// Used by the standalone pr-review workflow only. Archie's own PRs trigger
|
||||
// (StagePRReviewAndOpenPR, prreview_own_pr.go) needs the same polish-then-post
|
||||
// work but must not overwrite the StatusPROpen outcome OpenPR already set, so
|
||||
// it calls runPROutputPhase directly instead of this Stage.
|
||||
func stagePROutput() Stage {
|
||||
return Stage{Name: "output", Run: func(ctx context.Context, tc *TaskContext) error {
|
||||
detail, err := runPROutputPhase(ctx, tc)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
tc.Outcome = Outcome{Status: StatusCompleted, Detail: detail}
|
||||
return nil
|
||||
}}
|
||||
}
|
||||
|
||||
// runPROutputPhase is phase 9's actual work, factored out of stagePROutput so
|
||||
// a caller that must preserve its own outcome (archie's own PRs, which has
|
||||
// already set StatusPROpen by the time findings are ready to post) can run it
|
||||
// without stagePROutput's unconditional StatusCompleted assignment.
|
||||
func runPROutputPhase(ctx context.Context, tc *TaskContext) (string, error) {
|
||||
if budgetExhausted(tc, "output") {
|
||||
skipPhase(tc, "output")
|
||||
} else {
|
||||
polished := make([]prreview.ScoredFinding, len(tc.prReview.scored))
|
||||
forEachBounded(prReviewConcurrency, len(tc.prReview.scored), func(i int) {
|
||||
polished[i] = polishFinding(ctx, tc, tc.prReview.scored[i])
|
||||
})
|
||||
tc.prReview.scored = polished
|
||||
}
|
||||
|
||||
if err := postPRReview(ctx, tc); err != nil {
|
||||
return "", fmt.Errorf("post review: %w", err)
|
||||
}
|
||||
detail := fmt.Sprintf("posted %d finding(s) as %s", len(tc.prReview.scored), prreview.ReviewEventFor(tc.prReview.scored))
|
||||
if len(tc.prReview.skippedPhases) > 0 {
|
||||
detail += fmt.Sprintf("; skipped: %s", strings.Join(tc.prReview.skippedPhases, ", "))
|
||||
}
|
||||
return detail, nil
|
||||
}
|
||||
|
||||
// polishFinding runs one phase-9 polish call, tightening a finding's wording.
|
||||
// It keeps the original finding unchanged on any failure, per the PRD.
|
||||
func polishFinding(ctx context.Context, tc *TaskContext, f prreview.ScoredFinding) prreview.ScoredFinding {
|
||||
params := json.RawMessage(`{
|
||||
"type": "object",
|
||||
"properties": {"body": {"type": "string", "description": "The tightened finding body."}},
|
||||
"required": ["body"]
|
||||
}`)
|
||||
mission := fmt.Sprintf(
|
||||
"Tighten the wording of this review comment without changing its meaning or "+
|
||||
"dropping any concrete detail (file, line, failure scenario).\n\n"+
|
||||
"Title: %s\nBody: %s\n\nCall polish exactly once with the tightened body, then "+
|
||||
"call finish with status \"passed\".",
|
||||
f.Title, f.Body,
|
||||
)
|
||||
res, err := runPRReviewAgent(ctx, tc, tc.prReview.snapshotDir, "polish", "classification", mission, 4, []agentexec.CaptureTool{{
|
||||
Name: "polish", Description: "Record the tightened comment body. Call exactly once, before finish.",
|
||||
Parameters: params, RequiredFields: []string{"body"}, MaxCalls: 1,
|
||||
}})
|
||||
if err != nil {
|
||||
return f
|
||||
}
|
||||
calls := res.Captures["polish"]
|
||||
if len(calls) != 1 {
|
||||
return f
|
||||
}
|
||||
var captured struct {
|
||||
Body string `json:"body"`
|
||||
}
|
||||
if err := json.Unmarshal(calls[0], &captured); err != nil || captured.Body == "" {
|
||||
return f
|
||||
}
|
||||
f.Body = captured.Body
|
||||
return f
|
||||
}
|
||||
|
||||
// postPRReview posts every line-anchored finding as one forge review. A
|
||||
// finding with no line (LineStart <= 0) is dropped rather than posted
|
||||
// anywhere -- docs/prds/inline-review.md's PR-body fallback list for
|
||||
// whole-file findings, and REQUEST_CHANGES vs COMMENT event submission
|
||||
// (workflow.Forger.CreateReviewComments always posts a COMMENT-state
|
||||
// review; GitHub's implementation does not submit a review object at all,
|
||||
// only per-comment calls, so REQUEST_CHANGES needs a Forger/forge change,
|
||||
// not just a caller change here) are both real gaps this bead does not
|
||||
// close; see the follow-up bead this bead's commit files.
|
||||
func postPRReview(ctx context.Context, tc *TaskContext) error {
|
||||
comments := make([]ReviewComment, 0, len(tc.prReview.scored))
|
||||
for _, f := range tc.prReview.scored {
|
||||
if f.LineStart <= 0 {
|
||||
continue
|
||||
}
|
||||
comments = append(comments, ReviewComment{Path: f.File, Line: f.LineStart, Body: f.Body})
|
||||
}
|
||||
if len(comments) == 0 {
|
||||
return nil
|
||||
}
|
||||
return tc.Forge.CreateReviewComments(
|
||||
ctx, tc.Task.Owner, tc.Task.Repo, tc.Task.PRNumber, tc.prReview.headSHA, comments,
|
||||
)
|
||||
}
|
||||
|
||||
// runPRReviewAgent runs one read-only agent call rooted at workspace -- never
|
||||
// tc.Dir, per Isolation -- and discards the run/park distinction: any
|
||||
// non-passed result is a stage error. Every pr-review phase but the reviewer
|
||||
// fan-out (which must tell "did not finish" from "finished, found nothing"
|
||||
// apart) uses this. Every phase from anatomy onward passes
|
||||
// tc.prReview.snapshotDir; intake's AI-score call runs before that snapshot
|
||||
// exists, so it passes its own scratch workspace instead.
|
||||
func runPRReviewAgent(
|
||||
ctx context.Context, tc *TaskContext, workspace, name, role, mission string, maxSteps int, captureTools []agentexec.CaptureTool,
|
||||
) (agentexec.Result, error) {
|
||||
res, err := runPRReviewAgentRecorded(ctx, tc, workspace, name, role, mission, maxSteps, captureTools)
|
||||
if err != nil {
|
||||
return res, err
|
||||
}
|
||||
if res.Status != agentexec.StatusPassed {
|
||||
detail := res.Detail
|
||||
if detail == "" {
|
||||
detail = res.Summary
|
||||
}
|
||||
return res, fmt.Errorf("%s %s (%s): %s", name, res.Status, res.StopReason, clip(detail, 2000))
|
||||
}
|
||||
return res, nil
|
||||
}
|
||||
|
||||
// runPRReviewAgentRecorded is the shared request-building and StepExecution
|
||||
// recording underneath every pr-review agent call. It returns whatever the
|
||||
// runtime returned, status and all: callers that must distinguish "did not
|
||||
// finish" from "finished, found nothing" (the reviewer fan-out) read
|
||||
// res.Status themselves instead of getting an error for anything but a Go
|
||||
// error.
|
||||
func runPRReviewAgentRecorded(
|
||||
ctx context.Context, tc *TaskContext, workspace, name, role, mission string, maxSteps int, captureTools []agentexec.CaptureTool,
|
||||
) (agentexec.Result, error) {
|
||||
modelRef := tc.Cfg.Models[role]
|
||||
if modelRef == "" {
|
||||
return agentexec.Result{}, fmt.Errorf("no model configured for role %q (set [models] in config)", role)
|
||||
}
|
||||
req := agentexec.Request{
|
||||
Version: agentexec.ProtocolVersion, TaskID: tc.Task.ID, Attempt: tc.Task.Attempt,
|
||||
Stage: name, Workflow: tc.Task.Workflow, Model: modelRef,
|
||||
ContextWindow: modelContextBudget(tc.Cfg, modelRef),
|
||||
Mission: mission, ReadOnly: true,
|
||||
Budget: agentexec.Budget{MaxSteps: maxSteps, WallClock: tc.Cfg.Budgets.WallClock.Std()},
|
||||
CaptureTools: captureTools,
|
||||
}
|
||||
res, err := tc.RunAgentChild(ctx, name, func() (agentexec.Result, error) {
|
||||
return tc.Agent.Run(ctx, workspace, req, tc.toolCallReporter(name))
|
||||
})
|
||||
tc.prReview.tokensSpent.Add(int64(res.TokensUsed))
|
||||
return res, err
|
||||
}
|
||||
|
||||
// forEachBounded runs fn(0), fn(1), ..., fn(n-1), at most limit at a time,
|
||||
// and waits for all of them. It is phase 3, 4 and 9's shared concurrency
|
||||
// bound: tc.RunAgentChild already supports concurrent fan-out from inside
|
||||
// one stage body (each call independently records a StepExecution child), so
|
||||
// this needs only a semaphore, not an engine change.
|
||||
func forEachBounded(limit, n int, fn func(i int)) {
|
||||
if n == 0 {
|
||||
return
|
||||
}
|
||||
if limit <= 0 || limit > n {
|
||||
limit = n
|
||||
}
|
||||
sem := make(chan struct{}, limit)
|
||||
var wg sync.WaitGroup
|
||||
for i := range n {
|
||||
wg.Add(1)
|
||||
sem <- struct{}{}
|
||||
go func(i int) {
|
||||
defer wg.Done()
|
||||
defer func() { <-sem }()
|
||||
fn(i)
|
||||
}(i)
|
||||
}
|
||||
wg.Wait()
|
||||
}
|
||||
@@ -0,0 +1,133 @@
|
||||
package workflow
|
||||
|
||||
import (
|
||||
"context"
|
||||
"testing"
|
||||
|
||||
"github.com/samcharles93/archie-core/internal/agentexec"
|
||||
"github.com/samcharles93/archie-core/internal/domain/workflow/prreview"
|
||||
)
|
||||
|
||||
func TestStagePRPrecisionGateSkippedWhenConfigOff(t *testing.T) {
|
||||
tc := baseTaskContext(t)
|
||||
tc.prReview = &prReviewState{
|
||||
snapshotDir: t.TempDir(),
|
||||
findings: []prreview.Finding{{Title: "f0", Severity: prreview.SeverityNitpick}},
|
||||
}
|
||||
tc.Agent = &concurrentAgentRunner{byStage: func(string) (agentexec.Result, error) {
|
||||
t.Fatal("no agent call expected: review.precision_gate is off")
|
||||
return agentexec.Result{}, nil
|
||||
}}
|
||||
|
||||
if err := stagePRPrecisionGate().Run(context.Background(), tc); err != nil {
|
||||
t.Fatalf("precision-gate: %v", err)
|
||||
}
|
||||
if len(tc.prReview.findings) != 1 {
|
||||
t.Fatalf("findings = %d, want 1 (unfiltered: the gate never ran)", len(tc.prReview.findings))
|
||||
}
|
||||
}
|
||||
|
||||
func TestStagePRPrecisionGateDropsFindingsMarkedNotWorthPosting(t *testing.T) {
|
||||
tc := baseTaskContext(t)
|
||||
tc.Cfg.Review.PrecisionGate = true
|
||||
tc.prReview = &prReviewState{
|
||||
snapshotDir: t.TempDir(),
|
||||
findings: []prreview.Finding{
|
||||
{Title: "concrete", Severity: prreview.SeverityImportant},
|
||||
{Title: "nitpick", Severity: prreview.SeverityNitpick},
|
||||
},
|
||||
}
|
||||
tc.Agent = &concurrentAgentRunner{byStage: func(stage string) (agentexec.Result, error) {
|
||||
if stage == "precision-gate" {
|
||||
return captureResult("precision_gate", map[string]any{
|
||||
"verdicts": []map[string]any{{"index": 0, "keep": true}, {"index": 1, "keep": false}},
|
||||
}), nil
|
||||
}
|
||||
return passedResult("ok"), nil
|
||||
}}
|
||||
|
||||
if err := stagePRPrecisionGate().Run(context.Background(), tc); err != nil {
|
||||
t.Fatalf("precision-gate: %v", err)
|
||||
}
|
||||
if len(tc.prReview.findings) != 1 || tc.prReview.findings[0].Title != "concrete" {
|
||||
t.Fatalf("findings = %+v, want only the concrete finding kept", tc.prReview.findings)
|
||||
}
|
||||
}
|
||||
|
||||
func TestStagePRPrecisionGateFailedCallKeepsEveryFinding(t *testing.T) {
|
||||
tc := baseTaskContext(t)
|
||||
tc.Cfg.Review.PrecisionGate = true
|
||||
tc.prReview = &prReviewState{
|
||||
snapshotDir: t.TempDir(),
|
||||
findings: []prreview.Finding{
|
||||
{Title: "f0", Severity: prreview.SeverityImportant},
|
||||
{Title: "f1", Severity: prreview.SeverityNitpick},
|
||||
},
|
||||
}
|
||||
tc.Agent = &concurrentAgentRunner{byStage: func(string) (agentexec.Result, error) {
|
||||
return agentexec.Result{Status: agentexec.StatusParked}, nil
|
||||
}}
|
||||
|
||||
if err := stagePRPrecisionGate().Run(context.Background(), tc); err != nil {
|
||||
t.Fatalf("precision-gate: %v", err)
|
||||
}
|
||||
if len(tc.prReview.findings) != 2 {
|
||||
t.Fatalf("findings = %d, want 2 (recall-first: a failed call filters nothing)", len(tc.prReview.findings))
|
||||
}
|
||||
}
|
||||
|
||||
func TestStagePRPrecisionGateRunsNoAgentCallWhenNoFindings(t *testing.T) {
|
||||
tc := baseTaskContext(t)
|
||||
tc.Cfg.Review.PrecisionGate = true
|
||||
tc.prReview = &prReviewState{snapshotDir: t.TempDir()}
|
||||
tc.Agent = &concurrentAgentRunner{byStage: func(string) (agentexec.Result, error) {
|
||||
t.Fatal("no agent call expected with zero findings")
|
||||
return agentexec.Result{}, nil
|
||||
}}
|
||||
|
||||
if err := stagePRPrecisionGate().Run(context.Background(), tc); err != nil {
|
||||
t.Fatalf("precision-gate: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestStagePROperatorApprovalWaitsWhenConfiguredAndFindingsExist(t *testing.T) {
|
||||
tc := baseTaskContext(t)
|
||||
tc.Cfg.Review.ApproveBeforePost = true
|
||||
tc.prReview = &prReviewState{
|
||||
scored: []prreview.ScoredFinding{{Title: "f0", Score: 0.9}},
|
||||
}
|
||||
|
||||
if err := stagePROperatorApproval().Run(context.Background(), tc); err != nil {
|
||||
t.Fatalf("operator-approval: %v", err)
|
||||
}
|
||||
if tc.Outcome.Status != StatusWaitingHuman {
|
||||
t.Fatalf("outcome status = %q, want %q", tc.Outcome.Status, StatusWaitingHuman)
|
||||
}
|
||||
}
|
||||
|
||||
func TestStagePROperatorApprovalSkipsWhenConfigOff(t *testing.T) {
|
||||
tc := baseTaskContext(t)
|
||||
tc.prReview = &prReviewState{
|
||||
scored: []prreview.ScoredFinding{{Title: "f0", Score: 0.9}},
|
||||
}
|
||||
|
||||
if err := stagePROperatorApproval().Run(context.Background(), tc); err != nil {
|
||||
t.Fatalf("operator-approval: %v", err)
|
||||
}
|
||||
if tc.Outcome.Status != "" {
|
||||
t.Fatalf("outcome status = %q, want empty (config off, never waits)", tc.Outcome.Status)
|
||||
}
|
||||
}
|
||||
|
||||
func TestStagePROperatorApprovalNeverWaitsWithNoFindings(t *testing.T) {
|
||||
tc := baseTaskContext(t)
|
||||
tc.Cfg.Review.ApproveBeforePost = true
|
||||
tc.prReview = &prReviewState{}
|
||||
|
||||
if err := stagePROperatorApproval().Run(context.Background(), tc); err != nil {
|
||||
t.Fatalf("operator-approval: %v", err)
|
||||
}
|
||||
if tc.Outcome.Status != "" {
|
||||
t.Fatalf("outcome status = %q, want empty (zero findings never wait, per the PRD)", tc.Outcome.Status)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,644 @@
|
||||
package workflow
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"log/slog"
|
||||
"os"
|
||||
"slices"
|
||||
"strings"
|
||||
"sync"
|
||||
"sync/atomic"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/samcharles93/archie-core/internal/agentexec"
|
||||
"github.com/samcharles93/archie-core/internal/config"
|
||||
"github.com/samcharles93/archie-core/internal/domain/workflow/prreview"
|
||||
"github.com/samcharles93/archie-core/internal/events"
|
||||
"github.com/samcharles93/archie-core/internal/taskstate"
|
||||
)
|
||||
|
||||
// fakePRSource is a test double for PRSource: no forge call, just canned
|
||||
// metadata, diff and a snapshot copied from an in-memory file set.
|
||||
type fakePRSource struct {
|
||||
metadata PRMetadata
|
||||
diff string
|
||||
files map[string]string // relative path -> content, written into the snapshot dir
|
||||
headSHA string
|
||||
err error
|
||||
}
|
||||
|
||||
func (f *fakePRSource) Metadata(context.Context, string, string, int) (PRMetadata, error) {
|
||||
return f.metadata, f.err
|
||||
}
|
||||
|
||||
func (f *fakePRSource) Diff(context.Context, string, string, int) (string, error) {
|
||||
return f.diff, f.err
|
||||
}
|
||||
|
||||
func (f *fakePRSource) Snapshot(_ context.Context, _, _ string, _ int, destDir string) (string, error) {
|
||||
if f.err != nil {
|
||||
return "", f.err
|
||||
}
|
||||
for path, content := range f.files {
|
||||
if err := os.WriteFile(destDir+"/"+path, []byte(content), 0o644); err != nil {
|
||||
return "", err
|
||||
}
|
||||
}
|
||||
return f.headSHA, nil
|
||||
}
|
||||
|
||||
// concurrentAgentRunner counts simultaneous Run calls and dispatches by the
|
||||
// request's Stage name -- pr-review names every child step after its own
|
||||
// phase/dimension, so tests can script one behaviour per call site.
|
||||
type concurrentAgentRunner struct {
|
||||
mu sync.Mutex
|
||||
current int
|
||||
peak int
|
||||
|
||||
byStage func(stage string) (agentexec.Result, error)
|
||||
}
|
||||
|
||||
func (r *concurrentAgentRunner) Run(
|
||||
_ context.Context, _ string, req agentexec.Request, _ agentexec.ToolCallReporter,
|
||||
) (agentexec.Result, error) {
|
||||
r.mu.Lock()
|
||||
r.current++
|
||||
if r.current > r.peak {
|
||||
r.peak = r.current
|
||||
}
|
||||
r.mu.Unlock()
|
||||
defer func() {
|
||||
r.mu.Lock()
|
||||
r.current--
|
||||
r.mu.Unlock()
|
||||
}()
|
||||
res, err := r.byStage(req.Stage)
|
||||
res.Version, res.TaskID, res.Attempt, res.Stage = agentexec.ProtocolVersion, req.TaskID, req.Attempt, req.Stage
|
||||
return res, err
|
||||
}
|
||||
|
||||
// threadSafeStore is a Store fake safe for the concurrent StepExecution
|
||||
// writes phase 3, 4 and 9's bounded fan-out makes from goroutines --
|
||||
// recordingStore (changes_test.go) is not, and races under -race.
|
||||
type threadSafeStore struct {
|
||||
mu sync.Mutex
|
||||
nextID int64
|
||||
starts []StepStart
|
||||
finishes []StepFinish
|
||||
}
|
||||
|
||||
func (s *threadSafeStore) Update(context.Context, *Task) error { return nil }
|
||||
func (s *threadSafeStore) Transition(context.Context, int64, string, string, string) error {
|
||||
return nil
|
||||
}
|
||||
|
||||
func (s *threadSafeStore) InsertEvent(context.Context, events.Event) (int64, error) { return 0, nil }
|
||||
|
||||
func (s *threadSafeStore) StartStep(_ context.Context, start StepStart) (int64, events.Event, error) {
|
||||
s.mu.Lock()
|
||||
defer s.mu.Unlock()
|
||||
s.nextID++
|
||||
s.starts = append(s.starts, start)
|
||||
return s.nextID, events.Event{}, nil
|
||||
}
|
||||
|
||||
func (s *threadSafeStore) FinishStep(_ context.Context, finish StepFinish) (events.Event, error) {
|
||||
s.mu.Lock()
|
||||
defer s.mu.Unlock()
|
||||
s.finishes = append(s.finishes, finish)
|
||||
return events.Event{}, nil
|
||||
}
|
||||
|
||||
func passedResult(summary string) agentexec.Result {
|
||||
return agentexec.Result{Status: agentexec.StatusPassed, Summary: summary}
|
||||
}
|
||||
|
||||
func captureResult(tool string, payload any) agentexec.Result {
|
||||
data, err := json.Marshal(payload)
|
||||
if err != nil {
|
||||
panic(err)
|
||||
}
|
||||
return agentexec.Result{
|
||||
Status: agentexec.StatusPassed, Summary: "ok",
|
||||
Captures: map[string][]json.RawMessage{tool: {data}},
|
||||
}
|
||||
}
|
||||
|
||||
func baseTaskContext(t *testing.T) *TaskContext {
|
||||
t.Helper()
|
||||
return &TaskContext{
|
||||
Task: &Task{ID: 1, Attempt: 1, Owner: "acme", Repo: "widgets", PRNumber: 42, Workflow: "pr-review"},
|
||||
Cfg: config.Config{
|
||||
Models: map[string]string{"review": "provider/review", "classification": "provider/classification"},
|
||||
},
|
||||
Store: &threadSafeStore{},
|
||||
Log: slog.New(slog.DiscardHandler),
|
||||
}
|
||||
}
|
||||
|
||||
func TestStagePRIntakeClassifiesDepthAndScoresAIGenerated(t *testing.T) {
|
||||
tc := baseTaskContext(t)
|
||||
tc.PRSource = &fakePRSource{
|
||||
metadata: PRMetadata{Title: "t", Body: "b"},
|
||||
diff: smallDiff,
|
||||
}
|
||||
tc.Agent = &concurrentAgentRunner{byStage: func(string) (agentexec.Result, error) {
|
||||
return captureResult("score_ai_generated", map[string]any{"confidence": 0.9}), nil
|
||||
}}
|
||||
|
||||
if err := stagePRIntake().Run(context.Background(), tc); err != nil {
|
||||
t.Fatalf("intake: %v", err)
|
||||
}
|
||||
if tc.prReview == nil {
|
||||
t.Fatal("intake left no prReview state")
|
||||
}
|
||||
if tc.prReview.depth != prreview.DepthQuick {
|
||||
t.Errorf("depth = %q, want quick for a %d-line diff", tc.prReview.depth, tc.prReview.stats.TotalAdditions+tc.prReview.stats.TotalDeletions)
|
||||
}
|
||||
if tc.prReview.aiGenerated != 0.9 {
|
||||
t.Errorf("aiGenerated = %v, want 0.9 (from the captured score)", tc.prReview.aiGenerated)
|
||||
}
|
||||
}
|
||||
|
||||
func TestStagePRIntakeFailsWithNoPRSource(t *testing.T) {
|
||||
tc := baseTaskContext(t)
|
||||
if err := stagePRIntake().Run(context.Background(), tc); err == nil {
|
||||
t.Fatal("want an error with no PRSource configured")
|
||||
}
|
||||
}
|
||||
|
||||
func TestStagePRIntakeResolvesPRNumberFromInputsWhenUnset(t *testing.T) {
|
||||
tc := baseTaskContext(t)
|
||||
tc.Task.PRNumber = 0
|
||||
tc.Task.Inputs = map[string]any{"pr_number": json.Number("77")}
|
||||
tc.PRSource = &fakePRSource{metadata: PRMetadata{Title: "t", Body: "b"}, diff: smallDiff}
|
||||
tc.Agent = &concurrentAgentRunner{byStage: func(string) (agentexec.Result, error) {
|
||||
return captureResult("score_ai_generated", map[string]any{"confidence": 0.1}), nil
|
||||
}}
|
||||
|
||||
if err := stagePRIntake().Run(context.Background(), tc); err != nil {
|
||||
t.Fatalf("intake: %v", err)
|
||||
}
|
||||
if tc.Task.PRNumber != 77 {
|
||||
t.Errorf("PRNumber = %d, want 77 (resolved from Inputs)", tc.Task.PRNumber)
|
||||
}
|
||||
}
|
||||
|
||||
func TestStagePRIntakeFailsWithNoPRNumberAnywhere(t *testing.T) {
|
||||
tc := baseTaskContext(t)
|
||||
tc.Task.PRNumber = 0
|
||||
tc.PRSource = &fakePRSource{metadata: PRMetadata{Title: "t", Body: "b"}, diff: smallDiff}
|
||||
if err := stagePRIntake().Run(context.Background(), tc); err == nil {
|
||||
t.Fatal("want an error with no PR number set directly or via Inputs")
|
||||
}
|
||||
}
|
||||
|
||||
func TestStagePRIntakeAllowsNoPRNumberWhenEmbeddedInAnotherWorkflow(t *testing.T) {
|
||||
// Archie's own PRs (StagePRReviewAndOpenPR) embeds this stage inside the
|
||||
// implement workflow, reviewing a change with no PR number yet --
|
||||
// localPRSource ignores the number entirely. The missing-number error
|
||||
// must not fire here, only for the standalone pr-review workflow.
|
||||
tc := baseTaskContext(t)
|
||||
tc.Task.Workflow = "implement"
|
||||
tc.Task.PRNumber = 0
|
||||
tc.PRSource = &fakePRSource{metadata: PRMetadata{Title: "t", Body: "b"}, diff: smallDiff}
|
||||
tc.Agent = &concurrentAgentRunner{byStage: func(string) (agentexec.Result, error) {
|
||||
return captureResult("score_ai_generated", map[string]any{"confidence": 0.1}), nil
|
||||
}}
|
||||
|
||||
if err := stagePRIntake().Run(context.Background(), tc); err != nil {
|
||||
t.Fatalf("intake: %v (archie's own PRs must not require a PR number)", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestStagePRAnatomyBuildsSnapshotBlastRadiusAndNarrative(t *testing.T) {
|
||||
tc := baseTaskContext(t)
|
||||
tc.prReview = &prReviewState{
|
||||
diff: smallDiff,
|
||||
files: prreview.ParseDiff(smallDiff),
|
||||
}
|
||||
tc.PRSource = &fakePRSource{
|
||||
headSHA: "deadbeef",
|
||||
files: map[string]string{"main.go": "package main\n"},
|
||||
}
|
||||
tc.Agent = &concurrentAgentRunner{byStage: func(string) (agentexec.Result, error) {
|
||||
return passedResult("narrative here"), nil
|
||||
}}
|
||||
|
||||
if err := stagePRAnatomy().Run(context.Background(), tc); err != nil {
|
||||
t.Fatalf("anatomy: %v", err)
|
||||
}
|
||||
if tc.prReview.headSHA != "deadbeef" {
|
||||
t.Errorf("headSHA = %q, want deadbeef", tc.prReview.headSHA)
|
||||
}
|
||||
if tc.prReview.snapshotDir == "" {
|
||||
t.Fatal("no snapshot directory recorded")
|
||||
}
|
||||
defer os.RemoveAll(tc.prReview.snapshotDir)
|
||||
if _, err := os.Stat(tc.prReview.snapshotDir + "/main.go"); err != nil {
|
||||
t.Errorf("snapshot missing main.go: %v", err)
|
||||
}
|
||||
if tc.prReview.narrative != "narrative here" {
|
||||
t.Errorf("narrative = %q, want the agent's summary", tc.prReview.narrative)
|
||||
}
|
||||
}
|
||||
|
||||
func TestStagePRLensesMergesCapsAndAddsHallucinationDimension(t *testing.T) {
|
||||
tc := baseTaskContext(t)
|
||||
tc.prReview = &prReviewState{depth: prreview.DepthQuick, aiGenerated: 0.9, snapshotDir: t.TempDir()}
|
||||
|
||||
var calls atomic.Int32
|
||||
tc.Agent = &concurrentAgentRunner{byStage: func(stage string) (agentexec.Result, error) {
|
||||
calls.Add(1)
|
||||
// Priority 0.5, below the hallucination-check dimension's fixed
|
||||
// priority of 1 (dimension.go), so the cap's cut is deterministic:
|
||||
// the hallucination dimension always survives, one lens dimension
|
||||
// always does not.
|
||||
return captureResult("propose_dimensions", map[string]any{
|
||||
"dimensions": []map[string]any{
|
||||
{"name": stage, "prompt": "p", "target_files": []string{"a.go"}, "priority": 0.5},
|
||||
},
|
||||
}), nil
|
||||
}}
|
||||
|
||||
if err := stagePRLenses().Run(context.Background(), tc); err != nil {
|
||||
t.Fatalf("lenses: %v", err)
|
||||
}
|
||||
if calls.Load() != 3 {
|
||||
t.Errorf("agent calls = %d, want 3 (one per lens)", calls.Load())
|
||||
}
|
||||
// quick depth caps at 3 dimensions total; three lenses each proposed one
|
||||
// distinctly named dimension, plus the hallucination-check dimension a
|
||||
// 0.9 AI-generated score adds as a fourth candidate -- capped to 3, not
|
||||
// 3 lens dimensions plus the hallucination dimension on top.
|
||||
if len(tc.prReview.dimensions) != prreview.MaxDimensionsFor(prreview.DepthQuick) {
|
||||
t.Fatalf("dimensions = %d, want the quick cap of %d", len(tc.prReview.dimensions), prreview.MaxDimensionsFor(prreview.DepthQuick))
|
||||
}
|
||||
foundHallucination := false
|
||||
for _, d := range tc.prReview.dimensions {
|
||||
if d.Name == "hallucination-check" {
|
||||
foundHallucination = true
|
||||
}
|
||||
}
|
||||
if !foundHallucination {
|
||||
t.Error("hallucination-check dimension was cut even though its priority of 1 outranks every lens dimension's 0.5")
|
||||
}
|
||||
}
|
||||
|
||||
func TestStagePRReviewBoundsConcurrencyAndDistinguishesTimeoutFromCleanEmpty(t *testing.T) {
|
||||
tc := baseTaskContext(t)
|
||||
dims := make([]prreview.Dimension, 10)
|
||||
for i := range dims {
|
||||
dims[i] = prreview.Dimension{Name: fmt.Sprintf("dim-%d", i), Prompt: "p", TargetFiles: []string{"a.go"}}
|
||||
}
|
||||
tc.prReview = &prReviewState{dimensions: dims, snapshotDir: t.TempDir()}
|
||||
|
||||
runner := &concurrentAgentRunner{byStage: func(stage string) (agentexec.Result, error) {
|
||||
if stage == "reviewer-dim-0" {
|
||||
// Simulate a reviewer that exhausted its turn cap before its
|
||||
// terminal tool call: no Go error, but not agentexec.StatusPassed.
|
||||
return agentexec.Result{Status: agentexec.StatusParked, StopReason: "budget_exhausted"}, nil
|
||||
}
|
||||
return captureResult("report_findings", map[string]any{"findings": []any{}}), nil
|
||||
}}
|
||||
tc.Agent = runner
|
||||
store := &threadSafeStore{}
|
||||
tc.Store = store
|
||||
tc.StepID = 1 // non-zero, so startChildStep/finishChildStep actually record
|
||||
|
||||
if err := stagePRReview().Run(context.Background(), tc); err != nil {
|
||||
t.Fatalf("review: %v", err)
|
||||
}
|
||||
if runner.peak > prReviewConcurrency {
|
||||
t.Errorf("peak concurrent reviewer calls = %d, want at most %d", runner.peak, prReviewConcurrency)
|
||||
}
|
||||
if len(tc.prReview.findings) != 0 {
|
||||
t.Errorf("findings = %d, want 0 (every reviewer reported none)", len(tc.prReview.findings))
|
||||
}
|
||||
|
||||
store.mu.Lock()
|
||||
defer store.mu.Unlock()
|
||||
var timedOut, cleanEmpty int
|
||||
for _, f := range store.finishes {
|
||||
switch f.To {
|
||||
case taskstate.StepFailed:
|
||||
timedOut++
|
||||
case taskstate.StepSucceeded:
|
||||
cleanEmpty++
|
||||
default:
|
||||
t.Errorf("unexpected finish status %q", f.To)
|
||||
}
|
||||
}
|
||||
if timedOut != 1 {
|
||||
t.Errorf("StepFailed finishes = %d, want exactly 1 (the turn-cap-exhausted reviewer)", timedOut)
|
||||
}
|
||||
if cleanEmpty != 9 {
|
||||
t.Errorf("clean StepSucceeded finishes = %d, want 9", cleanEmpty)
|
||||
}
|
||||
}
|
||||
|
||||
func TestStagePRSynthesisScoresAndCapsFindings(t *testing.T) {
|
||||
tc := baseTaskContext(t)
|
||||
tc.prReview = &prReviewState{
|
||||
aiGenerated: 0.9,
|
||||
blastRadius: make([]string, 12),
|
||||
findings: []prreview.Finding{
|
||||
{File: "a.go", LineStart: 1, Severity: prreview.SeverityCritical, Title: "t", Body: "b", Evidence: "e", Confidence: 0.9},
|
||||
},
|
||||
}
|
||||
if err := stagePRSynthesis().Run(context.Background(), tc); err != nil {
|
||||
t.Fatalf("synthesis: %v", err)
|
||||
}
|
||||
if len(tc.prReview.scored) != 1 {
|
||||
t.Fatalf("scored = %d, want 1", len(tc.prReview.scored))
|
||||
}
|
||||
// blast radius over 10 files and a likely machine-written PR both apply
|
||||
// their multiplier (scoring.go), so the score exceeds the base severity
|
||||
// weight x confidence (1.0 x 0.9 = 0.9).
|
||||
if tc.prReview.scored[0].Score <= 0.9 {
|
||||
t.Errorf("score = %v, want more than the unmultiplied 0.9 (blast radius + AI-generated multipliers should have applied)", tc.prReview.scored[0].Score)
|
||||
}
|
||||
}
|
||||
|
||||
func TestStagePROutputPolishesPostsAndCompletes(t *testing.T) {
|
||||
tc := baseTaskContext(t)
|
||||
tc.prReview = &prReviewState{
|
||||
headSHA: "deadbeef",
|
||||
scored: []prreview.ScoredFinding{
|
||||
{File: "a.go", LineStart: 3, Title: "t1", Body: "original 1"},
|
||||
{File: "b.go", LineStart: 5, Title: "t2", Body: "original 2"},
|
||||
},
|
||||
}
|
||||
tc.Agent = &concurrentAgentRunner{byStage: func(string) (agentexec.Result, error) {
|
||||
return captureResult("polish", map[string]any{"body": "tightened"}), nil
|
||||
}}
|
||||
forge := &fakeForger{}
|
||||
tc.Forge = forge
|
||||
|
||||
if err := stagePROutput().Run(context.Background(), tc); err != nil {
|
||||
t.Fatalf("output: %v", err)
|
||||
}
|
||||
for _, f := range tc.prReview.scored {
|
||||
if f.Body != "tightened" {
|
||||
t.Errorf("finding %q body = %q, want the polish call's tightened body", f.Title, f.Body)
|
||||
}
|
||||
}
|
||||
if len(forge.comments) != 2 {
|
||||
t.Fatalf("posted comments = %d, want 2", len(forge.comments))
|
||||
}
|
||||
if forge.reviewedHeadSHA != "deadbeef" {
|
||||
t.Errorf("posted headSHA = %q, want deadbeef", forge.reviewedHeadSHA)
|
||||
}
|
||||
if tc.Outcome.Status != StatusCompleted {
|
||||
t.Errorf("outcome status = %q, want %q", tc.Outcome.Status, StatusCompleted)
|
||||
}
|
||||
}
|
||||
|
||||
func TestStagePROutputKeepsOriginalBodyWhenPolishFails(t *testing.T) {
|
||||
tc := baseTaskContext(t)
|
||||
tc.prReview = &prReviewState{scored: []prreview.ScoredFinding{{File: "a.go", LineStart: 1, Body: "original"}}}
|
||||
tc.Agent = &concurrentAgentRunner{byStage: func(string) (agentexec.Result, error) {
|
||||
return agentexec.Result{}, fmt.Errorf("model unavailable")
|
||||
}}
|
||||
tc.Forge = &fakeForger{}
|
||||
|
||||
if err := stagePROutput().Run(context.Background(), tc); err != nil {
|
||||
t.Fatalf("output: %v", err)
|
||||
}
|
||||
if tc.prReview.scored[0].Body != "original" {
|
||||
t.Errorf("body = %q, want the original body kept on polish failure", tc.prReview.scored[0].Body)
|
||||
}
|
||||
}
|
||||
|
||||
func TestBudgetExhaustedSkipsLensesAndRecordsIt(t *testing.T) {
|
||||
tc := baseTaskContext(t)
|
||||
tc.prReview = &prReviewState{
|
||||
depth: prreview.DepthQuick, snapshotDir: t.TempDir(), runStart: time.Now(),
|
||||
}
|
||||
// Spend past every phase's cumulative token allotment up front, so
|
||||
// lenses (the first budget-tracked phase) is already exhausted before
|
||||
// its own work would start.
|
||||
tc.prReview.tokensSpent.Store(int64(prReviewTotalBudget.MaxTokens))
|
||||
tc.Agent = &concurrentAgentRunner{byStage: func(string) (agentexec.Result, error) {
|
||||
t.Fatal("no lens call should run once the budget is already spent")
|
||||
return agentexec.Result{}, nil
|
||||
}}
|
||||
|
||||
if err := stagePRLenses().Run(context.Background(), tc); err != nil {
|
||||
t.Fatalf("lenses: %v", err)
|
||||
}
|
||||
if len(tc.prReview.dimensions) != 0 {
|
||||
t.Errorf("dimensions = %d, want 0 (lenses skipped, not run)", len(tc.prReview.dimensions))
|
||||
}
|
||||
if len(tc.prReview.skippedPhases) != 1 || tc.prReview.skippedPhases[0] != "lenses" {
|
||||
t.Errorf("skippedPhases = %v, want [\"lenses\"]", tc.prReview.skippedPhases)
|
||||
}
|
||||
}
|
||||
|
||||
func TestBudgetNotExhaustedRunsLensesNormally(t *testing.T) {
|
||||
tc := baseTaskContext(t)
|
||||
tc.prReview = &prReviewState{depth: prreview.DepthQuick, snapshotDir: t.TempDir(), runStart: time.Now()}
|
||||
tc.Agent = &concurrentAgentRunner{byStage: func(string) (agentexec.Result, error) {
|
||||
return captureResult("propose_dimensions", map[string]any{
|
||||
"dimensions": []map[string]any{{"name": "d", "prompt": "p", "target_files": []string{"a.go"}, "priority": 1.0}},
|
||||
}), nil
|
||||
}}
|
||||
|
||||
if err := stagePRLenses().Run(context.Background(), tc); err != nil {
|
||||
t.Fatalf("lenses: %v", err)
|
||||
}
|
||||
if len(tc.prReview.skippedPhases) != 0 {
|
||||
t.Errorf("skippedPhases = %v, want none: the budget was not spent", tc.prReview.skippedPhases)
|
||||
}
|
||||
if len(tc.prReview.dimensions) == 0 {
|
||||
t.Error("dimensions = 0, want at least the lens's own proposal: lenses should have run")
|
||||
}
|
||||
}
|
||||
|
||||
func TestStagePROutputReportsSkippedPhasesAndReviewEvent(t *testing.T) {
|
||||
tc := baseTaskContext(t)
|
||||
tc.prReview = &prReviewState{
|
||||
headSHA: "deadbeef",
|
||||
skippedPhases: []string{"verification", "coverage-consistency"},
|
||||
scored: []prreview.ScoredFinding{
|
||||
{File: "a.go", LineStart: 1, Title: "t", Body: "b", Blocking: true},
|
||||
},
|
||||
}
|
||||
tc.Agent = &concurrentAgentRunner{byStage: func(string) (agentexec.Result, error) {
|
||||
return captureResult("polish", map[string]any{"body": "tightened"}), nil
|
||||
}}
|
||||
tc.Forge = &fakeForger{}
|
||||
|
||||
if err := stagePROutput().Run(context.Background(), tc); err != nil {
|
||||
t.Fatalf("output: %v", err)
|
||||
}
|
||||
if !strings.Contains(tc.Outcome.Detail, "REQUEST_CHANGES") {
|
||||
t.Errorf("outcome detail = %q, want it to name REQUEST_CHANGES (a blocking finding survived)", tc.Outcome.Detail)
|
||||
}
|
||||
if !strings.Contains(tc.Outcome.Detail, "verification") || !strings.Contains(tc.Outcome.Detail, "coverage-consistency") {
|
||||
t.Errorf("outcome detail = %q, want it to name every skipped phase", tc.Outcome.Detail)
|
||||
}
|
||||
}
|
||||
|
||||
func TestStagePROutputSkipsPolishWhenBudgetExhausted(t *testing.T) {
|
||||
tc := baseTaskContext(t)
|
||||
tc.prReview = &prReviewState{
|
||||
runStart: time.Now(),
|
||||
scored: []prreview.ScoredFinding{{File: "a.go", LineStart: 1, Title: "t", Body: "original"}},
|
||||
}
|
||||
tc.prReview.tokensSpent.Store(int64(prReviewTotalBudget.MaxTokens))
|
||||
tc.Agent = &concurrentAgentRunner{byStage: func(string) (agentexec.Result, error) {
|
||||
t.Fatal("no polish call should run once the budget is already spent")
|
||||
return agentexec.Result{}, nil
|
||||
}}
|
||||
tc.Forge = &fakeForger{}
|
||||
|
||||
if err := stagePROutput().Run(context.Background(), tc); err != nil {
|
||||
t.Fatalf("output: %v", err)
|
||||
}
|
||||
if tc.prReview.scored[0].Body != "original" {
|
||||
t.Errorf("body = %q, want the original (unpolished) body", tc.prReview.scored[0].Body)
|
||||
}
|
||||
if !slices.Contains(tc.prReview.skippedPhases, "output") {
|
||||
t.Errorf("skippedPhases = %v, want it to include \"output\"", tc.prReview.skippedPhases)
|
||||
}
|
||||
}
|
||||
|
||||
// fakeForger records CreateReviewComments calls; every other Forger method
|
||||
// panics if called, since no pr-review stage should reach them.
|
||||
type fakeForger struct {
|
||||
comments []ReviewComment
|
||||
reviewedHeadSHA string
|
||||
}
|
||||
|
||||
func (f *fakeForger) CloseIssue(context.Context, string, string, int, string) error { panic("unused") }
|
||||
|
||||
func (f *fakeForger) CreatePR(context.Context, string, string, string, string, string, string) (int, error) {
|
||||
panic("unused")
|
||||
}
|
||||
|
||||
func (f *fakeForger) LinkBranch(context.Context, string, string, int, string) error { panic("unused") }
|
||||
|
||||
func (f *fakeForger) CreateReviewComments(
|
||||
_ context.Context, _, _ string, _ int, reviewedHeadSHA string, comments []ReviewComment,
|
||||
) error {
|
||||
f.comments = comments
|
||||
f.reviewedHeadSHA = reviewedHeadSHA
|
||||
return nil
|
||||
}
|
||||
|
||||
func (f *fakeForger) Comment(context.Context, string, string, int, string) (int64, error) {
|
||||
panic("unused")
|
||||
}
|
||||
|
||||
func (f *fakeForger) ReplyToReview(context.Context, string, string, int, int64, string) error {
|
||||
panic("unused")
|
||||
}
|
||||
|
||||
// smallDiff is a one-line, one-file diff: under the 100-line quick-depth
|
||||
// ceiling, so intake classifies it as quick.
|
||||
const smallDiff = `diff --git a/main.go b/main.go
|
||||
index e69de29..4b825dc 100644
|
||||
--- a/main.go
|
||||
+++ b/main.go
|
||||
@@ -0,0 +1 @@
|
||||
+package main
|
||||
`
|
||||
|
||||
func TestStagePRIntakeHonoursExplicitDepth(t *testing.T) {
|
||||
for _, tt := range []struct {
|
||||
value any
|
||||
want prreview.Depth
|
||||
invalid bool
|
||||
}{
|
||||
{"deep", prreview.DepthDeep, false},
|
||||
{"standard", prreview.DepthStandard, false},
|
||||
{"quick", prreview.DepthQuick, false},
|
||||
{"depe", "", true},
|
||||
{7, "", true},
|
||||
} {
|
||||
t.Run(fmt.Sprint(tt.value), func(t *testing.T) {
|
||||
tc := baseTaskContext(t)
|
||||
tc.Task.Inputs = map[string]any{"depth": tt.value}
|
||||
tc.PRSource = &fakePRSource{diff: smallDiff}
|
||||
tc.Agent = &concurrentAgentRunner{byStage: func(string) (agentexec.Result, error) {
|
||||
return captureResult("score_ai_generated", map[string]any{"confidence": 0}), nil
|
||||
}}
|
||||
err := stagePRIntake().Run(t.Context(), tc)
|
||||
if tt.invalid {
|
||||
if err == nil {
|
||||
t.Fatal("invalid depth accepted")
|
||||
}
|
||||
return
|
||||
}
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if tc.prReview.depth != tt.want {
|
||||
t.Fatalf("depth = %s, want %s", tc.prReview.depth, tt.want)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestReviewPullRequestUsesPipelineWithoutPosting(t *testing.T) {
|
||||
for _, mode := range []string{"complete", "lens fails", "reviewer fails", "budget skips merge"} {
|
||||
t.Run(mode, func(t *testing.T) {
|
||||
tc := baseTaskContext(t)
|
||||
tc.Task.Inputs = map[string]any{"depth": "deep"}
|
||||
tc.PRSource = &fakePRSource{diff: smallDiff, files: map[string]string{"a.go": "package a\n"}}
|
||||
var reachedMerge atomic.Bool
|
||||
tc.Agent = &concurrentAgentRunner{byStage: func(stage string) (agentexec.Result, error) {
|
||||
switch {
|
||||
case stage == "intake-ai-score":
|
||||
return captureResult("score_ai_generated", map[string]any{"confidence": 0}), nil
|
||||
case strings.HasPrefix(stage, "lens-"):
|
||||
if mode == "lens fails" {
|
||||
return agentexec.Result{}, fmt.Errorf("lens unavailable")
|
||||
}
|
||||
return captureResult("propose_dimensions", map[string]any{"dimensions": []map[string]any{{"name": "defect", "prompt": "review", "target_files": []string{"a.go"}, "priority": 1}}}), nil
|
||||
case strings.HasPrefix(stage, "reviewer-"):
|
||||
if mode == "reviewer fails" {
|
||||
return agentexec.Result{}, fmt.Errorf("reviewer unavailable")
|
||||
}
|
||||
return captureResult("report_findings", map[string]any{"findings": []map[string]any{{"file": "a.go", "line_start": 1, "severity": "important", "title": "defect", "body": "broken", "evidence": "package a", "confidence": 0.9}}}), nil
|
||||
case stage == "merge-gate":
|
||||
reachedMerge.Store(true)
|
||||
return captureResult("classify_blocking", map[string]any{"blocking": true}), nil
|
||||
case stage == "consistency" && mode == "budget skips merge":
|
||||
return agentexec.Result{Status: agentexec.StatusPassed, TokensUsed: 2_000_000}, nil
|
||||
default:
|
||||
return passedResult("ok"), nil
|
||||
}
|
||||
}}
|
||||
decision, err := ReviewPullRequest(t.Context(), tc)
|
||||
switch mode {
|
||||
case "lens fails":
|
||||
if err == nil || !strings.Contains(err.Error(), "lens unavailable") {
|
||||
t.Fatalf("error = %v", err)
|
||||
}
|
||||
case "reviewer fails":
|
||||
if err == nil || !strings.Contains(err.Error(), "reviewer(s) did not finish") {
|
||||
t.Fatalf("error = %v", err)
|
||||
}
|
||||
case "budget skips merge":
|
||||
if err != nil || len(decision.Comments) != 1 || reachedMerge.Load() || !slices.Contains(decision.SkippedPhases, "merge-gate") {
|
||||
t.Fatalf("decision = %+v, merge = %v, error = %v", decision, reachedMerge.Load(), err)
|
||||
}
|
||||
default:
|
||||
if err != nil || len(decision.Comments) != 1 || !reachedMerge.Load() || !decision.Comments[0].Blocking {
|
||||
t.Fatalf("decision = %+v, merge = %v, error = %v", decision, reachedMerge.Load(), err)
|
||||
}
|
||||
}
|
||||
if tc.prReview == nil || tc.prReview.depth != prreview.DepthDeep {
|
||||
t.Fatal("pipeline did not use explicit deep depth")
|
||||
}
|
||||
if _, err := os.Stat(tc.prReview.snapshotDir); !os.IsNotExist(err) {
|
||||
t.Fatalf("snapshot not cleaned: %v", err)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,674 @@
|
||||
package workflow
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"os"
|
||||
"strings"
|
||||
"sync"
|
||||
|
||||
"github.com/samcharles93/archie-core/internal/agentexec"
|
||||
"github.com/samcharles93/archie-core/internal/domain/workflow/prreview"
|
||||
)
|
||||
|
||||
// stagePRVerification is pipeline phase 5, over the review findings phase 4
|
||||
// produced: extract each finding's evidence package (code), check the
|
||||
// high-priority findings against theirs (an evidence verifier agent), have
|
||||
// an adversary confirm or challenge every finding and add anything every
|
||||
// reviewer missed, then flag within-cluster compound defects
|
||||
// (docs/prds/pr-review-agent.md, phase 5).
|
||||
func stagePRVerification() Stage {
|
||||
return Stage{Name: "verification", Run: func(ctx context.Context, tc *TaskContext) error {
|
||||
if len(tc.prReview.findings) == 0 {
|
||||
return nil
|
||||
}
|
||||
|
||||
verified, err := runEvidenceVerifier(ctx, tc, tc.prReview.findings)
|
||||
if err != nil {
|
||||
return fmt.Errorf("evidence verifier: %w", err)
|
||||
}
|
||||
|
||||
confirmed, err := runAdversary(ctx, tc, verified)
|
||||
if err != nil {
|
||||
return fmt.Errorf("adversary: %w", err)
|
||||
}
|
||||
|
||||
compounded, err := runCompoundCheck(ctx, tc, confirmed)
|
||||
if err != nil {
|
||||
return fmt.Errorf("compound check: %w", err)
|
||||
}
|
||||
|
||||
tc.prReview.findings = compounded
|
||||
return nil
|
||||
}}
|
||||
}
|
||||
|
||||
// stagePRPrecisionGate is the PRD's Precision dial: an optional
|
||||
// post-worthiness pass, active only when review.precision_gate is set,
|
||||
// between phase 4 (review) and phase 5 (verification). It keeps every
|
||||
// concrete, evidenced defect and drops nitpicks, style and unverifiable
|
||||
// claims, keeping a finding when unsure -- trading recall for precision and
|
||||
// cutting verification cost by filtering early. It is off by default: the
|
||||
// pipeline is recall-first.
|
||||
func stagePRPrecisionGate() Stage {
|
||||
return Stage{Name: "precision-gate", Run: func(ctx context.Context, tc *TaskContext) error {
|
||||
if !tc.Cfg.Review.PrecisionGate || len(tc.prReview.findings) == 0 {
|
||||
return nil
|
||||
}
|
||||
if budgetExhausted(tc, "precision-gate") {
|
||||
skipPhase(tc, "precision-gate")
|
||||
return nil
|
||||
}
|
||||
tc.prReview.findings = runPrecisionGate(ctx, tc, tc.prReview.findings)
|
||||
return nil
|
||||
}}
|
||||
}
|
||||
|
||||
// precisionGateSchema is the precision gate's capture-tool schema: one
|
||||
// keep/drop verdict per finding index, the same index-referencing shape the
|
||||
// evidence verifier and adversary already use.
|
||||
var precisionGateSchema = json.RawMessage(`{
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"verdicts": {
|
||||
"type": "array",
|
||||
"items": {
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"index": {"type": "integer"},
|
||||
"keep": {"type": "boolean"}
|
||||
},
|
||||
"required": ["index", "keep"]
|
||||
}
|
||||
}
|
||||
},
|
||||
"required": ["verdicts"]
|
||||
}`)
|
||||
|
||||
// runPrecisionGate runs the precision gate's single classification-role call
|
||||
// over every current finding. A failed or malformed call keeps every finding
|
||||
// unfiltered -- the same recall-first fallback runEvidenceVerifier and
|
||||
// runAdversary already use: a gate that could not run is not evidence any
|
||||
// finding should drop.
|
||||
func runPrecisionGate(ctx context.Context, tc *TaskContext, findings []prreview.Finding) []prreview.Finding {
|
||||
var listing strings.Builder
|
||||
for i, f := range findings {
|
||||
fmt.Fprintf(&listing, "Finding %d [%s]: %s (%s:%d)\n%s\nEvidence: %s\n\n",
|
||||
i, f.Severity, f.Title, f.File, f.LineStart, f.Body, f.Evidence)
|
||||
}
|
||||
mission := fmt.Sprintf(
|
||||
"Decide which of these findings are worth posting to a human reviewer. Keep every "+
|
||||
"concrete, evidenced defect. Drop nitpicks, pure style complaints, and any claim "+
|
||||
"you cannot verify from the evidence shown. Keep a finding when you are unsure.\n\n"+
|
||||
"%s"+
|
||||
"Call precision_gate exactly once with one verdict per finding index, then call "+
|
||||
"finish with status \"passed\".",
|
||||
listing.String(),
|
||||
)
|
||||
res, err := runPRReviewAgentRecorded(ctx, tc, tc.prReview.snapshotDir, "precision-gate", "classification", mission, 15, []agentexec.CaptureTool{{
|
||||
Name: "precision_gate", Description: "Record each finding's keep/drop verdict. Call exactly once, before finish.",
|
||||
Parameters: precisionGateSchema, RequiredFields: []string{"verdicts"}, MaxCalls: 1,
|
||||
}})
|
||||
if err != nil || res.Status != agentexec.StatusPassed {
|
||||
return findings //nolint:nilerr // recall-first: a gate that could not run is not evidence any finding should drop
|
||||
}
|
||||
calls := res.Captures["precision_gate"]
|
||||
if len(calls) != 1 {
|
||||
return findings
|
||||
}
|
||||
var captured struct {
|
||||
Verdicts []struct {
|
||||
Index int `json:"index"`
|
||||
Keep bool `json:"keep"`
|
||||
} `json:"verdicts"`
|
||||
}
|
||||
if json.Unmarshal(calls[0], &captured) != nil {
|
||||
return findings
|
||||
}
|
||||
drop := make(map[int]bool, len(captured.Verdicts))
|
||||
for _, v := range captured.Verdicts {
|
||||
if !v.Keep {
|
||||
drop[v.Index] = true
|
||||
}
|
||||
}
|
||||
kept := make([]prreview.Finding, 0, len(findings))
|
||||
for i, f := range findings {
|
||||
if drop[i] {
|
||||
continue
|
||||
}
|
||||
kept = append(kept, f)
|
||||
}
|
||||
return kept
|
||||
}
|
||||
|
||||
// stagePROperatorApproval is the PRD's Operator approval gate: active only
|
||||
// when review.approve_before_post is set, it runs immediately after
|
||||
// synthesis and before the merge gate, so an operator's re-review (which
|
||||
// reruns phases 3 to 8 with instructions folded in) never has to undo a
|
||||
// merge-gate verdict computed on a since-changed finding set. A review with
|
||||
// no findings never waits, per the PRD, and continues straight to the merge
|
||||
// gate. Setting tc.Outcome here ends the workflow run at this stage --
|
||||
// docs/prds/execution-tree-state-machine.md's engine loop stops as soon as a
|
||||
// stage sets a non-empty Outcome.Status -- so the merge gate and output
|
||||
// stages never run until an operator's response requeues the task.
|
||||
func stagePROperatorApproval() Stage {
|
||||
return Stage{Name: "operator-approval", Run: func(_ context.Context, tc *TaskContext) error {
|
||||
if !tc.Cfg.Review.ApproveBeforePost || len(tc.prReview.scored) == 0 {
|
||||
return nil
|
||||
}
|
||||
tc.Outcome = Outcome{
|
||||
Status: StatusWaitingHuman,
|
||||
Detail: fmt.Sprintf("%d finding(s) awaiting operator review before posting", len(tc.prReview.scored)),
|
||||
}
|
||||
return nil
|
||||
}}
|
||||
}
|
||||
|
||||
// evidencePackageFor extracts a finding's evidence package from the
|
||||
// snapshot, reusing afbk.1's ExtractEvidence. A file the snapshot no longer
|
||||
// has (a finding pointing at a path that does not exist) extracts an empty
|
||||
// package rather than failing the stage: that emptiness is itself the
|
||||
// evidence verifier's answer.
|
||||
func evidencePackageFor(tc *TaskContext, f prreview.Finding) prreview.EvidencePackage {
|
||||
pkg, err := prreview.ExtractEvidence(os.DirFS(tc.prReview.snapshotDir), prreview.EvidenceRequest{
|
||||
File: f.File, LineStart: f.LineStart, LineEnd: f.LineEnd,
|
||||
Title: f.Title, Body: f.Body, Evidence: f.Evidence,
|
||||
})
|
||||
if err != nil {
|
||||
return prreview.EvidencePackage{FindingTitle: f.Title, File: f.File}
|
||||
}
|
||||
return pkg
|
||||
}
|
||||
|
||||
// verifyFindingsSchema is the evidence verifier's capture-tool schema.
|
||||
var verifyFindingsSchema = json.RawMessage(`{
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"verdicts": {
|
||||
"type": "array",
|
||||
"items": {
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"index": {"type": "integer"},
|
||||
"supported": {"type": "boolean"},
|
||||
"reason": {"type": "string"}
|
||||
},
|
||||
"required": ["index", "supported"]
|
||||
}
|
||||
}
|
||||
},
|
||||
"required": ["verdicts"]
|
||||
}`)
|
||||
|
||||
// evidenceCandidate is one high-priority finding the evidence verifier is
|
||||
// asked to check, paired with its index in the caller's original slice: the
|
||||
// verifier's answer references findings by that index, not by content.
|
||||
type evidenceCandidate struct {
|
||||
index int
|
||||
finding prreview.Finding
|
||||
}
|
||||
|
||||
// highPriorityCandidates picks the findings runEvidenceVerifier checks:
|
||||
// every high-priority finding, each still carrying its position in findings.
|
||||
func highPriorityCandidates(findings []prreview.Finding) []evidenceCandidate {
|
||||
var candidates []evidenceCandidate
|
||||
for i, f := range findings {
|
||||
if prreview.IsHighPriority(f) {
|
||||
candidates = append(candidates, evidenceCandidate{index: i, finding: f})
|
||||
}
|
||||
}
|
||||
return candidates
|
||||
}
|
||||
|
||||
// evidenceListing renders each candidate's claim and evidence package as the
|
||||
// evidence verifier's mission text.
|
||||
func evidenceListing(tc *TaskContext, candidates []evidenceCandidate) string {
|
||||
var listing strings.Builder
|
||||
for _, c := range candidates {
|
||||
pkg := evidencePackageFor(tc, c.finding)
|
||||
fmt.Fprintf(&listing, "Finding %d: %s (%s:%d)\nClaim: %s\nQuoted evidence: %s\nCode at the cited lines:\n%s\n\n",
|
||||
c.index, c.finding.Title, c.finding.File, c.finding.LineStart, c.finding.Body, c.finding.Evidence, pkg.PrimaryCode)
|
||||
}
|
||||
return listing.String()
|
||||
}
|
||||
|
||||
// runEvidenceVerifier runs phase 5's evidence verifier over every
|
||||
// high-priority (critical or important) finding, dropping the ones it finds
|
||||
// unsupported by their own evidence package. A finding below high priority is
|
||||
// never sent, and always kept: the verifier exists to catch a hallucinated
|
||||
// critical, not to grade a nitpick. A failed or malformed verifier call keeps
|
||||
// every finding unfiltered -- the pipeline is recall-first, so a verifier
|
||||
// that could not run is not evidence any finding is wrong; the error is
|
||||
// intentionally swallowed here, not propagated.
|
||||
func runEvidenceVerifier(ctx context.Context, tc *TaskContext, findings []prreview.Finding) ([]prreview.Finding, error) {
|
||||
candidates := highPriorityCandidates(findings)
|
||||
if len(candidates) == 0 {
|
||||
return findings, nil
|
||||
}
|
||||
|
||||
mission := fmt.Sprintf(
|
||||
"Each finding below cites code as evidence for its claim. Check whether the "+
|
||||
"quoted code at the cited lines actually supports the claim -- not whether "+
|
||||
"the claim is a good catch, only whether the evidence is real and says what "+
|
||||
"the finding says it says.\n\n%s\n"+
|
||||
"Call verify_findings exactly once with one verdict per finding index, then "+
|
||||
"call finish with status \"passed\".",
|
||||
evidenceListing(tc, candidates),
|
||||
)
|
||||
res, err := runPRReviewAgentRecorded(ctx, tc, tc.prReview.snapshotDir, "evidence-verifier", "review", mission, 15, []agentexec.CaptureTool{{
|
||||
Name: "verify_findings", Description: "Record each finding's evidence verdict. Call exactly once, before finish.",
|
||||
Parameters: verifyFindingsSchema, RequiredFields: []string{"verdicts"}, MaxCalls: 1,
|
||||
}})
|
||||
if err != nil || res.Status != agentexec.StatusPassed {
|
||||
return findings, nil //nolint:nilerr // recall-first: a verifier that could not run is not evidence any finding is wrong, so nothing is filtered
|
||||
}
|
||||
calls := res.Captures["verify_findings"]
|
||||
if len(calls) != 1 {
|
||||
return findings, nil
|
||||
}
|
||||
var captured struct {
|
||||
Verdicts []struct {
|
||||
Index int `json:"index"`
|
||||
Supported bool `json:"supported"`
|
||||
} `json:"verdicts"`
|
||||
}
|
||||
if err := json.Unmarshal(calls[0], &captured); err != nil {
|
||||
return findings, nil //nolint:nilerr // malformed verifier output keeps findings unfiltered rather than failing the stage
|
||||
}
|
||||
unsupported := make(map[int]bool, len(captured.Verdicts))
|
||||
for _, v := range captured.Verdicts {
|
||||
if !v.Supported {
|
||||
unsupported[v.Index] = true
|
||||
}
|
||||
}
|
||||
|
||||
kept := make([]prreview.Finding, 0, len(findings))
|
||||
for i, f := range findings {
|
||||
if unsupported[i] {
|
||||
continue
|
||||
}
|
||||
kept = append(kept, f)
|
||||
}
|
||||
return kept, nil
|
||||
}
|
||||
|
||||
// adversaryVerdictsSchema is the adversary's verdict capture-tool schema.
|
||||
var adversaryVerdictsSchema = json.RawMessage(`{
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"verdicts": {
|
||||
"type": "array",
|
||||
"items": {
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"index": {"type": "integer"},
|
||||
"verdict": {"type": "string", "enum": ["confirmed", "challenged"]},
|
||||
"reason": {"type": "string"}
|
||||
},
|
||||
"required": ["index", "verdict"]
|
||||
}
|
||||
}
|
||||
},
|
||||
"required": ["verdicts"]
|
||||
}`)
|
||||
|
||||
// adversarySkepticism is the extra instruction the adversary's mission
|
||||
// carries when intake scored the PR as likely machine-written, per the PRD.
|
||||
func adversarySkepticism(aiGenerated float64) string {
|
||||
if prreview.IsAIGenerated(aiGenerated) {
|
||||
return "This PR's own description reads as likely machine-written: be more sceptical " +
|
||||
"than usual, since a machine-written description is more likely to accompany " +
|
||||
"machine-written code with the same failure modes."
|
||||
}
|
||||
return "This PR's own description does not read as machine-written; review normally."
|
||||
}
|
||||
|
||||
// findingsListing renders every finding, numbered by its index, for a
|
||||
// mission that must reference findings back by that index.
|
||||
func findingsListing(findings []prreview.Finding) string {
|
||||
var listing strings.Builder
|
||||
for i, f := range findings {
|
||||
fmt.Fprintf(&listing, "Finding %d [%s] %s (%s:%d): %s\nEvidence: %s\n\n",
|
||||
i, f.Severity, f.Title, f.File, f.LineStart, f.Body, f.Evidence)
|
||||
}
|
||||
return listing.String()
|
||||
}
|
||||
|
||||
// applyAdversaryVerdicts decodes one adversary_verdicts capture and sets each
|
||||
// referenced finding's Adversary field. An out-of-range index (malformed or
|
||||
// adversarial model output) is skipped rather than panicking.
|
||||
func applyAdversaryVerdicts(out []prreview.Finding, calls []json.RawMessage) {
|
||||
if len(calls) != 1 {
|
||||
return
|
||||
}
|
||||
var captured struct {
|
||||
Verdicts []struct {
|
||||
Index int `json:"index"`
|
||||
Verdict string `json:"verdict"`
|
||||
} `json:"verdicts"`
|
||||
}
|
||||
if json.Unmarshal(calls[0], &captured) != nil {
|
||||
return
|
||||
}
|
||||
for _, v := range captured.Verdicts {
|
||||
if v.Index < 0 || v.Index >= len(out) {
|
||||
continue
|
||||
}
|
||||
switch v.Verdict {
|
||||
case string(prreview.AdversaryConfirmed):
|
||||
out[v.Index].Adversary = prreview.AdversaryConfirmed
|
||||
case string(prreview.AdversaryChallenged):
|
||||
out[v.Index].Adversary = prreview.AdversaryChallenged
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// runAdversary runs phase 5's adversary over every surviving finding at
|
||||
// once: it confirms or challenges each by index, and may report new findings
|
||||
// no reviewer caught. It reads intake's machine-written confidence and is
|
||||
// told to be more sceptical above the threshold, per the PRD. A failed or
|
||||
// malformed call leaves every finding's Adversary verdict unset (neither
|
||||
// confirmed nor challenged) and adds nothing -- an adversary that could not
|
||||
// run has reached no verdict, which is different from confirming everything.
|
||||
func runAdversary(ctx context.Context, tc *TaskContext, findings []prreview.Finding) ([]prreview.Finding, error) {
|
||||
if len(findings) == 0 {
|
||||
return findings, nil
|
||||
}
|
||||
|
||||
mission := fmt.Sprintf(
|
||||
"You are the adversary reviewing this pull request's findings. For each finding "+
|
||||
"below, confirm it (you agree it is a real defect) or challenge it (you believe "+
|
||||
"it is wrong or not actually a problem). Read the snapshot yourself to check. "+
|
||||
"%s Also look for any defect every reviewer missed and report it as a new "+
|
||||
"finding.\n\n%s\n"+
|
||||
"Call adversary_verdicts exactly once with one verdict per finding index. If you "+
|
||||
"found anything new, call report_findings once too (skip it if you found "+
|
||||
"nothing new). Then call finish with status \"passed\".",
|
||||
adversarySkepticism(tc.prReview.aiGenerated), findingsListing(findings),
|
||||
)
|
||||
res, err := runPRReviewAgentRecorded(ctx, tc, tc.prReview.snapshotDir, "adversary", "review", mission, 25, []agentexec.CaptureTool{
|
||||
{
|
||||
Name: "adversary_verdicts", Description: "Record each finding's adversary verdict. Call exactly once, before finish.",
|
||||
Parameters: adversaryVerdictsSchema, RequiredFields: []string{"verdicts"}, MaxCalls: 1,
|
||||
},
|
||||
reportFindingsTool,
|
||||
})
|
||||
if err != nil || res.Status != agentexec.StatusPassed {
|
||||
return findings, nil //nolint:nilerr // a call that could not run reached no verdict, which the pipeline treats as neutral, not as a stage failure
|
||||
}
|
||||
|
||||
out := append([]prreview.Finding{}, findings...)
|
||||
applyAdversaryVerdicts(out, res.Captures["adversary_verdicts"])
|
||||
if extra, err := decodeReportedFindings(res.Captures["report_findings"], "adversary"); err == nil {
|
||||
out = append(out, extra...)
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
|
||||
// runCompoundCheck runs phase 5's compound-defect check once per cluster
|
||||
// that has more than one finding, in parallel, bounded the same as every
|
||||
// other pr-review fan-out. It flags the findings the agent names as
|
||||
// compounding (Finding.Compound = true), which is what scoring's compound
|
||||
// multiplier reads. A cluster with at most one finding cannot compound with
|
||||
// anything and is skipped without an agent call.
|
||||
func runCompoundCheck(ctx context.Context, tc *TaskContext, findings []prreview.Finding) ([]prreview.Finding, error) {
|
||||
grouped := prreview.ClusterFindings(tc.prReview.clusters, findings)
|
||||
var clusterIDs []string
|
||||
for id, members := range grouped {
|
||||
if len(members) > 1 {
|
||||
clusterIDs = append(clusterIDs, id)
|
||||
}
|
||||
}
|
||||
if len(clusterIDs) == 0 {
|
||||
return findings, nil
|
||||
}
|
||||
|
||||
flagged := make([]map[string]bool, len(clusterIDs))
|
||||
forEachBounded(prReviewConcurrency, len(clusterIDs), func(i int) {
|
||||
flagged[i] = runCompoundClusterCheck(ctx, tc, clusterIDs[i], grouped[clusterIDs[i]])
|
||||
})
|
||||
|
||||
compoundTitles := map[string]bool{}
|
||||
for _, set := range flagged {
|
||||
for title := range set {
|
||||
compoundTitles[title] = true
|
||||
}
|
||||
}
|
||||
out := append([]prreview.Finding{}, findings...)
|
||||
for i, f := range out {
|
||||
if compoundTitles[compoundKey(f)] {
|
||||
out[i].Compound = true
|
||||
}
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
|
||||
// compoundKey is how runCompoundCheck matches a cluster agent's flagged
|
||||
// member back to a finding in the full list: findings carry no stable ID, so
|
||||
// (file, line, title) is the identity a compound call's answer is matched on.
|
||||
func compoundKey(f prreview.Finding) string {
|
||||
return fmt.Sprintf("%s:%d:%s", f.File, f.LineStart, f.Title)
|
||||
}
|
||||
|
||||
// runCompoundClusterCheck runs one phase-5 compound call over one cluster's
|
||||
// findings, returning the set of member keys (compoundKey) the agent flagged
|
||||
// as only defective in combination. A failed or malformed call flags nothing.
|
||||
func runCompoundClusterCheck(ctx context.Context, tc *TaskContext, clusterID string, members []prreview.Finding) map[string]bool {
|
||||
var listing strings.Builder
|
||||
for i, f := range members {
|
||||
fmt.Fprintf(&listing, "Finding %d [%s] %s (%s:%d): %s\n\n", i, f.Severity, f.Title, f.File, f.LineStart, f.Body)
|
||||
}
|
||||
params := json.RawMessage(`{
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"indices": {"type": "array", "items": {"type": "integer"}}
|
||||
},
|
||||
"required": ["indices"]
|
||||
}`)
|
||||
mission := fmt.Sprintf(
|
||||
"These findings all touch the same cluster of files. Read them together: is "+
|
||||
"there a defect that only exists because of how two or more of them combine -- "+
|
||||
"one that neither finding describes on its own? If so, call compound_findings "+
|
||||
"once with the indices of the findings that compound (an empty array if none "+
|
||||
"do). Then call finish with status \"passed\".\n\n%s", listing.String(),
|
||||
)
|
||||
res, err := runPRReviewAgentRecorded(ctx, tc, tc.prReview.snapshotDir, "compound-"+clusterID, "review", mission, 15, []agentexec.CaptureTool{{
|
||||
Name: "compound_findings", Description: "Record which findings compound. Call exactly once, before finish.",
|
||||
Parameters: params, RequiredFields: []string{"indices"}, MaxCalls: 1,
|
||||
}})
|
||||
if err != nil || res.Status != agentexec.StatusPassed {
|
||||
return nil //nolint:nilerr // a cluster call that could not run flags nothing rather than failing the whole compound-check stage
|
||||
}
|
||||
calls := res.Captures["compound_findings"]
|
||||
if len(calls) != 1 {
|
||||
return nil
|
||||
}
|
||||
var captured struct {
|
||||
Indices []int `json:"indices"`
|
||||
}
|
||||
if err := json.Unmarshal(calls[0], &captured); err != nil {
|
||||
return nil
|
||||
}
|
||||
flagged := make(map[string]bool, len(captured.Indices))
|
||||
for _, index := range captured.Indices {
|
||||
if index < 0 || index >= len(members) {
|
||||
continue
|
||||
}
|
||||
flagged[compoundKey(members[index])] = true
|
||||
}
|
||||
return flagged
|
||||
}
|
||||
|
||||
// prReviewGapRounds is phase 6's cap on gap-review rounds: at most 2, per the
|
||||
// PRD, so a coverage gap the gap reviewers cannot close does not loop forever.
|
||||
const prReviewGapRounds = 2
|
||||
|
||||
// stagePRCoverageConsistency is pipeline phase 6: the coverage gate and
|
||||
// consistency verification run in parallel and both append to
|
||||
// tc.prReview.findings, guarded by prReviewState.findingsMu
|
||||
// (docs/prds/pr-review-agent.md, phase 6: "Coverage and consistency, in
|
||||
// parallel").
|
||||
func stagePRCoverageConsistency() Stage {
|
||||
return Stage{Name: "coverage-consistency", Run: func(ctx context.Context, tc *TaskContext) error {
|
||||
var wg sync.WaitGroup
|
||||
var consistencyErr error
|
||||
wg.Go(func() {
|
||||
consistencyErr = runConsistencyVerification(ctx, tc)
|
||||
})
|
||||
runCoverageGate(ctx, tc)
|
||||
wg.Wait()
|
||||
return consistencyErr
|
||||
}}
|
||||
}
|
||||
|
||||
// runCoverageGate is phase 6's coverage gate: for up to prReviewGapRounds
|
||||
// rounds, find the clusters and high-exposure blast-radius files no finding
|
||||
// covers, run one gap reviewer per gap (reusing runReviewer, the same
|
||||
// reviewer definition phase 4 uses), and append whatever they find. It stops
|
||||
// early once nothing is uncovered, so a change that phase 4 already covered
|
||||
// fully never spends a gap-review call.
|
||||
func runCoverageGate(ctx context.Context, tc *TaskContext) {
|
||||
for range prReviewGapRounds {
|
||||
tc.prReview.findingsMu.Lock()
|
||||
uncoveredClusters, uncoveredFiles := prreview.CoverageGaps(tc.prReview.clusters, tc.prReview.highExposure, tc.prReview.findings)
|
||||
tc.prReview.findingsMu.Unlock()
|
||||
if len(uncoveredClusters) == 0 && len(uncoveredFiles) == 0 {
|
||||
return
|
||||
}
|
||||
|
||||
dims := gapDimensions(uncoveredClusters, uncoveredFiles)
|
||||
found := make([][]prreview.Finding, len(dims))
|
||||
forEachBounded(prReviewConcurrency, len(dims), func(i int) {
|
||||
found[i] = runReviewer(ctx, tc, dims[i])
|
||||
})
|
||||
|
||||
tc.prReview.findingsMu.Lock()
|
||||
for _, f := range found {
|
||||
tc.prReview.findings = append(tc.prReview.findings, f...)
|
||||
}
|
||||
tc.prReview.findingsMu.Unlock()
|
||||
}
|
||||
}
|
||||
|
||||
// gapDimensions builds one reviewer dimension per coverage gap: a cluster's
|
||||
// files together (a cluster is reviewed as one unit, the way phase 4 would
|
||||
// have if a lens had proposed it), and one per uncovered high-exposure file.
|
||||
func gapDimensions(clusters []prreview.Cluster, files []string) []prreview.Dimension {
|
||||
dims := make([]prreview.Dimension, 0, len(clusters)+len(files))
|
||||
for _, cluster := range clusters {
|
||||
dims = append(dims, prreview.Dimension{
|
||||
Name: "gap-" + cluster.ID,
|
||||
Prompt: "This cluster of changed files was not reached by any review dimension. Review it for anything a reviewer would normally catch.",
|
||||
TargetFiles: cluster.Files,
|
||||
})
|
||||
}
|
||||
for _, file := range files {
|
||||
dims = append(dims, prreview.Dimension{
|
||||
Name: "gap-exposure-" + file,
|
||||
Prompt: "This file is reached by more than one changed file (high blast-radius exposure) but no reviewer covered it. Review it for anything the changes upstream may have broken here.",
|
||||
TargetFiles: []string{file},
|
||||
})
|
||||
}
|
||||
return dims
|
||||
}
|
||||
|
||||
// runConsistencyVerification is phase 6's consistency verification: one
|
||||
// agent call lists the cross-location obligations the changed code creates
|
||||
// and verifies each by reading both ends, reporting only the broken ones as
|
||||
// findings (the PRD's own wording -- "a broken obligation becomes a
|
||||
// finding"). It reuses report_findings, the same shape every other
|
||||
// findings-reporting call in the pipeline uses.
|
||||
func runConsistencyVerification(ctx context.Context, tc *TaskContext) error {
|
||||
mission := fmt.Sprintf(
|
||||
"Read this pull request's diff and the surrounding repository. List every "+
|
||||
"cross-location obligation the changed code creates: what must be true "+
|
||||
"elsewhere in the codebase for a changed line to be correct (a caller that "+
|
||||
"must pass a new argument, a schema a struct must still match, a doc that must "+
|
||||
"still describe the behaviour, an invariant a comment states that the new code "+
|
||||
"must uphold). For each obligation, verify it by reading both ends yourself. "+
|
||||
"Report only the obligations you found broken as findings -- an obligation you "+
|
||||
"checked and found intact is not a finding.\n\nDiff:\n%s\n\n"+
|
||||
"Call report_findings once (an empty array if every obligation you checked "+
|
||||
"held), then call finish with status \"passed\".",
|
||||
clip(tc.prReview.diff, 60000),
|
||||
)
|
||||
res, err := runPRReviewAgentRecorded(ctx, tc, tc.prReview.snapshotDir, "consistency", "review", mission, 25, []agentexec.CaptureTool{reportFindingsTool})
|
||||
if err != nil || res.Status != agentexec.StatusPassed {
|
||||
return nil //nolint:nilerr // a consistency call that could not run reports no broken obligations rather than failing the stage
|
||||
}
|
||||
found, err := decodeReportedFindings(res.Captures["report_findings"], "consistency")
|
||||
if err != nil {
|
||||
return nil //nolint:nilerr // malformed consistency output reports no broken obligations rather than failing the stage
|
||||
}
|
||||
|
||||
tc.prReview.findingsMu.Lock()
|
||||
tc.prReview.findings = append(tc.prReview.findings, found...)
|
||||
tc.prReview.findingsMu.Unlock()
|
||||
return nil
|
||||
}
|
||||
|
||||
// mergeGateSchema is the merge gate's classification-call capture-tool
|
||||
// schema.
|
||||
var mergeGateSchema = json.RawMessage(`{
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"blocking": {"type": "boolean", "description": "true only for a broken build, a security hole, data loss, a contract break or a regression."}
|
||||
},
|
||||
"required": ["blocking"]
|
||||
}`)
|
||||
|
||||
// stagePRMergeGate is pipeline phase 8: one classification call per
|
||||
// surviving finding decides blocking or advisory (docs/prds/pr-review-
|
||||
// agent.md, phase 8). Blocking is narrow, and a failed or malformed call
|
||||
// leaves a finding advisory -- the same fallback every other pr-review
|
||||
// verdict call in this pipeline uses when its agent call could not run.
|
||||
func stagePRMergeGate() Stage {
|
||||
return Stage{Name: "merge-gate", Run: func(ctx context.Context, tc *TaskContext) error {
|
||||
if len(tc.prReview.scored) == 0 {
|
||||
return nil
|
||||
}
|
||||
if budgetExhausted(tc, "merge-gate") {
|
||||
skipPhase(tc, "merge-gate")
|
||||
return nil
|
||||
}
|
||||
blocking := make([]bool, len(tc.prReview.scored))
|
||||
forEachBounded(prReviewConcurrency, len(tc.prReview.scored), func(i int) {
|
||||
blocking[i] = runMergeGateCall(ctx, tc, tc.prReview.scored[i])
|
||||
})
|
||||
for i := range tc.prReview.scored {
|
||||
tc.prReview.scored[i].Blocking = blocking[i]
|
||||
}
|
||||
return nil
|
||||
}}
|
||||
}
|
||||
|
||||
// runMergeGateCall runs one phase-8 classification call over one finding.
|
||||
func runMergeGateCall(ctx context.Context, tc *TaskContext, f prreview.ScoredFinding) bool {
|
||||
mission := fmt.Sprintf(
|
||||
"Decide whether this finding must block the change from merging. Blocking is "+
|
||||
"narrow: only a broken build, a security hole, data loss, a contract break, or "+
|
||||
"a regression qualifies. Everything else -- including a real but non-blocking "+
|
||||
"defect -- is advisory.\n\n[%s] %s (%s:%d)\n%s\n\n"+
|
||||
"Call classify_blocking exactly once, then call finish with status \"passed\".",
|
||||
f.Severity, f.Title, f.File, f.LineStart, f.Body,
|
||||
)
|
||||
res, err := runPRReviewAgentRecorded(ctx, tc, tc.prReview.snapshotDir, "merge-gate", "classification", mission, 6, []agentexec.CaptureTool{{
|
||||
Name: "classify_blocking", Description: "Record the blocking verdict. Call exactly once, before finish.",
|
||||
Parameters: mergeGateSchema, RequiredFields: []string{"blocking"}, MaxCalls: 1,
|
||||
}})
|
||||
if err != nil || res.Status != agentexec.StatusPassed {
|
||||
return false //nolint:nilerr // a merge-gate call that could not run leaves the finding advisory, per the PRD
|
||||
}
|
||||
calls := res.Captures["classify_blocking"]
|
||||
if len(calls) != 1 {
|
||||
return false
|
||||
}
|
||||
var captured struct {
|
||||
Blocking bool `json:"blocking"`
|
||||
}
|
||||
if json.Unmarshal(calls[0], &captured) != nil {
|
||||
return false
|
||||
}
|
||||
return captured.Blocking
|
||||
}
|
||||
@@ -0,0 +1,263 @@
|
||||
package workflow
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"sync/atomic"
|
||||
"testing"
|
||||
|
||||
"github.com/samcharles93/archie-core/internal/agentexec"
|
||||
"github.com/samcharles93/archie-core/internal/domain/workflow/prreview"
|
||||
)
|
||||
|
||||
// writeSnapshotFile is a small helper for tests that need a real file on disk
|
||||
// for evidence extraction to read.
|
||||
func writeSnapshotFile(t *testing.T, dir, path, content string) {
|
||||
t.Helper()
|
||||
full := filepath.Join(dir, path)
|
||||
if err := os.MkdirAll(filepath.Dir(full), 0o755); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := os.WriteFile(full, []byte(content), 0o644); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestStagePRVerificationDropsUnsupportedHighPriorityFindings(t *testing.T) {
|
||||
tc := baseTaskContext(t)
|
||||
dir := t.TempDir()
|
||||
writeSnapshotFile(t, dir, "a.go", "package a\n\nfunc F() int { return 1 }\n")
|
||||
tc.prReview = &prReviewState{
|
||||
snapshotDir: dir,
|
||||
findings: []prreview.Finding{
|
||||
{File: "a.go", LineStart: 3, LineEnd: 3, Severity: prreview.SeverityCritical, Title: "unsupported", Confidence: 0.9},
|
||||
{File: "a.go", LineStart: 3, LineEnd: 3, Severity: prreview.SeverityNitpick, Title: "low priority", Confidence: 0.5},
|
||||
},
|
||||
}
|
||||
tc.Agent = &concurrentAgentRunner{byStage: func(stage string) (agentexec.Result, error) {
|
||||
switch stage {
|
||||
case "evidence-verifier":
|
||||
return captureResult("verify_findings", map[string]any{
|
||||
"verdicts": []map[string]any{{"index": 0, "supported": false}},
|
||||
}), nil
|
||||
case "adversary":
|
||||
return captureResult("adversary_verdicts", map[string]any{"verdicts": []map[string]any{}}), nil
|
||||
default:
|
||||
return passedResult("ok"), nil
|
||||
}
|
||||
}}
|
||||
|
||||
if err := stagePRVerification().Run(context.Background(), tc); err != nil {
|
||||
t.Fatalf("verification: %v", err)
|
||||
}
|
||||
if len(tc.prReview.findings) != 1 {
|
||||
t.Fatalf("findings = %d, want 1 (the unsupported critical finding dropped)", len(tc.prReview.findings))
|
||||
}
|
||||
if tc.prReview.findings[0].Title != "low priority" {
|
||||
t.Errorf("surviving finding = %q, want the low-priority one the verifier never checked", tc.prReview.findings[0].Title)
|
||||
}
|
||||
}
|
||||
|
||||
func TestStagePRVerificationAppliesAdversaryVerdicts(t *testing.T) {
|
||||
tc := baseTaskContext(t)
|
||||
dir := t.TempDir()
|
||||
writeSnapshotFile(t, dir, "a.go", "package a\n\nfunc F() int { return 1 }\n")
|
||||
tc.prReview = &prReviewState{
|
||||
snapshotDir: dir,
|
||||
findings: []prreview.Finding{
|
||||
{File: "a.go", LineStart: 3, LineEnd: 3, Severity: prreview.SeverityImportant, Title: "f0", Confidence: 0.9},
|
||||
},
|
||||
}
|
||||
tc.Agent = &concurrentAgentRunner{byStage: func(stage string) (agentexec.Result, error) {
|
||||
switch stage {
|
||||
case "evidence-verifier":
|
||||
return captureResult("verify_findings", map[string]any{
|
||||
"verdicts": []map[string]any{{"index": 0, "supported": true}},
|
||||
}), nil
|
||||
case "adversary":
|
||||
return captureResult("adversary_verdicts", map[string]any{
|
||||
"verdicts": []map[string]any{{"index": 0, "verdict": "confirmed"}},
|
||||
}), nil
|
||||
default:
|
||||
return passedResult("ok"), nil
|
||||
}
|
||||
}}
|
||||
|
||||
if err := stagePRVerification().Run(context.Background(), tc); err != nil {
|
||||
t.Fatalf("verification: %v", err)
|
||||
}
|
||||
if len(tc.prReview.findings) != 1 {
|
||||
t.Fatalf("findings = %d, want 1", len(tc.prReview.findings))
|
||||
}
|
||||
if tc.prReview.findings[0].Adversary != prreview.AdversaryConfirmed {
|
||||
t.Errorf("adversary verdict = %q, want confirmed", tc.prReview.findings[0].Adversary)
|
||||
}
|
||||
}
|
||||
|
||||
func TestStagePRVerificationRunsNoAgentCallsWhenNoFindings(t *testing.T) {
|
||||
tc := baseTaskContext(t)
|
||||
tc.prReview = &prReviewState{snapshotDir: t.TempDir()}
|
||||
tc.Agent = &concurrentAgentRunner{byStage: func(string) (agentexec.Result, error) {
|
||||
t.Fatal("no agent call expected with zero findings")
|
||||
return agentexec.Result{}, nil
|
||||
}}
|
||||
if err := stagePRVerification().Run(context.Background(), tc); err != nil {
|
||||
t.Fatalf("verification: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestStagePRCompoundMarksFlaggedFindingsWithinCluster(t *testing.T) {
|
||||
tc := baseTaskContext(t)
|
||||
dir := t.TempDir()
|
||||
tc.prReview = &prReviewState{
|
||||
snapshotDir: dir,
|
||||
clusters: []prreview.Cluster{{ID: "cluster_0", Files: []string{"a.go", "b.go"}}},
|
||||
findings: []prreview.Finding{
|
||||
{File: "a.go", LineStart: 1, Title: "f0", Severity: prreview.SeverityImportant, Confidence: 0.9},
|
||||
{File: "b.go", LineStart: 1, Title: "f1", Severity: prreview.SeverityImportant, Confidence: 0.9},
|
||||
},
|
||||
}
|
||||
tc.Agent = &concurrentAgentRunner{byStage: func(stage string) (agentexec.Result, error) {
|
||||
if stage == "compound-cluster_0" {
|
||||
return captureResult("compound_findings", map[string]any{"indices": []int{0, 1}}), nil
|
||||
}
|
||||
return passedResult("ok"), nil
|
||||
}}
|
||||
|
||||
findings, err := runCompoundCheck(context.Background(), tc, tc.prReview.findings)
|
||||
if err != nil {
|
||||
t.Fatalf("runCompoundCheck: %v", err)
|
||||
}
|
||||
for _, f := range findings {
|
||||
if !f.Compound {
|
||||
t.Errorf("finding %q Compound = false, want true (both findings in the flagged cluster)", f.Title)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestStagePRCoverageConsistencyRunsGapReviewersAndConsistencyInParallel(t *testing.T) {
|
||||
tc := baseTaskContext(t)
|
||||
dir := t.TempDir()
|
||||
tc.prReview = &prReviewState{
|
||||
snapshotDir: dir,
|
||||
clusters: []prreview.Cluster{{ID: "cluster_0", Name: "pkg", Files: []string{"pkg/a.go"}}},
|
||||
highExposure: []string{"pkg/shared.go"},
|
||||
findings: nil,
|
||||
}
|
||||
tc.Agent = &concurrentAgentRunner{byStage: func(stage string) (agentexec.Result, error) {
|
||||
switch stage {
|
||||
case "consistency":
|
||||
return captureResult("report_findings", map[string]any{
|
||||
"findings": []map[string]any{
|
||||
{"file": "pkg/a.go", "line_start": 1, "severity": "important", "title": "broken obligation", "body": "b", "evidence": "e", "confidence": 0.8},
|
||||
},
|
||||
}), nil
|
||||
default:
|
||||
// Gap reviewers reuse runReviewer's report_findings shape.
|
||||
return captureResult("report_findings", map[string]any{"findings": []any{}}), nil
|
||||
}
|
||||
}}
|
||||
|
||||
if err := stagePRCoverageConsistency().Run(context.Background(), tc); err != nil {
|
||||
t.Fatalf("coverage-consistency: %v", err)
|
||||
}
|
||||
|
||||
foundConsistency := false
|
||||
for _, f := range tc.prReview.findings {
|
||||
if f.Title == "broken obligation" {
|
||||
foundConsistency = true
|
||||
}
|
||||
}
|
||||
if !foundConsistency {
|
||||
t.Error("consistency verification's finding was not appended")
|
||||
}
|
||||
}
|
||||
|
||||
func TestCoverageGapCapsAtTwoRounds(t *testing.T) {
|
||||
tc := baseTaskContext(t)
|
||||
tc.prReview = &prReviewState{
|
||||
snapshotDir: t.TempDir(),
|
||||
clusters: []prreview.Cluster{{ID: "cluster_0", Files: []string{"a.go"}}},
|
||||
}
|
||||
var calls int
|
||||
tc.Agent = &concurrentAgentRunner{byStage: func(string) (agentexec.Result, error) {
|
||||
calls++
|
||||
// Every gap reviewer call reports nothing, so the cluster stays
|
||||
// uncovered and a naive loop would run forever without the cap.
|
||||
return captureResult("report_findings", map[string]any{"findings": []any{}}), nil
|
||||
}}
|
||||
|
||||
runCoverageGate(context.Background(), tc)
|
||||
|
||||
if calls != 2 {
|
||||
t.Errorf("gap reviewer calls = %d, want exactly 2 (the round cap)", calls)
|
||||
}
|
||||
}
|
||||
|
||||
func TestStagePRMergeGateSetsBlockingIndependentlyPerFinding(t *testing.T) {
|
||||
tc := baseTaskContext(t)
|
||||
tc.prReview = &prReviewState{
|
||||
snapshotDir: t.TempDir(),
|
||||
scored: []prreview.ScoredFinding{
|
||||
{File: "a.go", LineStart: 1, Title: "sql injection"},
|
||||
{File: "b.go", LineStart: 2, Title: "unused variable"},
|
||||
},
|
||||
}
|
||||
// One call per finding: alternate the verdict by call order, so a bug
|
||||
// that broadcasts one verdict to every finding (rather than one call
|
||||
// each) would leave both findings the same instead of split.
|
||||
var calls atomic.Int32
|
||||
tc.Agent = &concurrentAgentRunner{byStage: func(string) (agentexec.Result, error) {
|
||||
n := calls.Add(1)
|
||||
return captureResult("classify_blocking", map[string]any{"blocking": n == 1}), nil
|
||||
}}
|
||||
|
||||
if err := stagePRMergeGate().Run(context.Background(), tc); err != nil {
|
||||
t.Fatalf("merge-gate: %v", err)
|
||||
}
|
||||
if calls.Load() != 2 {
|
||||
t.Fatalf("classification calls = %d, want 2 (one per finding)", calls.Load())
|
||||
}
|
||||
blockingCount := 0
|
||||
for _, f := range tc.prReview.scored {
|
||||
if f.Blocking {
|
||||
blockingCount++
|
||||
}
|
||||
}
|
||||
if blockingCount != 1 {
|
||||
t.Errorf("blocking findings = %d, want exactly 1 (one call returned true, the other false)", blockingCount)
|
||||
}
|
||||
}
|
||||
|
||||
func TestStagePRMergeGateFailedCallLeavesFindingAdvisory(t *testing.T) {
|
||||
tc := baseTaskContext(t)
|
||||
tc.prReview = &prReviewState{
|
||||
snapshotDir: t.TempDir(),
|
||||
scored: []prreview.ScoredFinding{{File: "a.go", LineStart: 1, Title: "t", Blocking: true}},
|
||||
}
|
||||
tc.Agent = &concurrentAgentRunner{byStage: func(string) (agentexec.Result, error) {
|
||||
return agentexec.Result{}, errors.New("model unavailable")
|
||||
}}
|
||||
|
||||
if err := stagePRMergeGate().Run(context.Background(), tc); err != nil {
|
||||
t.Fatalf("merge-gate: %v", err)
|
||||
}
|
||||
if tc.prReview.scored[0].Blocking {
|
||||
t.Error("Blocking = true, want false: a failed merge-gate call must leave the finding advisory, per the PRD")
|
||||
}
|
||||
}
|
||||
|
||||
func TestStagePRMergeGateSkipsEntirelyWhenNoFindings(t *testing.T) {
|
||||
tc := baseTaskContext(t)
|
||||
tc.prReview = &prReviewState{snapshotDir: t.TempDir()}
|
||||
tc.Agent = &concurrentAgentRunner{byStage: func(string) (agentexec.Result, error) {
|
||||
t.Fatal("no agent call should run when there are no findings to classify")
|
||||
return agentexec.Result{}, nil
|
||||
}}
|
||||
|
||||
if err := stagePRMergeGate().Run(context.Background(), tc); err != nil {
|
||||
t.Fatalf("merge-gate: %v", err)
|
||||
}
|
||||
}
|
||||
@@ -1,281 +0,0 @@
|
||||
package workflow
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
|
||||
"github.com/samcharles93/archie-core/internal/domain/workflow/task"
|
||||
"github.com/samcharles93/archie-core/internal/taskstate"
|
||||
)
|
||||
|
||||
// Reviewer runs an adversarial review of a code snapshot in its own
|
||||
// isolated context and reports what it found. Implementations must give
|
||||
// the reviewer no path to the implementer's conversation, credentials, or
|
||||
// git history -- see docs/prds/adversarial-self-review.md. A reviewer that
|
||||
// errors, is truncated, or reaches no conclusion returns a ReviewReport
|
||||
// with Status == ReviewStatusNotRun rather than a Go error: the stage
|
||||
// always has a report to act on, and "did not run" is a first-class
|
||||
// outcome, not a special case.
|
||||
type Reviewer interface {
|
||||
Review(ctx context.Context, req ReviewRequest) ReviewReport
|
||||
}
|
||||
|
||||
// ReviewRequest is everything an adversarial reviewer is given: a
|
||||
// .git-free snapshot of the reviewed commit's tracked files, the diff that
|
||||
// produced it, and the originating issue text. Nothing else -- commit
|
||||
// messages, branch name, reflog, and the implementer's own transcript are
|
||||
// structurally excluded, not merely withheld by instruction.
|
||||
type ReviewRequest struct {
|
||||
// SnapshotDir is a filesystem directory containing HEAD's tracked
|
||||
// files with no .git present.
|
||||
SnapshotDir string
|
||||
Diff string
|
||||
IssueText string
|
||||
// MaxSteps bounds the reviewer's tool-loop iterations. 0 means the
|
||||
// implementation's own default.
|
||||
MaxSteps int
|
||||
}
|
||||
|
||||
// reviewDetailBytes bounds how much of a park Detail the rendered review
|
||||
// findings occupy, matching the store's own park-reason cap headroom (see
|
||||
// baselineParkOutputBytes in implement.go).
|
||||
const reviewDetailBytes = 4000
|
||||
|
||||
// StageReview runs the adversarial self-review stage. It is a no-op unless
|
||||
// Repo.ReviewEnabled is true. A surviving confirmed error-level finding, or
|
||||
// a review that failed to run at all, parks the task (StatusParked)
|
||||
// with the findings in Detail and stops the workflow before StageOpenPR;
|
||||
// a clean pass, or only warn/plausible findings, leaves Outcome unset so
|
||||
// the workflow proceeds.
|
||||
func StageReview() Stage {
|
||||
return Stage{Name: "review", Run: func(ctx context.Context, tc *TaskContext) error {
|
||||
if !tc.Repo.ReviewEnabled {
|
||||
return nil
|
||||
}
|
||||
if tc.Reviewer == nil {
|
||||
tc.Outcome = Outcome{
|
||||
Status: StatusParked,
|
||||
Detail: "adversarial review is enabled (review_enabled) but no reviewer is configured",
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
report, err := runReview(ctx, tc)
|
||||
if err != nil {
|
||||
return fmt.Errorf("review: %w", err)
|
||||
}
|
||||
// Stash the report regardless of outcome: the PR-body findings
|
||||
// section (h019.6) needs it when the review passes, and the parked
|
||||
// task's park reason already carries it when it does not.
|
||||
tc.ReviewReport = report
|
||||
if !report.Passed() {
|
||||
tc.Outcome = Outcome{Status: StatusParked, Detail: renderReviewDetail(report)}
|
||||
}
|
||||
return nil
|
||||
}}
|
||||
}
|
||||
|
||||
// runReview builds the reviewer's isolated snapshot, diff and issue text,
|
||||
// invokes tc.Reviewer, and cleans up the snapshot regardless of outcome.
|
||||
func runReview(ctx context.Context, tc *TaskContext) (ReviewReport, error) {
|
||||
diff, err := tc.Trees.Diff(ctx, tc.Dir, tc.Repo.BaseBranch())
|
||||
if err != nil {
|
||||
return ReviewReport{}, fmt.Errorf("compute diff: %w", err)
|
||||
}
|
||||
|
||||
// Beside, not inside, the task worktree: a snapshot inside tc.Dir
|
||||
// would itself be an untracked, .git-adjacent directory the next
|
||||
// commit or gate run could pick up.
|
||||
snapshotDir, err := os.MkdirTemp(filepath.Dir(tc.Dir), "review-snapshot-*")
|
||||
if err != nil {
|
||||
return ReviewReport{}, fmt.Errorf("create review snapshot directory: %w", err)
|
||||
}
|
||||
defer os.RemoveAll(snapshotDir)
|
||||
|
||||
if err := tc.Trees.Snapshot(ctx, tc.Dir, snapshotDir); err != nil {
|
||||
return ReviewReport{}, fmt.Errorf("snapshot worktree for review: %w", err)
|
||||
}
|
||||
|
||||
req := ReviewRequest{
|
||||
SnapshotDir: snapshotDir,
|
||||
Diff: diff,
|
||||
IssueText: taskPromptBlock(tc.Task),
|
||||
MaxSteps: tc.Cfg.Budgets.MaxSteps,
|
||||
}
|
||||
// The reviewer is an agent call like any other: it records itself as a
|
||||
// child of the review stage's step, so an intervention during review can
|
||||
// say what the reviewer did. The reviewer runs on a report contract, not
|
||||
// the agent runner's result, so the recording maps the report's own
|
||||
// verdict: a review that ran is succeeded, one that never ran is failed,
|
||||
// and the tokens it accounted are not carried by the contract.
|
||||
stepID, _, err := tc.startChildStep(ctx, task.StepKindAgent, "review")
|
||||
if err != nil {
|
||||
return ReviewReport{}, err
|
||||
}
|
||||
report := tc.Reviewer.Review(ctx, req)
|
||||
to, detail := taskstate.StepSucceeded, report.Summary
|
||||
if report.Status == ReviewStatusNotRun {
|
||||
to, detail = taskstate.StepFailed, report.SkipReason
|
||||
}
|
||||
if ferr := tc.finishChildStep(ctx, stepID, to, detail, 0); ferr != nil {
|
||||
return ReviewReport{}, ferr
|
||||
}
|
||||
return report, nil
|
||||
}
|
||||
|
||||
// StagePostReviewComments posts the review's line-anchored findings as inline
|
||||
// comments on the pull request StageOpenPR has just opened (archie-core-q9au).
|
||||
//
|
||||
// Only line-anchored findings are posted. A whole-file finding has no line to
|
||||
// attach to and stays in the PR-body list, which remains the complete record and
|
||||
// the fallback. A blocking finding never reaches here either: StageReview parks
|
||||
// the task before StageOpenPR when a confirmed error-level finding survives, so
|
||||
// there is no PR to comment on.
|
||||
//
|
||||
// Best-effort by construction. The PR is already open and its body already
|
||||
// carries every finding, so a comment that failed to post must not park a task
|
||||
// whose actual work succeeded -- the same reasoning as OpenPR's best-effort
|
||||
// LinkBranch call. A failure is logged, never returned. That includes the forge
|
||||
// refusing the set because the pull request's head moved past ReviewedHeadSHA:
|
||||
// the line numbers would then describe a revision the author is no longer
|
||||
// looking at, and the PR body is the record that survives anyway.
|
||||
func StagePostReviewComments() Stage {
|
||||
return Stage{Name: "post-review-comments", Run: func(ctx context.Context, tc *TaskContext) error {
|
||||
if !tc.ReviewReport.Ran() {
|
||||
return nil
|
||||
}
|
||||
comments := lineAnchoredReviewComments(tc.ReviewReport)
|
||||
if len(comments) == 0 {
|
||||
return nil
|
||||
}
|
||||
// OpenPR records the number; without one there is nothing to anchor to,
|
||||
// and a guess would attach the findings to unrelated work.
|
||||
if tc.Task.PRNumber <= 0 {
|
||||
tc.Log.Warn("review comments not posted: the task carries no pull request number",
|
||||
"findings", len(comments))
|
||||
return nil
|
||||
}
|
||||
err := tc.Forge.CreateReviewComments(ctx, tc.Task.Owner, tc.Task.Repo, tc.Task.PRNumber, tc.ReviewedHeadSHA, comments)
|
||||
if err != nil {
|
||||
tc.Log.Error("review comments not posted; the pull request body still lists every finding",
|
||||
"err", err, "pr", tc.Task.PRNumber, "comments", len(comments))
|
||||
return nil
|
||||
}
|
||||
tc.Log.Info("review comments posted", "pr", tc.Task.PRNumber, "comments", len(comments))
|
||||
return nil
|
||||
}}
|
||||
}
|
||||
|
||||
// ReviewComment is one line-anchored comment to post on an open pull request:
|
||||
// the repo-relative path, the line in the reviewed revision, and the rendered
|
||||
// body -- which may carry a fenced suggestion block. It is what the review stage
|
||||
// hands the forge through Forger, so it names nothing forge-specific.
|
||||
type ReviewComment struct {
|
||||
Path string
|
||||
Line int
|
||||
Body string
|
||||
}
|
||||
|
||||
// lineAnchoredReviewComments renders the findings that can carry an inline
|
||||
// comment. It is the one place "which findings, with what body" is decided, so
|
||||
// the PR body and the inline comments cannot disagree about a finding beyond the
|
||||
// location the body has to state in text.
|
||||
func lineAnchoredReviewComments(report ReviewReport) []ReviewComment {
|
||||
comments := make([]ReviewComment, 0, len(report.Findings))
|
||||
for _, f := range report.Findings {
|
||||
if f.Line <= 0 {
|
||||
continue
|
||||
}
|
||||
comments = append(comments, ReviewComment{Path: f.File, Line: f.Line, Body: renderReviewCommentBody(f)})
|
||||
}
|
||||
return comments
|
||||
}
|
||||
|
||||
// renderReviewCommentBody renders one finding as the body of a comment anchored
|
||||
// to its line: the anchor carries the location, so the body states the defect,
|
||||
// the scenario that produces it, and -- for a confirmed defect the reviewer
|
||||
// supplied a replacement for -- the replacement as a fence GitHub applies in one
|
||||
// click.
|
||||
//
|
||||
// The fence is gated on confirmed here rather than left to the reviewer, because
|
||||
// it is a one-click apply: a wrong suggestion breaks the code and the author has
|
||||
// to notice. The prompt asks for a suggestion only on a confirmed, mechanically
|
||||
// fixable finding, but an agent's output is input, so the rule is enforced where
|
||||
// the fence is written.
|
||||
func renderReviewCommentBody(f ReviewFinding) string {
|
||||
var b strings.Builder
|
||||
fmt.Fprintf(&b, "**%s (%s)**: %s", f.Verdict, f.Level, f.Defect)
|
||||
if f.FailureScenario != "" {
|
||||
fmt.Fprintf(&b, "\n\n%s", f.FailureScenario)
|
||||
}
|
||||
if f.Verdict == ReviewVerdictConfirmed && f.Suggestion != "" {
|
||||
fmt.Fprintf(&b, "\n\n```suggestion\n%s\n```", strings.TrimRight(f.Suggestion, "\n"))
|
||||
}
|
||||
return b.String()
|
||||
}
|
||||
|
||||
// renderReviewDetail formats a review report as a park Detail: why the
|
||||
// task stopped, and what a human (or a retry) needs to know.
|
||||
func renderReviewDetail(report ReviewReport) string {
|
||||
if !report.Ran() {
|
||||
reason := report.SkipReason
|
||||
if reason == "" {
|
||||
reason = "no reason given"
|
||||
}
|
||||
return clip("adversarial review did not run -- "+reason, reviewDetailBytes)
|
||||
}
|
||||
var b strings.Builder
|
||||
b.WriteString("adversarial review found blocking defects:\n")
|
||||
for _, f := range report.Findings {
|
||||
if !f.Blocking() {
|
||||
continue
|
||||
}
|
||||
fmt.Fprintf(&b, "- %s:%d %s (%s)\n", f.File, f.Line, f.Defect, f.FailureScenario)
|
||||
}
|
||||
return clip(b.String(), reviewDetailBytes)
|
||||
}
|
||||
|
||||
// renderPRReviewSection renders the PR-body findings section (h019.6). It
|
||||
// returns "" when the review did not run, so a review that was disabled or
|
||||
// never wired does not claim one ran -- silence would be indistinguishable
|
||||
// from the review running and finding nothing. When it did run, the section
|
||||
// states the outcome explicitly, including the zero-findings case.
|
||||
func renderPRReviewSection(report ReviewReport) string {
|
||||
if !report.Ran() {
|
||||
return ""
|
||||
}
|
||||
var b strings.Builder
|
||||
b.WriteString("## Adversarial review\n\n")
|
||||
if len(report.Findings) == 0 {
|
||||
b.WriteString("Adversarial self-review ran on the committed diff and found no defects.\n")
|
||||
} else {
|
||||
b.WriteString("Adversarial self-review ran and found no blocking defects. Non-blocking notes:\n\n")
|
||||
for _, f := range report.Findings {
|
||||
loc := f.File
|
||||
if f.Line > 0 {
|
||||
loc = fmt.Sprintf("%s:%d", f.File, f.Line)
|
||||
}
|
||||
fmt.Fprintf(&b, "- `%s` (%s, %s): %s", loc, f.Verdict, f.Level, f.Defect)
|
||||
if f.FailureScenario != "" {
|
||||
fmt.Fprintf(&b, " — %s", f.FailureScenario)
|
||||
}
|
||||
b.WriteString("\n")
|
||||
}
|
||||
}
|
||||
// The cleared properties are what back "ran and found nothing": without
|
||||
// them a zero-finding review reads the same as one that never looked
|
||||
// (h019.5 calibration).
|
||||
if len(report.Checked) > 0 {
|
||||
b.WriteString("\nChecked and cleared:\n\n")
|
||||
for _, c := range report.Checked {
|
||||
fmt.Fprintf(&b, "- **%s** — %s\n", c.Property, c.Evidence)
|
||||
}
|
||||
}
|
||||
if report.Summary != "" {
|
||||
fmt.Fprintf(&b, "\nSummary: %s\n", report.Summary)
|
||||
}
|
||||
return b.String()
|
||||
}
|
||||
@@ -1,272 +0,0 @@
|
||||
package workflow
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"log/slog"
|
||||
"strings"
|
||||
"testing"
|
||||
)
|
||||
|
||||
// TestStagePostReviewCommentsPostsLineAnchoredFindings is the q9au stage
|
||||
// contract: a line-anchored finding reaches the forge as an inline comment,
|
||||
// a whole-file finding does not (it has no line to attach to and stays in the
|
||||
// PR body), and only a confirmed defect carries the one-click suggestion fence.
|
||||
func TestStagePostReviewCommentsPostsLineAnchoredFindings(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
finding ReviewFinding
|
||||
wantPosted bool
|
||||
wantSuggestion bool
|
||||
wantBodyHas []string
|
||||
wantBodyHasNot []string
|
||||
}{
|
||||
{
|
||||
name: "a confirmed defect carries its suggestion",
|
||||
finding: ReviewFinding{
|
||||
File: "a.go", Line: 12, Defect: "nil deref on an empty map",
|
||||
FailureScenario: "an issue with no labels panics the poller",
|
||||
Verdict: ReviewVerdictConfirmed, Level: ReviewLevelWarn,
|
||||
// Single-line only: a multi-line suggestion is rejected by
|
||||
// ReviewFinding.Validate (Issue 2), so the posted fence replaces
|
||||
// the anchored line exactly.
|
||||
Suggestion: "if len(labels) == 0 { return nil }",
|
||||
},
|
||||
wantPosted: true,
|
||||
wantSuggestion: true,
|
||||
wantBodyHas: []string{"nil deref on an empty map", "an issue with no labels panics the poller", "confirmed"},
|
||||
},
|
||||
{
|
||||
name: "a plausible worry never carries one, even when supplied",
|
||||
finding: ReviewFinding{
|
||||
File: "b.go", Line: 3, Defect: "possible race",
|
||||
FailureScenario: "two writers", Verdict: ReviewVerdictPlausible,
|
||||
Level: ReviewLevelWarn, Suggestion: "mu.Lock()",
|
||||
},
|
||||
wantPosted: true,
|
||||
wantSuggestion: false,
|
||||
wantBodyHas: []string{"possible race", "plausible"},
|
||||
wantBodyHasNot: []string{"```suggestion", "mu.Lock()"},
|
||||
},
|
||||
{
|
||||
name: "a confirmed defect with no mechanical fix has no fence",
|
||||
finding: ReviewFinding{
|
||||
File: "c.go", Line: 9, Defect: "wrong ordering",
|
||||
FailureScenario: "retry drops the newest event",
|
||||
Verdict: ReviewVerdictConfirmed, Level: ReviewLevelError,
|
||||
},
|
||||
wantPosted: true,
|
||||
wantSuggestion: false,
|
||||
wantBodyHasNot: []string{"```suggestion"},
|
||||
},
|
||||
{
|
||||
name: "a whole-file finding is not posted inline",
|
||||
finding: ReviewFinding{
|
||||
File: "README.md", Line: 0, Defect: "docs describe the old topology",
|
||||
Verdict: ReviewVerdictConfirmed, Level: ReviewLevelWarn,
|
||||
},
|
||||
wantPosted: false,
|
||||
},
|
||||
}
|
||||
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
f := &fakeForge{}
|
||||
tc := reviewCommentContext(f, ReviewReport{
|
||||
Status: ReviewStatusCompleted,
|
||||
Findings: []ReviewFinding{tt.finding},
|
||||
})
|
||||
|
||||
if err := StagePostReviewComments().Run(context.Background(), tc); err != nil {
|
||||
t.Fatalf("stage returned %v; a posting failure must never fail the task", err)
|
||||
}
|
||||
if got := len(f.reviewComments); (got > 0) != tt.wantPosted {
|
||||
t.Fatalf("posted %d comments, want posted=%v", got, tt.wantPosted)
|
||||
}
|
||||
if !tt.wantPosted {
|
||||
return
|
||||
}
|
||||
got := f.reviewComments[0]
|
||||
if got.Path != tt.finding.File || got.Line != tt.finding.Line {
|
||||
t.Errorf("anchor = %s:%d, want %s:%d", got.Path, got.Line, tt.finding.File, tt.finding.Line)
|
||||
}
|
||||
for _, want := range tt.wantBodyHas {
|
||||
if !strings.Contains(got.Body, want) {
|
||||
t.Errorf("body %q does not mention %q", got.Body, want)
|
||||
}
|
||||
}
|
||||
for _, unwanted := range tt.wantBodyHasNot {
|
||||
if strings.Contains(got.Body, unwanted) {
|
||||
t.Errorf("body %q contains %q, which must not be offered", got.Body, unwanted)
|
||||
}
|
||||
}
|
||||
hasFence := strings.Contains(got.Body, "```suggestion")
|
||||
if hasFence != tt.wantSuggestion {
|
||||
t.Errorf("suggestion fence present = %v, want %v (body %q)", hasFence, tt.wantSuggestion, got.Body)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// TestStagePostReviewCommentsPostsEveryAnchoredFindingInOneCall pins the batch
|
||||
// shape: Gitea carries inline comments only on a submitted review, so the set
|
||||
// must travel as one call rather than one call per finding.
|
||||
func TestStagePostReviewCommentsPostsEveryAnchoredFindingInOneCall(t *testing.T) {
|
||||
f := &fakeForge{}
|
||||
tc := reviewCommentContext(f, ReviewReport{
|
||||
Status: ReviewStatusCompleted,
|
||||
Findings: []ReviewFinding{
|
||||
{File: "a.go", Line: 1, Defect: "one", Verdict: ReviewVerdictConfirmed, Level: ReviewLevelWarn},
|
||||
{File: "b.go", Line: 2, Defect: "two", Verdict: ReviewVerdictPlausible, Level: ReviewLevelWarn},
|
||||
{File: "c.go", Line: 0, Defect: "file-level", Verdict: ReviewVerdictPlausible, Level: ReviewLevelWarn},
|
||||
},
|
||||
})
|
||||
|
||||
if err := StagePostReviewComments().Run(context.Background(), tc); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if f.reviewCalls != 1 {
|
||||
t.Errorf("CreateReviewComments called %d times, want 1", f.reviewCalls)
|
||||
}
|
||||
if len(f.reviewComments) != 2 {
|
||||
t.Errorf("posted %d comments, want the 2 anchored findings", len(f.reviewComments))
|
||||
}
|
||||
}
|
||||
|
||||
// TestStagePostReviewCommentsIsANoOpWithoutAReview keeps the stage silent when
|
||||
// there is nothing to post: a disabled or never-wired review must not reach the
|
||||
// forge at all, and a clean review has nothing to anchor.
|
||||
func TestStagePostReviewCommentsIsANoOpWithoutAReview(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
report ReviewReport
|
||||
}{
|
||||
{name: "the review never ran", report: ReviewReport{}},
|
||||
{name: "the review was skipped", report: ReviewReport{Status: ReviewStatusSkipped, SkipReason: "no reviewer"}},
|
||||
{name: "the review ran and found nothing", report: ReviewReport{Status: ReviewStatusCompleted}},
|
||||
{
|
||||
name: "the review found only a whole-file finding",
|
||||
report: ReviewReport{Status: ReviewStatusCompleted, Findings: []ReviewFinding{
|
||||
{File: "README.md", Line: 0, Defect: "stale", Verdict: ReviewVerdictPlausible, Level: ReviewLevelWarn},
|
||||
}},
|
||||
},
|
||||
}
|
||||
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
f := &fakeForge{}
|
||||
tc := reviewCommentContext(f, tt.report)
|
||||
if err := StagePostReviewComments().Run(context.Background(), tc); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if f.reviewCalls != 0 {
|
||||
t.Errorf("CreateReviewComments called %d times, want 0", f.reviewCalls)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// TestStagePostReviewCommentsSurvivesAPostingFailure is the "best effort"
|
||||
// clause: the PR is already open and its body already lists every finding, so a
|
||||
// failed comment must be logged, not turned into a parked task.
|
||||
func TestStagePostReviewCommentsSurvivesAPostingFailure(t *testing.T) {
|
||||
f := &fakeForge{reviewErr: errors.New("422 line is not part of the diff")}
|
||||
tc := reviewCommentContext(f, ReviewReport{
|
||||
Status: ReviewStatusCompleted,
|
||||
Findings: []ReviewFinding{
|
||||
{File: "a.go", Line: 4, Defect: "nil deref", Verdict: ReviewVerdictConfirmed, Level: ReviewLevelWarn},
|
||||
},
|
||||
})
|
||||
|
||||
if err := StagePostReviewComments().Run(context.Background(), tc); err != nil {
|
||||
t.Fatalf("stage returned %v; a posting failure must never fail the task", err)
|
||||
}
|
||||
if tc.Outcome.Status != "" {
|
||||
t.Errorf("outcome = %q, want unset: a failed comment must not end the workflow", tc.Outcome.Status)
|
||||
}
|
||||
}
|
||||
|
||||
// TestStagePostReviewCommentsNeedsAPullRequestNumber covers the one precondition
|
||||
// the stage cannot invent. A task with no recorded PR number has nothing to
|
||||
// anchor to, and guessing would attach the findings to unrelated work.
|
||||
func TestStagePostReviewCommentsNeedsAPullRequestNumber(t *testing.T) {
|
||||
f := &fakeForge{}
|
||||
tc := reviewCommentContext(f, ReviewReport{
|
||||
Status: ReviewStatusCompleted,
|
||||
Findings: []ReviewFinding{
|
||||
{File: "a.go", Line: 4, Defect: "nil deref", Verdict: ReviewVerdictConfirmed, Level: ReviewLevelWarn},
|
||||
},
|
||||
})
|
||||
tc.Task.PRNumber = 0
|
||||
|
||||
if err := StagePostReviewComments().Run(context.Background(), tc); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if f.reviewCalls != 0 {
|
||||
t.Errorf("CreateReviewComments called %d times, want 0 with no PR number", f.reviewCalls)
|
||||
}
|
||||
}
|
||||
|
||||
// TestStagePostReviewCommentsAddressesTheRecordedPullRequest pins that the
|
||||
// comment lands on the task's own PR, in the task's own repo.
|
||||
func TestStagePostReviewCommentsAddressesTheRecordedPullRequest(t *testing.T) {
|
||||
f := &fakeForge{}
|
||||
tc := reviewCommentContext(f, ReviewReport{
|
||||
Status: ReviewStatusCompleted,
|
||||
Findings: []ReviewFinding{
|
||||
{File: "a.go", Line: 4, Defect: "nil deref", Verdict: ReviewVerdictConfirmed, Level: ReviewLevelWarn},
|
||||
},
|
||||
})
|
||||
|
||||
if err := StagePostReviewComments().Run(context.Background(), tc); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if f.reviewOwner != "acme" || f.reviewRepo != "widget" || f.reviewNumber != 42 {
|
||||
t.Errorf("anchored at %s/%s#%d, want acme/widget#42", f.reviewOwner, f.reviewRepo, f.reviewNumber)
|
||||
}
|
||||
}
|
||||
|
||||
// TestStagePostReviewCommentsThreadsTheReviewedHeadSHA is the producer half of
|
||||
// the head-drift guard: the forge can only refuse a set whose line numbers
|
||||
// describe a revision the pull request has left if the stage tells it which
|
||||
// revision those numbers were measured on.
|
||||
func TestStagePostReviewCommentsThreadsTheReviewedHeadSHA(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
reviewed string
|
||||
}{
|
||||
{name: "a measured revision reaches the forge", reviewed: "1a2b3c4d"},
|
||||
{
|
||||
name: "an unmeasured revision is passed through as empty so the forge posts unverified",
|
||||
},
|
||||
}
|
||||
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
f := &fakeForge{}
|
||||
tc := reviewCommentContext(f, ReviewReport{
|
||||
Status: ReviewStatusCompleted,
|
||||
Findings: []ReviewFinding{
|
||||
{File: "a.go", Line: 4, Defect: "nil deref", Verdict: ReviewVerdictConfirmed, Level: ReviewLevelWarn},
|
||||
},
|
||||
})
|
||||
tc.ReviewedHeadSHA = tt.reviewed
|
||||
|
||||
if err := StagePostReviewComments().Run(context.Background(), tc); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if f.reviewHeadSHA != tt.reviewed {
|
||||
t.Errorf("forge was handed reviewed head %q, want %q", f.reviewHeadSHA, tt.reviewed)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func reviewCommentContext(f *fakeForge, report ReviewReport) *TaskContext {
|
||||
return &TaskContext{
|
||||
Task: &Task{ID: 1, Owner: "acme", Repo: "widget", IssueNumber: 7, PRNumber: 42},
|
||||
Forge: f,
|
||||
Log: slog.New(slog.DiscardHandler),
|
||||
ReviewReport: report,
|
||||
}
|
||||
}
|
||||
@@ -1,320 +0,0 @@
|
||||
package workflow
|
||||
|
||||
import (
|
||||
"errors"
|
||||
"fmt"
|
||||
"strings"
|
||||
)
|
||||
|
||||
// ReviewLevel represents the severity level of a review finding.
|
||||
// In house style, "error" blocks PR creation when confirmed; "warn" is advisory.
|
||||
type ReviewLevel string
|
||||
|
||||
const (
|
||||
// ReviewLevelError indicates a severe defect that blocks workflow execution when confirmed.
|
||||
ReviewLevelError ReviewLevel = "error"
|
||||
// ReviewLevelWarn indicates an advisory finding that is logged but never blocks.
|
||||
ReviewLevelWarn ReviewLevel = "warn"
|
||||
)
|
||||
|
||||
// ReviewVerdict distinguishes confirmed defects from plausible worries.
|
||||
// The load-bearing rule is that ONLY a confirmed defect may block.
|
||||
type ReviewVerdict string
|
||||
|
||||
const (
|
||||
// ReviewVerdictConfirmed indicates a verified defect backed by a concrete failure scenario.
|
||||
ReviewVerdictConfirmed ReviewVerdict = "confirmed"
|
||||
// ReviewVerdictPlausible indicates a potential defect or worry that is plausible but unverified.
|
||||
ReviewVerdictPlausible ReviewVerdict = "plausible"
|
||||
)
|
||||
|
||||
// ReviewDisposition records what happened once a finding was acted on.
|
||||
type ReviewDisposition string
|
||||
|
||||
const (
|
||||
// ReviewDispositionFixed indicates the defect was addressed and resolved in code.
|
||||
ReviewDispositionFixed ReviewDisposition = "fixed"
|
||||
// ReviewDispositionSkipped indicates the finding was reviewed and intentionally bypassed.
|
||||
ReviewDispositionSkipped ReviewDisposition = "skipped"
|
||||
// ReviewDispositionNoChangeNeeded indicates the finding was investigated and judged as requiring no change.
|
||||
ReviewDispositionNoChangeNeeded ReviewDisposition = "no_change_needed"
|
||||
)
|
||||
|
||||
// ReviewCategory classifies findings against this repo's review checklist.
|
||||
type ReviewCategory string
|
||||
|
||||
const (
|
||||
// ReviewCategoryDeadCode identifies unreferenced or dead code paths.
|
||||
ReviewCategoryDeadCode ReviewCategory = "dead-code"
|
||||
// ReviewCategoryUncheckedError identifies ignored or unchecked error returns.
|
||||
ReviewCategoryUncheckedError ReviewCategory = "unchecked-error"
|
||||
// ReviewCategoryHardcodedValue identifies hardcoded values that should be parameters or constants.
|
||||
ReviewCategoryHardcodedValue ReviewCategory = "hardcoded-value"
|
||||
// ReviewCategoryInterfaceSatisfaction identifies interface compliance or satisfaction defects.
|
||||
ReviewCategoryInterfaceSatisfaction ReviewCategory = "interface-satisfaction"
|
||||
// ReviewCategoryNilRisk identifies nil-pointer dereference risks.
|
||||
ReviewCategoryNilRisk ReviewCategory = "nil-risk"
|
||||
// ReviewCategoryGoroutineLeak identifies leaking goroutines or unbound concurrency.
|
||||
ReviewCategoryGoroutineLeak ReviewCategory = "goroutine-leak"
|
||||
// ReviewCategoryRace identifies data races or unsynchronized access to shared mutable state.
|
||||
ReviewCategoryRace ReviewCategory = "race"
|
||||
// ReviewCategoryOther identifies defects not matching specific checklist items.
|
||||
ReviewCategoryOther ReviewCategory = "other"
|
||||
)
|
||||
|
||||
// ReviewStatus represents the execution status of an adversarial review stage.
|
||||
type ReviewStatus string
|
||||
|
||||
const (
|
||||
// ReviewStatusNotRun indicates the reviewer agent was not executed.
|
||||
ReviewStatusNotRun ReviewStatus = "not_run"
|
||||
// ReviewStatusCompleted indicates the reviewer agent ran to completion.
|
||||
ReviewStatusCompleted ReviewStatus = "completed"
|
||||
// ReviewStatusSkipped indicates the reviewer was intentionally skipped.
|
||||
ReviewStatusSkipped ReviewStatus = "skipped"
|
||||
)
|
||||
|
||||
// ReviewFinding represents one defect found during adversarial review.
|
||||
// It is serialisable across the agent-execution JSON data boundary.
|
||||
//
|
||||
// A finding describes wrong behaviour or a crash. Style, formatting, naming,
|
||||
// and preference nits are out of contract -- they are the padding that trains
|
||||
// an operator to skim past a findings list.
|
||||
type ReviewFinding struct {
|
||||
// File is the repo-relative file path where the defect is located.
|
||||
File string `json:"file"`
|
||||
// Line is the line number where the defect is located.
|
||||
Line int `json:"line"`
|
||||
// Defect is a one-sentence statement of the defect.
|
||||
Defect string `json:"defect"`
|
||||
// FailureScenario describes the concrete inputs or state that produce the wrong output or crash.
|
||||
FailureScenario string `json:"failure_scenario"`
|
||||
// Suggestion is the replacement text for the anchored line, when the
|
||||
// reviewer can state the fix mechanically. Only a confirmed finding may
|
||||
// carry one: it is rendered as a one-click apply, and a wrong one-click
|
||||
// fix is worse than no fix (h019.5 calibration). Empty means "no
|
||||
// mechanical fix", not "no fix needed".
|
||||
Suggestion string `json:"suggestion,omitempty"`
|
||||
// Verdict distinguishes confirmed defects from plausible hypotheses. Only confirmed findings may block.
|
||||
Verdict ReviewVerdict `json:"verdict"`
|
||||
// Disposition records the action taken once the finding is addressed.
|
||||
Disposition ReviewDisposition `json:"disposition,omitempty"`
|
||||
// Level is "error" (blocking when confirmed) or "warn" (advisory).
|
||||
Level ReviewLevel `json:"level"`
|
||||
// Category classifies the defect according to the review checklist.
|
||||
Category ReviewCategory `json:"category"`
|
||||
}
|
||||
|
||||
// ReviewCheck records a property the reviewer verified clean. It is the
|
||||
// symmetric counterpart of ReviewFinding: where a finding carries a failure
|
||||
// scenario as its evidence, a check carries how the property was verified. A
|
||||
// zero-finding review is credible only when it lists what it checked, so a
|
||||
// check without evidence is an assertion, not a verification.
|
||||
type ReviewCheck struct {
|
||||
// Property is what was verified clean, e.g. "nil-safety of Foo callers".
|
||||
Property string `json:"property"`
|
||||
// Evidence is how it was verified, e.g. "read all 4 callers; each nil-checks".
|
||||
Evidence string `json:"evidence"`
|
||||
}
|
||||
|
||||
// ReviewReport represents the aggregate result of an adversarial review.
|
||||
// It explicitly distinguishes "reviewer looked and found nothing" (Status == ReviewStatusCompleted, Findings empty)
|
||||
// from "reviewer never ran" (Status == ReviewStatusNotRun or uninitialised).
|
||||
type ReviewReport struct {
|
||||
// Status explicitly records whether the review was run, not run, or skipped.
|
||||
Status ReviewStatus `json:"status"`
|
||||
// Findings contains all findings produced by the review.
|
||||
Findings []ReviewFinding `json:"findings,omitempty"`
|
||||
// Checked records the properties the reviewer verified clean. A
|
||||
// zero-finding review is credible only when it says what it checked;
|
||||
// this is what separates "I checked and it is fine" from "I did not look".
|
||||
Checked []ReviewCheck `json:"checked,omitempty"`
|
||||
// Summary is a human-readable high-level overview of the review.
|
||||
Summary string `json:"summary,omitempty"`
|
||||
// SkipReason explains why the review was not run or was skipped.
|
||||
SkipReason string `json:"skip_reason,omitempty"`
|
||||
}
|
||||
|
||||
// Blocking reports whether the finding is a blocking defect.
|
||||
// The load-bearing rule: ONLY a finding with Verdict == ReviewVerdictConfirmed
|
||||
// AND Level == ReviewLevelError blocks. A plausible finding, however severe
|
||||
// its level claims to be, never blocks on its own.
|
||||
func (f ReviewFinding) Blocking() bool {
|
||||
return f.Verdict == ReviewVerdictConfirmed && f.Level == ReviewLevelError
|
||||
}
|
||||
|
||||
// ReviewBlocking reports whether any finding in the slice is blocking.
|
||||
// The load-bearing rule: ONLY a finding with Verdict == ReviewVerdictConfirmed
|
||||
// and Level == ReviewLevelError blocks. Plausible findings and warn-level findings
|
||||
// never block.
|
||||
func ReviewBlocking(findings []ReviewFinding) bool {
|
||||
for _, f := range findings {
|
||||
if f.Blocking() {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
// Ran reports whether the adversarial reviewer actually executed to completion.
|
||||
// Reviews that were not run (Status == ReviewStatusNotRun or uninitialised) or skipped
|
||||
// return false.
|
||||
func (r ReviewReport) Ran() bool {
|
||||
return r.Status == ReviewStatusCompleted
|
||||
}
|
||||
|
||||
// Blocking reports whether any finding within the report is blocking.
|
||||
func (r ReviewReport) Blocking() bool {
|
||||
return ReviewBlocking(r.Findings)
|
||||
}
|
||||
|
||||
// Passed reports whether the review ran to completion and produced no blocking findings.
|
||||
// A review that did not run (or has zero-value status) returns false because "not run"
|
||||
// is structurally distinct from "ran and found 0 defects".
|
||||
func (r ReviewReport) Passed() bool {
|
||||
return r.Ran() && !r.Blocking()
|
||||
}
|
||||
|
||||
// Validate checks that the finding satisfies contract invariants.
|
||||
func (f ReviewFinding) Validate() error {
|
||||
if err := f.validateAnchor(); err != nil {
|
||||
return err
|
||||
}
|
||||
if err := f.validateClassification(); err != nil {
|
||||
return err
|
||||
}
|
||||
return f.validateDisposition()
|
||||
}
|
||||
|
||||
// validateAnchor checks where the defect is and what fix, if any, accompanies
|
||||
// it.
|
||||
func (f ReviewFinding) validateAnchor() error {
|
||||
if f.File == "" {
|
||||
return errors.New("review finding file is required")
|
||||
}
|
||||
if f.Line < 0 {
|
||||
return fmt.Errorf("review finding line cannot be negative: %d", f.Line)
|
||||
}
|
||||
if f.Defect == "" {
|
||||
return errors.New("review finding defect statement is required")
|
||||
}
|
||||
if f.FailureScenario == "" {
|
||||
return errors.New("review finding failure scenario is required")
|
||||
}
|
||||
// A suggestion is applied in one click, so only a confirmed defect may
|
||||
// carry one: a wrong suggestion breaks the code and the author has to
|
||||
// notice, where a plausible worry gets a sentence (h019.5 calibration).
|
||||
// Enforced at the value so the agent's record_finding call is rejected with
|
||||
// something the model can act on, instead of the suggestion being dropped
|
||||
// somewhere later where nothing tells it.
|
||||
if f.Suggestion != "" && f.Verdict != ReviewVerdictConfirmed {
|
||||
return fmt.Errorf("review finding %s:%d offers a one-click suggestion with verdict %q; only %q may", f.File, f.Line, f.Verdict, ReviewVerdictConfirmed)
|
||||
}
|
||||
// A suggestion is a one-line replacement GitHub applies with a single click,
|
||||
// so it must be single-line: an embedded newline renders a multi-line fence
|
||||
// that GitHub applies only up to the first line, silently corrupting the file
|
||||
// (docs/prds/inline-review.md). Enforced at the value so the agent's
|
||||
// record_finding call is rejected with something the model can act on.
|
||||
if f.Suggestion != "" && strings.ContainsAny(f.Suggestion, "\n\r") {
|
||||
return fmt.Errorf("review finding %s:%d suggestion must be single-line; got %d line(s)", f.File, f.Line, strings.Count(f.Suggestion, "\n")+1)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// validateClassification checks the three enums that classify the finding.
|
||||
func (f ReviewFinding) validateClassification() error {
|
||||
switch f.Verdict {
|
||||
case ReviewVerdictConfirmed, ReviewVerdictPlausible:
|
||||
default:
|
||||
return fmt.Errorf("invalid review verdict %q (must be %q or %q)", f.Verdict, ReviewVerdictConfirmed, ReviewVerdictPlausible)
|
||||
}
|
||||
switch f.Level {
|
||||
case ReviewLevelError, ReviewLevelWarn:
|
||||
default:
|
||||
return fmt.Errorf("invalid review level %q (must be %q or %q)", f.Level, ReviewLevelError, ReviewLevelWarn)
|
||||
}
|
||||
switch f.Category {
|
||||
case ReviewCategoryDeadCode,
|
||||
ReviewCategoryUncheckedError,
|
||||
ReviewCategoryHardcodedValue,
|
||||
ReviewCategoryInterfaceSatisfaction,
|
||||
ReviewCategoryNilRisk,
|
||||
ReviewCategoryGoroutineLeak,
|
||||
ReviewCategoryRace,
|
||||
ReviewCategoryOther:
|
||||
default:
|
||||
return fmt.Errorf("invalid review category %q", f.Category)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// validateDisposition checks the optional record of what was done about the
|
||||
// finding once it was acted on.
|
||||
func (f ReviewFinding) validateDisposition() error {
|
||||
switch f.Disposition {
|
||||
case "", ReviewDispositionFixed, ReviewDispositionSkipped, ReviewDispositionNoChangeNeeded:
|
||||
default:
|
||||
return fmt.Errorf("invalid review disposition %q", f.Disposition)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// Validate checks that the check satisfies contract invariants. Evidence is
|
||||
// required: a check without evidence is an assertion, not a verification.
|
||||
func (c ReviewCheck) Validate() error {
|
||||
if c.Property == "" {
|
||||
return errors.New("review check property is required")
|
||||
}
|
||||
if c.Evidence == "" {
|
||||
return errors.New("review check evidence is required")
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// Validate checks that the report satisfies contract invariants.
|
||||
func (r ReviewReport) Validate() error {
|
||||
switch r.Status {
|
||||
case ReviewStatusNotRun, ReviewStatusCompleted, ReviewStatusSkipped:
|
||||
default:
|
||||
return fmt.Errorf("invalid review status %q", r.Status)
|
||||
}
|
||||
for i, f := range r.Findings {
|
||||
if err := f.Validate(); err != nil {
|
||||
return fmt.Errorf("findings[%d]: %w", i, err)
|
||||
}
|
||||
}
|
||||
for i, c := range r.Checked {
|
||||
if err := c.Validate(); err != nil {
|
||||
return fmt.Errorf("checked[%d]: %w", i, err)
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// NewCompletedReviewReport constructs a completed review report.
|
||||
func NewCompletedReviewReport(findings []ReviewFinding, summary string) ReviewReport {
|
||||
if findings == nil {
|
||||
findings = []ReviewFinding{}
|
||||
}
|
||||
return ReviewReport{
|
||||
Status: ReviewStatusCompleted,
|
||||
Findings: findings,
|
||||
Summary: summary,
|
||||
}
|
||||
}
|
||||
|
||||
// NewNotRunReviewReport constructs a report representing an unexecuted review.
|
||||
func NewNotRunReviewReport(reason string) ReviewReport {
|
||||
return ReviewReport{
|
||||
Status: ReviewStatusNotRun,
|
||||
SkipReason: reason,
|
||||
}
|
||||
}
|
||||
|
||||
// NewSkippedReviewReport constructs a report representing a skipped review.
|
||||
func NewSkippedReviewReport(reason string) ReviewReport {
|
||||
return ReviewReport{
|
||||
Status: ReviewStatusSkipped,
|
||||
SkipReason: reason,
|
||||
}
|
||||
}
|
||||
@@ -1,969 +0,0 @@
|
||||
package workflow
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"reflect"
|
||||
"testing"
|
||||
)
|
||||
|
||||
func TestReviewFindingBlockingMatrix(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
verdict ReviewVerdict
|
||||
level ReviewLevel
|
||||
wantBlocking bool
|
||||
}{
|
||||
{
|
||||
name: "confirmed error blocks",
|
||||
verdict: ReviewVerdictConfirmed,
|
||||
level: ReviewLevelError,
|
||||
wantBlocking: true,
|
||||
},
|
||||
{
|
||||
name: "confirmed warn does not block",
|
||||
verdict: ReviewVerdictConfirmed,
|
||||
level: ReviewLevelWarn,
|
||||
wantBlocking: false,
|
||||
},
|
||||
{
|
||||
name: "confirmed empty level does not block",
|
||||
verdict: ReviewVerdictConfirmed,
|
||||
level: "",
|
||||
wantBlocking: false,
|
||||
},
|
||||
{
|
||||
name: "confirmed unknown level does not block",
|
||||
verdict: ReviewVerdictConfirmed,
|
||||
level: "critical",
|
||||
wantBlocking: false,
|
||||
},
|
||||
{
|
||||
name: "plausible error MUST NOT block",
|
||||
verdict: ReviewVerdictPlausible,
|
||||
level: ReviewLevelError,
|
||||
wantBlocking: false,
|
||||
},
|
||||
{
|
||||
name: "plausible warn does not block",
|
||||
verdict: ReviewVerdictPlausible,
|
||||
level: ReviewLevelWarn,
|
||||
wantBlocking: false,
|
||||
},
|
||||
{
|
||||
name: "plausible empty level does not block",
|
||||
verdict: ReviewVerdictPlausible,
|
||||
level: "",
|
||||
wantBlocking: false,
|
||||
},
|
||||
{
|
||||
name: "empty verdict error does not block",
|
||||
verdict: "",
|
||||
level: ReviewLevelError,
|
||||
wantBlocking: false,
|
||||
},
|
||||
{
|
||||
name: "empty verdict warn does not block",
|
||||
verdict: "",
|
||||
level: ReviewLevelWarn,
|
||||
wantBlocking: false,
|
||||
},
|
||||
{
|
||||
name: "empty verdict and level does not block",
|
||||
verdict: "",
|
||||
level: "",
|
||||
wantBlocking: false,
|
||||
},
|
||||
{
|
||||
name: "unknown verdict with error does not block",
|
||||
verdict: "likely",
|
||||
level: ReviewLevelError,
|
||||
wantBlocking: false,
|
||||
},
|
||||
{
|
||||
name: "uppercase confirmed error does not block (case sensitive)",
|
||||
verdict: "CONFIRMED",
|
||||
level: "ERROR",
|
||||
wantBlocking: false,
|
||||
},
|
||||
{
|
||||
name: "title case confirmed error does not block (case sensitive)",
|
||||
verdict: "Confirmed",
|
||||
level: "Error",
|
||||
wantBlocking: false,
|
||||
},
|
||||
}
|
||||
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
f := ReviewFinding{
|
||||
File: "internal/domain/workflow/review_findings.go",
|
||||
Line: 42,
|
||||
Defect: "sample defect statement",
|
||||
FailureScenario: "sample failure scenario",
|
||||
Verdict: tt.verdict,
|
||||
Level: tt.level,
|
||||
Category: ReviewCategoryNilRisk,
|
||||
}
|
||||
|
||||
if got := f.Blocking(); got != tt.wantBlocking {
|
||||
t.Errorf("ReviewFinding.Blocking() = %v, want %v (verdict=%q, level=%q)",
|
||||
got, tt.wantBlocking, tt.verdict, tt.level)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestReviewBlockingSlice(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
findings []ReviewFinding
|
||||
wantBlocking bool
|
||||
}{
|
||||
{
|
||||
name: "nil slice returns false",
|
||||
findings: nil,
|
||||
wantBlocking: false,
|
||||
},
|
||||
{
|
||||
name: "empty slice returns false",
|
||||
findings: []ReviewFinding{},
|
||||
wantBlocking: false,
|
||||
},
|
||||
{
|
||||
name: "single confirmed error returns true",
|
||||
findings: []ReviewFinding{
|
||||
{
|
||||
File: "main.go",
|
||||
Line: 10,
|
||||
Defect: "nil pointer dereference",
|
||||
FailureScenario: "running with nil config crashes",
|
||||
Verdict: ReviewVerdictConfirmed,
|
||||
Level: ReviewLevelError,
|
||||
Category: ReviewCategoryNilRisk,
|
||||
},
|
||||
},
|
||||
wantBlocking: true,
|
||||
},
|
||||
{
|
||||
name: "warn-only confirmed findings do not block",
|
||||
findings: []ReviewFinding{
|
||||
{
|
||||
File: "main.go",
|
||||
Line: 10,
|
||||
Defect: "unused helper function",
|
||||
FailureScenario: "function is never called",
|
||||
Verdict: ReviewVerdictConfirmed,
|
||||
Level: ReviewLevelWarn,
|
||||
Category: ReviewCategoryDeadCode,
|
||||
},
|
||||
{
|
||||
File: "util.go",
|
||||
Line: 20,
|
||||
Defect: "magic number in loop",
|
||||
FailureScenario: "hardcoded 100 instead of constant",
|
||||
Verdict: ReviewVerdictConfirmed,
|
||||
Level: ReviewLevelWarn,
|
||||
Category: ReviewCategoryHardcodedValue,
|
||||
},
|
||||
},
|
||||
wantBlocking: false,
|
||||
},
|
||||
{
|
||||
name: "plausible error findings do not block",
|
||||
findings: []ReviewFinding{
|
||||
{
|
||||
File: "server.go",
|
||||
Line: 50,
|
||||
Defect: "potential race condition",
|
||||
FailureScenario: "concurrent writes might race if mutex is missing",
|
||||
Verdict: ReviewVerdictPlausible,
|
||||
Level: ReviewLevelError,
|
||||
Category: ReviewCategoryRace,
|
||||
},
|
||||
},
|
||||
wantBlocking: false,
|
||||
},
|
||||
{
|
||||
name: "mixed findings with one confirmed error returns true",
|
||||
findings: []ReviewFinding{
|
||||
{
|
||||
File: "doc.go",
|
||||
Line: 1,
|
||||
Defect: "typo in comment",
|
||||
FailureScenario: "spelling mistake in doc comment",
|
||||
Verdict: ReviewVerdictConfirmed,
|
||||
Level: ReviewLevelWarn,
|
||||
Category: ReviewCategoryOther,
|
||||
},
|
||||
{
|
||||
File: "handler.go",
|
||||
Line: 45,
|
||||
Defect: "unchecked error return from Write",
|
||||
FailureScenario: "client disconnect causes unhandled error",
|
||||
Verdict: ReviewVerdictConfirmed,
|
||||
Level: ReviewLevelError,
|
||||
Category: ReviewCategoryUncheckedError,
|
||||
},
|
||||
{
|
||||
File: "worker.go",
|
||||
Line: 100,
|
||||
Defect: "unbounded goroutine launch",
|
||||
FailureScenario: "burst of 10k items exhausts memory",
|
||||
Verdict: ReviewVerdictPlausible,
|
||||
Level: ReviewLevelError,
|
||||
Category: ReviewCategoryGoroutineLeak,
|
||||
},
|
||||
},
|
||||
wantBlocking: true,
|
||||
},
|
||||
{
|
||||
name: "confirmed error as first element returns true",
|
||||
findings: []ReviewFinding{
|
||||
{
|
||||
File: "first.go",
|
||||
Line: 1,
|
||||
Defect: "blocking issue",
|
||||
FailureScenario: "crashes immediately",
|
||||
Verdict: ReviewVerdictConfirmed,
|
||||
Level: ReviewLevelError,
|
||||
Category: ReviewCategoryNilRisk,
|
||||
},
|
||||
{
|
||||
File: "second.go",
|
||||
Line: 2,
|
||||
Defect: "advisory issue",
|
||||
FailureScenario: "minor warning",
|
||||
Verdict: ReviewVerdictConfirmed,
|
||||
Level: ReviewLevelWarn,
|
||||
Category: ReviewCategoryOther,
|
||||
},
|
||||
},
|
||||
wantBlocking: true,
|
||||
},
|
||||
}
|
||||
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
if got := ReviewBlocking(tt.findings); got != tt.wantBlocking {
|
||||
t.Errorf("ReviewBlocking() = %v, want %v", got, tt.wantBlocking)
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
t.Run("large slice detects confirmed error buried among warns", func(t *testing.T) {
|
||||
findings := make([]ReviewFinding, 1000)
|
||||
for i := range findings {
|
||||
findings[i] = ReviewFinding{
|
||||
File: "file.go",
|
||||
Line: i,
|
||||
Defect: "warn finding",
|
||||
FailureScenario: "advisory scenario",
|
||||
Verdict: ReviewVerdictConfirmed,
|
||||
Level: ReviewLevelWarn,
|
||||
Category: ReviewCategoryOther,
|
||||
}
|
||||
}
|
||||
findings[750] = ReviewFinding{
|
||||
File: "file.go",
|
||||
Line: 750,
|
||||
Defect: "confirmed error",
|
||||
FailureScenario: "fatal crash on nil context",
|
||||
Verdict: ReviewVerdictConfirmed,
|
||||
Level: ReviewLevelError,
|
||||
Category: ReviewCategoryNilRisk,
|
||||
}
|
||||
|
||||
if !ReviewBlocking(findings) {
|
||||
t.Error("ReviewBlocking() = false, want true for confirmed error at index 750")
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
func TestReviewReportStatusDistinction(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
report ReviewReport
|
||||
wantRan bool
|
||||
wantBlocking bool
|
||||
wantPassed bool
|
||||
}{
|
||||
{
|
||||
name: "ran and found zero findings passes",
|
||||
report: ReviewReport{
|
||||
Status: ReviewStatusCompleted,
|
||||
Findings: []ReviewFinding{},
|
||||
Summary: "adversarial review completed with 0 findings",
|
||||
},
|
||||
wantRan: true,
|
||||
wantBlocking: false,
|
||||
wantPassed: true,
|
||||
},
|
||||
{
|
||||
name: "ran and found nil findings slice passes",
|
||||
report: ReviewReport{
|
||||
Status: ReviewStatusCompleted,
|
||||
Findings: nil,
|
||||
Summary: "adversarial review completed cleanly",
|
||||
},
|
||||
wantRan: true,
|
||||
wantBlocking: false,
|
||||
wantPassed: true,
|
||||
},
|
||||
{
|
||||
name: "ran and found only warn findings passes",
|
||||
report: ReviewReport{
|
||||
Status: ReviewStatusCompleted,
|
||||
Findings: []ReviewFinding{
|
||||
{
|
||||
File: "a.go",
|
||||
Line: 12,
|
||||
Defect: "advisory note",
|
||||
FailureScenario: "minor style difference",
|
||||
Verdict: ReviewVerdictConfirmed,
|
||||
Level: ReviewLevelWarn,
|
||||
Category: ReviewCategoryOther,
|
||||
},
|
||||
},
|
||||
Summary: "advisories found",
|
||||
},
|
||||
wantRan: true,
|
||||
wantBlocking: false,
|
||||
wantPassed: true,
|
||||
},
|
||||
{
|
||||
name: "ran and found plausible error findings passes",
|
||||
report: ReviewReport{
|
||||
Status: ReviewStatusCompleted,
|
||||
Findings: []ReviewFinding{
|
||||
{
|
||||
File: "b.go",
|
||||
Line: 34,
|
||||
Defect: "hypothetical deadlock",
|
||||
FailureScenario: "if channel buffers fill up",
|
||||
Verdict: ReviewVerdictPlausible,
|
||||
Level: ReviewLevelError,
|
||||
Category: ReviewCategoryGoroutineLeak,
|
||||
},
|
||||
},
|
||||
Summary: "plausible error found",
|
||||
},
|
||||
wantRan: true,
|
||||
wantBlocking: false,
|
||||
wantPassed: true,
|
||||
},
|
||||
{
|
||||
name: "ran and found confirmed error blocks and does not pass",
|
||||
report: ReviewReport{
|
||||
Status: ReviewStatusCompleted,
|
||||
Findings: []ReviewFinding{
|
||||
{
|
||||
File: "c.go",
|
||||
Line: 56,
|
||||
Defect: "unhandled error in critical section",
|
||||
FailureScenario: "database error causes silent data corruption",
|
||||
Verdict: ReviewVerdictConfirmed,
|
||||
Level: ReviewLevelError,
|
||||
Category: ReviewCategoryUncheckedError,
|
||||
},
|
||||
},
|
||||
Summary: "confirmed blocking error",
|
||||
},
|
||||
wantRan: true,
|
||||
wantBlocking: true,
|
||||
wantPassed: false,
|
||||
},
|
||||
{
|
||||
name: "did not run MUST NOT be considered passed even with empty findings",
|
||||
report: ReviewReport{
|
||||
Status: ReviewStatusNotRun,
|
||||
Findings: []ReviewFinding{},
|
||||
SkipReason: "stage not reached due to prior failure",
|
||||
},
|
||||
wantRan: false,
|
||||
wantBlocking: false,
|
||||
wantPassed: false,
|
||||
},
|
||||
{
|
||||
name: "did not run with nil findings does not pass",
|
||||
report: ReviewReport{
|
||||
Status: ReviewStatusNotRun,
|
||||
Findings: nil,
|
||||
SkipReason: "pipeline stopped early",
|
||||
},
|
||||
wantRan: false,
|
||||
wantBlocking: false,
|
||||
wantPassed: false,
|
||||
},
|
||||
{
|
||||
name: "skipped review does not count as ran and does not pass",
|
||||
report: ReviewReport{
|
||||
Status: ReviewStatusSkipped,
|
||||
Findings: []ReviewFinding{},
|
||||
SkipReason: "documentation-only diff skipped by policy",
|
||||
},
|
||||
wantRan: false,
|
||||
wantBlocking: false,
|
||||
wantPassed: false,
|
||||
},
|
||||
{
|
||||
name: "zero-value report does not count as ran and does not pass",
|
||||
report: ReviewReport{},
|
||||
wantRan: false,
|
||||
wantBlocking: false,
|
||||
wantPassed: false,
|
||||
},
|
||||
}
|
||||
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
if got := tt.report.Ran(); got != tt.wantRan {
|
||||
t.Errorf("ReviewReport.Ran() = %v, want %v", got, tt.wantRan)
|
||||
}
|
||||
if got := tt.report.Blocking(); got != tt.wantBlocking {
|
||||
t.Errorf("ReviewReport.Blocking() = %v, want %v", got, tt.wantBlocking)
|
||||
}
|
||||
if got := tt.report.Passed(); got != tt.wantPassed {
|
||||
t.Errorf("ReviewReport.Passed() = %v, want %v", got, tt.wantPassed)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestReviewReportConstructors(t *testing.T) {
|
||||
t.Run("NewCompletedReviewReport sets completed status", func(t *testing.T) {
|
||||
findings := []ReviewFinding{
|
||||
{
|
||||
File: "pkg/foo.go",
|
||||
Line: 15,
|
||||
Defect: "hardcoded port",
|
||||
FailureScenario: "port 8080 conflicts in container",
|
||||
Verdict: ReviewVerdictConfirmed,
|
||||
Level: ReviewLevelWarn,
|
||||
Category: ReviewCategoryHardcodedValue,
|
||||
},
|
||||
}
|
||||
report := NewCompletedReviewReport(findings, "completed successfully")
|
||||
if report.Status != ReviewStatusCompleted {
|
||||
t.Errorf("Status = %q, want %q", report.Status, ReviewStatusCompleted)
|
||||
}
|
||||
if !report.Ran() {
|
||||
t.Error("Ran() = false, want true")
|
||||
}
|
||||
if len(report.Findings) != 1 {
|
||||
t.Errorf("len(Findings) = %d, want 1", len(report.Findings))
|
||||
}
|
||||
if report.Summary != "completed successfully" {
|
||||
t.Errorf("Summary = %q, want %q", report.Summary, "completed successfully")
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("NewCompletedReviewReport with nil findings initializes empty slice", func(t *testing.T) {
|
||||
report := NewCompletedReviewReport(nil, "clean run")
|
||||
if report.Findings == nil {
|
||||
t.Error("Findings should be non-nil empty slice")
|
||||
}
|
||||
if len(report.Findings) != 0 {
|
||||
t.Errorf("len(Findings) = %d, want 0", len(report.Findings))
|
||||
}
|
||||
if !report.Passed() {
|
||||
t.Error("Passed() = false, want true for clean completed review")
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("NewNotRunReviewReport sets not_run status", func(t *testing.T) {
|
||||
report := NewNotRunReviewReport("prerequisite stage failed")
|
||||
if report.Status != ReviewStatusNotRun {
|
||||
t.Errorf("Status = %q, want %q", report.Status, ReviewStatusNotRun)
|
||||
}
|
||||
if report.Ran() {
|
||||
t.Error("Ran() = true, want false")
|
||||
}
|
||||
if report.Passed() {
|
||||
t.Error("Passed() = true, want false")
|
||||
}
|
||||
if report.SkipReason != "prerequisite stage failed" {
|
||||
t.Errorf("SkipReason = %q, want %q", report.SkipReason, "prerequisite stage failed")
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("NewSkippedReviewReport sets skipped status", func(t *testing.T) {
|
||||
report := NewSkippedReviewReport("markdown change only")
|
||||
if report.Status != ReviewStatusSkipped {
|
||||
t.Errorf("Status = %q, want %q", report.Status, ReviewStatusSkipped)
|
||||
}
|
||||
if report.Ran() {
|
||||
t.Error("Ran() = true, want false")
|
||||
}
|
||||
if report.Passed() {
|
||||
t.Error("Passed() = true, want false")
|
||||
}
|
||||
if report.SkipReason != "markdown change only" {
|
||||
t.Errorf("SkipReason = %q, want %q", report.SkipReason, "markdown change only")
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
func TestReviewFindingJSONRoundTrip(t *testing.T) {
|
||||
original := ReviewFinding{
|
||||
File: "internal/domain/workflow/steps.go",
|
||||
Line: 142,
|
||||
Defect: "Goroutine spawned without error channel drain causes leak",
|
||||
FailureScenario: "When context is cancelled before worker exits, the goroutine remains blocked on ch <- result forever",
|
||||
Verdict: ReviewVerdictConfirmed,
|
||||
Disposition: ReviewDispositionFixed,
|
||||
Level: ReviewLevelError,
|
||||
Category: ReviewCategoryGoroutineLeak,
|
||||
}
|
||||
|
||||
data, err := json.Marshal(original)
|
||||
if err != nil {
|
||||
t.Fatalf("json.Marshal(ReviewFinding) failed: %v", err)
|
||||
}
|
||||
|
||||
var decoded ReviewFinding
|
||||
if err := json.Unmarshal(data, &decoded); err != nil {
|
||||
t.Fatalf("json.Unmarshal(ReviewFinding) failed: %v", err)
|
||||
}
|
||||
|
||||
if !reflect.DeepEqual(original, decoded) {
|
||||
t.Errorf("JSON round-trip mismatch:\n got: %#v\n want: %#v", decoded, original)
|
||||
}
|
||||
|
||||
// Explicitly assert that verdict and disposition survived the round trip
|
||||
if decoded.Verdict != ReviewVerdictConfirmed {
|
||||
t.Errorf("Verdict = %q, want %q", decoded.Verdict, ReviewVerdictConfirmed)
|
||||
}
|
||||
if decoded.Disposition != ReviewDispositionFixed {
|
||||
t.Errorf("Disposition = %q, want %q", decoded.Disposition, ReviewDispositionFixed)
|
||||
}
|
||||
if decoded.Level != ReviewLevelError {
|
||||
t.Errorf("Level = %q, want %q", decoded.Level, ReviewLevelError)
|
||||
}
|
||||
if decoded.Category != ReviewCategoryGoroutineLeak {
|
||||
t.Errorf("Category = %q, want %q", decoded.Category, ReviewCategoryGoroutineLeak)
|
||||
}
|
||||
if !decoded.Blocking() {
|
||||
t.Error("Decoded finding should remain blocking after round-trip")
|
||||
}
|
||||
}
|
||||
|
||||
func TestReviewReportJSONRoundTrip(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
report ReviewReport
|
||||
}{
|
||||
{
|
||||
name: "completed with multiple findings",
|
||||
report: ReviewReport{
|
||||
Status: ReviewStatusCompleted,
|
||||
Findings: []ReviewFinding{
|
||||
{
|
||||
File: "pkg/auth/token.go",
|
||||
Line: 88,
|
||||
Defect: "Missing signature verification before claims decoding",
|
||||
FailureScenario: "Forged JWT with 'none' alg is accepted as valid",
|
||||
Verdict: ReviewVerdictConfirmed,
|
||||
Disposition: ReviewDispositionNoChangeNeeded,
|
||||
Level: ReviewLevelError,
|
||||
Category: ReviewCategoryInterfaceSatisfaction,
|
||||
},
|
||||
{
|
||||
File: "pkg/auth/token.go",
|
||||
Line: 120,
|
||||
Defect: "Debug log prints raw token",
|
||||
FailureScenario: "Verbose logging exposes bearer token in daemon logs",
|
||||
Verdict: ReviewVerdictPlausible,
|
||||
Disposition: ReviewDispositionSkipped,
|
||||
Level: ReviewLevelWarn,
|
||||
Category: ReviewCategoryOther,
|
||||
},
|
||||
},
|
||||
Summary: "Found 1 confirmed defect and 1 advisory",
|
||||
},
|
||||
},
|
||||
{
|
||||
name: "completed with zero findings",
|
||||
report: ReviewReport{
|
||||
Status: ReviewStatusCompleted,
|
||||
Findings: []ReviewFinding{},
|
||||
Summary: "Clean review pass",
|
||||
},
|
||||
},
|
||||
{
|
||||
name: "zero findings with checked properties",
|
||||
report: ReviewReport{
|
||||
Status: ReviewStatusCompleted,
|
||||
Findings: []ReviewFinding{},
|
||||
Checked: []ReviewCheck{
|
||||
{Property: "nil-safety of Foo callers", Evidence: "read all 4 callers; each nil-checks"},
|
||||
{Property: "error propagation in Bar", Evidence: "traced Bar through to the caller's error return"},
|
||||
},
|
||||
Summary: "Checked and cleared",
|
||||
},
|
||||
},
|
||||
{
|
||||
name: "not run report",
|
||||
report: ReviewReport{
|
||||
Status: ReviewStatusNotRun,
|
||||
SkipReason: "gate failed prior to review",
|
||||
},
|
||||
},
|
||||
{
|
||||
name: "skipped report",
|
||||
report: ReviewReport{
|
||||
Status: ReviewStatusSkipped,
|
||||
SkipReason: "doc changes only",
|
||||
},
|
||||
},
|
||||
}
|
||||
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
data, err := json.Marshal(tt.report)
|
||||
if err != nil {
|
||||
t.Fatalf("json.Marshal(ReviewReport) failed: %v", err)
|
||||
}
|
||||
|
||||
var decoded ReviewReport
|
||||
if err := json.Unmarshal(data, &decoded); err != nil {
|
||||
t.Fatalf("json.Unmarshal(ReviewReport) failed: %v", err)
|
||||
}
|
||||
|
||||
// For empty slice, JSON unmarshals as nil if omitted or [] if present.
|
||||
// Compare semantic state.
|
||||
if decoded.Status != tt.report.Status {
|
||||
t.Errorf("Status = %q, want %q", decoded.Status, tt.report.Status)
|
||||
}
|
||||
if decoded.Summary != tt.report.Summary {
|
||||
t.Errorf("Summary = %q, want %q", decoded.Summary, tt.report.Summary)
|
||||
}
|
||||
if decoded.SkipReason != tt.report.SkipReason {
|
||||
t.Errorf("SkipReason = %q, want %q", decoded.SkipReason, tt.report.SkipReason)
|
||||
}
|
||||
if decoded.Ran() != tt.report.Ran() {
|
||||
t.Errorf("Ran() = %v, want %v", decoded.Ran(), tt.report.Ran())
|
||||
}
|
||||
if decoded.Blocking() != tt.report.Blocking() {
|
||||
t.Errorf("Blocking() = %v, want %v", decoded.Blocking(), tt.report.Blocking())
|
||||
}
|
||||
if decoded.Passed() != tt.report.Passed() {
|
||||
t.Errorf("Passed() = %v, want %v", decoded.Passed(), tt.report.Passed())
|
||||
}
|
||||
if len(decoded.Findings) != len(tt.report.Findings) {
|
||||
t.Fatalf("len(Findings) = %d, want %d", len(decoded.Findings), len(tt.report.Findings))
|
||||
}
|
||||
for i := range decoded.Findings {
|
||||
if !reflect.DeepEqual(decoded.Findings[i], tt.report.Findings[i]) {
|
||||
t.Errorf("Findings[%d] mismatch:\n got: %#v\n want: %#v",
|
||||
i, decoded.Findings[i], tt.report.Findings[i])
|
||||
}
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestReviewFindingValidation(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
finding ReviewFinding
|
||||
wantError bool
|
||||
}{
|
||||
{
|
||||
name: "valid full finding with confirmed verdict and error level",
|
||||
finding: ReviewFinding{
|
||||
File: "internal/domain/workflow/steps.go",
|
||||
Line: 42,
|
||||
Defect: "Nil check missing before dereferencing context",
|
||||
FailureScenario: "Calling step with nil context causes panic",
|
||||
Verdict: ReviewVerdictConfirmed,
|
||||
Disposition: ReviewDispositionFixed,
|
||||
Level: ReviewLevelError,
|
||||
Category: ReviewCategoryNilRisk,
|
||||
},
|
||||
wantError: false,
|
||||
},
|
||||
{
|
||||
name: "valid finding without disposition (new unacted finding)",
|
||||
finding: ReviewFinding{
|
||||
File: "internal/domain/workflow/steps.go",
|
||||
Line: 10,
|
||||
Defect: "Unchecked error return",
|
||||
FailureScenario: "Write failure ignored silently",
|
||||
Verdict: ReviewVerdictPlausible,
|
||||
Level: ReviewLevelWarn,
|
||||
Category: ReviewCategoryUncheckedError,
|
||||
},
|
||||
wantError: false,
|
||||
},
|
||||
{
|
||||
name: "valid finding with line 0",
|
||||
finding: ReviewFinding{
|
||||
File: "README.md",
|
||||
Line: 0,
|
||||
Defect: "Missing architecture section",
|
||||
FailureScenario: "New developers cannot find component map",
|
||||
Verdict: ReviewVerdictConfirmed,
|
||||
Level: ReviewLevelWarn,
|
||||
Category: ReviewCategoryOther,
|
||||
},
|
||||
wantError: false,
|
||||
},
|
||||
{
|
||||
name: "all valid checklist categories accepted",
|
||||
finding: ReviewFinding{
|
||||
File: "file.go",
|
||||
Line: 1,
|
||||
Defect: "Valid defect",
|
||||
FailureScenario: "Valid scenario",
|
||||
Verdict: ReviewVerdictConfirmed,
|
||||
Level: ReviewLevelError,
|
||||
Category: ReviewCategoryDeadCode,
|
||||
},
|
||||
wantError: false,
|
||||
},
|
||||
{
|
||||
name: "invalid empty file",
|
||||
finding: ReviewFinding{
|
||||
File: "",
|
||||
Line: 10,
|
||||
Defect: "Defect without file",
|
||||
FailureScenario: "Scenario",
|
||||
Verdict: ReviewVerdictConfirmed,
|
||||
Level: ReviewLevelError,
|
||||
Category: ReviewCategoryNilRisk,
|
||||
},
|
||||
wantError: true,
|
||||
},
|
||||
{
|
||||
name: "invalid negative line",
|
||||
finding: ReviewFinding{
|
||||
File: "file.go",
|
||||
Line: -5,
|
||||
Defect: "Negative line finding",
|
||||
FailureScenario: "Scenario",
|
||||
Verdict: ReviewVerdictConfirmed,
|
||||
Level: ReviewLevelError,
|
||||
Category: ReviewCategoryNilRisk,
|
||||
},
|
||||
wantError: true,
|
||||
},
|
||||
{
|
||||
name: "invalid empty defect",
|
||||
finding: ReviewFinding{
|
||||
File: "file.go",
|
||||
Line: 10,
|
||||
Defect: "",
|
||||
FailureScenario: "Scenario",
|
||||
Verdict: ReviewVerdictConfirmed,
|
||||
Level: ReviewLevelError,
|
||||
Category: ReviewCategoryNilRisk,
|
||||
},
|
||||
wantError: true,
|
||||
},
|
||||
{
|
||||
name: "invalid empty failure scenario",
|
||||
finding: ReviewFinding{
|
||||
File: "file.go",
|
||||
Line: 10,
|
||||
Defect: "Defect without scenario",
|
||||
FailureScenario: "",
|
||||
Verdict: ReviewVerdictConfirmed,
|
||||
Level: ReviewLevelError,
|
||||
Category: ReviewCategoryNilRisk,
|
||||
},
|
||||
wantError: true,
|
||||
},
|
||||
{
|
||||
name: "multi-line suggestion is rejected",
|
||||
finding: ReviewFinding{
|
||||
File: "file.go",
|
||||
Line: 10,
|
||||
Defect: "Defect with multi-line suggestion",
|
||||
FailureScenario: "Scenario",
|
||||
Verdict: ReviewVerdictConfirmed,
|
||||
Level: ReviewLevelError,
|
||||
Category: ReviewCategoryNilRisk,
|
||||
Suggestion: "return nil\nmu.Lock()",
|
||||
},
|
||||
wantError: true,
|
||||
},
|
||||
{
|
||||
name: "single-line suggestion on a confirmed finding is valid",
|
||||
finding: ReviewFinding{
|
||||
File: "file.go",
|
||||
Line: 10,
|
||||
Defect: "Defect with single-line suggestion",
|
||||
FailureScenario: "Scenario",
|
||||
Verdict: ReviewVerdictConfirmed,
|
||||
Level: ReviewLevelError,
|
||||
Category: ReviewCategoryNilRisk,
|
||||
Suggestion: "mu.Lock()",
|
||||
},
|
||||
wantError: false,
|
||||
},
|
||||
{
|
||||
name: "invalid verdict",
|
||||
finding: ReviewFinding{
|
||||
File: "file.go",
|
||||
Line: 10,
|
||||
Defect: "Defect with bogus verdict",
|
||||
FailureScenario: "Scenario",
|
||||
Verdict: "maybe",
|
||||
Level: ReviewLevelError,
|
||||
Category: ReviewCategoryNilRisk,
|
||||
},
|
||||
wantError: true,
|
||||
},
|
||||
{
|
||||
name: "invalid level",
|
||||
finding: ReviewFinding{
|
||||
File: "file.go",
|
||||
Line: 10,
|
||||
Defect: "Defect with bogus level",
|
||||
FailureScenario: "Scenario",
|
||||
Verdict: ReviewVerdictConfirmed,
|
||||
Level: "critical",
|
||||
Category: ReviewCategoryNilRisk,
|
||||
},
|
||||
wantError: true,
|
||||
},
|
||||
{
|
||||
name: "invalid category",
|
||||
finding: ReviewFinding{
|
||||
File: "file.go",
|
||||
Line: 10,
|
||||
Defect: "Defect with bogus category",
|
||||
FailureScenario: "Scenario",
|
||||
Verdict: ReviewVerdictConfirmed,
|
||||
Level: ReviewLevelError,
|
||||
Category: "unknown-category",
|
||||
},
|
||||
wantError: true,
|
||||
},
|
||||
{
|
||||
name: "invalid disposition",
|
||||
finding: ReviewFinding{
|
||||
File: "file.go",
|
||||
Line: 10,
|
||||
Defect: "Defect with bogus disposition",
|
||||
FailureScenario: "Scenario",
|
||||
Verdict: ReviewVerdictConfirmed,
|
||||
Disposition: "ignored",
|
||||
Level: ReviewLevelError,
|
||||
Category: ReviewCategoryNilRisk,
|
||||
},
|
||||
wantError: true,
|
||||
},
|
||||
}
|
||||
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
err := tt.finding.Validate()
|
||||
if (err != nil) != tt.wantError {
|
||||
t.Errorf("ReviewFinding.Validate() error = %v, wantError %v", err, tt.wantError)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestReviewReportValidation(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
report ReviewReport
|
||||
wantError bool
|
||||
}{
|
||||
{
|
||||
name: "valid completed report with valid findings",
|
||||
report: ReviewReport{
|
||||
Status: ReviewStatusCompleted,
|
||||
Findings: []ReviewFinding{
|
||||
{
|
||||
File: "file.go",
|
||||
Line: 10,
|
||||
Defect: "Valid defect",
|
||||
FailureScenario: "Valid scenario",
|
||||
Verdict: ReviewVerdictConfirmed,
|
||||
Level: ReviewLevelError,
|
||||
Category: ReviewCategoryNilRisk,
|
||||
},
|
||||
},
|
||||
Summary: "valid summary",
|
||||
},
|
||||
wantError: false,
|
||||
},
|
||||
{
|
||||
name: "valid not_run report",
|
||||
report: ReviewReport{
|
||||
Status: ReviewStatusNotRun,
|
||||
SkipReason: "gate failed",
|
||||
},
|
||||
wantError: false,
|
||||
},
|
||||
{
|
||||
name: "valid skipped report",
|
||||
report: ReviewReport{
|
||||
Status: ReviewStatusSkipped,
|
||||
SkipReason: "policy skip",
|
||||
},
|
||||
wantError: false,
|
||||
},
|
||||
{
|
||||
name: "invalid report status",
|
||||
report: ReviewReport{
|
||||
Status: "in_progress",
|
||||
},
|
||||
wantError: true,
|
||||
},
|
||||
{
|
||||
name: "valid completed report with checked properties",
|
||||
report: ReviewReport{
|
||||
Status: ReviewStatusCompleted,
|
||||
Checked: []ReviewCheck{{Property: "nil-safety of Foo callers", Evidence: "read all 4 callers"}},
|
||||
},
|
||||
wantError: false,
|
||||
},
|
||||
{
|
||||
name: "checked entry without evidence is rejected",
|
||||
report: ReviewReport{
|
||||
Status: ReviewStatusCompleted,
|
||||
Checked: []ReviewCheck{{Property: "looks fine"}},
|
||||
},
|
||||
wantError: true,
|
||||
},
|
||||
{
|
||||
name: "checked entry without property is rejected",
|
||||
report: ReviewReport{
|
||||
Status: ReviewStatusCompleted,
|
||||
Checked: []ReviewCheck{{Evidence: "read the file"}},
|
||||
},
|
||||
wantError: true,
|
||||
},
|
||||
{
|
||||
name: "invalid finding inside findings slice",
|
||||
report: ReviewReport{
|
||||
Status: ReviewStatusCompleted,
|
||||
Findings: []ReviewFinding{
|
||||
{
|
||||
File: "",
|
||||
Line: 10,
|
||||
Defect: "Missing file",
|
||||
FailureScenario: "Scenario",
|
||||
Verdict: ReviewVerdictConfirmed,
|
||||
Level: ReviewLevelError,
|
||||
Category: ReviewCategoryNilRisk,
|
||||
},
|
||||
},
|
||||
},
|
||||
wantError: true,
|
||||
},
|
||||
}
|
||||
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
err := tt.report.Validate()
|
||||
if (err != nil) != tt.wantError {
|
||||
t.Errorf("ReviewReport.Validate() error = %v, wantError %v", err, tt.wantError)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
@@ -1,56 +0,0 @@
|
||||
package workflow
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"strings"
|
||||
)
|
||||
|
||||
// RenderMarkdown renders an adversarial review report as the artifact source
|
||||
// the collaborative editor publishes. It is a pure function over the report:
|
||||
// no I/O, no infrastructure, so the rendered form is testable in isolation.
|
||||
//
|
||||
// A completed review renders its summary, every finding as its own subsection
|
||||
// (file, line, verdict, level, failure scenario, and the mechanical suggestion
|
||||
// and disposition when present) and the verified properties as a "checked
|
||||
// clean" list. A review that did not run renders only its reason, because a
|
||||
// report that never executed has no findings or checks to stand behind.
|
||||
func (r ReviewReport) RenderMarkdown(title string) string {
|
||||
var b strings.Builder
|
||||
fmt.Fprintf(&b, "# %s\n\n", title)
|
||||
|
||||
if r.Status == ReviewStatusNotRun || (r.SkipReason != "" && len(r.Findings) == 0 && len(r.Checked) == 0) {
|
||||
fmt.Fprintf(&b, "> This review did not run: %s\n", r.SkipReason)
|
||||
return strings.TrimRight(b.String(), "\n") + "\n"
|
||||
}
|
||||
|
||||
if r.Summary != "" {
|
||||
b.WriteString(r.Summary)
|
||||
b.WriteString("\n\n")
|
||||
}
|
||||
|
||||
if len(r.Findings) > 0 {
|
||||
b.WriteString("## Findings\n\n")
|
||||
for _, f := range r.Findings {
|
||||
fmt.Fprintf(&b, "### %s:%d — %s (%s)\n\n", f.File, f.Line, f.Level, f.Verdict)
|
||||
fmt.Fprintf(&b, "**Defect:** %s\n\n", f.Defect)
|
||||
if f.FailureScenario != "" {
|
||||
fmt.Fprintf(&b, "**Failure scenario:** %s\n\n", f.FailureScenario)
|
||||
}
|
||||
if f.Suggestion != "" {
|
||||
fmt.Fprintf(&b, "Suggested fix: %s\n\n", f.Suggestion)
|
||||
}
|
||||
if f.Disposition != "" {
|
||||
fmt.Fprintf(&b, "Disposition: %s\n\n", f.Disposition)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if len(r.Checked) > 0 {
|
||||
b.WriteString("## Checked clean\n\n")
|
||||
for _, c := range r.Checked {
|
||||
fmt.Fprintf(&b, "- %s — %s\n", c.Property, c.Evidence)
|
||||
}
|
||||
}
|
||||
|
||||
return strings.TrimRight(b.String(), "\n") + "\n"
|
||||
}
|
||||
@@ -1,78 +0,0 @@
|
||||
package workflow
|
||||
|
||||
import (
|
||||
"strings"
|
||||
"testing"
|
||||
)
|
||||
|
||||
func mdFixture() ReviewReport {
|
||||
disposition := ReviewDisposition("fixed")
|
||||
return ReviewReport{
|
||||
Status: ReviewStatusCompleted,
|
||||
Summary: "The nil-deref path is reachable from a malformed payload.",
|
||||
Findings: []ReviewFinding{
|
||||
{
|
||||
File: "handler.go", Line: 42, Defect: "nil deref on empty payload",
|
||||
FailureScenario: "POST with a body lacking the field crashes the handler.",
|
||||
Suggestion: "guard the field before use", Verdict: ReviewVerdictConfirmed,
|
||||
Level: ReviewLevelError, Category: "correctness", Disposition: disposition,
|
||||
},
|
||||
{
|
||||
File: "store.go", Line: 9, Defect: "query loses the filter",
|
||||
FailureScenario: "paged reads return rows from other scopes",
|
||||
Verdict: ReviewVerdictPlausible, Level: ReviewLevelWarn,
|
||||
},
|
||||
},
|
||||
Checked: []ReviewCheck{
|
||||
{Property: "nil-safety of Foo callers", Evidence: "read all 4 callers; each nil-checks"},
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
func TestReviewReportRendersFindingsAsMarkdown(t *testing.T) {
|
||||
got := mdFixture().RenderMarkdown("Review of acme/widget#42")
|
||||
for _, want := range []string{
|
||||
"# Review of acme/widget#42",
|
||||
"The nil-deref path is reachable",
|
||||
"### handler.go:42",
|
||||
"**Defect:** nil deref on empty payload",
|
||||
"**Failure scenario:** POST with a body lacking the field crashes the handler.",
|
||||
"error (confirmed)",
|
||||
"Suggested fix: guard the field before use",
|
||||
"Disposition: fixed",
|
||||
"### store.go:9",
|
||||
"warn (plausible)",
|
||||
"## Checked clean",
|
||||
"- nil-safety of Foo callers — read all 4 callers; each nil-checks",
|
||||
} {
|
||||
if !strings.Contains(got, want) {
|
||||
t.Fatalf("rendered report missing %q:\n%s", want, got)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestReviewReportRendersNotRunStatus(t *testing.T) {
|
||||
report := NewNotRunReviewReport("no reviewer model configured")
|
||||
got := report.RenderMarkdown("Review of acme/widget#42")
|
||||
if !strings.Contains(got, "no reviewer model configured") {
|
||||
t.Fatalf("skip reason missing:\n%s", got)
|
||||
}
|
||||
if strings.Contains(got, "## Findings") || strings.Contains(got, "## Checked clean") {
|
||||
t.Fatalf("a not-run report must not render finding or check sections:\n%s", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestReviewReportRendersACleanReview(t *testing.T) {
|
||||
report := ReviewReport{
|
||||
Status: ReviewStatusCompleted,
|
||||
Summary: "No defects found.",
|
||||
Checked: []ReviewCheck{{Property: "p", Evidence: "e"}},
|
||||
}
|
||||
got := report.RenderMarkdown("Review")
|
||||
if strings.Contains(got, "### ") {
|
||||
t.Fatalf("a zero-finding review must not render finding sections:\n%s", got)
|
||||
}
|
||||
if !strings.Contains(got, "No defects found.") || !strings.Contains(got, "- p — e") {
|
||||
t.Fatalf("summary and checks missing:\n%s", got)
|
||||
}
|
||||
}
|
||||
@@ -1,125 +0,0 @@
|
||||
package workflow
|
||||
|
||||
import (
|
||||
"strings"
|
||||
"testing"
|
||||
)
|
||||
|
||||
func TestRenderPRReviewSection(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
report ReviewReport
|
||||
empty bool
|
||||
want []string
|
||||
}{
|
||||
{
|
||||
name: "zero value renders nothing",
|
||||
report: ReviewReport{},
|
||||
empty: true,
|
||||
},
|
||||
{
|
||||
name: "not run renders nothing",
|
||||
report: NewNotRunReviewReport("provider down"),
|
||||
empty: true,
|
||||
},
|
||||
{
|
||||
name: "zero findings is explicit",
|
||||
report: NewCompletedReviewReport(nil, "checked and cleared"),
|
||||
want: []string{"Adversarial self-review ran", "found no defects"},
|
||||
},
|
||||
{
|
||||
name: "checked properties back a zero-finding review",
|
||||
report: ReviewReport{
|
||||
Status: ReviewStatusCompleted,
|
||||
Checked: []ReviewCheck{{Property: "nil-safety of Foo callers", Evidence: "read all 4 callers"}},
|
||||
},
|
||||
want: []string{"found no defects", "Checked and cleared", "nil-safety of Foo callers", "read all 4 callers"},
|
||||
},
|
||||
{
|
||||
name: "checked properties also render alongside findings",
|
||||
report: ReviewReport{
|
||||
Status: ReviewStatusCompleted,
|
||||
Findings: []ReviewFinding{
|
||||
{File: "a.go", Line: 1, Defect: "warn", FailureScenario: "x", Verdict: ReviewVerdictPlausible, Level: ReviewLevelWarn, Category: ReviewCategoryOther},
|
||||
},
|
||||
Checked: []ReviewCheck{{Property: "error paths", Evidence: "traced two"}},
|
||||
},
|
||||
want: []string{"Non-blocking notes", "Checked and cleared", "error paths"},
|
||||
},
|
||||
{
|
||||
name: "non-blocking findings listed",
|
||||
report: NewCompletedReviewReport([]ReviewFinding{
|
||||
{File: "a.go", Line: 12, Defect: "style nit", FailureScenario: "x", Verdict: ReviewVerdictPlausible, Level: ReviewLevelWarn, Category: ReviewCategoryOther},
|
||||
}, "one note"),
|
||||
want: []string{"found no blocking defects", "a.go:12", "style nit"},
|
||||
},
|
||||
{
|
||||
name: "whole-file finding omits line",
|
||||
report: NewCompletedReviewReport([]ReviewFinding{
|
||||
{File: "README.md", Defect: "typo", FailureScenario: "x", Verdict: ReviewVerdictPlausible, Level: ReviewLevelWarn, Category: ReviewCategoryOther},
|
||||
}, ""),
|
||||
want: []string{"README.md", "typo"},
|
||||
},
|
||||
}
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
got := renderPRReviewSection(tt.report)
|
||||
if tt.empty {
|
||||
if got != "" {
|
||||
t.Fatalf("renderPRReviewSection = %q, want empty", got)
|
||||
}
|
||||
return
|
||||
}
|
||||
for _, w := range tt.want {
|
||||
if !strings.Contains(got, w) {
|
||||
t.Errorf("renderPRReviewSection missing %q in:\n%s", w, got)
|
||||
}
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestImplementPRBodyOmitsFindingsWhenReviewDidNotRun(t *testing.T) {
|
||||
tc := &TaskContext{BuildSummary: "did the thing", Task: &Task{}}
|
||||
body := implementPRBody(tc)
|
||||
if strings.Contains(body, "Adversarial review") {
|
||||
t.Fatalf("PR body includes a review section when no review ran:\n%s", body)
|
||||
}
|
||||
if !strings.Contains(body, "did the thing") {
|
||||
t.Fatalf("PR body lost the build summary:\n%s", body)
|
||||
}
|
||||
}
|
||||
|
||||
func TestImplementPRBodyCarriesZeroFindingSection(t *testing.T) {
|
||||
tc := &TaskContext{
|
||||
BuildSummary: "did the thing",
|
||||
Task: &Task{},
|
||||
ReviewReport: NewCompletedReviewReport(nil, "checked and cleared"),
|
||||
}
|
||||
body := implementPRBody(tc)
|
||||
if !strings.Contains(body, "found no defects") {
|
||||
t.Fatalf("PR body missing the zero-findings review section:\n%s", body)
|
||||
}
|
||||
if !strings.Contains(body, "did the thing") {
|
||||
t.Fatalf("PR body lost the build summary:\n%s", body)
|
||||
}
|
||||
}
|
||||
|
||||
// TestStageReviewStashesReportForPRBody pins the integration between
|
||||
// StageReview and implementPRBody: a passing review must land on
|
||||
// tc.ReviewReport, or the PR body silently omits the findings section
|
||||
// (renderPRReviewSection returns "" for a zero-value report).
|
||||
func TestStageReviewStashesReportForPRBody(t *testing.T) {
|
||||
tc, _ := reviewTaskContext(t, true)
|
||||
tc.Reviewer = &fakeReviewer{report: NewCompletedReviewReport(nil, "clean")}
|
||||
|
||||
if err := StageReview().Run(t.Context(), tc); err != nil {
|
||||
t.Fatalf("StageReview().Run() error = %v", err)
|
||||
}
|
||||
if !tc.ReviewReport.Ran() {
|
||||
t.Fatal("ReviewReport not stashed after a passing review; the PR body would omit the findings section")
|
||||
}
|
||||
if tc.ReviewReport.Status != ReviewStatusCompleted {
|
||||
t.Errorf("ReviewReport.Status = %q, want %q", tc.ReviewReport.Status, ReviewStatusCompleted)
|
||||
}
|
||||
}
|
||||
@@ -1,203 +0,0 @@
|
||||
package workflow
|
||||
|
||||
import (
|
||||
"context"
|
||||
"log/slog"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"github.com/samcharles93/archie-core/internal/config"
|
||||
"github.com/samcharles93/archie-core/internal/worktree"
|
||||
)
|
||||
|
||||
// fakeReviewer records the request it was given and returns a canned
|
||||
// report, standing in for the app-layer agent.Subagent-backed Reviewer.
|
||||
//
|
||||
// It reads what it needs from req.SnapshotDir synchronously, inside
|
||||
// Review, because the stage cleans the snapshot up as soon as Review
|
||||
// returns -- a real Reviewer implementation faces the same constraint.
|
||||
type fakeReviewer struct {
|
||||
gotRequest ReviewRequest
|
||||
snapshotContent string
|
||||
snapshotHadGit bool
|
||||
called bool
|
||||
report ReviewReport
|
||||
}
|
||||
|
||||
func (f *fakeReviewer) Review(_ context.Context, req ReviewRequest) ReviewReport {
|
||||
f.called = true
|
||||
f.gotRequest = req
|
||||
if content, err := os.ReadFile(filepath.Join(req.SnapshotDir, "feature.go")); err == nil {
|
||||
f.snapshotContent = string(content)
|
||||
}
|
||||
if _, err := os.Stat(filepath.Join(req.SnapshotDir, ".git")); err == nil {
|
||||
f.snapshotHadGit = true
|
||||
}
|
||||
return f.report
|
||||
}
|
||||
|
||||
// reviewTaskContext builds a TaskContext over a real git repo with a
|
||||
// tracked, committed change against base, so StageReview's Diff/Snapshot
|
||||
// calls have real content to work with.
|
||||
func reviewTaskContext(t *testing.T, enabled bool) (*TaskContext, string) {
|
||||
t.Helper()
|
||||
dir := gitRepoWithOriginRef(t, "main")
|
||||
if err := os.WriteFile(filepath.Join(dir, "feature.go"), []byte("package main\n"), 0o644); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
runGit(t, dir, "add", "-A")
|
||||
runGit(t, dir, "commit", "-m", "feat: add feature.go")
|
||||
|
||||
tc := &TaskContext{
|
||||
Task: &Task{ID: 1, Owner: "acme", Repo: "todo", IssueNumber: 42, Title: "Add feature", Body: "Please add it."},
|
||||
Repo: config.Repo{Owner: "acme", Name: "todo", Base: "main", ReviewEnabled: enabled},
|
||||
Cfg: config.Config{},
|
||||
Trees: &worktree.Manager{WorkDir: t.TempDir()},
|
||||
Dir: dir,
|
||||
Log: slog.New(slog.DiscardHandler),
|
||||
}
|
||||
return tc, dir
|
||||
}
|
||||
|
||||
func TestStageReviewIsInertUnlessEnabled(t *testing.T) {
|
||||
tc, _ := reviewTaskContext(t, false)
|
||||
reviewer := &fakeReviewer{}
|
||||
tc.Reviewer = reviewer
|
||||
|
||||
if err := StageReview().Run(t.Context(), tc); err != nil {
|
||||
t.Fatalf("StageReview().Run() error = %v", err)
|
||||
}
|
||||
if reviewer.called {
|
||||
t.Error("StageReview called the reviewer despite Repo.ReviewEnabled == false")
|
||||
}
|
||||
if tc.Outcome.Status != "" {
|
||||
t.Errorf("Outcome.Status = %q, want unset", tc.Outcome.Status)
|
||||
}
|
||||
}
|
||||
|
||||
func TestStageReviewParksWhenEnabledWithNoReviewerWired(t *testing.T) {
|
||||
tc, _ := reviewTaskContext(t, true)
|
||||
tc.Reviewer = nil
|
||||
|
||||
if err := StageReview().Run(t.Context(), tc); err != nil {
|
||||
t.Fatalf("StageReview().Run() error = %v", err)
|
||||
}
|
||||
if tc.Outcome.Status != StatusParked {
|
||||
t.Errorf("Outcome.Status = %q, want %q", tc.Outcome.Status, StatusParked)
|
||||
}
|
||||
}
|
||||
|
||||
func TestStageReviewParksFailClosedWhenReviewDidNotRun(t *testing.T) {
|
||||
tc, _ := reviewTaskContext(t, true)
|
||||
reviewer := &fakeReviewer{report: NewNotRunReviewReport("provider outage")}
|
||||
tc.Reviewer = reviewer
|
||||
|
||||
if err := StageReview().Run(t.Context(), tc); err != nil {
|
||||
t.Fatalf("StageReview().Run() error = %v", err)
|
||||
}
|
||||
if !reviewer.called {
|
||||
t.Fatal("reviewer was not called")
|
||||
}
|
||||
if tc.Outcome.Status != StatusParked {
|
||||
t.Errorf("Outcome.Status = %q, want %q", tc.Outcome.Status, StatusParked)
|
||||
}
|
||||
if got := tc.Outcome.Detail; !containsAll(got, "did not run", "provider outage") {
|
||||
t.Errorf("Outcome.Detail = %q, want it to say the review did not run and why", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestStageReviewParksOnConfirmedErrorFinding(t *testing.T) {
|
||||
tc, _ := reviewTaskContext(t, true)
|
||||
blocking := ReviewFinding{
|
||||
File: "feature.go", Line: 1, Defect: "nil deref",
|
||||
FailureScenario: "calling Foo(nil) panics",
|
||||
Verdict: ReviewVerdictConfirmed, Level: ReviewLevelError, Category: ReviewCategoryNilRisk,
|
||||
}
|
||||
reviewer := &fakeReviewer{report: NewCompletedReviewReport([]ReviewFinding{blocking}, "found one defect")}
|
||||
tc.Reviewer = reviewer
|
||||
|
||||
if err := StageReview().Run(t.Context(), tc); err != nil {
|
||||
t.Fatalf("StageReview().Run() error = %v", err)
|
||||
}
|
||||
if tc.Outcome.Status != StatusParked {
|
||||
t.Errorf("Outcome.Status = %q, want %q", tc.Outcome.Status, StatusParked)
|
||||
}
|
||||
if got := tc.Outcome.Detail; !containsAll(got, "feature.go", "nil deref") {
|
||||
t.Errorf("Outcome.Detail = %q, want the blocking finding rendered", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestStageReviewDoesNotBlockOnWarnOrPlausibleFindings(t *testing.T) {
|
||||
tc, _ := reviewTaskContext(t, true)
|
||||
findings := []ReviewFinding{
|
||||
{
|
||||
File: "feature.go", Line: 1, Defect: "could be tidier",
|
||||
FailureScenario: "n/a", Verdict: ReviewVerdictConfirmed, Level: ReviewLevelWarn,
|
||||
Category: ReviewCategoryOther,
|
||||
},
|
||||
{
|
||||
File: "feature.go", Line: 2, Defect: "maybe a race",
|
||||
FailureScenario: "under heavy concurrency", Verdict: ReviewVerdictPlausible, Level: ReviewLevelError,
|
||||
Category: ReviewCategoryRace,
|
||||
},
|
||||
}
|
||||
reviewer := &fakeReviewer{report: NewCompletedReviewReport(findings, "found two non-blocking findings")}
|
||||
tc.Reviewer = reviewer
|
||||
|
||||
if err := StageReview().Run(t.Context(), tc); err != nil {
|
||||
t.Fatalf("StageReview().Run() error = %v", err)
|
||||
}
|
||||
if tc.Outcome.Status != "" {
|
||||
t.Errorf("Outcome.Status = %q, want unset (non-blocking findings must not park)", tc.Outcome.Status)
|
||||
}
|
||||
}
|
||||
|
||||
func TestStageReviewGivesReviewerAnIsolatedSnapshotDiffAndIssueText(t *testing.T) {
|
||||
tc, dir := reviewTaskContext(t, true)
|
||||
reviewer := &fakeReviewer{report: NewCompletedReviewReport(nil, "clean")}
|
||||
tc.Reviewer = reviewer
|
||||
|
||||
if err := StageReview().Run(t.Context(), tc); err != nil {
|
||||
t.Fatalf("StageReview().Run() error = %v", err)
|
||||
}
|
||||
if !reviewer.called {
|
||||
t.Fatal("reviewer was not called")
|
||||
}
|
||||
req := reviewer.gotRequest
|
||||
|
||||
if req.SnapshotDir == "" {
|
||||
t.Fatal("ReviewRequest.SnapshotDir is empty")
|
||||
}
|
||||
if req.SnapshotDir == dir {
|
||||
t.Errorf("ReviewRequest.SnapshotDir = %q reuses the task worktree; want a distinct snapshot", req.SnapshotDir)
|
||||
}
|
||||
if reviewer.snapshotContent != "package main\n" {
|
||||
t.Errorf("snapshotted feature.go = %q, want %q", reviewer.snapshotContent, "package main\n")
|
||||
}
|
||||
if reviewer.snapshotHadGit {
|
||||
t.Error("ReviewRequest.SnapshotDir contains .git, want it stripped")
|
||||
}
|
||||
if req.Diff == "" {
|
||||
t.Error("ReviewRequest.Diff is empty, want the feature.go diff")
|
||||
}
|
||||
if !containsAll(req.IssueText, "Add feature", "Please add it.") {
|
||||
t.Errorf("ReviewRequest.IssueText = %q, want it to contain the task title and body", req.IssueText)
|
||||
}
|
||||
|
||||
// The snapshot is scratch, cleaned up once the stage returns -- it
|
||||
// must not leak into the task worktree's directory tree permanently.
|
||||
if _, err := os.Stat(req.SnapshotDir); !os.IsNotExist(err) {
|
||||
t.Errorf("ReviewRequest.SnapshotDir %q still exists after the stage returned, want it cleaned up", req.SnapshotDir)
|
||||
}
|
||||
}
|
||||
|
||||
func containsAll(s string, substrs ...string) bool {
|
||||
for _, sub := range substrs {
|
||||
if !strings.Contains(s, sub) {
|
||||
return false
|
||||
}
|
||||
}
|
||||
return true
|
||||
}
|
||||
@@ -266,14 +266,7 @@ func OpenPR(ctx context.Context, tc *TaskContext, body string) error {
|
||||
// it. This is the one point where the number is known and the worktree is
|
||||
// still there to measure, so it is captured here rather than left for a
|
||||
// read to reconstruct. Reporting only: captureChanges never fails the stage.
|
||||
//
|
||||
// The same read answers which revision this run's line numbers describe:
|
||||
// nothing commits between StageReview and here, so the worktree still holds
|
||||
// the head the review read, and the stage that posts line-anchored comments
|
||||
// needs it to refuse a post once the PR's head has moved past it.
|
||||
if stats := tc.captureChanges(ctx, capturedAfterOpenPR); stats.HeadSHA != "" {
|
||||
tc.ReviewedHeadSHA = stats.HeadSHA
|
||||
}
|
||||
tc.captureChanges(ctx, capturedAfterOpenPR)
|
||||
return nil
|
||||
}
|
||||
|
||||
|
||||
@@ -9,6 +9,7 @@ package task
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"time"
|
||||
|
||||
"github.com/samcharles93/archie-core/internal/domain/org"
|
||||
@@ -151,6 +152,34 @@ func (t Task) IsForgeBacked() bool {
|
||||
return t.Source != SourceChat
|
||||
}
|
||||
|
||||
// EffectivePRNumber resolves the pull request a pr-review task targets:
|
||||
// PRNumber when a producer already set it directly (archie's own PRs, which
|
||||
// never goes through binding/chat input assignment), otherwise the pr_number
|
||||
// input a binding (watched repositories) or a chat task (the operator
|
||||
// trigger) assigned generically through Inputs. Returns 0 when neither is
|
||||
// set. DecodeInputs (the store's round-trip) decodes a JSON number as
|
||||
// json.Number; a caller that built Inputs by hand in Go may use float64 or
|
||||
// int instead, so all three are accepted.
|
||||
func (t Task) EffectivePRNumber() int {
|
||||
if t.PRNumber != 0 {
|
||||
return t.PRNumber
|
||||
}
|
||||
switch n := t.Inputs["pr_number"].(type) {
|
||||
case json.Number:
|
||||
i, err := n.Int64()
|
||||
if err != nil {
|
||||
return 0
|
||||
}
|
||||
return int(i)
|
||||
case float64:
|
||||
return int(n)
|
||||
case int:
|
||||
return n
|
||||
default:
|
||||
return 0
|
||||
}
|
||||
}
|
||||
|
||||
// Store is the narrow, consumer-owned subset of the State Store contract
|
||||
// that workflow stages call mid-run. The PostgreSQL store and staterpc.Client
|
||||
// satisfy it; this is the interface archie-agent uses over the gRPC State
|
||||
|
||||
@@ -0,0 +1,29 @@
|
||||
package task
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"testing"
|
||||
)
|
||||
|
||||
func TestEffectivePRNumber(t *testing.T) {
|
||||
for _, test := range []struct {
|
||||
name string
|
||||
task Task
|
||||
want int
|
||||
}{
|
||||
{"PRNumber set directly wins", Task{PRNumber: 7, Inputs: map[string]any{"pr_number": json.Number("99")}}, 7},
|
||||
{"json.Number from a store round trip", Task{Inputs: map[string]any{"pr_number": json.Number("42")}}, 42},
|
||||
{"float64 from a hand-built map", Task{Inputs: map[string]any{"pr_number": float64(13)}}, 13},
|
||||
{"int from a hand-built map", Task{Inputs: map[string]any{"pr_number": 5}}, 5},
|
||||
{"missing input", Task{Inputs: map[string]any{}}, 0},
|
||||
{"nil Inputs", Task{}, 0},
|
||||
{"wrong type", Task{Inputs: map[string]any{"pr_number": "not a number"}}, 0},
|
||||
{"malformed json.Number", Task{Inputs: map[string]any{"pr_number": json.Number("not-a-number")}}, 0},
|
||||
} {
|
||||
t.Run(test.name, func(t *testing.T) {
|
||||
if got := test.task.EffectivePRNumber(); got != test.want {
|
||||
t.Errorf("EffectivePRNumber() = %d, want %d", got, test.want)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
@@ -54,6 +54,17 @@ type Forger interface {
|
||||
ReplyToReview(ctx context.Context, owner, repo string, number int, commentID int64, body string) error
|
||||
}
|
||||
|
||||
// ReviewComment is one line-anchored comment to post on an open pull
|
||||
// request: the repo-relative path, the line in the reviewed revision, and
|
||||
// the rendered body -- which may carry a fenced suggestion block. It names
|
||||
// nothing forge-specific, so both the pr-review pipeline (prreview_stages.go)
|
||||
// and the remediate workflow's callers can hand it to Forger unchanged.
|
||||
type ReviewComment struct {
|
||||
Path string
|
||||
Line int
|
||||
Body string
|
||||
}
|
||||
|
||||
// Trees is the subset of *worktree.Manager that workflow stages call
|
||||
// mid-run. *worktree.Manager (the daemon's real manager, holding the push
|
||||
// token) and a hybrid RPC-backed implementation (archie-agent proxies
|
||||
@@ -88,11 +99,17 @@ type TaskContext struct {
|
||||
// workflow.call step: such a step in a runner with no capability
|
||||
// fails the run with a named error (docs/prds/workflow-calls.md).
|
||||
Calls task.Caller
|
||||
// Reviewer runs the adversarial self-review stage (StageReview). Nil
|
||||
// is only safe when Repo.ReviewEnabled is false; StageReview parks
|
||||
// rather than silently skipping if it is enabled with no Reviewer
|
||||
// wired.
|
||||
Reviewer Reviewer
|
||||
// PRSource fetches a pull request under review and its head snapshot for
|
||||
// the pr-review workflow (see prreview_stages.go). Nil is only safe for a
|
||||
// workflow that never runs pr-review's stages: they fail the run with a
|
||||
// named error rather than silently skipping, the same rule Reviewer and
|
||||
// Calls follow.
|
||||
PRSource PRSource
|
||||
// prReview is the pr-review workflow's cross-stage scratch state (see
|
||||
// prreview_stages.go). It is unexported, following the pattern set by
|
||||
// the feasibility workflow's decision field: pipeline-private state has
|
||||
// no business being read or set outside its own stages.
|
||||
prReview *prReviewState
|
||||
Bus *events.Bus // nil-safe via Emit
|
||||
Log *slog.Logger
|
||||
// CustomStages is retained for source compatibility only. Repository Go
|
||||
@@ -161,20 +178,6 @@ type TaskContext struct {
|
||||
// reported prompt-token sum was cache hits (billed at a steep discount)
|
||||
// rather than fresh, full-price tokens; see formatTokenUsage.
|
||||
RunUsage agentexec.Usage
|
||||
// ReviewReport is the adversarial self-review stage's result, stashed
|
||||
// by StageReview so StageOpenPR can render a findings section on the PR
|
||||
// body (h019.6). Zero value means the review did not run (disabled or
|
||||
// skipped); ReviewReport.Ran() is the test for "render the section".
|
||||
ReviewReport ReviewReport
|
||||
// ReviewedHeadSHA is the worktree head at the moment the pull request was
|
||||
// opened, which is the revision the review's line numbers were measured on
|
||||
// (StageReview runs against that same worktree and commits nothing, so no
|
||||
// revision intervenes). StagePostReviewComments threads it to the forge so
|
||||
// a comment whose line numbers describe a head the pull request has since
|
||||
// moved past is refused instead of attaching to unrelated code. Empty when
|
||||
// the revision could not be read -- posting then proceeds unverified rather
|
||||
// than dropping every finding.
|
||||
ReviewedHeadSHA string
|
||||
}
|
||||
|
||||
// Emit publishes an observability event stamped with the task's
|
||||
@@ -258,6 +261,14 @@ type Stage struct {
|
||||
type Workflow struct {
|
||||
Name string
|
||||
Stages []Stage
|
||||
// Interface declares this workflow's inputs, repository mode and agent
|
||||
// profile to whatever starts it -- a binding's CheckWorkflow reads it the
|
||||
// same way it reads a YAML-defined workflow's WorkflowInterface. Zero
|
||||
// value (no declared inputs) is the default every builtin had before
|
||||
// pr-review's triggers needed one; ShippedDefinitions only emits an
|
||||
// inputs: block when this is non-empty, so every other builtin's
|
||||
// generated YAML is byte-identical to before.
|
||||
Interface task.WorkflowInterface
|
||||
}
|
||||
|
||||
// Registry maps workflow names to definitions.
|
||||
|
||||
@@ -6,6 +6,7 @@ package forge
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"io"
|
||||
"log/slog"
|
||||
"strings"
|
||||
"time"
|
||||
@@ -96,6 +97,21 @@ type PullRequestReader interface {
|
||||
GetPullRequest(ctx context.Context, owner, repo string, number int) (PullRequest, error)
|
||||
}
|
||||
|
||||
// PullRequestDiffReader fetches a pull request's unified diff over the
|
||||
// forge's HTTP API. Implementations that cannot read PRs do not implement
|
||||
// it, matching PullRequestReader.
|
||||
type PullRequestDiffReader interface {
|
||||
GetPullRequestDiff(ctx context.Context, owner, repo string, number int) (string, error)
|
||||
}
|
||||
|
||||
// RepoArchiveReader fetches a gzipped tar archive of a repository at a ref
|
||||
// over the forge's HTTP API, with no git-level credential involved: the
|
||||
// caller reads the returned stream and closes it. Implementations that
|
||||
// cannot read repository contents do not implement it.
|
||||
type RepoArchiveReader interface {
|
||||
GetRepoArchive(ctx context.Context, owner, repo, ref string) (io.ReadCloser, error)
|
||||
}
|
||||
|
||||
// RepoForge is repository-level operations.
|
||||
type RepoForge interface {
|
||||
// AcceptInvitations auto-accepts pending repository invitations.
|
||||
|
||||
@@ -171,6 +171,27 @@ func (c *GiteaClient) GetPullRequest(ctx context.Context, owner, repo string, nu
|
||||
}, nil
|
||||
}
|
||||
|
||||
// GetPullRequestDiff fetches the pull request's unified diff over the Gitea
|
||||
// API, never a local git clone.
|
||||
func (c *GiteaClient) GetPullRequestDiff(_ context.Context, owner, repo string, number int) (string, error) {
|
||||
diff, _, err := c.cli.GetPullRequestDiff(owner, repo, int64(number), gitea.PullRequestDiffOptions{})
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("get pull request diff %s/%s#%d: %w", owner, repo, number, err)
|
||||
}
|
||||
return string(diff), nil
|
||||
}
|
||||
|
||||
// GetRepoArchive fetches a gzipped tar archive of the repository at ref
|
||||
// directly from Gitea's own API -- unlike GitHub, no separate unauthenticated
|
||||
// fetch is needed.
|
||||
func (c *GiteaClient) GetRepoArchive(_ context.Context, owner, repo, ref string) (io.ReadCloser, error) {
|
||||
body, _, err := c.cli.GetArchiveReader(owner, repo, ref, gitea.TarGZArchive)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("get repo archive %s/%s@%s: %w", owner, repo, ref, err)
|
||||
}
|
||||
return body, nil
|
||||
}
|
||||
|
||||
// ListReviews returns the reviews on a PR with ID > sinceID.
|
||||
func (c *GiteaClient) ListReviews(ctx context.Context, owner, repo string, number int, sinceID int64) ([]Review, error) {
|
||||
var out []Review
|
||||
|
||||
@@ -1,6 +1,7 @@
|
||||
package forge
|
||||
|
||||
import (
|
||||
"io"
|
||||
"net/http"
|
||||
"testing"
|
||||
)
|
||||
@@ -381,3 +382,38 @@ func TestLabelNamesGitea(t *testing.T) {
|
||||
t.Fatalf("labelNamesGitea(nil) = %v, want nil", names)
|
||||
}
|
||||
}
|
||||
|
||||
func TestGiteaGetPullRequestDiff(t *testing.T) {
|
||||
c, mux := newTestGiteaClient(t)
|
||||
mux.HandleFunc("GET /api/v1/repos/o/r/pulls/7.diff", func(w http.ResponseWriter, r *http.Request) {
|
||||
_, _ = w.Write([]byte("diff --git a/f.go b/f.go\n+added\n"))
|
||||
})
|
||||
|
||||
diff, err := c.GetPullRequestDiff(t.Context(), "o", "r", 7)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if diff != "diff --git a/f.go b/f.go\n+added\n" {
|
||||
t.Fatalf("diff = %q", diff)
|
||||
}
|
||||
}
|
||||
|
||||
func TestGiteaGetRepoArchive(t *testing.T) {
|
||||
c, mux := newTestGiteaClient(t)
|
||||
mux.HandleFunc("GET /api/v1/repos/o/r/archive/deadbeef.tar.gz", func(w http.ResponseWriter, r *http.Request) {
|
||||
_, _ = w.Write([]byte("fake-tar-bytes"))
|
||||
})
|
||||
|
||||
rc, err := c.GetRepoArchive(t.Context(), "o", "r", "deadbeef")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer rc.Close()
|
||||
body, err := io.ReadAll(rc)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if string(body) != "fake-tar-bytes" {
|
||||
t.Fatalf("archive body = %q", body)
|
||||
}
|
||||
}
|
||||
@@ -8,7 +8,9 @@ import (
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"log/slog"
|
||||
"net/http"
|
||||
"strings"
|
||||
"sync"
|
||||
|
||||
@@ -190,6 +192,42 @@ func (c *GitHubClient) GetPullRequest(ctx context.Context, owner, repo string, n
|
||||
}, nil
|
||||
}
|
||||
|
||||
// GetPullRequestDiff fetches the pull request's unified diff over the
|
||||
// GitHub API, never a local git clone.
|
||||
func (c *GitHubClient) GetPullRequestDiff(ctx context.Context, owner, repo string, number int) (string, error) {
|
||||
diff, _, err := c.gh.PullRequests.GetRaw(ctx, owner, repo, number, github.RawOptions{Type: github.Diff})
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("get pull request diff %s/%s#%d: %w", owner, repo, number, err)
|
||||
}
|
||||
return diff, nil
|
||||
}
|
||||
|
||||
// GetRepoArchive fetches a gzipped tar archive of the repository at ref.
|
||||
// GetArchiveLink only resolves the download's final location; the archive's
|
||||
// bytes still have to be fetched with a plain, unauthenticated GET (the URL
|
||||
// GitHub redirects to is a pre-signed storage link, not a GitHub API
|
||||
// endpoint), so no forge token is attached to this second request.
|
||||
func (c *GitHubClient) GetRepoArchive(ctx context.Context, owner, repo, ref string) (io.ReadCloser, error) {
|
||||
url, _, err := c.gh.Repositories.GetArchiveLink(ctx, owner, repo, github.Tarball, &github.RepositoryContentGetOptions{Ref: ref}, 5)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("resolve archive link %s/%s@%s: %w", owner, repo, ref, err)
|
||||
}
|
||||
req, err := http.NewRequestWithContext(ctx, http.MethodGet, url.String(), nil)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("build archive request %s/%s@%s: %w", owner, repo, ref, err)
|
||||
}
|
||||
resp, err := http.DefaultClient.Do(req)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("fetch archive %s/%s@%s: %w", owner, repo, ref, err)
|
||||
}
|
||||
if resp.StatusCode != http.StatusOK {
|
||||
body, _ := io.ReadAll(io.LimitReader(resp.Body, 2048))
|
||||
_ = resp.Body.Close()
|
||||
return nil, fmt.Errorf("fetch archive %s/%s@%s: status %s: %s", owner, repo, ref, resp.Status, body)
|
||||
}
|
||||
return resp.Body, nil
|
||||
}
|
||||
|
||||
// ListReviews returns the reviews on a PR with ID > sinceID.
|
||||
func (c *GitHubClient) ListReviews(ctx context.Context, owner, repo string, number int, sinceID int64) ([]Review, error) {
|
||||
var out []Review
|
||||
|
||||
@@ -2,6 +2,7 @@ package forge
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"io"
|
||||
"net/http"
|
||||
"testing"
|
||||
)
|
||||
@@ -382,3 +383,65 @@ func TestAcceptInvitationsListError(t *testing.T) {
|
||||
t.Fatal("expected error")
|
||||
}
|
||||
}
|
||||
|
||||
func TestGetPullRequestDiff(t *testing.T) {
|
||||
c, mux := newTestClient(t)
|
||||
mux.HandleFunc("GET /repos/o/r/pulls/7", func(w http.ResponseWriter, r *http.Request) {
|
||||
if got := r.Header.Get("Accept"); got == "" {
|
||||
t.Errorf("Accept header not set for a raw diff request")
|
||||
}
|
||||
_, _ = w.Write([]byte("diff --git a/f.go b/f.go\n+added\n"))
|
||||
})
|
||||
|
||||
diff, err := c.GetPullRequestDiff(t.Context(), "o", "r", 7)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if diff != "diff --git a/f.go b/f.go\n+added\n" {
|
||||
t.Fatalf("diff = %q", diff)
|
||||
}
|
||||
}
|
||||
|
||||
func TestGetRepoArchive(t *testing.T) {
|
||||
c, mux := newTestClient(t)
|
||||
mux.HandleFunc("GET /repos/o/r/tarball/deadbeef", func(w http.ResponseWriter, r *http.Request) {
|
||||
w.Header().Set("Location", "http://"+r.Host+"/archive-blob/deadbeef.tar.gz")
|
||||
w.WriteHeader(http.StatusFound)
|
||||
})
|
||||
mux.HandleFunc("GET /archive-blob/deadbeef.tar.gz", func(w http.ResponseWriter, r *http.Request) {
|
||||
// Real GitHub archive links are unauthenticated storage URLs; assert
|
||||
// no forge token leaks onto this second request.
|
||||
if got := r.Header.Get("Authorization"); got != "" {
|
||||
t.Errorf("Authorization header present on the archive blob request: %q", got)
|
||||
}
|
||||
_, _ = w.Write([]byte("fake-tar-bytes"))
|
||||
})
|
||||
|
||||
rc, err := c.GetRepoArchive(t.Context(), "o", "r", "deadbeef")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer rc.Close()
|
||||
body, err := io.ReadAll(rc)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if string(body) != "fake-tar-bytes" {
|
||||
t.Fatalf("archive body = %q", body)
|
||||
}
|
||||
}
|
||||
|
||||
func TestGetRepoArchiveNonOKStatus(t *testing.T) {
|
||||
c, mux := newTestClient(t)
|
||||
mux.HandleFunc("GET /repos/o/r/tarball/badref", func(w http.ResponseWriter, r *http.Request) {
|
||||
w.Header().Set("Location", "http://"+r.Host+"/archive-blob/badref.tar.gz")
|
||||
w.WriteHeader(http.StatusFound)
|
||||
})
|
||||
mux.HandleFunc("GET /archive-blob/badref.tar.gz", func(w http.ResponseWriter, r *http.Request) {
|
||||
w.WriteHeader(http.StatusNotFound)
|
||||
})
|
||||
|
||||
if _, err := c.GetRepoArchive(t.Context(), "o", "r", "badref"); err == nil {
|
||||
t.Fatal("want an error for a non-200 archive blob response")
|
||||
}
|
||||
}
|
||||
@@ -1,105 +0,0 @@
|
||||
package gateway
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"strings"
|
||||
|
||||
"github.com/samcharles93/archie-core/internal/tools"
|
||||
)
|
||||
|
||||
// PRReviewResult is the structured outcome of an operator-triggered PR
|
||||
// review, returned to the channel that requested it. It mirrors
|
||||
// workflow.ReviewReport without importing the domain package: the gateway
|
||||
// states what it needs and the daemon adapts between the two (the
|
||||
// ChatTaskActor pattern). Status is "completed", "not_run", or "skipped";
|
||||
// a zero-finding "completed" result is a review that ran and cleared, and
|
||||
// is structurally distinct from "not_run".
|
||||
type PRReviewResult struct {
|
||||
Repo string `json:"repo"`
|
||||
Number int `json:"number"`
|
||||
Model string `json:"model,omitempty"`
|
||||
Status string `json:"status"`
|
||||
Summary string `json:"summary,omitempty"`
|
||||
// Reason explains why a not_run/skipped review did not execute.
|
||||
Reason string `json:"reason,omitempty"`
|
||||
Findings []PRReviewFinding `json:"findings"`
|
||||
// ArtifactURL is the editor deep link for the published review artifact,
|
||||
// empty when artifact publishing is disabled or failed.
|
||||
ArtifactURL string `json:"artifact_url,omitempty"`
|
||||
}
|
||||
|
||||
// PRReviewFinding is one defect from an operator-triggered review.
|
||||
type PRReviewFinding struct {
|
||||
File string `json:"file"`
|
||||
Line int `json:"line"`
|
||||
Defect string `json:"defect"`
|
||||
FailureScenario string `json:"failure_scenario,omitempty"`
|
||||
Verdict string `json:"verdict"`
|
||||
Level string `json:"level"`
|
||||
Category string `json:"category,omitempty"`
|
||||
Blocking bool `json:"blocking"`
|
||||
}
|
||||
|
||||
// ChatPRReviewer is the surface review_pr needs. The gateway states what it
|
||||
// needs and the daemon supplies an adapter over the forge, worktree manager
|
||||
// and reviewer runtime, so this package keeps its independence from
|
||||
// internal/forge, internal/worktree and internal/domain/workflow.
|
||||
//
|
||||
// identity follows the ChatTaskActor convention: nil denotes an
|
||||
// authenticated dashboard operator who may review any configured
|
||||
// repository; a non-nil identity is a chat identity scoped to its own
|
||||
// configured repositories.
|
||||
type ChatPRReviewer interface {
|
||||
ReviewPR(ctx context.Context, identity *string, repo string, number int) (PRReviewResult, error)
|
||||
}
|
||||
|
||||
// ReviewTools builds the review_pr chat tool. A nil reviewer omits the tool
|
||||
// rather than registering one that always fails, so a daemon without
|
||||
// operator review support advertises nothing.
|
||||
func ReviewTools(reviewer ChatPRReviewer, identity string) []tools.ToolEntry {
|
||||
if reviewer == nil {
|
||||
return nil
|
||||
}
|
||||
return []tools.ToolEntry{reviewPRTool(reviewer, identity)}
|
||||
}
|
||||
|
||||
func reviewPRTool(reviewer ChatPRReviewer, identity string) tools.ToolEntry {
|
||||
return tools.ToolEntry{
|
||||
Name: "review_pr",
|
||||
Toolset: "tasks",
|
||||
Description: "Run the adversarial reviewer against an existing pull request and return its structured findings. " +
|
||||
"Use it when the user asks to review a PR, re-check one after a push, or see what the reviewer found. " +
|
||||
"Only a confirmed error-level finding blocks; a clean result reports zero findings explicitly.",
|
||||
Classification: tools.ClassMutating,
|
||||
Schema: tools.JSONSchema{
|
||||
"type": "object",
|
||||
"properties": map[string]any{
|
||||
"repo": map[string]any{
|
||||
"type": "string",
|
||||
"description": "Repository of the PR as \"owner/name\". Must be a repository this identity is configured for.",
|
||||
},
|
||||
"pr_number": map[string]any{
|
||||
"type": "integer",
|
||||
"description": "Pull request number to review.",
|
||||
},
|
||||
},
|
||||
"required": []any{"repo", "pr_number"},
|
||||
},
|
||||
Handler: func(ctx context.Context, input map[string]any) (any, error) {
|
||||
repo := strings.TrimSpace(asString(input["repo"]))
|
||||
if repo == "" {
|
||||
return nil, fmt.Errorf("review_pr: repo is required")
|
||||
}
|
||||
number, ok := asInt64(input["pr_number"])
|
||||
if !ok || number <= 0 {
|
||||
return nil, fmt.Errorf("review_pr: pr_number must be a positive integer")
|
||||
}
|
||||
result, err := reviewer.ReviewPR(ctx, &identity, repo, int(number))
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("review_pr: %w", err)
|
||||
}
|
||||
return result, nil
|
||||
},
|
||||
}
|
||||
}
|
||||
@@ -1,105 +0,0 @@
|
||||
package gateway
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"testing"
|
||||
)
|
||||
|
||||
type fakePRReviewer struct {
|
||||
gotIdentity *string
|
||||
gotRepo string
|
||||
gotNumber int
|
||||
result PRReviewResult
|
||||
err error
|
||||
}
|
||||
|
||||
func (f *fakePRReviewer) ReviewPR(_ context.Context, identity *string, repo string, number int) (PRReviewResult, error) {
|
||||
f.gotIdentity = identity
|
||||
f.gotRepo = repo
|
||||
f.gotNumber = number
|
||||
return f.result, f.err
|
||||
}
|
||||
|
||||
func TestReviewToolOmittedWhenReviewerNil(t *testing.T) {
|
||||
if entries := ReviewTools(nil, "archie"); len(entries) != 0 {
|
||||
t.Fatalf("ReviewTools(nil) = %d entries, want 0 -- a tool that cannot run must not be advertised", len(entries))
|
||||
}
|
||||
}
|
||||
|
||||
func TestReviewToolAppearsWhenReviewerNonNil(t *testing.T) {
|
||||
entries := ReviewTools(&fakePRReviewer{}, "archie")
|
||||
if len(entries) != 1 {
|
||||
t.Fatalf("ReviewTools() = %d entries, want 1", len(entries))
|
||||
}
|
||||
if err := entries[0].Validate(); err != nil {
|
||||
t.Fatalf("entry.Validate() = %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestReviewPRToolBindsIdentityAndReturnsFindings(t *testing.T) {
|
||||
reviewer := &fakePRReviewer{
|
||||
result: PRReviewResult{
|
||||
Repo: "acme/widget", Number: 7, Model: "openai/gpt-5", Status: "completed",
|
||||
Findings: []PRReviewFinding{{File: "a.go", Line: 3, Defect: "nil deref", FailureScenario: "x", Verdict: "confirmed", Level: "error", Category: "nil-risk", Blocking: true}},
|
||||
},
|
||||
}
|
||||
entry := toolNamed(t, ReviewTools(reviewer, "archie"), "review_pr")
|
||||
|
||||
out, err := entry.Handler(context.Background(), map[string]any{
|
||||
"repo": "acme/widget", "pr_number": float64(7), "identity": "someone-else",
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
// The model supplied an identity; it must be ignored in favour of the
|
||||
// bound identity.
|
||||
if reviewer.gotIdentity == nil || *reviewer.gotIdentity != "archie" {
|
||||
t.Fatalf("identity = %v, want bound \"archie\"", reviewer.gotIdentity)
|
||||
}
|
||||
if reviewer.gotRepo != "acme/widget" || reviewer.gotNumber != 7 {
|
||||
t.Fatalf("ReviewPR called with (%q, %d), want (acme/widget, 7)", reviewer.gotRepo, reviewer.gotNumber)
|
||||
}
|
||||
|
||||
result, ok := out.(PRReviewResult)
|
||||
if !ok {
|
||||
t.Fatalf("handler returned %T, want PRReviewResult", out)
|
||||
}
|
||||
if result.Status != "completed" || len(result.Findings) != 1 || !result.Findings[0].Blocking {
|
||||
t.Fatalf("result = %+v, want completed with one blocking finding", result)
|
||||
}
|
||||
}
|
||||
|
||||
func TestReviewPRToolValidatesInput(t *testing.T) {
|
||||
reviewer := &fakePRReviewer{}
|
||||
entry := toolNamed(t, ReviewTools(reviewer, "archie"), "review_pr")
|
||||
|
||||
tests := []struct {
|
||||
name string
|
||||
input map[string]any
|
||||
}{
|
||||
{"missing repo", map[string]any{"pr_number": float64(7)}},
|
||||
{"missing number", map[string]any{"repo": "acme/widget"}},
|
||||
{"zero number", map[string]any{"repo": "acme/widget", "pr_number": float64(0)}},
|
||||
{"negative number", map[string]any{"repo": "acme/widget", "pr_number": float64(-1)}},
|
||||
}
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
if _, err := entry.Handler(context.Background(), tt.input); err == nil {
|
||||
t.Fatalf("handler error = nil, want validation error for %v", tt.input)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestReviewPRToolSurfacesReviewerError(t *testing.T) {
|
||||
reviewer := &fakePRReviewer{err: errors.New("review already in progress")}
|
||||
entry := toolNamed(t, ReviewTools(reviewer, "archie"), "review_pr")
|
||||
|
||||
if _, err := entry.Handler(context.Background(), map[string]any{
|
||||
"repo": "acme/widget", "pr_number": float64(7),
|
||||
}); err == nil {
|
||||
t.Fatal("handler error = nil, want the reviewer failure surfaced")
|
||||
}
|
||||
}
|
||||
@@ -229,8 +229,7 @@ func taskSpawnTool(creator TaskCreator, identity string) tools.ToolEntry {
|
||||
return tools.ToolEntry{
|
||||
Name: "task_spawn",
|
||||
Toolset: "tasks",
|
||||
Description: "Queue a new task for this instance to work on in its own worktree, " +
|
||||
"opening a pull request for review when it finishes. " +
|
||||
Description: "Queue a workflow task for this instance. The selected workflow determines its outcome. " +
|
||||
"Use it when the user asks for work to be carried out rather than answered in chat.",
|
||||
Classification: tools.ClassMutating,
|
||||
Schema: tools.JSONSchema{
|
||||
@@ -246,7 +245,11 @@ func taskSpawnTool(creator TaskCreator, identity string) tools.ToolEntry {
|
||||
},
|
||||
"workflow": map[string]any{
|
||||
"type": "string",
|
||||
"description": "Workflow to run (e.g. implement, tdd, feasibility). Omit to let the daemon route it.",
|
||||
"description": "Workflow to run. Omit to let the daemon route it.",
|
||||
},
|
||||
"inputs": map[string]any{
|
||||
"type": "object",
|
||||
"description": "Inputs declared by the selected workflow. The daemon validates their names, types and required values before execution.",
|
||||
},
|
||||
},
|
||||
"required": []any{"title"},
|
||||
@@ -257,10 +260,20 @@ func taskSpawnTool(creator TaskCreator, identity string) tools.ToolEntry {
|
||||
return nil, fmt.Errorf("task_spawn: title is required")
|
||||
}
|
||||
|
||||
var inputs map[string]any
|
||||
if value, supplied := input["inputs"]; supplied {
|
||||
var ok bool
|
||||
inputs, ok = value.(map[string]any)
|
||||
if !ok {
|
||||
return nil, fmt.Errorf("task_spawn: inputs must be an object")
|
||||
}
|
||||
}
|
||||
|
||||
id, err := creator.CreateTask(ctx, SpawnRequest{
|
||||
Title: title,
|
||||
Repo: strings.TrimSpace(asString(input["repo"])),
|
||||
Workflow: strings.TrimSpace(asString(input["workflow"])),
|
||||
Inputs: inputs,
|
||||
// The bound identity, never input["identity"]. CreateTask
|
||||
// enforces the repository allow-list for it.
|
||||
Identity: identity,
|
||||
|
||||
@@ -4,6 +4,7 @@ import (
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"reflect"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
@@ -187,12 +188,13 @@ func TestTaskListPropagatesStoreError(t *testing.T) {
|
||||
|
||||
func TestTaskSpawn(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
input map[string]any
|
||||
creator *fakeCreator
|
||||
wantErr bool
|
||||
wantTitle string
|
||||
wantRepo string
|
||||
name string
|
||||
input map[string]any
|
||||
creator *fakeCreator
|
||||
wantErr bool
|
||||
wantTitle string
|
||||
wantRepo string
|
||||
wantInputs map[string]any
|
||||
}{
|
||||
{
|
||||
name: "title only",
|
||||
@@ -207,6 +209,20 @@ func TestTaskSpawn(t *testing.T) {
|
||||
wantTitle: "port it",
|
||||
wantRepo: "acme/app",
|
||||
},
|
||||
{
|
||||
name: "workflow inputs are forwarded as given",
|
||||
input: map[string]any{"title": "review it", "repo": "acme/app", "workflow": "pr-review", "inputs": map[string]any{"pr_number": float64(7)}},
|
||||
creator: &fakeCreator{id: 46},
|
||||
wantTitle: "review it",
|
||||
wantRepo: "acme/app",
|
||||
wantInputs: map[string]any{"pr_number": float64(7)},
|
||||
},
|
||||
{
|
||||
name: "inputs that are not an object are refused",
|
||||
input: map[string]any{"title": "x", "workflow": "pr-review", "inputs": "pr_number=7"},
|
||||
creator: &fakeCreator{id: 47},
|
||||
wantErr: true,
|
||||
},
|
||||
{
|
||||
name: "missing title is refused",
|
||||
input: map[string]any{},
|
||||
@@ -246,6 +262,9 @@ func TestTaskSpawn(t *testing.T) {
|
||||
if tc.creator.got.Repo != tc.wantRepo {
|
||||
t.Errorf("repo = %q, want %q", tc.creator.got.Repo, tc.wantRepo)
|
||||
}
|
||||
if !reflect.DeepEqual(tc.creator.got.Inputs, tc.wantInputs) {
|
||||
t.Errorf("inputs = %#v, want %#v", tc.creator.got.Inputs, tc.wantInputs)
|
||||
}
|
||||
// Identity is bound, never taken from the model.
|
||||
if tc.creator.got.Identity != "archie" {
|
||||
t.Errorf("identity = %q, want the bound %q", tc.creator.got.Identity, "archie")
|
||||
|
||||
@@ -45,7 +45,7 @@ type taskProfile struct {
|
||||
// internal/domain/workflow; the daemon supplies an adapter closure over
|
||||
// the store.TaskStore method of the same name.
|
||||
type chatTaskWriter interface {
|
||||
EnqueueChatTask(ctx context.Context, owner, repo, title, body, workflow, identity string) (taskID int64, err error)
|
||||
EnqueueChatTask(ctx context.Context, owner, repo, title, body, workflow, identity string, inputs map[string]any) (taskID int64, err error)
|
||||
}
|
||||
|
||||
// NewStoreTaskCreator returns a TaskCreator that enqueues chat-spawned
|
||||
@@ -103,7 +103,7 @@ func (c *StoreTaskCreator) CreateTask(ctx context.Context, req SpawnRequest) (in
|
||||
if owner == "" || repo == "" {
|
||||
return 0, fmt.Errorf("no repo configured for chat-spawned tasks")
|
||||
}
|
||||
return c.store.EnqueueChatTask(ctx, owner, repo, req.Title, req.Body, req.Workflow, req.Identity)
|
||||
return c.store.EnqueueChatTask(ctx, owner, repo, req.Title, req.Body, req.Workflow, req.Identity, req.Inputs)
|
||||
}
|
||||
|
||||
func splitOwnerRepo(s string) (owner, repo string, ok bool) {
|
||||
|
||||
Loaded 100 of 129 files, more files were not shown because too many files have changed in this diff.
Show more
Reference in new issue
Block a user