feat(pr-review): benchmark live pipeline through generic workflow spawn

This commit is contained in:
sam committed 2026-09-29 00:35:34 +10:00
1 parent 081ee4a8ba
commit ed698fced1
23 files changed
+647 -232

No files matched your search

+1 -1
View File
@@ -154,7 +154,7 @@ tasks:
- bash tools/webhook-send.sh {{.CLI_ARGS}}
bench:prreview:
desc: Score PR review findings on the Martian benchmark (-- -findings DIR; --help for flags)
desc: Review or rescore the Martian benchmark (-- -review-model PROVIDER/MODEL -classification-model PROVIDER/MODEL; --help for flags)
cmds:
- go run ./cmd/archie-prbench {{.CLI_ARGS}}
+10 -3
View File
@@ -20,13 +20,20 @@ func main() { os.Exit(run()) }
func run() int {
var opts prbench.Options
var problems string
flag.StringVar(&opts.Config, "config", configuration.DefaultConfigPath(), "configuration file whose providers serve the judge model")
flag.StringVar(&opts.Fixtures, "findings", "", "directory of <problem id>.json findings to score")
flag.StringVar(&opts.Config, "config", configuration.DefaultConfigPath(), "configuration file whose providers serve the reviewer and judge")
flag.StringVar(&opts.Fixtures, "findings", "", "directory of <problem id>.json findings to rescore; omit to review live")
flag.StringVar(&opts.ReviewModel, "review-model", "", "provider/model for the review role in live runs")
flag.StringVar(&opts.ClassifyModel, "classification-model", "", "provider/model for the classification role in live runs")
flag.StringVar(&opts.JudgeModel, "judge-model", "anthropic/claude-sonnet-4.6", "provider/model of the independent judge")
flag.StringVar(&opts.Out, "out", "prbench-out", "directory for per-problem results and summary.json")
flag.StringVar(&opts.Out, "out", "bin/prbench-results", "directory for per-problem results and summary.json")
flag.StringVar(&opts.Resume, "resume", "", "reuse complete results from a previous run with the same models")
flag.StringVar(&problems, "problems", "", "comma-separated problem ids (default: all)")
flag.IntVar(&opts.Concurrency, "concurrency", 4, "problems judged at once")
flag.Parse()
opts.GitHubToken = os.Getenv("GH_TOKEN")
if opts.GitHubToken == "" {
opts.GitHubToken = os.Getenv("GITHUB_TOKEN")
}
if problems != "" {
opts.Problems = strings.Split(problems, ",")
}
+1 -1
View File
@@ -154,7 +154,7 @@
"url": "/dev/development/workflows/",
"title": "Workflows",
"section": "Development",
"body": "# Workflows\n\nAuthority: `docs/architecture/agent-system.md` for the lifecycle,\n`docs/prds/event-automation.md` for inputs, repository modes and profiles.\n\nWorkflow definitions are YAML stored as a control-plane resource. The State\nStore validates them and the agent compiles them, each against the step\nvocabulary it registers, so both must see the same step types.\n\n## Adding a step type\n\n1. Write the factory in `internal/domain/workflow` as a `StepType`. It decodes\n and validates its settings and returns an error for anything it cannot\n honour. The factory runs when a definition is saved and again when it is\n compiled, so a bad definition is refused on save.\n2. Register it through a provider in\n `internal/infrastructure/workflowsteps/workflowsteps.go`. That one list is\n what every process registers, which is how the validating and executing\n sides agree.\n3. An agent-driven step builds on `AgentStage`; do not start a second path to\n the agent runner.\n4. A step that needs no repository goes in `repoFreeStepTypes` in\n `definition.go`, or workflows declaring `repository: none` or `optional`\n cannot use it. A step that needs one may assume `tc.Dir` holds a worktree\n only when the workflow's repository is `required`.\n5. A step that ends the workflow sets `tc.Outcome`. A workflow that runs out of\n steps without an outcome is parked as a definition bug.\n6. Add a parse test (settings accepted and refused) and a run test with fake\n `Trees` and agent runner.\n\n## Changing the definition format\n\n- The fields a caller needs without the engine (inputs, repository, profile)\n live in `task.WorkflowInterface`, because the dashboard process must not\n import the workflow engine (`cmd/archie-ui/architecture_test.go`). Anything a\n binding or the dashboard checks goes there.\n- The YAML decoder rejects unknown keys. Existing stored definitions must keep\n parsing, so a new key is optional with a default that preserves today's\n behaviour.\n- Examples in `examples/workflows/` are parsed against the registered\n vocabulary by `TestExampleWorkflowsParse`. Add one for a new capability.\n"
"body": "# Workflows\n\nAuthority: `docs/architecture/agent-system.md` for the lifecycle,\n`docs/prds/event-automation.md` for inputs, repository modes and profiles.\n\nWorkflow definitions are YAML stored as a control-plane resource. The State\nStore validates them and the agent compiles them, each against the step\nvocabulary it registers, so both must see the same step types.\n\n## Adding a step type\n\n1. Write the factory in `internal/domain/workflow` as a `StepType`. It decodes\n and validates its settings and returns an error for anything it cannot\n honour. The factory runs when a definition is saved and again when it is\n compiled, so a bad definition is refused on save.\n2. Register it through a provider in\n `internal/infrastructure/workflowsteps/workflowsteps.go`. That one list is\n what every process registers, which is how the validating and executing\n sides agree.\n3. An agent-driven step builds on `AgentStage`; do not start a second path to\n the agent runner.\n4. A step that needs no repository goes in `repoFreeStepTypes` in\n `definition.go`, or workflows declaring `repository: none` or `optional`\n cannot use it. A step that needs one may assume `tc.Dir` holds a worktree\n only when the workflow's repository is `required`.\n5. A step that ends the workflow sets `tc.Outcome`. A workflow that runs out of\n steps without an outcome is parked as a definition bug.\n6. Add a parse test (settings accepted and refused) and a run test with fake\n `Trees` and agent runner.\n\n## Changing the definition format\n\n- The fields a caller needs without the engine (inputs, repository, profile)\n live in `task.WorkflowInterface`, because the dashboard process must not\n import the workflow engine (`cmd/archie-ui/architecture_test.go`). Anything a\n binding or the dashboard checks goes there.\n- The YAML decoder rejects unknown keys. Existing stored definitions must keep\n parsing, so a new key is optional with a default that preserves today's\n behaviour.\n- Examples in `examples/workflows/` are parsed against the registered\n vocabulary by `TestExampleWorkflowsParse`. Add one for a new capability.\n\n## Starting a workflow from chat\n\n`task_spawn` accepts `workflow` and an `inputs` object. The gateway forwards\ninputs through the task creation contract; it does not interpret individual\nworkflows. The daemon checks inputs against the pinned `WorkflowInterface`\nbefore acquiring a container and parks invalid requests with the reason.\n"
},
{
"url": "/dev/prds/01-project-management/",
+10
View File
@@ -11,6 +11,12 @@
"title": "Building your first playbook binding",
"section": "Guides",
"body": "# Building your first playbook binding\n\nThis walks through the whole no-code path the event-capture epic promises:\npoint an external app's webhook at archie, see the real payload it sends,\nmap the fields you care about, bind them to a workflow, and watch the next\nmatching event turn into a task -- no code, no redeploy, no restart. See\n`docs/architecture/bindings.md` for the model and threat model behind each\nstep below.\n\n## 1. Send a test event\n\nPoint whatever external app you want archie to react to at:\n\n```\nPOST http://\u003cyour-archied-host\u003e:8484/webhooks/capture/\u003csource\u003e\n```\n\n`\u003csource\u003e` is a name you choose for this sender (e.g. `sentry`, `github-ci`,\n`stripe`) -- it's just a URL path segment, not a preset list. Send one real\nevent however is easiest for a first test, e.g.:\n\n```bash\ncurl -X POST http://localhost:8484/webhooks/capture/sentry \\\n -H \"Content-Type: application/json\" \\\n -d '{\"issue\": {\"title\": \"NullPointerException in checkout\"}, \"level\": \"error\"}'\n```\n\nThis event isn't signed yet, so it's captured but marked **unauthenticated**\n-- visible for you to design against, but incapable of triggering anything\n(`docs/architecture/bindings.md`'s threat-model section explains why: an\nunauthenticated event can never match a binding, no matter how it looks).\n\n## 2. Inspect the captured payload\n\nOpen the dashboard's **Event inspector** (`/captures`). Your test event\nappears there with its real body -- this is schema-by-example: you're\nlooking at an actual payload from the actual sender, not guessing at a\nschema in advance.\n\n## 3. Map the fields you care about\n\nOpen **Field mappings** (`/mappings`) → **New mapping**. Pick your captured\nevent from the dropdown, then click through its payload tree on the right --\nevery key and value is clickable. Clicking a value binds it as a field\n(name, JSON path, and an inferred type you can change). Add whichever fields\nyour workflow needs (e.g. the issue title), mark the ones that must be\npresent as **Required**, hit **Preview** to confirm they resolve correctly\nagainst the real payload, then **Save**.\n\n## 4. Create the binding\n\nOpen **Playbook bindings** (`/bindings`) → **New binding**:\n\n- **Name** -- anything descriptive.\n- **Source** -- the same `\u003csource\u003e` segment from step 1 (e.g. `sentry`).\n- **Field mapping** -- the one you just saved.\n- **Workflow** -- which registered workflow should run on a match.\n- **Repo pin** (optional) -- leave both blank to use the single configured\n repo; set owner/repo explicitly if you run a multi-repo deployment.\n- **Shared secret** -- a real secret (16+ bytes), never `test1234`. The\n sender signs every request body with this via HMAC-SHA256 so archie can\n tell a genuine event from anyone who finds the URL.\n\nSave. The binding starts in **draft** -- nothing fires yet.\n\n## 5. Approve it\n\nA **draft** or freshly-**edited** binding needs an explicit human approval\nbefore it can ever match a live event -- this is deliberate, not a missing\nstep: an operator authoring a binding is a considered act, an event arriving\nis not, and nothing should self-arm (`docs/architecture/bindings.md`'s\nthreat-model section, point 2). Click **Approve**. Status becomes **armed**.\n\n## 6. Fire it for real\n\nSend the same kind of event again, this time **signed** with the binding's\nsecret:\n\n```bash\nBODY='{\"issue\": {\"title\": \"NullPointerException in checkout\"}, \"level\": \"error\"}'\nSECRET=\"the-secret-you-set-in-step-4\"\nSIG=$(printf '%s' \"$BODY\" | openssl dgst -sha256 -hmac \"$SECRET\" | sed 's/^.* //')\n\ncurl -X POST http://localhost:8484/webhooks/capture/sentry \\\n -H \"Content-Type: application/json\" \\\n -H \"X-Hub-Signature-256: sha256=$SIG\" \\\n -d \"$BODY\"\n```\n\nThis event is captured as **authenticated**, matched against your armed\nbinding, and dispatched into a new task through the exact same\ngate/worktree/sandbox pipeline every other task goes through -- see\n`docs/architecture/messaging-and-work-intake.md`'s \"Intake provenance\". Check\nthe **Tasks** page; a new task should appear, its body built from the fields\nyour mapping resolved.\n\n## Editing a live binding\n\nAny edit to an armed binding -- even a typo fix -- drops it back to\n**pending_approval** automatically. This can't be bypassed by editing; it\nhas to be re-approved. When editing, leave the **Shared secret** field blank\nto keep the existing secret; only fill it in if you're deliberately\nrotating it.\n"
},
{
"url": "/docs/guides/pr-review-benchmark/",
"title": "PR review benchmark",
"section": "Guides",
"body": "# PR review benchmark\n\nRun `scripts/pr-review-benchmark.sh` with `PRBENCH_REVIEW_MODEL` and\n`PRBENCH_CLASSIFICATION_MODEL` set to configured `provider/model` references.\nThe judge defaults to `anthropic/claude-sonnet-4.6`; set\n`PRBENCH_JUDGE_MODEL` to another configured model when needed. The script uses\n`~/.config/archie/config.toml` for providers and `GH_TOKEN` or `GITHUB_TOKEN`\nfor read access to the benchmark pull requests. It posts no forge review.\n\nThe default run reviews the 38 runnable Martian pull requests once each at\n`deep`, judges the comments against 102 golden comments, and writes\n`bin/prbench-results/run.json`, `summary.json`, and one JSON result per pull\nrequest. That output directory is ignored by git. A completed full run fails\nwhen any problem fails or golden micro-recall is below 0.70. Golden-only\nprecision is reported alongside recall. Set `PRBENCH_OUT` to keep separate\nruns; `-problems ID` limits a smoke run to one pull request. Completed results\nare saved as each PR finishes. Pass `-resume DIR` to reuse completed results\nfrom a run with the same models; each reused problem must match the current\nbenchmark entry, and failed problems are rerun.\nA smoke run alone is not the acceptance run.\n\nThe benchmark passes only each problem's ID and pull request URL to the\nreviewer. The judge receives the golden comments after the review has ended.\nThe runner uses the production decision phases through the merge gate,\nincluding scoring and the 25-comment cap, then stops before polish and forge\nposting. A reviewer that fails to finish makes that problem fail. The results\nrecord budget-skipped phases and the selected models. Model substitutions do\nnot satisfy the PRD's named-model criterion even if recall clears the threshold.\n"
}
],
"nav": [
@@ -24,6 +30,10 @@
{
"title": "Building your first playbook binding",
"url": "/docs/guides/first-playbook/"
},
{
"title": "PR review benchmark",
"url": "/docs/guides/pr-review-benchmark/"
}
]
}
+7
View File
@@ -39,3 +39,10 @@ vocabulary it registers, so both must see the same step types.
behaviour.
- Examples in `examples/workflows/` are parsed against the registered
vocabulary by `TestExampleWorkflowsParse`. Add one for a new capability.
## Starting a workflow from chat
`task_spawn` accepts `workflow` and an `inputs` object. The gateway forwards
inputs through the task creation contract; it does not interpret individual
workflows. The daemon checks inputs against the pinned `WorkflowInterface`
before acquiring a container and parks invalid requests with the reason.
+28
View File
@@ -0,0 +1,28 @@
# PR review benchmark
Run `scripts/pr-review-benchmark.sh` with `PRBENCH_REVIEW_MODEL` and
`PRBENCH_CLASSIFICATION_MODEL` set to configured `provider/model` references.
The judge defaults to `anthropic/claude-sonnet-4.6`; set
`PRBENCH_JUDGE_MODEL` to another configured model when needed. The script uses
`~/.config/archie/config.toml` for providers and `GH_TOKEN` or `GITHUB_TOKEN`
for read access to the benchmark pull requests. It posts no forge review.
The default run reviews the 38 runnable Martian pull requests once each at
`deep`, judges the comments against 102 golden comments, and writes
`bin/prbench-results/run.json`, `summary.json`, and one JSON result per pull
request. That output directory is ignored by git. A completed full run fails
when any problem fails or golden micro-recall is below 0.70. Golden-only
precision is reported alongside recall. Set `PRBENCH_OUT` to keep separate
runs; `-problems ID` limits a smoke run to one pull request. Completed results
are saved as each PR finishes. Pass `-resume DIR` to reuse completed results
from a run with the same models; each reused problem must match the current
benchmark entry, and failed problems are rerun.
A smoke run alone is not the acceptance run.
The benchmark passes only each problem's ID and pull request URL to the
reviewer. The judge receives the golden comments after the review has ended.
The runner uses the production decision phases through the merge gate,
including scoring and the 25-comment cap, then stops before polish and forge
posting. A reviewer that fails to finish makes that problem fail. The results
record budget-skipped phases and the selected models. Model substitutions do
not satisfy the PRD's named-model criterion even if recall clears the threshold.
+106
View File
@@ -0,0 +1,106 @@
package prbench
import (
"context"
"fmt"
"log/slog"
"net/url"
"strconv"
"strings"
"sync/atomic"
"github.com/samcharles93/ai-sdk/runtime"
"github.com/samcharles93/archie-core/internal/agentexec"
"github.com/samcharles93/archie-core/internal/config"
"github.com/samcharles93/archie-core/internal/domain/workflow"
"github.com/samcharles93/archie-core/internal/domain/workflow/prreview"
"github.com/samcharles93/archie-core/internal/domain/workflow/prreview/bench"
"github.com/samcharles93/archie-core/internal/forge"
"github.com/samcharles93/archie-core/internal/infrastructure/prsource"
)
func newBenchmarkReviewer(opts Options, cfg config.Config, rt *runtime.Runtime) (bench.Reviewer, error) {
if opts.Fixtures != "" {
return fixtureReviewer{dir: opts.Fixtures}, nil
}
if opts.ReviewModel == "" || opts.ClassifyModel == "" {
return nil, fmt.Errorf("live review requires both -review-model and -classification-model")
}
client, err := forge.NewGitHub(opts.GitHubToken, "https://github.com", slog.Default())
if err != nil {
return nil, err
}
prReader, prOK := client.(forge.PullRequestReader)
diffReader, diffOK := client.(forge.PullRequestDiffReader)
archiveReader, archiveOK := client.(forge.RepoArchiveReader)
if !prOK || !diffOK || !archiveOK {
return nil, fmt.Errorf("GitHub client cannot read pull request snapshots")
}
cfg.Models = map[string]string{"review": opts.ReviewModel, "classification": opts.ClassifyModel}
cfg.Review.ApproveBeforePost = false
cfg.Review.PrecisionGate = false
return &liveReviewer{
cfg: cfg, agent: agentexec.NewLoopRunner(rt, slog.Default()),
source: prsource.New(forgeSource{PullRequestReader: prReader, PullRequestDiffReader: diffReader, RepoArchiveReader: archiveReader}),
}, nil
}
type liveReviewer struct {
cfg config.Config
agent agentexec.Runner
source workflow.PRSource
nextID atomic.Int64
}
func (r *liveReviewer) Review(ctx context.Context, p bench.Problem) (bench.Review, error) {
owner, repo, number, err := parsePRURL(p.PRURL)
if err != nil {
return bench.Review{}, err
}
slog.Info("benchmark review started", "problem", p.ID)
tc := &workflow.TaskContext{
Task: &workflow.Task{ID: r.nextID.Add(1), Attempt: 1, Owner: owner, Repo: repo, PRNumber: number, Workflow: "pr-review", Inputs: map[string]any{"depth": "deep"}},
Cfg: r.cfg, Agent: r.agent, PRSource: r.source, Log: slog.Default(),
}
decision, err := workflow.ReviewPullRequest(ctx, tc)
if err != nil {
return bench.Review{}, err
}
// Non-nil empty comments means a completed pipeline found nothing.
if decision.Comments == nil {
decision.Comments = []prreview.ScoredFinding{}
}
slog.Info("benchmark review completed", "problem", p.ID, "comments", len(decision.Comments), "skipped_phases", decision.SkippedPhases)
return bench.Review{Comments: decision.Comments, SkippedPhases: decision.SkippedPhases}, nil
}
func parsePRURL(raw string) (string, string, int, error) {
u, err := url.Parse(raw)
if err != nil {
return "", "", 0, fmt.Errorf("invalid pull request URL: %w", err)
}
parts := strings.Split(strings.Trim(u.Path, "/"), "/")
if u.Scheme != "https" || u.Host != "github.com" || len(parts) != 4 || parts[2] != "pull" || u.RawQuery != "" || u.Fragment != "" || u.User != nil {
return "", "", 0, fmt.Errorf("benchmark requires a GitHub pull request URL")
}
n, err := strconv.Atoi(parts[3])
if err != nil || n <= 0 || parts[0] == "" || parts[1] == "" {
return "", "", 0, fmt.Errorf("invalid pull request URL")
}
return parts[0], parts[1], n, nil
}
type forgeSource struct {
forge.PullRequestReader
forge.PullRequestDiffReader
forge.RepoArchiveReader
}
func (s forgeSource) GetPullRequest(ctx context.Context, owner, repo string, number int) (prsource.PullRequest, error) {
p, err := s.PullRequestReader.GetPullRequest(ctx, owner, repo, number)
if err != nil {
return prsource.PullRequest{}, err
}
return prsource.PullRequest{Title: p.Title, Body: p.Body, HeadSHA: p.HeadSHA}, nil
}
+136 -15
View File
@@ -6,13 +6,14 @@ import (
"context"
_ "embed"
"encoding/json"
"errors"
"fmt"
"io"
"log/slog"
"os"
"path/filepath"
"reflect"
"slices"
"sync"
"github.com/samcharles93/archie-core/internal/agentexec"
"github.com/samcharles93/archie-core/internal/app/archied"
@@ -48,20 +49,21 @@ func Problems(ids []string) ([]bench.Problem, error) {
// Options configures one benchmark run.
type Options struct {
Config string
Fixtures string
JudgeModel string
Out string
Problems []string
Concurrency int
Config string
Fixtures string
JudgeModel string
ReviewModel string
ClassifyModel string
GitHubToken string
Resume string
Out string
Problems []string
Concurrency int
}
// Run reviews and judges the selected problems, writes one result file per
// problem and a summary to Options.Out, and prints the summary to w.
func Run(ctx context.Context, opts Options, w io.Writer) (bench.Summary, error) {
if opts.Fixtures == "" {
return bench.Summary{}, errors.New("no reviewer: pass a findings fixture directory")
}
problems, err := Problems(opts.Problems)
if err != nil {
return bench.Summary{}, err
@@ -78,10 +80,17 @@ func Run(ctx context.Context, opts Options, w io.Writer) (bench.Summary, error)
return bench.Summary{}, fmt.Errorf("%s configures no providers for the judge", opts.Config)
}
judge := &modelJudge{runtime: rt, modelRef: opts.JudgeModel}
reviewer, err := newBenchmarkReviewer(opts, doc.Config, rt)
if err != nil {
return bench.Summary{}, err
}
results := bench.Run(ctx, problems, fixtureReviewer{dir: opts.Fixtures}, judge, opts.Concurrency)
results, err := runReviews(ctx, problems, reviewer, judge, opts)
if err != nil {
return bench.Summary{}, err
}
summary := bench.Summarize(results)
if err := writeResults(opts.Out, results, summary); err != nil {
if err := writeResults(opts.Out, results, summary, opts); err != nil {
return summary, err
}
for _, r := range results {
@@ -92,10 +101,51 @@ func Run(ctx context.Context, opts Options, w io.Writer) (bench.Summary, error)
fmt.Fprintf(w, "problems %d (failed %d) recall %.3f (%d/%d) golden-only precision %.3f (%d/%d)\n",
summary.Problems, summary.Failed, summary.Recall, summary.Hits, summary.Goldens,
summary.Precision, summary.Matched, summary.Comments)
if summary.Failed != 0 {
return summary, fmt.Errorf("%d benchmark problem(s) failed; results are incomplete", summary.Failed)
}
if opts.Fixtures == "" && len(opts.Problems) == 0 && summary.Recall < 0.70 {
return summary, fmt.Errorf("golden micro-recall %.3f is below 0.70", summary.Recall)
}
return summary, nil
}
func writeResults(dir string, results []bench.Result, summary bench.Summary) error {
func runReviews(ctx context.Context, problems []bench.Problem, reviewer bench.Reviewer, judge bench.Judge, opts Options) ([]bench.Result, error) {
results, pending, err := resumeResults(problems, opts)
if err != nil {
return nil, err
}
if err := os.MkdirAll(filepath.Join(opts.Out, "results"), 0o755); err != nil {
return nil, err
}
if err := writeJSON(filepath.Join(opts.Out, "run.json"), metadata(opts)); err != nil {
return nil, err
}
var saveMu sync.Mutex
var saveErr error
newResults := bench.Run(ctx, pending, reviewer, judge, opts.Concurrency, func(r bench.Result) {
if err := writeJSON(filepath.Join(opts.Out, "results", r.Problem.ID+".json"), r); err != nil {
saveMu.Lock()
saveErr = err
saveMu.Unlock()
}
})
if saveErr != nil {
return nil, fmt.Errorf("save benchmark result: %w", saveErr)
}
byID := make(map[string]bench.Result, len(newResults))
for _, r := range newResults {
byID[r.Problem.ID] = r
}
for i, r := range results {
if r.Problem.ID == "" {
results[i] = byID[problems[i].ID]
}
}
return results, nil
}
func writeResults(dir string, results []bench.Result, summary bench.Summary, opts Options) error {
if err := os.MkdirAll(filepath.Join(dir, "results"), 0o755); err != nil {
return err
}
@@ -104,7 +154,66 @@ func writeResults(dir string, results []bench.Result, summary bench.Summary) err
return err
}
}
return writeJSON(filepath.Join(dir, "summary.json"), summary)
if err := writeJSON(filepath.Join(dir, "summary.json"), summary); err != nil {
return err
}
return writeJSON(filepath.Join(dir, "run.json"), metadata(opts))
}
type runMetadata struct {
ReviewModel string `json:"review_model"`
ClassificationModel string `json:"classification_model"`
JudgeModel string `json:"judge_model"`
Live bool `json:"live"`
}
func metadata(opts Options) runMetadata {
return runMetadata{opts.ReviewModel, opts.ClassifyModel, opts.JudgeModel, opts.Fixtures == ""}
}
func resumeResults(problems []bench.Problem, opts Options) ([]bench.Result, []bench.Problem, error) {
results := make([]bench.Result, len(problems))
if opts.Resume == "" {
return results, problems, nil
}
data, err := os.ReadFile(filepath.Join(opts.Resume, "run.json"))
if err != nil {
return nil, nil, fmt.Errorf("read benchmark resume metadata: %w", err)
}
var previous runMetadata
if err := json.Unmarshal(data, &previous); err != nil {
return nil, nil, fmt.Errorf("decode benchmark resume metadata: %w", err)
}
if previous != metadata(opts) {
return nil, nil, fmt.Errorf("benchmark resume models or mode differ from this run")
}
var pending []bench.Problem
for i, p := range problems {
data, err := os.ReadFile(filepath.Join(opts.Resume, "results", p.ID+".json"))
if os.IsNotExist(err) {
pending = append(pending, p)
continue
}
if err != nil {
return nil, nil, fmt.Errorf("read benchmark result %s: %w", p.ID, err)
}
var r bench.Result
if err := json.Unmarshal(data, &r); err != nil {
return nil, nil, fmt.Errorf("decode benchmark result %s: %w", p.ID, err)
}
if !reflect.DeepEqual(r.Problem, p) {
return nil, nil, fmt.Errorf("benchmark result %s belongs to a different problem", p.ID)
}
if r.Err != "" {
pending = append(pending, p)
continue
}
if len(r.Verdicts) != len(p.Goldens) {
return nil, nil, fmt.Errorf("benchmark result %s is incomplete or belongs to a different problem", p.ID)
}
results[i] = r
}
return results, pending, nil
}
func writeJSON(path string, v any) error {
@@ -112,5 +221,17 @@ func writeJSON(path string, v any) error {
if err != nil {
return err
}
return os.WriteFile(path, append(data, '\n'), 0o644)
f, err := os.CreateTemp(filepath.Dir(path), ".prbench-*")
if err != nil {
return err
}
defer os.Remove(f.Name())
if _, err := f.Write(append(data, '\n')); err != nil {
f.Close()
return err
}
if err := f.Close(); err != nil {
return err
}
return os.Rename(f.Name(), path)
}
+18
View File
@@ -63,3 +63,21 @@ func TestFixtureReviewer(t *testing.T) {
})
}
}
func TestResumeResultsOnlyReusesCompleteMatchingRun(t *testing.T) {
dir := t.TempDir()
opts := Options{ReviewModel: "openai/review", ClassifyModel: "openai/classify", JudgeModel: "openai/judge", Resume: dir}
p := bench.Problem{ID: "first", PRURL: "https://github.com/acme/repo/pull/1", Goldens: []bench.Golden{{Comment: "defect"}}}
other := bench.Problem{ID: "second", PRURL: "https://github.com/acme/repo/pull/2", Goldens: []bench.Golden{{Comment: "other"}}}
if err := writeResults(dir, []bench.Result{{Problem: p, Verdicts: []bench.Verdict{{Golden: 0, Comment: bench.NoComment}}}}, bench.Summary{}, opts); err != nil {
t.Fatal(err)
}
results, pending, err := resumeResults([]bench.Problem{p, other}, opts)
if err != nil || len(pending) != 1 || pending[0].ID != other.ID || results[0].Problem.ID != p.ID {
t.Fatalf("results = %+v, pending = %+v, err = %v", results, pending, err)
}
opts.JudgeModel = "openai/different"
if _, _, err := resumeResults([]bench.Problem{p}, opts); err == nil {
t.Fatal("mixed model results were accepted")
}
}
+6
View File
@@ -1784,6 +1784,12 @@ func (d *Daemon) pinTaskProfile(ctx context.Context, task *workflow.Task) (confi
return config.AgentProfile{}, false
}
iface, err := workflowtask.ParseWorkflowInterface(task.WorkflowDefinitionYAML)
if err == nil {
// Every producer meets the declared interface here, including the
// ones that cannot read definitions when they enqueue (chat and
// dashboard spawns go through the Gateway, which never does).
err = iface.CheckInputs(task.Inputs)
}
if err == nil {
var profile config.AgentProfile
if profile, err = d.configFor(task).Containers.Profile(iface.Profile); err == nil {
+46
View File
@@ -2412,6 +2412,52 @@ func TestPinTaskProfileRefusesCapturesTheHarnessCannotServe(t *testing.T) {
}
}
// TestPinTaskProfileRefusesInputsTheWorkflowDoesNotAccept holds every
// producer to the workflow's declared interface at the one point each task
// meets its definition. Bindings check inputs before they enqueue; a chat or
// dashboard spawn cannot, because the Gateway never reads workflow
// definitions, so without this a spawn missing a required input would run
// and fail deep inside the workflow instead of parking with the reason.
func TestPinTaskProfileRefusesInputsTheWorkflowDoesNotAccept(t *testing.T) {
d, s, _ := daemonWithNATS(t)
d.WorkflowDefinitions = &workflowDefinitionsStub{collection: workflow.WorkflowDefinitionCollection{Definitions: []workflow.WorkflowDefinitionEntry{
{ID: "investigate", YAML: investigateYAML},
}}, version: 1}
ctx := context.Background()
for _, tt := range []struct {
name string
inputs map[string]any
ok bool
reason string
}{
{"declared inputs run", map[string]any{"src_ip": "10.0.0.1"}, true, ""},
{"missing required input parks", nil, false, `input "src_ip" is required`},
{"undeclared input parks", map[string]any{"src_ip": "10.0.0.1", "extra": "x"}, false, `input "extra" is not declared`},
{"mistyped input parks", map[string]any{"src_ip": 7}, false, `input "src_ip" is number, want string`},
} {
t.Run(tt.name, func(t *testing.T) {
if _, err := s.EnqueueChatTask(ctx, "acme", "widget", tt.name, "", "investigate", "", tt.inputs); err != nil {
t.Fatal(err)
}
task, err := s.ClaimNext(ctx)
if err != nil || task == nil {
t.Fatalf("claim: (%v, %v)", task, err)
}
if _, ok := d.pinTaskProfile(ctx, task); ok != tt.ok {
t.Fatalf("pinTaskProfile ok = %v, want %v", ok, tt.ok)
}
if tt.ok {
return
}
got, err := s.TaskByID(ctx, task.ID)
if err != nil || got.Status != workflow.StatusParked || !strings.Contains(got.ParkReason, tt.reason) {
t.Fatalf("task = %+v, %v; want parked naming %q", got, err, tt.reason)
}
})
}
}
// TestWorkflowCallGivesACalleeItsOwnProfile is the harness PRD's "a stage may
// name a different profile" (docs/prds/external-agent-harness.md, Selection):
// there is no mid-container runner switch, and none is built here. A stage
@@ -29,11 +29,16 @@ type Problem struct {
// Review is what a reviewer returns for one problem: its findings and the run
// context that scores them.
type Review struct {
// Comments is the pipeline's final selection. Non-nil bypasses fixture scoring.
Comments []prreview.ScoredFinding
SkippedPhases []string
Findings []prreview.Finding
Inputs prreview.ScoreInputs
}
// Reviewer reviews one problem's pull request blind: it never sees the goldens.
// Reviewer reviews one problem's pull request blind: Run passes only ID and
// PRURL, never the goldens.
type Reviewer interface {
Review(ctx context.Context, p Problem) (Review, error)
}
@@ -61,10 +66,11 @@ type Judge interface {
// Result is one problem's outcome. Err is set when the reviewer or the judge
// failed, and then the problem is not scored at all: a failure is not a miss.
type Result struct {
Problem Problem `json:"problem"`
Comments []prreview.ScoredFinding `json:"comments"`
Verdicts []Verdict `json:"verdicts"`
Err string `json:"error,omitempty"`
Problem Problem `json:"problem"`
Comments []prreview.ScoredFinding `json:"comments"`
Verdicts []Verdict `json:"verdicts"`
SkippedPhases []string `json:"skipped_phases,omitempty"`
Err string `json:"error,omitempty"`
}
// Hits is how many goldens a posted comment caught.
@@ -90,8 +96,9 @@ func (r Result) MatchedComments() int {
}
// Run reviews and judges every problem, at most concurrency at a time, and
// returns one result per problem in the order given.
func Run(ctx context.Context, problems []Problem, reviewer Reviewer, judge Judge, concurrency int) []Result {
// returns one result per problem in the order given. onResult, when supplied,
// runs as each problem finishes and may be called concurrently.
func Run(ctx context.Context, problems []Problem, reviewer Reviewer, judge Judge, concurrency int, onResult ...func(Result)) []Result {
results := make([]Result, len(problems))
slots := make(chan struct{}, max(concurrency, 1))
var wg sync.WaitGroup
@@ -99,7 +106,11 @@ func Run(ctx context.Context, problems []Problem, reviewer Reviewer, judge Judge
wg.Go(func() {
slots <- struct{}{}
defer func() { <-slots }()
results[i] = runOne(ctx, p, reviewer, judge)
result := runOne(ctx, p, reviewer, judge)
results[i] = result
if len(onResult) != 0 {
onResult[0](result)
}
})
}
wg.Wait()
@@ -108,12 +119,16 @@ func Run(ctx context.Context, problems []Problem, reviewer Reviewer, judge Judge
func runOne(ctx context.Context, p Problem, reviewer Reviewer, judge Judge) Result {
res := Result{Problem: p}
review, err := reviewer.Review(ctx, p)
review, err := reviewer.Review(ctx, Problem{ID: p.ID, PRURL: p.PRURL})
if err != nil {
res.Err = fmt.Sprintf("review: %v", err)
return res
}
res.Comments = prreview.CapInlineComments(prreview.Score(review.Findings, review.Inputs))
res.SkippedPhases = review.SkippedPhases
res.Comments = review.Comments
if res.Comments == nil {
res.Comments = prreview.CapInlineComments(prreview.Score(review.Findings, review.Inputs))
}
if len(res.Comments) == 0 {
res.Verdicts = misses(p)
return res
@@ -139,3 +139,32 @@ func TestSummarize(t *testing.T) {
t.Errorf("Summarize = %+v\nwant %+v", got, want)
}
}
type blindReviewer struct{ t *testing.T }
func (r blindReviewer) Review(_ context.Context, p Problem) (Review, error) {
if len(p.Goldens) != 0 {
r.t.Error("reviewer received golden answers")
}
if p.ID != "blind" || p.PRURL != "https://example.org/pr/1" {
r.t.Error("reviewer lost PR identity")
}
return Review{}, nil
}
func TestRunKeepsGoldensFromReviewer(t *testing.T) {
p := Problem{ID: "blind", PRURL: "https://example.org/pr/1", Goldens: []Golden{{Comment: "secret answer"}}}
r := Run(t.Context(), []Problem{p}, blindReviewer{t}, &fakeJudge{}, 1)
if len(r[0].Problem.Goldens) != 1 || len(r[0].Verdicts) != 1 {
t.Fatal("scoring lost goldens")
}
}
func TestRunUsesPipelineCommentsWithoutRescoring(t *testing.T) {
p := problem("pipeline", 1)
comments := []prreview.ScoredFinding{{File: "final.go", LineStart: 4, Severity: prreview.SeverityImportant, Confidence: 0.1, Score: 7}}
r := Run(t.Context(), []Problem{p}, fakeReviewer{p.ID: {Comments: comments, SkippedPhases: []string{"merge-gate"}}}, &fakeJudge{verdicts: map[string][]Verdict{p.ID: {{Golden: 0, Comment: 0}}}}, 1)
if r[0].Err != "" || len(r[0].Comments) != 1 || r[0].Comments[0].Score != 7 || len(r[0].SkippedPhases) != 1 || r[0].SkippedPhases[0] != "merge-gate" {
t.Fatalf("result = %+v", r[0])
}
}
@@ -0,0 +1,51 @@
package workflow
import (
"context"
"fmt"
"os"
"github.com/samcharles93/archie-core/internal/domain/workflow/prreview"
)
// ReviewPullRequest runs the same decision phases as PRReview through its
// merge gate, without phase 9 or any forge writes. The caller supplies a fresh
// context with operator approval disabled.
type PRReviewDecision struct {
Comments []prreview.ScoredFinding
SkippedPhases []string
}
func ReviewPullRequest(ctx context.Context, tc *TaskContext) (PRReviewDecision, error) {
if tc.Cfg.Review.ApproveBeforePost {
return PRReviewDecision{}, fmt.Errorf("headless review cannot wait for operator approval")
}
ctx, cancel := context.WithTimeout(ctx, prReviewTotalBudget.WallClock)
defer cancel()
defer func() {
if tc.prReview != nil && tc.prReview.snapshotDir != "" {
_ = os.RemoveAll(tc.prReview.snapshotDir)
}
}()
for _, stage := range prReviewDecisionStages() {
if err := ctx.Err(); err != nil {
return PRReviewDecision{}, err
}
if err := stage.Run(ctx, tc); err != nil {
return PRReviewDecision{}, fmt.Errorf("pr-review %s: %w", stage.Name, err)
}
if tc.Outcome.Status != "" {
return PRReviewDecision{}, fmt.Errorf("pr-review stopped: %s", tc.Outcome.Detail)
}
}
if err := ctx.Err(); err != nil {
return PRReviewDecision{}, err
}
if failed := tc.prReview.reviewerFailures.Load(); failed != 0 {
return PRReviewDecision{}, fmt.Errorf("pr-review: %d reviewer(s) did not finish", failed)
}
return PRReviewDecision{
Comments: tc.prReview.scored,
SkippedPhases: append([]string(nil), tc.prReview.skippedPhases...),
}, nil
}
+18 -6
View File
@@ -68,10 +68,11 @@ type prReviewState struct {
clusters []prreview.Cluster
highExposure []string
narrative string
dimensions []prreview.Dimension
findings []prreview.Finding
scored []prreview.ScoredFinding
narrative string
dimensions []prreview.Dimension
findings []prreview.Finding
scored []prreview.ScoredFinding
reviewerFailures atomic.Int64
// findingsMu guards findings during phase 6, where the coverage gate and
// consistency verification append to it from two goroutines running in
@@ -159,6 +160,7 @@ func PRReview() Workflow {
Interface: task.WorkflowInterface{
Inputs: map[string]task.InputSpec{
"pr_number": {Type: "number", Required: true},
"depth": {Type: "string"},
},
},
Stages: append(prReviewDecisionStages(), stagePROutput()),
@@ -212,6 +214,14 @@ func stagePRIntake() Stage {
if tc.Task.PRNumber == 0 && tc.Task.Workflow == "pr-review" {
return fmt.Errorf("pr-review: no pull request number (set directly or via the pr_number input)")
}
var explicit prreview.Depth
if value, supplied := tc.Task.Inputs["depth"]; supplied {
depth, ok := value.(string)
if !ok || (depth != "quick" && depth != "standard" && depth != "deep") {
return fmt.Errorf("pr-review: depth must be quick, standard or deep")
}
explicit = prreview.Depth(depth)
}
meta, err := tc.PRSource.Metadata(ctx, tc.Task.Owner, tc.Task.Repo, tc.Task.PRNumber)
if err != nil {
return fmt.Errorf("fetch pull request metadata: %w", err)
@@ -222,7 +232,7 @@ func stagePRIntake() Stage {
}
files := prreview.ParseDiff(diff)
stats := prreview.SummarizeFiles(files)
depth := prreview.ResolveDepth(stats.TotalAdditions+stats.TotalDeletions, "")
depth := prreview.ResolveDepth(stats.TotalAdditions+stats.TotalDeletions, explicit)
// tc.prReview must exist before scoreAIGenerated runs: every
// pr-review agent call, including this one, records its token spend
@@ -301,11 +311,11 @@ func stagePRAnatomy() Stage {
if err != nil {
return fmt.Errorf("create pull request snapshot directory: %w", err)
}
tc.prReview.snapshotDir = dir
headSHA, err := tc.PRSource.Snapshot(ctx, tc.Task.Owner, tc.Task.Repo, tc.Task.PRNumber, dir)
if err != nil {
return fmt.Errorf("snapshot pull request head: %w", err)
}
tc.prReview.snapshotDir = dir
tc.prReview.headSHA = headSHA
changed := make([]string, 0, len(tc.prReview.files))
@@ -498,10 +508,12 @@ func runReviewer(ctx context.Context, tc *TaskContext, dim prreview.Dimension) [
name := "reviewer-" + dim.Name
res, runErr := runReviewerAgent(ctx, tc, name, mission, []agentexec.CaptureTool{reportFindingsTool})
if runErr != nil || res.Status != agentexec.StatusPassed {
tc.prReview.reviewerFailures.Add(1)
return nil
}
findings, err := decodeReportedFindings(res.Captures["report_findings"], dim.Name)
if err != nil {
tc.prReview.reviewerFailures.Add(1)
return nil
}
return findings
@@ -547,3 +547,98 @@ index e69de29..4b825dc 100644
@@ -0,0 +1 @@
+package main
`
func TestStagePRIntakeHonoursExplicitDepth(t *testing.T) {
for _, tt := range []struct {
value any
want prreview.Depth
invalid bool
}{
{"deep", prreview.DepthDeep, false},
{"standard", prreview.DepthStandard, false},
{"quick", prreview.DepthQuick, false},
{"depe", "", true},
{7, "", true},
} {
t.Run(fmt.Sprint(tt.value), func(t *testing.T) {
tc := baseTaskContext(t)
tc.Task.Inputs = map[string]any{"depth": tt.value}
tc.PRSource = &fakePRSource{diff: smallDiff}
tc.Agent = &concurrentAgentRunner{byStage: func(string) (agentexec.Result, error) {
return captureResult("score_ai_generated", map[string]any{"confidence": 0}), nil
}}
err := stagePRIntake().Run(t.Context(), tc)
if tt.invalid {
if err == nil {
t.Fatal("invalid depth accepted")
}
return
}
if err != nil {
t.Fatal(err)
}
if tc.prReview.depth != tt.want {
t.Fatalf("depth = %s, want %s", tc.prReview.depth, tt.want)
}
})
}
}
func TestReviewPullRequestUsesPipelineWithoutPosting(t *testing.T) {
for _, mode := range []string{"complete", "lens fails", "reviewer fails", "budget skips merge"} {
t.Run(mode, func(t *testing.T) {
tc := baseTaskContext(t)
tc.Task.Inputs = map[string]any{"depth": "deep"}
tc.PRSource = &fakePRSource{diff: smallDiff, files: map[string]string{"a.go": "package a\n"}}
var reachedMerge atomic.Bool
tc.Agent = &concurrentAgentRunner{byStage: func(stage string) (agentexec.Result, error) {
switch {
case stage == "intake-ai-score":
return captureResult("score_ai_generated", map[string]any{"confidence": 0}), nil
case strings.HasPrefix(stage, "lens-"):
if mode == "lens fails" {
return agentexec.Result{}, fmt.Errorf("lens unavailable")
}
return captureResult("propose_dimensions", map[string]any{"dimensions": []map[string]any{{"name": "defect", "prompt": "review", "target_files": []string{"a.go"}, "priority": 1}}}), nil
case strings.HasPrefix(stage, "reviewer-"):
if mode == "reviewer fails" {
return agentexec.Result{}, fmt.Errorf("reviewer unavailable")
}
return captureResult("report_findings", map[string]any{"findings": []map[string]any{{"file": "a.go", "line_start": 1, "severity": "important", "title": "defect", "body": "broken", "evidence": "package a", "confidence": 0.9}}}), nil
case stage == "merge-gate":
reachedMerge.Store(true)
return captureResult("classify_blocking", map[string]any{"blocking": true}), nil
case stage == "consistency" && mode == "budget skips merge":
return agentexec.Result{Status: agentexec.StatusPassed, TokensUsed: 2_000_000}, nil
default:
return passedResult("ok"), nil
}
}}
decision, err := ReviewPullRequest(t.Context(), tc)
switch mode {
case "lens fails":
if err == nil || !strings.Contains(err.Error(), "lens unavailable") {
t.Fatalf("error = %v", err)
}
case "reviewer fails":
if err == nil || !strings.Contains(err.Error(), "reviewer(s) did not finish") {
t.Fatalf("error = %v", err)
}
case "budget skips merge":
if err != nil || len(decision.Comments) != 1 || reachedMerge.Load() || !slices.Contains(decision.SkippedPhases, "merge-gate") {
t.Fatalf("decision = %+v, merge = %v, error = %v", decision, reachedMerge.Load(), err)
}
default:
if err != nil || len(decision.Comments) != 1 || !reachedMerge.Load() || !decision.Comments[0].Blocking {
t.Fatalf("decision = %+v, merge = %v, error = %v", decision, reachedMerge.Load(), err)
}
}
if tc.prReview == nil || tc.prReview.depth != prreview.DepthDeep {
t.Fatal("pipeline did not use explicit deep depth")
}
if _, err := os.Stat(tc.prReview.snapshotDir); !os.IsNotExist(err) {
t.Fatalf("snapshot not cleaned: %v", err)
}
})
}
}
-85
View File
@@ -1,85 +0,0 @@
package gateway
import (
"context"
"fmt"
"strings"
"github.com/samcharles93/archie-core/internal/tools"
)
// ReviewTools builds the review_pr chat tool. A nil creator omits the tool
// rather than registering one that always fails, so a daemon without a task
// store advertises nothing.
//
// The tool queues a pr-review task; it does not review in the turn. The
// pipeline needs a sandbox, a prefetched snapshot and minutes of model calls,
// so the operator gets a task ID back and watches the task like any other
// (docs/prds/pr-review-agent.md, "Triggers").
func ReviewTools(creator TaskCreator, identity string) []tools.ToolEntry {
if creator == nil {
return nil
}
return []tools.ToolEntry{reviewPRTool(creator, identity)}
}
// prReviewWorkflow is the workflow a queued operator review runs. Its
// declared interface requires pr_number, so the tool that names it is also
// the only chat path that can satisfy it.
const prReviewWorkflow = "pr-review"
func reviewPRTool(creator TaskCreator, identity string) tools.ToolEntry {
return tools.ToolEntry{
Name: "review_pr",
Toolset: "tasks",
Description: "Queue a review of an existing pull request. Use it when the user asks to review a PR, " +
"re-check one after a push, or see what the reviewer found. The review runs as a task in its own " +
"sandbox and posts its findings to the pull request, after operator approval when that is enabled; " +
"this returns the task ID immediately.",
Classification: tools.ClassMutating,
Schema: tools.JSONSchema{
"type": "object",
"properties": map[string]any{
"repo": map[string]any{
"type": "string",
"description": "Repository of the PR as \"owner/name\". Must be a repository this identity is configured for.",
},
"pr_number": map[string]any{
"type": "integer",
"description": "Pull request number to review.",
},
},
"required": []any{"repo", "pr_number"},
},
Handler: func(ctx context.Context, input map[string]any) (any, error) {
repo := strings.TrimSpace(asString(input["repo"]))
if repo == "" {
return nil, fmt.Errorf("review_pr: repo is required")
}
number, ok := asInt64(input["pr_number"])
if !ok || number <= 0 {
return nil, fmt.Errorf("review_pr: pr_number must be a positive integer")
}
id, err := creator.CreateTask(ctx, SpawnRequest{
Title: fmt.Sprintf("review %s#%d", repo, number),
Body: fmt.Sprintf("Operator-requested review of pull request %s#%d.", repo, number),
Repo: repo,
Workflow: prReviewWorkflow,
// int, not int64: task.Task.EffectivePRNumber reads the
// input as json.Number, float64 or int, and an int64 would
// resolve to 0 for a task that names a real PR.
Inputs: map[string]any{"pr_number": int(number)},
// The bound identity, never input["identity"]: CreateTask
// enforces the repository allow-list for it.
Identity: identity,
})
if err != nil {
return nil, fmt.Errorf("review_pr: %w", err)
}
return TaskSpawnResult{
ID: id,
Message: fmt.Sprintf("queued pr-review task %d for %s#%d", id, repo, number),
}, nil
},
}
}
-98
View File
@@ -1,98 +0,0 @@
package gateway
import (
"context"
"errors"
"testing"
)
func TestReviewToolOmittedWhenCreatorNil(t *testing.T) {
if entries := ReviewTools(nil, "archie"); len(entries) != 0 {
t.Fatalf("ReviewTools(nil) = %d entries, want 0 -- a tool that cannot run must not be advertised", len(entries))
}
}
func TestReviewToolAppearsWhenCreatorNonNil(t *testing.T) {
entries := ReviewTools(&fakeCreator{}, "archie")
if len(entries) != 1 {
t.Fatalf("ReviewTools() = %d entries, want 1", len(entries))
}
if err := entries[0].Validate(); err != nil {
t.Fatalf("entry.Validate() = %v", err)
}
}
func TestReviewPRToolQueuesReviewTask(t *testing.T) {
creator := &fakeCreator{id: 42}
entry := toolNamed(t, ReviewTools(creator, "archie"), "review_pr")
out, err := entry.Handler(context.Background(), map[string]any{
"repo": "acme/widget", "pr_number": float64(7), "identity": "someone-else",
})
if err != nil {
t.Fatal(err)
}
got := creator.got
if got.Workflow != "pr-review" {
t.Errorf("workflow = %q, want pr-review -- the operator trigger runs the pipeline, it does not review in the turn", got.Workflow)
}
if got.Repo != "acme/widget" {
t.Errorf("repo = %q, want acme/widget -- CreateTask authorizes against it", got.Repo)
}
// The model supplied an identity; it must be ignored in favour of the
// bound identity.
if got.Identity != "archie" {
t.Errorf("identity = %q, want bound %q", got.Identity, "archie")
}
if n, ok := got.Inputs["pr_number"].(int); !ok || n != 7 {
t.Errorf("inputs[pr_number] = %#v, want int 7", got.Inputs["pr_number"])
}
if got.Title == "" || got.Body == "" {
t.Errorf("title/body = %q/%q, want both set so the queued task is identifiable", got.Title, got.Body)
}
result, ok := out.(TaskSpawnResult)
if !ok {
t.Fatalf("handler returned %T, want TaskSpawnResult", out)
}
if result.ID != 42 {
t.Errorf("result.ID = %d, want 42", result.ID)
}
}
func TestReviewPRToolValidatesInput(t *testing.T) {
creator := &fakeCreator{}
entry := toolNamed(t, ReviewTools(creator, "archie"), "review_pr")
tests := []struct {
name string
input map[string]any
}{
{"missing repo", map[string]any{"pr_number": float64(7)}},
{"missing number", map[string]any{"repo": "acme/widget"}},
{"zero number", map[string]any{"repo": "acme/widget", "pr_number": float64(0)}},
{"negative number", map[string]any{"repo": "acme/widget", "pr_number": float64(-1)}},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
if _, err := entry.Handler(context.Background(), tt.input); err == nil {
t.Fatalf("handler error = nil, want validation error for %v", tt.input)
}
})
}
if len(creator.got.Inputs) != 0 || creator.got.Repo != "" {
t.Errorf("rejected input still reached CreateTask: %+v", creator.got)
}
}
func TestReviewPRToolSurfacesCreatorError(t *testing.T) {
creator := &fakeCreator{err: errors.New("repo not configured")}
entry := toolNamed(t, ReviewTools(creator, "archie"), "review_pr")
if _, err := entry.Handler(context.Background(), map[string]any{
"repo": "acme/widget", "pr_number": float64(7),
}); err == nil {
t.Fatal("handler error = nil, want the creator failure surfaced")
}
}
+16 -3
View File
@@ -229,8 +229,7 @@ func taskSpawnTool(creator TaskCreator, identity string) tools.ToolEntry {
return tools.ToolEntry{
Name: "task_spawn",
Toolset: "tasks",
Description: "Queue a new task for this instance to work on in its own worktree, " +
"opening a pull request for review when it finishes. " +
Description: "Queue a workflow task for this instance. The selected workflow determines its outcome. " +
"Use it when the user asks for work to be carried out rather than answered in chat.",
Classification: tools.ClassMutating,
Schema: tools.JSONSchema{
@@ -246,7 +245,11 @@ func taskSpawnTool(creator TaskCreator, identity string) tools.ToolEntry {
},
"workflow": map[string]any{
"type": "string",
"description": "Workflow to run (e.g. implement, tdd, feasibility). Omit to let the daemon route it.",
"description": "Workflow to run. Omit to let the daemon route it.",
},
"inputs": map[string]any{
"type": "object",
"description": "Inputs declared by the selected workflow. The daemon validates their names, types and required values before execution.",
},
},
"required": []any{"title"},
@@ -257,10 +260,20 @@ func taskSpawnTool(creator TaskCreator, identity string) tools.ToolEntry {
return nil, fmt.Errorf("task_spawn: title is required")
}
var inputs map[string]any
if value, supplied := input["inputs"]; supplied {
var ok bool
inputs, ok = value.(map[string]any)
if !ok {
return nil, fmt.Errorf("task_spawn: inputs must be an object")
}
}
id, err := creator.CreateTask(ctx, SpawnRequest{
Title: title,
Repo: strings.TrimSpace(asString(input["repo"])),
Workflow: strings.TrimSpace(asString(input["workflow"])),
Inputs: inputs,
// The bound identity, never input["identity"]. CreateTask
// enforces the repository allow-list for it.
Identity: identity,
+25 -6
View File
@@ -4,6 +4,7 @@ import (
"context"
"errors"
"fmt"
"reflect"
"strings"
"testing"
"time"
@@ -187,12 +188,13 @@ func TestTaskListPropagatesStoreError(t *testing.T) {
func TestTaskSpawn(t *testing.T) {
tests := []struct {
name string
input map[string]any
creator *fakeCreator
wantErr bool
wantTitle string
wantRepo string
name string
input map[string]any
creator *fakeCreator
wantErr bool
wantTitle string
wantRepo string
wantInputs map[string]any
}{
{
name: "title only",
@@ -207,6 +209,20 @@ func TestTaskSpawn(t *testing.T) {
wantTitle: "port it",
wantRepo: "acme/app",
},
{
name: "workflow inputs are forwarded as given",
input: map[string]any{"title": "review it", "repo": "acme/app", "workflow": "pr-review", "inputs": map[string]any{"pr_number": float64(7)}},
creator: &fakeCreator{id: 46},
wantTitle: "review it",
wantRepo: "acme/app",
wantInputs: map[string]any{"pr_number": float64(7)},
},
{
name: "inputs that are not an object are refused",
input: map[string]any{"title": "x", "workflow": "pr-review", "inputs": "pr_number=7"},
creator: &fakeCreator{id: 47},
wantErr: true,
},
{
name: "missing title is refused",
input: map[string]any{},
@@ -246,6 +262,9 @@ func TestTaskSpawn(t *testing.T) {
if tc.creator.got.Repo != tc.wantRepo {
t.Errorf("repo = %q, want %q", tc.creator.got.Repo, tc.wantRepo)
}
if !reflect.DeepEqual(tc.creator.got.Inputs, tc.wantInputs) {
t.Errorf("inputs = %#v, want %#v", tc.creator.got.Inputs, tc.wantInputs)
}
// Identity is bound, never taken from the model.
if tc.creator.got.Identity != "archie" {
t.Errorf("identity = %q, want the bound %q", tc.creator.got.Identity, "archie")
-1
View File
@@ -313,7 +313,6 @@ func (r *TurnRunner) prepareTurn(ctx context.Context, sessionID string, in Inbou
TaskTools(r.TaskLister, r.Tasks, r.TaskLogs, r.TaskActor, r.TaskIdentity),
SessionTools(r.Sessions, r.Router.SessionTracker(), r.Router.sessionPlatform(in), in.Message)...,
)
extraTools = append(extraTools, ReviewTools(r.Tasks, r.TaskIdentity)...)
// The dashboard tools (page_index, dashboard_navigate) belong to the web
// UI only: a non-web channel has no dashboard to point at.
extraTools = append(extraTools, PageIndexTools(r.Channel)...)
+16
View File
@@ -0,0 +1,16 @@
#!/usr/bin/env bash
# Run the approved PR-review benchmark without posting to pull requests.
set -euo pipefail
cd "$(dirname "$0")/.."
: "${PRBENCH_REVIEW_MODEL:?set PRBENCH_REVIEW_MODEL to a configured provider/model}"
: "${PRBENCH_CLASSIFICATION_MODEL:?set PRBENCH_CLASSIFICATION_MODEL to a configured provider/model}"
: "${PRBENCH_JUDGE_MODEL:=anthropic/claude-sonnet-4.6}"
: "${PRBENCH_OUT:=bin/prbench-results}"
if [[ -z "${GH_TOKEN:-}" && -z "${GITHUB_TOKEN:-}" ]] && command -v gh >/dev/null; then
export GH_TOKEN="$(gh auth token)"
fi
exec go run ./cmd/archie-prbench \
-review-model "$PRBENCH_REVIEW_MODEL" \
-classification-model "$PRBENCH_CLASSIFICATION_MODEL" \
-judge-model "$PRBENCH_JUDGE_MODEL" \
-out "$PRBENCH_OUT" "$@"
+3 -3
View File
@@ -35,10 +35,10 @@ func TestTheSplitIsByAudience(t *testing.T) {
development := documentFor(t, generated, developmentDocs)
publicURLs := urlSet(t, public)
if len(publicURLs) != 2 {
t.Errorf("public set = %v, want the landing page and the guide", sortedKeys(publicURLs))
if len(publicURLs) != 3 {
t.Errorf("public set = %v, want the landing page and two guides", sortedKeys(publicURLs))
}
for _, want := range []string{"/docs/", "/docs/guides/first-playbook/"} {
for _, want := range []string{"/docs/", "/docs/guides/first-playbook/", "/docs/guides/pr-review-benchmark/"} {
if !publicURLs[want] {
t.Errorf("%s is not in the public set", want)
}