Files
archie/cmd/archie-prbench/main.go
T
sam 1358c3d759 feat(prreview): benchmark harness scoring findings on the Martian problems
Scores a reviewer's findings on the 38 runnable Martian Code-Review-Bench
PRs: the deterministic core turns findings into posted comments, an
independent judge model matches them to the goldens, and the run reports
golden micro-recall and golden-only precision. The reviewer is a findings
fixture directory until the pr-review workflow lands.

Run with: task bench:prreview -- -findings DIR
2026-09-26 10:30:12 +10:00

42 lines
1.4 KiB
Go

// Command archie-prbench scores the PR reviewer on the Martian
// Code-Review-Bench problems. See docs/prds/pr-review-agent.md, Verification.
package main
import (
"context"
"flag"
"fmt"
"os"
"os/signal"
"strings"
"syscall"
"github.com/samcharles93/archie-core/internal/app/prbench"
"github.com/samcharles93/archie-core/internal/infrastructure/configuration"
)
func main() { os.Exit(run()) }
func run() int {
var opts prbench.Options
var problems string
flag.StringVar(&opts.Config, "config", configuration.DefaultConfigPath(), "configuration file whose providers serve the judge model")
flag.StringVar(&opts.Fixtures, "findings", "", "directory of <problem id>.json findings to score")
flag.StringVar(&opts.JudgeModel, "judge-model", "anthropic/claude-sonnet-4.6", "provider/model of the independent judge")
flag.StringVar(&opts.Out, "out", "prbench-out", "directory for per-problem results and summary.json")
flag.StringVar(&problems, "problems", "", "comma-separated problem ids (default: all)")
flag.IntVar(&opts.Concurrency, "concurrency", 4, "problems judged at once")
flag.Parse()
if problems != "" {
opts.Problems = strings.Split(problems, ",")
}
ctx, stop := signal.NotifyContext(context.Background(), os.Interrupt, syscall.SIGTERM)
defer stop()
if _, err := prbench.Run(ctx, opts, os.Stdout); err != nil {
fmt.Fprintln(os.Stderr, "archie-prbench:", err)
return 1
}
return 0
}