Files
archie/internal/app/prbench/prbench.go
T
sam 1358c3d759 feat(prreview): benchmark harness scoring findings on the Martian problems
Scores a reviewer's findings on the 38 runnable Martian Code-Review-Bench
PRs: the deterministic core turns findings into posted comments, an
independent judge model matches them to the goldens, and the run reports
golden micro-recall and golden-only precision. The reviewer is a findings
fixture directory until the pr-review workflow lands.

Run with: task bench:prreview -- -findings DIR
2026-09-26 10:30:12 +10:00

117 lines
3.5 KiB
Go

// Package prbench wires the PR reviewer benchmark: the Martian Code-Review-Bench
// problems, a reviewer, and a judge model on the built-in runner.
package prbench
import (
"context"
_ "embed"
"encoding/json"
"errors"
"fmt"
"io"
"log/slog"
"os"
"path/filepath"
"slices"
"github.com/samcharles93/archie-core/internal/agentexec"
"github.com/samcharles93/archie-core/internal/app/archied"
"github.com/samcharles93/archie-core/internal/domain/workflow/prreview/bench"
"github.com/samcharles93/archie-core/internal/infrastructure/configuration"
)
// martian holds the 38 runnable problems of the Martian Code-Review-Bench
// offline set: those whose pull request still exists upstream.
//
//go:embed martian.json
var martian []byte
// Problems returns the benchmark problems, optionally narrowed to ids.
func Problems(ids []string) ([]bench.Problem, error) {
var all []bench.Problem
if err := json.Unmarshal(martian, &all); err != nil {
return nil, fmt.Errorf("decode benchmark problems: %w", err)
}
if len(ids) == 0 {
return all, nil
}
var out []bench.Problem
for _, id := range ids {
i := slices.IndexFunc(all, func(p bench.Problem) bool { return p.ID == id })
if i < 0 {
return nil, fmt.Errorf("no benchmark problem %q", id)
}
out = append(out, all[i])
}
return out, nil
}
// Options configures one benchmark run.
type Options struct {
Config string
Fixtures string
JudgeModel string
Out string
Problems []string
Concurrency int
}
// Run reviews and judges the selected problems, writes one result file per
// problem and a summary to Options.Out, and prints the summary to w.
func Run(ctx context.Context, opts Options, w io.Writer) (bench.Summary, error) {
if opts.Fixtures == "" {
return bench.Summary{}, errors.New("no reviewer: pass a findings fixture directory")
}
problems, err := Problems(opts.Problems)
if err != nil {
return bench.Summary{}, err
}
doc, err := configuration.New(nil).File(opts.Config)
if err != nil {
return bench.Summary{}, err
}
if err := archied.ResolveProviders(&doc.Config, slog.Default()); err != nil {
return bench.Summary{}, err
}
rt := agentexec.NewRuntime(agentexec.ProvidersFromConfig(doc.Config.Providers))
if rt == nil {
return bench.Summary{}, fmt.Errorf("%s configures no providers for the judge", opts.Config)
}
judge := &modelJudge{runtime: rt, modelRef: opts.JudgeModel}
results := bench.Run(ctx, problems, fixtureReviewer{dir: opts.Fixtures}, judge, opts.Concurrency)
summary := bench.Summarize(results)
if err := writeResults(opts.Out, results, summary); err != nil {
return summary, err
}
for _, r := range results {
if r.Err != "" {
fmt.Fprintf(w, "%s: %s\n", r.Problem.ID, r.Err)
}
}
fmt.Fprintf(w, "problems %d (failed %d) recall %.3f (%d/%d) golden-only precision %.3f (%d/%d)\n",
summary.Problems, summary.Failed, summary.Recall, summary.Hits, summary.Goldens,
summary.Precision, summary.Matched, summary.Comments)
return summary, nil
}
func writeResults(dir string, results []bench.Result, summary bench.Summary) error {
if err := os.MkdirAll(filepath.Join(dir, "results"), 0o755); err != nil {
return err
}
for _, r := range results {
if err := writeJSON(filepath.Join(dir, "results", r.Problem.ID+".json"), r); err != nil {
return err
}
}
return writeJSON(filepath.Join(dir, "summary.json"), summary)
}
func writeJSON(path string, v any) error {
data, err := json.MarshalIndent(v, "", " ")
if err != nil {
return err
}
return os.WriteFile(path, append(data, '\n'), 0o644)
}