Scores a reviewer's findings on the 38 runnable Martian Code-Review-Bench PRs: the deterministic core turns findings into posted comments, an independent judge model matches them to the goldens, and the run reports golden micro-recall and golden-only precision. The reviewer is a findings fixture directory until the pr-review workflow lands. Run with: task bench:prreview -- -findings DIR
117 lines
3.5 KiB
Go
117 lines
3.5 KiB
Go
// Package prbench wires the PR reviewer benchmark: the Martian Code-Review-Bench
|
|
// problems, a reviewer, and a judge model on the built-in runner.
|
|
package prbench
|
|
|
|
import (
|
|
"context"
|
|
_ "embed"
|
|
"encoding/json"
|
|
"errors"
|
|
"fmt"
|
|
"io"
|
|
"log/slog"
|
|
"os"
|
|
"path/filepath"
|
|
"slices"
|
|
|
|
"github.com/samcharles93/archie-core/internal/agentexec"
|
|
"github.com/samcharles93/archie-core/internal/app/archied"
|
|
"github.com/samcharles93/archie-core/internal/domain/workflow/prreview/bench"
|
|
"github.com/samcharles93/archie-core/internal/infrastructure/configuration"
|
|
)
|
|
|
|
// martian holds the 38 runnable problems of the Martian Code-Review-Bench
|
|
// offline set: those whose pull request still exists upstream.
|
|
//
|
|
//go:embed martian.json
|
|
var martian []byte
|
|
|
|
// Problems returns the benchmark problems, optionally narrowed to ids.
|
|
func Problems(ids []string) ([]bench.Problem, error) {
|
|
var all []bench.Problem
|
|
if err := json.Unmarshal(martian, &all); err != nil {
|
|
return nil, fmt.Errorf("decode benchmark problems: %w", err)
|
|
}
|
|
if len(ids) == 0 {
|
|
return all, nil
|
|
}
|
|
var out []bench.Problem
|
|
for _, id := range ids {
|
|
i := slices.IndexFunc(all, func(p bench.Problem) bool { return p.ID == id })
|
|
if i < 0 {
|
|
return nil, fmt.Errorf("no benchmark problem %q", id)
|
|
}
|
|
out = append(out, all[i])
|
|
}
|
|
return out, nil
|
|
}
|
|
|
|
// Options configures one benchmark run.
|
|
type Options struct {
|
|
Config string
|
|
Fixtures string
|
|
JudgeModel string
|
|
Out string
|
|
Problems []string
|
|
Concurrency int
|
|
}
|
|
|
|
// Run reviews and judges the selected problems, writes one result file per
|
|
// problem and a summary to Options.Out, and prints the summary to w.
|
|
func Run(ctx context.Context, opts Options, w io.Writer) (bench.Summary, error) {
|
|
if opts.Fixtures == "" {
|
|
return bench.Summary{}, errors.New("no reviewer: pass a findings fixture directory")
|
|
}
|
|
problems, err := Problems(opts.Problems)
|
|
if err != nil {
|
|
return bench.Summary{}, err
|
|
}
|
|
doc, err := configuration.New(nil).File(opts.Config)
|
|
if err != nil {
|
|
return bench.Summary{}, err
|
|
}
|
|
if err := archied.ResolveProviders(&doc.Config, slog.Default()); err != nil {
|
|
return bench.Summary{}, err
|
|
}
|
|
rt := agentexec.NewRuntime(agentexec.ProvidersFromConfig(doc.Config.Providers))
|
|
if rt == nil {
|
|
return bench.Summary{}, fmt.Errorf("%s configures no providers for the judge", opts.Config)
|
|
}
|
|
judge := &modelJudge{runtime: rt, modelRef: opts.JudgeModel}
|
|
|
|
results := bench.Run(ctx, problems, fixtureReviewer{dir: opts.Fixtures}, judge, opts.Concurrency)
|
|
summary := bench.Summarize(results)
|
|
if err := writeResults(opts.Out, results, summary); err != nil {
|
|
return summary, err
|
|
}
|
|
for _, r := range results {
|
|
if r.Err != "" {
|
|
fmt.Fprintf(w, "%s: %s\n", r.Problem.ID, r.Err)
|
|
}
|
|
}
|
|
fmt.Fprintf(w, "problems %d (failed %d) recall %.3f (%d/%d) golden-only precision %.3f (%d/%d)\n",
|
|
summary.Problems, summary.Failed, summary.Recall, summary.Hits, summary.Goldens,
|
|
summary.Precision, summary.Matched, summary.Comments)
|
|
return summary, nil
|
|
}
|
|
|
|
func writeResults(dir string, results []bench.Result, summary bench.Summary) error {
|
|
if err := os.MkdirAll(filepath.Join(dir, "results"), 0o755); err != nil {
|
|
return err
|
|
}
|
|
for _, r := range results {
|
|
if err := writeJSON(filepath.Join(dir, "results", r.Problem.ID+".json"), r); err != nil {
|
|
return err
|
|
}
|
|
}
|
|
return writeJSON(filepath.Join(dir, "summary.json"), summary)
|
|
}
|
|
|
|
func writeJSON(path string, v any) error {
|
|
data, err := json.MarshalIndent(v, "", " ")
|
|
if err != nil {
|
|
return err
|
|
}
|
|
return os.WriteFile(path, append(data, '\n'), 0o644)
|
|
}
|