198 lines
7.0 KiB
Go
198 lines
7.0 KiB
Go
// Package prbench runs the PR reviewer benchmark: the Martian Code-Review-Bench
|
|
// problems, the production review pipeline, and a judge model, all on the
|
|
// providers and model aliases the instance already configures.
|
|
package prbench
|
|
|
|
import (
|
|
"context"
|
|
_ "embed"
|
|
"encoding/json"
|
|
"errors"
|
|
"flag"
|
|
"fmt"
|
|
"io"
|
|
"log/slog"
|
|
"os"
|
|
"os/signal"
|
|
"path/filepath"
|
|
"slices"
|
|
"strings"
|
|
"syscall"
|
|
"time"
|
|
|
|
"github.com/samcharles93/archie-core/internal/agentexec"
|
|
"github.com/samcharles93/archie-core/internal/agentexec/modelloop"
|
|
"github.com/samcharles93/archie-core/internal/app/servicekit"
|
|
"github.com/samcharles93/archie-core/internal/buildinfo"
|
|
"github.com/samcharles93/archie-core/internal/config"
|
|
"github.com/samcharles93/archie-core/internal/domain/workflow/prreview/bench"
|
|
"github.com/samcharles93/archie-core/internal/infrastructure/configuration"
|
|
"github.com/samcharles93/archie-core/internal/infrastructure/prsource"
|
|
)
|
|
|
|
// acceptanceRecall is the PRD's golden micro-recall bar for a full run.
|
|
const acceptanceRecall = 0.70
|
|
|
|
// martian holds the 38 runnable problems of the Martian Code-Review-Bench
|
|
// offline set: those whose pull request still exists upstream.
|
|
//
|
|
//go:embed martian.json
|
|
var martian []byte
|
|
|
|
// Problems returns the benchmark problems, optionally narrowed to ids.
|
|
func Problems(ids []string) ([]bench.Problem, error) {
|
|
var all []bench.Problem
|
|
if err := json.Unmarshal(martian, &all); err != nil {
|
|
return nil, fmt.Errorf("decode benchmark problems: %w", err)
|
|
}
|
|
if len(ids) == 0 {
|
|
return all, nil
|
|
}
|
|
var out []bench.Problem
|
|
for _, id := range ids {
|
|
i := slices.IndexFunc(all, func(p bench.Problem) bool { return p.ID == id })
|
|
if i < 0 {
|
|
return nil, fmt.Errorf("no benchmark problem %q", id)
|
|
}
|
|
out = append(out, all[i])
|
|
}
|
|
return out, nil
|
|
}
|
|
|
|
// Options configures one benchmark run. Model and Judge each name a model
|
|
// alias of the config or a provider/model ref; empty means the "default" alias.
|
|
type Options struct {
|
|
Config string
|
|
Model string
|
|
Judge string
|
|
OutDir string
|
|
Problems []string
|
|
Concurrency int
|
|
}
|
|
|
|
// Run reviews and judges the selected problems, writes the report into
|
|
// Options.OutDir, and prints it to w.
|
|
func Run(ctx context.Context, opts Options, w io.Writer) (bench.Summary, error) {
|
|
problems, err := Problems(opts.Problems)
|
|
if err != nil {
|
|
return bench.Summary{}, err
|
|
}
|
|
doc, err := configuration.New(nil).File(opts.Config)
|
|
if err != nil {
|
|
return bench.Summary{}, err
|
|
}
|
|
cfg := doc.Config
|
|
if err := servicekit.ResolveProviders(&cfg, slog.Default()); err != nil {
|
|
return bench.Summary{}, err
|
|
}
|
|
reviewRef, err := resolveRef(cfg.Models, opts.Model)
|
|
if err != nil {
|
|
return bench.Summary{}, err
|
|
}
|
|
judgeRef, err := resolveRef(cfg.Models, opts.Judge)
|
|
if err != nil {
|
|
return bench.Summary{}, err
|
|
}
|
|
rt := modelloop.NewRuntime(agentexec.ProvidersFromConfig(cfg.Providers), cfg.ModelLimits)
|
|
if rt == nil {
|
|
return bench.Summary{}, fmt.Errorf("%s configures no providers", opts.Config)
|
|
}
|
|
// The pipeline resolves its model through the default alias.
|
|
cfg.Models = map[string]string{config.DefaultModelAlias: reviewRef}
|
|
reviewer := &liveReviewer{cfg: cfg, agent: modelloop.NewLoopRunner(rt, slog.Default()), source: prsource.New(newPublicGitHub())}
|
|
judge := &modelJudge{runtime: rt, modelRef: judgeRef}
|
|
|
|
results := bench.Run(ctx, problems, reviewer, judge, opts.Concurrency)
|
|
summary := bench.Summarize(results)
|
|
report := render(results, summary, reviewRef, judgeRef, len(opts.Problems) == 0)
|
|
path, err := writeReport(opts.OutDir, report)
|
|
if err != nil {
|
|
return summary, err
|
|
}
|
|
fmt.Fprint(w, report)
|
|
fmt.Fprintf(w, "\nreport written to %s\n", path)
|
|
if summary.Failed != 0 {
|
|
return summary, fmt.Errorf("%d benchmark problem(s) failed; results are incomplete", summary.Failed)
|
|
}
|
|
return summary, nil
|
|
}
|
|
|
|
// Main is the `archied prbench` subcommand.
|
|
func Main(args []string, stdout, stderr io.Writer) int {
|
|
var opts Options
|
|
var problems string
|
|
flags := flag.NewFlagSet("archied prbench", flag.ContinueOnError)
|
|
flags.SetOutput(stderr)
|
|
flags.StringVar(&opts.Config, "config", configuration.DefaultConfigPath(), "config file whose providers and model aliases serve the run")
|
|
flags.StringVar(&opts.Model, "model", "", "model alias or provider/model for the reviewer (default: the default alias)")
|
|
flags.StringVar(&opts.Judge, "judge-model", "", "model alias or provider/model for the judge (default: the default alias; use a different model from the reviewer's for an independent judge)")
|
|
flags.StringVar(&opts.OutDir, "out", "docs/benchmarks", "directory the report is written to")
|
|
flags.StringVar(&problems, "problems", "", "comma-separated problem ids for a partial run (default: all 38)")
|
|
flags.IntVar(&opts.Concurrency, "concurrency", 4, "problems reviewed at once")
|
|
if err := flags.Parse(args); err != nil {
|
|
if errors.Is(err, flag.ErrHelp) {
|
|
return 0
|
|
}
|
|
return 2
|
|
}
|
|
if problems != "" {
|
|
opts.Problems = strings.Split(problems, ",")
|
|
}
|
|
ctx, stop := signal.NotifyContext(context.Background(), os.Interrupt, syscall.SIGTERM)
|
|
defer stop()
|
|
if _, err := Run(ctx, opts, stdout); err != nil {
|
|
fmt.Fprintln(stderr, "archied prbench:", err)
|
|
return 1
|
|
}
|
|
return 0
|
|
}
|
|
|
|
// resolveRef accepts a provider/model ref as is and otherwise looks the alias up.
|
|
func resolveRef(aliases map[string]string, v string) (string, error) {
|
|
v = strings.TrimSpace(v)
|
|
if strings.Contains(v, "/") {
|
|
return v, nil
|
|
}
|
|
return config.ResolveModel(aliases, v, config.PurposeAgent)
|
|
}
|
|
|
|
func render(results []bench.Result, s bench.Summary, reviewRef, judgeRef string, full bool) string {
|
|
var b strings.Builder
|
|
scope, verdict := "partial run (not the acceptance run)", ""
|
|
if full {
|
|
scope = "full run"
|
|
verdict = fmt.Sprintf("\nAcceptance (recall >= %.2f): %s\n", acceptanceRecall, passFail(s.Failed == 0 && s.Recall >= acceptanceRecall))
|
|
}
|
|
fmt.Fprintf(&b, "# PR review benchmark\n\nDate: %s\nArchie version: %s\nReview model: %s\nJudge model: %s\nScope: %s\n\n",
|
|
time.Now().UTC().Format("2006-01-02"), buildinfo.Version, reviewRef, judgeRef, scope)
|
|
fmt.Fprintf(&b, "Problems %d (failed %d) recall %.3f (%d/%d) golden-only precision %.3f (%d/%d)\n%s\n",
|
|
s.Problems, s.Failed, s.Recall, s.Hits, s.Goldens, s.Precision, s.Matched, s.Comments, verdict)
|
|
b.WriteString("| Problem | Recall | Comments | Notes |\n| --- | --- | --- | --- |\n")
|
|
for _, r := range results {
|
|
recall, notes := fmt.Sprintf("%d/%d", r.Hits(), len(r.Problem.Goldens)), ""
|
|
switch {
|
|
case r.Err != "":
|
|
recall, notes = "-", "failed: "+strings.ReplaceAll(r.Err, "|", "/")
|
|
case len(r.SkippedPhases) != 0:
|
|
notes = "skipped phases: " + strings.Join(r.SkippedPhases, ", ")
|
|
}
|
|
fmt.Fprintf(&b, "| %s | %s | %d | %s |\n", r.Problem.ID, recall, len(r.Comments), notes)
|
|
}
|
|
return b.String()
|
|
}
|
|
|
|
func passFail(ok bool) string {
|
|
if ok {
|
|
return "PASS"
|
|
}
|
|
return "FAIL"
|
|
}
|
|
|
|
func writeReport(dir, report string) (string, error) {
|
|
if err := os.MkdirAll(dir, 0o755); err != nil {
|
|
return "", err
|
|
}
|
|
path := filepath.Join(dir, "pr-review-"+time.Now().UTC().Format("20060102-150405")+".md")
|
|
return path, os.WriteFile(path, []byte(report), 0o644)
|
|
}
|