Files
archie/internal/app/prbench/prbench.go
T

198 lines
7.0 KiB
Go

// Package prbench runs the PR reviewer benchmark: the Martian Code-Review-Bench
// problems, the production review pipeline, and a judge model, all on the
// providers and model aliases the instance already configures.
package prbench
import (
"context"
_ "embed"
"encoding/json"
"errors"
"flag"
"fmt"
"io"
"log/slog"
"os"
"os/signal"
"path/filepath"
"slices"
"strings"
"syscall"
"time"
"github.com/samcharles93/archie-core/internal/agentexec"
"github.com/samcharles93/archie-core/internal/agentexec/modelloop"
"github.com/samcharles93/archie-core/internal/app/servicekit"
"github.com/samcharles93/archie-core/internal/buildinfo"
"github.com/samcharles93/archie-core/internal/config"
"github.com/samcharles93/archie-core/internal/domain/workflow/prreview/bench"
"github.com/samcharles93/archie-core/internal/infrastructure/configuration"
"github.com/samcharles93/archie-core/internal/infrastructure/prsource"
)
// acceptanceRecall is the PRD's golden micro-recall bar for a full run.
const acceptanceRecall = 0.70
// martian holds the 38 runnable problems of the Martian Code-Review-Bench
// offline set: those whose pull request still exists upstream.
//
//go:embed martian.json
var martian []byte
// Problems returns the benchmark problems, optionally narrowed to ids.
func Problems(ids []string) ([]bench.Problem, error) {
var all []bench.Problem
if err := json.Unmarshal(martian, &all); err != nil {
return nil, fmt.Errorf("decode benchmark problems: %w", err)
}
if len(ids) == 0 {
return all, nil
}
var out []bench.Problem
for _, id := range ids {
i := slices.IndexFunc(all, func(p bench.Problem) bool { return p.ID == id })
if i < 0 {
return nil, fmt.Errorf("no benchmark problem %q", id)
}
out = append(out, all[i])
}
return out, nil
}
// Options configures one benchmark run. Model and Judge each name a model
// alias of the config or a provider/model ref; empty means the "default" alias.
type Options struct {
Config string
Model string
Judge string
OutDir string
Problems []string
Concurrency int
}
// Run reviews and judges the selected problems, writes the report into
// Options.OutDir, and prints it to w.
func Run(ctx context.Context, opts Options, w io.Writer) (bench.Summary, error) {
problems, err := Problems(opts.Problems)
if err != nil {
return bench.Summary{}, err
}
doc, err := configuration.New(nil).File(opts.Config)
if err != nil {
return bench.Summary{}, err
}
cfg := doc.Config
if err := servicekit.ResolveProviders(&cfg, slog.Default()); err != nil {
return bench.Summary{}, err
}
reviewRef, err := resolveRef(cfg.Models, opts.Model)
if err != nil {
return bench.Summary{}, err
}
judgeRef, err := resolveRef(cfg.Models, opts.Judge)
if err != nil {
return bench.Summary{}, err
}
rt := modelloop.NewRuntime(agentexec.ProvidersFromConfig(cfg.Providers), cfg.ModelLimits)
if rt == nil {
return bench.Summary{}, fmt.Errorf("%s configures no providers", opts.Config)
}
// The pipeline resolves its model through the default alias.
cfg.Models = map[string]string{config.DefaultModelAlias: reviewRef}
reviewer := &liveReviewer{cfg: cfg, agent: modelloop.NewLoopRunner(rt, slog.Default()), source: prsource.New(newPublicGitHub())}
judge := &modelJudge{runtime: rt, modelRef: judgeRef}
results := bench.Run(ctx, problems, reviewer, judge, opts.Concurrency)
summary := bench.Summarize(results)
report := render(results, summary, reviewRef, judgeRef, len(opts.Problems) == 0)
path, err := writeReport(opts.OutDir, report)
if err != nil {
return summary, err
}
fmt.Fprint(w, report)
fmt.Fprintf(w, "\nreport written to %s\n", path)
if summary.Failed != 0 {
return summary, fmt.Errorf("%d benchmark problem(s) failed; results are incomplete", summary.Failed)
}
return summary, nil
}
// Main is the `archied prbench` subcommand.
func Main(args []string, stdout, stderr io.Writer) int {
var opts Options
var problems string
flags := flag.NewFlagSet("archied prbench", flag.ContinueOnError)
flags.SetOutput(stderr)
flags.StringVar(&opts.Config, "config", configuration.DefaultConfigPath(), "config file whose providers and model aliases serve the run")
flags.StringVar(&opts.Model, "model", "", "model alias or provider/model for the reviewer (default: the default alias)")
flags.StringVar(&opts.Judge, "judge-model", "", "model alias or provider/model for the judge (default: the default alias; use a different model from the reviewer's for an independent judge)")
flags.StringVar(&opts.OutDir, "out", "docs/benchmarks", "directory the report is written to")
flags.StringVar(&problems, "problems", "", "comma-separated problem ids for a partial run (default: all 38)")
flags.IntVar(&opts.Concurrency, "concurrency", 4, "problems reviewed at once")
if err := flags.Parse(args); err != nil {
if errors.Is(err, flag.ErrHelp) {
return 0
}
return 2
}
if problems != "" {
opts.Problems = strings.Split(problems, ",")
}
ctx, stop := signal.NotifyContext(context.Background(), os.Interrupt, syscall.SIGTERM)
defer stop()
if _, err := Run(ctx, opts, stdout); err != nil {
fmt.Fprintln(stderr, "archied prbench:", err)
return 1
}
return 0
}
// resolveRef accepts a provider/model ref as is and otherwise looks the alias up.
func resolveRef(aliases map[string]string, v string) (string, error) {
v = strings.TrimSpace(v)
if strings.Contains(v, "/") {
return v, nil
}
return config.ResolveModel(aliases, v, config.PurposeAgent)
}
func render(results []bench.Result, s bench.Summary, reviewRef, judgeRef string, full bool) string {
var b strings.Builder
scope, verdict := "partial run (not the acceptance run)", ""
if full {
scope = "full run"
verdict = fmt.Sprintf("\nAcceptance (recall >= %.2f): %s\n", acceptanceRecall, passFail(s.Failed == 0 && s.Recall >= acceptanceRecall))
}
fmt.Fprintf(&b, "# PR review benchmark\n\nDate: %s\nArchie version: %s\nReview model: %s\nJudge model: %s\nScope: %s\n\n",
time.Now().UTC().Format("2006-01-02"), buildinfo.Version, reviewRef, judgeRef, scope)
fmt.Fprintf(&b, "Problems %d (failed %d) recall %.3f (%d/%d) golden-only precision %.3f (%d/%d)\n%s\n",
s.Problems, s.Failed, s.Recall, s.Hits, s.Goldens, s.Precision, s.Matched, s.Comments, verdict)
b.WriteString("| Problem | Recall | Comments | Notes |\n| --- | --- | --- | --- |\n")
for _, r := range results {
recall, notes := fmt.Sprintf("%d/%d", r.Hits(), len(r.Problem.Goldens)), ""
switch {
case r.Err != "":
recall, notes = "-", "failed: "+strings.ReplaceAll(r.Err, "|", "/")
case len(r.SkippedPhases) != 0:
notes = "skipped phases: " + strings.Join(r.SkippedPhases, ", ")
}
fmt.Fprintf(&b, "| %s | %s | %d | %s |\n", r.Problem.ID, recall, len(r.Comments), notes)
}
return b.String()
}
func passFail(ok bool) string {
if ok {
return "PASS"
}
return "FAIL"
}
func writeReport(dir, report string) (string, error) {
if err := os.MkdirAll(dir, 0o755); err != nil {
return "", err
}
path := filepath.Join(dir, "pr-review-"+time.Now().UTC().Format("20060102-150405")+".md")
return path, os.WriteFile(path, []byte(report), 0o644)
}