Files
archie/internal/app/prbench/prbench_test.go
T
sam 1358c3d759 feat(prreview): benchmark harness scoring findings on the Martian problems
Scores a reviewer's findings on the 38 runnable Martian Code-Review-Bench
PRs: the deterministic core turns findings into posted comments, an
independent judge model matches them to the goldens, and the run reports
golden micro-recall and golden-only precision. The reviewer is a findings
fixture directory until the pr-review workflow lands.

Run with: task bench:prreview -- -findings DIR
2026-09-26 10:30:12 +10:00

66 lines
1.8 KiB
Go

package prbench
import (
"os"
"path/filepath"
"strings"
"testing"
"github.com/samcharles93/archie-core/internal/domain/workflow/prreview/bench"
)
func TestProblemsAreTheRunnableSet(t *testing.T) {
all, err := Problems(nil)
if err != nil {
t.Fatal(err)
}
goldens := 0
seen := map[string]bool{}
for _, p := range all {
if seen[p.ID] || p.PRURL == "" || len(p.Goldens) == 0 {
t.Errorf("problem %q: duplicate, or missing its PR or goldens", p.ID)
}
seen[p.ID] = true
goldens += len(p.Goldens)
}
if len(all) != 38 || goldens != 102 {
t.Errorf("got %d problems with %d goldens, want 38 with 102", len(all), goldens)
}
if _, err := Problems([]string{"nope#1"}); err == nil {
t.Error("an unknown problem id was accepted")
}
}
func TestFixtureReviewer(t *testing.T) {
tests := []struct {
name string
body string
wantErr string
want int
}{
{name: "findings", body: `{"blast_radius_files": 12, "findings": [{"file": "a.go", "line_start": 3, "severity": "critical", "confidence": 0.8}]}`, want: 1},
{name: "unknown severity", body: `{"findings": [{"file": "a.go", "severity": "High"}]}`, wantErr: "unknown severity"},
{name: "missing file", wantErr: "no such file"},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
dir := t.TempDir()
if tt.body != "" {
if err := os.WriteFile(filepath.Join(dir, "x#1.json"), []byte(tt.body), 0o644); err != nil {
t.Fatal(err)
}
}
got, err := fixtureReviewer{dir: dir}.Review(t.Context(), bench.Problem{ID: "x#1"})
if tt.wantErr != "" {
if err == nil || !strings.Contains(err.Error(), tt.wantErr) {
t.Fatalf("err = %v, want containing %q", err, tt.wantErr)
}
return
}
if err != nil || len(got.Findings) != tt.want || got.Inputs.BlastRadiusFiles != 12 {
t.Fatalf("Review = %+v, %v", got, err)
}
})
}
}