package prbench import ( "context" "encoding/json" "fmt" "os" "path/filepath" "github.com/samcharles93/archie-core/internal/domain/workflow/prreview" "github.com/samcharles93/archie-core/internal/domain/workflow/prreview/bench" ) // fixtureReviewer reads each problem's findings from /.json, so the // deterministic core and the judge can be measured before a live reviewer // exists, and a live run's findings can be rescored without reviewing again. type fixtureReviewer struct{ dir string } type fixture struct { AIGenerated float64 `json:"ai_generated"` BlastRadiusFiles int `json:"blast_radius_files"` Findings []fixtureFinding `json:"findings"` } type fixtureFinding struct { Dimension string `json:"dimension"` File string `json:"file"` LineStart int `json:"line_start"` LineEnd int `json:"line_end"` Severity string `json:"severity"` Category string `json:"category"` Title string `json:"title"` Body string `json:"body"` Suggestion string `json:"suggestion"` Evidence string `json:"evidence"` Confidence float64 `json:"confidence"` Tags []string `json:"tags"` Adversary string `json:"adversary"` Compound bool `json:"compound"` Blocking bool `json:"blocking"` } func (f fixtureReviewer) Review(_ context.Context, p bench.Problem) (bench.Review, error) { data, err := os.ReadFile(filepath.Join(f.dir, p.ID+".json")) if err != nil { return bench.Review{}, err } var fx fixture if err := json.Unmarshal(data, &fx); err != nil { return bench.Review{}, fmt.Errorf("decode %s findings: %w", p.ID, err) } review := bench.Review{Inputs: prreview.ScoreInputs{AIGenerated: fx.AIGenerated, BlastRadiusFiles: fx.BlastRadiusFiles}} for i, ff := range fx.Findings { switch prreview.Severity(ff.Severity) { case prreview.SeverityCritical, prreview.SeverityImportant, prreview.SeveritySuggestion, prreview.SeverityNitpick: default: return bench.Review{}, fmt.Errorf("%s finding %d: unknown severity %q", p.ID, i, ff.Severity) } review.Findings = append(review.Findings, prreview.Finding{ Dimension: ff.Dimension, File: ff.File, LineStart: ff.LineStart, LineEnd: ff.LineEnd, Severity: prreview.Severity(ff.Severity), Category: ff.Category, Title: ff.Title, Body: ff.Body, Suggestion: ff.Suggestion, Evidence: ff.Evidence, Confidence: ff.Confidence, Tags: ff.Tags, Adversary: prreview.AdversaryVerdict(ff.Adversary), Compound: ff.Compound, Blocking: ff.Blocking, }) } return review, nil }