Files
archie/internal/domain/workflow/prreview/scoring.go
T

418 lines
13 KiB
Go

package prreview
import (
"cmp"
"math"
"slices"
"strings"
)
// Severity is how bad a finding is, on the reviewer's own rubric. It drives
// weights, floors and ordering; whether the finding must be fixed before the
// change ships is a separate question, answered by the merge gate.
type Severity string
const (
SeverityCritical Severity = "critical"
SeverityImportant Severity = "important"
SeveritySuggestion Severity = "suggestion"
SeverityNitpick Severity = "nitpick"
)
// AdversaryVerdict is what the adversary decided about one finding.
type AdversaryVerdict string
const (
// AdversaryConfirmed is a finding the adversary reproduced or agreed with.
AdversaryConfirmed AdversaryVerdict = "confirmed"
// AdversaryChallenged is a finding the adversary argued against.
AdversaryChallenged AdversaryVerdict = "challenged"
)
// ReviewEvent is the verdict a forge review submits. It has no approving value:
// archie reviews changes it did not write and does not own the merge decision,
// so the strongest thing it can say is that a blocking finding has to be fixed.
type ReviewEvent string
const (
// ReviewEventComment posts the findings without gating the merge.
ReviewEventComment ReviewEvent = "COMMENT"
// ReviewEventRequestChanges asks for the blocking findings to be fixed
// before the change merges.
ReviewEventRequestChanges ReviewEvent = "REQUEST_CHANGES"
)
// The multipliers a finding or a run activates. They compound: a compound
// finding the adversary confirmed on a machine-written change with a wide blast
// radius carries all four.
const (
MultiplierCompound = "compound"
MultiplierAdversaryConfirmed = "adversary_confirmed"
MultiplierAdversaryChallenged = "adversary_challenged"
MultiplierAIGenerated = "ai_generated_pr"
MultiplierBlastRadiusHigh = "blast_radius_high"
)
const (
multiplierCompound = 1.5
multiplierAdversaryConfirmed = 1.3
multiplierAdversaryChallenged = 0.5
multiplierAIGenerated = 1.2
multiplierBlastRadiusHigh = 1.2
)
const (
// BlastRadiusFileThreshold is the number of reached files above which a
// change is treated as wide-reaching.
BlastRadiusFileThreshold = 10
// AIGeneratedThreshold is the intake confidence above which a change counts
// as machine-written. Above, not at: a coin-flip is not evidence.
AIGeneratedThreshold = 0.5
// MaxInlineComments is how many inline comments one review posts. The cap is
// a budget on the reader's attention, so what it cuts is the tail of the
// ranked list, never a filter on what was found.
MaxInlineComments = 25
)
// Finding is one review finding before scoring: what the reviewer observed,
// where, and the two flags that only a verdict can set.
type Finding struct {
Dimension string
File string
LineStart int
LineEnd int
Severity Severity
// No producer sets it yet, so an empty category is a category -- two
// findings that both leave it empty are duplicates of each other when they
// also share a file and a line.
Category string
Title string
Body string
Suggestion string
Evidence string
Confidence float64
Tags []string
// Adversary is the verdict the adversary reached about this finding, empty
// when it did not reach one. It rides on the finding rather than on a
// side-list keyed by title, so a verdict cannot land on another finding
// that happens to share a title.
Adversary AdversaryVerdict
// Compound reports that this defect only exists in combination with
// others, which is what the compound multiplier is for.
Compound bool
// Blocking is the merge gate's verdict: only a broken build, a security
// hole, data loss, a contract break or a regression is blocking, and a
// failed gate call leaves this false.
Blocking bool
}
// ScoredFinding is a finding after scoring: ranked, canonicalised, and carrying
// the multipliers that moved it.
type ScoredFinding struct {
Dimension string
File string
LineStart int
LineEnd int
Severity Severity
Category string
Title string
Body string
Suggestion string
Evidence string
Confidence float64
Tags []string
Score float64
Multipliers []string
Blocking bool
Adversary AdversaryVerdict
}
// ScoreInputs is the run context that moves every finding's score.
type ScoreInputs struct {
// AIGenerated is intake's confidence, 0..1, that the change was written by
// a machine.
AIGenerated float64
// BlastRadiusFiles is how many unchanged files the change reaches.
BlastRadiusFiles int
}
// Score drops findings below their severity's confidence floor, scores and
// merges the rest, and ranks them. The result is deterministic.
func Score(findings []Finding, inputs ScoreInputs) []ScoredFinding {
scored := make([]ScoredFinding, 0, len(findings))
for _, finding := range findings {
severity := normalizeSeverity(finding.Severity)
if finding.Confidence < confidenceFloor(severity) {
continue
}
score, applied := scoreOf(finding, severity, inputs)
scored = append(scored, ScoredFinding{
Dimension: finding.Dimension,
File: finding.File,
LineStart: finding.LineStart,
LineEnd: finding.LineEnd,
Severity: severity,
Category: finding.Category,
Title: finding.Title,
Body: finding.Body,
Suggestion: finding.Suggestion,
Evidence: finding.Evidence,
Confidence: finding.Confidence,
Tags: finding.Tags,
Score: score,
Multipliers: applied,
Blocking: finding.Blocking,
Adversary: finding.Adversary,
})
}
ranked := mergeDuplicates(scored)
slices.SortFunc(ranked, compareScored)
return ranked
}
// mergeDuplicates collapses the findings that are one claim about one place.
// Overlap is transitive, so a chain 1-2, 2-3, 3-4 is one claim and not two.
func mergeDuplicates(findings []ScoredFinding) []ScoredFinding {
ordered := slices.Clone(findings)
slices.SortFunc(ordered, compareIdentity)
merged := make([]ScoredFinding, 0, len(ordered))
for start := 0; start < len(ordered); {
end := start + 1
for end < len(ordered) && sameKey(ordered[start], ordered[end]) {
end++
}
merged = append(merged, mergeRun(ordered[start:end])...)
start = end
}
return merged
}
// sameKey reports whether two findings of a canonically ordered list are in the
// same (file, category) group, which is how far the grouping reaches before the
// line ranges decide.
func sameKey(a, b ScoredFinding) bool {
return a.File == b.File && a.Category == b.Category
}
// mergeRun merges overlapping line ranges in one (file, category) group,
// keeping the best-scoring member over the union of lines. Blocking if any
// member is.
func mergeRun(group []ScoredFinding) []ScoredFinding {
clusters := make([]findingCluster, 0, len(group))
for _, member := range group {
if len(clusters) > 0 && member.LineStart <= clusters[len(clusters)-1].last {
clusters[len(clusters)-1].add(member)
continue
}
clusters = append(clusters, newFindingCluster(member))
}
merged := make([]ScoredFinding, 0, len(clusters))
for _, cluster := range clusters {
merged = append(merged, cluster.finding())
}
return merged
}
// findingCluster is a run of overlapping findings.
type findingCluster struct {
best ScoredFinding
first int
last int
blocking bool
}
func newFindingCluster(member ScoredFinding) findingCluster {
return findingCluster{best: member, first: member.LineStart, last: member.LineEnd, blocking: member.Blocking}
}
// add folds a member whose range overlaps the cluster into it.
func (c *findingCluster) add(member ScoredFinding) {
c.last = max(c.last, member.LineEnd)
c.blocking = c.blocking || member.Blocking
if compareScored(member, c.best) < 0 {
c.best = member
}
}
// finding is the one finding the cluster contributes.
func (c *findingCluster) finding() ScoredFinding {
merged := c.best
merged.LineStart = c.first
merged.LineEnd = c.last
merged.Blocking = c.blocking
return merged
}
// compareScored is the order findings are ranked in: the highest score first,
// ties broken by the finding itself. It is a total order over what a scored
// finding carries, so it is also what picks a cluster's member and what makes
// any permutation of the same findings produce the same list in the same order.
func compareScored(a, b ScoredFinding) int {
if order := cmp.Compare(b.Score, a.Score); order != 0 {
return order
}
return compareIdentity(a, b)
}
// compareIdentity orders two findings by their own content: where they are,
// then what they say. It carries no score, so it is the order the merge walks
// rows in -- it has to be by line for the ranges to be unioned -- and it is the
// tie-break that keeps tied findings out of the order they arrived in.
func compareIdentity(a, b ScoredFinding) int {
if order := cmp.Or(
cmp.Compare(a.File, b.File),
cmp.Compare(a.Category, b.Category),
cmp.Compare(a.LineStart, b.LineStart),
cmp.Compare(a.LineEnd, b.LineEnd),
cmp.Compare(string(a.Severity), string(b.Severity)),
cmp.Compare(a.Dimension, b.Dimension),
cmp.Compare(a.Title, b.Title),
cmp.Compare(a.Body, b.Body),
cmp.Compare(a.Suggestion, b.Suggestion),
cmp.Compare(a.Evidence, b.Evidence),
cmp.Compare(a.Confidence, b.Confidence),
slices.Compare(a.Tags, b.Tags),
slices.Compare(a.Multipliers, b.Multipliers),
); order != 0 {
return order
}
// The gating finding comes first, so a pair that differs in nothing else
// still has one order.
if a.Blocking == b.Blocking {
return 0
}
if a.Blocking {
return -1
}
return 1
}
// scoreOf applies the rubric: the severity's weight times the reviewer's
// confidence, then every multiplier the finding and the run activate. The
// order is fixed, because the same findings have to produce the same score.
func scoreOf(finding Finding, severity Severity, inputs ScoreInputs) (float64, []string) {
score := severityWeight(severity) * finding.Confidence
var applied []string
if finding.Compound {
score *= multiplierCompound
applied = append(applied, MultiplierCompound)
}
switch finding.Adversary {
case AdversaryConfirmed:
score *= multiplierAdversaryConfirmed
applied = append(applied, MultiplierAdversaryConfirmed)
case AdversaryChallenged:
score *= multiplierAdversaryChallenged
applied = append(applied, MultiplierAdversaryChallenged)
}
if inputs.AIGenerated > AIGeneratedThreshold {
score *= multiplierAIGenerated
applied = append(applied, MultiplierAIGenerated)
}
if inputs.BlastRadiusFiles > BlastRadiusFileThreshold {
score *= multiplierBlastRadiusHigh
applied = append(applied, MultiplierBlastRadiusHigh)
}
return round3(score), applied
}
// round3 rounds a score to three decimals. A score is read by humans and
// compared by the sort, and a full float64 mantissa invites ties that are not
// ties.
func round3(score float64) float64 { return math.Round(score*1000) / 1000 }
// severityWeight is the rubric's weight for a severity.
func severityWeight(severity Severity) float64 {
switch severity {
case SeverityCritical:
return 1
case SeverityImportant:
return 0.7
case SeveritySuggestion:
return 0.3
default:
return 0.1
}
}
// confidenceFloor is the confidence under which a finding of this severity is
// dropped: a vague crash is worth a look, a vague nitpick is not worth a
// reader's time.
func confidenceFloor(severity Severity) float64 {
switch severity {
case SeverityCritical:
return 0.2
case SeverityImportant:
return 0.3
default:
return 0.4
}
}
// severityAliases maps the words a reviewer model emits onto the rubric.
var severityAliases = map[Severity]Severity{
Severity("high"): SeverityCritical,
Severity("blocker"): SeverityCritical,
Severity("medium"): SeverityImportant,
Severity("major"): SeverityImportant,
Severity("minor"): SeveritySuggestion,
Severity("low"): SeveritySuggestion,
Severity("info"): SeverityNitpick,
Severity("trivia"): SeverityNitpick,
Severity("trivial"): SeverityNitpick,
}
// normalizeSeverity maps a severity to one of the four rubric words;
// unknown words become suggestion.
func normalizeSeverity(severity Severity) Severity {
lowered := Severity(strings.ToLower(strings.TrimSpace(string(severity))))
switch lowered {
case SeverityCritical, SeverityImportant, SeveritySuggestion, SeverityNitpick:
return lowered
}
if canonical, known := severityAliases[lowered]; known {
return canonical
}
return SeveritySuggestion
}
// CapInlineComments cuts a ranked list down to the comments one review posts.
// It is applied to the ranked list, so what it cuts is the lowest-scoring tail.
func CapInlineComments(findings []ScoredFinding) []ScoredFinding {
if len(findings) > MaxInlineComments {
return findings[:MaxInlineComments]
}
return findings
}
// IsAIGenerated reports whether intake's machine-written confidence clears
// AIGeneratedThreshold: a coin-flip is not evidence, so it takes strictly
// more than the threshold to count.
func IsAIGenerated(confidence float64) bool {
return confidence > AIGeneratedThreshold
}
func IsHighPriority(f Finding) bool {
switch normalizeSeverity(f.Severity) {
case SeverityCritical, SeverityImportant:
return true
default:
return false
}
}
// ReviewEventFor returns REQUEST_CHANGES when any finding blocks, else
// COMMENT. It never approves.
func ReviewEventFor(findings []ScoredFinding) ReviewEvent {
for _, finding := range findings {
if finding.Blocking {
return ReviewEventRequestChanges
}
}
return ReviewEventComment
}