feat(prreview): benchmark harness scoring findings on the Martian problems

Scores a reviewer's findings on the 38 runnable Martian Code-Review-Bench
PRs: the deterministic core turns findings into posted comments, an
independent judge model matches them to the goldens, and the run reports
golden micro-recall and golden-only precision. The reviewer is a findings
fixture directory until the pr-review workflow lands.

Run with: task bench:prreview -- -findings DIR
This commit is contained in:
sam committed 2026-09-26 10:30:12 +10:00
1 parent 5c4e6351a0
commit 1c7dd08f43
11 files changed
+1379 -1

No files matched your search

+5
View File
@@ -153,6 +153,11 @@ tasks:
cmds:
- bash tools/webhook-send.sh {{.CLI_ARGS}}
bench:prreview:
desc: Score PR review findings on the Martian benchmark (-- -findings DIR; --help for flags)
cmds:
- go run ./cmd/archie-prbench {{.CLI_ARGS}}
dev:
desc: Run development services and Vite at http://localhost:5173
deps: [dev:daemon, dev:gateway, dev:state-store, dev:dashboard, dev:ui]
+41
View File
@@ -0,0 +1,41 @@
// Command archie-prbench scores the PR reviewer on the Martian
// Code-Review-Bench problems. See docs/prds/pr-review-agent.md, Verification.
package main
import (
"context"
"flag"
"fmt"
"os"
"os/signal"
"strings"
"syscall"
"github.com/samcharles93/archie-core/internal/app/prbench"
"github.com/samcharles93/archie-core/internal/infrastructure/configuration"
)
func main() { os.Exit(run()) }
func run() int {
var opts prbench.Options
var problems string
flag.StringVar(&opts.Config, "config", configuration.DefaultConfigPath(), "configuration file whose providers serve the judge model")
flag.StringVar(&opts.Fixtures, "findings", "", "directory of <problem id>.json findings to score")
flag.StringVar(&opts.JudgeModel, "judge-model", "anthropic/claude-sonnet-4.6", "provider/model of the independent judge")
flag.StringVar(&opts.Out, "out", "prbench-out", "directory for per-problem results and summary.json")
flag.StringVar(&problems, "problems", "", "comma-separated problem ids (default: all)")
flag.IntVar(&opts.Concurrency, "concurrency", 4, "problems judged at once")
flag.Parse()
if problems != "" {
opts.Problems = strings.Split(problems, ",")
}
ctx, stop := signal.NotifyContext(context.Background(), os.Interrupt, syscall.SIGTERM)
defer stop()
if _, err := prbench.Run(ctx, opts, os.Stdout); err != nil {
fmt.Fprintln(os.Stderr, "archie-prbench:", err)
return 1
}
return 0
}
+2 -1
View File
@@ -125,7 +125,8 @@ func TestPullRequestsRunTheDefinitiveGateWithoutPublishAccess(t *testing.T) {
// Naming each one with its reason keeps an omission a decision rather than an
// oversight -- which is exactly how archied's channels were lost.
var unpackagedCommands = map[string]string{
"archie-agent": "runs only inside the Docker image build-and-push publishes, never as a host binary",
"archie-agent": "runs only inside the Docker image build-and-push publishes, never as a host binary",
"archie-prbench": "a developer benchmark run from the repository with task bench:prreview",
}
// cliOnlyCommands ship in the zip but must never be restarted by the updater:
+10
View File
@@ -109,3 +109,13 @@ func providerSecretEnvName(scope, providerID string) string {
sum := sha256.Sum256([]byte(scope + "\x00" + providerID))
return fmt.Sprintf("ARCHIE_PROVIDER_%X_API_KEY", sum[:8])
}
// ResolveProviders resolves cfg's provider credentials the way the daemon does
// at boot, for a tool that calls models outside the daemon.
func ResolveProviders(cfg *config.Config, log *slog.Logger) error {
secrets, err := configuredSecretRegistry(cfg, log)
if err != nil {
return err
}
return resolveProviderSecrets(cfg, secrets, log)
}
+67
View File
@@ -0,0 +1,67 @@
package prbench
import (
"context"
"encoding/json"
"fmt"
"os"
"path/filepath"
"github.com/samcharles93/archie-core/internal/domain/workflow/prreview"
"github.com/samcharles93/archie-core/internal/domain/workflow/prreview/bench"
)
// fixtureReviewer reads each problem's findings from <dir>/<id>.json, so the
// deterministic core and the judge can be measured before a live reviewer
// exists, and a live run's findings can be rescored without reviewing again.
type fixtureReviewer struct{ dir string }
type fixture struct {
AIGenerated float64 `json:"ai_generated"`
BlastRadiusFiles int `json:"blast_radius_files"`
Findings []fixtureFinding `json:"findings"`
}
type fixtureFinding struct {
Dimension string `json:"dimension"`
File string `json:"file"`
LineStart int `json:"line_start"`
LineEnd int `json:"line_end"`
Severity string `json:"severity"`
Category string `json:"category"`
Title string `json:"title"`
Body string `json:"body"`
Suggestion string `json:"suggestion"`
Evidence string `json:"evidence"`
Confidence float64 `json:"confidence"`
Tags []string `json:"tags"`
Adversary string `json:"adversary"`
Compound bool `json:"compound"`
Blocking bool `json:"blocking"`
}
func (f fixtureReviewer) Review(_ context.Context, p bench.Problem) (bench.Review, error) {
data, err := os.ReadFile(filepath.Join(f.dir, p.ID+".json"))
if err != nil {
return bench.Review{}, err
}
var fx fixture
if err := json.Unmarshal(data, &fx); err != nil {
return bench.Review{}, fmt.Errorf("decode %s findings: %w", p.ID, err)
}
review := bench.Review{Inputs: prreview.ScoreInputs{AIGenerated: fx.AIGenerated, BlastRadiusFiles: fx.BlastRadiusFiles}}
for i, ff := range fx.Findings {
switch prreview.Severity(ff.Severity) {
case prreview.SeverityCritical, prreview.SeverityImportant, prreview.SeveritySuggestion, prreview.SeverityNitpick:
default:
return bench.Review{}, fmt.Errorf("%s finding %d: unknown severity %q", p.ID, i, ff.Severity)
}
review.Findings = append(review.Findings, prreview.Finding{
Dimension: ff.Dimension, File: ff.File, LineStart: ff.LineStart, LineEnd: ff.LineEnd,
Severity: prreview.Severity(ff.Severity), Category: ff.Category, Title: ff.Title, Body: ff.Body,
Suggestion: ff.Suggestion, Evidence: ff.Evidence, Confidence: ff.Confidence, Tags: ff.Tags,
Adversary: prreview.AdversaryVerdict(ff.Adversary), Compound: ff.Compound, Blocking: ff.Blocking,
})
}
return review, nil
}
+90
View File
@@ -0,0 +1,90 @@
package prbench
import (
"context"
"fmt"
"strings"
"github.com/samcharles93/ai-sdk/agent"
"github.com/samcharles93/ai-sdk/core"
"github.com/samcharles93/ai-sdk/runtime"
"github.com/samcharles93/archie-core/internal/domain/workflow/prreview"
"github.com/samcharles93/archie-core/internal/domain/workflow/prreview/bench"
)
// modelJudge asks one model, independent of the reviewer's, which posted
// comment catches each golden.
type modelJudge struct {
runtime *runtime.Runtime
modelRef string
}
func (j *modelJudge) Judge(ctx context.Context, p bench.Problem, comments []prreview.ScoredFinding) ([]bench.Verdict, error) {
provider, model, err := j.runtime.ChatProvider(ctx, j.modelRef)
if err != nil {
return nil, fmt.Errorf("resolve judge model %q: %w", j.modelRef, err)
}
var verdicts []bench.Verdict
sub := agent.Subagent{
Provider: provider,
Model: model,
System: judgeSystemPrompt,
Tools: core.ToolSet{"record_verdict": recordVerdictTool(len(p.Goldens), len(comments), &verdicts)},
MaxSteps: 2*len(p.Goldens) + 3,
}
// The verdicts are the judgement, not the closing reply: a run that recorded
// some and then ran out of steps is checked for completeness by the caller.
if _, err := sub.Run(ctx, judgePrompt(p, comments)); err != nil && len(verdicts) == 0 {
return nil, err
}
return verdicts, nil
}
const judgeSystemPrompt = `You score a code reviewer against known defects. You are given the pull request's known defects (goldens) and the numbered comments the reviewer posted.
For each golden, decide whether any posted comment identifies the same underlying defect: the same location and the same root cause. A comment about the same file but a different problem is not a match, and neither is a vague comment that would only be right by coincidence. Wording does not need to match.
Call record_verdict exactly once per golden. Pass the index of the matching comment, or -1 when no comment matches, and a one-sentence reason. When several comments match, pass the one that states the defect most directly.`
func judgePrompt(p bench.Problem, comments []prreview.ScoredFinding) string {
var b strings.Builder
fmt.Fprintf(&b, "Pull request: %s\n\n<goldens>\n", p.PRURL)
for i, g := range p.Goldens {
fmt.Fprintf(&b, "[%d] (%s) %s\n", i, g.Severity, g.Comment)
}
b.WriteString("</goldens>\n\n<comments>\n")
for i, c := range comments {
fmt.Fprintf(&b, "[%d] %s:%d-%d (%s) %s\n%s\n\n", i, c.File, c.LineStart, c.LineEnd, c.Severity, c.Title, c.Body)
}
b.WriteString("</comments>\n")
return b.String()
}
type verdictInput struct {
Golden int `json:"golden" jsonschema:"description=Index of the golden this verdict is for."`
Comment int `json:"comment" jsonschema:"description=Index of the comment that identifies the same defect, or -1 when none does."`
Reason string `json:"reason" jsonschema:"description=One sentence on why the comment does or does not match."`
}
func recordVerdictTool(goldens, comments int, verdicts *[]bench.Verdict) *core.Tool {
return core.NewTypedTool(
"record_verdict",
"Record whether a posted comment catches one golden. Call once per golden.",
func(_ context.Context, in verdictInput) (string, error) {
if in.Golden < 0 || in.Golden >= goldens {
return fmt.Sprintf("record_verdict rejected: golden must be 0..%d", goldens-1), nil
}
if in.Comment < bench.NoComment || in.Comment >= comments {
return fmt.Sprintf("record_verdict rejected: comment must be -1..%d", comments-1), nil
}
for _, v := range *verdicts {
if v.Golden == in.Golden {
return fmt.Sprintf("record_verdict rejected: golden %d already has a verdict", in.Golden), nil
}
}
*verdicts = append(*verdicts, bench.Verdict{Golden: in.Golden, Comment: in.Comment, Reason: in.Reason})
return "verdict recorded", nil
},
)
}
+638
View File
@@ -0,0 +1,638 @@
[
{
"id": "cal_dot_com#10600",
"pr_url": "https://github.com/calcom/cal.com/pull/10600",
"goldens": [
{
"comment": "The exported function TwoFactor handles backup codes and is in BackupCode.tsx. Inconsistent naming.",
"severity": "Low"
},
{
"comment": "Error message mentions 'backup code login' but this is a disable endpoint, not login",
"severity": "Low"
},
{
"comment": "Backup code validation is case-sensitive due to the use of indexOf(). This causes validation to fail if a user enters uppercase hex characters, as backup codes should be case-insensitive for a better user experience.",
"severity": "Medium"
},
{
"comment": "Because backupCodes are decrypted and mutated in memory before being written back, two concurrent login requests using the same backupCode could both pass this check and update, so a single backup code may effectively be accepted more than once if used concurrently, weakening the intended one-time-use semantics.",
"severity": "High"
}
]
},
{
"id": "cal_dot_com#10967",
"pr_url": "https://github.com/calcom/cal.com/pull/10967",
"goldens": [
{
"comment": "Potential null reference if mainHostDestinationCalendar is undefined if evt.destinationCalendar is null or an empty array ",
"severity": "High"
},
{
"comment": "The optional chaining on mainHostDestinationCalendar?.integration is redundant since you already check mainHostDestinationCalendar in the ternary condition.",
"severity": "Low"
},
{
"comment": "Logic error: when externalCalendarId is provided, you're searching for a calendar where externalId === externalCalendarId, but this will always fail since you're looking for a calendar that matches itself. Should likely find by credentialId or use different logic.",
"severity": "High"
},
{
"comment": "Logic inversion in organization creation: The slug property is now conditionally set when IS_TEAM_BILLING_ENABLED is true, instead of when it's false as originally intended. This change, combined with requestedSlug still being set when IS_TEAM_BILLING_ENABLED is true, results in both properties being set when billing is enabled, and neither when disabled",
"severity": "Medium"
},
{
"comment": "The Calendar interface now requires createEvent(event, credentialId), but some implementations (e.g., Lark/Office365) still declare createEvent(event) only—this breaks the interface contract (also applies to other locations in the PR).",
"severity": "Low"
}
]
},
{
"id": "cal_dot_com#11059",
"pr_url": "https://github.com/calcom/cal.com/pull/11059",
"goldens": [
{
"comment": "The parseRefreshTokenResponse function incorrectly sets refresh_token to the hardcoded string 'refresh_token' when it's missing from the OAuth refresh token response. This invalidates the token, breaking subsequent token refreshes and causing authentication failures.",
"severity": "High"
},
{
"comment": "Invalid Zod schema syntax. Computed property keys like [z.string().toString()] are not valid in Zod object schemas and will cause runtime errors. ",
"severity": "High"
},
{
"comment": "parseRefreshTokenResponse returns a Zod safeParse result ({ success, data, error }), not the credential key object. Persisting that as key stores the wrapper instead of the token payload; we should store the parsed data or use schema parse.",
"severity": "High"
},
{
"comment": "When APP_CREDENTIAL_SHARING_ENABLED and CALCOM_CREDENTIAL_SYNC_ENDPOINT are set, the refreshFunction helper returns the fetch Response, but several callers (for example GoogleCalendarService.refreshAccessToken expecting res.data, and HubspotCalendarService.refreshAccessToken expecting a HubspotToken) assume it returns the integration-specific token object. That mismatch will cause runtime errors in the sync-enabled path unless the return type or those call sites are adjusted.",
"severity": "High"
},
{
"comment": "When the sync endpoint path is used, res is a fetch Response and has no .data; res?.data will be undefined and token.access_token will throw at runtime. This relies on a consistent return shape from refreshOAuthTokens, which isn’t guaranteed currently.",
"severity": "High"
}
]
},
{
"id": "cal_dot_com#14740",
"pr_url": "https://github.com/calcom/cal.com/pull/14740",
"goldens": [
{
"comment": "Case sensitivity bypass in email blacklist",
"severity": "High"
},
{
"comment": "The logic for checking team admin/owner permissions is incorrect. This condition uses AND (&&) which requires both isTeamAdmin AND isTeamOwner to be true, but it should use OR (||) since a user needs to be either an admin OR an owner to have permission.",
"severity": "Critical"
},
{
"comment": "This calls the email sender with the original guests, so existing attendees included in the input will be treated as new when sending notifications, leading to incorrect emails.",
"severity": "Medium"
},
{
"comment": "uniqueGuests filters out existing attendees and blacklisted emails but does not deduplicate duplicates within the input; createMany can insert duplicate attendee rows if the client submits repeated emails.",
"severity": "Medium"
},
{
"comment": "Starting with an array containing an empty string may cause validation issues. Consider starting with an empty array [] and handling the empty state in the MultiEmail component instead.",
"severity": "Low"
}
]
},
{
"id": "cal_dot_com#14943",
"pr_url": "https://github.com/calcom/cal.com/pull/14943",
"goldens": [
{
"comment": "Using retryCount: reminder.retryCount + 1 reads a possibly stale value and can lose increments under concurrency; consider an atomic increment via Prisma (increment: 1) to avoid race conditions (also applies to the similar update in the catch block).",
"severity": "High"
},
{
"comment": "The deletion logic in scheduleSMSReminders.ts incorrectly deletes non-SMS workflow reminders (e.g., Email, WhatsApp) that have retryCount > 1. This occurs because the retryCount condition within the OR clause for deletion lacks a method: WorkflowMethods.SMS filter, causing it to apply to all reminder types instead of only SMS reminders, which is the intended scope of this function.",
"severity": "High"
}
]
},
{
"id": "cal_dot_com#22345",
"pr_url": "https://github.com/calcom/cal.com/pull/22345",
"goldens": [
{
"comment": "In getBaseConditions(), the else if (filterConditions) and final else branches are unreachable. This is because getAuthorizationConditions() always returns a non-null Prisma.Sql object, making authConditions always truthy, which means only the first two if/else if conditions are ever evaluated.",
"severity": "Low"
},
{
"comment": "Fetching userIdsFromOrg only when teamsFromOrg.length > 0 can exclude org-level members for orgs without child teams; consider deriving from teamIds (which includes orgId) or removing the guard so org-only orgs still include member user bookings.",
"severity": "Medium"
}
]
},
{
"id": "cal_dot_com#22532",
"pr_url": "https://github.com/calcom/cal.com/pull/22532",
"goldens": [
{
"comment": "The updateManyByCredentialId call uses an empty data object, which prevents Prisma's @updatedAt decorator from updating the updatedAt timestamp. This results in inaccurate cache status tracking, as the timestamp isn't updated when the cache is refreshed. To fix this, explicitly set the updatedAt field.",
"severity": "Medium"
},
{
"comment": "logic: macOS-specific sed syntax with empty string after -i flag will fail on Linux systems",
"severity": "Low"
}
]
},
{
"id": "cal_dot_com#7232",
"pr_url": "https://github.com/calcom/cal.com/pull/7232",
"goldens": [
{
"comment": "Asynchronous functions deleteScheduledEmailReminder and deleteScheduledSMSReminder are called without await inside forEach loops. This occurs during booking rescheduling/cancellation, and workflow/workflow step deletion/updates. Consequently, scheduled workflow reminders may not be reliably cancelled, potentially leaving them active.",
"severity": "Medium"
},
{
"comment": "When immediateDelete is true, the deleteScheduledEmailReminder function cancels the SendGrid email but fails to delete the corresponding WorkflowReminder record from the database. This creates orphaned database entries and is inconsistent with the immediateDelete: false path, which marks the record as cancelled. The SendGrid DELETE API call is also omitted in this path.",
"severity": "High"
}
]
},
{
"id": "cal_dot_com#8087",
"pr_url": "https://github.com/calcom/cal.com/pull/8087",
"goldens": [
{
"comment": "Consider adding try-catch around the await to handle import failures gracefully",
"severity": "Low"
},
{
"comment": "The code uses forEach with async callbacks, which causes asynchronous operations (e.g., calendar/video event deletions, payment refunds) to run concurrently without being awaited. This 'fire-and-forget' behavior leads to unhandled promise rejections, race conditions, and incomplete cleanup, as surrounding try-catch blocks cannot properly handle errors from these unawaited promises. Replace forEach with for...of loops or Promise.all() with map() to ensure proper sequential execution and error handling.",
"severity": "Critical"
}
]
},
{
"id": "cal_dot_com#8330",
"pr_url": "https://github.com/calcom/cal.com/pull/8330",
"goldens": [
{
"comment": "Incorrect end time calculation using slotStartTime instead of slotEndTime",
"severity": "Medium"
},
{
"comment": "Using === for dayjs object comparison will always return false as it compares object references, not values. Use .isSame() method instead: dayjs(date.start).add(utcOffset, 'minutes').isSame(dayjs(date.end).add(utcOffset, minutes))",
"severity": "Medium"
}
]
},
{
"id": "grafana#103633",
"pr_url": "https://github.com/grafana/grafana/pull/103633",
"goldens": [
{
"comment": "The Check operation exhibits asymmetric cache trust logic: cached permission grants are trusted and returned immediately, but cached denials from the same permission cache are ignored, leading to a fresh database lookup. This allows stale cached grants to provide access to revoked resources, posing a security risk. ",
"severity": "High"
},
{
"comment": "The test comment says the cached permissions 'allow access', but the map stores false for dashboards:uid:dash1, so checkPermission will still treat this scope as not allowed.",
"severity": "Low"
}
]
},
{
"id": "grafana#106778",
"pr_url": "https://github.com/grafana/grafana/pull/106778",
"goldens": [
{
"comment": "The rendered GrafanaRuleListItem is missing the required key prop for React list items. This can cause rendering issues when the list order changes.",
"severity": "Medium"
},
{
"comment": "RuleActionsButtons is invoked with only promRule, but SilenceGrafanaRuleDrawer inside RuleActionsButtons still depends on a Grafana Ruler rule being present, so for Grafana rules coming from list views the 'Silence notifications' menu entry (now driven by Grafana Prom abilities) will toggle showSilenceDrawer without ever rendering the drawer. This means clicking 'Silence notifications' for these rules has no visible effect, even when abilities indicate silencing is allowed.",
"severity": "High"
}
]
},
{
"id": "grafana#107534",
"pr_url": "https://github.com/grafana/grafana/pull/107534",
"goldens": [
{
"comment": "The applyTemplateVariables method is called with request.filters as the third parameter, but this parameter is not used in the corresponding test setup.",
"severity": "Low"
}
]
},
{
"id": "grafana#76186",
"pr_url": "https://github.com/grafana/grafana/pull/76186",
"goldens": [
{
"comment": "The ContextualLoggerMiddleware methods (QueryData, CallResource, CheckHealth, CollectMetrics) panic when a nil request is received. This occurs because they directly access req.PluginContext (via the instrumentContext function) without first checking if req is nil. This is a regression, as previous middleware layers gracefully handled nil requests.",
"severity": "High"
},
{
"comment": "The traceID is no longer logged for plugin requests. During a refactoring, the tracing import and the logic to extract and add traceID from the context to log parameters were removed from the LoggerMiddleware. The newly introduced ContextualLoggerMiddleware does not add this information, resulting in missing traceID in plugin request logs and impacting debugging and request tracing capabilities.",
"severity": "Low"
}
]
},
{
"id": "grafana#79265",
"pr_url": "https://github.com/grafana/grafana/pull/79265",
"goldens": [
{
"comment": "Race condition: Multiple concurrent requests could pass the device count check simultaneously and create devices beyond the limit. Consider using a database transaction or lock.",
"severity": "High"
},
{
"comment": "Anonymous authentication now fails entirely if anonDeviceService.TagDevice returns ErrDeviceLimitReached. Previously, device tagging was asynchronous and non-blocking. This change prevents anonymous users from authenticating when the device limit is reached.",
"severity": "Medium"
},
{
"comment": "This call won’t compile: dbSession.Exec(args...) is given a []interface{} where the first element is the query, but Exec’s signature requires a first parameter of type string (not an interface{} splat).",
"severity": "Medium"
},
{
"comment": "Returning ErrDeviceLimitReached when no rows were updated is misleading; the device might not exist.",
"severity": "Low"
},
{
"comment": "Time window calculation inconsistency: Using device.UpdatedAt.UTC().Add(-anonymousDeviceExpiration) as the lower bound but device.UpdatedAt as the current time may not match the intended logic. Consider using time.Now().UTC() consistently.",
"severity": "Low"
}
]
},
{
"id": "grafana#80329",
"pr_url": "https://github.com/grafana/grafana/pull/80329",
"goldens": [
{
"comment": "The code uses Error log level for what appears to be debugging information. This will pollute error logs in production. Consider using Debug or Info level instead.",
"severity": "Low"
}
]
},
{
"id": "grafana#90045",
"pr_url": "https://github.com/grafana/grafana/pull/90045",
"goldens": [
{
"comment": "The context is being created with d.Log instead of the log variable that was initialized with additional context values (name, kind, method). This means those values won't be propagated to the logging context.",
"severity": "Medium"
},
{
"comment": "Bug: calling recordLegacyDuration when storage operation fails should be recordStorageDuration.",
"severity": "High"
},
{
"comment": "Inconsistency: using name instead of options.Kind for metrics recording differs from other methods.",
"severity": "Medium"
}
]
},
{
"id": "grafana#90939",
"pr_url": "https://github.com/grafana/grafana/pull/90939",
"goldens": [
{
"comment": "The GetWebAssets function implements an incomplete double-checked locking pattern for caching web assets. The function first checks if the cache is populated using a read lock (RLock), and if the cache is empty, it acquires a write lock to populate it. However, it fails to re-check whether the cache was populated by another goroutine while waiting to acquire the write lock.",
"severity": "Medium"
},
{
"comment": "In addition to the missing double-check, the function has a critical flaw in its error handling: it unconditionally assigns the fetch result to the cache (line 69: entryPointAssetsCache = result) regardless of whether the fetch succeeded or failed. When an error occurs during asset fetching, result is nil, and this nil value overwrites any previously valid cache entry.",
"severity": "High"
}
]
},
{
"id": "grafana#94942",
"pr_url": "https://github.com/grafana/grafana/pull/94942",
"goldens": [
{
"comment": "The enableSqlExpressions function has flawed logic that always returns false, effectively disabling SQL expressions unconditionally:",
"severity": "Critical"
},
{
"comment": "Several methods such as NewInMemoryDB().RunCommands and db.QueryFramesInto return 'not implemented'.",
"severity": "High"
}
]
},
{
"id": "grafana#97529",
"pr_url": "https://github.com/grafana/grafana/pull/97529",
"goldens": [
{
"comment": "A race condition in BuildIndex allows multiple goroutines to concurrently build the same expensive index for the same key. This is caused by moving the b.cacheMu lock from protecting the entire function to only protecting the final cache assignment. ",
"severity": "High"
},
{
"comment": "Calling s.search.TotalDocs() here may race with concurrent index creation: TotalDocs iterates b.cache without synchronization, and the event watcher goroutine started just above could trigger BuildIndex writes concurrently, potentially causing a concurrent map read/write panic.",
"severity": "High"
}
]
},
{
"id": "keycloak#32918",
"pr_url": "https://github.com/keycloak/keycloak/pull/32918",
"goldens": [
{
"comment": "Recursive caching call using session instead of delegate",
"severity": "Critical"
},
{
"comment": "Cleanup reference uses incorrect alias - should be 'idp-alias-' + i instead of 'alias'.",
"severity": "Medium"
}
]
},
{
"id": "keycloak#33832",
"pr_url": "https://github.com/keycloak/keycloak/pull/33832",
"goldens": [
{
"comment": "Returns wrong provider (default keystore instead of BouncyCastle)",
"severity": "High"
},
{
"comment": "Dead code exists where ASN1Encoder instances are created and written to, but their results are immediately discarded. The actual encoding is performed by new ASN1Encoder instances created in the subsequent return statement, rendering the earlier operations useless.",
"severity": "Low"
}
]
},
{
"id": "keycloak#36880",
"pr_url": "https://github.com/keycloak/keycloak/pull/36880",
"goldens": [
{
"comment": "Inconsistent feature flag bug causing orphaned permissions. The AdminPermissions event listener, responsible for cleaning up permissions upon role, client, or group removal, is incorrectly guarded by the ADMIN_FINE_GRAINED_AUTHZ (V1) feature flag. This is inconsistent with other methods in the class that use ADMIN_FINE_GRAINED_AUTHZ_V2. Consequently, if ADMIN_FINE_GRAINED_AUTHZ_V2 is enabled but V1 is not, the permission cleanup logic will not execute, leading to orphaned permission data. Cleanup should occur regardless of which fine-grained authorization version is enabled.",
"severity": "High"
},
{
"comment": "In hasPermission(ClientModel client, String scope), the resource lookup uses findByName(server, client.getId(), server.getId()), but AdminPermissionsSchema.getOrCreateResource creates per-client resources with the owner set to resourceServer.getClientId(), so this lookup will never find those resources and will always fall back to the 'all-clients' resource, effectively ignoring client-specific permissions.",
"severity": "High"
},
{
"comment": "In getClientsWithPermission(String scope), iterating resourceStore.findByType(server, AdminPermissionsSchema.CLIENTS_RESOURCE_TYPE) and returning resource.getName() will only ever consider the type-level 'Clients' resource (per-client resources have no type) and return its name, while AvailableRoleMappingResource#getRoleIdsWithPermissions expects actual client IDs to pass to realm.getClientById, which can lead to incorrect behavior or a null client and subsequent failures.",
"severity": "High"
}
]
},
{
"id": "keycloak#36882",
"pr_url": "https://github.com/keycloak/keycloak/pull/36882",
"goldens": [
{
"comment": "Incorrect method call for exit codes. The picocli.exit() method calls System.exit() directly, which is problematic:",
"severity": "Medium"
}
]
},
{
"id": "keycloak#37038",
"pr_url": "https://github.com/keycloak/keycloak/pull/37038",
"goldens": [
{
"comment": "Incorrect permission check in canManage() method",
"severity": "High"
},
{
"comment": "In getGroupIdsWithViewPermission, hasPermission is called with groupResource.getId() and the same groupResource.getId() is added to granted, but hasPermission resolves resources by name (treating the argument as a group id) and the GroupPermissionEvaluator contract says this method returns group IDs that are later used as UserModel.GROUPS and in getUsersCount group filters. This mismatch means per-group VIEW_MEMBERS/MANAGE_MEMBERS permissions may not yield the expected group IDs for filtering and counts, and evaluation may effectively only look at the type-level 'all-groups' resource; consider revisiting whether this should operate on the underlying group ids (resource names) instead so it aligns with the JPA queries and the interface contract.",
"severity": "High"
}
]
},
{
"id": "keycloak#37429",
"pr_url": "https://github.com/keycloak/keycloak/pull/37429",
"goldens": [
{
"comment": "The translation is in Italian instead of Lithuanian. This should be translated to Lithuanian to match the file's locale (messages_lt.properties).",
"severity": "Medium"
},
{
"comment": "The totpStep1 value uses Traditional Chinese terms in the Simplified Chinese file (zh_CN), which is likely incorrect for this locale. Please verify the locale‑appropriate translation.",
"severity": "Medium"
},
{
"comment": "The anchor sanitization logic has a potential issue where it consumes English matcher groups without proper validation. If the translated text has more anchor tags than the English text, this could lead to incorrect validation results.",
"severity": "Low"
},
{
"comment": "The method name 'santizeAnchors' should be 'sanitizeAnchors' (missing 'i').",
"severity": "Low"
}
]
},
{
"id": "keycloak#37634",
"pr_url": "https://github.com/keycloak/keycloak/pull/37634",
"goldens": [
{
"comment": "Wrong parameter in null check (grantType vs. rawTokenId)",
"severity": "Critical"
},
{
"comment": "In isAccessTokenId, the substring for the grant shortcut and the equality check look inverted: the grant shortcut occupies indices 4–5 (substring(4,6)), and a match should return true (combined with UUID check), not false.",
"severity": "High"
},
{
"comment": "Javadoc mentions \"usually like 3-letters shortcut\" but some implementations use 2-letter shortcuts (\"ac\", \"cc\", \"rt\", \"te\", \"pc\", \"ci\", \"ro\"). Consider updating documentation to reflect actual usage pattern.",
"severity": "Low"
},
{
"comment": " Catching generic RuntimeException is too broad. The implementation throws IllegalArgumentException specifically - catch that instead for more precise testing.",
"severity": "Low"
}
]
},
{
"id": "keycloak#38446",
"pr_url": "https://github.com/keycloak/keycloak/pull/38446",
"goldens": [
{
"comment": "Unsafe raw List deserialization without type safety. Calling Optional.get() directly on the Optional returned by RecoveryAuthnCodesUtils.getCredential(user) without checking isPresent() can lead to a NoSuchElementException if the Optional is empty.",
"severity": "Medium"
},
{
"comment": "After creating the RecoveryAuthnCodesCredentialModel, consider setting its id from the stored credential (e.g., myUser.recoveryCodes.getId()); otherwise getId() will be null and downstream removal by id (e.g., removeStoredCredentialById in the authenticator flow) may not work.",
"severity": "Low"
}
]
},
{
"id": "keycloak#40940",
"pr_url": "https://github.com/keycloak/keycloak/pull/40940",
"goldens": [
{
"comment": "Returning null from getSubGroupsCount() violates the GroupModel contract (Javadoc says it never returns null) and may lead to NPEs in callers that expect a non-null count.",
"severity": "Critical"
},
{
"comment": "The reader thread isn’t waited for; flipping deletedAll to true and asserting immediately can race and miss exceptions added just after the flag change, making this test flaky.",
"severity": "Medium"
}
]
},
{
"id": "keycloak#41249",
"pr_url": "https://github.com/keycloak/keycloak/pull/41249",
"goldens": [
{
"comment": "ConditionalPasskeysEnabled() called without UserModel parameter",
"severity": "Medium"
},
{
"comment": "With isConditionalPasskeysEnabled(UserModel user) requiring user != null, authenticate(...) will not call webauthnAuth.fillContextForm(context) on the initial login page where context.getUser() is still null, so conditional passkey UI will not be set up for first-time passkey login. Consider whether this should also be enabled when no user has been selected yet so ID-less passkey authentication on the initial login form continues to work.",
"severity": "Medium"
}
]
},
{
"id": "sentry#67876",
"pr_url": "https://github.com/getsentry/sentry/pull/67876",
"goldens": [
{
"comment": "Null reference if github_authenticated_user state is missing",
"severity": "Medium"
},
{
"comment": "OAuth state uses pipeline.signature (static) instead of a per-request random value",
"severity": "Medium"
},
{
"comment": "The code attempts to access integration.metadata[sender][login] without checking for the existence of the sender key. This causes a KeyError for integrations where the sender metadata was not set during creation",
"severity": "High"
}
]
},
{
"id": "sentry#77754",
"pr_url": "https://github.com/getsentry/sentry/pull/77754",
"goldens": [
{
"comment": "Shared mutable default in dataclass timestamp",
"severity": "Medium"
},
{
"comment": "The method name has a typo: test_from_dict_inalid_data should be test_from_dict_invalid_data.",
"severity": "Low"
},
{
"comment": "Method name says 'empty_array' but tests empty dict - consider renaming to 'test_from_dict_empty_dict' for clarity.",
"severity": "Low"
},
{
"comment": "to_dict() returns a datetime for queued; if this dict is passed in task kwargs (e.g., via apply_async), JSON serialization may fail depending on the serializer, which can cause enqueue errors.",
"severity": "Medium"
}
]
},
{
"id": "sentry#80168",
"pr_url": "https://github.com/getsentry/sentry/pull/80168",
"goldens": [
{
"comment": "MetricAlertDetectorHandler inherits from StatefulDetectorHandler but only contains pass, failing to implement its required abstract methods: counter_names (property), get_dedupe_value(), get_group_key_values(), and build_occurrence_and_event_data(). This will cause a TypeError at runtime when the class is instantiated.",
"severity": "High"
},
{
"comment": "Docstring says this returns a list of DetectorEvaluationResult, but the method now returns a dict keyed by DetectorGroupKey. Consider updating the docstring to match the new return type.",
"severity": "Low"
}
]
},
{
"id": "sentry#80528",
"pr_url": "https://github.com/getsentry/sentry/pull/80528",
"goldens": [
{
"comment": "The function modifies the config variable to include display values but then returns the original monitor.config instead of the modified version.",
"severity": "High"
},
{
"comment": "The code fetches MonitorCheckIn objects by ID when the required data already exists in previous_checkins. This creates an unnecessary database query.",
"severity": "Low"
}
]
},
{
"id": "sentry#92393",
"pr_url": "https://github.com/getsentry/sentry/pull/92393",
"goldens": [
{
"comment": "OptimizedCursorPaginator negative-offset branch slices QuerySet with a negative start index",
"severity": "Critical"
},
{
"comment": "BasePaginator negative-offset branch slices QuerySet with a negative start index",
"severity": "High"
},
{
"comment": "OptimizedCursorPaginator.get_item_key uses floor/ceil on a datetime key (order_by='-datetime'), causing TypeError.",
"severity": "High"
}
]
},
{
"id": "sentry#93824",
"pr_url": "https://github.com/getsentry/sentry/pull/93824",
"goldens": [
{
"comment": "Inconsistent metric tagging with 'shard' and 'shards'",
"severity": "Medium"
},
{
"comment": "Fixed sleep in tests can be flaky; wait on condition instead",
"severity": "Low"
},
{
"comment": "Because flusher processes are created via multiprocessing.get_context('spawn').Process, they are instances of multiprocessing.context.SpawnProcess, which on POSIX is not a subclass of multiprocessing.Process, so this isinstance check will always be false and hung processes won't be killed here.",
"severity": "High"
},
{
"comment": "Sleep in test_consumer.py won’t actually wait because time.sleep was monkeypatched above; consider restoring sleep or using a different sync to ensure the flusher has time to process.",
"severity": "Medium"
},
{
"comment": "Breaking out of the loop when the deadline has elapsed can skip terminating remaining flusher processes, potentially leaving them running after shutdown; consider ensuring termination is attempted even if the deadline is exceeded.",
"severity": "Medium"
}
]
},
{
"id": "sentry#94376",
"pr_url": "https://github.com/getsentry/sentry/pull/94376",
"goldens": [
{
"comment": "sample_rate = 0.0 is falsy and skipped",
"severity": "Low"
},
{
"comment": "Using Python’s built-in hash() to build cache keys is non-deterministic across processes (hash randomization), so keys won’t match across workers and invalidate_upsampling_cache may fail to delete them. Use a deterministic serialization of project_ids for the cache key.",
"severity": "Low"
},
{
"comment": "The upsampling eligibility check passes the outer dataset instead of the actual dataset used by scoped_dataset. In paths where the query ultimately runs against discover (e.g., dashboard split) while the original dataset is metrics, upsampling may be skipped even when all projects are allowlisted.",
"severity": "Medium"
}
]
},
{
"id": "sentry#95633",
"pr_url": "https://github.com/getsentry/sentry/pull/95633",
"goldens": [
{
"comment": "The queue.shutdown() method with 'immediate=False' parameter may not exist in the standard Python queue module. This could cause AttributeError at runtime. Verify the correct API or implement a custom shutdown mechanism.",
"severity": "High"
},
{
"comment": "The magic number 50 for max_wait is used repeatedly throughout the tests. Consider extracting this as a named constant to improve maintainability.",
"severity": "Low"
},
{
"comment": "The test test_thread_queue_parallel_error_handling has a docstring that doesn't match the test implementation.",
"severity": "Low"
}
]
}
]
+116
View File
@@ -0,0 +1,116 @@
// Package prbench wires the PR reviewer benchmark: the Martian Code-Review-Bench
// problems, a reviewer, and a judge model on the built-in runner.
package prbench
import (
"context"
_ "embed"
"encoding/json"
"errors"
"fmt"
"io"
"log/slog"
"os"
"path/filepath"
"slices"
"github.com/samcharles93/archie-core/internal/agentexec"
"github.com/samcharles93/archie-core/internal/app/archied"
"github.com/samcharles93/archie-core/internal/domain/workflow/prreview/bench"
"github.com/samcharles93/archie-core/internal/infrastructure/configuration"
)
// martian holds the 38 runnable problems of the Martian Code-Review-Bench
// offline set: those whose pull request still exists upstream.
//
//go:embed martian.json
var martian []byte
// Problems returns the benchmark problems, optionally narrowed to ids.
func Problems(ids []string) ([]bench.Problem, error) {
var all []bench.Problem
if err := json.Unmarshal(martian, &all); err != nil {
return nil, fmt.Errorf("decode benchmark problems: %w", err)
}
if len(ids) == 0 {
return all, nil
}
var out []bench.Problem
for _, id := range ids {
i := slices.IndexFunc(all, func(p bench.Problem) bool { return p.ID == id })
if i < 0 {
return nil, fmt.Errorf("no benchmark problem %q", id)
}
out = append(out, all[i])
}
return out, nil
}
// Options configures one benchmark run.
type Options struct {
Config string
Fixtures string
JudgeModel string
Out string
Problems []string
Concurrency int
}
// Run reviews and judges the selected problems, writes one result file per
// problem and a summary to Options.Out, and prints the summary to w.
func Run(ctx context.Context, opts Options, w io.Writer) (bench.Summary, error) {
if opts.Fixtures == "" {
return bench.Summary{}, errors.New("no reviewer: pass a findings fixture directory")
}
problems, err := Problems(opts.Problems)
if err != nil {
return bench.Summary{}, err
}
doc, err := configuration.New(nil).File(opts.Config)
if err != nil {
return bench.Summary{}, err
}
if err := archied.ResolveProviders(&doc.Config, slog.Default()); err != nil {
return bench.Summary{}, err
}
rt := agentexec.NewRuntime(agentexec.ProvidersFromConfig(doc.Config.Providers))
if rt == nil {
return bench.Summary{}, fmt.Errorf("%s configures no providers for the judge", opts.Config)
}
judge := &modelJudge{runtime: rt, modelRef: opts.JudgeModel}
results := bench.Run(ctx, problems, fixtureReviewer{dir: opts.Fixtures}, judge, opts.Concurrency)
summary := bench.Summarize(results)
if err := writeResults(opts.Out, results, summary); err != nil {
return summary, err
}
for _, r := range results {
if r.Err != "" {
fmt.Fprintf(w, "%s: %s\n", r.Problem.ID, r.Err)
}
}
fmt.Fprintf(w, "problems %d (failed %d) recall %.3f (%d/%d) golden-only precision %.3f (%d/%d)\n",
summary.Problems, summary.Failed, summary.Recall, summary.Hits, summary.Goldens,
summary.Precision, summary.Matched, summary.Comments)
return summary, nil
}
func writeResults(dir string, results []bench.Result, summary bench.Summary) error {
if err := os.MkdirAll(filepath.Join(dir, "results"), 0o755); err != nil {
return err
}
for _, r := range results {
if err := writeJSON(filepath.Join(dir, "results", r.Problem.ID+".json"), r); err != nil {
return err
}
}
return writeJSON(filepath.Join(dir, "summary.json"), summary)
}
func writeJSON(path string, v any) error {
data, err := json.MarshalIndent(v, "", " ")
if err != nil {
return err
}
return os.WriteFile(path, append(data, '\n'), 0o644)
}
+65
View File
@@ -0,0 +1,65 @@
package prbench
import (
"os"
"path/filepath"
"strings"
"testing"
"github.com/samcharles93/archie-core/internal/domain/workflow/prreview/bench"
)
func TestProblemsAreTheRunnableSet(t *testing.T) {
all, err := Problems(nil)
if err != nil {
t.Fatal(err)
}
goldens := 0
seen := map[string]bool{}
for _, p := range all {
if seen[p.ID] || p.PRURL == "" || len(p.Goldens) == 0 {
t.Errorf("problem %q: duplicate, or missing its PR or goldens", p.ID)
}
seen[p.ID] = true
goldens += len(p.Goldens)
}
if len(all) != 38 || goldens != 102 {
t.Errorf("got %d problems with %d goldens, want 38 with 102", len(all), goldens)
}
if _, err := Problems([]string{"nope#1"}); err == nil {
t.Error("an unknown problem id was accepted")
}
}
func TestFixtureReviewer(t *testing.T) {
tests := []struct {
name string
body string
wantErr string
want int
}{
{name: "findings", body: `{"blast_radius_files": 12, "findings": [{"file": "a.go", "line_start": 3, "severity": "critical", "confidence": 0.8}]}`, want: 1},
{name: "unknown severity", body: `{"findings": [{"file": "a.go", "severity": "High"}]}`, wantErr: "unknown severity"},
{name: "missing file", wantErr: "no such file"},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
dir := t.TempDir()
if tt.body != "" {
if err := os.WriteFile(filepath.Join(dir, "x#1.json"), []byte(tt.body), 0o644); err != nil {
t.Fatal(err)
}
}
got, err := fixtureReviewer{dir: dir}.Review(t.Context(), bench.Problem{ID: "x#1"})
if tt.wantErr != "" {
if err == nil || !strings.Contains(err.Error(), tt.wantErr) {
t.Fatalf("err = %v, want containing %q", err, tt.wantErr)
}
return
}
if err != nil || len(got.Findings) != tt.want || got.Inputs.BlastRadiusFiles != 12 {
t.Fatalf("Review = %+v, %v", got, err)
}
})
}
}
@@ -0,0 +1,204 @@
// Package bench measures the PR reviewer against a set of pull requests whose
// real defects are known: each problem's golden comments. A reviewer produces
// findings, the deterministic core turns them into the comments a review would
// post, and an independent judge decides which golden each comment catches.
// See docs/prds/pr-review-agent.md, Verification.
package bench
import (
"context"
"fmt"
"sync"
"github.com/samcharles93/archie-core/internal/domain/workflow/prreview"
)
// Golden is one known defect in a problem's pull request.
type Golden struct {
Comment string `json:"comment"`
Severity string `json:"severity"`
}
// Problem is one benchmark pull request and its known defects.
type Problem struct {
ID string `json:"id"`
PRURL string `json:"pr_url"`
Goldens []Golden `json:"goldens"`
}
// Review is what a reviewer returns for one problem: its findings and the run
// context that scores them.
type Review struct {
Findings []prreview.Finding
Inputs prreview.ScoreInputs
}
// Reviewer reviews one problem's pull request blind: it never sees the goldens.
type Reviewer interface {
Review(ctx context.Context, p Problem) (Review, error)
}
// NoComment is a verdict's Comment when no posted comment catches the golden.
const NoComment = -1
// Verdict is the judge's decision about one golden: which posted comment, by
// index, identifies the same defect, or NoComment.
type Verdict struct {
Golden int `json:"golden"`
Comment int `json:"comment"`
Reason string `json:"reason"`
}
// Hit reports whether a posted comment caught the golden.
func (v Verdict) Hit() bool { return v.Comment != NoComment }
// Judge matches a problem's goldens against the comments a review posted. It
// returns one verdict per golden.
type Judge interface {
Judge(ctx context.Context, p Problem, comments []prreview.ScoredFinding) ([]Verdict, error)
}
// Result is one problem's outcome. Err is set when the reviewer or the judge
// failed, and then the problem is not scored at all: a failure is not a miss.
type Result struct {
Problem Problem `json:"problem"`
Comments []prreview.ScoredFinding `json:"comments"`
Verdicts []Verdict `json:"verdicts"`
Err string `json:"error,omitempty"`
}
// Hits is how many goldens a posted comment caught.
func (r Result) Hits() int {
n := 0
for _, v := range r.Verdicts {
if v.Hit() {
n++
}
}
return n
}
// MatchedComments is how many posted comments caught at least one golden.
func (r Result) MatchedComments() int {
seen := map[int]bool{}
for _, v := range r.Verdicts {
if v.Hit() {
seen[v.Comment] = true
}
}
return len(seen)
}
// Run reviews and judges every problem, at most concurrency at a time, and
// returns one result per problem in the order given.
func Run(ctx context.Context, problems []Problem, reviewer Reviewer, judge Judge, concurrency int) []Result {
results := make([]Result, len(problems))
slots := make(chan struct{}, max(concurrency, 1))
var wg sync.WaitGroup
for i, p := range problems {
wg.Go(func() {
slots <- struct{}{}
defer func() { <-slots }()
results[i] = runOne(ctx, p, reviewer, judge)
})
}
wg.Wait()
return results
}
func runOne(ctx context.Context, p Problem, reviewer Reviewer, judge Judge) Result {
res := Result{Problem: p}
review, err := reviewer.Review(ctx, p)
if err != nil {
res.Err = fmt.Sprintf("review: %v", err)
return res
}
res.Comments = prreview.CapInlineComments(prreview.Score(review.Findings, review.Inputs))
if len(res.Comments) == 0 {
res.Verdicts = misses(p)
return res
}
verdicts, err := judge.Judge(ctx, p, res.Comments)
if err == nil {
err = checkVerdicts(p, len(res.Comments), verdicts)
}
if err != nil {
res.Err = fmt.Sprintf("judge: %v", err)
return res
}
res.Verdicts = verdicts
return res
}
func misses(p Problem) []Verdict {
out := make([]Verdict, len(p.Goldens))
for i := range out {
out[i] = Verdict{Golden: i, Comment: NoComment, Reason: "the review posted no comments"}
}
return out
}
// checkVerdicts requires exactly one verdict per golden, each naming a comment
// that was posted, so a judge that skips a golden cannot read as a miss.
func checkVerdicts(p Problem, comments int, verdicts []Verdict) error {
seen := make([]bool, len(p.Goldens))
for _, v := range verdicts {
if v.Golden < 0 || v.Golden >= len(p.Goldens) {
return fmt.Errorf("verdict for golden %d, which does not exist", v.Golden)
}
if seen[v.Golden] {
return fmt.Errorf("two verdicts for golden %d", v.Golden)
}
seen[v.Golden] = true
if v.Comment != NoComment && (v.Comment < 0 || v.Comment >= comments) {
return fmt.Errorf("golden %d matched comment %d, which was not posted", v.Golden, v.Comment)
}
}
for i, ok := range seen {
if !ok {
return fmt.Errorf("no verdict for golden %d", i)
}
}
return nil
}
// Summary is the benchmark's aggregate over the problems that were scored.
type Summary struct {
Problems int `json:"problems"`
Failed int `json:"failed"`
Goldens int `json:"goldens"`
Hits int `json:"hits"`
Comments int `json:"comments"`
Matched int `json:"matched_comments"`
// Recall is golden micro-recall: goldens caught over goldens scored.
Recall float64 `json:"recall"`
// Precision is golden-only precision: posted comments that caught a golden
// over posted comments. A real defect no golden names counts against it.
Precision float64 `json:"precision"`
}
// Summarize aggregates results. A failed problem counts only in Failed.
func Summarize(results []Result) Summary {
var s Summary
for _, r := range results {
s.Problems++
if r.Err != "" {
s.Failed++
continue
}
s.Goldens += len(r.Problem.Goldens)
s.Hits += r.Hits()
s.Comments += len(r.Comments)
s.Matched += r.MatchedComments()
}
s.Recall = ratio(s.Hits, s.Goldens)
s.Precision = ratio(s.Matched, s.Comments)
return s
}
func ratio(n, d int) float64 {
if d == 0 {
return 0
}
return float64(n) / float64(d)
}
@@ -0,0 +1,141 @@
package bench
import (
"context"
"errors"
"strings"
"testing"
"github.com/samcharles93/archie-core/internal/domain/workflow/prreview"
)
type fakeReviewer map[string]Review
func (f fakeReviewer) Review(_ context.Context, p Problem) (Review, error) {
r, ok := f[p.ID]
if !ok {
return Review{}, errors.New("no review")
}
return r, nil
}
type fakeJudge struct {
verdicts map[string][]Verdict
calls int
}
func (f *fakeJudge) Judge(_ context.Context, p Problem, _ []prreview.ScoredFinding) ([]Verdict, error) {
f.calls++
return f.verdicts[p.ID], nil
}
func problem(id string, goldens int) Problem {
return Problem{ID: id, Goldens: make([]Golden, goldens)}
}
func finding(file string, line int, confidence float64) prreview.Finding {
return prreview.Finding{File: file, LineStart: line, LineEnd: line, Severity: prreview.SeverityImportant, Title: file, Confidence: confidence}
}
func TestRunScoresEachProblem(t *testing.T) {
tests := []struct {
name string
review Review
verdicts []Verdict
wantErr string
wantHits int
wantPosts int
wantCalls int
}{
{
name: "hit and miss",
review: Review{Findings: []prreview.Finding{finding("a.go", 1, 0.9), finding("b.go", 1, 0.9)}},
verdicts: []Verdict{{Golden: 0, Comment: 1}, {Golden: 1, Comment: NoComment}},
wantHits: 1,
wantPosts: 2,
wantCalls: 1,
},
{
name: "every finding under the floor posts nothing and skips the judge",
review: Review{Findings: []prreview.Finding{finding("a.go", 1, 0.1)}},
wantCalls: 0,
},
{
name: "a judge that skips a golden fails the problem",
review: Review{Findings: []prreview.Finding{finding("a.go", 1, 0.9)}},
verdicts: []Verdict{{Golden: 0, Comment: 0}},
wantErr: "no verdict for golden 1",
wantPosts: 1,
wantCalls: 1,
},
{
name: "a judge that names an unposted comment fails the problem",
review: Review{Findings: []prreview.Finding{finding("a.go", 1, 0.9)}},
verdicts: []Verdict{{Golden: 0, Comment: 3}, {Golden: 1, Comment: NoComment}},
wantErr: "comment 3",
wantPosts: 1,
wantCalls: 1,
},
{
name: "duplicate verdicts fail the problem",
review: Review{Findings: []prreview.Finding{finding("a.go", 1, 0.9)}},
verdicts: []Verdict{{Golden: 0, Comment: 0}, {Golden: 0, Comment: 0}},
wantErr: "two verdicts",
wantPosts: 1,
wantCalls: 1,
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
p := problem("p", 2)
judge := &fakeJudge{verdicts: map[string][]Verdict{"p": tt.verdicts}}
got := Run(t.Context(), []Problem{p}, fakeReviewer{"p": tt.review}, judge, 1)[0]
if tt.wantErr == "" && got.Err != "" || !strings.Contains(got.Err, tt.wantErr) {
t.Fatalf("Err = %q, want containing %q", got.Err, tt.wantErr)
}
if tt.wantErr == "" && len(got.Verdicts) != len(p.Goldens) {
t.Errorf("verdicts = %d, want one per golden", len(got.Verdicts))
}
if got.Hits() != tt.wantHits || len(got.Comments) != tt.wantPosts || judge.calls != tt.wantCalls {
t.Errorf("hits, comments, judge calls = %d, %d, %d; want %d, %d, %d",
got.Hits(), len(got.Comments), judge.calls, tt.wantHits, tt.wantPosts, tt.wantCalls)
}
})
}
}
func TestRunKeepsProblemOrder(t *testing.T) {
problems := []Problem{problem("a", 0), problem("b", 0), problem("c", 0)}
results := Run(t.Context(), problems, fakeReviewer{"a": {}, "c": {}}, &fakeJudge{}, 3)
for i, r := range results {
if r.Problem.ID != problems[i].ID {
t.Fatalf("result %d is %s, want %s", i, r.Problem.ID, problems[i].ID)
}
}
if results[1].Err == "" {
t.Error("the reviewer's failure on b was not recorded")
}
}
func TestSummarize(t *testing.T) {
comments := make([]prreview.ScoredFinding, 4)
results := []Result{
{
Problem: problem("a", 3),
Comments: comments,
// Two goldens caught by one comment: two hits, one matched comment.
Verdicts: []Verdict{{Golden: 0, Comment: 2}, {Golden: 1, Comment: 2}, {Golden: 2, Comment: NoComment}},
},
{
Problem: problem("b", 1),
Comments: comments[:1],
Verdicts: []Verdict{{Golden: 0, Comment: 0}},
},
{Problem: problem("c", 5), Comments: comments, Err: "judge: down"},
}
got := Summarize(results)
want := Summary{Problems: 3, Failed: 1, Goldens: 4, Hits: 3, Comments: 5, Matched: 2, Recall: 0.75, Precision: 0.4}
if got != want {
t.Errorf("Summarize = %+v\nwant %+v", got, want)
}
}