// Package eval measures whether a judge is good enough for this use case. // // It answers the only question that matters before a classifier is allowed to // intercept live traffic: does it catch the calls that should be caught, without // blocking the calls that should not be? The deterministic lens is measured on // the same cases, so the judge's marginal value is a number rather than a claim. package eval import ( "bufio" "bytes" "context" "encoding/json" "errors" "fmt" "io" "os" "strings" "text/tabwriter" "github.com/samcharles93/archipelago/guards/integrity/internal/audit" "github.com/samcharles93/archipelago/guards/integrity/internal/judge" ) // Expected is the ground truth for one case. type Expected struct { Legitimacy string `json:"legitimacy,omitempty"` Validity string `json:"validity,omitempty"` Decision string `json:"decision"` } // Case is one labelled call. type Case struct { ID string `json:"id"` Why string `json:"why,omitempty"` Expected Expected `json:"expected"` Bundle json.RawMessage `json:"bundle"` } // LoadCases reads labelled cases as JSON lines. func LoadCases(r io.Reader) ([]Case, error) { scanner := bufio.NewScanner(r) scanner.Buffer(make([]byte, 0, 1<<20), 16<<20) var cases []Case line := 0 for scanner.Scan() { line++ raw := scanner.Bytes() if len(raw) == 0 || raw[0] == '#' { continue } dec := json.NewDecoder(bytes.NewReader(raw)) dec.DisallowUnknownFields() var c Case if err := dec.Decode(&c); err != nil { return nil, fmt.Errorf("line %d: %w", line, err) } if err := validateCase(&c); err != nil { return nil, fmt.Errorf("line %d: %w", line, err) } cases = append(cases, c) } if err := scanner.Err(); err != nil { return nil, fmt.Errorf("read cases: %w", err) } if len(cases) == 0 { return nil, errors.New("no cases") } return cases, nil } // LoadCasesFile reads cases from a path, or from stdin when path is "-". func LoadCasesFile(path string) ([]Case, error) { if path == "-" { return LoadCases(os.Stdin) } f, err := os.Open(path) //nolint:gosec // operator-supplied path if err != nil { return nil, err } defer func() { _ = f.Close() }() return LoadCases(f) } func validateCase(c *Case) error { if c.ID == "" { return errors.New("id is required") } if len(c.Bundle) == 0 { return errors.New("bundle is required") } if !oneOf(c.Expected.Decision, audit.DecisionAllow, audit.DecisionReview, audit.DecisionBlock) { return fmt.Errorf("expected.decision %q must be allow, review or block", c.Expected.Decision) } if c.Expected.Legitimacy != "" && !oneOf(c.Expected.Legitimacy, audit.Legitimate, audit.Nefarious, audit.Inconclusive) { return fmt.Errorf("expected.legitimacy %q is not a legitimacy verdict", c.Expected.Legitimacy) } if c.Expected.Validity != "" && !oneOf(c.Expected.Validity, audit.Valid, audit.Invalid, audit.Inconclusive) { return fmt.Errorf("expected.validity %q is not a validity verdict", c.Expected.Validity) } if _, err := audit.DecodeBundle(bytes.NewReader(c.Bundle)); err != nil { return fmt.Errorf("bundle: %w", err) } return nil } func oneOf(v string, allowed ...string) bool { for _, a := range allowed { if v == a { return true } } return false } // Options configures a run. type Options struct { Judge judge.Judge JudgeOptions judge.Options // Names the judge in the report when the judge itself is generic. Label string } // Result is one case's outcome under both the lens and the judge. type Result struct { Case Case Baseline *audit.Report Judged *audit.Report Judge *judge.Verdict Error string } // Scores summarises how well one configuration did. type Scores struct { Total int `json:"total"` Correct int `json:"correct"` // Accuracy is the fraction of cases whose decision matched the label. Accuracy float64 `json:"accuracy"` // Confusion is expected decision -> observed decision, per case. Confusion map[string]map[string]int `json:"confusion"` // Nefarious precision and recall treat "the output is a vector for // wrongdoing" as the positive class, because that is the class a gate exists // to catch and a false positive in it is what makes operators stop trusting // the gate. NefariousExpected int `json:"nefarious_expected"` NefariousCalled int `json:"nefarious_called"` NefariousCorrect int `json:"nefarious_correct"` NefariousRecall float64 `json:"nefarious_recall"` NefariousPrecision float64 `json:"nefarious_precision"` // BenignTotal and FalseBlocks measure the cost of being wrong about a good // call, which is the other half of the trade. BenignTotal int `json:"benign_total"` FalseBlocks int `json:"false_blocks"` FalseBlockRate float64 `json:"false_block_rate"` UnderBlocks int `json:"under_blocks"` // expected a block, did not produce one LegitimacyMatch int `json:"legitimacy_match"` } // Summary is a full comparison of the lens against the lens plus a judge. type Summary struct { Cases []Result `json:"cases"` Baseline Scores `json:"baseline"` Judged Scores `json:"judged"` // Fixed and Broke are the judge's marginal value: cases where the judge // changed the decision and the change was right, and where it was wrong. Fixed int `json:"fixed"` Broke int `json:"broke"` Changed int `json:"changed"` JudgeLabel string `json:"judge_label,omitempty"` JudgeAuthority string `json:"judge_authority,omitempty"` JudgeErrors int `json:"judge_errors"` JudgeLatencyMS int `json:"judge_latency_ms_total"` JudgeTokens int `json:"judge_tokens_total"` JudgeCostUSD float64 `json:"judge_cost_usd_total"` JudgeCallsMade int `json:"judge_calls_made"` JudgeUnanswered int `json:"judge_unanswered"` } // Run measures the lens alone and the lens plus a judge on the same cases. // // A nil judge is allowed: the run then reports the baseline only, which is the // right thing to do when no classifier is reachable. func Run(ctx context.Context, a *audit.Auditor, opts Options, cases []Case) Summary { s := Summary{JudgeLabel: opts.Label, JudgeAuthority: string(opts.JudgeOptions.Authority)} for _, c := range cases { res := Result{Case: c} b, err := audit.DecodeBundle(bytes.NewReader(c.Bundle)) if err != nil { res.Error = err.Error() s.Cases = append(s.Cases, res) continue } res.Baseline = a.Audit(b) res.Judged = a.Audit(b) if opts.Judge != nil { v := opts.Judge.Judge(ctx, b, res.Judged, opts.JudgeOptions) res.Judge = &v s.JudgeCallsMade++ s.JudgeLatencyMS += v.LatencyMS if v.Usage != nil { s.JudgeTokens += v.Usage.TotalTokens s.JudgeCostUSD += v.Usage.CostUSD } if !v.Ok() { s.JudgeErrors++ if v.Error != "" && strings.Contains(v.Error, "no answers") { s.JudgeUnanswered++ } } judge.Apply(res.Judged, v) } s.Cases = append(s.Cases, res) } s.Baseline = score(s.Cases, func(r Result) *audit.Report { return r.Baseline }) s.Judged = score(s.Cases, func(r Result) *audit.Report { return r.Judged }) for _, r := range s.Cases { if r.Baseline == nil || r.Judged == nil { continue } if r.Baseline.Decision == r.Judged.Decision { continue } s.Changed++ baselineRight := r.Baseline.Decision == r.Case.Expected.Decision judgedRight := r.Judged.Decision == r.Case.Expected.Decision switch { case judgedRight && !baselineRight: s.Fixed++ case baselineRight && !judgedRight: s.Broke++ } } return s } func score(cases []Result, report func(Result) *audit.Report) Scores { s := Scores{Total: len(cases), Confusion: map[string]map[string]int{}} for _, r := range cases { rep := report(r) if rep == nil { s.Total-- continue } expected := r.Case.Expected if s.Confusion[expected.Decision] == nil { s.Confusion[expected.Decision] = map[string]int{} } s.Confusion[expected.Decision][rep.Decision]++ if rep.Decision == expected.Decision { s.Correct++ } if expected.Legitimacy != "" && rep.Legitimacy.Verdict == expected.Legitimacy { s.LegitimacyMatch++ } switch expected.Legitimacy { case audit.Nefarious: s.NefariousExpected++ if rep.Legitimacy.Verdict == audit.Nefarious { s.NefariousCorrect++ } case audit.Legitimate: s.BenignTotal++ // A legitimate call that is correctly blocked on the validity axis // is not a false block. if rep.Decision == audit.DecisionBlock && expected.Decision != audit.DecisionBlock { s.FalseBlocks++ } } if rep.Legitimacy.Verdict == audit.Nefarious { s.NefariousCalled++ } if expected.Decision == audit.DecisionBlock && rep.Decision != audit.DecisionBlock { s.UnderBlocks++ } } s.Accuracy = ratio(s.Correct, s.Total) s.NefariousRecall = ratio(s.NefariousCorrect, s.NefariousExpected) s.NefariousPrecision = ratio(s.NefariousCorrect, s.NefariousCalled) s.FalseBlockRate = ratio(s.FalseBlocks, s.BenignTotal) return s } func ratio(numerator, denominator int) float64 { if denominator == 0 { return 0 } return float64(numerator) / float64(denominator) } // Gate fails when the judged run misses any floor the operator set. A gate that // cannot fail is decoration, so every floor is checked explicitly. func (s Summary) Gate(minAccuracy, minRecall, minPrecision, maxFalseBlockRate float64) error { var problems []string if s.Judged.Accuracy < minAccuracy { problems = append(problems, fmt.Sprintf("accuracy %.2f is below %.2f", s.Judged.Accuracy, minAccuracy)) } if s.Judged.NefariousRecall < minRecall { problems = append(problems, fmt.Sprintf("nefarious recall %.2f is below %.2f", s.Judged.NefariousRecall, minRecall)) } if s.Judged.NefariousPrecision < minPrecision { problems = append(problems, fmt.Sprintf("nefarious precision %.2f is below %.2f", s.Judged.NefariousPrecision, minPrecision)) } if s.Judged.FalseBlockRate > maxFalseBlockRate { problems = append(problems, fmt.Sprintf("false block rate %.2f is above %.2f", s.Judged.FalseBlockRate, maxFalseBlockRate)) } if s.JudgeErrors > 0 { problems = append(problems, fmt.Sprintf("%d judge calls failed", s.JudgeErrors)) } if len(problems) > 0 { return errors.New(strings.Join(problems, "; ")) } return nil } // Text renders the per-case table and the comparison. func (s Summary) Text() string { var b strings.Builder // Writing into a tabwriter buffer cannot fail; its Flush reports the error, // and this is a rendered report rather than a fidelity-critical artefact. w := tabwriter.NewWriter(&b, 0, 0, 2, ' ', 0) _, _ = fmt.Fprintln(w, "case\texpected\tlens\tjudged\tjudge\tnote") for _, r := range s.Cases { if r.Baseline == nil || r.Judged == nil { _, _ = fmt.Fprintf(w, "%s\t%s\t-\t-\t-\tERROR %s\n", r.Case.ID, r.Case.Expected.Decision, r.Error) continue } judgeCell := "-" if r.Judge != nil { switch { case !r.Judge.Ok(): judgeCell = "error" case r.Judge.Legitimacy.Value != "": judgeCell = fmt.Sprintf("%s %.2f", r.Judge.Legitimacy.Value, r.Judge.Legitimacy.Confidence) default: judgeCell = "signals only" } } _, _ = fmt.Fprintf(w, "%s\t%s\t%s\t%s\t%s\t%s\n", r.Case.ID, r.Case.Expected.Decision, r.Baseline.Decision, r.Judged.Decision, judgeCell, note(r)) } _ = w.Flush() b.WriteString("\n") writeScores(&b, "lens only", s.Baseline) writeScores(&b, "lens + judge", s.Judged) fmt.Fprintf(&b, "\njudge marginal value: fixed %d, broke %d (of %d changed decisions)\n", s.Fixed, s.Broke, s.Changed) if s.JudgeLabel != "" { authority := s.JudgeAuthority if authority == "" { authority = string(judge.AuthorityAdvisory) } fmt.Fprintf(&b, "judge: %s (authority %s), %d calls, %d errors, %d ms total, %d tokens, $%.4f\n", s.JudgeLabel, authority, s.JudgeCallsMade, s.JudgeErrors, s.JudgeLatencyMS, s.JudgeTokens, s.JudgeCostUSD) if s.JudgeCallsMade > 0 { fmt.Fprintf(&b, "per call: %d ms, %d tokens, $%.6f\n", s.JudgeLatencyMS/max(s.JudgeCallsMade, 1), s.JudgeTokens/max(s.JudgeCallsMade, 1), s.JudgeCostUSD/float64(max(s.JudgeCallsMade, 1))) } } return b.String() } func writeScores(b *strings.Builder, label string, s Scores) { fmt.Fprintf(b, "%-14s accuracy %2d/%-2d = %.2f | nefarious recall %.2f (%d/%d) precision %.2f (%d/%d)\n", label, s.Correct, s.Total, s.Accuracy, s.NefariousRecall, s.NefariousCorrect, s.NefariousExpected, s.NefariousPrecision, s.NefariousCorrect, s.NefariousCalled) fmt.Fprintf(b, "%-14s benign %d, false blocks %d (%.2f), missed blocks %d, confusion %s\n", "", s.BenignTotal, s.FalseBlocks, s.FalseBlockRate, s.UnderBlocks, confusionString(s.Confusion)) } func confusionString(confusion map[string]map[string]int) string { keys := []string{audit.DecisionBlock, audit.DecisionReview, audit.DecisionAllow} parts := make([]string, 0, len(keys)) for _, want := range keys { got := confusion[want] if len(got) == 0 { continue } inner := make([]string, 0, len(keys)) for _, observed := range keys { if n := got[observed]; n > 0 { inner = append(inner, fmt.Sprintf("%s=%d", observed, n)) } } parts = append(parts, strings.Join([]string{want, strings.Join(inner, " ")}, "->")) } return strings.Join(parts, ", ") } func note(r Result) string { if r.Baseline == nil || r.Judged == nil { return "" } baselineRight := r.Baseline.Decision == r.Case.Expected.Decision judgedRight := r.Judged.Decision == r.Case.Expected.Decision switch { case judgedRight && !baselineRight: return "fixed by judge" case baselineRight && !judgedRight: return "broken by judge" case judgedRight: return "ok" default: return "missed" } } // WriteJSON emits the full summary including per-case reports. func WriteJSON(w io.Writer, s Summary) error { enc := json.NewEncoder(w) enc.SetIndent("", " ") if err := enc.Encode(s); err != nil { return fmt.Errorf("encode summary: %w", err) } return nil }