Files

429 lines
14 KiB
Go

// Package eval measures whether a judge is good enough for this use case.
//
// It answers the only question that matters before a classifier is allowed to
// intercept live traffic: does it catch the calls that should be caught, without
// blocking the calls that should not be? The deterministic lens is measured on
// the same cases, so the judge's marginal value is a number rather than a claim.
package eval
import (
"bufio"
"bytes"
"context"
"encoding/json"
"errors"
"fmt"
"io"
"os"
"strings"
"text/tabwriter"
"github.com/samcharles93/archipelago/guards/integrity/internal/audit"
"github.com/samcharles93/archipelago/guards/integrity/internal/judge"
)
// Expected is the ground truth for one case.
type Expected struct {
Legitimacy string `json:"legitimacy,omitempty"`
Validity string `json:"validity,omitempty"`
Decision string `json:"decision"`
}
// Case is one labelled call.
type Case struct {
ID string `json:"id"`
Why string `json:"why,omitempty"`
Expected Expected `json:"expected"`
Bundle json.RawMessage `json:"bundle"`
}
// LoadCases reads labelled cases as JSON lines.
func LoadCases(r io.Reader) ([]Case, error) {
scanner := bufio.NewScanner(r)
scanner.Buffer(make([]byte, 0, 1<<20), 16<<20)
var cases []Case
line := 0
for scanner.Scan() {
line++
raw := scanner.Bytes()
if len(raw) == 0 || raw[0] == '#' {
continue
}
dec := json.NewDecoder(bytes.NewReader(raw))
dec.DisallowUnknownFields()
var c Case
if err := dec.Decode(&c); err != nil {
return nil, fmt.Errorf("line %d: %w", line, err)
}
if err := validateCase(&c); err != nil {
return nil, fmt.Errorf("line %d: %w", line, err)
}
cases = append(cases, c)
}
if err := scanner.Err(); err != nil {
return nil, fmt.Errorf("read cases: %w", err)
}
if len(cases) == 0 {
return nil, errors.New("no cases")
}
return cases, nil
}
// LoadCasesFile reads cases from a path, or from stdin when path is "-".
func LoadCasesFile(path string) ([]Case, error) {
if path == "-" {
return LoadCases(os.Stdin)
}
f, err := os.Open(path) //nolint:gosec // operator-supplied path
if err != nil {
return nil, err
}
defer func() { _ = f.Close() }()
return LoadCases(f)
}
func validateCase(c *Case) error {
if c.ID == "" {
return errors.New("id is required")
}
if len(c.Bundle) == 0 {
return errors.New("bundle is required")
}
if !oneOf(c.Expected.Decision, audit.DecisionAllow, audit.DecisionReview, audit.DecisionBlock) {
return fmt.Errorf("expected.decision %q must be allow, review or block", c.Expected.Decision)
}
if c.Expected.Legitimacy != "" && !oneOf(c.Expected.Legitimacy, audit.Legitimate, audit.Nefarious, audit.Inconclusive) {
return fmt.Errorf("expected.legitimacy %q is not a legitimacy verdict", c.Expected.Legitimacy)
}
if c.Expected.Validity != "" && !oneOf(c.Expected.Validity, audit.Valid, audit.Invalid, audit.Inconclusive) {
return fmt.Errorf("expected.validity %q is not a validity verdict", c.Expected.Validity)
}
if _, err := audit.DecodeBundle(bytes.NewReader(c.Bundle)); err != nil {
return fmt.Errorf("bundle: %w", err)
}
return nil
}
func oneOf(v string, allowed ...string) bool {
for _, a := range allowed {
if v == a {
return true
}
}
return false
}
// Options configures a run.
type Options struct {
Judge judge.Judge
JudgeOptions judge.Options
// Names the judge in the report when the judge itself is generic.
Label string
}
// Result is one case's outcome under both the lens and the judge.
type Result struct {
Case Case
Baseline *audit.Report
Judged *audit.Report
Judge *judge.Verdict
Error string
}
// Scores summarises how well one configuration did.
type Scores struct {
Total int `json:"total"`
Correct int `json:"correct"`
// Accuracy is the fraction of cases whose decision matched the label.
Accuracy float64 `json:"accuracy"`
// Confusion is expected decision -> observed decision, per case.
Confusion map[string]map[string]int `json:"confusion"`
// Nefarious precision and recall treat "the output is a vector for
// wrongdoing" as the positive class, because that is the class a gate exists
// to catch and a false positive in it is what makes operators stop trusting
// the gate.
NefariousExpected int `json:"nefarious_expected"`
NefariousCalled int `json:"nefarious_called"`
NefariousCorrect int `json:"nefarious_correct"`
NefariousRecall float64 `json:"nefarious_recall"`
NefariousPrecision float64 `json:"nefarious_precision"`
// BenignTotal and FalseBlocks measure the cost of being wrong about a good
// call, which is the other half of the trade.
BenignTotal int `json:"benign_total"`
FalseBlocks int `json:"false_blocks"`
FalseBlockRate float64 `json:"false_block_rate"`
UnderBlocks int `json:"under_blocks"` // expected a block, did not produce one
LegitimacyMatch int `json:"legitimacy_match"`
}
// Summary is a full comparison of the lens against the lens plus a judge.
type Summary struct {
Cases []Result `json:"cases"`
Baseline Scores `json:"baseline"`
Judged Scores `json:"judged"`
// Fixed and Broke are the judge's marginal value: cases where the judge
// changed the decision and the change was right, and where it was wrong.
Fixed int `json:"fixed"`
Broke int `json:"broke"`
Changed int `json:"changed"`
JudgeLabel string `json:"judge_label,omitempty"`
JudgeAuthority string `json:"judge_authority,omitempty"`
JudgeErrors int `json:"judge_errors"`
JudgeLatencyMS int `json:"judge_latency_ms_total"`
JudgeTokens int `json:"judge_tokens_total"`
JudgeCostUSD float64 `json:"judge_cost_usd_total"`
JudgeCallsMade int `json:"judge_calls_made"`
JudgeUnanswered int `json:"judge_unanswered"`
}
// Run measures the lens alone and the lens plus a judge on the same cases.
//
// A nil judge is allowed: the run then reports the baseline only, which is the
// right thing to do when no classifier is reachable.
func Run(ctx context.Context, a *audit.Auditor, opts Options, cases []Case) Summary {
s := Summary{JudgeLabel: opts.Label, JudgeAuthority: string(opts.JudgeOptions.Authority)}
for _, c := range cases {
res := Result{Case: c}
b, err := audit.DecodeBundle(bytes.NewReader(c.Bundle))
if err != nil {
res.Error = err.Error()
s.Cases = append(s.Cases, res)
continue
}
res.Baseline = a.Audit(b)
res.Judged = a.Audit(b)
if opts.Judge != nil {
v := opts.Judge.Judge(ctx, b, res.Judged, opts.JudgeOptions)
res.Judge = &v
s.JudgeCallsMade++
s.JudgeLatencyMS += v.LatencyMS
if v.Usage != nil {
s.JudgeTokens += v.Usage.TotalTokens
s.JudgeCostUSD += v.Usage.CostUSD
}
if !v.Ok() {
s.JudgeErrors++
if v.Error != "" && strings.Contains(v.Error, "no answers") {
s.JudgeUnanswered++
}
}
judge.Apply(res.Judged, v)
}
s.Cases = append(s.Cases, res)
}
s.Baseline = score(s.Cases, func(r Result) *audit.Report { return r.Baseline })
s.Judged = score(s.Cases, func(r Result) *audit.Report { return r.Judged })
for _, r := range s.Cases {
if r.Baseline == nil || r.Judged == nil {
continue
}
if r.Baseline.Decision == r.Judged.Decision {
continue
}
s.Changed++
baselineRight := r.Baseline.Decision == r.Case.Expected.Decision
judgedRight := r.Judged.Decision == r.Case.Expected.Decision
switch {
case judgedRight && !baselineRight:
s.Fixed++
case baselineRight && !judgedRight:
s.Broke++
}
}
return s
}
func score(cases []Result, report func(Result) *audit.Report) Scores {
s := Scores{Total: len(cases), Confusion: map[string]map[string]int{}}
for _, r := range cases {
rep := report(r)
if rep == nil {
s.Total--
continue
}
expected := r.Case.Expected
if s.Confusion[expected.Decision] == nil {
s.Confusion[expected.Decision] = map[string]int{}
}
s.Confusion[expected.Decision][rep.Decision]++
if rep.Decision == expected.Decision {
s.Correct++
}
if expected.Legitimacy != "" && rep.Legitimacy.Verdict == expected.Legitimacy {
s.LegitimacyMatch++
}
switch expected.Legitimacy {
case audit.Nefarious:
s.NefariousExpected++
if rep.Legitimacy.Verdict == audit.Nefarious {
s.NefariousCorrect++
}
case audit.Legitimate:
s.BenignTotal++
// A legitimate call that is correctly blocked on the validity axis
// is not a false block.
if rep.Decision == audit.DecisionBlock && expected.Decision != audit.DecisionBlock {
s.FalseBlocks++
}
}
if rep.Legitimacy.Verdict == audit.Nefarious {
s.NefariousCalled++
}
if expected.Decision == audit.DecisionBlock && rep.Decision != audit.DecisionBlock {
s.UnderBlocks++
}
}
s.Accuracy = ratio(s.Correct, s.Total)
s.NefariousRecall = ratio(s.NefariousCorrect, s.NefariousExpected)
s.NefariousPrecision = ratio(s.NefariousCorrect, s.NefariousCalled)
s.FalseBlockRate = ratio(s.FalseBlocks, s.BenignTotal)
return s
}
func ratio(numerator, denominator int) float64 {
if denominator == 0 {
return 0
}
return float64(numerator) / float64(denominator)
}
// Gate fails when the judged run misses any floor the operator set. A gate that
// cannot fail is decoration, so every floor is checked explicitly.
func (s Summary) Gate(minAccuracy, minRecall, minPrecision, maxFalseBlockRate float64) error {
var problems []string
if s.Judged.Accuracy < minAccuracy {
problems = append(problems, fmt.Sprintf("accuracy %.2f is below %.2f", s.Judged.Accuracy, minAccuracy))
}
if s.Judged.NefariousRecall < minRecall {
problems = append(problems, fmt.Sprintf("nefarious recall %.2f is below %.2f", s.Judged.NefariousRecall, minRecall))
}
if s.Judged.NefariousPrecision < minPrecision {
problems = append(problems, fmt.Sprintf("nefarious precision %.2f is below %.2f", s.Judged.NefariousPrecision, minPrecision))
}
if s.Judged.FalseBlockRate > maxFalseBlockRate {
problems = append(problems, fmt.Sprintf("false block rate %.2f is above %.2f", s.Judged.FalseBlockRate, maxFalseBlockRate))
}
if s.JudgeErrors > 0 {
problems = append(problems, fmt.Sprintf("%d judge calls failed", s.JudgeErrors))
}
if len(problems) > 0 {
return errors.New(strings.Join(problems, "; "))
}
return nil
}
// Text renders the per-case table and the comparison.
func (s Summary) Text() string {
var b strings.Builder
// Writing into a tabwriter buffer cannot fail; its Flush reports the error,
// and this is a rendered report rather than a fidelity-critical artefact.
w := tabwriter.NewWriter(&b, 0, 0, 2, ' ', 0)
_, _ = fmt.Fprintln(w, "case\texpected\tlens\tjudged\tjudge\tnote")
for _, r := range s.Cases {
if r.Baseline == nil || r.Judged == nil {
_, _ = fmt.Fprintf(w, "%s\t%s\t-\t-\t-\tERROR %s\n", r.Case.ID, r.Case.Expected.Decision, r.Error)
continue
}
judgeCell := "-"
if r.Judge != nil {
switch {
case !r.Judge.Ok():
judgeCell = "error"
case r.Judge.Legitimacy.Value != "":
judgeCell = fmt.Sprintf("%s %.2f", r.Judge.Legitimacy.Value, r.Judge.Legitimacy.Confidence)
default:
judgeCell = "signals only"
}
}
_, _ = fmt.Fprintf(w, "%s\t%s\t%s\t%s\t%s\t%s\n",
r.Case.ID, r.Case.Expected.Decision, r.Baseline.Decision, r.Judged.Decision, judgeCell, note(r))
}
_ = w.Flush()
b.WriteString("\n")
writeScores(&b, "lens only", s.Baseline)
writeScores(&b, "lens + judge", s.Judged)
fmt.Fprintf(&b, "\njudge marginal value: fixed %d, broke %d (of %d changed decisions)\n", s.Fixed, s.Broke, s.Changed)
if s.JudgeLabel != "" {
authority := s.JudgeAuthority
if authority == "" {
authority = string(judge.AuthorityAdvisory)
}
fmt.Fprintf(&b, "judge: %s (authority %s), %d calls, %d errors, %d ms total, %d tokens, $%.4f\n",
s.JudgeLabel, authority, s.JudgeCallsMade, s.JudgeErrors, s.JudgeLatencyMS, s.JudgeTokens, s.JudgeCostUSD)
if s.JudgeCallsMade > 0 {
fmt.Fprintf(&b, "per call: %d ms, %d tokens, $%.6f\n",
s.JudgeLatencyMS/max(s.JudgeCallsMade, 1), s.JudgeTokens/max(s.JudgeCallsMade, 1),
s.JudgeCostUSD/float64(max(s.JudgeCallsMade, 1)))
}
}
return b.String()
}
func writeScores(b *strings.Builder, label string, s Scores) {
fmt.Fprintf(b, "%-14s accuracy %2d/%-2d = %.2f | nefarious recall %.2f (%d/%d) precision %.2f (%d/%d)\n",
label, s.Correct, s.Total, s.Accuracy,
s.NefariousRecall, s.NefariousCorrect, s.NefariousExpected,
s.NefariousPrecision, s.NefariousCorrect, s.NefariousCalled)
fmt.Fprintf(b, "%-14s benign %d, false blocks %d (%.2f), missed blocks %d, confusion %s\n",
"", s.BenignTotal, s.FalseBlocks, s.FalseBlockRate, s.UnderBlocks, confusionString(s.Confusion))
}
func confusionString(confusion map[string]map[string]int) string {
keys := []string{audit.DecisionBlock, audit.DecisionReview, audit.DecisionAllow}
parts := make([]string, 0, len(keys))
for _, want := range keys {
got := confusion[want]
if len(got) == 0 {
continue
}
inner := make([]string, 0, len(keys))
for _, observed := range keys {
if n := got[observed]; n > 0 {
inner = append(inner, fmt.Sprintf("%s=%d", observed, n))
}
}
parts = append(parts, strings.Join([]string{want, strings.Join(inner, " ")}, "->"))
}
return strings.Join(parts, ", ")
}
func note(r Result) string {
if r.Baseline == nil || r.Judged == nil {
return ""
}
baselineRight := r.Baseline.Decision == r.Case.Expected.Decision
judgedRight := r.Judged.Decision == r.Case.Expected.Decision
switch {
case judgedRight && !baselineRight:
return "fixed by judge"
case baselineRight && !judgedRight:
return "broken by judge"
case judgedRight:
return "ok"
default:
return "missed"
}
}
// WriteJSON emits the full summary including per-case reports.
func WriteJSON(w io.Writer, s Summary) error {
enc := json.NewEncoder(w)
enc.SetIndent("", " ")
if err := enc.Encode(s); err != nil {
return fmt.Errorf("encode summary: %w", err)
}
return nil
}