429 lines
14 KiB
Go
429 lines
14 KiB
Go
// Package eval measures whether a judge is good enough for this use case.
|
|
//
|
|
// It answers the only question that matters before a classifier is allowed to
|
|
// intercept live traffic: does it catch the calls that should be caught, without
|
|
// blocking the calls that should not be? The deterministic lens is measured on
|
|
// the same cases, so the judge's marginal value is a number rather than a claim.
|
|
package eval
|
|
|
|
import (
|
|
"bufio"
|
|
"bytes"
|
|
"context"
|
|
"encoding/json"
|
|
"errors"
|
|
"fmt"
|
|
"io"
|
|
"os"
|
|
"strings"
|
|
"text/tabwriter"
|
|
|
|
"github.com/samcharles93/archipelago/guards/integrity/internal/audit"
|
|
"github.com/samcharles93/archipelago/guards/integrity/internal/judge"
|
|
)
|
|
|
|
// Expected is the ground truth for one case.
|
|
type Expected struct {
|
|
Legitimacy string `json:"legitimacy,omitempty"`
|
|
Validity string `json:"validity,omitempty"`
|
|
Decision string `json:"decision"`
|
|
}
|
|
|
|
// Case is one labelled call.
|
|
type Case struct {
|
|
ID string `json:"id"`
|
|
Why string `json:"why,omitempty"`
|
|
Expected Expected `json:"expected"`
|
|
Bundle json.RawMessage `json:"bundle"`
|
|
}
|
|
|
|
// LoadCases reads labelled cases as JSON lines.
|
|
func LoadCases(r io.Reader) ([]Case, error) {
|
|
scanner := bufio.NewScanner(r)
|
|
scanner.Buffer(make([]byte, 0, 1<<20), 16<<20)
|
|
var cases []Case
|
|
line := 0
|
|
for scanner.Scan() {
|
|
line++
|
|
raw := scanner.Bytes()
|
|
if len(raw) == 0 || raw[0] == '#' {
|
|
continue
|
|
}
|
|
dec := json.NewDecoder(bytes.NewReader(raw))
|
|
dec.DisallowUnknownFields()
|
|
var c Case
|
|
if err := dec.Decode(&c); err != nil {
|
|
return nil, fmt.Errorf("line %d: %w", line, err)
|
|
}
|
|
if err := validateCase(&c); err != nil {
|
|
return nil, fmt.Errorf("line %d: %w", line, err)
|
|
}
|
|
cases = append(cases, c)
|
|
}
|
|
if err := scanner.Err(); err != nil {
|
|
return nil, fmt.Errorf("read cases: %w", err)
|
|
}
|
|
if len(cases) == 0 {
|
|
return nil, errors.New("no cases")
|
|
}
|
|
return cases, nil
|
|
}
|
|
|
|
// LoadCasesFile reads cases from a path, or from stdin when path is "-".
|
|
func LoadCasesFile(path string) ([]Case, error) {
|
|
if path == "-" {
|
|
return LoadCases(os.Stdin)
|
|
}
|
|
f, err := os.Open(path) //nolint:gosec // operator-supplied path
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
defer func() { _ = f.Close() }()
|
|
return LoadCases(f)
|
|
}
|
|
|
|
func validateCase(c *Case) error {
|
|
if c.ID == "" {
|
|
return errors.New("id is required")
|
|
}
|
|
if len(c.Bundle) == 0 {
|
|
return errors.New("bundle is required")
|
|
}
|
|
if !oneOf(c.Expected.Decision, audit.DecisionAllow, audit.DecisionReview, audit.DecisionBlock) {
|
|
return fmt.Errorf("expected.decision %q must be allow, review or block", c.Expected.Decision)
|
|
}
|
|
if c.Expected.Legitimacy != "" && !oneOf(c.Expected.Legitimacy, audit.Legitimate, audit.Nefarious, audit.Inconclusive) {
|
|
return fmt.Errorf("expected.legitimacy %q is not a legitimacy verdict", c.Expected.Legitimacy)
|
|
}
|
|
if c.Expected.Validity != "" && !oneOf(c.Expected.Validity, audit.Valid, audit.Invalid, audit.Inconclusive) {
|
|
return fmt.Errorf("expected.validity %q is not a validity verdict", c.Expected.Validity)
|
|
}
|
|
if _, err := audit.DecodeBundle(bytes.NewReader(c.Bundle)); err != nil {
|
|
return fmt.Errorf("bundle: %w", err)
|
|
}
|
|
return nil
|
|
}
|
|
|
|
func oneOf(v string, allowed ...string) bool {
|
|
for _, a := range allowed {
|
|
if v == a {
|
|
return true
|
|
}
|
|
}
|
|
return false
|
|
}
|
|
|
|
// Options configures a run.
|
|
type Options struct {
|
|
Judge judge.Judge
|
|
JudgeOptions judge.Options
|
|
// Names the judge in the report when the judge itself is generic.
|
|
Label string
|
|
}
|
|
|
|
// Result is one case's outcome under both the lens and the judge.
|
|
type Result struct {
|
|
Case Case
|
|
Baseline *audit.Report
|
|
Judged *audit.Report
|
|
Judge *judge.Verdict
|
|
Error string
|
|
}
|
|
|
|
// Scores summarises how well one configuration did.
|
|
type Scores struct {
|
|
Total int `json:"total"`
|
|
Correct int `json:"correct"`
|
|
// Accuracy is the fraction of cases whose decision matched the label.
|
|
Accuracy float64 `json:"accuracy"`
|
|
// Confusion is expected decision -> observed decision, per case.
|
|
Confusion map[string]map[string]int `json:"confusion"`
|
|
|
|
// Nefarious precision and recall treat "the output is a vector for
|
|
// wrongdoing" as the positive class, because that is the class a gate exists
|
|
// to catch and a false positive in it is what makes operators stop trusting
|
|
// the gate.
|
|
NefariousExpected int `json:"nefarious_expected"`
|
|
NefariousCalled int `json:"nefarious_called"`
|
|
NefariousCorrect int `json:"nefarious_correct"`
|
|
NefariousRecall float64 `json:"nefarious_recall"`
|
|
NefariousPrecision float64 `json:"nefarious_precision"`
|
|
|
|
// BenignTotal and FalseBlocks measure the cost of being wrong about a good
|
|
// call, which is the other half of the trade.
|
|
BenignTotal int `json:"benign_total"`
|
|
FalseBlocks int `json:"false_blocks"`
|
|
FalseBlockRate float64 `json:"false_block_rate"`
|
|
UnderBlocks int `json:"under_blocks"` // expected a block, did not produce one
|
|
LegitimacyMatch int `json:"legitimacy_match"`
|
|
}
|
|
|
|
// Summary is a full comparison of the lens against the lens plus a judge.
|
|
type Summary struct {
|
|
Cases []Result `json:"cases"`
|
|
Baseline Scores `json:"baseline"`
|
|
Judged Scores `json:"judged"`
|
|
|
|
// Fixed and Broke are the judge's marginal value: cases where the judge
|
|
// changed the decision and the change was right, and where it was wrong.
|
|
Fixed int `json:"fixed"`
|
|
Broke int `json:"broke"`
|
|
Changed int `json:"changed"`
|
|
|
|
JudgeLabel string `json:"judge_label,omitempty"`
|
|
JudgeAuthority string `json:"judge_authority,omitempty"`
|
|
JudgeErrors int `json:"judge_errors"`
|
|
JudgeLatencyMS int `json:"judge_latency_ms_total"`
|
|
JudgeTokens int `json:"judge_tokens_total"`
|
|
JudgeCostUSD float64 `json:"judge_cost_usd_total"`
|
|
JudgeCallsMade int `json:"judge_calls_made"`
|
|
JudgeUnanswered int `json:"judge_unanswered"`
|
|
}
|
|
|
|
// Run measures the lens alone and the lens plus a judge on the same cases.
|
|
//
|
|
// A nil judge is allowed: the run then reports the baseline only, which is the
|
|
// right thing to do when no classifier is reachable.
|
|
func Run(ctx context.Context, a *audit.Auditor, opts Options, cases []Case) Summary {
|
|
s := Summary{JudgeLabel: opts.Label, JudgeAuthority: string(opts.JudgeOptions.Authority)}
|
|
for _, c := range cases {
|
|
res := Result{Case: c}
|
|
|
|
b, err := audit.DecodeBundle(bytes.NewReader(c.Bundle))
|
|
if err != nil {
|
|
res.Error = err.Error()
|
|
s.Cases = append(s.Cases, res)
|
|
continue
|
|
}
|
|
res.Baseline = a.Audit(b)
|
|
|
|
res.Judged = a.Audit(b)
|
|
if opts.Judge != nil {
|
|
v := opts.Judge.Judge(ctx, b, res.Judged, opts.JudgeOptions)
|
|
res.Judge = &v
|
|
s.JudgeCallsMade++
|
|
s.JudgeLatencyMS += v.LatencyMS
|
|
if v.Usage != nil {
|
|
s.JudgeTokens += v.Usage.TotalTokens
|
|
s.JudgeCostUSD += v.Usage.CostUSD
|
|
}
|
|
if !v.Ok() {
|
|
s.JudgeErrors++
|
|
if v.Error != "" && strings.Contains(v.Error, "no answers") {
|
|
s.JudgeUnanswered++
|
|
}
|
|
}
|
|
judge.Apply(res.Judged, v)
|
|
}
|
|
s.Cases = append(s.Cases, res)
|
|
}
|
|
|
|
s.Baseline = score(s.Cases, func(r Result) *audit.Report { return r.Baseline })
|
|
s.Judged = score(s.Cases, func(r Result) *audit.Report { return r.Judged })
|
|
|
|
for _, r := range s.Cases {
|
|
if r.Baseline == nil || r.Judged == nil {
|
|
continue
|
|
}
|
|
if r.Baseline.Decision == r.Judged.Decision {
|
|
continue
|
|
}
|
|
s.Changed++
|
|
baselineRight := r.Baseline.Decision == r.Case.Expected.Decision
|
|
judgedRight := r.Judged.Decision == r.Case.Expected.Decision
|
|
switch {
|
|
case judgedRight && !baselineRight:
|
|
s.Fixed++
|
|
case baselineRight && !judgedRight:
|
|
s.Broke++
|
|
}
|
|
}
|
|
return s
|
|
}
|
|
|
|
func score(cases []Result, report func(Result) *audit.Report) Scores {
|
|
s := Scores{Total: len(cases), Confusion: map[string]map[string]int{}}
|
|
for _, r := range cases {
|
|
rep := report(r)
|
|
if rep == nil {
|
|
s.Total--
|
|
continue
|
|
}
|
|
expected := r.Case.Expected
|
|
|
|
if s.Confusion[expected.Decision] == nil {
|
|
s.Confusion[expected.Decision] = map[string]int{}
|
|
}
|
|
s.Confusion[expected.Decision][rep.Decision]++
|
|
if rep.Decision == expected.Decision {
|
|
s.Correct++
|
|
}
|
|
if expected.Legitimacy != "" && rep.Legitimacy.Verdict == expected.Legitimacy {
|
|
s.LegitimacyMatch++
|
|
}
|
|
|
|
switch expected.Legitimacy {
|
|
case audit.Nefarious:
|
|
s.NefariousExpected++
|
|
if rep.Legitimacy.Verdict == audit.Nefarious {
|
|
s.NefariousCorrect++
|
|
}
|
|
case audit.Legitimate:
|
|
s.BenignTotal++
|
|
// A legitimate call that is correctly blocked on the validity axis
|
|
// is not a false block.
|
|
if rep.Decision == audit.DecisionBlock && expected.Decision != audit.DecisionBlock {
|
|
s.FalseBlocks++
|
|
}
|
|
}
|
|
if rep.Legitimacy.Verdict == audit.Nefarious {
|
|
s.NefariousCalled++
|
|
}
|
|
if expected.Decision == audit.DecisionBlock && rep.Decision != audit.DecisionBlock {
|
|
s.UnderBlocks++
|
|
}
|
|
}
|
|
|
|
s.Accuracy = ratio(s.Correct, s.Total)
|
|
s.NefariousRecall = ratio(s.NefariousCorrect, s.NefariousExpected)
|
|
s.NefariousPrecision = ratio(s.NefariousCorrect, s.NefariousCalled)
|
|
s.FalseBlockRate = ratio(s.FalseBlocks, s.BenignTotal)
|
|
return s
|
|
}
|
|
|
|
func ratio(numerator, denominator int) float64 {
|
|
if denominator == 0 {
|
|
return 0
|
|
}
|
|
return float64(numerator) / float64(denominator)
|
|
}
|
|
|
|
// Gate fails when the judged run misses any floor the operator set. A gate that
|
|
// cannot fail is decoration, so every floor is checked explicitly.
|
|
func (s Summary) Gate(minAccuracy, minRecall, minPrecision, maxFalseBlockRate float64) error {
|
|
var problems []string
|
|
if s.Judged.Accuracy < minAccuracy {
|
|
problems = append(problems, fmt.Sprintf("accuracy %.2f is below %.2f", s.Judged.Accuracy, minAccuracy))
|
|
}
|
|
if s.Judged.NefariousRecall < minRecall {
|
|
problems = append(problems, fmt.Sprintf("nefarious recall %.2f is below %.2f", s.Judged.NefariousRecall, minRecall))
|
|
}
|
|
if s.Judged.NefariousPrecision < minPrecision {
|
|
problems = append(problems, fmt.Sprintf("nefarious precision %.2f is below %.2f", s.Judged.NefariousPrecision, minPrecision))
|
|
}
|
|
if s.Judged.FalseBlockRate > maxFalseBlockRate {
|
|
problems = append(problems, fmt.Sprintf("false block rate %.2f is above %.2f", s.Judged.FalseBlockRate, maxFalseBlockRate))
|
|
}
|
|
if s.JudgeErrors > 0 {
|
|
problems = append(problems, fmt.Sprintf("%d judge calls failed", s.JudgeErrors))
|
|
}
|
|
if len(problems) > 0 {
|
|
return errors.New(strings.Join(problems, "; "))
|
|
}
|
|
return nil
|
|
}
|
|
|
|
// Text renders the per-case table and the comparison.
|
|
func (s Summary) Text() string {
|
|
var b strings.Builder
|
|
// Writing into a tabwriter buffer cannot fail; its Flush reports the error,
|
|
// and this is a rendered report rather than a fidelity-critical artefact.
|
|
w := tabwriter.NewWriter(&b, 0, 0, 2, ' ', 0)
|
|
_, _ = fmt.Fprintln(w, "case\texpected\tlens\tjudged\tjudge\tnote")
|
|
for _, r := range s.Cases {
|
|
if r.Baseline == nil || r.Judged == nil {
|
|
_, _ = fmt.Fprintf(w, "%s\t%s\t-\t-\t-\tERROR %s\n", r.Case.ID, r.Case.Expected.Decision, r.Error)
|
|
continue
|
|
}
|
|
judgeCell := "-"
|
|
if r.Judge != nil {
|
|
switch {
|
|
case !r.Judge.Ok():
|
|
judgeCell = "error"
|
|
case r.Judge.Legitimacy.Value != "":
|
|
judgeCell = fmt.Sprintf("%s %.2f", r.Judge.Legitimacy.Value, r.Judge.Legitimacy.Confidence)
|
|
default:
|
|
judgeCell = "signals only"
|
|
}
|
|
}
|
|
_, _ = fmt.Fprintf(w, "%s\t%s\t%s\t%s\t%s\t%s\n",
|
|
r.Case.ID, r.Case.Expected.Decision, r.Baseline.Decision, r.Judged.Decision, judgeCell, note(r))
|
|
}
|
|
_ = w.Flush()
|
|
|
|
b.WriteString("\n")
|
|
writeScores(&b, "lens only", s.Baseline)
|
|
writeScores(&b, "lens + judge", s.Judged)
|
|
fmt.Fprintf(&b, "\njudge marginal value: fixed %d, broke %d (of %d changed decisions)\n", s.Fixed, s.Broke, s.Changed)
|
|
if s.JudgeLabel != "" {
|
|
authority := s.JudgeAuthority
|
|
if authority == "" {
|
|
authority = string(judge.AuthorityAdvisory)
|
|
}
|
|
fmt.Fprintf(&b, "judge: %s (authority %s), %d calls, %d errors, %d ms total, %d tokens, $%.4f\n",
|
|
s.JudgeLabel, authority, s.JudgeCallsMade, s.JudgeErrors, s.JudgeLatencyMS, s.JudgeTokens, s.JudgeCostUSD)
|
|
if s.JudgeCallsMade > 0 {
|
|
fmt.Fprintf(&b, "per call: %d ms, %d tokens, $%.6f\n",
|
|
s.JudgeLatencyMS/max(s.JudgeCallsMade, 1), s.JudgeTokens/max(s.JudgeCallsMade, 1),
|
|
s.JudgeCostUSD/float64(max(s.JudgeCallsMade, 1)))
|
|
}
|
|
}
|
|
return b.String()
|
|
}
|
|
|
|
func writeScores(b *strings.Builder, label string, s Scores) {
|
|
fmt.Fprintf(b, "%-14s accuracy %2d/%-2d = %.2f | nefarious recall %.2f (%d/%d) precision %.2f (%d/%d)\n",
|
|
label, s.Correct, s.Total, s.Accuracy,
|
|
s.NefariousRecall, s.NefariousCorrect, s.NefariousExpected,
|
|
s.NefariousPrecision, s.NefariousCorrect, s.NefariousCalled)
|
|
fmt.Fprintf(b, "%-14s benign %d, false blocks %d (%.2f), missed blocks %d, confusion %s\n",
|
|
"", s.BenignTotal, s.FalseBlocks, s.FalseBlockRate, s.UnderBlocks, confusionString(s.Confusion))
|
|
}
|
|
|
|
func confusionString(confusion map[string]map[string]int) string {
|
|
keys := []string{audit.DecisionBlock, audit.DecisionReview, audit.DecisionAllow}
|
|
parts := make([]string, 0, len(keys))
|
|
for _, want := range keys {
|
|
got := confusion[want]
|
|
if len(got) == 0 {
|
|
continue
|
|
}
|
|
inner := make([]string, 0, len(keys))
|
|
for _, observed := range keys {
|
|
if n := got[observed]; n > 0 {
|
|
inner = append(inner, fmt.Sprintf("%s=%d", observed, n))
|
|
}
|
|
}
|
|
parts = append(parts, strings.Join([]string{want, strings.Join(inner, " ")}, "->"))
|
|
}
|
|
return strings.Join(parts, ", ")
|
|
}
|
|
|
|
func note(r Result) string {
|
|
if r.Baseline == nil || r.Judged == nil {
|
|
return ""
|
|
}
|
|
baselineRight := r.Baseline.Decision == r.Case.Expected.Decision
|
|
judgedRight := r.Judged.Decision == r.Case.Expected.Decision
|
|
switch {
|
|
case judgedRight && !baselineRight:
|
|
return "fixed by judge"
|
|
case baselineRight && !judgedRight:
|
|
return "broken by judge"
|
|
case judgedRight:
|
|
return "ok"
|
|
default:
|
|
return "missed"
|
|
}
|
|
}
|
|
|
|
// WriteJSON emits the full summary including per-case reports.
|
|
func WriteJSON(w io.Writer, s Summary) error {
|
|
enc := json.NewEncoder(w)
|
|
enc.SetIndent("", " ")
|
|
if err := enc.Encode(s); err != nil {
|
|
return fmt.Errorf("encode summary: %w", err)
|
|
}
|
|
return nil
|
|
}
|