From 54fe5028cbbdd0dd1680134eeb6d7329e2800e9c Mon Sep 17 00:00:00 2001 From: Sam Catlow Date: Mon, 16 Nov 2026 11:11:45 +1100 Subject: [PATCH] feat(judgegate): an eval writes every case's answer, confidence and reason beside its label for review --- guards/confidence/cmd/eval/main.go | 11 ++++--- guards/policy/cmd/eval/main.go | 11 ++++--- sdk/judgegate/eval.go | 23 ++++++++++--- sdk/judgegate/judgegate.go | 52 ++++++++++++++++++++++-------- 4 files changed, 70 insertions(+), 27 deletions(-) diff --git a/guards/confidence/cmd/eval/main.go b/guards/confidence/cmd/eval/main.go index fea1e3a..8c29b2f 100644 --- a/guards/confidence/cmd/eval/main.go +++ b/guards/confidence/cmd/eval/main.go @@ -58,14 +58,17 @@ func load(path string) ([]evalCase, error) { return f.Cases, nil } -func judgeCase(ctx context.Context, client systemone.Client, c evalCase) (flagged, want bool, err error) { +func judgeCase(ctx context.Context, client systemone.Client, c evalCase) (judgegate.Graded, error) { + g := judgegate.Graded{ID: c.ID, Label: c.Label, Want: c.Label == judge.Unsupported} j, err := judge.New(client) if err != nil { - return false, false, err + return g, err } out, err := j.Assess(ctx, c.Case) if err != nil { - return false, false, err + return g, err } - return out.Label == judge.Unsupported, c.Label == judge.Unsupported, nil + g.Answer, g.Confidence, g.Reason = out.Label, out.Confidence, out.Gap + g.Flagged = out.Label == judge.Unsupported + return g, nil } diff --git a/guards/policy/cmd/eval/main.go b/guards/policy/cmd/eval/main.go index 93c1243..426e7b5 100644 --- a/guards/policy/cmd/eval/main.go +++ b/guards/policy/cmd/eval/main.go @@ -71,14 +71,17 @@ func load(path string) ([]evalCase, error) { return f.Cases, nil } -func judgeCase(ctx context.Context, client systemone.Client, c evalCase) (flagged, want bool, err error) { +func judgeCase(ctx context.Context, client systemone.Client, c evalCase) (judgegate.Graded, error) { + g := judgegate.Graded{ID: c.ID, Label: c.Label, Want: c.Label == judge.Violates} j, err := judge.New(client, c.rules, judge.DefaultMinConfidence) if err != nil { - return false, false, err + return g, err } out, err := j.Assess(ctx, c.Action) if err != nil { - return false, false, err + return g, err } - return out.Label == judge.Violates, c.Label == judge.Violates, nil + g.Answer, g.Confidence, g.Reason = out.Label, out.Confidence, out.Rule + g.Flagged = out.Label == judge.Violates + return g, nil } diff --git a/sdk/judgegate/eval.go b/sdk/judgegate/eval.go index 1c71e4e..c442e2f 100644 --- a/sdk/judgegate/eval.go +++ b/sdk/judgegate/eval.go @@ -23,9 +23,9 @@ type EvalSpec[T any] struct { Threshold Threshold // Load reads the labelled cases from path. Load func(path string) ([]T, error) - // Judge asks the judge about one case, reporting whether it flagged the - // case and whether the label says it should be flagged. - Judge func(ctx context.Context, c systemone.Client, tc T) (flagged, want bool, err error) + // Judge asks the judge about one case and grades its answer against the + // case's label. On an error it still returns the case's id and label. + Judge func(ctx context.Context, c systemone.Client, tc T) (Graded, error) } // RunEval is a guard's cmd/eval: it measures the judge named on the command @@ -35,6 +35,7 @@ func RunEval[T any](spec EvalSpec[T]) error { model := flag.String("model", "", "judge model id") cases := flag.String("cases", "eval/cases.json", "labelled cases") out := flag.String("out", "eval_result.json", "where to write the result") + answers := flag.String("answers", "eval/answers.json", "where to write every case's answer for review") flag.Parse() if *base == "" || *model == "" { return errors.New("-base and -model are required") @@ -44,11 +45,11 @@ func RunEval[T any](spec EvalSpec[T]) error { return err } client := systemone.Client{BaseURL: *base, Model: *model, APIKey: os.Getenv(spec.KeyEnv), Label: "judge"} - measured := Measure(context.Background(), loaded, func(ctx context.Context, tc T) (bool, bool, error) { + measured := Measure(context.Background(), loaded, func(ctx context.Context, tc T) (Graded, error) { return spec.Judge(ctx, client, tc) }) rec := NewRecord(*model, time.Now(), measured, spec.Threshold) - fmt.Printf("%s: %d cases answered, %d judge errors, missed cases %v\n", spec.Name, measured.Cases, measured.Errors, measured.Missed) + fmt.Printf("%s: %d cases answered, %d judge errors, disagreeing with the label %v\n", spec.Name, measured.Cases, measured.Errors, measured.Disagreed) if measured.FirstError != "" { fmt.Println("first judge error:", measured.FirstError) } @@ -61,6 +62,18 @@ func RunEval[T any](spec EvalSpec[T]) error { if err := os.WriteFile(*out, append(data, '\n'), 0o600); err != nil { return err } + review := struct { + Model string `json:"model"` + Date string `json:"date"` + Answers []Graded `json:"answers"` + }{rec.Model, rec.Date, measured.Answers} + data, err = json.MarshalIndent(review, "", " ") + if err != nil { + return err + } + if err := os.WriteFile(*answers, append(data, '\n'), 0o600); err != nil { + return err + } if short := rec.Shortfalls(); len(short) > 0 { fmt.Println("the judge stays advisory:", short) } else { diff --git a/sdk/judgegate/judgegate.go b/sdk/judgegate/judgegate.go index 51934ff..a5573fa 100644 --- a/sdk/judgegate/judgegate.go +++ b/sdk/judgegate/judgegate.go @@ -165,34 +165,58 @@ type Measured struct { Cases int // Errors counts cases where the judge failed; they are left out of Score. Errors int - // Missed lists the index of each case the judge got wrong. - Missed []int + // Disagreed lists the index of each case where the judge and the label + // differ. + Disagreed []int // FirstError is the first judge failure, to say what went wrong. FirstError string + // Answers is the judge's answer to every case, in case order, for a person + // to review against the labels. + Answers []Graded } -// Measure runs every case through judge. judge reports whether the judge -// flagged the case and whether the label says it should be flagged. -func Measure[T any](ctx context.Context, cases []T, judge func(context.Context, T) (flagged, want bool, err error)) Measured { +// Graded is the judge's answer to one labelled case. +type Graded struct { + ID string `json:"id"` + // Label is what the case file says; Answer is what the judge said. + Label string `json:"label"` + Answer string `json:"answer,omitempty"` + // Flagged is whether the answer counts as flagging the case, and Want + // whether the label does. + Flagged bool `json:"flagged"` + Want bool `json:"want"` + Confidence float64 `json:"confidence,omitempty"` + // Reason is what the answer rests on: the rule cited or the gap found. + Reason string `json:"reason,omitempty"` + Error string `json:"error,omitempty"` +} + +// Measure runs every case through judge and scores the answers against the +// labels. A case the judge fails on is kept in Answers and left out of Score. +func Measure[T any](ctx context.Context, cases []T, judge func(context.Context, T) (Graded, error)) Measured { var m Measured var tp, fp, fn, tn int for i, c := range cases { - flagged, want, err := judge(ctx, c) - switch { - case err != nil: + g, err := judge(ctx, c) + if err != nil { + g.Error = err.Error() + m.Answers = append(m.Answers, g) m.Errors++ if m.FirstError == "" { - m.FirstError = err.Error() + m.FirstError = g.Error } continue - case flagged && want: + } + m.Answers = append(m.Answers, g) + switch { + case g.Flagged && g.Want: tp++ - case flagged: + case g.Flagged: fp++ - m.Missed = append(m.Missed, i) - case want: + m.Disagreed = append(m.Disagreed, i) + case g.Want: fn++ - m.Missed = append(m.Missed, i) + m.Disagreed = append(m.Disagreed, i) default: tn++ }