feat(judgegate): an eval writes every case's answer, confidence and reason beside its label for review
publish / publish (push) Successful in 1m19s

This commit is contained in:
sam committed 2026-11-16 11:11:45 +11:00
1 parent bd087e3e6d
commit 54fe5028cb
4 files changed
+70 -27

No files matched your search

+7 -4
View File
@@ -58,14 +58,17 @@ func load(path string) ([]evalCase, error) {
return f.Cases, nil
}
func judgeCase(ctx context.Context, client systemone.Client, c evalCase) (flagged, want bool, err error) {
func judgeCase(ctx context.Context, client systemone.Client, c evalCase) (judgegate.Graded, error) {
g := judgegate.Graded{ID: c.ID, Label: c.Label, Want: c.Label == judge.Unsupported}
j, err := judge.New(client)
if err != nil {
return false, false, err
return g, err
}
out, err := j.Assess(ctx, c.Case)
if err != nil {
return false, false, err
return g, err
}
return out.Label == judge.Unsupported, c.Label == judge.Unsupported, nil
g.Answer, g.Confidence, g.Reason = out.Label, out.Confidence, out.Gap
g.Flagged = out.Label == judge.Unsupported
return g, nil
}
+7 -4
View File
@@ -71,14 +71,17 @@ func load(path string) ([]evalCase, error) {
return f.Cases, nil
}
func judgeCase(ctx context.Context, client systemone.Client, c evalCase) (flagged, want bool, err error) {
func judgeCase(ctx context.Context, client systemone.Client, c evalCase) (judgegate.Graded, error) {
g := judgegate.Graded{ID: c.ID, Label: c.Label, Want: c.Label == judge.Violates}
j, err := judge.New(client, c.rules, judge.DefaultMinConfidence)
if err != nil {
return false, false, err
return g, err
}
out, err := j.Assess(ctx, c.Action)
if err != nil {
return false, false, err
return g, err
}
return out.Label == judge.Violates, c.Label == judge.Violates, nil
g.Answer, g.Confidence, g.Reason = out.Label, out.Confidence, out.Rule
g.Flagged = out.Label == judge.Violates
return g, nil
}
+18 -5
View File
@@ -23,9 +23,9 @@ type EvalSpec[T any] struct {
Threshold Threshold
// Load reads the labelled cases from path.
Load func(path string) ([]T, error)
// Judge asks the judge about one case, reporting whether it flagged the
// case and whether the label says it should be flagged.
Judge func(ctx context.Context, c systemone.Client, tc T) (flagged, want bool, err error)
// Judge asks the judge about one case and grades its answer against the
// case's label. On an error it still returns the case's id and label.
Judge func(ctx context.Context, c systemone.Client, tc T) (Graded, error)
}
// RunEval is a guard's cmd/eval: it measures the judge named on the command
@@ -35,6 +35,7 @@ func RunEval[T any](spec EvalSpec[T]) error {
model := flag.String("model", "", "judge model id")
cases := flag.String("cases", "eval/cases.json", "labelled cases")
out := flag.String("out", "eval_result.json", "where to write the result")
answers := flag.String("answers", "eval/answers.json", "where to write every case's answer for review")
flag.Parse()
if *base == "" || *model == "" {
return errors.New("-base and -model are required")
@@ -44,11 +45,11 @@ func RunEval[T any](spec EvalSpec[T]) error {
return err
}
client := systemone.Client{BaseURL: *base, Model: *model, APIKey: os.Getenv(spec.KeyEnv), Label: "judge"}
measured := Measure(context.Background(), loaded, func(ctx context.Context, tc T) (bool, bool, error) {
measured := Measure(context.Background(), loaded, func(ctx context.Context, tc T) (Graded, error) {
return spec.Judge(ctx, client, tc)
})
rec := NewRecord(*model, time.Now(), measured, spec.Threshold)
fmt.Printf("%s: %d cases answered, %d judge errors, missed cases %v\n", spec.Name, measured.Cases, measured.Errors, measured.Missed)
fmt.Printf("%s: %d cases answered, %d judge errors, disagreeing with the label %v\n", spec.Name, measured.Cases, measured.Errors, measured.Disagreed)
if measured.FirstError != "" {
fmt.Println("first judge error:", measured.FirstError)
}
@@ -61,6 +62,18 @@ func RunEval[T any](spec EvalSpec[T]) error {
if err := os.WriteFile(*out, append(data, '\n'), 0o600); err != nil {
return err
}
review := struct {
Model string `json:"model"`
Date string `json:"date"`
Answers []Graded `json:"answers"`
}{rec.Model, rec.Date, measured.Answers}
data, err = json.MarshalIndent(review, "", " ")
if err != nil {
return err
}
if err := os.WriteFile(*answers, append(data, '\n'), 0o600); err != nil {
return err
}
if short := rec.Shortfalls(); len(short) > 0 {
fmt.Println("the judge stays advisory:", short)
} else {
+38 -14
View File
@@ -165,34 +165,58 @@ type Measured struct {
Cases int
// Errors counts cases where the judge failed; they are left out of Score.
Errors int
// Missed lists the index of each case the judge got wrong.
Missed []int
// Disagreed lists the index of each case where the judge and the label
// differ.
Disagreed []int
// FirstError is the first judge failure, to say what went wrong.
FirstError string
// Answers is the judge's answer to every case, in case order, for a person
// to review against the labels.
Answers []Graded
}
// Measure runs every case through judge. judge reports whether the judge
// flagged the case and whether the label says it should be flagged.
func Measure[T any](ctx context.Context, cases []T, judge func(context.Context, T) (flagged, want bool, err error)) Measured {
// Graded is the judge's answer to one labelled case.
type Graded struct {
ID string `json:"id"`
// Label is what the case file says; Answer is what the judge said.
Label string `json:"label"`
Answer string `json:"answer,omitempty"`
// Flagged is whether the answer counts as flagging the case, and Want
// whether the label does.
Flagged bool `json:"flagged"`
Want bool `json:"want"`
Confidence float64 `json:"confidence,omitempty"`
// Reason is what the answer rests on: the rule cited or the gap found.
Reason string `json:"reason,omitempty"`
Error string `json:"error,omitempty"`
}
// Measure runs every case through judge and scores the answers against the
// labels. A case the judge fails on is kept in Answers and left out of Score.
func Measure[T any](ctx context.Context, cases []T, judge func(context.Context, T) (Graded, error)) Measured {
var m Measured
var tp, fp, fn, tn int
for i, c := range cases {
flagged, want, err := judge(ctx, c)
switch {
case err != nil:
g, err := judge(ctx, c)
if err != nil {
g.Error = err.Error()
m.Answers = append(m.Answers, g)
m.Errors++
if m.FirstError == "" {
m.FirstError = err.Error()
m.FirstError = g.Error
}
continue
case flagged && want:
}
m.Answers = append(m.Answers, g)
switch {
case g.Flagged && g.Want:
tp++
case flagged:
case g.Flagged:
fp++
m.Missed = append(m.Missed, i)
case want:
m.Disagreed = append(m.Disagreed, i)
case g.Want:
fn++
m.Missed = append(m.Missed, i)
m.Disagreed = append(m.Disagreed, i)
default:
tn++
}