Files
archipelago/guards/confidence/cmd/eval/main.go
T

77 lines
2.2 KiB
Go

// eval measures the confidence guard's judge on the labelled cases and writes
// the result the guard reads at start. Run it from guards/confidence:
//
// go run ./cmd/eval -api systemone -base <base URL> -model <model> -out eval_result.json
// go run ./cmd/eval -api openai-decisions -model gpt-6-luna -out eval_result.openai-decisions.json
//
// The API key is read from CONFIDENCE_JUDGE_API_KEY for systemone and from
// OPENAI_API_KEY for openai-decisions.
// The positive class is a diagnosis its evidence does not support.
package main
import (
"context"
"errors"
"fmt"
"os"
"github.com/samcharles93/archipelago/guards/confidence/internal/judge"
"github.com/samcharles93/archipelago/sdk/judgegate"
)
var threshold = judgegate.Threshold{
MinRecall: 0.8,
MinPrecision: 0.8,
MaxFalseFlagRate: 0.05,
Why: "the gate the integrity guard uses for a judge allowed to set a ceiling: flag four in five unsupported diagnoses, be right four times in five when flagging one, and flag at most one supported diagnosis in twenty",
}
type evalCase struct {
ID string `json:"id"`
judge.Case
// Label is supported, partial or unsupported.
Label string `json:"label"`
}
func main() {
if err := judgegate.RunEval(judgegate.EvalSpec[evalCase]{
Name: "confidence", KeyEnv: "CONFIDENCE_JUDGE_API_KEY", Threshold: threshold, Load: load, Judge: judgeCase,
}); err != nil {
fmt.Fprintln(os.Stderr, "eval:", err)
os.Exit(1)
}
}
func load(path string) ([]evalCase, error) {
data, err := judgegate.ReadFile(path)
if err != nil {
return nil, err
}
var f struct {
Cases []evalCase `json:"cases"`
}
if err := judgegate.ReadJSON(data, &f); err != nil {
return nil, fmt.Errorf("cases: %w", err)
}
if len(f.Cases) == 0 {
return nil, errors.New("no cases")
}
return f.Cases, nil
}
func judgeCase(ctx context.Context, client judgegate.Client, c evalCase) (judgegate.Graded, error) {
g := judgegate.Graded{ID: c.ID, Label: c.Label, Want: c.Label == judge.Unsupported}
j, err := judge.New(client)
if err != nil {
return g, err
}
out, err := j.Assess(ctx, c.Case)
g = out.Call.Attach(g)
if err != nil {
return g, err
}
g.Answer, g.Confidence, g.Reason = out.Label, out.Confidence, out.Gap
g.Flagged = out.Label == judge.Unsupported
return g, nil
}