31 lines
1.0 KiB
Go
31 lines
1.0 KiB
Go
package eval
|
|
|
|
import (
|
|
"time"
|
|
|
|
"github.com/samcharles93/archipelago/sdk/judgegate"
|
|
)
|
|
|
|
// Record is the recorded outcome of measuring the judge model against the
|
|
// labelled cases: the shared record the guard reads at start, and what the
|
|
// lens alone scored on the same cases.
|
|
type Record struct {
|
|
judgegate.Record
|
|
Lens judgegate.Score `json:"lens"`
|
|
Fixed int `json:"fixed"`
|
|
Broke int `json:"broke"`
|
|
}
|
|
|
|
// NewRecord records a run against a threshold. The positive class is a nefarious
|
|
// call.
|
|
func NewRecord(model string, at time.Time, s Summary, t judgegate.Threshold) Record {
|
|
score := func(x Scores) judgegate.Score {
|
|
return judgegate.Score{Accuracy: x.Accuracy, Recall: x.NefariousRecall, Precision: x.NefariousPrecision, FalseFlagRate: x.FalseBlockRate}
|
|
}
|
|
measured := judgegate.Measured{Score: score(s.Judged), Cases: s.Judged.Total, Errors: s.JudgeErrors}
|
|
return Record{
|
|
Record: judgegate.NewRecord(model, at, measured, t),
|
|
Lens: score(s.Baseline), Fixed: s.Fixed, Broke: s.Broke,
|
|
}
|
|
}
|