108 lines
3.5 KiB
Go
108 lines
3.5 KiB
Go
package judgegate
|
|
|
|
import (
|
|
"crypto/sha256"
|
|
"encoding/hex"
|
|
"time"
|
|
|
|
"github.com/samcharles93/archipelago/sdk/guard"
|
|
"github.com/samcharles93/archipelago/sdk/systemone"
|
|
)
|
|
|
|
// Standing is a judge's standing from the recorded eval: whether its verdicts
|
|
// count, and if not, why.
|
|
type Standing struct {
|
|
Trusted bool
|
|
Reason string
|
|
EvalDate string
|
|
}
|
|
|
|
// StandingOf reads a shipped evaluation result and says where it leaves model at
|
|
// now. An unreadable result leaves the judge untrusted, with the cause as reason.
|
|
func StandingOf(evalJSON []byte, model string, now time.Time, maxAge time.Duration) Standing {
|
|
s := Standing{Reason: AdvisoryReason(evalJSON, model, now, maxAge)}
|
|
s.Trusted = s.Reason == ""
|
|
if rec, err := ParseRecord(evalJSON); err == nil {
|
|
s.EvalDate = rec.Date
|
|
}
|
|
return s
|
|
}
|
|
|
|
// Call is one judge call: who answered, with what result id, and what it was
|
|
// judged on beyond the action.
|
|
type Call struct {
|
|
Model string
|
|
ResultID string
|
|
// Answers is the service's answers as JSON.
|
|
Answers []byte
|
|
// Basis is the text the judge was given beyond the action.
|
|
Basis string
|
|
// Request and Response are the exact bodies sent and received, and the
|
|
// path, status and latency that describe the call. They are set even when
|
|
// the call failed.
|
|
Request []byte
|
|
Response []byte
|
|
EndpointPath string
|
|
HTTPStatus int
|
|
LatencyMS int64
|
|
}
|
|
|
|
// CallOf describes a call made to model that returned res, whether or not it
|
|
// succeeded: a failed call still names the model, the basis it was given, and
|
|
// what was sent and received.
|
|
func CallOf(model, basis string, res systemone.Result) Call {
|
|
c := Call{
|
|
Model: model, ResultID: res.ID, Basis: basis,
|
|
Request: res.Request, Response: res.Response,
|
|
EndpointPath: res.EndpointPath, HTTPStatus: res.HTTPStatus, LatencyMS: res.Latency.Milliseconds(),
|
|
}
|
|
if res.Model != "" {
|
|
c.Model = res.Model
|
|
}
|
|
if len(res.Answers) > 0 {
|
|
c.Answers = res.AnswersJSON()
|
|
}
|
|
return c
|
|
}
|
|
|
|
// Attach copies what the call sent and received into a review record, so a
|
|
// case's answer keeps its payload even when the judge failed.
|
|
func (c Call) Attach(g Graded) Graded {
|
|
g.Request, g.Response = string(c.Request), string(c.Response)
|
|
g.EndpointPath, g.HTTPStatus, g.LatencyMS = c.EndpointPath, c.HTTPStatus, c.LatencyMS
|
|
return g
|
|
}
|
|
|
|
// Record is what the call answered, kept whole so the verdict can be audited
|
|
// without asking the model again.
|
|
func (c Call) Record(answer string, confidence float64, reason string, st Standing) guard.JudgeRecord {
|
|
r := guard.JudgeRecord{
|
|
Model: c.Model, ResultID: c.ResultID, Answer: answer, Confidence: confidence, Reason: reason,
|
|
Basis: c.Basis, Answers: c.Answers,
|
|
Trusted: st.Trusted, AdvisoryReason: st.Reason, EvalDate: st.EvalDate,
|
|
Request: c.Request, Response: c.Response,
|
|
EndpointPath: c.EndpointPath, HTTPStatus: c.HTTPStatus, LatencyMS: c.LatencyMS,
|
|
}
|
|
if c.Basis != "" {
|
|
sum := sha256.Sum256([]byte(c.Basis))
|
|
r.BasisDigest = hex.EncodeToString(sum[:])
|
|
}
|
|
return r
|
|
}
|
|
|
|
// Failed is the record of a call that errored: the error is the reason.
|
|
func (c Call) Failed(err error, st Standing) guard.JudgeRecord {
|
|
return c.Record("", 0, "the judge call failed: "+err.Error(), st)
|
|
}
|
|
|
|
// FailedVerdict is what a guard answers when its judge could not: the action
|
|
// holds at propose and wait, as an error would hold it, and the record keeps
|
|
// what was attempted.
|
|
func FailedVerdict(rec guard.JudgeRecord) guard.Verdict {
|
|
return guard.Verdict{
|
|
Ceiling: guard.ProposeAndWait,
|
|
BlindSpots: []string{rec.Reason},
|
|
Judges: []guard.JudgeRecord{rec},
|
|
}
|
|
}
|