392 lines
13 KiB
Go
392 lines
13 KiB
Go
package audit
|
|
|
|
import (
|
|
"fmt"
|
|
"sort"
|
|
"strings"
|
|
)
|
|
|
|
// Verdict values. Legitimacy answers whether the output is a safe, honest thing
|
|
// to act on; validity answers whether it satisfies the declared contract.
|
|
const (
|
|
Legitimate = "legitimate"
|
|
Nefarious = "nefarious"
|
|
Inconclusive = "inconclusive"
|
|
|
|
Valid = "valid"
|
|
Invalid = "invalid"
|
|
|
|
DecisionAllow = "allow"
|
|
DecisionReview = "review"
|
|
DecisionBlock = "block"
|
|
)
|
|
|
|
// Confidence values: proven means a rule matched text present in the bundle;
|
|
// judge-confirmed means a classifier asserted the signal is real; likely means an
|
|
// unconfirmed heuristic that a human must judge.
|
|
const (
|
|
ConfidenceProven = "proven"
|
|
ConfidenceJudge = "judge-confirmed"
|
|
ConfidenceLikely = "likely"
|
|
JudgeRulePrefix = "JUDGE-"
|
|
JudgeLegitimacyID = "JUDGE-LEG"
|
|
JudgeValidityID = "JUDGE-VAL"
|
|
)
|
|
|
|
// notAudited states the lens's permanent blind spots. It is reported with every
|
|
// verdict so a clean result is never read as a guarantee.
|
|
var notAudited = []string{
|
|
"model internals, provider logs, and training data",
|
|
"tool calls that were executed: only the requested call is visible, never its effect",
|
|
"context outside the bundle: other turns, other sessions, and the caller's own state",
|
|
"non-English and paraphrased variants of the phrase rules",
|
|
}
|
|
|
|
// Evidence is the proof for a finding: where, and what was actually there.
|
|
type Evidence struct {
|
|
Locator string `json:"locator"`
|
|
Quote string `json:"quote"`
|
|
Detail string `json:"detail,omitempty"`
|
|
}
|
|
|
|
// Finding is one rule hit, carrying its own mechanism so it stands alone.
|
|
type Finding struct {
|
|
RuleID string `json:"rule_id"`
|
|
Title string `json:"title"`
|
|
Axis string `json:"axis"`
|
|
Kind string `json:"kind"`
|
|
Severity string `json:"severity"`
|
|
Confidence string `json:"confidence"`
|
|
Mechanism string `json:"mechanism"`
|
|
Remediation string `json:"remediation"`
|
|
Evidence []Evidence `json:"evidence"`
|
|
// JudgeProbability is the classifier's probability that this signal is real.
|
|
// It is recorded even when it did not change the outcome, because it is the
|
|
// evidence a later eval or fine-tune is built from.
|
|
JudgeProbability *float64 `json:"judge_probability,omitempty"`
|
|
// Dismissed means a judge ruled the finding out. The finding is kept in the
|
|
// report - nothing is deleted from an audit - but it no longer affects the
|
|
// verdict.
|
|
Dismissed bool `json:"dismissed,omitempty"`
|
|
// Provenance says where the flagged text already appeared in the context
|
|
// window before the response: "user", "operator", "read", or empty when it
|
|
// appeared nowhere and the model produced it itself. ProvenanceAt locates the
|
|
// message it was traced to.
|
|
Provenance string `json:"provenance,omitempty"`
|
|
ProvenanceAt string `json:"provenance_at,omitempty"`
|
|
}
|
|
|
|
// Verdict is one axis of the judgement plus the reason for it.
|
|
type Verdict struct {
|
|
Verdict string `json:"verdict"`
|
|
Reason string `json:"reason"`
|
|
}
|
|
|
|
// Skip records a rule that was not applicable, so silence is never mistaken for
|
|
// a pass.
|
|
type Skip struct {
|
|
RuleID string `json:"rule_id"`
|
|
Reason string `json:"reason"`
|
|
}
|
|
|
|
// Coverage states what was actually evaluated.
|
|
type Coverage struct {
|
|
ContractProvided bool `json:"contract_provided"`
|
|
RulesEvaluated int `json:"rules_evaluated"`
|
|
AxesEvaluated map[string]int `json:"axes_evaluated,omitempty"`
|
|
Skipped []Skip `json:"skipped,omitempty"`
|
|
NotAudited []string `json:"not_audited"`
|
|
}
|
|
|
|
// JudgeRecord is what a judge was asked and answered, stored in the report so a
|
|
// judge-influenced decision stays auditable and becomes eval data.
|
|
type JudgeRecord struct {
|
|
Provider string `json:"provider,omitempty"`
|
|
Deployment string `json:"deployment,omitempty"`
|
|
ResponseID string `json:"response_id,omitempty"`
|
|
Model string `json:"model,omitempty"`
|
|
Authority string `json:"authority,omitempty"`
|
|
MinConfidence float64 `json:"min_confidence,omitempty"`
|
|
Legitimacy string `json:"legitimacy,omitempty"`
|
|
Confidence float64 `json:"confidence,omitempty"`
|
|
Probabilities map[string]float64 `json:"probabilities,omitempty"`
|
|
Validity string `json:"validity,omitempty"`
|
|
ValidityConfidence float64 `json:"validity_confidence,omitempty"`
|
|
ValidityProbabilities map[string]float64 `json:"validity_probabilities,omitempty"`
|
|
SignalProbabilities map[string]float64 `json:"signal_probabilities,omitempty"`
|
|
InputTokens int `json:"input_tokens,omitempty"`
|
|
OutputTokens int `json:"output_tokens,omitempty"`
|
|
CostUSD float64 `json:"cost_usd,omitempty"`
|
|
LatencyMS int `json:"latency_ms,omitempty"`
|
|
Error string `json:"error,omitempty"`
|
|
ChangedLegitimacy bool `json:"changed_legitimacy,omitempty"`
|
|
ChangedValidity bool `json:"changed_validity,omitempty"`
|
|
TruncatedInput bool `json:"truncated_input,omitempty"`
|
|
ProvenViolationsIgnored bool `json:"proven_violations_ignored,omitempty"`
|
|
}
|
|
|
|
// Report is the audit result for one bundle.
|
|
type Report struct {
|
|
BundleID string `json:"bundle_id,omitempty"`
|
|
Subject string `json:"subject,omitempty"`
|
|
Legitimacy Verdict `json:"legitimacy"`
|
|
Validity Verdict `json:"validity"`
|
|
Decision string `json:"decision"`
|
|
Reason string `json:"reason"`
|
|
Findings []Finding `json:"findings"`
|
|
Coverage Coverage `json:"coverage"`
|
|
Judge *JudgeRecord `json:"judge,omitempty"`
|
|
}
|
|
|
|
// Result is what a primitive reports: at most one finding, or a skip with a
|
|
// reason. A primitive that neither fires nor skips returns the zero Result.
|
|
type Result struct {
|
|
Finding *Finding
|
|
Skipped bool
|
|
SkipReason string
|
|
}
|
|
|
|
func skip(reason string) Result { return Result{Skipped: true, SkipReason: reason} }
|
|
|
|
func hit(ev ...Evidence) Result { return Result{Finding: &Finding{Evidence: ev}} }
|
|
|
|
// evalContext is the prepared view of one bundle shared by all rules.
|
|
type evalContext struct {
|
|
bundle *Bundle
|
|
surfaces surfaces
|
|
}
|
|
|
|
func newEvalContext(b *Bundle) *evalContext {
|
|
return &evalContext{bundle: b, surfaces: b.surfaces()}
|
|
}
|
|
|
|
// Auditor is a prepared lens: rules compiled once, reusable across bundles.
|
|
type Auditor struct {
|
|
rules []preparedRule
|
|
contextMessages int
|
|
}
|
|
|
|
// WithContext returns an auditor that traces each finding against the last n
|
|
// request messages. Zero means DefaultContextMessages and a negative n the whole
|
|
// request.
|
|
func (a *Auditor) WithContext(n int) *Auditor {
|
|
out := *a
|
|
out.contextMessages = n
|
|
return &out
|
|
}
|
|
|
|
// ContextMessages is the size of the window findings are traced against.
|
|
func (a *Auditor) ContextMessages() int {
|
|
if a.contextMessages == 0 {
|
|
return DefaultContextMessages
|
|
}
|
|
return a.contextMessages
|
|
}
|
|
|
|
// NewAuditor validates and compiles a lens.
|
|
func NewAuditor(rules []Rule) (*Auditor, error) {
|
|
prepared, err := Prepare(rules)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
return &Auditor{rules: prepared}, nil
|
|
}
|
|
|
|
// Rules returns the lens, in declaration order.
|
|
func (a *Auditor) Rules() []Rule {
|
|
out := make([]Rule, 0, len(a.rules))
|
|
for _, r := range a.rules {
|
|
out = append(out, r.Rule)
|
|
}
|
|
return out
|
|
}
|
|
|
|
// Audit judges one bundle. It does not mutate the bundle.
|
|
func (a *Auditor) Audit(b *Bundle) *Report {
|
|
ctx := newEvalContext(b)
|
|
ran := make(map[string]int)
|
|
rep := &Report{
|
|
BundleID: b.ID,
|
|
Subject: subjectLabel(b.Subject),
|
|
Findings: []Finding{},
|
|
Coverage: Coverage{
|
|
ContractProvided: b.Contract != nil,
|
|
NotAudited: notAudited,
|
|
},
|
|
}
|
|
|
|
window := b.contextWindow(a.ContextMessages())
|
|
for _, pr := range a.rules {
|
|
res := pr.checker.run(ctx)
|
|
rep.Coverage.RulesEvaluated++
|
|
if res.Skipped {
|
|
rep.Coverage.Skipped = append(rep.Coverage.Skipped, Skip{RuleID: pr.ID, Reason: res.SkipReason})
|
|
continue
|
|
}
|
|
ran[pr.Axis]++
|
|
if res.Finding == nil {
|
|
continue
|
|
}
|
|
f := *res.Finding
|
|
f.RuleID = pr.ID
|
|
f.Title = pr.Title
|
|
f.Axis = pr.Axis
|
|
f.Kind = pr.Kind
|
|
f.Severity = pr.Severity
|
|
f.Mechanism = pr.Rationale
|
|
f.Remediation = pr.Remediation
|
|
f.Confidence = ConfidenceProven
|
|
if pr.Kind == KindSignal {
|
|
f.Confidence = ConfidenceLikely
|
|
}
|
|
b.traceFinding(&f, window, pr.Check.Primitive == "secret_like")
|
|
rep.Findings = append(rep.Findings, f)
|
|
}
|
|
|
|
rep.Coverage.AxesEvaluated = ran
|
|
rep.Recompute()
|
|
return rep
|
|
}
|
|
|
|
// Recompute re-derives every verdict from the current findings. The audit calls
|
|
// it, and so does a judge that has confirmed, dismissed or added findings, so the
|
|
// aggregation rules stay in exactly one place.
|
|
func (r *Report) Recompute() {
|
|
sortFindings(r.Findings)
|
|
active := make([]Finding, 0, len(r.Findings))
|
|
for _, f := range r.Findings {
|
|
if !f.Dismissed {
|
|
active = append(active, f)
|
|
}
|
|
}
|
|
r.Legitimacy = judgeLegitimacy(active)
|
|
r.Validity = judgeValidity(active, r.Coverage)
|
|
r.Decision, r.Reason = decide(r, active)
|
|
}
|
|
|
|
// Audit is the one-shot convenience form of NewAuditor + Audit.
|
|
func Audit(b *Bundle, rules []Rule) (*Report, error) {
|
|
a, err := NewAuditor(rules)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
return a.Audit(b), nil
|
|
}
|
|
|
|
func judgeLegitimacy(findings []Finding) Verdict {
|
|
var violations, signals []Finding
|
|
for _, f := range findings {
|
|
if f.Axis != AxisLegitimacy {
|
|
continue
|
|
}
|
|
if f.Kind == KindViolation {
|
|
violations = append(violations, f)
|
|
} else {
|
|
signals = append(signals, f)
|
|
}
|
|
}
|
|
switch {
|
|
case len(violations) > 0:
|
|
return Verdict{Nefarious, "trust violations: " + ruleIDs(violations) + " (see the confidence on each finding)"}
|
|
case len(signals) > 0:
|
|
return Verdict{Inconclusive, "signals requiring human review before the output is trusted: " + ruleIDs(signals)}
|
|
default:
|
|
return Verdict{Legitimate, "no legitimacy rule matched"}
|
|
}
|
|
}
|
|
|
|
func judgeValidity(findings []Finding, cov Coverage) Verdict {
|
|
var violations, signals []Finding
|
|
for _, f := range findings {
|
|
if f.Axis != AxisValidity {
|
|
continue
|
|
}
|
|
if f.Kind == KindViolation {
|
|
violations = append(violations, f)
|
|
} else {
|
|
signals = append(signals, f)
|
|
}
|
|
}
|
|
switch {
|
|
case len(violations) > 0:
|
|
return Verdict{Invalid, "contract violations: " + ruleIDs(violations)}
|
|
case len(signals) > 0:
|
|
return Verdict{Inconclusive, "grounding or consistency signals: " + ruleIDs(signals)}
|
|
case cov.AxesEvaluated[AxisValidity] == 0:
|
|
return Verdict{Inconclusive, "no validity rule was applicable: " + firstValiditySkipReason(cov)}
|
|
case !cov.ContractProvided:
|
|
return Verdict{Inconclusive, "no contract supplied, so conformance cannot be established"}
|
|
default:
|
|
return Verdict{Valid, "no contract rule was violated"}
|
|
}
|
|
}
|
|
|
|
// decide turns verdicts into the operational outcome. Absence of a contract is
|
|
// an admission of scope, not a defect, so it never forces review on its own.
|
|
func decide(rep *Report, active []Finding) (string, string) {
|
|
switch {
|
|
case rep.Legitimacy.Verdict == Nefarious:
|
|
return DecisionBlock, rep.Legitimacy.Reason
|
|
case rep.Validity.Verdict == Invalid:
|
|
return DecisionBlock, rep.Validity.Reason
|
|
}
|
|
for _, f := range active {
|
|
if f.Axis == AxisRecord {
|
|
return DecisionReview, "record integrity is not established: " + f.RuleID + " (" + f.Title + ")"
|
|
}
|
|
}
|
|
if len(active) > 0 {
|
|
return DecisionReview, "signals require human review: " + ruleIDs(active)
|
|
}
|
|
return DecisionAllow, "no rule matched"
|
|
}
|
|
|
|
func firstValiditySkipReason(cov Coverage) string {
|
|
for _, s := range cov.Skipped {
|
|
if strings.HasPrefix(s.RuleID, "VAL-") {
|
|
return "no validity rule was applicable: " + s.Reason
|
|
}
|
|
}
|
|
return "no validity rule was applicable"
|
|
}
|
|
|
|
func ruleIDs(findings []Finding) string {
|
|
ids := make([]string, 0, len(findings))
|
|
for _, f := range findings {
|
|
ids = append(ids, f.RuleID)
|
|
}
|
|
sort.Strings(ids)
|
|
return strings.Join(ids, ", ")
|
|
}
|
|
|
|
func sortFindings(findings []Finding) {
|
|
rank := map[string]int{SevP0: 0, SevP1: 1, SevP2: 2}
|
|
sort.SliceStable(findings, func(i, j int) bool {
|
|
a, b := findings[i], findings[j]
|
|
if rank[a.Severity] != rank[b.Severity] {
|
|
return rank[a.Severity] < rank[b.Severity]
|
|
}
|
|
if a.Axis != b.Axis {
|
|
return a.Axis < b.Axis
|
|
}
|
|
return a.RuleID < b.RuleID
|
|
})
|
|
}
|
|
|
|
func subjectLabel(s Subject) string {
|
|
switch {
|
|
case s.Provider != "" && s.Model != "":
|
|
return s.Provider + "/" + s.Model
|
|
case s.Model != "":
|
|
return s.Model
|
|
default:
|
|
return ""
|
|
}
|
|
}
|
|
|
|
// Summary is the one-line form used by both the text report and callers that
|
|
// want a compact feed.
|
|
func (f Finding) Summary() string {
|
|
return fmt.Sprintf("[%s %s/%s] %s — %s", f.Severity, f.Axis, f.Kind, f.RuleID, f.Title)
|
|
}
|