Files

392 lines
13 KiB
Go

package audit
import (
"fmt"
"sort"
"strings"
)
// Verdict values. Legitimacy answers whether the output is a safe, honest thing
// to act on; validity answers whether it satisfies the declared contract.
const (
Legitimate = "legitimate"
Nefarious = "nefarious"
Inconclusive = "inconclusive"
Valid = "valid"
Invalid = "invalid"
DecisionAllow = "allow"
DecisionReview = "review"
DecisionBlock = "block"
)
// Confidence values: proven means a rule matched text present in the bundle;
// judge-confirmed means a classifier asserted the signal is real; likely means an
// unconfirmed heuristic that a human must judge.
const (
ConfidenceProven = "proven"
ConfidenceJudge = "judge-confirmed"
ConfidenceLikely = "likely"
JudgeRulePrefix = "JUDGE-"
JudgeLegitimacyID = "JUDGE-LEG"
JudgeValidityID = "JUDGE-VAL"
)
// notAudited states the lens's permanent blind spots. It is reported with every
// verdict so a clean result is never read as a guarantee.
var notAudited = []string{
"model internals, provider logs, and training data",
"tool calls that were executed: only the requested call is visible, never its effect",
"context outside the bundle: other turns, other sessions, and the caller's own state",
"non-English and paraphrased variants of the phrase rules",
}
// Evidence is the proof for a finding: where, and what was actually there.
type Evidence struct {
Locator string `json:"locator"`
Quote string `json:"quote"`
Detail string `json:"detail,omitempty"`
}
// Finding is one rule hit, carrying its own mechanism so it stands alone.
type Finding struct {
RuleID string `json:"rule_id"`
Title string `json:"title"`
Axis string `json:"axis"`
Kind string `json:"kind"`
Severity string `json:"severity"`
Confidence string `json:"confidence"`
Mechanism string `json:"mechanism"`
Remediation string `json:"remediation"`
Evidence []Evidence `json:"evidence"`
// JudgeProbability is the classifier's probability that this signal is real.
// It is recorded even when it did not change the outcome, because it is the
// evidence a later eval or fine-tune is built from.
JudgeProbability *float64 `json:"judge_probability,omitempty"`
// Dismissed means a judge ruled the finding out. The finding is kept in the
// report - nothing is deleted from an audit - but it no longer affects the
// verdict.
Dismissed bool `json:"dismissed,omitempty"`
// Provenance says where the flagged text already appeared in the context
// window before the response: "user", "operator", "read", or empty when it
// appeared nowhere and the model produced it itself. ProvenanceAt locates the
// message it was traced to.
Provenance string `json:"provenance,omitempty"`
ProvenanceAt string `json:"provenance_at,omitempty"`
}
// Verdict is one axis of the judgement plus the reason for it.
type Verdict struct {
Verdict string `json:"verdict"`
Reason string `json:"reason"`
}
// Skip records a rule that was not applicable, so silence is never mistaken for
// a pass.
type Skip struct {
RuleID string `json:"rule_id"`
Reason string `json:"reason"`
}
// Coverage states what was actually evaluated.
type Coverage struct {
ContractProvided bool `json:"contract_provided"`
RulesEvaluated int `json:"rules_evaluated"`
AxesEvaluated map[string]int `json:"axes_evaluated,omitempty"`
Skipped []Skip `json:"skipped,omitempty"`
NotAudited []string `json:"not_audited"`
}
// JudgeRecord is what a judge was asked and answered, stored in the report so a
// judge-influenced decision stays auditable and becomes eval data.
type JudgeRecord struct {
Provider string `json:"provider,omitempty"`
Deployment string `json:"deployment,omitempty"`
ResponseID string `json:"response_id,omitempty"`
Model string `json:"model,omitempty"`
Authority string `json:"authority,omitempty"`
MinConfidence float64 `json:"min_confidence,omitempty"`
Legitimacy string `json:"legitimacy,omitempty"`
Confidence float64 `json:"confidence,omitempty"`
Probabilities map[string]float64 `json:"probabilities,omitempty"`
Validity string `json:"validity,omitempty"`
ValidityConfidence float64 `json:"validity_confidence,omitempty"`
ValidityProbabilities map[string]float64 `json:"validity_probabilities,omitempty"`
SignalProbabilities map[string]float64 `json:"signal_probabilities,omitempty"`
InputTokens int `json:"input_tokens,omitempty"`
OutputTokens int `json:"output_tokens,omitempty"`
CostUSD float64 `json:"cost_usd,omitempty"`
LatencyMS int `json:"latency_ms,omitempty"`
Error string `json:"error,omitempty"`
ChangedLegitimacy bool `json:"changed_legitimacy,omitempty"`
ChangedValidity bool `json:"changed_validity,omitempty"`
TruncatedInput bool `json:"truncated_input,omitempty"`
ProvenViolationsIgnored bool `json:"proven_violations_ignored,omitempty"`
}
// Report is the audit result for one bundle.
type Report struct {
BundleID string `json:"bundle_id,omitempty"`
Subject string `json:"subject,omitempty"`
Legitimacy Verdict `json:"legitimacy"`
Validity Verdict `json:"validity"`
Decision string `json:"decision"`
Reason string `json:"reason"`
Findings []Finding `json:"findings"`
Coverage Coverage `json:"coverage"`
Judge *JudgeRecord `json:"judge,omitempty"`
}
// Result is what a primitive reports: at most one finding, or a skip with a
// reason. A primitive that neither fires nor skips returns the zero Result.
type Result struct {
Finding *Finding
Skipped bool
SkipReason string
}
func skip(reason string) Result { return Result{Skipped: true, SkipReason: reason} }
func hit(ev ...Evidence) Result { return Result{Finding: &Finding{Evidence: ev}} }
// evalContext is the prepared view of one bundle shared by all rules.
type evalContext struct {
bundle *Bundle
surfaces surfaces
}
func newEvalContext(b *Bundle) *evalContext {
return &evalContext{bundle: b, surfaces: b.surfaces()}
}
// Auditor is a prepared lens: rules compiled once, reusable across bundles.
type Auditor struct {
rules []preparedRule
contextMessages int
}
// WithContext returns an auditor that traces each finding against the last n
// request messages. Zero means DefaultContextMessages and a negative n the whole
// request.
func (a *Auditor) WithContext(n int) *Auditor {
out := *a
out.contextMessages = n
return &out
}
// ContextMessages is the size of the window findings are traced against.
func (a *Auditor) ContextMessages() int {
if a.contextMessages == 0 {
return DefaultContextMessages
}
return a.contextMessages
}
// NewAuditor validates and compiles a lens.
func NewAuditor(rules []Rule) (*Auditor, error) {
prepared, err := Prepare(rules)
if err != nil {
return nil, err
}
return &Auditor{rules: prepared}, nil
}
// Rules returns the lens, in declaration order.
func (a *Auditor) Rules() []Rule {
out := make([]Rule, 0, len(a.rules))
for _, r := range a.rules {
out = append(out, r.Rule)
}
return out
}
// Audit judges one bundle. It does not mutate the bundle.
func (a *Auditor) Audit(b *Bundle) *Report {
ctx := newEvalContext(b)
ran := make(map[string]int)
rep := &Report{
BundleID: b.ID,
Subject: subjectLabel(b.Subject),
Findings: []Finding{},
Coverage: Coverage{
ContractProvided: b.Contract != nil,
NotAudited: notAudited,
},
}
window := b.contextWindow(a.ContextMessages())
for _, pr := range a.rules {
res := pr.checker.run(ctx)
rep.Coverage.RulesEvaluated++
if res.Skipped {
rep.Coverage.Skipped = append(rep.Coverage.Skipped, Skip{RuleID: pr.ID, Reason: res.SkipReason})
continue
}
ran[pr.Axis]++
if res.Finding == nil {
continue
}
f := *res.Finding
f.RuleID = pr.ID
f.Title = pr.Title
f.Axis = pr.Axis
f.Kind = pr.Kind
f.Severity = pr.Severity
f.Mechanism = pr.Rationale
f.Remediation = pr.Remediation
f.Confidence = ConfidenceProven
if pr.Kind == KindSignal {
f.Confidence = ConfidenceLikely
}
b.traceFinding(&f, window, pr.Check.Primitive == "secret_like")
rep.Findings = append(rep.Findings, f)
}
rep.Coverage.AxesEvaluated = ran
rep.Recompute()
return rep
}
// Recompute re-derives every verdict from the current findings. The audit calls
// it, and so does a judge that has confirmed, dismissed or added findings, so the
// aggregation rules stay in exactly one place.
func (r *Report) Recompute() {
sortFindings(r.Findings)
active := make([]Finding, 0, len(r.Findings))
for _, f := range r.Findings {
if !f.Dismissed {
active = append(active, f)
}
}
r.Legitimacy = judgeLegitimacy(active)
r.Validity = judgeValidity(active, r.Coverage)
r.Decision, r.Reason = decide(r, active)
}
// Audit is the one-shot convenience form of NewAuditor + Audit.
func Audit(b *Bundle, rules []Rule) (*Report, error) {
a, err := NewAuditor(rules)
if err != nil {
return nil, err
}
return a.Audit(b), nil
}
func judgeLegitimacy(findings []Finding) Verdict {
var violations, signals []Finding
for _, f := range findings {
if f.Axis != AxisLegitimacy {
continue
}
if f.Kind == KindViolation {
violations = append(violations, f)
} else {
signals = append(signals, f)
}
}
switch {
case len(violations) > 0:
return Verdict{Nefarious, "trust violations: " + ruleIDs(violations) + " (see the confidence on each finding)"}
case len(signals) > 0:
return Verdict{Inconclusive, "signals requiring human review before the output is trusted: " + ruleIDs(signals)}
default:
return Verdict{Legitimate, "no legitimacy rule matched"}
}
}
func judgeValidity(findings []Finding, cov Coverage) Verdict {
var violations, signals []Finding
for _, f := range findings {
if f.Axis != AxisValidity {
continue
}
if f.Kind == KindViolation {
violations = append(violations, f)
} else {
signals = append(signals, f)
}
}
switch {
case len(violations) > 0:
return Verdict{Invalid, "contract violations: " + ruleIDs(violations)}
case len(signals) > 0:
return Verdict{Inconclusive, "grounding or consistency signals: " + ruleIDs(signals)}
case cov.AxesEvaluated[AxisValidity] == 0:
return Verdict{Inconclusive, "no validity rule was applicable: " + firstValiditySkipReason(cov)}
case !cov.ContractProvided:
return Verdict{Inconclusive, "no contract supplied, so conformance cannot be established"}
default:
return Verdict{Valid, "no contract rule was violated"}
}
}
// decide turns verdicts into the operational outcome. Absence of a contract is
// an admission of scope, not a defect, so it never forces review on its own.
func decide(rep *Report, active []Finding) (string, string) {
switch {
case rep.Legitimacy.Verdict == Nefarious:
return DecisionBlock, rep.Legitimacy.Reason
case rep.Validity.Verdict == Invalid:
return DecisionBlock, rep.Validity.Reason
}
for _, f := range active {
if f.Axis == AxisRecord {
return DecisionReview, "record integrity is not established: " + f.RuleID + " (" + f.Title + ")"
}
}
if len(active) > 0 {
return DecisionReview, "signals require human review: " + ruleIDs(active)
}
return DecisionAllow, "no rule matched"
}
func firstValiditySkipReason(cov Coverage) string {
for _, s := range cov.Skipped {
if strings.HasPrefix(s.RuleID, "VAL-") {
return "no validity rule was applicable: " + s.Reason
}
}
return "no validity rule was applicable"
}
func ruleIDs(findings []Finding) string {
ids := make([]string, 0, len(findings))
for _, f := range findings {
ids = append(ids, f.RuleID)
}
sort.Strings(ids)
return strings.Join(ids, ", ")
}
func sortFindings(findings []Finding) {
rank := map[string]int{SevP0: 0, SevP1: 1, SevP2: 2}
sort.SliceStable(findings, func(i, j int) bool {
a, b := findings[i], findings[j]
if rank[a.Severity] != rank[b.Severity] {
return rank[a.Severity] < rank[b.Severity]
}
if a.Axis != b.Axis {
return a.Axis < b.Axis
}
return a.RuleID < b.RuleID
})
}
func subjectLabel(s Subject) string {
switch {
case s.Provider != "" && s.Model != "":
return s.Provider + "/" + s.Model
case s.Model != "":
return s.Model
default:
return ""
}
}
// Summary is the one-line form used by both the text report and callers that
// want a compact feed.
func (f Finding) Summary() string {
return fmt.Sprintf("[%s %s/%s] %s — %s", f.Severity, f.Axis, f.Kind, f.RuleID, f.Title)
}