244 lines
8.7 KiB
Go
244 lines
8.7 KiB
Go
// Package audit judges recorded LLM calls against a declarative rule lens.
|
|
//
|
|
// The auditor is static and offline by design: it never calls a model and never
|
|
// touches the network, so a verdict is reproducible from the bundle alone. Every
|
|
// finding carries a locator and a verbatim excerpt from the bundle, which is what
|
|
// makes a verdict reviewable rather than persuasive.
|
|
package audit
|
|
|
|
import (
|
|
"crypto/sha256"
|
|
"encoding/hex"
|
|
"encoding/json"
|
|
"errors"
|
|
"fmt"
|
|
"io"
|
|
"os"
|
|
"strings"
|
|
)
|
|
|
|
// Bundle is one recorded LLM call: the request, the response, and the context
|
|
// needed to judge it. It is the only input the auditor reads.
|
|
type Bundle struct {
|
|
ID string `json:"id,omitempty"`
|
|
Subject Subject `json:"subject,omitempty"`
|
|
Request Request `json:"request"`
|
|
Response Response `json:"response"`
|
|
Contract *Contract `json:"contract,omitempty"`
|
|
Sources []Source `json:"sources,omitempty"`
|
|
Integrity *Integrity `json:"integrity,omitempty"`
|
|
}
|
|
|
|
// Subject records who produced the call. Absent fields are a coverage finding,
|
|
// not an error: an unattributed call can be judged but not held to account.
|
|
type Subject struct {
|
|
Provider string `json:"provider,omitempty"`
|
|
Model string `json:"model,omitempty"`
|
|
Actor string `json:"actor,omitempty"`
|
|
Timestamp string `json:"timestamp,omitempty"`
|
|
Purpose string `json:"purpose,omitempty"`
|
|
}
|
|
|
|
// Request is the input side of the call.
|
|
type Request struct {
|
|
Messages []Message `json:"messages,omitempty"`
|
|
Tools []Tool `json:"tools,omitempty"`
|
|
// ToolCalls are the calls already made in this conversation, as the harness
|
|
// replays them. An agent harness resends the whole transcript on every turn, so
|
|
// the last request of a session carries its complete tool history - which is what
|
|
// makes a trajectory auditable from a single call.
|
|
//
|
|
// Note for aggregation: because the transcript is replayed, this field is not
|
|
// additive across calls. Summing it counts the same call once per subsequent
|
|
// turn. See internal/session for the dedup rule.
|
|
ToolCalls []ToolCall `json:"tool_calls,omitempty"`
|
|
Parameters map[string]any `json:"parameters,omitempty"`
|
|
// NewFrom is the index of the first message this call added to the
|
|
// conversation; earlier messages were already present in a previous call of
|
|
// the same session. Nil means unknown, and every message counts as new. It is
|
|
// set by the record source, not sent by the caller, so it is excluded from the
|
|
// request hash.
|
|
NewFrom *int `json:"new_from,omitempty"`
|
|
}
|
|
|
|
// Message is one request-side turn.
|
|
type Message struct {
|
|
Role string `json:"role"`
|
|
Content string `json:"content,omitempty"`
|
|
}
|
|
|
|
// Tool is a tool the caller declared as available for this call.
|
|
type Tool struct {
|
|
Name string `json:"name"`
|
|
Parameters map[string]any `json:"parameters,omitempty"`
|
|
}
|
|
|
|
// Response is the output side of the call. Content is prose, Output is the
|
|
// structured payload when the caller asked for JSON, ToolCalls are requested
|
|
// actions. Tool-only responses legitimately have empty Content.
|
|
type Response struct {
|
|
Content string `json:"content,omitempty"`
|
|
FinishReason string `json:"finish_reason,omitempty"`
|
|
ToolCalls []ToolCall `json:"tool_calls,omitempty"`
|
|
Output any `json:"output,omitempty"`
|
|
Usage *Usage `json:"usage,omitempty"`
|
|
Refusal bool `json:"refusal,omitempty"`
|
|
}
|
|
|
|
// ToolCall is one action the model asked the caller to perform.
|
|
type ToolCall struct {
|
|
Name string `json:"name"`
|
|
Arguments map[string]any `json:"arguments,omitempty"`
|
|
// ArgumentsRaw is the arguments exactly as the transport delivered them, when
|
|
// it delivered a string. A model can emit invalid JSON here, and that is a
|
|
// validity failure rather than a missing value, so the raw text is kept.
|
|
ArgumentsRaw string `json:"arguments_raw,omitempty"`
|
|
ID string `json:"id,omitempty"`
|
|
}
|
|
|
|
// Usage is optional accounting; its absence is a completeness finding.
|
|
//
|
|
// CacheReadTokens matters more than it looks. An agent harness replays the whole
|
|
// transcript every turn, so most of the traffic is re-read context rather than new
|
|
// input, and on a long run the cache-read count dwarfs both other numbers.
|
|
type Usage struct {
|
|
PromptTokens int `json:"prompt_tokens,omitempty"`
|
|
CompletionTokens int `json:"completion_tokens,omitempty"`
|
|
CacheReadTokens int `json:"cache_read_tokens,omitempty"`
|
|
}
|
|
|
|
// Contract is what the call was supposed to produce. Without it the auditor can
|
|
// still judge trustworthiness but cannot certify validity, and says so.
|
|
type Contract struct {
|
|
Task string `json:"task,omitempty"`
|
|
OutputSchema map[string]any `json:"output_schema,omitempty"`
|
|
RequiredTools []string `json:"required_tools,omitempty"`
|
|
RequireGrounding bool `json:"require_grounding,omitempty"`
|
|
MustInclude []string `json:"must_include,omitempty"`
|
|
MustNotInclude []string `json:"must_not_include,omitempty"`
|
|
ExpectedFinishReasons []string `json:"expected_finish_reasons,omitempty"`
|
|
ExpectedRefusal *bool `json:"expected_refusal,omitempty"`
|
|
}
|
|
|
|
// Source is retrieved context the response may be grounded in.
|
|
type Source struct {
|
|
ID string `json:"id,omitempty"`
|
|
URI string `json:"uri,omitempty"`
|
|
Text string `json:"text,omitempty"`
|
|
}
|
|
|
|
// Integrity is the chain-of-custody record for the call.
|
|
type Integrity struct {
|
|
// RequestSHA256 is the canonical request hash, as printed by `auditor hash`.
|
|
RequestSHA256 string `json:"request_sha256,omitempty"`
|
|
}
|
|
|
|
// LoadBundle reads a bundle from a path, or from stdin when path is "-".
|
|
func LoadBundle(path string) (*Bundle, error) {
|
|
var r io.Reader
|
|
if path == "-" {
|
|
r = os.Stdin
|
|
} else {
|
|
f, err := os.Open(path)
|
|
if err != nil {
|
|
return nil, fmt.Errorf("open bundle %s: %w", path, err)
|
|
}
|
|
defer func() { _ = f.Close() }()
|
|
r = f
|
|
}
|
|
b, err := DecodeBundle(r)
|
|
if err != nil {
|
|
return nil, fmt.Errorf("decode bundle %s: %w", path, err)
|
|
}
|
|
return b, nil
|
|
}
|
|
|
|
// DecodeBundle decodes strictly. An unknown field is an error rather than
|
|
// ignored, because a bundle that has drifted from the documented schema is
|
|
// exactly the case where silent tolerance would produce a false clean verdict.
|
|
func DecodeBundle(r io.Reader) (*Bundle, error) {
|
|
dec := json.NewDecoder(r)
|
|
dec.UseNumber()
|
|
dec.DisallowUnknownFields()
|
|
|
|
var b Bundle
|
|
if err := dec.Decode(&b); err != nil {
|
|
return nil, err
|
|
}
|
|
var extra any
|
|
if err := dec.Decode(&extra); !errors.Is(err, io.EOF) {
|
|
return nil, errors.New("trailing data after bundle")
|
|
}
|
|
if err := b.Validate(); err != nil {
|
|
return nil, err
|
|
}
|
|
return &b, nil
|
|
}
|
|
|
|
// Validate rejects bundles that cannot be audited at all. It deliberately does
|
|
// not check completeness: a sparse bundle is audited and reported as such.
|
|
func (b *Bundle) Validate() error {
|
|
if b.Response.empty() {
|
|
return errors.New("response is empty: nothing to audit")
|
|
}
|
|
for i, t := range b.Request.Tools {
|
|
if t.Name == "" {
|
|
return fmt.Errorf("request.tools[%d]: name is required", i)
|
|
}
|
|
}
|
|
for i, c := range b.Response.ToolCalls {
|
|
if c.Name == "" {
|
|
return fmt.Errorf("response.tool_calls[%d]: name is required", i)
|
|
}
|
|
}
|
|
for i, m := range b.Request.Messages {
|
|
if m.Role == "" {
|
|
return fmt.Errorf("request.messages[%d]: role is required", i)
|
|
}
|
|
}
|
|
if n := b.Request.NewFrom; n != nil && (*n < 0 || *n > len(b.Request.Messages)) {
|
|
return fmt.Errorf("request.new_from %d is outside 0..%d", *n, len(b.Request.Messages))
|
|
}
|
|
if b.Integrity != nil && b.Integrity.RequestSHA256 != "" && !isHex64(b.Integrity.RequestSHA256) {
|
|
return errors.New("integrity.request_sha256 must be 64 lowercase hex characters")
|
|
}
|
|
return nil
|
|
}
|
|
|
|
func (r Response) empty() bool {
|
|
return r.Content == "" && r.FinishReason == "" && r.Output == nil &&
|
|
len(r.ToolCalls) == 0 && !r.Refusal
|
|
}
|
|
|
|
// isHex64 requires lowercase, matching the documented format: the declared hash
|
|
// is machine-produced by `auditor hash`, so upper case means a hand-edit.
|
|
func isHex64(s string) bool {
|
|
if len(s) != 64 || s != strings.ToLower(s) {
|
|
return false
|
|
}
|
|
_, err := hex.DecodeString(s)
|
|
return err == nil
|
|
}
|
|
|
|
// CanonicalRequest returns the byte form the request hash is taken over: compact
|
|
// JSON of the decoded request, object keys sorted, numbers preserved as written.
|
|
func (b *Bundle) CanonicalRequest() ([]byte, error) {
|
|
request := b.Request
|
|
request.NewFrom = nil
|
|
raw, err := json.Marshal(request)
|
|
if err != nil {
|
|
return nil, fmt.Errorf("canonicalise request: %w", err)
|
|
}
|
|
return raw, nil
|
|
}
|
|
|
|
// RequestHash returns the hex SHA-256 of CanonicalRequest.
|
|
func (b *Bundle) RequestHash() (string, error) {
|
|
raw, err := b.CanonicalRequest()
|
|
if err != nil {
|
|
return "", err
|
|
}
|
|
sum := sha256.Sum256(raw)
|
|
return hex.EncodeToString(sum[:]), nil
|
|
}
|