1299 lines
35 KiB
Go
1299 lines
35 KiB
Go
package audit
|
|
|
|
import (
|
|
"encoding/base64"
|
|
"encoding/json"
|
|
"errors"
|
|
"fmt"
|
|
"net/url"
|
|
"regexp"
|
|
"strings"
|
|
"unicode"
|
|
"unicode/utf8"
|
|
)
|
|
|
|
// This file holds the primitive checks the lens is built from. A primitive
|
|
// reports at most one finding, carrying every piece of evidence it found, so a
|
|
// rule maps to exactly one finding and the aggregation stays predictable.
|
|
|
|
const (
|
|
defaultMaxEvidence = 5
|
|
defaultMaxClaims = 25
|
|
// minIdentifierLen keeps short field values out of grounding checks while
|
|
// still catching ticket and invoice style identifiers.
|
|
minIdentifierLen = 6
|
|
)
|
|
|
|
// ---------------------------------------------------------------------------
|
|
// text_match
|
|
// ---------------------------------------------------------------------------
|
|
|
|
type textMatchParams struct {
|
|
Patterns []string `json:"patterns"`
|
|
Scope []string `json:"scope,omitempty"`
|
|
Mode string `json:"mode,omitempty"` // any (default) | all
|
|
Negate bool `json:"negate,omitempty"`
|
|
IgnoreCase bool `json:"ignore_case,omitempty"`
|
|
MaxEvidence int `json:"max_evidence,omitempty"`
|
|
Segment string `json:"segment,omitempty"`
|
|
}
|
|
|
|
const segmentCommand = "command"
|
|
|
|
type textMatch struct {
|
|
patterns []*regexp.Regexp
|
|
raw []string
|
|
scope []string
|
|
mode string
|
|
negate bool
|
|
max int
|
|
segment string
|
|
}
|
|
|
|
func newTextMatch(raw json.RawMessage) (Checker, error) {
|
|
var p textMatchParams
|
|
if err := decodeParams(raw, &p); err != nil {
|
|
return nil, err
|
|
}
|
|
m := textMatch{scope: p.Scope, mode: p.Mode, negate: p.Negate, max: p.MaxEvidence, raw: p.Patterns, segment: p.Segment}
|
|
if m.mode == "" {
|
|
m.mode = "any"
|
|
}
|
|
if m.mode != "any" && m.mode != "all" {
|
|
return nil, fmt.Errorf("mode %q must be any or all", m.mode)
|
|
}
|
|
if m.max <= 0 {
|
|
m.max = defaultMaxEvidence
|
|
}
|
|
if m.segment != "" && m.segment != segmentCommand {
|
|
return nil, fmt.Errorf("segment %q must be %q or omitted", m.segment, segmentCommand)
|
|
}
|
|
if m.segment != "" && m.negate {
|
|
return nil, errors.New("segment cannot be combined with negate")
|
|
}
|
|
if err := validateScope(p.Scope); err != nil {
|
|
return nil, err
|
|
}
|
|
if len(p.Patterns) == 0 {
|
|
return nil, errors.New("patterns must not be empty")
|
|
}
|
|
for _, pat := range p.Patterns {
|
|
expr := pat
|
|
if p.IgnoreCase {
|
|
expr = "(?i)" + pat
|
|
}
|
|
re, err := regexp.Compile(expr)
|
|
if err != nil {
|
|
return nil, fmt.Errorf("pattern %q: %w", pat, err)
|
|
}
|
|
m.patterns = append(m.patterns, re)
|
|
}
|
|
return m, nil
|
|
}
|
|
|
|
func (m textMatch) run(ctx *evalContext) Result {
|
|
if m.segment == segmentCommand {
|
|
return m.runSegments(ctx)
|
|
}
|
|
scanned := ctx.surfaces.selectSurfaces(m.scope)
|
|
|
|
if m.negate {
|
|
return m.runNegation(scanned)
|
|
}
|
|
|
|
var evidence []Evidence
|
|
for _, sur := range scanned {
|
|
for i, re := range m.patterns {
|
|
loc := re.FindStringIndex(sur.Text)
|
|
if loc == nil {
|
|
continue
|
|
}
|
|
evidence = append(evidence, Evidence{
|
|
Locator: sur.Locator,
|
|
Quote: excerpt(sur.Text[loc[0]:loc[1]], 200),
|
|
Detail: fmt.Sprintf("matched /%s/", m.raw[i]),
|
|
})
|
|
if len(evidence) >= m.max {
|
|
return hit(evidence...)
|
|
}
|
|
}
|
|
}
|
|
if len(evidence) == 0 {
|
|
return Result{}
|
|
}
|
|
return hit(evidence...)
|
|
}
|
|
|
|
// runSegments matches every pattern against each shell command segment of each
|
|
// decoded string in scope, so a pattern anchored with ^ sees the start of a
|
|
// command and never the start of a longer text. Evidence quotes the segment.
|
|
func (m textMatch) runSegments(ctx *evalContext) Result {
|
|
var evidence []Evidence
|
|
for _, sur := range ctx.decodedSurfaces(m.scope) {
|
|
candidates := commandCandidates(sur.Text)
|
|
for i, re := range m.patterns {
|
|
for _, candidate := range candidates {
|
|
if !re.MatchString(candidate) {
|
|
continue
|
|
}
|
|
evidence = append(evidence, Evidence{
|
|
Locator: sur.Locator,
|
|
Quote: excerpt(candidate, 200),
|
|
Detail: fmt.Sprintf("matched /%s/", m.raw[i]),
|
|
})
|
|
break
|
|
}
|
|
if len(evidence) >= m.max {
|
|
return hit(evidence...)
|
|
}
|
|
}
|
|
}
|
|
if len(evidence) == 0 {
|
|
return Result{}
|
|
}
|
|
return hit(evidence...)
|
|
}
|
|
|
|
// runNegation implements the "must contain" form: the rule fires when the
|
|
// required text is absent, so evidence is the surrounding context plus a
|
|
// statement of what was missing.
|
|
func (m textMatch) runNegation(scanned []Surface) Result {
|
|
matched := make([]bool, len(m.patterns))
|
|
for _, sur := range scanned {
|
|
for i, re := range m.patterns {
|
|
if re.MatchString(sur.Text) {
|
|
matched[i] = true
|
|
}
|
|
}
|
|
}
|
|
need := 1
|
|
if m.mode == "all" {
|
|
need = len(m.patterns)
|
|
}
|
|
found := 0
|
|
for _, ok := range matched {
|
|
if ok {
|
|
found++
|
|
}
|
|
}
|
|
if found >= need {
|
|
return Result{}
|
|
}
|
|
|
|
var missing []string
|
|
for i, ok := range matched {
|
|
if !ok {
|
|
missing = append(missing, m.raw[i])
|
|
}
|
|
}
|
|
ev := Evidence{Detail: "no match for: " + strings.Join(missing, ", ")}
|
|
if len(scanned) > 0 {
|
|
ev.Locator = scanned[0].Locator
|
|
ev.Quote = excerpt(scanned[0].Text, 200)
|
|
} else {
|
|
ev.Locator = strings.Join(m.scope, "+")
|
|
}
|
|
return hit(ev)
|
|
}
|
|
|
|
// ---------------------------------------------------------------------------
|
|
// hidden_text
|
|
// ---------------------------------------------------------------------------
|
|
|
|
type hiddenTextParams struct {
|
|
Detectors []string `json:"detectors"`
|
|
Scope []string `json:"scope,omitempty"`
|
|
}
|
|
|
|
type hiddenText struct {
|
|
detectors map[string]bool
|
|
scope []string
|
|
}
|
|
|
|
var (
|
|
htmlCommentRe = regexp.MustCompile(`(?s)<!--.*?-->`)
|
|
cssHiddenRe = regexp.MustCompile(`(?i)(display\s*:\s*none|visibility\s*:\s*hidden|font-size\s*:\s*0(px|em|rem)?\b|opacity\s*:\s*0(\.0+)?\b)`)
|
|
encodedBlobRe = regexp.MustCompile(`[A-Za-z0-9+/=_-]{200,}`)
|
|
)
|
|
|
|
func newHiddenText(raw json.RawMessage) (Checker, error) {
|
|
var p hiddenTextParams
|
|
if err := decodeParams(raw, &p); err != nil {
|
|
return nil, err
|
|
}
|
|
if len(p.Detectors) == 0 {
|
|
return nil, errors.New("detectors must not be empty")
|
|
}
|
|
known := map[string]bool{
|
|
"invisible": true,
|
|
"html_comment": true,
|
|
"css_hidden": true,
|
|
"encoded_blob": true,
|
|
}
|
|
h := hiddenText{detectors: map[string]bool{}, scope: p.Scope}
|
|
if err := validateScope(p.Scope); err != nil {
|
|
return nil, err
|
|
}
|
|
for _, d := range p.Detectors {
|
|
if !known[d] {
|
|
return nil, fmt.Errorf("unknown detector %q", d)
|
|
}
|
|
h.detectors[d] = true
|
|
}
|
|
return h, nil
|
|
}
|
|
|
|
func (h hiddenText) run(ctx *evalContext) Result {
|
|
scanned := ctx.surfaces.selectSurfaces(h.scope)
|
|
if len(scanned) == 0 {
|
|
return skip("bundle has no scannable surface in scope")
|
|
}
|
|
var evidence []Evidence
|
|
if h.detectors["invisible"] {
|
|
for _, sur := range ctx.decodedSurfaces(h.scope) {
|
|
evidence = append(evidence, invisibleEvidence(sur)...)
|
|
if len(evidence) >= defaultMaxEvidence {
|
|
break
|
|
}
|
|
}
|
|
}
|
|
for _, sur := range scanned {
|
|
if len(evidence) >= defaultMaxEvidence {
|
|
break
|
|
}
|
|
if h.detectors["html_comment"] {
|
|
evidence = append(evidence, matches(sur, htmlCommentRe, "HTML comment")...)
|
|
}
|
|
if h.detectors["css_hidden"] {
|
|
evidence = append(evidence, matches(sur, cssHiddenRe, "CSS hiding")...)
|
|
}
|
|
if h.detectors["encoded_blob"] {
|
|
evidence = append(evidence, encodedBlobs(sur)...)
|
|
}
|
|
if len(evidence) >= defaultMaxEvidence {
|
|
break
|
|
}
|
|
}
|
|
if len(evidence) == 0 {
|
|
return Result{}
|
|
}
|
|
return hit(evidence...)
|
|
}
|
|
|
|
func invisibleEvidence(sur Surface) []Evidence {
|
|
var out []Evidence
|
|
var run []rune
|
|
flush := func() {
|
|
if len(run) == 0 {
|
|
return
|
|
}
|
|
names := make([]string, 0, len(run))
|
|
for i, r := range run {
|
|
if i == 4 {
|
|
names = append(names, "...")
|
|
break
|
|
}
|
|
names = append(names, fmt.Sprintf("%U", r))
|
|
}
|
|
out = append(out, Evidence{
|
|
Locator: sur.Locator,
|
|
Quote: string(run),
|
|
Detail: "invisible characters: " + strings.Join(names, ", "),
|
|
})
|
|
run = nil
|
|
}
|
|
runes := []rune(sur.Text)
|
|
for i, r := range runes {
|
|
if isConcealing(runes, i) {
|
|
run = append(run, r)
|
|
continue
|
|
}
|
|
flush()
|
|
}
|
|
flush()
|
|
return out
|
|
}
|
|
|
|
// isConcealing reports whether the rune at i hides text from a reader. Format
|
|
// characters conceal, except the two places ordinary text puts them: a byte order
|
|
// mark at the very start of a string, and a zero width joiner inside an emoji
|
|
// sequence.
|
|
func isConcealing(runes []rune, i int) bool {
|
|
r := runes[i]
|
|
if !isInvisible(r) {
|
|
return false
|
|
}
|
|
switch {
|
|
case r == bom && i == 0:
|
|
return false
|
|
case r == zeroWidthJoiner && joinsEmoji(runes, i):
|
|
return false
|
|
}
|
|
return true
|
|
}
|
|
|
|
const (
|
|
bom = '\uFEFF'
|
|
zeroWidthJoiner = '\u200D'
|
|
variationSel16 = '\uFE0F'
|
|
)
|
|
|
|
func joinsEmoji(runes []rune, i int) bool {
|
|
before := i - 1
|
|
if before >= 0 && runes[before] == variationSel16 {
|
|
before--
|
|
}
|
|
return before >= 0 && i+1 < len(runes) && isEmoji(runes[before]) && isEmoji(runes[i+1])
|
|
}
|
|
|
|
// isEmoji covers the Unicode blocks that hold emoji. It is deliberately broad: it
|
|
// only decides whether a joiner sits between two pictographs, and a joiner between
|
|
// a letter and anything stays concealment.
|
|
func isEmoji(r rune) bool {
|
|
switch {
|
|
case r >= 0x1F000 && r <= 0x1FAFF:
|
|
return true
|
|
case r >= 0x2190 && r <= 0x21FF, r >= 0x2300 && r <= 0x23FF, r >= 0x25A0 && r <= 0x25FF:
|
|
return true
|
|
case r >= 0x2600 && r <= 0x27BF, r >= 0x2900 && r <= 0x297F, r >= 0x2B00 && r <= 0x2BFF:
|
|
return true
|
|
}
|
|
return false
|
|
}
|
|
|
|
// isInvisible reports Unicode format characters, which include zero-width
|
|
// characters, bidirectional overrides, and the tag block used to smuggle
|
|
// instructions past a human reader.
|
|
func isInvisible(r rune) bool {
|
|
if r >= 0xE0000 && r <= 0xE007F {
|
|
return true
|
|
}
|
|
return unicode.Is(unicode.Cf, r)
|
|
}
|
|
|
|
func matches(sur Surface, re *regexp.Regexp, label string) []Evidence {
|
|
var out []Evidence
|
|
for _, loc := range re.FindAllStringIndex(sur.Text, defaultMaxEvidence) {
|
|
out = append(out, Evidence{
|
|
Locator: sur.Locator,
|
|
Quote: excerpt(sur.Text[loc[0]:loc[1]], 200),
|
|
Detail: label,
|
|
})
|
|
}
|
|
return out
|
|
}
|
|
|
|
func encodedBlobs(sur Surface) []Evidence {
|
|
var out []Evidence
|
|
for _, loc := range encodedBlobRe.FindAllStringIndex(sur.Text, defaultMaxEvidence) {
|
|
// A data: URI carries an intentional base64 image, not a hidden payload.
|
|
start := loc[0] - 16
|
|
if start < 0 {
|
|
start = 0
|
|
}
|
|
if strings.Contains(strings.ToLower(sur.Text[start:loc[0]]), "base64,") {
|
|
continue
|
|
}
|
|
out = append(out, Evidence{
|
|
Locator: sur.Locator,
|
|
Quote: excerpt(sur.Text[loc[0]:loc[1]], 120),
|
|
Detail: fmt.Sprintf("opaque blob of %d characters", loc[1]-loc[0]),
|
|
})
|
|
}
|
|
return out
|
|
}
|
|
|
|
// ---------------------------------------------------------------------------
|
|
// secret_like
|
|
// ---------------------------------------------------------------------------
|
|
|
|
type secretLike struct {
|
|
patterns []*regexp.Regexp
|
|
labels []string
|
|
scope []string
|
|
}
|
|
|
|
var secretPatterns = []struct {
|
|
label string
|
|
expr string
|
|
}{
|
|
{"PEM private key", `-----BEGIN [A-Z ]*PRIVATE KEY-----`},
|
|
{"AWS access key id", `\bAKIA[0-9A-Z]{16}\b`},
|
|
{"GitHub token", `\bgh[pousr]_[A-Za-z0-9]{36,}\b`},
|
|
{"Slack token", `\bxox[baprs]-[A-Za-z0-9-]{10,}\b`},
|
|
{"OpenAI-style key", `\bsk-[A-Za-z0-9_-]{20,}\b`},
|
|
{"JSON web token", `\beyJ[A-Za-z0-9_-]{10,}\.[A-Za-z0-9_-]{10,}\.[A-Za-z0-9_-]{10,}\b`},
|
|
{"labelled secret", `(?i)\b(api[_-]?key|secret|password|passwd|access[_-]?token|bearer)\b\s*[:=]\s*["']?[A-Za-z0-9_\-./+]{16,}`},
|
|
{"credentialed connection string", `(?i)\b(postgres(ql)?|mysql|mongodb(\+srv)?|redis|amqp|mssql)://[^:@/\s]+:[^@/\s]+@`},
|
|
}
|
|
|
|
func newSecretLike(raw json.RawMessage) (Checker, error) {
|
|
var p struct {
|
|
Scope []string `json:"scope,omitempty"`
|
|
}
|
|
if err := decodeParams(raw, &p); err != nil {
|
|
return nil, err
|
|
}
|
|
if err := validateScope(p.Scope); err != nil {
|
|
return nil, err
|
|
}
|
|
s := secretLike{scope: p.Scope}
|
|
for _, sp := range secretPatterns {
|
|
re, err := regexp.Compile(sp.expr)
|
|
if err != nil {
|
|
return nil, fmt.Errorf("built-in pattern %s: %w", sp.label, err)
|
|
}
|
|
s.patterns = append(s.patterns, re)
|
|
s.labels = append(s.labels, sp.label)
|
|
}
|
|
return s, nil
|
|
}
|
|
|
|
func (s secretLike) run(ctx *evalContext) Result {
|
|
var evidence []Evidence
|
|
for _, sur := range ctx.surfaces.selectSurfaces(s.scope) {
|
|
for i, re := range s.patterns {
|
|
loc := re.FindStringIndex(sur.Text)
|
|
if loc == nil {
|
|
continue
|
|
}
|
|
// Evidence is masked: a report must not become the leak.
|
|
evidence = append(evidence, Evidence{
|
|
Locator: sur.Locator,
|
|
Quote: maskSecret(sur.Text[loc[0]:loc[1]]),
|
|
Detail: s.labels[i],
|
|
})
|
|
if len(evidence) >= defaultMaxEvidence {
|
|
return hit(evidence...)
|
|
}
|
|
}
|
|
}
|
|
if len(evidence) == 0 {
|
|
return Result{}
|
|
}
|
|
return hit(evidence...)
|
|
}
|
|
|
|
// maskSecret keeps just enough shape to locate the secret without reproducing it.
|
|
func maskSecret(s string) string {
|
|
if len(s) <= 8 {
|
|
return strings.Repeat("*", len(s))
|
|
}
|
|
return s[:4] + strings.Repeat("*", 6) + s[len(s)-2:]
|
|
}
|
|
|
|
// ---------------------------------------------------------------------------
|
|
// link_hygiene
|
|
// ---------------------------------------------------------------------------
|
|
|
|
type linkHygieneParams struct {
|
|
Check string `json:"check"` // encoded | segment | host
|
|
Scope []string `json:"scope,omitempty"`
|
|
AllowHosts []string `json:"allow_hosts,omitempty"`
|
|
FlagHosts []string `json:"flag_hosts,omitempty"`
|
|
MinEncoded int `json:"min_encoded,omitempty"`
|
|
}
|
|
|
|
type linkHygiene struct {
|
|
check string
|
|
scope []string
|
|
allowHosts []string
|
|
flagHosts []string
|
|
minEncoded int
|
|
}
|
|
|
|
var (
|
|
urlRe = regexp.MustCompile(`https?://[^\s"'<>\\)\]]+`)
|
|
opaqueSegRe = regexp.MustCompile(`[A-Za-z0-9_+=/-]{40,}`)
|
|
minEncodedLen = 24
|
|
minDecodedLen = 16
|
|
minSegmentSize = 40
|
|
)
|
|
|
|
func newLinkHygiene(raw json.RawMessage) (Checker, error) {
|
|
var p linkHygieneParams
|
|
if err := decodeParams(raw, &p); err != nil {
|
|
return nil, err
|
|
}
|
|
switch p.Check {
|
|
case "encoded", "segment", "host":
|
|
default:
|
|
return nil, fmt.Errorf("check %q must be encoded, segment or host", p.Check)
|
|
}
|
|
if err := validateScope(p.Scope); err != nil {
|
|
return nil, err
|
|
}
|
|
l := linkHygiene{check: p.Check, scope: p.Scope, minEncoded: p.MinEncoded}
|
|
if l.minEncoded <= 0 {
|
|
l.minEncoded = minEncodedLen
|
|
}
|
|
for _, h := range p.AllowHosts {
|
|
l.allowHosts = append(l.allowHosts, strings.ToLower(strings.TrimPrefix(h, ".")))
|
|
}
|
|
for _, h := range p.FlagHosts {
|
|
l.flagHosts = append(l.flagHosts, strings.ToLower(strings.TrimPrefix(h, ".")))
|
|
}
|
|
if p.Check == "host" && len(l.allowHosts) == 0 && len(l.flagHosts) == 0 {
|
|
return nil, errors.New("check host needs allow_hosts or flag_hosts")
|
|
}
|
|
return l, nil
|
|
}
|
|
|
|
func (l linkHygiene) run(ctx *evalContext) Result {
|
|
var evidence []Evidence
|
|
for _, sur := range ctx.surfaces.selectSurfaces(l.scope) {
|
|
for _, raw := range urlRe.FindAllString(sur.Text, 20) {
|
|
ev, ok := l.inspect(sur.Locator, raw)
|
|
if !ok {
|
|
continue
|
|
}
|
|
evidence = append(evidence, ev)
|
|
if len(evidence) >= defaultMaxEvidence {
|
|
return hit(evidence...)
|
|
}
|
|
}
|
|
}
|
|
if len(evidence) == 0 {
|
|
return Result{}
|
|
}
|
|
return hit(evidence...)
|
|
}
|
|
|
|
func (l linkHygiene) inspect(locator, raw string) (Evidence, bool) {
|
|
u, err := url.Parse(strings.TrimRight(raw, ".,;)"))
|
|
if err != nil {
|
|
return Evidence{}, false
|
|
}
|
|
host := strings.ToLower(u.Hostname())
|
|
|
|
switch l.check {
|
|
case "host":
|
|
if matchesHost(host, l.allowHosts) {
|
|
return Evidence{}, false
|
|
}
|
|
if matchesHost(host, l.flagHosts) {
|
|
return Evidence{Locator: locator, Quote: excerpt(raw, 200), Detail: "link host is a known drop or shortener: " + host}, true
|
|
}
|
|
if len(l.allowHosts) > 0 {
|
|
return Evidence{Locator: locator, Quote: excerpt(raw, 200), Detail: "link host is not in the allowlist: " + host}, true
|
|
}
|
|
return Evidence{}, false
|
|
case "encoded":
|
|
for _, seg := range segments(u) {
|
|
if decoded, ok := opaquePayload(seg, l.minEncoded); ok {
|
|
return Evidence{
|
|
Locator: locator,
|
|
Quote: excerpt(raw, 200),
|
|
Detail: fmt.Sprintf("URL segment decodes to %d bytes of non-text data", decoded),
|
|
}, true
|
|
}
|
|
}
|
|
case "segment":
|
|
for _, seg := range segments(u) {
|
|
if len(seg) >= minSegmentSize && opaqueSegRe.MatchString(seg) {
|
|
return Evidence{
|
|
Locator: locator,
|
|
Quote: excerpt(raw, 200),
|
|
Detail: fmt.Sprintf("opaque %d-character segment in link", len(seg)),
|
|
}, true
|
|
}
|
|
}
|
|
}
|
|
return Evidence{}, false
|
|
}
|
|
|
|
func segments(u *url.URL) []string {
|
|
var out []string
|
|
for _, part := range strings.Split(u.Path, "/") {
|
|
if part != "" {
|
|
out = append(out, part)
|
|
}
|
|
}
|
|
for _, values := range u.Query() {
|
|
out = append(out, values...)
|
|
}
|
|
return out
|
|
}
|
|
|
|
// opaquePayload reports a segment that decodes as base64 to a substantial amount
|
|
// of non-printable data. Readable encodings are deliberately excluded here so
|
|
// that a false positive cannot block: those go to the weaker segment rule
|
|
// instead.
|
|
//
|
|
// Hex is deliberately not decoded here. A hex string in a URL is a session id,
|
|
// hash or signature far more often than it is a payload, and hex of any random
|
|
// bytes is mostly non-printable, so accepting it blocked ordinary signed links.
|
|
// A hex payload still reaches the weak rule, where a human or a judge decides.
|
|
func opaquePayload(seg string, minLen int) (int, bool) {
|
|
if len(seg) < minLen || looksLikeIdentifier(seg) || distinctRunes(seg) < 4 || !hasBase64Marker(seg) {
|
|
return 0, false
|
|
}
|
|
decoders := []func(string) ([]byte, error){
|
|
base64.RawURLEncoding.DecodeString,
|
|
base64.RawStdEncoding.DecodeString,
|
|
}
|
|
for _, decode := range decoders {
|
|
raw, err := decode(seg)
|
|
if err != nil || len(raw) < minDecodedLen {
|
|
continue
|
|
}
|
|
nonPrintable := 0
|
|
for _, b := range raw {
|
|
if b < 0x20 || b > 0x7e {
|
|
nonPrintable++
|
|
}
|
|
}
|
|
if float64(nonPrintable)/float64(len(raw)) <= 0.3 {
|
|
continue
|
|
}
|
|
return len(raw), true
|
|
}
|
|
return 0, false
|
|
}
|
|
|
|
// looksLikeIdentifier reports a segment shaped like an identifier rather than
|
|
// encoded data: pure hex, or lowercase alphanumeric words joined by hyphens.
|
|
//
|
|
// This guard exists because decoding is not evidence. Any alphanumeric string of
|
|
// the right length decodes as base64, and hex of random bytes, hashes and signed
|
|
// session ids all decode to non-printable data, so a rule that only asked "does
|
|
// this decode" blocked ordinary links. Found by the eval set.
|
|
func looksLikeIdentifier(seg string) bool {
|
|
if isHexOnly(seg) {
|
|
return true
|
|
}
|
|
parts := strings.Split(seg, "-")
|
|
if len(parts) < 2 {
|
|
return false
|
|
}
|
|
for _, part := range parts {
|
|
if part == "" {
|
|
return false
|
|
}
|
|
for _, r := range part {
|
|
if !unicode.IsLower(r) && !unicode.IsDigit(r) {
|
|
return false
|
|
}
|
|
}
|
|
}
|
|
return true
|
|
}
|
|
|
|
func isHexOnly(s string) bool {
|
|
if s == "" {
|
|
return false
|
|
}
|
|
for _, r := range s {
|
|
if (r >= '0' && r <= '9') || (r >= 'a' && r <= 'f') || (r >= 'A' && r <= 'F') {
|
|
continue
|
|
}
|
|
return false
|
|
}
|
|
return true
|
|
}
|
|
|
|
// hasBase64Marker reports the characters that separate base64 of binary data
|
|
// from a lowercase identifier. The base64 alphabet mixes case, so any non-trivial
|
|
// binary payload almost always contains an upper-case letter; a lower-case-only
|
|
// encoding is deliberately left to the weaker segment rule, where a human or a
|
|
// judge decides.
|
|
func hasBase64Marker(s string) bool {
|
|
for _, r := range s {
|
|
if (r >= 'A' && r <= 'Z') || r == '+' || r == '/' || r == '=' {
|
|
return true
|
|
}
|
|
}
|
|
return false
|
|
}
|
|
|
|
func distinctRunes(s string) int {
|
|
seen := make(map[rune]bool, len(s))
|
|
for _, r := range s {
|
|
seen[r] = true
|
|
}
|
|
return len(seen)
|
|
}
|
|
|
|
func matchesHost(host string, list []string) bool {
|
|
for _, candidate := range list {
|
|
if host == candidate || strings.HasSuffix(host, "."+candidate) {
|
|
return true
|
|
}
|
|
}
|
|
return false
|
|
}
|
|
|
|
// ---------------------------------------------------------------------------
|
|
// tool_calls
|
|
// ---------------------------------------------------------------------------
|
|
|
|
type toolCallsParams struct {
|
|
Checks []string `json:"checks"`
|
|
}
|
|
|
|
type toolCalls struct {
|
|
declared bool
|
|
required bool
|
|
duplicates bool
|
|
}
|
|
|
|
func newToolCalls(raw json.RawMessage) (Checker, error) {
|
|
var p toolCallsParams
|
|
if err := decodeParams(raw, &p); err != nil {
|
|
return nil, err
|
|
}
|
|
if len(p.Checks) == 0 {
|
|
return nil, errors.New("checks must not be empty")
|
|
}
|
|
t := toolCalls{}
|
|
for _, c := range p.Checks {
|
|
switch c {
|
|
case "declared":
|
|
t.declared = true
|
|
case "required":
|
|
t.required = true
|
|
case "duplicates":
|
|
t.duplicates = true
|
|
default:
|
|
return nil, fmt.Errorf("unknown check %q", c)
|
|
}
|
|
}
|
|
return t, nil
|
|
}
|
|
|
|
func (t toolCalls) run(ctx *evalContext) Result {
|
|
b := ctx.bundle
|
|
if len(b.Response.ToolCalls) == 0 {
|
|
if t.required && b.Contract != nil && len(b.Contract.RequiredTools) > 0 {
|
|
return hit(Evidence{
|
|
Locator: "response.tool_calls",
|
|
Detail: "no tool was called; contract requires: " + strings.Join(b.Contract.RequiredTools, ", "),
|
|
})
|
|
}
|
|
return Result{}
|
|
}
|
|
|
|
checkDeclared := t.declared
|
|
if checkDeclared && len(b.Request.Tools) == 0 {
|
|
if !t.required && !t.duplicates {
|
|
return skip("request declares no tools, so tool authority cannot be checked")
|
|
}
|
|
checkDeclared = false
|
|
}
|
|
|
|
var evidence []Evidence
|
|
|
|
if checkDeclared {
|
|
var declared []string
|
|
for _, tool := range b.Request.Tools {
|
|
declared = append(declared, tool.Name)
|
|
}
|
|
for i, call := range b.Response.ToolCalls {
|
|
if _, ok := b.toolByName(call.Name); ok {
|
|
continue
|
|
}
|
|
evidence = append(evidence, Evidence{
|
|
Locator: locatorf("response.tool_calls[%d].name", i),
|
|
Quote: call.Name,
|
|
Detail: "tool was not declared for this call; declared: " + strings.Join(declared, ", "),
|
|
})
|
|
}
|
|
}
|
|
|
|
if t.required && b.Contract != nil {
|
|
called := map[string]bool{}
|
|
for _, call := range b.Response.ToolCalls {
|
|
called[call.Name] = true
|
|
}
|
|
for _, name := range b.Contract.RequiredTools {
|
|
if !called[name] {
|
|
evidence = append(evidence, Evidence{
|
|
Locator: "response.tool_calls",
|
|
Detail: "required tool was not called: " + name,
|
|
})
|
|
}
|
|
}
|
|
}
|
|
|
|
if t.duplicates {
|
|
first := map[string]string{}
|
|
for i, call := range b.Response.ToolCalls {
|
|
args, err := json.Marshal(call.Arguments)
|
|
if err != nil {
|
|
continue
|
|
}
|
|
key := call.Name
|
|
if prev, seen := first[key]; seen {
|
|
if prev != string(args) {
|
|
evidence = append(evidence, Evidence{
|
|
Locator: locatorf("response.tool_calls[%d].arguments", i),
|
|
Quote: excerpt(string(args), 200),
|
|
Detail: "same tool called again with different arguments: " + call.Name,
|
|
})
|
|
}
|
|
continue
|
|
}
|
|
first[key] = string(args)
|
|
}
|
|
}
|
|
|
|
if len(evidence) == 0 {
|
|
return Result{}
|
|
}
|
|
return hit(evidence...)
|
|
}
|
|
|
|
// ---------------------------------------------------------------------------
|
|
// tool_arguments
|
|
// ---------------------------------------------------------------------------
|
|
|
|
// toolArguments validates every requested call's arguments against the declared
|
|
// parameter schema, and reports arguments that are not valid JSON even when no
|
|
// schema was declared, because that is a validity failure on its own.
|
|
type toolArguments struct{}
|
|
|
|
func newToolArguments(raw json.RawMessage) (Checker, error) {
|
|
var p struct{}
|
|
if err := decodeParams(raw, &p); err != nil {
|
|
return nil, err
|
|
}
|
|
return toolArguments{}, nil
|
|
}
|
|
|
|
func (toolArguments) run(ctx *evalContext) Result {
|
|
b := ctx.bundle
|
|
if len(b.Response.ToolCalls) == 0 {
|
|
return Result{}
|
|
}
|
|
|
|
// Unparseable arguments are reported before any schema question, because the
|
|
// fault does not depend on a schema being declared.
|
|
var evidence []Evidence
|
|
var checkable []int
|
|
for i, call := range b.Response.ToolCalls {
|
|
if call.Arguments == nil && call.ArgumentsRaw != "" {
|
|
evidence = append(evidence, Evidence{
|
|
Locator: locatorf("response.tool_calls[%d].arguments", i),
|
|
Quote: excerpt(call.ArgumentsRaw, 160),
|
|
Detail: "arguments are not valid JSON, so no handler can read them as written",
|
|
})
|
|
continue
|
|
}
|
|
if _, ok := b.toolByName(call.Name); ok {
|
|
checkable = append(checkable, i)
|
|
}
|
|
}
|
|
|
|
if len(checkable) == 0 {
|
|
if len(evidence) > 0 {
|
|
return hit(evidence...)
|
|
}
|
|
if len(b.Request.Tools) == 0 {
|
|
return skip("no tool schemas were declared, so arguments cannot be validated")
|
|
}
|
|
return skip("no tool call matched a declared tool schema")
|
|
}
|
|
|
|
for _, i := range checkable {
|
|
call := b.Response.ToolCalls[i]
|
|
tool, _ := b.toolByName(call.Name)
|
|
if tool.Parameters == nil {
|
|
continue
|
|
}
|
|
problems, unsupported := validateValue(tool.Parameters, anyOrEmptyObject(call.Arguments), locatorf("response.tool_calls[%d].arguments", i))
|
|
for _, p := range problems {
|
|
evidence = append(evidence, Evidence{Locator: locatorf("response.tool_calls[%d].arguments", i), Detail: p})
|
|
}
|
|
for _, u := range unsupported {
|
|
evidence = append(evidence, Evidence{Locator: locatorf("response.tool_calls[%d]", i), Detail: "not validated: " + u})
|
|
}
|
|
if len(evidence) >= 8 {
|
|
break
|
|
}
|
|
}
|
|
if len(evidence) == 0 {
|
|
return Result{}
|
|
}
|
|
return hit(evidence...)
|
|
}
|
|
|
|
func anyOrEmptyObject(args map[string]any) any {
|
|
if args == nil {
|
|
return map[string]any{}
|
|
}
|
|
return args
|
|
}
|
|
|
|
// ---------------------------------------------------------------------------
|
|
// schema_conformance
|
|
// ---------------------------------------------------------------------------
|
|
|
|
type schemaConformance struct{}
|
|
|
|
func newSchemaConformance(raw json.RawMessage) (Checker, error) {
|
|
var p struct{}
|
|
if err := decodeParams(raw, &p); err != nil {
|
|
return nil, err
|
|
}
|
|
return schemaConformance{}, nil
|
|
}
|
|
|
|
func (schemaConformance) run(ctx *evalContext) Result {
|
|
b := ctx.bundle
|
|
if b.Contract == nil || b.Contract.OutputSchema == nil {
|
|
return skip("contract declares no output_schema")
|
|
}
|
|
value, ok := structuredOutput(b)
|
|
if !ok {
|
|
return skip("response carries no structured output to check")
|
|
}
|
|
problems, unsupported := validateValue(b.Contract.OutputSchema, value, "response.output")
|
|
if len(problems) == 0 {
|
|
if len(unsupported) > 0 {
|
|
return skip("output_schema uses keywords this auditor cannot evaluate: " + strings.Join(unsupported, "; "))
|
|
}
|
|
return Result{}
|
|
}
|
|
evidence := make([]Evidence, 0, len(problems))
|
|
for _, p := range problems {
|
|
evidence = append(evidence, Evidence{Locator: "response.output", Detail: p})
|
|
if len(evidence) >= 8 {
|
|
break
|
|
}
|
|
}
|
|
return hit(evidence...)
|
|
}
|
|
|
|
// structuredOutput prefers the explicit output field and falls back to content
|
|
// that parses as a JSON object or array.
|
|
func structuredOutput(b *Bundle) (any, bool) {
|
|
if b.Response.Output != nil {
|
|
return b.Response.Output, true
|
|
}
|
|
content := strings.TrimSpace(b.Response.Content)
|
|
if content == "" || (content[0] != '{' && content[0] != '[') {
|
|
return nil, false
|
|
}
|
|
dec := json.NewDecoder(strings.NewReader(content))
|
|
dec.UseNumber()
|
|
var value any
|
|
if err := dec.Decode(&value); err != nil {
|
|
return nil, false
|
|
}
|
|
return value, true
|
|
}
|
|
|
|
// ---------------------------------------------------------------------------
|
|
// contract_terms
|
|
// ---------------------------------------------------------------------------
|
|
|
|
type contractTerms struct{}
|
|
|
|
func newContractTerms(raw json.RawMessage) (Checker, error) {
|
|
var p struct{}
|
|
if err := decodeParams(raw, &p); err != nil {
|
|
return nil, err
|
|
}
|
|
return contractTerms{}, nil
|
|
}
|
|
|
|
func (contractTerms) run(ctx *evalContext) Result {
|
|
b := ctx.bundle
|
|
if b.Contract == nil {
|
|
return skip("no contract supplied")
|
|
}
|
|
c := b.Contract
|
|
if len(c.MustInclude) == 0 && len(c.MustNotInclude) == 0 &&
|
|
len(c.ExpectedFinishReasons) == 0 && c.ExpectedRefusal == nil {
|
|
return skip("contract declares no content terms")
|
|
}
|
|
|
|
responseText := strings.Join(surfaceTexts(ctx.surfaces.selectSurfaces(DefaultScope)), "\n")
|
|
var evidence []Evidence
|
|
|
|
for _, want := range c.MustInclude {
|
|
if !strings.Contains(responseText, want) {
|
|
evidence = append(evidence, Evidence{
|
|
Locator: "response.content",
|
|
Detail: fmt.Sprintf("required text is absent: %q", want),
|
|
})
|
|
}
|
|
}
|
|
for _, banned := range c.MustNotInclude {
|
|
if idx := strings.Index(responseText, banned); idx >= 0 {
|
|
evidence = append(evidence, Evidence{
|
|
Locator: "response.content",
|
|
Quote: excerpt(responseText[idx:], 200),
|
|
Detail: fmt.Sprintf("forbidden text is present: %q", banned),
|
|
})
|
|
}
|
|
}
|
|
if len(c.ExpectedFinishReasons) > 0 && !containsString(c.ExpectedFinishReasons, b.Response.FinishReason) {
|
|
evidence = append(evidence, Evidence{
|
|
Locator: "response.finish_reason",
|
|
Quote: b.Response.FinishReason,
|
|
Detail: "finish reason is not one of: " + strings.Join(c.ExpectedFinishReasons, ", "),
|
|
})
|
|
}
|
|
if c.ExpectedRefusal != nil && *c.ExpectedRefusal != b.Response.Refusal {
|
|
evidence = append(evidence, Evidence{
|
|
Locator: "response.refusal",
|
|
Detail: fmt.Sprintf("contract expects refusal=%t, response refusal=%t", *c.ExpectedRefusal, b.Response.Refusal),
|
|
})
|
|
}
|
|
if len(evidence) == 0 {
|
|
return Result{}
|
|
}
|
|
return hit(evidence...)
|
|
}
|
|
|
|
func surfaceTexts(surfaces []Surface) []string {
|
|
out := make([]string, 0, len(surfaces))
|
|
for _, sur := range surfaces {
|
|
out = append(out, sur.Text)
|
|
}
|
|
return out
|
|
}
|
|
|
|
func containsString(list []string, want string) bool {
|
|
for _, item := range list {
|
|
if item == want {
|
|
return true
|
|
}
|
|
}
|
|
return false
|
|
}
|
|
|
|
// ---------------------------------------------------------------------------
|
|
// grounded_claims
|
|
// ---------------------------------------------------------------------------
|
|
|
|
type groundedParams struct {
|
|
MinQuoteLength int `json:"min_quote_len,omitempty"`
|
|
MaxClaims int `json:"max_claims,omitempty"`
|
|
}
|
|
|
|
type groundedClaims struct {
|
|
minQuote int
|
|
max int
|
|
}
|
|
|
|
var quoteRe = regexp.MustCompile(`"([^"\n]{8,160})"|“([^”\n]{8,160})”`)
|
|
|
|
func newGroundedClaims(raw json.RawMessage) (Checker, error) {
|
|
var p groundedParams
|
|
if err := decodeParams(raw, &p); err != nil {
|
|
return nil, err
|
|
}
|
|
g := groundedClaims{minQuote: p.MinQuoteLength, max: p.MaxClaims}
|
|
if g.minQuote <= 0 {
|
|
g.minQuote = 16
|
|
}
|
|
if g.max <= 0 {
|
|
g.max = defaultMaxClaims
|
|
}
|
|
return g, nil
|
|
}
|
|
|
|
func (g groundedClaims) run(ctx *evalContext) Result {
|
|
b := ctx.bundle
|
|
if b.Contract == nil || !b.Contract.RequireGrounding {
|
|
return skip("contract does not require grounding")
|
|
}
|
|
if len(b.Sources) == 0 && b.requestText() == "" {
|
|
return skip("grounding is required but no sources or request text were supplied: unverifiable")
|
|
}
|
|
corpus := strings.ToLower(b.requestText())
|
|
for _, s := range b.Sources {
|
|
corpus += strings.ToLower(s.Text) + strings.ToLower(s.URI)
|
|
}
|
|
|
|
var evidence []Evidence
|
|
for _, claim := range g.claims(b.Response.Content) {
|
|
if strings.Contains(corpus, strings.ToLower(claim)) {
|
|
continue
|
|
}
|
|
evidence = append(evidence, Evidence{
|
|
Locator: "response.content",
|
|
Quote: excerpt(claim, 160),
|
|
Detail: "not found in the supplied sources or request text",
|
|
})
|
|
if len(evidence) >= g.max {
|
|
break
|
|
}
|
|
}
|
|
if len(evidence) == 0 {
|
|
return Result{}
|
|
}
|
|
return hit(evidence...)
|
|
}
|
|
|
|
// claims extracts the assertions worth checking: links, quoted spans that carry
|
|
// a fact, and identifier-like tokens. Prose paraphrases are deliberately not
|
|
// extracted, which is the main reason this rule is a signal and not a violation.
|
|
func (g groundedClaims) claims(content string) []string {
|
|
seen := map[string]bool{}
|
|
var out []string
|
|
add := func(s string, minLen int) {
|
|
s = strings.TrimSpace(s)
|
|
if utf8.RuneCountInString(s) < minLen || seen[s] {
|
|
return
|
|
}
|
|
seen[s] = true
|
|
out = append(out, s)
|
|
}
|
|
|
|
for _, m := range quoteRe.FindAllStringSubmatch(content, -1) {
|
|
for _, group := range m[1:] {
|
|
if group != "" && looksFactual(group) {
|
|
add(group, g.minQuote)
|
|
}
|
|
}
|
|
}
|
|
for _, token := range strings.FieldsFunc(content, func(r rune) bool {
|
|
return !unicode.IsLetter(r) && !unicode.IsDigit(r) && r != '-' && r != '_'
|
|
}) {
|
|
if hasLetterAndDigit(token) {
|
|
add(token, minIdentifierLen)
|
|
}
|
|
}
|
|
return out
|
|
}
|
|
|
|
func looksFactual(s string) bool {
|
|
for _, r := range s {
|
|
if unicode.IsDigit(r) {
|
|
return true
|
|
}
|
|
}
|
|
for _, token := range strings.Fields(s) {
|
|
if token == strings.ToUpper(token) && len([]rune(token)) >= 4 {
|
|
return true
|
|
}
|
|
}
|
|
return false
|
|
}
|
|
|
|
func hasLetterAndDigit(s string) bool {
|
|
var letter, digit bool
|
|
for _, r := range s {
|
|
if unicode.IsLetter(r) {
|
|
letter = true
|
|
}
|
|
if unicode.IsDigit(r) {
|
|
digit = true
|
|
}
|
|
}
|
|
return letter && digit
|
|
}
|
|
|
|
// ---------------------------------------------------------------------------
|
|
// record_completeness
|
|
// ---------------------------------------------------------------------------
|
|
|
|
type recordParams struct {
|
|
Require []string `json:"require"`
|
|
}
|
|
|
|
type recordCompleteness struct {
|
|
require []string
|
|
}
|
|
|
|
var recordFields = map[string]func(*Bundle) bool{
|
|
"subject.provider": func(b *Bundle) bool { return b.Subject.Provider != "" },
|
|
"subject.model": func(b *Bundle) bool { return b.Subject.Model != "" },
|
|
"subject.actor": func(b *Bundle) bool { return b.Subject.Actor != "" },
|
|
"subject.timestamp": func(b *Bundle) bool { return b.Subject.Timestamp != "" },
|
|
"response.finish_reason": func(b *Bundle) bool { return b.Response.FinishReason != "" },
|
|
"response.usage": func(b *Bundle) bool { return b.Response.Usage != nil },
|
|
"integrity.request_sha256": func(b *Bundle) bool { return integrityHash(b) != "" },
|
|
}
|
|
|
|
func newRecordCompleteness(raw json.RawMessage) (Checker, error) {
|
|
var p recordParams
|
|
if err := decodeParams(raw, &p); err != nil {
|
|
return nil, err
|
|
}
|
|
if len(p.Require) == 0 {
|
|
return nil, errors.New("require must not be empty")
|
|
}
|
|
for _, f := range p.Require {
|
|
if _, ok := recordFields[f]; !ok {
|
|
return nil, fmt.Errorf("unknown record field %q", f)
|
|
}
|
|
}
|
|
return recordCompleteness{require: p.Require}, nil
|
|
}
|
|
|
|
func (r recordCompleteness) run(ctx *evalContext) Result {
|
|
var evidence []Evidence
|
|
for _, field := range r.require {
|
|
if !recordFields[field](ctx.bundle) {
|
|
evidence = append(evidence, Evidence{Locator: field, Detail: "field is absent"})
|
|
}
|
|
}
|
|
if len(evidence) == 0 {
|
|
return Result{}
|
|
}
|
|
return hit(evidence...)
|
|
}
|
|
|
|
// ---------------------------------------------------------------------------
|
|
// request_hash_mismatch
|
|
// ---------------------------------------------------------------------------
|
|
|
|
type requestHash struct{}
|
|
|
|
func newRequestHash(raw json.RawMessage) (Checker, error) {
|
|
var p struct {
|
|
Algorithm string `json:"algorithm,omitempty"`
|
|
}
|
|
if err := decodeParams(raw, &p); err != nil {
|
|
return nil, err
|
|
}
|
|
if p.Algorithm != "" && p.Algorithm != "sha256" {
|
|
return nil, fmt.Errorf("algorithm %q is not supported; only sha256", p.Algorithm)
|
|
}
|
|
return requestHash{}, nil
|
|
}
|
|
|
|
func (requestHash) run(ctx *evalContext) Result {
|
|
b := ctx.bundle
|
|
declared := integrityHash(b)
|
|
if declared == "" {
|
|
return skip("no integrity.request_sha256 declared")
|
|
}
|
|
computed, err := b.RequestHash()
|
|
if err != nil {
|
|
return skip("request could not be canonicalised: " + err.Error())
|
|
}
|
|
if declared == computed {
|
|
return Result{}
|
|
}
|
|
return hit(Evidence{
|
|
Locator: "integrity.request_sha256",
|
|
Quote: declared,
|
|
Detail: "computed hash of the presented request is " + computed,
|
|
})
|
|
}
|
|
|
|
func integrityHash(b *Bundle) string {
|
|
if b.Integrity == nil {
|
|
return ""
|
|
}
|
|
return b.Integrity.RequestSHA256
|
|
}
|
|
|
|
// ---------------------------------------------------------------------------
|
|
// shared validation
|
|
// ---------------------------------------------------------------------------
|
|
|
|
func validateScope(scope []string) error {
|
|
for _, s := range scope {
|
|
if !oneOf(s, SurfaceContent, SurfaceOutput, SurfaceArguments, SurfaceRequest, SurfaceRequestNew) {
|
|
return fmt.Errorf("unknown surface %q", s)
|
|
}
|
|
}
|
|
return nil
|
|
}
|