feat(webhook): harden captured payload redaction
RedactPayload only redacted values under a fixed set of sensitive key names, so a sender that names a secret field something else leaked it into captured_events.body. Harden the heuristic: - broaden the key-name markers (compound names like access_token, client_secret, signing_key, x_api_key, webhook_secret, refresh_token) and keep gateway.sensitiveParameterKey identical to preserve the documented agreement between the two redaction paths. - add value-shape detection: a compact JWT or a PEM private-key block is redacted under any key, catching the 'sender named the field something the heuristic misses' gap. - keep it conservative so schema-by-example field mapping still works: only unambiguous shapes (eyJ-prefixed JWTs, PEM key blocks) are redacted by value, never ordinary content. Best-effort per docs/prds/webhook-intake-security.md point 5. Refs archie-core-t2db (captured_events.body redaction)
This commit is contained in:
1 parent
7ee6ee5554
commit
c13ec2f225
4 files changed
+138
-24
No files matched your search
@@ -75,10 +75,14 @@ old material must be retained.
|
||||
|
||||
## Non-goals / deferred
|
||||
|
||||
- **`captured_events.body` redaction upgrade** is deferred (confirmed with
|
||||
maintainer). The body redaction is a separate best-effort key-name heuristic
|
||||
with its own documented limits and its own capture-path surface; it is not
|
||||
bundled into this store-scoped change.
|
||||
- **`captured_events.body` redaction hardening** was completed alongside this
|
||||
work (not deferred): `webhookguard.RedactPayload` now redacts an expanded set
|
||||
of secret key markers (compound names like `access_token`, `client_secret`,
|
||||
`signing_key`, `x_api_key`, …) as well as unambiguous value shapes (compact
|
||||
JWTs, PEM private-key blocks) under any key. `gateway.sensitiveParameterKey`
|
||||
was broadened identically to preserve the documented agreement. It remains
|
||||
best-effort per `docs/prds/webhook-intake-security.md` point 5 -- a sender
|
||||
that picks a name the heuristic misses is still captured.
|
||||
- **Key re-derivation on config reload** is a startup concern: the store holds
|
||||
the cipher for its lifetime, so a reload that changes `[bindings]` keys
|
||||
requires a daemon restart to take effect. Documented, accepted.
|
||||
|
||||
@@ -216,8 +216,11 @@ func redactToolParameters(value any, sensitive bool) any {
|
||||
func sensitiveParameterKey(key string) bool {
|
||||
normalized := strings.NewReplacer("-", "_", " ", "_").Replace(strings.ToLower(key))
|
||||
for _, marker := range []string{
|
||||
"token", "secret", "password", "passwd", "api_key", "apikey",
|
||||
"authorization", "credential", "cookie", "private_key",
|
||||
"token", "secret", "password", "passwd", "pwd", "passphrase",
|
||||
"api_key", "apikey", "api_secret", "client_secret", "private_key", "secret_key",
|
||||
"signing_key", "authorization", "bearer", "credential", "cookie", "session",
|
||||
"access_token", "refresh_token", "id_token", "bot_token", "webhook_secret",
|
||||
"x_api_key", "x_auth_token", "jwt", "oauth", "csrf", "otp", "signature",
|
||||
} {
|
||||
if strings.Contains(normalized, marker) {
|
||||
return true
|
||||
|
||||
@@ -3,20 +3,33 @@ package webhookguard
|
||||
import (
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"regexp"
|
||||
"strings"
|
||||
)
|
||||
|
||||
// redactedValue replaces a value whose key looks sensitive.
|
||||
// redactedValue replaces a value whose key (or shape) looks sensitive.
|
||||
const redactedValue = "[redacted]"
|
||||
|
||||
// sensitiveKeyMarkers is the key-name heuristic, matching
|
||||
// internal/gateway/stream.go's sensitiveParameterKey exactly so the two
|
||||
// redaction paths agree on what counts as sensitive.
|
||||
// redaction paths agree on what counts as sensitive. It is intentionally
|
||||
// conservative: a value whose key matches is redacted wholesale, so a marker
|
||||
// that is too broad (e.g. bare "key" or "auth", which would match "Key: 42"
|
||||
// or "author") would destroy the schema-by-example mapping use case the
|
||||
// captured payload exists to serve.
|
||||
var sensitiveKeyMarkers = []string{
|
||||
"token", "secret", "password", "passwd", "api_key", "apikey",
|
||||
"authorization", "credential", "cookie", "private_key",
|
||||
"token", "secret", "password", "passwd", "pwd", "passphrase",
|
||||
"api_key", "apikey", "api_secret", "client_secret", "private_key", "secret_key",
|
||||
"signing_key", "authorization", "bearer", "credential", "cookie", "session",
|
||||
"access_token", "refresh_token", "id_token", "bot_token", "webhook_secret",
|
||||
"x_api_key", "x_auth_token", "jwt", "oauth", "csrf", "otp", "signature",
|
||||
}
|
||||
|
||||
// jwtTokenRe matches a compact JWT (header.payload.signature) -- the header's
|
||||
// base64url always begins "eyJ" -- which is an unambiguous secret token even
|
||||
// when a sender puts it under a key the name heuristic misses.
|
||||
var jwtTokenRe = regexp.MustCompile(`^eyJ[A-Za-z0-9_-]+\.[A-Za-z0-9_-]+\.[A-Za-z0-9_-]+$`)
|
||||
|
||||
// RedactPayload replaces values under sensitive-looking JSON keys with a
|
||||
// redaction marker and returns the re-encoded payload, for capturing an
|
||||
// inbound webhook body before persisting it.
|
||||
@@ -27,12 +40,16 @@ var sensitiveKeyMarkers = []string{
|
||||
// for a chat transcript. A captured webhook payload exists to be inspected
|
||||
// and mapped field-by-field (see docs/prds/event-sources-and-reactions.md
|
||||
// and t2db.3's schema-by-example mapping) -- redacting every string would
|
||||
// make that impossible. Only a value whose key matches the marker list is
|
||||
// touched; everything else is returned exactly as decoded.
|
||||
// make that impossible. Only a value whose key matches the marker list, or
|
||||
// whose value is an unambiguous secret shape, is touched; everything else is
|
||||
// returned exactly as decoded.
|
||||
//
|
||||
// A key match redacts its entire value, including a nested object or array
|
||||
// -- "credentials": {"host": ..., "user": ...} is wholesale replaced, not
|
||||
// recursed into, matching the intent of the key name.
|
||||
//
|
||||
// This is best-effort, not a guarantee: a sender that names a secret field
|
||||
// something the heuristic misses is still captured with its secret.
|
||||
func RedactPayload(payload []byte) ([]byte, error) {
|
||||
var value any
|
||||
decoder := json.NewDecoder(strings.NewReader(string(payload)))
|
||||
@@ -49,23 +66,43 @@ func RedactPayload(payload []byte) ([]byte, error) {
|
||||
}
|
||||
|
||||
func redactValue(value any) any {
|
||||
m, ok := value.(map[string]any)
|
||||
if !ok {
|
||||
if arr, ok := value.([]any); ok {
|
||||
for i, child := range arr {
|
||||
arr[i] = redactValue(child)
|
||||
switch typed := value.(type) {
|
||||
case map[string]any:
|
||||
for key, child := range typed {
|
||||
if sensitiveKey(key) {
|
||||
typed[key] = redactedValue
|
||||
continue
|
||||
}
|
||||
typed[key] = redactValue(child)
|
||||
}
|
||||
return typed
|
||||
case []any:
|
||||
for i, child := range typed {
|
||||
typed[i] = redactValue(child)
|
||||
}
|
||||
return typed
|
||||
case string:
|
||||
if sensitiveValueShape(typed) {
|
||||
return redactedValue
|
||||
}
|
||||
return typed
|
||||
default:
|
||||
return value
|
||||
}
|
||||
for key, child := range m {
|
||||
if sensitiveKey(key) {
|
||||
m[key] = redactedValue
|
||||
continue
|
||||
}
|
||||
m[key] = redactValue(child)
|
||||
}
|
||||
|
||||
// sensitiveValueShape reports whether a string is an unambiguous secret even
|
||||
// under a non-sensitive key: a compact JWT or a PEM private-key block. These
|
||||
// shapes are never a meaningful field-mapping value, so redacting them does
|
||||
// not hurt the schema-by-example use case.
|
||||
func sensitiveValueShape(s string) bool {
|
||||
if len(s) < 20 {
|
||||
return false
|
||||
}
|
||||
return m
|
||||
if jwtTokenRe.MatchString(s) {
|
||||
return true
|
||||
}
|
||||
return strings.Contains(s, "-----BEGIN") && strings.Contains(s, "PRIVATE KEY-----")
|
||||
}
|
||||
|
||||
func sensitiveKey(key string) bool {
|
||||
|
||||
@@ -59,6 +59,14 @@ func TestRedactPayloadRedactsSensitiveKeys(t *testing.T) {
|
||||
{"private key", "private_key"},
|
||||
{"cookie", "cookie"},
|
||||
{"credential", "credential"},
|
||||
{"access_token", "access_token"},
|
||||
{"client_secret", "client_secret"},
|
||||
{"signing_key", "signing_key"},
|
||||
{"webhook_secret", "webhook_secret"},
|
||||
{"x-api-key", "x-api-key"},
|
||||
{"x_auth_token", "x_auth_token"},
|
||||
{"otp", "otp"},
|
||||
{"refresh_token", "refresh_token"},
|
||||
}
|
||||
|
||||
for _, test := range tests {
|
||||
@@ -125,3 +133,65 @@ func TestRedactPayloadRejectsInvalidJSON(t *testing.T) {
|
||||
t.Error("RedactPayload() succeeded on invalid JSON, want error")
|
||||
}
|
||||
}
|
||||
|
||||
// TestRedactPayloadRedactsSecretValueShapes pins the value-shape heuristic:
|
||||
// a compact JWT or a PEM private-key block is redacted even when it sits under
|
||||
// a key the name heuristic would never flag -- the exact "sender names a secret
|
||||
// field something the heuristic misses" gap docs/prds/webhook-intake-
|
||||
// security.md point 5 documents as best-effort.
|
||||
func TestRedactPayloadRedactsSecretValueShapes(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
value any
|
||||
}{
|
||||
{
|
||||
"compact JWT",
|
||||
"eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJzdWIiOiIxMjM0NTY3ODkwIiwibmFtZSI6IkpvaG4gRG9lIn0.SflKxwRJSMeKKF2QT4fwpMeJf36POk6yJV_adQssw5c",
|
||||
},
|
||||
{
|
||||
"PEM private key",
|
||||
"-----BEGIN RSA PRIVATE KEY-----\nMIIEowIBAAKCAQEA7...\n-----END RSA PRIVATE KEY-----",
|
||||
},
|
||||
}
|
||||
for _, test := range tests {
|
||||
t.Run(test.name, func(t *testing.T) {
|
||||
// Under an innocuous key, not a secret-named one.
|
||||
input, err := json.Marshal(map[string]any{"payload": test.value})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
got, err := webhookguard.RedactPayload(input)
|
||||
if err != nil {
|
||||
t.Fatalf("RedactPayload() error = %v", err)
|
||||
}
|
||||
var decoded map[string]any
|
||||
if err := json.Unmarshal(got, &decoded); err != nil {
|
||||
t.Fatalf("redacted output is not valid JSON: %v", err)
|
||||
}
|
||||
if decoded["payload"] == test.value {
|
||||
t.Errorf("secret-shaped value under 'payload' survived redaction")
|
||||
}
|
||||
if decoded["payload"] != "[redacted]" {
|
||||
t.Errorf("decoded payload = %#v, want %q", decoded["payload"], "[redacted]")
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// TestRedactPayloadKeepsOrdinaryStrings verifies the value-shape heuristic
|
||||
// does not over-redact ordinary content (the field-mapping use case depends on
|
||||
// leaving non-secret values intact).
|
||||
func TestRedactPayloadKeepsOrdinaryStrings(t *testing.T) {
|
||||
input := []byte(`{"description":"Fix the build pipeline","buildId":"build-123"}`)
|
||||
got, err := webhookguard.RedactPayload(input)
|
||||
if err != nil {
|
||||
t.Fatalf("RedactPayload() error = %v", err)
|
||||
}
|
||||
var decoded map[string]any
|
||||
if err := json.Unmarshal(got, &decoded); err != nil {
|
||||
t.Fatalf("redacted output is not valid JSON: %v", err)
|
||||
}
|
||||
if decoded["description"] != "Fix the build pipeline" || decoded["buildId"] != "build-123" {
|
||||
t.Errorf("ordinary strings were redacted: %#v", decoded)
|
||||
}
|
||||
}
|
||||
Reference in new issue
Block a user