Files
archie/internal/store/captures_test.go
T
sam 37bf8ef05a feat(capture): store unbound webhook events, retention-bounded
Adds POST /webhooks/capture/{source}: an unauthenticated endpoint (mounted
alongside /healthz, bypassing dashboard token auth by design) that persists
any inbound webhook archie has no binding for yet, so it can be inspected
and mapped later (t2db.2/t2db.3).

docs/prds/event-capture-storage.md decides SQLite over NATS JetStream for
storage: NATS is not unconditionally available until 7d5u.2 ships embedded
support, and capture's single-writer/single-reader access pattern doesn't
need JetStream's fan-out/replay machinery anyway.

internal/store/captures.go adds a captured_events table with prune-on-write
(retention age + row-count cap, both configurable). The HTTP handler applies
webhookguard redaction before persisting, and enforces two more independent
disk-bound mechanics before that: a per-request body-size cap and a
per-remote-address rate limit. New [capture] config section with documented
defaults; all fields optional, capture is on by default.

Adversarial review (manual, after the usual fresh-subagent pass hit a
session API limit mid-run) caught and fixed three real defects before this
landed, each with a regression test confirmed to fail against the pre-fix
code:
- the body-size cap silently no-op'd when CaptureMaxBodyBytes was left
  unset, the one bound that limits a single payload's size rather than row
  count or write rate
- the rate limiter was keyed by the request's own source path segment --
  unregistered and attacker-chosen by design for this endpoint, violating
  webhookguard.RateLimiter's documented bounded-key-space assumption and
  letting a sender bypass the limit by rotating the segment; rekeyed to
  remote address
- the retention prune compared time.RFC3339Nano strings, whose trimmed
  variable-width fractional seconds break lexicographic ordering within the
  same second; switched to fixed-width RFC3339

Closes archie-core-t2db.1.
2026-08-22 04:51:25 +10:00

170 lines
5.8 KiB
Go

package store
import (
"testing"
"time"
)
func TestInsertCaptureRoundTrips(t *testing.T) {
s := openTest(t)
id, err := s.InsertCapture(t.Context(), CapturedEvent{
Source: "github",
RemoteAddr: "203.0.113.5",
ContentType: "application/json",
Headers: `{"x-hub-signature-256":"[redacted]"}`,
Body: `{"action":"opened"}`,
Authenticated: false,
}, 7*24*time.Hour, 0)
if err != nil {
t.Fatalf("InsertCapture: %v", err)
}
if id == 0 {
t.Fatalf("InsertCapture returned zero id")
}
got, err := s.ListCaptures(t.Context(), 10)
if err != nil {
t.Fatalf("ListCaptures: %v", err)
}
if len(got) != 1 {
t.Fatalf("ListCaptures = %d rows, want 1", len(got))
}
c := got[0]
if c.ID != id || c.Source != "github" || c.RemoteAddr != "203.0.113.5" ||
c.ContentType != "application/json" || c.Body != `{"action":"opened"}` ||
c.Authenticated {
t.Fatalf("round-tripped capture = %+v, want matching fields", c)
}
if c.ReceivedAt.IsZero() {
t.Fatalf("ReceivedAt is zero, want a timestamp stamped at insert")
}
}
func TestListCapturesOrdersNewestFirst(t *testing.T) {
s := openTest(t)
for i := range 3 {
if _, err := s.InsertCapture(t.Context(), CapturedEvent{Source: "src", Body: "{}"}, 0, 0); err != nil {
t.Fatalf("InsertCapture[%d]: %v", i, err)
}
}
got, err := s.ListCaptures(t.Context(), 10)
if err != nil {
t.Fatalf("ListCaptures: %v", err)
}
if len(got) != 3 {
t.Fatalf("ListCaptures = %d rows, want 3", len(got))
}
if got[0].ID <= got[1].ID || got[1].ID <= got[2].ID {
t.Fatalf("ListCaptures not newest-first: ids = %d, %d, %d", got[0].ID, got[1].ID, got[2].ID)
}
}
func TestInsertCapturePrunesBeyondMaxEvents(t *testing.T) {
s := openTest(t)
var lastID int64
for i := range 5 {
id, err := s.InsertCapture(t.Context(), CapturedEvent{Source: "src", Body: "{}"}, 0, 3)
if err != nil {
t.Fatalf("InsertCapture[%d]: %v", i, err)
}
lastID = id
}
got, err := s.ListCaptures(t.Context(), 100)
if err != nil {
t.Fatalf("ListCaptures: %v", err)
}
if len(got) != 3 {
t.Fatalf("row count = %d, want 3 (maxEvents cap)", len(got))
}
if got[0].ID != lastID {
t.Fatalf("newest surviving row id = %d, want %d (the last insert)", got[0].ID, lastID)
}
// The two oldest of the five inserts must have been pruned.
for _, c := range got {
if c.ID <= lastID-3 {
t.Fatalf("found pruned row id %d still present after maxEvents=3 cap", c.ID)
}
}
}
func TestInsertCapturePrunesOlderThanRetention(t *testing.T) {
s := openTest(t)
// Insert a row and manually age it beyond the retention window, since
// InsertCapture always stamps "now" and the test cannot wait real days.
id, err := s.InsertCapture(t.Context(), CapturedEvent{Source: "old", Body: "{}"}, 0, 0)
if err != nil {
t.Fatalf("InsertCapture: %v", err)
}
stale := time.Now().UTC().Add(-8 * 24 * time.Hour).Format(captureTimeLayout)
if _, err := s.db.ExecContext(t.Context(),
`UPDATE captured_events SET received_at = ? WHERE id = ?`, stale, id); err != nil {
t.Fatalf("backdate row: %v", err)
}
// A fresh insert with a 7-day retention should prune the now-stale row.
if _, err := s.InsertCapture(t.Context(), CapturedEvent{Source: "new", Body: "{}"}, 7*24*time.Hour, 0); err != nil {
t.Fatalf("InsertCapture: %v", err)
}
got, err := s.ListCaptures(t.Context(), 100)
if err != nil {
t.Fatalf("ListCaptures: %v", err)
}
for _, c := range got {
if c.ID == id {
t.Fatalf("stale row id %d survived a 7-day retention prune", id)
}
}
if len(got) != 1 {
t.Fatalf("row count = %d, want 1 (only the fresh insert)", len(got))
}
}
// TestCaptureTimeLayoutSortsLexicographicallyLikeChronologically pins a real
// defect that time.RFC3339Nano has: it trims trailing fractional-second
// zeros and omits the fractional part entirely when it is exactly zero, so
// "15:04:05Z" (an exact second) sorts LEXICOGRAPHICALLY AFTER "15:04:05.1Z"
// (0.1s later) because 'Z' (0x5A) > '.' (0x2E) -- backwards from
// chronological order. A retention prune using `WHERE received_at < ?` on a
// column written that way is unreliable within the same second.
// captureTimeLayout must be fixed-width so string order and time order
// always agree.
//
// This compares captureTimeLayout's string output directly rather than
// going through InsertCapture's real prune, because the defect only shows up
// when two timestamps share the same whole-second value and differ only in
// fractional presence -- a case InsertCapture's own cutoff (computed from
// the real wall clock) cannot be steered into deterministically in a test.
func TestCaptureTimeLayoutSortsLexicographicallyLikeChronologically(t *testing.T) {
exactSecond := time.Date(2026, 1, 1, 15, 4, 5, 0, time.UTC)
fractionLater := time.Date(2026, 1, 1, 15, 4, 5, 100_000_000, time.UTC) // same second, 0.1s later
if !exactSecond.Before(fractionLater) {
t.Fatalf("test setup: %v must be before %v", exactSecond, fractionLater)
}
es, fs := exactSecond.Format(captureTimeLayout), fractionLater.Format(captureTimeLayout)
// The layout is second-precision, so two timestamps within the same
// second are allowed to format identically (that's the accepted
// precision tradeoff, not a bug) -- but a later time must never sort as
// lexicographically LESS than an earlier one.
if es > fs {
t.Fatalf("captureTimeLayout string order disagrees with chronological order: %q should not sort after %q", es, fs)
}
}
func TestInsertCaptureZeroRetentionAndMaxEventsMeansUnbounded(t *testing.T) {
s := openTest(t)
for i := range 10 {
if _, err := s.InsertCapture(t.Context(), CapturedEvent{Source: "src", Body: "{}"}, 0, 0); err != nil {
t.Fatalf("InsertCapture[%d]: %v", i, err)
}
}
got, err := s.ListCaptures(t.Context(), 100)
if err != nil {
t.Fatalf("ListCaptures: %v", err)
}
if len(got) != 10 {
t.Fatalf("row count = %d, want 10 (retention=0, maxEvents=0 disables both prunes)", len(got))
}
}