refactor(events,deliveries): split source saving and plain the new prose (#2021, #2016)

This commit is contained in:
sam committed 2026-11-21 13:52:33 +11:00
1 parent 29a06f7c54
commit f47e60db00
14 files changed
+138 -109

No files matched your search

+6 -6
View File
@@ -948,7 +948,7 @@ components:
source:
type: string
token:
description: Shown once, for a token source; only its hash is stored
description: Returned once for a token source; only its hash is stored
type: string
url:
description: Where the sender posts its events
@@ -2084,18 +2084,18 @@ components:
description: The connector that parses this source's events
type: string
hmac_header:
description: The header that carries the signature, for hmac
description: The header that carries the signature (hmac only)
type: string
hmac_prefix:
description: Text before the hex digest in that header, such as sha256=, for hmac
description: Text before the hex digest in the signature header, such as sha256= (hmac only)
examples:
- sha256=
type: string
hmac_secret:
description: "The shared signing secret, for hmac. Write-only: it is sealed and never returned"
description: "The shared signing secret (hmac only). Write-only: it is sealed and never returned"
type: string
verification:
description: "How a delivery proves itself: a bearer token, an HMAC-SHA256 signature of the body, or nothing"
description: "How deliveries are verified: a bearer token, an HMAC-SHA256 signature of the body, or no verification"
enum:
- token
- hmac
@@ -5381,7 +5381,7 @@ paths:
schema:
$ref: "#/components/schemas/ErrorModel"
description: Error
summary: Add a source, or change how it verifies deliveries and rotate its token
summary: Add or update a source, its verification scheme and its token
tags:
- Events
/api/sources/{name}/key-fields:
@@ -60,8 +60,8 @@ type ConnectorServiceClient interface {
// delivery is never offered to an agent: hard rules apply to it, guards
// don't. The host retries a failed delivery, so a connector should make a
// repeat with the same idempotency key harmless where its service allows.
// A connector answers InvalidArgument, FailedPrecondition, PermissionDenied
// or NotFound when the service refuses the delivery for good: the host
// A connector returns InvalidArgument, FailedPrecondition, PermissionDenied
// or NotFound when the service rejects the delivery permanently: the host
// records the reason and stops retrying. Any other status is retried.
Deliver(ctx context.Context, in *DeliverRequest, opts ...grpc.CallOption) (*DeliverResponse, error)
// Listen streams the events of a source the connector holds its own
@@ -213,8 +213,8 @@ type ConnectorServiceServer interface {
// delivery is never offered to an agent: hard rules apply to it, guards
// don't. The host retries a failed delivery, so a connector should make a
// repeat with the same idempotency key harmless where its service allows.
// A connector answers InvalidArgument, FailedPrecondition, PermissionDenied
// or NotFound when the service refuses the delivery for good: the host
// A connector returns InvalidArgument, FailedPrecondition, PermissionDenied
// or NotFound when the service rejects the delivery permanently: the host
// records the reason and stops retrying. Any other status is retried.
Deliver(context.Context, *DeliverRequest) (*DeliverResponse, error)
// Listen streams the events of a source the connector holds its own
@@ -1,7 +1,7 @@
-- +goose Up
-- How a source's deliveries are verified. A token source checks the bearer
-- token and an hmac source checks a signature over the raw body with a shared
-- secret, which is kept sealed. A source that chose none checks nothing.
-- token. An hmac source checks a signature over the raw body using a shared
-- secret, which is stored sealed. A source set to none performs no check.
ALTER TABLE events.sources
ADD COLUMN verification text NOT NULL DEFAULT 'token'
CHECK (verification IN ('token', 'hmac', 'none')),
@@ -1,6 +1,7 @@
-- +goose Up
-- A delivery the channel refuses for good, or that keeps failing past the retry
-- limit, is failed rather than retried: failed_at stops the worker claiming it.
-- A delivery that the channel rejects permanently, or that still fails at the
-- retry limit, is marked failed instead of retried. The worker does not claim
-- rows with failed_at set.
ALTER TABLE deliveries.messages ADD COLUMN failed_at timestamptz;
DROP INDEX deliveries.messages_due;
CREATE INDEX messages_due ON deliveries.messages (next_attempt_at) WHERE sent_at IS NULL AND failed_at IS NULL;
+7 -6
View File
@@ -31,10 +31,11 @@ var (
// ErrUnknownName names an action, lookup or event type the connector
// never declared.
ErrUnknownName = errors.New("not declared by the connector")
// ErrRejected means a delivery was refused in a way a retry can't change:
// the connector answered InvalidArgument, FailedPrecondition,
// PermissionDenied or NotFound, or the input broke the declared schema.
ErrRejected = errors.New("delivery rejected for good")
// ErrRejected means a delivery was rejected permanently, so retrying it
// cannot succeed: the connector returned InvalidArgument,
// FailedPrecondition, PermissionDenied or NotFound, or the input did not
// match the declared schema.
ErrRejected = errors.New("delivery permanently rejected")
)
// Reversibility says whether and how an action can be undone.
@@ -249,8 +250,8 @@ func (c *Connector) Deliver(ctx context.Context, name string, input jsontext.Val
return resp.GetExternalId(), nil
}
// rejectedForGood reports whether a connector's error says the same delivery
// can never succeed.
// rejectedForGood reports whether a connector's error status means that
// retrying the same delivery cannot succeed.
func rejectedForGood(err error) bool {
switch status.Code(err) {
case codes.InvalidArgument, codes.FailedPrecondition, codes.PermissionDenied, codes.NotFound:
+4 -3
View File
@@ -4,7 +4,8 @@
// dashboard, is the first channel: its messages are rows a client reads from a
// cursor, with a live stream that only says when to read. Every accepted
// connector that delivers messages is another: each gets its own row, retried
// until the connector confirms it, refuses it for good, or the attempts run out.
// until the connector confirms it, rejects it permanently, or the retry limit
// is reached.
package deliveries
import (
@@ -228,8 +229,8 @@ func (m *Module) deliver(ctx context.Context, msg store.Message) {
}
}
// recordFailure stores why an attempt failed, and ends the delivery when the
// channel refused it for good or the attempts are spent.
// recordFailure records why an attempt failed. It marks the delivery failed
// when the channel rejected it permanently or the retry limit is reached.
func (m *Module) recordFailure(ctx context.Context, msg store.Message, cause error, took time.Duration) {
record := m.store.Failed
reason := cause.Error()
+2 -2
View File
@@ -171,8 +171,8 @@ func (s *Store) Failed(ctx context.Context, seq int64, attempt int, reason strin
`UPDATE deliveries.messages SET last_error = $2 WHERE seq = $1`, reason)
}
// GaveUp records the attempt that ended a delivery for good: the message is
// marked failed and no worker claims it again.
// GaveUp records the final attempt of a delivery and marks the message failed,
// so no worker claims it again.
func (s *Store) GaveUp(ctx context.Context, seq int64, attempt int, reason string, took time.Duration) error {
return s.attempt(ctx, seq, attempt, attemptOutcome{failure: reason, took: took},
`UPDATE deliveries.messages SET failed_at = now(), last_error = $2 WHERE seq = $1`, reason)
+82 -57
View File
@@ -29,7 +29,7 @@ func (m *Module) Register(api huma.API) {
}, m.sources)
huma.Register(api, huma.Operation{
OperationID: "put-source", Method: http.MethodPut, Path: "/api/sources/{name}",
Summary: "Add a source, or change how it verifies deliveries and rotate its token", Tags: tags,
Summary: "Add or update a source, its verification scheme and its token", Tags: tags,
}, m.putSource)
huma.Register(api, huma.Operation{
OperationID: "put-source-key-fields", Method: http.MethodPut, Path: "/api/sources/{name}/key-fields",
@@ -61,25 +61,26 @@ type putSourceInput struct {
Name string `path:"name" doc:"Lowercase letters, digits and hyphens; the webhook URL ends in it"`
Body struct {
Connector string `json:"connector" doc:"The connector that parses this source's events"`
// Verification defaults to a bearer token so a source that names none
// keeps working.
Verification string `json:"verification,omitzero" enum:"token,hmac,none" doc:"How a delivery proves itself: a bearer token, an HMAC-SHA256 signature of the body, or nothing"`
HMACHeader string `json:"hmac_header,omitzero" doc:"The header that carries the signature, for hmac"`
HMACPrefix string `json:"hmac_prefix,omitzero" doc:"Text before the hex digest in that header, such as sha256=, for hmac" example:"sha256="`
HMACSecret string `json:"hmac_secret,omitzero" doc:"The shared signing secret, for hmac. Write-only: it is sealed and never returned"`
// A new source without a scheme uses a bearer token, so existing
// senders keep working.
Verification string `json:"verification,omitzero" enum:"token,hmac,none" doc:"How deliveries are verified: a bearer token, an HMAC-SHA256 signature of the body, or no verification"`
HMACHeader string `json:"hmac_header,omitzero" doc:"The header that carries the signature (hmac only)"`
HMACPrefix string `json:"hmac_prefix,omitzero" doc:"Text before the hex digest in the signature header, such as sha256= (hmac only)" example:"sha256="`
HMACSecret string `json:"hmac_secret,omitzero" doc:"The shared signing secret (hmac only). Write-only: it is sealed and never returned"`
}
}
type sourceCreated struct {
Source string `json:"source"`
Token string `json:"token,omitzero" doc:"Shown once, for a token source; only its hash is stored"`
Token string `json:"token,omitzero" doc:"Returned once for a token source; only its hash is stored"`
URL string `json:"url" doc:"Where the sender posts its events"`
}
// putSource creates a source or changes it. A save that names no verification
// keeps the stored scheme and its credentials. A token is returned once, when
// a token scheme is chosen, and stored only as its hash. An hmac secret is
// sealed and never returned; leaving it blank keeps the sealed one.
// putSource creates or updates a source. A save that omits verification keeps
// the stored scheme and its credentials. A token is returned only when the
// token scheme is chosen explicitly or the source is new, and only its hash is
// stored. An HMAC secret is sealed and never returned; a blank secret keeps the
// sealed one.
func (m *Module) putSource(ctx context.Context, in *putSourceInput) (*httpapi.Reply[sourceCreated], error) {
if !validName.MatchString(in.Name) || !validName.MatchString(in.Body.Connector) {
return nil, huma.Error400BadRequest("a source needs a lowercase name and a connector")
@@ -88,55 +89,77 @@ func (m *Module) putSource(ctx context.Context, in *putSourceInput) (*httpapi.Re
if err != nil && !errors.Is(err, store.ErrNotFound) {
return nil, huma.Error500InternalServerError("couldn't read the source", err)
}
found := err == nil
created := sourceCreated{Source: in.Name, URL: "/hooks/" + in.Name}
src := store.Source{OrgID: m.orgID, Name: in.Name, Connector: in.Body.Connector, Verification: in.Body.Verification}
switch {
case src.Verification == "" && found:
// Naming no scheme keeps the stored one with its token or secret, so
// a save can't downgrade a source.
src = existing
src.Connector = in.Body.Connector
case src.Verification == "":
src.Verification = store.Token
var stored *store.Source
if err == nil {
stored = &existing
}
if src.Verification == store.Token && (!found || in.Body.Verification != "") {
token, err := newToken()
src, token, err := m.sourceToSave(in, stored)
if err != nil {
return nil, huma.Error500InternalServerError("couldn't make a token", err)
}
src.TokenSHA256, created.Token = httpjson.TokenHash(token), token
}
switch src.Verification {
case store.Token, store.None:
if in.Body.Verification != "" && src.Verification == store.None {
src.TokenSHA256 = ""
}
case store.HMAC:
if in.Body.Verification != "" {
src.HMACHeader, src.HMACPrefix, src.TokenSHA256 = in.Body.HMACHeader, in.Body.HMACPrefix, ""
switch {
case in.Body.HMACSecret != "":
sealed, err := m.sealer.Seal(secretLabel(m.orgID, in.Name), []byte(in.Body.HMACSecret))
if err != nil {
return nil, huma.Error500InternalServerError("couldn't seal the secret", err)
}
src.HMACSecretSealed = sealed
case found && existing.Verification == store.HMAC:
// A blank secret keeps the sealed one.
src.HMACSecretSealed = existing.HMACSecretSealed
}
}
if src.HMACHeader == "" || len(src.HMACSecretSealed) == 0 {
return nil, huma.Error400BadRequest("an hmac source needs the signature header and a secret")
}
default:
return nil, huma.Error400BadRequest("verification is token, hmac or none")
return nil, err
}
if err := m.store.UpsertSource(ctx, src); err != nil {
return nil, huma.Error500InternalServerError("couldn't save the source", err)
}
return httpapi.OK(created), nil
return httpapi.OK(sourceCreated{Source: in.Name, Token: token, URL: "/hooks/" + in.Name}), nil
}
// sourceToSave builds the row to store from the request and the stored source,
// if there is one. It returns the new token when one was issued.
func (m *Module) sourceToSave(in *putSourceInput, stored *store.Source) (store.Source, string, error) {
explicit := in.Body.Verification != ""
src := store.Source{OrgID: m.orgID, Name: in.Name, Connector: in.Body.Connector, Verification: in.Body.Verification}
switch {
case !explicit && stored != nil:
// Omitting the scheme keeps the stored one with its token or secret, so
// a save cannot downgrade a source.
src = *stored
src.Connector = in.Body.Connector
case !explicit:
src.Verification = store.Token
}
switch src.Verification {
case store.Token:
if stored != nil && !explicit {
return src, "", nil
}
token, err := newToken()
if err != nil {
return src, "", huma.Error500InternalServerError("couldn't generate a token", err)
}
src.TokenSHA256 = httpjson.TokenHash(token)
return src, token, nil
case store.None:
return src, "", nil
case store.HMAC:
if explicit {
if err := m.setHMAC(&src, in, stored); err != nil {
return src, "", err
}
}
if src.HMACHeader == "" || len(src.HMACSecretSealed) == 0 {
return src, "", huma.Error400BadRequest("an HMAC source needs a signature header and a secret")
}
return src, "", nil
default:
return src, "", huma.Error400BadRequest("verification must be token, hmac or none")
}
}
// setHMAC applies the requested HMAC settings. A blank secret keeps the sealed
// secret of a source that already uses HMAC.
func (m *Module) setHMAC(src *store.Source, in *putSourceInput, stored *store.Source) error {
src.HMACHeader, src.HMACPrefix = in.Body.HMACHeader, in.Body.HMACPrefix
switch {
case in.Body.HMACSecret != "":
sealed, err := m.sealer.Seal(secretLabel(m.orgID, in.Name), []byte(in.Body.HMACSecret))
if err != nil {
return huma.Error500InternalServerError("couldn't seal the secret", err)
}
src.HMACSecretSealed = sealed
case stored != nil && stored.Verification == store.HMAC:
src.HMACSecretSealed = stored.HMACSecretSealed
}
return nil
}
type putTriggerInput struct {
@@ -279,8 +302,10 @@ func (m *Module) sources(ctx context.Context, _ *struct{}) (*httpapi.Reply[[]sou
}
out := make([]sourceView, 0, len(sources))
for _, src := range sources {
view := sourceView{Name: src.Name, Connector: src.Connector, TokenSet: src.TokenSHA256 != "", KeyFields: []string{},
Verification: src.Verification, HMACHeader: src.HMACHeader, HMACPrefix: src.HMACPrefix, HMACSecretSet: len(src.HMACSecretSealed) > 0}
view := sourceView{
Name: src.Name, Connector: src.Connector, TokenSet: src.TokenSHA256 != "", KeyFields: []string{},
Verification: src.Verification, HMACHeader: src.HMACHeader, HMACPrefix: src.HMACPrefix, HMACSecretSet: len(src.HMACSecretSealed) > 0,
}
if len(src.KeyFields) > 0 {
if err := json.Unmarshal(src.KeyFields, &view.KeyFields); err != nil {
m.log.ErrorContext(ctx, "read source key fields", "source", src.Name, "error", err)
+11 -10
View File
@@ -87,7 +87,7 @@ func NewModule(db *sqlx.DB, log *slog.Logger, orgID string, sealer Sealer, c Con
return m
}
// sourceReader is the one read intake makes of the store.
// sourceReader is the store method that intake uses to look up a source.
type sourceReader interface {
Source(ctx context.Context, orgID, name string) (store.Source, error)
}
@@ -98,9 +98,9 @@ func (m *Module) Routes(mux *http.ServeMux) {
mux.HandleFunc("POST /hooks/{source}", m.intake)
}
// intake has the source's connector parse a verified delivery and stores each
// event before answering. A re-delivered event is a constraint conflict,
// counted as a duplicate.
// intake verifies a delivery, parses it with the source's connector and stores
// each event before responding. A redelivered event violates a uniqueness
// constraint and is counted as a duplicate.
func (m *Module) intake(w http.ResponseWriter, r *http.Request) {
src, parsed, body, status, err := m.accept(r)
if err != nil {
@@ -115,10 +115,11 @@ func (m *Module) intake(w http.ResponseWriter, r *http.Request) {
httpjson.Write(w, http.StatusAccepted, map[string]int{"stored": stored, "duplicates": duplicates})
}
// accept verifies a delivery against its source's scheme, then has the
// connector parse it. A delivery that fails verification is refused with 401
// before the connector sees its body, and the refusal is logged. An unknown
// source and a failed delivery answer alike, so neither is probed.
// accept verifies a delivery against its source's scheme, then passes it to
// the connector for parsing. A delivery that fails verification is rejected
// with 401 before the connector sees its body, and the rejection is logged.
// An unknown source and a failed verification get the same response, so a
// caller cannot use it to discover source names.
func (m *Module) accept(r *http.Request) (src store.Source, parsed []connectors.Event, body []byte, status int, err error) {
ctx := r.Context()
src, err = m.reader.Source(ctx, m.orgID, r.PathValue("source"))
@@ -163,8 +164,8 @@ var (
errConnectorDown = errors.New("connector isn't running")
)
// refuse logs why a delivery was turned away and picks its status: 401 when it
// failed to prove itself, 503 when verification couldn't run.
// refuse logs why a delivery was rejected and chooses the status: 401 when
// verification failed, 503 when verification could not run.
func (m *Module) refuse(ctx context.Context, src store.Source, err error) (store.Source, []connectors.Event, []byte, int, error) {
m.log.WarnContext(ctx, "refused webhook delivery", "source", src.Name, "verification", src.Verification, "reason", err.Error())
if errors.Is(err, errUnverified) {
+5 -4
View File
@@ -18,7 +18,7 @@ import (
// ErrNotFound means the named row doesn't exist.
var ErrNotFound = errors.New("not found")
// The ways a source verifies a delivery.
// Verification schemes for a source's deliveries.
const (
Token = "token"
HMAC = "hmac"
@@ -31,10 +31,11 @@ type Source struct {
Name string `db:"name"`
Connector string `db:"connector"`
TokenSHA256 string `db:"token_sha256"`
// Verification is how a delivery proves itself: Token, HMAC or None.
// Verification is the scheme that deliveries must pass: Token, HMAC or None.
Verification string `db:"verification"`
// HMACHeader carries the signature, HMACPrefix precedes its hex digest,
// and HMACSecretSealed is the shared secret, sealed.
// HMACHeader names the header that carries the signature, HMACPrefix is
// the text before its hex digest, and HMACSecretSealed is the sealed
// shared secret.
HMACHeader string `db:"hmac_header"`
HMACPrefix string `db:"hmac_prefix"`
HMACSecretSealed []byte `db:"hmac_secret_sealed"`
+5 -5
View File
@@ -26,8 +26,8 @@ func secretLabel(orgID, name string) string {
return "events.sources/" + orgID + "/" + name + "/hmac"
}
// errUnverified means a delivery didn't prove itself. Its text is safe to
// log: it names the scheme, never a secret or a signature.
// errUnverified means a delivery failed verification. Its text is safe to log:
// it names the scheme, never a secret or a signature.
var errUnverified = errors.New("delivery not verified")
// checkToken verifies a token source before its body is read.
@@ -39,9 +39,9 @@ func checkToken(src store.Source, r *http.Request) error {
}
// verifyHMAC checks the signature header against an HMAC-SHA256 of the raw
// body, which the connector hasn't seen yet. The header holds the hex digest
// after the source's prefix, in either case. An absent or malformed
// signature never verifies.
// body, before the connector sees it. The header holds the hex digest, in
// either case, after the source's prefix. A missing or malformed signature
// always fails.
//
// Ported from internal/webhookguard/hmac.go (digest comparison) and
// internal/infrastructure/captureintake/receiver.go (header-borne signature),
+2 -3
View File
@@ -5,7 +5,6 @@ import (
"crypto/hmac"
"crypto/sha256"
"encoding/hex"
"io"
"log/slog"
"net/http"
"net/http/httptest"
@@ -73,13 +72,13 @@ func TestHMACSourceVerifiesBeforeParsing(t *testing.T) {
parsed := false
m := &Module{
orgID: "org", reader: fakeSources{src}, sealer: sealer,
log: slog.New(slog.NewTextHandler(io.Discard, nil)),
log: slog.New(slog.DiscardHandler),
parse: func(context.Context, string, []byte, map[string]string) ([]connectors.Event, error) {
parsed = true
return nil, nil
},
}
req := httptest.NewRequest(http.MethodPost, "/hooks/hook", strings.NewReader(body))
req := httptest.NewRequestWithContext(t.Context(), http.MethodPost, "/hooks/hook", strings.NewReader(body))
req.SetPathValue("source", "hook")
if tt.header != "" {
req.Header.Set("X-Signature", tt.header)
+2 -2
View File
@@ -30,8 +30,8 @@ service ConnectorService {
// delivery is never offered to an agent: hard rules apply to it, guards
// don't. The host retries a failed delivery, so a connector should make a
// repeat with the same idempotency key harmless where its service allows.
// A connector answers InvalidArgument, FailedPrecondition, PermissionDenied
// or NotFound when the service refuses the delivery for good: the host
// A connector returns InvalidArgument, FailedPrecondition, PermissionDenied
// or NotFound when the service rejects the delivery permanently: the host
// records the reason and stops retrying. Any other status is retried.
rpc Deliver(DeliverRequest) returns (DeliverResponse);
// Listen streams the events of a source the connector holds its own
+3 -3
View File
@@ -29,7 +29,7 @@ onMounted(() => {
const columns: DataTableColumn<EventSource_>[] = [
{ id: "name", label: "Source", fixed: true, sort: (s) => s.name },
{ id: "connector", label: "Connector", sort: (s) => s.connector },
{ id: "verification", label: "Verified by" },
{ id: "verification", label: "Verification" },
{ id: "key", label: "Grouped by" },
];
@@ -131,7 +131,7 @@ const connectorNames = computed(() => (connectors.data.value ?? []).map((c) => c
<DataTable :columns="columns" :rows="data" :row-key="(s) => s.name" storage-key="sources">
<template #cell-name="{ row }"><span class="font-medium">{{ row.name }}</span></template>
<template #cell-connector="{ row }">{{ row.connector }}</template>
<template #cell-verification="{ row }">{{ row.verification === "hmac" ? `HMAC (${row.hmac_header})` : row.verification === "none" ? "Nothing" : "Bearer token" }}</template>
<template #cell-verification="{ row }">{{ row.verification === "hmac" ? `HMAC (${row.hmac_header})` : row.verification === "none" ? "None" : "Bearer token" }}</template>
<template #cell-key="{ row }"><span class="font-mono text-caption">{{ (row.key_fields ?? []).join(", ") || "Connector default" }}</span></template>
<template #actions="{ row }">
<MoreMenu :label="`${row.name} actions`">
@@ -160,7 +160,7 @@ const connectorNames = computed(() => (connectors.data.value ?? []).map((c) => c
<SelectContent>
<SelectItem value="token">Bearer token</SelectItem>
<SelectItem value="hmac">HMAC signature</SelectItem>
<SelectItem value="none">Nothing</SelectItem>
<SelectItem value="none">None</SelectItem>
</SelectContent>
</Select>
<template v-if="draft.verification === 'hmac'">