refactor(events,deliveries): split source saving and plain the new prose (#2021, #2016)

This commit is contained in:
sam committed 2026-11-21 13:52:33 +11:00
1 parent 29a06f7c54
commit f47e60db00
14 files changed
+139 -110

No files matched your search

+6 -6
View File
@@ -948,7 +948,7 @@ components:
source: source:
type: string type: string
token: token:
description: Shown once, for a token source; only its hash is stored description: Returned once for a token source; only its hash is stored
type: string type: string
url: url:
description: Where the sender posts its events description: Where the sender posts its events
@@ -2084,18 +2084,18 @@ components:
description: The connector that parses this source's events description: The connector that parses this source's events
type: string type: string
hmac_header: hmac_header:
description: The header that carries the signature, for hmac description: The header that carries the signature (hmac only)
type: string type: string
hmac_prefix: hmac_prefix:
description: Text before the hex digest in that header, such as sha256=, for hmac description: Text before the hex digest in the signature header, such as sha256= (hmac only)
examples: examples:
- sha256= - sha256=
type: string type: string
hmac_secret: hmac_secret:
description: "The shared signing secret, for hmac. Write-only: it is sealed and never returned" description: "The shared signing secret (hmac only). Write-only: it is sealed and never returned"
type: string type: string
verification: verification:
description: "How a delivery proves itself: a bearer token, an HMAC-SHA256 signature of the body, or nothing" description: "How deliveries are verified: a bearer token, an HMAC-SHA256 signature of the body, or no verification"
enum: enum:
- token - token
- hmac - hmac
@@ -5381,7 +5381,7 @@ paths:
schema: schema:
$ref: "#/components/schemas/ErrorModel" $ref: "#/components/schemas/ErrorModel"
description: Error description: Error
summary: Add a source, or change how it verifies deliveries and rotate its token summary: Add or update a source, its verification scheme and its token
tags: tags:
- Events - Events
/api/sources/{name}/key-fields: /api/sources/{name}/key-fields:
@@ -60,8 +60,8 @@ type ConnectorServiceClient interface {
// delivery is never offered to an agent: hard rules apply to it, guards // delivery is never offered to an agent: hard rules apply to it, guards
// don't. The host retries a failed delivery, so a connector should make a // don't. The host retries a failed delivery, so a connector should make a
// repeat with the same idempotency key harmless where its service allows. // repeat with the same idempotency key harmless where its service allows.
// A connector answers InvalidArgument, FailedPrecondition, PermissionDenied // A connector returns InvalidArgument, FailedPrecondition, PermissionDenied
// or NotFound when the service refuses the delivery for good: the host // or NotFound when the service rejects the delivery permanently: the host
// records the reason and stops retrying. Any other status is retried. // records the reason and stops retrying. Any other status is retried.
Deliver(ctx context.Context, in *DeliverRequest, opts ...grpc.CallOption) (*DeliverResponse, error) Deliver(ctx context.Context, in *DeliverRequest, opts ...grpc.CallOption) (*DeliverResponse, error)
// Listen streams the events of a source the connector holds its own // Listen streams the events of a source the connector holds its own
@@ -213,8 +213,8 @@ type ConnectorServiceServer interface {
// delivery is never offered to an agent: hard rules apply to it, guards // delivery is never offered to an agent: hard rules apply to it, guards
// don't. The host retries a failed delivery, so a connector should make a // don't. The host retries a failed delivery, so a connector should make a
// repeat with the same idempotency key harmless where its service allows. // repeat with the same idempotency key harmless where its service allows.
// A connector answers InvalidArgument, FailedPrecondition, PermissionDenied // A connector returns InvalidArgument, FailedPrecondition, PermissionDenied
// or NotFound when the service refuses the delivery for good: the host // or NotFound when the service rejects the delivery permanently: the host
// records the reason and stops retrying. Any other status is retried. // records the reason and stops retrying. Any other status is retried.
Deliver(context.Context, *DeliverRequest) (*DeliverResponse, error) Deliver(context.Context, *DeliverRequest) (*DeliverResponse, error)
// Listen streams the events of a source the connector holds its own // Listen streams the events of a source the connector holds its own
@@ -1,7 +1,7 @@
-- +goose Up -- +goose Up
-- How a source's deliveries are verified. A token source checks the bearer -- How a source's deliveries are verified. A token source checks the bearer
-- token and an hmac source checks a signature over the raw body with a shared -- token. An hmac source checks a signature over the raw body using a shared
-- secret, which is kept sealed. A source that chose none checks nothing. -- secret, which is stored sealed. A source set to none performs no check.
ALTER TABLE events.sources ALTER TABLE events.sources
ADD COLUMN verification text NOT NULL DEFAULT 'token' ADD COLUMN verification text NOT NULL DEFAULT 'token'
CHECK (verification IN ('token', 'hmac', 'none')), CHECK (verification IN ('token', 'hmac', 'none')),
@@ -1,6 +1,7 @@
-- +goose Up -- +goose Up
-- A delivery the channel refuses for good, or that keeps failing past the retry -- A delivery that the channel rejects permanently, or that still fails at the
-- limit, is failed rather than retried: failed_at stops the worker claiming it. -- retry limit, is marked failed instead of retried. The worker does not claim
-- rows with failed_at set.
ALTER TABLE deliveries.messages ADD COLUMN failed_at timestamptz; ALTER TABLE deliveries.messages ADD COLUMN failed_at timestamptz;
DROP INDEX deliveries.messages_due; DROP INDEX deliveries.messages_due;
CREATE INDEX messages_due ON deliveries.messages (next_attempt_at) WHERE sent_at IS NULL AND failed_at IS NULL; CREATE INDEX messages_due ON deliveries.messages (next_attempt_at) WHERE sent_at IS NULL AND failed_at IS NULL;
+7 -6
View File
@@ -31,10 +31,11 @@ var (
// ErrUnknownName names an action, lookup or event type the connector // ErrUnknownName names an action, lookup or event type the connector
// never declared. // never declared.
ErrUnknownName = errors.New("not declared by the connector") ErrUnknownName = errors.New("not declared by the connector")
// ErrRejected means a delivery was refused in a way a retry can't change: // ErrRejected means a delivery was rejected permanently, so retrying it
// the connector answered InvalidArgument, FailedPrecondition, // cannot succeed: the connector returned InvalidArgument,
// PermissionDenied or NotFound, or the input broke the declared schema. // FailedPrecondition, PermissionDenied or NotFound, or the input did not
ErrRejected = errors.New("delivery rejected for good") // match the declared schema.
ErrRejected = errors.New("delivery permanently rejected")
) )
// Reversibility says whether and how an action can be undone. // Reversibility says whether and how an action can be undone.
@@ -249,8 +250,8 @@ func (c *Connector) Deliver(ctx context.Context, name string, input jsontext.Val
return resp.GetExternalId(), nil return resp.GetExternalId(), nil
} }
// rejectedForGood reports whether a connector's error says the same delivery // rejectedForGood reports whether a connector's error status means that
// can never succeed. // retrying the same delivery cannot succeed.
func rejectedForGood(err error) bool { func rejectedForGood(err error) bool {
switch status.Code(err) { switch status.Code(err) {
case codes.InvalidArgument, codes.FailedPrecondition, codes.PermissionDenied, codes.NotFound: case codes.InvalidArgument, codes.FailedPrecondition, codes.PermissionDenied, codes.NotFound:
+4 -3
View File
@@ -4,7 +4,8 @@
// dashboard, is the first channel: its messages are rows a client reads from a // dashboard, is the first channel: its messages are rows a client reads from a
// cursor, with a live stream that only says when to read. Every accepted // cursor, with a live stream that only says when to read. Every accepted
// connector that delivers messages is another: each gets its own row, retried // connector that delivers messages is another: each gets its own row, retried
// until the connector confirms it, refuses it for good, or the attempts run out. // until the connector confirms it, rejects it permanently, or the retry limit
// is reached.
package deliveries package deliveries
import ( import (
@@ -228,8 +229,8 @@ func (m *Module) deliver(ctx context.Context, msg store.Message) {
} }
} }
// recordFailure stores why an attempt failed, and ends the delivery when the // recordFailure records why an attempt failed. It marks the delivery failed
// channel refused it for good or the attempts are spent. // when the channel rejected it permanently or the retry limit is reached.
func (m *Module) recordFailure(ctx context.Context, msg store.Message, cause error, took time.Duration) { func (m *Module) recordFailure(ctx context.Context, msg store.Message, cause error, took time.Duration) {
record := m.store.Failed record := m.store.Failed
reason := cause.Error() reason := cause.Error()
+2 -2
View File
@@ -171,8 +171,8 @@ func (s *Store) Failed(ctx context.Context, seq int64, attempt int, reason strin
`UPDATE deliveries.messages SET last_error = $2 WHERE seq = $1`, reason) `UPDATE deliveries.messages SET last_error = $2 WHERE seq = $1`, reason)
} }
// GaveUp records the attempt that ended a delivery for good: the message is // GaveUp records the final attempt of a delivery and marks the message failed,
// marked failed and no worker claims it again. // so no worker claims it again.
func (s *Store) GaveUp(ctx context.Context, seq int64, attempt int, reason string, took time.Duration) error { func (s *Store) GaveUp(ctx context.Context, seq int64, attempt int, reason string, took time.Duration) error {
return s.attempt(ctx, seq, attempt, attemptOutcome{failure: reason, took: took}, return s.attempt(ctx, seq, attempt, attemptOutcome{failure: reason, took: took},
`UPDATE deliveries.messages SET failed_at = now(), last_error = $2 WHERE seq = $1`, reason) `UPDATE deliveries.messages SET failed_at = now(), last_error = $2 WHERE seq = $1`, reason)
+83 -58
View File
@@ -29,7 +29,7 @@ func (m *Module) Register(api huma.API) {
}, m.sources) }, m.sources)
huma.Register(api, huma.Operation{ huma.Register(api, huma.Operation{
OperationID: "put-source", Method: http.MethodPut, Path: "/api/sources/{name}", OperationID: "put-source", Method: http.MethodPut, Path: "/api/sources/{name}",
Summary: "Add a source, or change how it verifies deliveries and rotate its token", Tags: tags, Summary: "Add or update a source, its verification scheme and its token", Tags: tags,
}, m.putSource) }, m.putSource)
huma.Register(api, huma.Operation{ huma.Register(api, huma.Operation{
OperationID: "put-source-key-fields", Method: http.MethodPut, Path: "/api/sources/{name}/key-fields", OperationID: "put-source-key-fields", Method: http.MethodPut, Path: "/api/sources/{name}/key-fields",
@@ -61,25 +61,26 @@ type putSourceInput struct {
Name string `path:"name" doc:"Lowercase letters, digits and hyphens; the webhook URL ends in it"` Name string `path:"name" doc:"Lowercase letters, digits and hyphens; the webhook URL ends in it"`
Body struct { Body struct {
Connector string `json:"connector" doc:"The connector that parses this source's events"` Connector string `json:"connector" doc:"The connector that parses this source's events"`
// Verification defaults to a bearer token so a source that names none // A new source without a scheme uses a bearer token, so existing
// keeps working. // senders keep working.
Verification string `json:"verification,omitzero" enum:"token,hmac,none" doc:"How a delivery proves itself: a bearer token, an HMAC-SHA256 signature of the body, or nothing"` Verification string `json:"verification,omitzero" enum:"token,hmac,none" doc:"How deliveries are verified: a bearer token, an HMAC-SHA256 signature of the body, or no verification"`
HMACHeader string `json:"hmac_header,omitzero" doc:"The header that carries the signature, for hmac"` HMACHeader string `json:"hmac_header,omitzero" doc:"The header that carries the signature (hmac only)"`
HMACPrefix string `json:"hmac_prefix,omitzero" doc:"Text before the hex digest in that header, such as sha256=, for hmac" example:"sha256="` HMACPrefix string `json:"hmac_prefix,omitzero" doc:"Text before the hex digest in the signature header, such as sha256= (hmac only)" example:"sha256="`
HMACSecret string `json:"hmac_secret,omitzero" doc:"The shared signing secret, for hmac. Write-only: it is sealed and never returned"` HMACSecret string `json:"hmac_secret,omitzero" doc:"The shared signing secret (hmac only). Write-only: it is sealed and never returned"`
} }
} }
type sourceCreated struct { type sourceCreated struct {
Source string `json:"source"` Source string `json:"source"`
Token string `json:"token,omitzero" doc:"Shown once, for a token source; only its hash is stored"` Token string `json:"token,omitzero" doc:"Returned once for a token source; only its hash is stored"`
URL string `json:"url" doc:"Where the sender posts its events"` URL string `json:"url" doc:"Where the sender posts its events"`
} }
// putSource creates a source or changes it. A save that names no verification // putSource creates or updates a source. A save that omits verification keeps
// keeps the stored scheme and its credentials. A token is returned once, when // the stored scheme and its credentials. A token is returned only when the
// a token scheme is chosen, and stored only as its hash. An hmac secret is // token scheme is chosen explicitly or the source is new, and only its hash is
// sealed and never returned; leaving it blank keeps the sealed one. // stored. An HMAC secret is sealed and never returned; a blank secret keeps the
// sealed one.
func (m *Module) putSource(ctx context.Context, in *putSourceInput) (*httpapi.Reply[sourceCreated], error) { func (m *Module) putSource(ctx context.Context, in *putSourceInput) (*httpapi.Reply[sourceCreated], error) {
if !validName.MatchString(in.Name) || !validName.MatchString(in.Body.Connector) { if !validName.MatchString(in.Name) || !validName.MatchString(in.Body.Connector) {
return nil, huma.Error400BadRequest("a source needs a lowercase name and a connector") return nil, huma.Error400BadRequest("a source needs a lowercase name and a connector")
@@ -88,55 +89,77 @@ func (m *Module) putSource(ctx context.Context, in *putSourceInput) (*httpapi.Re
if err != nil && !errors.Is(err, store.ErrNotFound) { if err != nil && !errors.Is(err, store.ErrNotFound) {
return nil, huma.Error500InternalServerError("couldn't read the source", err) return nil, huma.Error500InternalServerError("couldn't read the source", err)
} }
found := err == nil var stored *store.Source
created := sourceCreated{Source: in.Name, URL: "/hooks/" + in.Name} if err == nil {
src := store.Source{OrgID: m.orgID, Name: in.Name, Connector: in.Body.Connector, Verification: in.Body.Verification} stored = &existing
switch {
case src.Verification == "" && found:
// Naming no scheme keeps the stored one with its token or secret, so
// a save can't downgrade a source.
src = existing
src.Connector = in.Body.Connector
case src.Verification == "":
src.Verification = store.Token
} }
if src.Verification == store.Token && (!found || in.Body.Verification != "") { src, token, err := m.sourceToSave(in, stored)
token, err := newToken() if err != nil {
if err != nil { return nil, err
return nil, huma.Error500InternalServerError("couldn't make a token", err)
}
src.TokenSHA256, created.Token = httpjson.TokenHash(token), token
}
switch src.Verification {
case store.Token, store.None:
if in.Body.Verification != "" && src.Verification == store.None {
src.TokenSHA256 = ""
}
case store.HMAC:
if in.Body.Verification != "" {
src.HMACHeader, src.HMACPrefix, src.TokenSHA256 = in.Body.HMACHeader, in.Body.HMACPrefix, ""
switch {
case in.Body.HMACSecret != "":
sealed, err := m.sealer.Seal(secretLabel(m.orgID, in.Name), []byte(in.Body.HMACSecret))
if err != nil {
return nil, huma.Error500InternalServerError("couldn't seal the secret", err)
}
src.HMACSecretSealed = sealed
case found && existing.Verification == store.HMAC:
// A blank secret keeps the sealed one.
src.HMACSecretSealed = existing.HMACSecretSealed
}
}
if src.HMACHeader == "" || len(src.HMACSecretSealed) == 0 {
return nil, huma.Error400BadRequest("an hmac source needs the signature header and a secret")
}
default:
return nil, huma.Error400BadRequest("verification is token, hmac or none")
} }
if err := m.store.UpsertSource(ctx, src); err != nil { if err := m.store.UpsertSource(ctx, src); err != nil {
return nil, huma.Error500InternalServerError("couldn't save the source", err) return nil, huma.Error500InternalServerError("couldn't save the source", err)
} }
return httpapi.OK(created), nil return httpapi.OK(sourceCreated{Source: in.Name, Token: token, URL: "/hooks/" + in.Name}), nil
}
// sourceToSave builds the row to store from the request and the stored source,
// if there is one. It returns the new token when one was issued.
func (m *Module) sourceToSave(in *putSourceInput, stored *store.Source) (store.Source, string, error) {
explicit := in.Body.Verification != ""
src := store.Source{OrgID: m.orgID, Name: in.Name, Connector: in.Body.Connector, Verification: in.Body.Verification}
switch {
case !explicit && stored != nil:
// Omitting the scheme keeps the stored one with its token or secret, so
// a save cannot downgrade a source.
src = *stored
src.Connector = in.Body.Connector
case !explicit:
src.Verification = store.Token
}
switch src.Verification {
case store.Token:
if stored != nil && !explicit {
return src, "", nil
}
token, err := newToken()
if err != nil {
return src, "", huma.Error500InternalServerError("couldn't generate a token", err)
}
src.TokenSHA256 = httpjson.TokenHash(token)
return src, token, nil
case store.None:
return src, "", nil
case store.HMAC:
if explicit {
if err := m.setHMAC(&src, in, stored); err != nil {
return src, "", err
}
}
if src.HMACHeader == "" || len(src.HMACSecretSealed) == 0 {
return src, "", huma.Error400BadRequest("an HMAC source needs a signature header and a secret")
}
return src, "", nil
default:
return src, "", huma.Error400BadRequest("verification must be token, hmac or none")
}
}
// setHMAC applies the requested HMAC settings. A blank secret keeps the sealed
// secret of a source that already uses HMAC.
func (m *Module) setHMAC(src *store.Source, in *putSourceInput, stored *store.Source) error {
src.HMACHeader, src.HMACPrefix = in.Body.HMACHeader, in.Body.HMACPrefix
switch {
case in.Body.HMACSecret != "":
sealed, err := m.sealer.Seal(secretLabel(m.orgID, in.Name), []byte(in.Body.HMACSecret))
if err != nil {
return huma.Error500InternalServerError("couldn't seal the secret", err)
}
src.HMACSecretSealed = sealed
case stored != nil && stored.Verification == store.HMAC:
src.HMACSecretSealed = stored.HMACSecretSealed
}
return nil
} }
type putTriggerInput struct { type putTriggerInput struct {
@@ -279,8 +302,10 @@ func (m *Module) sources(ctx context.Context, _ *struct{}) (*httpapi.Reply[[]sou
} }
out := make([]sourceView, 0, len(sources)) out := make([]sourceView, 0, len(sources))
for _, src := range sources { for _, src := range sources {
view := sourceView{Name: src.Name, Connector: src.Connector, TokenSet: src.TokenSHA256 != "", KeyFields: []string{}, view := sourceView{
Verification: src.Verification, HMACHeader: src.HMACHeader, HMACPrefix: src.HMACPrefix, HMACSecretSet: len(src.HMACSecretSealed) > 0} Name: src.Name, Connector: src.Connector, TokenSet: src.TokenSHA256 != "", KeyFields: []string{},
Verification: src.Verification, HMACHeader: src.HMACHeader, HMACPrefix: src.HMACPrefix, HMACSecretSet: len(src.HMACSecretSealed) > 0,
}
if len(src.KeyFields) > 0 { if len(src.KeyFields) > 0 {
if err := json.Unmarshal(src.KeyFields, &view.KeyFields); err != nil { if err := json.Unmarshal(src.KeyFields, &view.KeyFields); err != nil {
m.log.ErrorContext(ctx, "read source key fields", "source", src.Name, "error", err) m.log.ErrorContext(ctx, "read source key fields", "source", src.Name, "error", err)
+11 -10
View File
@@ -87,7 +87,7 @@ func NewModule(db *sqlx.DB, log *slog.Logger, orgID string, sealer Sealer, c Con
return m return m
} }
// sourceReader is the one read intake makes of the store. // sourceReader is the store method that intake uses to look up a source.
type sourceReader interface { type sourceReader interface {
Source(ctx context.Context, orgID, name string) (store.Source, error) Source(ctx context.Context, orgID, name string) (store.Source, error)
} }
@@ -98,9 +98,9 @@ func (m *Module) Routes(mux *http.ServeMux) {
mux.HandleFunc("POST /hooks/{source}", m.intake) mux.HandleFunc("POST /hooks/{source}", m.intake)
} }
// intake has the source's connector parse a verified delivery and stores each // intake verifies a delivery, parses it with the source's connector and stores
// event before answering. A re-delivered event is a constraint conflict, // each event before responding. A redelivered event violates a uniqueness
// counted as a duplicate. // constraint and is counted as a duplicate.
func (m *Module) intake(w http.ResponseWriter, r *http.Request) { func (m *Module) intake(w http.ResponseWriter, r *http.Request) {
src, parsed, body, status, err := m.accept(r) src, parsed, body, status, err := m.accept(r)
if err != nil { if err != nil {
@@ -115,10 +115,11 @@ func (m *Module) intake(w http.ResponseWriter, r *http.Request) {
httpjson.Write(w, http.StatusAccepted, map[string]int{"stored": stored, "duplicates": duplicates}) httpjson.Write(w, http.StatusAccepted, map[string]int{"stored": stored, "duplicates": duplicates})
} }
// accept verifies a delivery against its source's scheme, then has the // accept verifies a delivery against its source's scheme, then passes it to
// connector parse it. A delivery that fails verification is refused with 401 // the connector for parsing. A delivery that fails verification is rejected
// before the connector sees its body, and the refusal is logged. An unknown // with 401 before the connector sees its body, and the rejection is logged.
// source and a failed delivery answer alike, so neither is probed. // An unknown source and a failed verification get the same response, so a
// caller cannot use it to discover source names.
func (m *Module) accept(r *http.Request) (src store.Source, parsed []connectors.Event, body []byte, status int, err error) { func (m *Module) accept(r *http.Request) (src store.Source, parsed []connectors.Event, body []byte, status int, err error) {
ctx := r.Context() ctx := r.Context()
src, err = m.reader.Source(ctx, m.orgID, r.PathValue("source")) src, err = m.reader.Source(ctx, m.orgID, r.PathValue("source"))
@@ -163,8 +164,8 @@ var (
errConnectorDown = errors.New("connector isn't running") errConnectorDown = errors.New("connector isn't running")
) )
// refuse logs why a delivery was turned away and picks its status: 401 when it // refuse logs why a delivery was rejected and chooses the status: 401 when
// failed to prove itself, 503 when verification couldn't run. // verification failed, 503 when verification could not run.
func (m *Module) refuse(ctx context.Context, src store.Source, err error) (store.Source, []connectors.Event, []byte, int, error) { func (m *Module) refuse(ctx context.Context, src store.Source, err error) (store.Source, []connectors.Event, []byte, int, error) {
m.log.WarnContext(ctx, "refused webhook delivery", "source", src.Name, "verification", src.Verification, "reason", err.Error()) m.log.WarnContext(ctx, "refused webhook delivery", "source", src.Name, "verification", src.Verification, "reason", err.Error())
if errors.Is(err, errUnverified) { if errors.Is(err, errUnverified) {
+5 -4
View File
@@ -18,7 +18,7 @@ import (
// ErrNotFound means the named row doesn't exist. // ErrNotFound means the named row doesn't exist.
var ErrNotFound = errors.New("not found") var ErrNotFound = errors.New("not found")
// The ways a source verifies a delivery. // Verification schemes for a source's deliveries.
const ( const (
Token = "token" Token = "token"
HMAC = "hmac" HMAC = "hmac"
@@ -31,10 +31,11 @@ type Source struct {
Name string `db:"name"` Name string `db:"name"`
Connector string `db:"connector"` Connector string `db:"connector"`
TokenSHA256 string `db:"token_sha256"` TokenSHA256 string `db:"token_sha256"`
// Verification is how a delivery proves itself: Token, HMAC or None. // Verification is the scheme that deliveries must pass: Token, HMAC or None.
Verification string `db:"verification"` Verification string `db:"verification"`
// HMACHeader carries the signature, HMACPrefix precedes its hex digest, // HMACHeader names the header that carries the signature, HMACPrefix is
// and HMACSecretSealed is the shared secret, sealed. // the text before its hex digest, and HMACSecretSealed is the sealed
// shared secret.
HMACHeader string `db:"hmac_header"` HMACHeader string `db:"hmac_header"`
HMACPrefix string `db:"hmac_prefix"` HMACPrefix string `db:"hmac_prefix"`
HMACSecretSealed []byte `db:"hmac_secret_sealed"` HMACSecretSealed []byte `db:"hmac_secret_sealed"`
+5 -5
View File
@@ -26,8 +26,8 @@ func secretLabel(orgID, name string) string {
return "events.sources/" + orgID + "/" + name + "/hmac" return "events.sources/" + orgID + "/" + name + "/hmac"
} }
// errUnverified means a delivery didn't prove itself. Its text is safe to // errUnverified means a delivery failed verification. Its text is safe to log:
// log: it names the scheme, never a secret or a signature. // it names the scheme, never a secret or a signature.
var errUnverified = errors.New("delivery not verified") var errUnverified = errors.New("delivery not verified")
// checkToken verifies a token source before its body is read. // checkToken verifies a token source before its body is read.
@@ -39,9 +39,9 @@ func checkToken(src store.Source, r *http.Request) error {
} }
// verifyHMAC checks the signature header against an HMAC-SHA256 of the raw // verifyHMAC checks the signature header against an HMAC-SHA256 of the raw
// body, which the connector hasn't seen yet. The header holds the hex digest // body, before the connector sees it. The header holds the hex digest, in
// after the source's prefix, in either case. An absent or malformed // either case, after the source's prefix. A missing or malformed signature
// signature never verifies. // always fails.
// //
// Ported from internal/webhookguard/hmac.go (digest comparison) and // Ported from internal/webhookguard/hmac.go (digest comparison) and
// internal/infrastructure/captureintake/receiver.go (header-borne signature), // internal/infrastructure/captureintake/receiver.go (header-borne signature),
+2 -3
View File
@@ -5,7 +5,6 @@ import (
"crypto/hmac" "crypto/hmac"
"crypto/sha256" "crypto/sha256"
"encoding/hex" "encoding/hex"
"io"
"log/slog" "log/slog"
"net/http" "net/http"
"net/http/httptest" "net/http/httptest"
@@ -73,13 +72,13 @@ func TestHMACSourceVerifiesBeforeParsing(t *testing.T) {
parsed := false parsed := false
m := &Module{ m := &Module{
orgID: "org", reader: fakeSources{src}, sealer: sealer, orgID: "org", reader: fakeSources{src}, sealer: sealer,
log: slog.New(slog.NewTextHandler(io.Discard, nil)), log: slog.New(slog.DiscardHandler),
parse: func(context.Context, string, []byte, map[string]string) ([]connectors.Event, error) { parse: func(context.Context, string, []byte, map[string]string) ([]connectors.Event, error) {
parsed = true parsed = true
return nil, nil return nil, nil
}, },
} }
req := httptest.NewRequest(http.MethodPost, "/hooks/hook", strings.NewReader(body)) req := httptest.NewRequestWithContext(t.Context(), http.MethodPost, "/hooks/hook", strings.NewReader(body))
req.SetPathValue("source", "hook") req.SetPathValue("source", "hook")
if tt.header != "" { if tt.header != "" {
req.Header.Set("X-Signature", tt.header) req.Header.Set("X-Signature", tt.header)
+2 -2
View File
@@ -30,8 +30,8 @@ service ConnectorService {
// delivery is never offered to an agent: hard rules apply to it, guards // delivery is never offered to an agent: hard rules apply to it, guards
// don't. The host retries a failed delivery, so a connector should make a // don't. The host retries a failed delivery, so a connector should make a
// repeat with the same idempotency key harmless where its service allows. // repeat with the same idempotency key harmless where its service allows.
// A connector answers InvalidArgument, FailedPrecondition, PermissionDenied // A connector returns InvalidArgument, FailedPrecondition, PermissionDenied
// or NotFound when the service refuses the delivery for good: the host // or NotFound when the service rejects the delivery permanently: the host
// records the reason and stops retrying. Any other status is retried. // records the reason and stops retrying. Any other status is retried.
rpc Deliver(DeliverRequest) returns (DeliverResponse); rpc Deliver(DeliverRequest) returns (DeliverResponse);
// Listen streams the events of a source the connector holds its own // Listen streams the events of a source the connector holds its own
+3 -3
View File
@@ -29,7 +29,7 @@ onMounted(() => {
const columns: DataTableColumn<EventSource_>[] = [ const columns: DataTableColumn<EventSource_>[] = [
{ id: "name", label: "Source", fixed: true, sort: (s) => s.name }, { id: "name", label: "Source", fixed: true, sort: (s) => s.name },
{ id: "connector", label: "Connector", sort: (s) => s.connector }, { id: "connector", label: "Connector", sort: (s) => s.connector },
{ id: "verification", label: "Verified by" }, { id: "verification", label: "Verification" },
{ id: "key", label: "Grouped by" }, { id: "key", label: "Grouped by" },
]; ];
@@ -131,7 +131,7 @@ const connectorNames = computed(() => (connectors.data.value ?? []).map((c) => c
<DataTable :columns="columns" :rows="data" :row-key="(s) => s.name" storage-key="sources"> <DataTable :columns="columns" :rows="data" :row-key="(s) => s.name" storage-key="sources">
<template #cell-name="{ row }"><span class="font-medium">{{ row.name }}</span></template> <template #cell-name="{ row }"><span class="font-medium">{{ row.name }}</span></template>
<template #cell-connector="{ row }">{{ row.connector }}</template> <template #cell-connector="{ row }">{{ row.connector }}</template>
<template #cell-verification="{ row }">{{ row.verification === "hmac" ? `HMAC (${row.hmac_header})` : row.verification === "none" ? "Nothing" : "Bearer token" }}</template> <template #cell-verification="{ row }">{{ row.verification === "hmac" ? `HMAC (${row.hmac_header})` : row.verification === "none" ? "None" : "Bearer token" }}</template>
<template #cell-key="{ row }"><span class="font-mono text-caption">{{ (row.key_fields ?? []).join(", ") || "Connector default" }}</span></template> <template #cell-key="{ row }"><span class="font-mono text-caption">{{ (row.key_fields ?? []).join(", ") || "Connector default" }}</span></template>
<template #actions="{ row }"> <template #actions="{ row }">
<MoreMenu :label="`${row.name} actions`"> <MoreMenu :label="`${row.name} actions`">
@@ -160,7 +160,7 @@ const connectorNames = computed(() => (connectors.data.value ?? []).map((c) => c
<SelectContent> <SelectContent>
<SelectItem value="token">Bearer token</SelectItem> <SelectItem value="token">Bearer token</SelectItem>
<SelectItem value="hmac">HMAC signature</SelectItem> <SelectItem value="hmac">HMAC signature</SelectItem>
<SelectItem value="none">Nothing</SelectItem> <SelectItem value="none">None</SelectItem>
</SelectContent> </SelectContent>
</Select> </Select>
<template v-if="draft.verification === 'hmac'"> <template v-if="draft.verification === 'hmac'">