This commit is contained in:
14 files changed
+139
-110
No files matched your search
+6
-6
@@ -948,7 +948,7 @@ components:
|
|||||||
source:
|
source:
|
||||||
type: string
|
type: string
|
||||||
token:
|
token:
|
||||||
description: Shown once, for a token source; only its hash is stored
|
description: Returned once for a token source; only its hash is stored
|
||||||
type: string
|
type: string
|
||||||
url:
|
url:
|
||||||
description: Where the sender posts its events
|
description: Where the sender posts its events
|
||||||
@@ -2084,18 +2084,18 @@ components:
|
|||||||
description: The connector that parses this source's events
|
description: The connector that parses this source's events
|
||||||
type: string
|
type: string
|
||||||
hmac_header:
|
hmac_header:
|
||||||
description: The header that carries the signature, for hmac
|
description: The header that carries the signature (hmac only)
|
||||||
type: string
|
type: string
|
||||||
hmac_prefix:
|
hmac_prefix:
|
||||||
description: Text before the hex digest in that header, such as sha256=, for hmac
|
description: Text before the hex digest in the signature header, such as sha256= (hmac only)
|
||||||
examples:
|
examples:
|
||||||
- sha256=
|
- sha256=
|
||||||
type: string
|
type: string
|
||||||
hmac_secret:
|
hmac_secret:
|
||||||
description: "The shared signing secret, for hmac. Write-only: it is sealed and never returned"
|
description: "The shared signing secret (hmac only). Write-only: it is sealed and never returned"
|
||||||
type: string
|
type: string
|
||||||
verification:
|
verification:
|
||||||
description: "How a delivery proves itself: a bearer token, an HMAC-SHA256 signature of the body, or nothing"
|
description: "How deliveries are verified: a bearer token, an HMAC-SHA256 signature of the body, or no verification"
|
||||||
enum:
|
enum:
|
||||||
- token
|
- token
|
||||||
- hmac
|
- hmac
|
||||||
@@ -5381,7 +5381,7 @@ paths:
|
|||||||
schema:
|
schema:
|
||||||
$ref: "#/components/schemas/ErrorModel"
|
$ref: "#/components/schemas/ErrorModel"
|
||||||
description: Error
|
description: Error
|
||||||
summary: Add a source, or change how it verifies deliveries and rotate its token
|
summary: Add or update a source, its verification scheme and its token
|
||||||
tags:
|
tags:
|
||||||
- Events
|
- Events
|
||||||
/api/sources/{name}/key-fields:
|
/api/sources/{name}/key-fields:
|
||||||
|
|||||||
@@ -60,8 +60,8 @@ type ConnectorServiceClient interface {
|
|||||||
// delivery is never offered to an agent: hard rules apply to it, guards
|
// delivery is never offered to an agent: hard rules apply to it, guards
|
||||||
// don't. The host retries a failed delivery, so a connector should make a
|
// don't. The host retries a failed delivery, so a connector should make a
|
||||||
// repeat with the same idempotency key harmless where its service allows.
|
// repeat with the same idempotency key harmless where its service allows.
|
||||||
// A connector answers InvalidArgument, FailedPrecondition, PermissionDenied
|
// A connector returns InvalidArgument, FailedPrecondition, PermissionDenied
|
||||||
// or NotFound when the service refuses the delivery for good: the host
|
// or NotFound when the service rejects the delivery permanently: the host
|
||||||
// records the reason and stops retrying. Any other status is retried.
|
// records the reason and stops retrying. Any other status is retried.
|
||||||
Deliver(ctx context.Context, in *DeliverRequest, opts ...grpc.CallOption) (*DeliverResponse, error)
|
Deliver(ctx context.Context, in *DeliverRequest, opts ...grpc.CallOption) (*DeliverResponse, error)
|
||||||
// Listen streams the events of a source the connector holds its own
|
// Listen streams the events of a source the connector holds its own
|
||||||
@@ -213,8 +213,8 @@ type ConnectorServiceServer interface {
|
|||||||
// delivery is never offered to an agent: hard rules apply to it, guards
|
// delivery is never offered to an agent: hard rules apply to it, guards
|
||||||
// don't. The host retries a failed delivery, so a connector should make a
|
// don't. The host retries a failed delivery, so a connector should make a
|
||||||
// repeat with the same idempotency key harmless where its service allows.
|
// repeat with the same idempotency key harmless where its service allows.
|
||||||
// A connector answers InvalidArgument, FailedPrecondition, PermissionDenied
|
// A connector returns InvalidArgument, FailedPrecondition, PermissionDenied
|
||||||
// or NotFound when the service refuses the delivery for good: the host
|
// or NotFound when the service rejects the delivery permanently: the host
|
||||||
// records the reason and stops retrying. Any other status is retried.
|
// records the reason and stops retrying. Any other status is retried.
|
||||||
Deliver(context.Context, *DeliverRequest) (*DeliverResponse, error)
|
Deliver(context.Context, *DeliverRequest) (*DeliverResponse, error)
|
||||||
// Listen streams the events of a source the connector holds its own
|
// Listen streams the events of a source the connector holds its own
|
||||||
|
|||||||
@@ -1,7 +1,7 @@
|
|||||||
-- +goose Up
|
-- +goose Up
|
||||||
-- How a source's deliveries are verified. A token source checks the bearer
|
-- How a source's deliveries are verified. A token source checks the bearer
|
||||||
-- token and an hmac source checks a signature over the raw body with a shared
|
-- token. An hmac source checks a signature over the raw body using a shared
|
||||||
-- secret, which is kept sealed. A source that chose none checks nothing.
|
-- secret, which is stored sealed. A source set to none performs no check.
|
||||||
ALTER TABLE events.sources
|
ALTER TABLE events.sources
|
||||||
ADD COLUMN verification text NOT NULL DEFAULT 'token'
|
ADD COLUMN verification text NOT NULL DEFAULT 'token'
|
||||||
CHECK (verification IN ('token', 'hmac', 'none')),
|
CHECK (verification IN ('token', 'hmac', 'none')),
|
||||||
|
|||||||
@@ -1,6 +1,7 @@
|
|||||||
-- +goose Up
|
-- +goose Up
|
||||||
-- A delivery the channel refuses for good, or that keeps failing past the retry
|
-- A delivery that the channel rejects permanently, or that still fails at the
|
||||||
-- limit, is failed rather than retried: failed_at stops the worker claiming it.
|
-- retry limit, is marked failed instead of retried. The worker does not claim
|
||||||
|
-- rows with failed_at set.
|
||||||
ALTER TABLE deliveries.messages ADD COLUMN failed_at timestamptz;
|
ALTER TABLE deliveries.messages ADD COLUMN failed_at timestamptz;
|
||||||
DROP INDEX deliveries.messages_due;
|
DROP INDEX deliveries.messages_due;
|
||||||
CREATE INDEX messages_due ON deliveries.messages (next_attempt_at) WHERE sent_at IS NULL AND failed_at IS NULL;
|
CREATE INDEX messages_due ON deliveries.messages (next_attempt_at) WHERE sent_at IS NULL AND failed_at IS NULL;
|
||||||
|
|||||||
@@ -31,10 +31,11 @@ var (
|
|||||||
// ErrUnknownName names an action, lookup or event type the connector
|
// ErrUnknownName names an action, lookup or event type the connector
|
||||||
// never declared.
|
// never declared.
|
||||||
ErrUnknownName = errors.New("not declared by the connector")
|
ErrUnknownName = errors.New("not declared by the connector")
|
||||||
// ErrRejected means a delivery was refused in a way a retry can't change:
|
// ErrRejected means a delivery was rejected permanently, so retrying it
|
||||||
// the connector answered InvalidArgument, FailedPrecondition,
|
// cannot succeed: the connector returned InvalidArgument,
|
||||||
// PermissionDenied or NotFound, or the input broke the declared schema.
|
// FailedPrecondition, PermissionDenied or NotFound, or the input did not
|
||||||
ErrRejected = errors.New("delivery rejected for good")
|
// match the declared schema.
|
||||||
|
ErrRejected = errors.New("delivery permanently rejected")
|
||||||
)
|
)
|
||||||
|
|
||||||
// Reversibility says whether and how an action can be undone.
|
// Reversibility says whether and how an action can be undone.
|
||||||
@@ -249,8 +250,8 @@ func (c *Connector) Deliver(ctx context.Context, name string, input jsontext.Val
|
|||||||
return resp.GetExternalId(), nil
|
return resp.GetExternalId(), nil
|
||||||
}
|
}
|
||||||
|
|
||||||
// rejectedForGood reports whether a connector's error says the same delivery
|
// rejectedForGood reports whether a connector's error status means that
|
||||||
// can never succeed.
|
// retrying the same delivery cannot succeed.
|
||||||
func rejectedForGood(err error) bool {
|
func rejectedForGood(err error) bool {
|
||||||
switch status.Code(err) {
|
switch status.Code(err) {
|
||||||
case codes.InvalidArgument, codes.FailedPrecondition, codes.PermissionDenied, codes.NotFound:
|
case codes.InvalidArgument, codes.FailedPrecondition, codes.PermissionDenied, codes.NotFound:
|
||||||
|
|||||||
@@ -4,7 +4,8 @@
|
|||||||
// dashboard, is the first channel: its messages are rows a client reads from a
|
// dashboard, is the first channel: its messages are rows a client reads from a
|
||||||
// cursor, with a live stream that only says when to read. Every accepted
|
// cursor, with a live stream that only says when to read. Every accepted
|
||||||
// connector that delivers messages is another: each gets its own row, retried
|
// connector that delivers messages is another: each gets its own row, retried
|
||||||
// until the connector confirms it, refuses it for good, or the attempts run out.
|
// until the connector confirms it, rejects it permanently, or the retry limit
|
||||||
|
// is reached.
|
||||||
package deliveries
|
package deliveries
|
||||||
|
|
||||||
import (
|
import (
|
||||||
@@ -228,8 +229,8 @@ func (m *Module) deliver(ctx context.Context, msg store.Message) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
// recordFailure stores why an attempt failed, and ends the delivery when the
|
// recordFailure records why an attempt failed. It marks the delivery failed
|
||||||
// channel refused it for good or the attempts are spent.
|
// when the channel rejected it permanently or the retry limit is reached.
|
||||||
func (m *Module) recordFailure(ctx context.Context, msg store.Message, cause error, took time.Duration) {
|
func (m *Module) recordFailure(ctx context.Context, msg store.Message, cause error, took time.Duration) {
|
||||||
record := m.store.Failed
|
record := m.store.Failed
|
||||||
reason := cause.Error()
|
reason := cause.Error()
|
||||||
|
|||||||
@@ -171,8 +171,8 @@ func (s *Store) Failed(ctx context.Context, seq int64, attempt int, reason strin
|
|||||||
`UPDATE deliveries.messages SET last_error = $2 WHERE seq = $1`, reason)
|
`UPDATE deliveries.messages SET last_error = $2 WHERE seq = $1`, reason)
|
||||||
}
|
}
|
||||||
|
|
||||||
// GaveUp records the attempt that ended a delivery for good: the message is
|
// GaveUp records the final attempt of a delivery and marks the message failed,
|
||||||
// marked failed and no worker claims it again.
|
// so no worker claims it again.
|
||||||
func (s *Store) GaveUp(ctx context.Context, seq int64, attempt int, reason string, took time.Duration) error {
|
func (s *Store) GaveUp(ctx context.Context, seq int64, attempt int, reason string, took time.Duration) error {
|
||||||
return s.attempt(ctx, seq, attempt, attemptOutcome{failure: reason, took: took},
|
return s.attempt(ctx, seq, attempt, attemptOutcome{failure: reason, took: took},
|
||||||
`UPDATE deliveries.messages SET failed_at = now(), last_error = $2 WHERE seq = $1`, reason)
|
`UPDATE deliveries.messages SET failed_at = now(), last_error = $2 WHERE seq = $1`, reason)
|
||||||
|
|||||||
+83
-58
@@ -29,7 +29,7 @@ func (m *Module) Register(api huma.API) {
|
|||||||
}, m.sources)
|
}, m.sources)
|
||||||
huma.Register(api, huma.Operation{
|
huma.Register(api, huma.Operation{
|
||||||
OperationID: "put-source", Method: http.MethodPut, Path: "/api/sources/{name}",
|
OperationID: "put-source", Method: http.MethodPut, Path: "/api/sources/{name}",
|
||||||
Summary: "Add a source, or change how it verifies deliveries and rotate its token", Tags: tags,
|
Summary: "Add or update a source, its verification scheme and its token", Tags: tags,
|
||||||
}, m.putSource)
|
}, m.putSource)
|
||||||
huma.Register(api, huma.Operation{
|
huma.Register(api, huma.Operation{
|
||||||
OperationID: "put-source-key-fields", Method: http.MethodPut, Path: "/api/sources/{name}/key-fields",
|
OperationID: "put-source-key-fields", Method: http.MethodPut, Path: "/api/sources/{name}/key-fields",
|
||||||
@@ -61,25 +61,26 @@ type putSourceInput struct {
|
|||||||
Name string `path:"name" doc:"Lowercase letters, digits and hyphens; the webhook URL ends in it"`
|
Name string `path:"name" doc:"Lowercase letters, digits and hyphens; the webhook URL ends in it"`
|
||||||
Body struct {
|
Body struct {
|
||||||
Connector string `json:"connector" doc:"The connector that parses this source's events"`
|
Connector string `json:"connector" doc:"The connector that parses this source's events"`
|
||||||
// Verification defaults to a bearer token so a source that names none
|
// A new source without a scheme uses a bearer token, so existing
|
||||||
// keeps working.
|
// senders keep working.
|
||||||
Verification string `json:"verification,omitzero" enum:"token,hmac,none" doc:"How a delivery proves itself: a bearer token, an HMAC-SHA256 signature of the body, or nothing"`
|
Verification string `json:"verification,omitzero" enum:"token,hmac,none" doc:"How deliveries are verified: a bearer token, an HMAC-SHA256 signature of the body, or no verification"`
|
||||||
HMACHeader string `json:"hmac_header,omitzero" doc:"The header that carries the signature, for hmac"`
|
HMACHeader string `json:"hmac_header,omitzero" doc:"The header that carries the signature (hmac only)"`
|
||||||
HMACPrefix string `json:"hmac_prefix,omitzero" doc:"Text before the hex digest in that header, such as sha256=, for hmac" example:"sha256="`
|
HMACPrefix string `json:"hmac_prefix,omitzero" doc:"Text before the hex digest in the signature header, such as sha256= (hmac only)" example:"sha256="`
|
||||||
HMACSecret string `json:"hmac_secret,omitzero" doc:"The shared signing secret, for hmac. Write-only: it is sealed and never returned"`
|
HMACSecret string `json:"hmac_secret,omitzero" doc:"The shared signing secret (hmac only). Write-only: it is sealed and never returned"`
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
type sourceCreated struct {
|
type sourceCreated struct {
|
||||||
Source string `json:"source"`
|
Source string `json:"source"`
|
||||||
Token string `json:"token,omitzero" doc:"Shown once, for a token source; only its hash is stored"`
|
Token string `json:"token,omitzero" doc:"Returned once for a token source; only its hash is stored"`
|
||||||
URL string `json:"url" doc:"Where the sender posts its events"`
|
URL string `json:"url" doc:"Where the sender posts its events"`
|
||||||
}
|
}
|
||||||
|
|
||||||
// putSource creates a source or changes it. A save that names no verification
|
// putSource creates or updates a source. A save that omits verification keeps
|
||||||
// keeps the stored scheme and its credentials. A token is returned once, when
|
// the stored scheme and its credentials. A token is returned only when the
|
||||||
// a token scheme is chosen, and stored only as its hash. An hmac secret is
|
// token scheme is chosen explicitly or the source is new, and only its hash is
|
||||||
// sealed and never returned; leaving it blank keeps the sealed one.
|
// stored. An HMAC secret is sealed and never returned; a blank secret keeps the
|
||||||
|
// sealed one.
|
||||||
func (m *Module) putSource(ctx context.Context, in *putSourceInput) (*httpapi.Reply[sourceCreated], error) {
|
func (m *Module) putSource(ctx context.Context, in *putSourceInput) (*httpapi.Reply[sourceCreated], error) {
|
||||||
if !validName.MatchString(in.Name) || !validName.MatchString(in.Body.Connector) {
|
if !validName.MatchString(in.Name) || !validName.MatchString(in.Body.Connector) {
|
||||||
return nil, huma.Error400BadRequest("a source needs a lowercase name and a connector")
|
return nil, huma.Error400BadRequest("a source needs a lowercase name and a connector")
|
||||||
@@ -88,55 +89,77 @@ func (m *Module) putSource(ctx context.Context, in *putSourceInput) (*httpapi.Re
|
|||||||
if err != nil && !errors.Is(err, store.ErrNotFound) {
|
if err != nil && !errors.Is(err, store.ErrNotFound) {
|
||||||
return nil, huma.Error500InternalServerError("couldn't read the source", err)
|
return nil, huma.Error500InternalServerError("couldn't read the source", err)
|
||||||
}
|
}
|
||||||
found := err == nil
|
var stored *store.Source
|
||||||
created := sourceCreated{Source: in.Name, URL: "/hooks/" + in.Name}
|
if err == nil {
|
||||||
src := store.Source{OrgID: m.orgID, Name: in.Name, Connector: in.Body.Connector, Verification: in.Body.Verification}
|
stored = &existing
|
||||||
switch {
|
|
||||||
case src.Verification == "" && found:
|
|
||||||
// Naming no scheme keeps the stored one with its token or secret, so
|
|
||||||
// a save can't downgrade a source.
|
|
||||||
src = existing
|
|
||||||
src.Connector = in.Body.Connector
|
|
||||||
case src.Verification == "":
|
|
||||||
src.Verification = store.Token
|
|
||||||
}
|
}
|
||||||
if src.Verification == store.Token && (!found || in.Body.Verification != "") {
|
src, token, err := m.sourceToSave(in, stored)
|
||||||
token, err := newToken()
|
if err != nil {
|
||||||
if err != nil {
|
return nil, err
|
||||||
return nil, huma.Error500InternalServerError("couldn't make a token", err)
|
|
||||||
}
|
|
||||||
src.TokenSHA256, created.Token = httpjson.TokenHash(token), token
|
|
||||||
}
|
|
||||||
switch src.Verification {
|
|
||||||
case store.Token, store.None:
|
|
||||||
if in.Body.Verification != "" && src.Verification == store.None {
|
|
||||||
src.TokenSHA256 = ""
|
|
||||||
}
|
|
||||||
case store.HMAC:
|
|
||||||
if in.Body.Verification != "" {
|
|
||||||
src.HMACHeader, src.HMACPrefix, src.TokenSHA256 = in.Body.HMACHeader, in.Body.HMACPrefix, ""
|
|
||||||
switch {
|
|
||||||
case in.Body.HMACSecret != "":
|
|
||||||
sealed, err := m.sealer.Seal(secretLabel(m.orgID, in.Name), []byte(in.Body.HMACSecret))
|
|
||||||
if err != nil {
|
|
||||||
return nil, huma.Error500InternalServerError("couldn't seal the secret", err)
|
|
||||||
}
|
|
||||||
src.HMACSecretSealed = sealed
|
|
||||||
case found && existing.Verification == store.HMAC:
|
|
||||||
// A blank secret keeps the sealed one.
|
|
||||||
src.HMACSecretSealed = existing.HMACSecretSealed
|
|
||||||
}
|
|
||||||
}
|
|
||||||
if src.HMACHeader == "" || len(src.HMACSecretSealed) == 0 {
|
|
||||||
return nil, huma.Error400BadRequest("an hmac source needs the signature header and a secret")
|
|
||||||
}
|
|
||||||
default:
|
|
||||||
return nil, huma.Error400BadRequest("verification is token, hmac or none")
|
|
||||||
}
|
}
|
||||||
if err := m.store.UpsertSource(ctx, src); err != nil {
|
if err := m.store.UpsertSource(ctx, src); err != nil {
|
||||||
return nil, huma.Error500InternalServerError("couldn't save the source", err)
|
return nil, huma.Error500InternalServerError("couldn't save the source", err)
|
||||||
}
|
}
|
||||||
return httpapi.OK(created), nil
|
return httpapi.OK(sourceCreated{Source: in.Name, Token: token, URL: "/hooks/" + in.Name}), nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// sourceToSave builds the row to store from the request and the stored source,
|
||||||
|
// if there is one. It returns the new token when one was issued.
|
||||||
|
func (m *Module) sourceToSave(in *putSourceInput, stored *store.Source) (store.Source, string, error) {
|
||||||
|
explicit := in.Body.Verification != ""
|
||||||
|
src := store.Source{OrgID: m.orgID, Name: in.Name, Connector: in.Body.Connector, Verification: in.Body.Verification}
|
||||||
|
switch {
|
||||||
|
case !explicit && stored != nil:
|
||||||
|
// Omitting the scheme keeps the stored one with its token or secret, so
|
||||||
|
// a save cannot downgrade a source.
|
||||||
|
src = *stored
|
||||||
|
src.Connector = in.Body.Connector
|
||||||
|
case !explicit:
|
||||||
|
src.Verification = store.Token
|
||||||
|
}
|
||||||
|
switch src.Verification {
|
||||||
|
case store.Token:
|
||||||
|
if stored != nil && !explicit {
|
||||||
|
return src, "", nil
|
||||||
|
}
|
||||||
|
token, err := newToken()
|
||||||
|
if err != nil {
|
||||||
|
return src, "", huma.Error500InternalServerError("couldn't generate a token", err)
|
||||||
|
}
|
||||||
|
src.TokenSHA256 = httpjson.TokenHash(token)
|
||||||
|
return src, token, nil
|
||||||
|
case store.None:
|
||||||
|
return src, "", nil
|
||||||
|
case store.HMAC:
|
||||||
|
if explicit {
|
||||||
|
if err := m.setHMAC(&src, in, stored); err != nil {
|
||||||
|
return src, "", err
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if src.HMACHeader == "" || len(src.HMACSecretSealed) == 0 {
|
||||||
|
return src, "", huma.Error400BadRequest("an HMAC source needs a signature header and a secret")
|
||||||
|
}
|
||||||
|
return src, "", nil
|
||||||
|
default:
|
||||||
|
return src, "", huma.Error400BadRequest("verification must be token, hmac or none")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// setHMAC applies the requested HMAC settings. A blank secret keeps the sealed
|
||||||
|
// secret of a source that already uses HMAC.
|
||||||
|
func (m *Module) setHMAC(src *store.Source, in *putSourceInput, stored *store.Source) error {
|
||||||
|
src.HMACHeader, src.HMACPrefix = in.Body.HMACHeader, in.Body.HMACPrefix
|
||||||
|
switch {
|
||||||
|
case in.Body.HMACSecret != "":
|
||||||
|
sealed, err := m.sealer.Seal(secretLabel(m.orgID, in.Name), []byte(in.Body.HMACSecret))
|
||||||
|
if err != nil {
|
||||||
|
return huma.Error500InternalServerError("couldn't seal the secret", err)
|
||||||
|
}
|
||||||
|
src.HMACSecretSealed = sealed
|
||||||
|
case stored != nil && stored.Verification == store.HMAC:
|
||||||
|
src.HMACSecretSealed = stored.HMACSecretSealed
|
||||||
|
}
|
||||||
|
return nil
|
||||||
}
|
}
|
||||||
|
|
||||||
type putTriggerInput struct {
|
type putTriggerInput struct {
|
||||||
@@ -279,8 +302,10 @@ func (m *Module) sources(ctx context.Context, _ *struct{}) (*httpapi.Reply[[]sou
|
|||||||
}
|
}
|
||||||
out := make([]sourceView, 0, len(sources))
|
out := make([]sourceView, 0, len(sources))
|
||||||
for _, src := range sources {
|
for _, src := range sources {
|
||||||
view := sourceView{Name: src.Name, Connector: src.Connector, TokenSet: src.TokenSHA256 != "", KeyFields: []string{},
|
view := sourceView{
|
||||||
Verification: src.Verification, HMACHeader: src.HMACHeader, HMACPrefix: src.HMACPrefix, HMACSecretSet: len(src.HMACSecretSealed) > 0}
|
Name: src.Name, Connector: src.Connector, TokenSet: src.TokenSHA256 != "", KeyFields: []string{},
|
||||||
|
Verification: src.Verification, HMACHeader: src.HMACHeader, HMACPrefix: src.HMACPrefix, HMACSecretSet: len(src.HMACSecretSealed) > 0,
|
||||||
|
}
|
||||||
if len(src.KeyFields) > 0 {
|
if len(src.KeyFields) > 0 {
|
||||||
if err := json.Unmarshal(src.KeyFields, &view.KeyFields); err != nil {
|
if err := json.Unmarshal(src.KeyFields, &view.KeyFields); err != nil {
|
||||||
m.log.ErrorContext(ctx, "read source key fields", "source", src.Name, "error", err)
|
m.log.ErrorContext(ctx, "read source key fields", "source", src.Name, "error", err)
|
||||||
|
|||||||
+11
-10
@@ -87,7 +87,7 @@ func NewModule(db *sqlx.DB, log *slog.Logger, orgID string, sealer Sealer, c Con
|
|||||||
return m
|
return m
|
||||||
}
|
}
|
||||||
|
|
||||||
// sourceReader is the one read intake makes of the store.
|
// sourceReader is the store method that intake uses to look up a source.
|
||||||
type sourceReader interface {
|
type sourceReader interface {
|
||||||
Source(ctx context.Context, orgID, name string) (store.Source, error)
|
Source(ctx context.Context, orgID, name string) (store.Source, error)
|
||||||
}
|
}
|
||||||
@@ -98,9 +98,9 @@ func (m *Module) Routes(mux *http.ServeMux) {
|
|||||||
mux.HandleFunc("POST /hooks/{source}", m.intake)
|
mux.HandleFunc("POST /hooks/{source}", m.intake)
|
||||||
}
|
}
|
||||||
|
|
||||||
// intake has the source's connector parse a verified delivery and stores each
|
// intake verifies a delivery, parses it with the source's connector and stores
|
||||||
// event before answering. A re-delivered event is a constraint conflict,
|
// each event before responding. A redelivered event violates a uniqueness
|
||||||
// counted as a duplicate.
|
// constraint and is counted as a duplicate.
|
||||||
func (m *Module) intake(w http.ResponseWriter, r *http.Request) {
|
func (m *Module) intake(w http.ResponseWriter, r *http.Request) {
|
||||||
src, parsed, body, status, err := m.accept(r)
|
src, parsed, body, status, err := m.accept(r)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
@@ -115,10 +115,11 @@ func (m *Module) intake(w http.ResponseWriter, r *http.Request) {
|
|||||||
httpjson.Write(w, http.StatusAccepted, map[string]int{"stored": stored, "duplicates": duplicates})
|
httpjson.Write(w, http.StatusAccepted, map[string]int{"stored": stored, "duplicates": duplicates})
|
||||||
}
|
}
|
||||||
|
|
||||||
// accept verifies a delivery against its source's scheme, then has the
|
// accept verifies a delivery against its source's scheme, then passes it to
|
||||||
// connector parse it. A delivery that fails verification is refused with 401
|
// the connector for parsing. A delivery that fails verification is rejected
|
||||||
// before the connector sees its body, and the refusal is logged. An unknown
|
// with 401 before the connector sees its body, and the rejection is logged.
|
||||||
// source and a failed delivery answer alike, so neither is probed.
|
// An unknown source and a failed verification get the same response, so a
|
||||||
|
// caller cannot use it to discover source names.
|
||||||
func (m *Module) accept(r *http.Request) (src store.Source, parsed []connectors.Event, body []byte, status int, err error) {
|
func (m *Module) accept(r *http.Request) (src store.Source, parsed []connectors.Event, body []byte, status int, err error) {
|
||||||
ctx := r.Context()
|
ctx := r.Context()
|
||||||
src, err = m.reader.Source(ctx, m.orgID, r.PathValue("source"))
|
src, err = m.reader.Source(ctx, m.orgID, r.PathValue("source"))
|
||||||
@@ -163,8 +164,8 @@ var (
|
|||||||
errConnectorDown = errors.New("connector isn't running")
|
errConnectorDown = errors.New("connector isn't running")
|
||||||
)
|
)
|
||||||
|
|
||||||
// refuse logs why a delivery was turned away and picks its status: 401 when it
|
// refuse logs why a delivery was rejected and chooses the status: 401 when
|
||||||
// failed to prove itself, 503 when verification couldn't run.
|
// verification failed, 503 when verification could not run.
|
||||||
func (m *Module) refuse(ctx context.Context, src store.Source, err error) (store.Source, []connectors.Event, []byte, int, error) {
|
func (m *Module) refuse(ctx context.Context, src store.Source, err error) (store.Source, []connectors.Event, []byte, int, error) {
|
||||||
m.log.WarnContext(ctx, "refused webhook delivery", "source", src.Name, "verification", src.Verification, "reason", err.Error())
|
m.log.WarnContext(ctx, "refused webhook delivery", "source", src.Name, "verification", src.Verification, "reason", err.Error())
|
||||||
if errors.Is(err, errUnverified) {
|
if errors.Is(err, errUnverified) {
|
||||||
|
|||||||
@@ -18,7 +18,7 @@ import (
|
|||||||
// ErrNotFound means the named row doesn't exist.
|
// ErrNotFound means the named row doesn't exist.
|
||||||
var ErrNotFound = errors.New("not found")
|
var ErrNotFound = errors.New("not found")
|
||||||
|
|
||||||
// The ways a source verifies a delivery.
|
// Verification schemes for a source's deliveries.
|
||||||
const (
|
const (
|
||||||
Token = "token"
|
Token = "token"
|
||||||
HMAC = "hmac"
|
HMAC = "hmac"
|
||||||
@@ -31,10 +31,11 @@ type Source struct {
|
|||||||
Name string `db:"name"`
|
Name string `db:"name"`
|
||||||
Connector string `db:"connector"`
|
Connector string `db:"connector"`
|
||||||
TokenSHA256 string `db:"token_sha256"`
|
TokenSHA256 string `db:"token_sha256"`
|
||||||
// Verification is how a delivery proves itself: Token, HMAC or None.
|
// Verification is the scheme that deliveries must pass: Token, HMAC or None.
|
||||||
Verification string `db:"verification"`
|
Verification string `db:"verification"`
|
||||||
// HMACHeader carries the signature, HMACPrefix precedes its hex digest,
|
// HMACHeader names the header that carries the signature, HMACPrefix is
|
||||||
// and HMACSecretSealed is the shared secret, sealed.
|
// the text before its hex digest, and HMACSecretSealed is the sealed
|
||||||
|
// shared secret.
|
||||||
HMACHeader string `db:"hmac_header"`
|
HMACHeader string `db:"hmac_header"`
|
||||||
HMACPrefix string `db:"hmac_prefix"`
|
HMACPrefix string `db:"hmac_prefix"`
|
||||||
HMACSecretSealed []byte `db:"hmac_secret_sealed"`
|
HMACSecretSealed []byte `db:"hmac_secret_sealed"`
|
||||||
|
|||||||
@@ -26,8 +26,8 @@ func secretLabel(orgID, name string) string {
|
|||||||
return "events.sources/" + orgID + "/" + name + "/hmac"
|
return "events.sources/" + orgID + "/" + name + "/hmac"
|
||||||
}
|
}
|
||||||
|
|
||||||
// errUnverified means a delivery didn't prove itself. Its text is safe to
|
// errUnverified means a delivery failed verification. Its text is safe to log:
|
||||||
// log: it names the scheme, never a secret or a signature.
|
// it names the scheme, never a secret or a signature.
|
||||||
var errUnverified = errors.New("delivery not verified")
|
var errUnverified = errors.New("delivery not verified")
|
||||||
|
|
||||||
// checkToken verifies a token source before its body is read.
|
// checkToken verifies a token source before its body is read.
|
||||||
@@ -39,9 +39,9 @@ func checkToken(src store.Source, r *http.Request) error {
|
|||||||
}
|
}
|
||||||
|
|
||||||
// verifyHMAC checks the signature header against an HMAC-SHA256 of the raw
|
// verifyHMAC checks the signature header against an HMAC-SHA256 of the raw
|
||||||
// body, which the connector hasn't seen yet. The header holds the hex digest
|
// body, before the connector sees it. The header holds the hex digest, in
|
||||||
// after the source's prefix, in either case. An absent or malformed
|
// either case, after the source's prefix. A missing or malformed signature
|
||||||
// signature never verifies.
|
// always fails.
|
||||||
//
|
//
|
||||||
// Ported from internal/webhookguard/hmac.go (digest comparison) and
|
// Ported from internal/webhookguard/hmac.go (digest comparison) and
|
||||||
// internal/infrastructure/captureintake/receiver.go (header-borne signature),
|
// internal/infrastructure/captureintake/receiver.go (header-borne signature),
|
||||||
|
|||||||
@@ -5,7 +5,6 @@ import (
|
|||||||
"crypto/hmac"
|
"crypto/hmac"
|
||||||
"crypto/sha256"
|
"crypto/sha256"
|
||||||
"encoding/hex"
|
"encoding/hex"
|
||||||
"io"
|
|
||||||
"log/slog"
|
"log/slog"
|
||||||
"net/http"
|
"net/http"
|
||||||
"net/http/httptest"
|
"net/http/httptest"
|
||||||
@@ -73,13 +72,13 @@ func TestHMACSourceVerifiesBeforeParsing(t *testing.T) {
|
|||||||
parsed := false
|
parsed := false
|
||||||
m := &Module{
|
m := &Module{
|
||||||
orgID: "org", reader: fakeSources{src}, sealer: sealer,
|
orgID: "org", reader: fakeSources{src}, sealer: sealer,
|
||||||
log: slog.New(slog.NewTextHandler(io.Discard, nil)),
|
log: slog.New(slog.DiscardHandler),
|
||||||
parse: func(context.Context, string, []byte, map[string]string) ([]connectors.Event, error) {
|
parse: func(context.Context, string, []byte, map[string]string) ([]connectors.Event, error) {
|
||||||
parsed = true
|
parsed = true
|
||||||
return nil, nil
|
return nil, nil
|
||||||
},
|
},
|
||||||
}
|
}
|
||||||
req := httptest.NewRequest(http.MethodPost, "/hooks/hook", strings.NewReader(body))
|
req := httptest.NewRequestWithContext(t.Context(), http.MethodPost, "/hooks/hook", strings.NewReader(body))
|
||||||
req.SetPathValue("source", "hook")
|
req.SetPathValue("source", "hook")
|
||||||
if tt.header != "" {
|
if tt.header != "" {
|
||||||
req.Header.Set("X-Signature", tt.header)
|
req.Header.Set("X-Signature", tt.header)
|
||||||
|
|||||||
@@ -30,8 +30,8 @@ service ConnectorService {
|
|||||||
// delivery is never offered to an agent: hard rules apply to it, guards
|
// delivery is never offered to an agent: hard rules apply to it, guards
|
||||||
// don't. The host retries a failed delivery, so a connector should make a
|
// don't. The host retries a failed delivery, so a connector should make a
|
||||||
// repeat with the same idempotency key harmless where its service allows.
|
// repeat with the same idempotency key harmless where its service allows.
|
||||||
// A connector answers InvalidArgument, FailedPrecondition, PermissionDenied
|
// A connector returns InvalidArgument, FailedPrecondition, PermissionDenied
|
||||||
// or NotFound when the service refuses the delivery for good: the host
|
// or NotFound when the service rejects the delivery permanently: the host
|
||||||
// records the reason and stops retrying. Any other status is retried.
|
// records the reason and stops retrying. Any other status is retried.
|
||||||
rpc Deliver(DeliverRequest) returns (DeliverResponse);
|
rpc Deliver(DeliverRequest) returns (DeliverResponse);
|
||||||
// Listen streams the events of a source the connector holds its own
|
// Listen streams the events of a source the connector holds its own
|
||||||
|
|||||||
@@ -29,7 +29,7 @@ onMounted(() => {
|
|||||||
const columns: DataTableColumn<EventSource_>[] = [
|
const columns: DataTableColumn<EventSource_>[] = [
|
||||||
{ id: "name", label: "Source", fixed: true, sort: (s) => s.name },
|
{ id: "name", label: "Source", fixed: true, sort: (s) => s.name },
|
||||||
{ id: "connector", label: "Connector", sort: (s) => s.connector },
|
{ id: "connector", label: "Connector", sort: (s) => s.connector },
|
||||||
{ id: "verification", label: "Verified by" },
|
{ id: "verification", label: "Verification" },
|
||||||
{ id: "key", label: "Grouped by" },
|
{ id: "key", label: "Grouped by" },
|
||||||
];
|
];
|
||||||
|
|
||||||
@@ -131,7 +131,7 @@ const connectorNames = computed(() => (connectors.data.value ?? []).map((c) => c
|
|||||||
<DataTable :columns="columns" :rows="data" :row-key="(s) => s.name" storage-key="sources">
|
<DataTable :columns="columns" :rows="data" :row-key="(s) => s.name" storage-key="sources">
|
||||||
<template #cell-name="{ row }"><span class="font-medium">{{ row.name }}</span></template>
|
<template #cell-name="{ row }"><span class="font-medium">{{ row.name }}</span></template>
|
||||||
<template #cell-connector="{ row }">{{ row.connector }}</template>
|
<template #cell-connector="{ row }">{{ row.connector }}</template>
|
||||||
<template #cell-verification="{ row }">{{ row.verification === "hmac" ? `HMAC (${row.hmac_header})` : row.verification === "none" ? "Nothing" : "Bearer token" }}</template>
|
<template #cell-verification="{ row }">{{ row.verification === "hmac" ? `HMAC (${row.hmac_header})` : row.verification === "none" ? "None" : "Bearer token" }}</template>
|
||||||
<template #cell-key="{ row }"><span class="font-mono text-caption">{{ (row.key_fields ?? []).join(", ") || "Connector default" }}</span></template>
|
<template #cell-key="{ row }"><span class="font-mono text-caption">{{ (row.key_fields ?? []).join(", ") || "Connector default" }}</span></template>
|
||||||
<template #actions="{ row }">
|
<template #actions="{ row }">
|
||||||
<MoreMenu :label="`${row.name} actions`">
|
<MoreMenu :label="`${row.name} actions`">
|
||||||
@@ -160,7 +160,7 @@ const connectorNames = computed(() => (connectors.data.value ?? []).map((c) => c
|
|||||||
<SelectContent>
|
<SelectContent>
|
||||||
<SelectItem value="token">Bearer token</SelectItem>
|
<SelectItem value="token">Bearer token</SelectItem>
|
||||||
<SelectItem value="hmac">HMAC signature</SelectItem>
|
<SelectItem value="hmac">HMAC signature</SelectItem>
|
||||||
<SelectItem value="none">Nothing</SelectItem>
|
<SelectItem value="none">None</SelectItem>
|
||||||
</SelectContent>
|
</SelectContent>
|
||||||
</Select>
|
</Select>
|
||||||
<template v-if="draft.verification === 'hmac'">
|
<template v-if="draft.verification === 'hmac'">
|
||||||
|
|||||||
Reference in new issue
Block a user