Compare commits

...
Author SHA1 Message Date
sam 955995fe2b fix(events): connector listener sources pass the delivery policy (#2106)
Every way of storing events now goes through admit, and save takes the
admission it returns. A refused listener batch is dropped and recorded as
a denial, like a refused webhook delivery.
2026-11-22 22:18:11 +11:00
4 changed files with 134 additions and 25 deletions

No files matched your search

+1 -1
View File
@@ -93,7 +93,7 @@ func (o *operations) Start(ctx context.Context) error {
o.settings.UseAuthorizer(checked)
o.allow.UseAuthorizer(checked)
o.tell.UseAccess(checked)
o.hear.UseDeliveryPolicy(access)
o.hear.UseDeliveryPolicy(governance.RecordingDelivery(access, o.settings, o.log.With("module", "governance")))
o.talk.Use(chat.Deps{
People: o.settings, Access: access, Settings: o.settings, Toolbox: o.conns, Judge: o.judge,
Approver: o.allow, Memory: o.remember, Runs: o.engine, Notifier: o.tell,
+57 -24
View File
@@ -144,7 +144,7 @@ func (m *Module) intake(w http.ResponseWriter, r *http.Request) {
httpjson.Error(w, status, err.Error())
return
}
stored, duplicates, err := m.save(r.Context(), d.src, d.events, d.body, d.arrival)
stored, duplicates, err := m.save(r.Context(), d.admission, d.events, d.body, d.arrival)
if err != nil {
httpjson.Error(w, http.StatusServiceUnavailable, err.Error())
return
@@ -154,10 +154,12 @@ func (m *Module) intake(w http.ResponseWriter, r *http.Request) {
// delivery is a webhook delivery that passed verification and was parsed.
type delivery struct {
src store.Source
events []connectors.Event
body []byte
arrival arrival
src store.Source
// admission is set once the source's delivery policy allowed the sender.
admission admission
events []connectors.Event
body []byte
arrival arrival
}
// accept verifies a delivery against its source's scheme, then passes it to
@@ -203,7 +205,9 @@ func (m *Module) accept(r *http.Request) (d delivery, status int, err error) {
return m.refuse(ctx, d, err)
}
}
if status, err := m.deliveryAllowed(ctx, src, r); err != nil {
addr := m.clientAddr(ctx, r)
d.admission, status, err = m.admit(ctx, src, addr)
if err != nil {
return d, status, err
}
d.events, err = m.parse(ctx, src.Connector, d.body, map[string]string{"Content-Type": r.Header.Get("Content-Type")})
@@ -213,7 +217,7 @@ func (m *Module) accept(r *http.Request) (d delivery, status int, err error) {
if err != nil {
return d, http.StatusBadRequest, err
}
d.arrival.addr = senderAddr(r)
d.arrival.addr = addr
d.arrival.headers = r.Header
return d, 0, nil
}
@@ -238,30 +242,36 @@ var (
errDeliveryUnavailable = errors.New("delivery policy unavailable")
)
// deliveryAllowed applies the source's delivery policy to the sender's address.
// A verified delivery still needs it. When the policy chain can't decide the
// delivery is refused with 503, so the sender retries and nothing is stored.
func (m *Module) deliveryAllowed(ctx context.Context, src store.Source, r *http.Request) (int, error) {
// admission is a sender the source's delivery policy allowed. Only admit makes
// one and save requires one, so every way of accepting events asks the policy.
type admission struct {
src store.Source
}
// admit applies the source's delivery policy to the sender's address, which is
// empty for a source that has no remote sender, such as a connector's listener
// stream. A verified delivery still needs it. When the policy chain can't
// decide, the status is 503 so a webhook sender retries and nothing is stored.
func (m *Module) admit(ctx context.Context, src store.Source, addr string) (admission, int, error) {
if m.delivery == nil {
return http.StatusServiceUnavailable, errDeliveryUnavailable
return admission{}, http.StatusServiceUnavailable, errDeliveryUnavailable
}
addr := senderAddr(r)
// Delivery policies belong to the operating organisation, whatever the
// source's own org id says.
d := m.delivery.AuthorizeDelivery(ctx, governance.DefaultOrgID, src.Name, addr)
switch {
case d.Allowed:
return 0, nil
return admission{src: src}, 0, nil
case errors.Is(d.Err, governance.ErrChainUnavailable):
m.log.WarnContext(ctx, "delivery policy unavailable", "source", src.Name)
return http.StatusServiceUnavailable, errDeliveryUnavailable
return admission{}, http.StatusServiceUnavailable, errDeliveryUnavailable
}
m.log.WarnContext(ctx, "refused webhook delivery", "source", src.Name, "reason", "delivery policy", "level", d.Level, "policies", d.Policies)
return http.StatusForbidden, errDeliveryDenied
m.log.WarnContext(ctx, "refused delivery", "source", src.Name, "reason", "delivery policy", "level", d.Level, "policies", d.Policies)
return admission{}, http.StatusForbidden, errDeliveryDenied
}
// senderAddr is the address a request came from, without its port.
func senderAddr(r *http.Request) string {
// clientAddr is the address a request came from, without its port.
func (m *Module) clientAddr(_ context.Context, r *http.Request) string {
addr, _, err := net.SplitHostPort(r.RemoteAddr)
if err != nil {
return r.RemoteAddr
@@ -280,7 +290,8 @@ func (m *Module) parseWith(ctx context.Context, connector string, body []byte, h
// save stores each event with the payload it arrived in, recording at arrival
// whether it echoes a delivery Archie sent, so those never reach grouping.
func (m *Module) save(ctx context.Context, src store.Source, parsed []connectors.Event, body []byte, a arrival) (stored, duplicates int, err error) {
func (m *Module) save(ctx context.Context, adm admission, parsed []connectors.Event, body []byte, a arrival) (stored, duplicates int, err error) {
src := adm.src
for i, e := range parsed {
run, own, err := m.deliveries.SentBy(ctx, src.Connector, e.SourceEventID, e.LinkKey)
if err != nil {
@@ -365,10 +376,7 @@ func (m *Module) listen(ctx context.Context, src store.Source) {
err = fmt.Errorf("connector %q isn't running", src.Connector)
} else {
err = conn.Listen(ctx, cursor, func(events []connectors.Event, next string) error {
if _, _, err := m.save(ctx, src, events, received(events), arrival{}); err != nil {
return err
}
if err := m.store.SetCursor(ctx, m.orgID, src.Name, next); err != nil {
if err := m.listenBatch(ctx, src, events, next); err != nil {
return err
}
cursor = next
@@ -386,6 +394,31 @@ func (m *Module) listen(ctx context.Context, src store.Source) {
}
}
// listenBatch stores one streamed batch, then moves the source's cursor past it.
func (m *Module) listenBatch(ctx context.Context, src store.Source, events []connectors.Event, next string) error {
if err := m.storeListened(ctx, src, events); err != nil {
return err
}
return m.store.SetCursor(ctx, m.orgID, src.Name, next)
}
// storeListened stores a batch a listening connector streamed, once the
// source's delivery policy allows it. A listener has no remote sender, so the
// policy sees no address: rules on the source apply, and a rule that admits
// only some addresses admits no listener. A refused batch is dropped and
// not an error, since asking again would get the same answer.
func (m *Module) storeListened(ctx context.Context, src store.Source, events []connectors.Event) error {
adm, _, err := m.admit(ctx, src, "")
if errors.Is(err, errDeliveryDenied) {
return nil
}
if err != nil {
return err
}
_, _, err = m.save(ctx, adm, events, received(events), arrival{})
return err
}
// answer settles the approval request an answer event names. An answer that
// can't settle it is logged and done with: the request's own record already
// says why. A channel account no account has linked is told how to link it.
+48
View File
@@ -0,0 +1,48 @@
package events
import (
"log/slog"
"testing"
"github.com/samcharles93/archie-core/modules/connectors"
"github.com/samcharles93/archie-core/modules/events/internal/store"
"github.com/samcharles93/archie-core/modules/governance"
)
// TestListenedEventsPassTheDeliveryPolicyBeforeStoring guards connector
// listener intake: a streamed batch is checked against its source's delivery
// policy like a webhook delivery. A refusal stores nothing and doesn't fail
// the stream, so the cursor moves on, while an undecidable policy fails it so
// the batch is fetched again.
func TestListenedEventsPassTheDeliveryPolicyBeforeStoring(t *testing.T) {
tests := []struct {
name string
policy governance.Decision
unwired bool
wantErr bool
wantStored int
}{
{name: "an allowed batch is stored", policy: governance.Allowed(), wantStored: 1},
{name: "a denied batch is dropped without failing the stream", policy: governance.DeniedAt(governance.LevelObject, []string{"tg-deny"})},
{name: "an undecidable policy fails the stream so the batch is fetched again", policy: governance.Unavailable(), wantErr: true},
{name: "no policy wired fails the stream", unwired: true, wantErr: true},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
policy := &deliveryPolicy{decision: tt.policy}
inserter := &onceInserter{stored: map[string]bool{}}
m := &Module{orgID: "default", log: slog.New(slog.DiscardHandler), inserter: inserter, deliveries: noEcho{}}
if !tt.unwired {
m.UseDeliveryPolicy(policy)
}
src := store.Source{OrgID: "default", Name: "tg", Connector: "telegram"}
err := m.storeListened(t.Context(), src, []connectors.Event{{Type: "message", SourceEventID: "evt-1"}})
if (err != nil) != tt.wantErr {
t.Fatalf("error = %v, want error %v", err, tt.wantErr)
}
if len(inserter.stored) != tt.wantStored {
t.Fatalf("events stored = %d, want %d", len(inserter.stored), tt.wantStored)
}
})
}
}
+28
View File
@@ -111,3 +111,31 @@ func (m *Module) mayUse(ctx context.Context, p Principal, action Action, kind Re
}
return chain.Authorize(ctx, p, action, Resource{Kind: kind, Org: p.Org}, AccessContext{Time: time.Now()}).Allowed
}
type recordingDelivery struct {
DeliveryAuthorizer
denials DenialRecorder
log *slog.Logger
}
// RecordingDelivery wraps a so every refused delivery is stored as a denial,
// whichever way the event arrived. A decision that could not be made is not a
// denial and is not stored.
func RecordingDelivery(a DeliveryAuthorizer, denials DenialRecorder, log *slog.Logger) DeliveryAuthorizer {
return recordingDelivery{DeliveryAuthorizer: a, denials: denials, log: log}
}
func (r recordingDelivery) AuthorizeDelivery(ctx context.Context, orgID OrgID, sourcePath, addr string) Decision {
d := r.DeliveryAuthorizer.AuthorizeDelivery(ctx, orgID, sourcePath, addr)
if d.Allowed || errors.Is(d.Err, ErrChainUnavailable) {
return d
}
err := r.denials.RecordDenial(context.WithoutCancel(ctx), Denial{
Principal: SenderID, Org: orgID, Action: ActionDeliver, Kind: KindSource, ResourceID: sourcePath,
Level: d.Level, Policies: d.Policies, At: time.Now(), Count: 1,
})
if err != nil {
r.log.ErrorContext(ctx, "record delivery denial", "source", sourcePath, "error", err)
}
return d
}