diff --git a/internal/app/archie/operations.go b/internal/app/archie/operations.go index b443b15e3..38f30554d 100644 --- a/internal/app/archie/operations.go +++ b/internal/app/archie/operations.go @@ -93,7 +93,7 @@ func (o *operations) Start(ctx context.Context) error { o.settings.UseAuthorizer(checked) o.allow.UseAuthorizer(checked) o.tell.UseAccess(checked) - o.hear.UseDeliveryPolicy(access) + o.hear.UseDeliveryPolicy(governance.RecordingDelivery(access, o.settings, o.log.With("module", "governance"))) o.talk.Use(chat.Deps{ People: o.settings, Access: access, Settings: o.settings, Toolbox: o.conns, Judge: o.judge, Approver: o.allow, Memory: o.remember, Runs: o.engine, Notifier: o.tell, diff --git a/modules/events/events.go b/modules/events/events.go index 8efbee4c4..17251e9ea 100644 --- a/modules/events/events.go +++ b/modules/events/events.go @@ -144,7 +144,7 @@ func (m *Module) intake(w http.ResponseWriter, r *http.Request) { httpjson.Error(w, status, err.Error()) return } - stored, duplicates, err := m.save(r.Context(), d.src, d.events, d.body, d.arrival) + stored, duplicates, err := m.save(r.Context(), d.admission, d.events, d.body, d.arrival) if err != nil { httpjson.Error(w, http.StatusServiceUnavailable, err.Error()) return @@ -154,10 +154,12 @@ func (m *Module) intake(w http.ResponseWriter, r *http.Request) { // delivery is a webhook delivery that passed verification and was parsed. type delivery struct { - src store.Source - events []connectors.Event - body []byte - arrival arrival + src store.Source + // admission is set once the source's delivery policy allowed the sender. + admission admission + events []connectors.Event + body []byte + arrival arrival } // accept verifies a delivery against its source's scheme, then passes it to @@ -203,7 +205,9 @@ func (m *Module) accept(r *http.Request) (d delivery, status int, err error) { return m.refuse(ctx, d, err) } } - if status, err := m.deliveryAllowed(ctx, src, r); err != nil { + addr := m.clientAddr(ctx, r) + d.admission, status, err = m.admit(ctx, src, addr) + if err != nil { return d, status, err } d.events, err = m.parse(ctx, src.Connector, d.body, map[string]string{"Content-Type": r.Header.Get("Content-Type")}) @@ -213,7 +217,7 @@ func (m *Module) accept(r *http.Request) (d delivery, status int, err error) { if err != nil { return d, http.StatusBadRequest, err } - d.arrival.addr = senderAddr(r) + d.arrival.addr = addr d.arrival.headers = r.Header return d, 0, nil } @@ -238,30 +242,36 @@ var ( errDeliveryUnavailable = errors.New("delivery policy unavailable") ) -// deliveryAllowed applies the source's delivery policy to the sender's address. -// A verified delivery still needs it. When the policy chain can't decide the -// delivery is refused with 503, so the sender retries and nothing is stored. -func (m *Module) deliveryAllowed(ctx context.Context, src store.Source, r *http.Request) (int, error) { +// admission is a sender the source's delivery policy allowed. Only admit makes +// one and save requires one, so every way of accepting events asks the policy. +type admission struct { + src store.Source +} + +// admit applies the source's delivery policy to the sender's address, which is +// empty for a source that has no remote sender, such as a connector's listener +// stream. A verified delivery still needs it. When the policy chain can't +// decide, the status is 503 so a webhook sender retries and nothing is stored. +func (m *Module) admit(ctx context.Context, src store.Source, addr string) (admission, int, error) { if m.delivery == nil { - return http.StatusServiceUnavailable, errDeliveryUnavailable + return admission{}, http.StatusServiceUnavailable, errDeliveryUnavailable } - addr := senderAddr(r) // Delivery policies belong to the operating organisation, whatever the // source's own org id says. d := m.delivery.AuthorizeDelivery(ctx, governance.DefaultOrgID, src.Name, addr) switch { case d.Allowed: - return 0, nil + return admission{src: src}, 0, nil case errors.Is(d.Err, governance.ErrChainUnavailable): m.log.WarnContext(ctx, "delivery policy unavailable", "source", src.Name) - return http.StatusServiceUnavailable, errDeliveryUnavailable + return admission{}, http.StatusServiceUnavailable, errDeliveryUnavailable } - m.log.WarnContext(ctx, "refused webhook delivery", "source", src.Name, "reason", "delivery policy", "level", d.Level, "policies", d.Policies) - return http.StatusForbidden, errDeliveryDenied + m.log.WarnContext(ctx, "refused delivery", "source", src.Name, "reason", "delivery policy", "level", d.Level, "policies", d.Policies) + return admission{}, http.StatusForbidden, errDeliveryDenied } -// senderAddr is the address a request came from, without its port. -func senderAddr(r *http.Request) string { +// clientAddr is the address a request came from, without its port. +func (m *Module) clientAddr(_ context.Context, r *http.Request) string { addr, _, err := net.SplitHostPort(r.RemoteAddr) if err != nil { return r.RemoteAddr @@ -280,7 +290,8 @@ func (m *Module) parseWith(ctx context.Context, connector string, body []byte, h // save stores each event with the payload it arrived in, recording at arrival // whether it echoes a delivery Archie sent, so those never reach grouping. -func (m *Module) save(ctx context.Context, src store.Source, parsed []connectors.Event, body []byte, a arrival) (stored, duplicates int, err error) { +func (m *Module) save(ctx context.Context, adm admission, parsed []connectors.Event, body []byte, a arrival) (stored, duplicates int, err error) { + src := adm.src for i, e := range parsed { run, own, err := m.deliveries.SentBy(ctx, src.Connector, e.SourceEventID, e.LinkKey) if err != nil { @@ -365,10 +376,7 @@ func (m *Module) listen(ctx context.Context, src store.Source) { err = fmt.Errorf("connector %q isn't running", src.Connector) } else { err = conn.Listen(ctx, cursor, func(events []connectors.Event, next string) error { - if _, _, err := m.save(ctx, src, events, received(events), arrival{}); err != nil { - return err - } - if err := m.store.SetCursor(ctx, m.orgID, src.Name, next); err != nil { + if err := m.listenBatch(ctx, src, events, next); err != nil { return err } cursor = next @@ -386,6 +394,31 @@ func (m *Module) listen(ctx context.Context, src store.Source) { } } +// listenBatch stores one streamed batch, then moves the source's cursor past it. +func (m *Module) listenBatch(ctx context.Context, src store.Source, events []connectors.Event, next string) error { + if err := m.storeListened(ctx, src, events); err != nil { + return err + } + return m.store.SetCursor(ctx, m.orgID, src.Name, next) +} + +// storeListened stores a batch a listening connector streamed, once the +// source's delivery policy allows it. A listener has no remote sender, so the +// policy sees no address: rules on the source apply, and a rule that admits +// only some addresses admits no listener. A refused batch is dropped and +// not an error, since asking again would get the same answer. +func (m *Module) storeListened(ctx context.Context, src store.Source, events []connectors.Event) error { + adm, _, err := m.admit(ctx, src, "") + if errors.Is(err, errDeliveryDenied) { + return nil + } + if err != nil { + return err + } + _, _, err = m.save(ctx, adm, events, received(events), arrival{}) + return err +} + // answer settles the approval request an answer event names. An answer that // can't settle it is logged and done with: the request's own record already // says why. A channel account no account has linked is told how to link it. diff --git a/modules/events/listen_test.go b/modules/events/listen_test.go new file mode 100644 index 000000000..bda6e7c67 --- /dev/null +++ b/modules/events/listen_test.go @@ -0,0 +1,48 @@ +package events + +import ( + "log/slog" + "testing" + + "github.com/samcharles93/archie-core/modules/connectors" + "github.com/samcharles93/archie-core/modules/events/internal/store" + "github.com/samcharles93/archie-core/modules/governance" +) + +// TestListenedEventsPassTheDeliveryPolicyBeforeStoring guards connector +// listener intake: a streamed batch is checked against its source's delivery +// policy like a webhook delivery. A refusal stores nothing and doesn't fail +// the stream, so the cursor moves on, while an undecidable policy fails it so +// the batch is fetched again. +func TestListenedEventsPassTheDeliveryPolicyBeforeStoring(t *testing.T) { + tests := []struct { + name string + policy governance.Decision + unwired bool + wantErr bool + wantStored int + }{ + {name: "an allowed batch is stored", policy: governance.Allowed(), wantStored: 1}, + {name: "a denied batch is dropped without failing the stream", policy: governance.DeniedAt(governance.LevelObject, []string{"tg-deny"})}, + {name: "an undecidable policy fails the stream so the batch is fetched again", policy: governance.Unavailable(), wantErr: true}, + {name: "no policy wired fails the stream", unwired: true, wantErr: true}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + policy := &deliveryPolicy{decision: tt.policy} + inserter := &onceInserter{stored: map[string]bool{}} + m := &Module{orgID: "default", log: slog.New(slog.DiscardHandler), inserter: inserter, deliveries: noEcho{}} + if !tt.unwired { + m.UseDeliveryPolicy(policy) + } + src := store.Source{OrgID: "default", Name: "tg", Connector: "telegram"} + err := m.storeListened(t.Context(), src, []connectors.Event{{Type: "message", SourceEventID: "evt-1"}}) + if (err != nil) != tt.wantErr { + t.Fatalf("error = %v, want error %v", err, tt.wantErr) + } + if len(inserter.stored) != tt.wantStored { + t.Fatalf("events stored = %d, want %d", len(inserter.stored), tt.wantStored) + } + }) + } +} diff --git a/modules/governance/enforce.go b/modules/governance/enforce.go index 84b5dfa1e..ca4b2dc94 100644 --- a/modules/governance/enforce.go +++ b/modules/governance/enforce.go @@ -111,3 +111,31 @@ func (m *Module) mayUse(ctx context.Context, p Principal, action Action, kind Re } return chain.Authorize(ctx, p, action, Resource{Kind: kind, Org: p.Org}, AccessContext{Time: time.Now()}).Allowed } + +type recordingDelivery struct { + DeliveryAuthorizer + denials DenialRecorder + log *slog.Logger +} + +// RecordingDelivery wraps a so every refused delivery is stored as a denial, +// whichever way the event arrived. A decision that could not be made is not a +// denial and is not stored. +func RecordingDelivery(a DeliveryAuthorizer, denials DenialRecorder, log *slog.Logger) DeliveryAuthorizer { + return recordingDelivery{DeliveryAuthorizer: a, denials: denials, log: log} +} + +func (r recordingDelivery) AuthorizeDelivery(ctx context.Context, orgID OrgID, sourcePath, addr string) Decision { + d := r.DeliveryAuthorizer.AuthorizeDelivery(ctx, orgID, sourcePath, addr) + if d.Allowed || errors.Is(d.Err, ErrChainUnavailable) { + return d + } + err := r.denials.RecordDenial(context.WithoutCancel(ctx), Denial{ + Principal: SenderID, Org: orgID, Action: ActionDeliver, Kind: KindSource, ResourceID: sourcePath, + Level: d.Level, Policies: d.Policies, At: time.Now(), Count: 1, + }) + if err != nil { + r.log.ErrorContext(ctx, "record delivery denial", "source", sourcePath, "error", err) + } + return d +}