diff --git a/api/openapi.yaml b/api/openapi.yaml index 271667359..89fa3c73e 100644 --- a/api/openapi.yaml +++ b/api/openapi.yaml @@ -133,6 +133,14 @@ components: required: - public_url type: object + CaptureInputBody: + additionalProperties: false + properties: + capture: + type: boolean + required: + - capture + type: object ChangePasswordInputBody: additionalProperties: false properties: @@ -796,6 +804,22 @@ components: required: - text type: object + CreateEventTypeInputBody: + additionalProperties: false + properties: + example: + $ref: "#/components/schemas/ExampleStruct" + name: + type: string + rule: + $ref: "#/components/schemas/EventtypeRule" + description: The conditions that identify the type; inferred from the example when absent + source: + type: string + required: + - source + - name + type: object CreateTokenInputBody: additionalProperties: false properties: @@ -890,6 +914,15 @@ components: - body - created_at type: object + DeliveryHeaderInputBody: + additionalProperties: false + properties: + header: + description: A header name; empty stops de-duplicating retries by header + type: string + required: + - header + type: object DisabledInputBody: additionalProperties: false properties: @@ -1030,6 +1063,72 @@ components: format: uri type: string type: object + EventsCaptureView: + additionalProperties: false + properties: + body: + type: string + content_type: + type: string + event_type: + type: string + headers: + type: string + id: + type: string + received_at: + format: date-time + type: string + remote_addr: + type: string + source: + type: string + verification: + enum: + - signed + - token + - unsigned + - "" + type: string + required: + - id + - received_at + - source + - headers + - body + - verification + type: object + EventsCapturesBody: + additionalProperties: false + properties: + captures: + items: + $ref: "#/components/schemas/EventsCaptureView" + type: + - array + - "null" + enabled: + type: boolean + required: + - captures + - enabled + type: object + EventsDispatchView: + additionalProperties: false + properties: + created_at: + format: date-time + type: string + event_id: + type: string + run_id: + type: string + trigger: + type: string + required: + - event_id + - run_id + type: object EventsEventDetail: additionalProperties: false properties: @@ -1044,6 +1143,50 @@ components: - body - runs type: object + EventsEventTypeView: + additionalProperties: false + properties: + id: + type: string + name: + type: string + rule: + $ref: "#/components/schemas/EventtypeRule" + schema: + additionalProperties: + type: string + type: object + source: + type: string + required: + - id + - source + - name + - rule + - schema + type: object + EventsEventTypesBody: + additionalProperties: false + properties: + enabled: + type: boolean + event_types: + items: + $ref: "#/components/schemas/EventsEventTypeView" + type: + - array + - "null" + proposals: + items: + $ref: "#/components/schemas/EventtypeProposal" + type: + - array + - "null" + required: + - enabled + - event_types + - proposals + type: object EventsEventView: additionalProperties: false properties: @@ -1067,6 +1210,8 @@ components: type: string type: type: string + verification: + type: string required: - id - source @@ -1110,6 +1255,82 @@ components: required: - key_fields type: object + EventsMappingBody: + additionalProperties: false + properties: + event_type_id: + format: uuid + type: string + fields: + items: + $ref: "#/components/schemas/MappingField" + type: + - array + - "null" + name: + type: string + source_hint: + type: string + required: + - name + - event_type_id + - fields + type: object + EventsMappingView: + additionalProperties: false + properties: + created_at: + format: date-time + type: string + event_type_id: + type: string + fields: + items: + $ref: "#/components/schemas/MappingField" + type: + - array + - "null" + id: + type: string + last_matched_at: + format: date-time + type: string + match_count: + format: int64 + type: integer + name: + type: string + source_hint: + type: string + updated_at: + format: date-time + type: string + required: + - id + - name + - source_hint + - event_type_id + - fields + - match_count + - created_at + - updated_at + type: object + EventsPreviewBody: + additionalProperties: false + properties: + failures: + items: + $ref: "#/components/schemas/MappingFailure" + type: + - array + - "null" + values: + additionalProperties: {} + type: object + required: + - values + - failures + type: object EventsProposedTrigger: additionalProperties: false properties: @@ -1126,9 +1347,20 @@ components: - filter - workflow type: object + EventsSavedID: + additionalProperties: false + properties: + id: + type: string + required: + - id + type: object EventsSourceCreated: additionalProperties: false properties: + secret: + description: A signing secret generated for an HMAC source, returned once + type: string source: type: string token: @@ -1144,8 +1376,14 @@ components: EventsSourceView: additionalProperties: false properties: + capture: + type: boolean connector: type: string + delivery_header: + type: string + display_name: + type: string hmac_header: type: string hmac_prefix: @@ -1160,6 +1398,15 @@ components: - "null" name: type: string + secret: + type: string + signing: + enum: + - signed + - token + - unsigned_pending_approval + - unsigned + type: string token_set: type: boolean verification: @@ -1171,6 +1418,8 @@ components: - token_set - verification - hmac_secret_set + - signing + - capture type: object EventsTriggerRun: additionalProperties: false @@ -1198,16 +1447,28 @@ components: type: boolean event_type: type: string + event_type_id: + type: string every: type: string filter: additionalProperties: type: string type: object + mapping_id: + type: string name: type: string source: type: string + status: + enum: + - pending_approval + - armed + - paused + type: string + unsigned: + type: boolean workflow: type: string required: @@ -1217,6 +1478,92 @@ components: - filter - workflow - enabled + - status + - unsigned + type: object + EventtypeHeaderCondition: + additionalProperties: false + properties: + name: + type: string + value: + type: string + required: + - name + - value + type: object + EventtypePayloadCondition: + additionalProperties: false + properties: + op: + type: string + path: + type: string + value: + type: string + required: + - path + - op + type: object + EventtypeProposal: + additionalProperties: false + properties: + count: + format: int64 + type: integer + headers: + additionalProperties: + type: string + type: object + key: + type: string + rule: + $ref: "#/components/schemas/EventtypeRule" + sample_id: + type: string + schema: + additionalProperties: + type: string + type: object + source: + type: string + required: + - source + - key + - schema + - headers + - count + - sample_id + - rule + type: object + EventtypeRule: + additionalProperties: false + properties: + headers: + items: + $ref: "#/components/schemas/EventtypeHeaderCondition" + type: + - array + - "null" + payload: + items: + $ref: "#/components/schemas/EventtypePayloadCondition" + type: + - array + - "null" + required: + - headers + - payload + type: object + ExampleStruct: + additionalProperties: false + properties: + body: + type: string + headers: + additionalProperties: + type: string + type: object type: object GovernanceAuthStatusBody: additionalProperties: false @@ -2364,6 +2711,37 @@ components: - created_by - created_at type: object + MappingFailure: + additionalProperties: false + properties: + field_name: + type: string + path: + type: string + reason: + type: string + required: + - field_name + - path + - reason + type: object + MappingField: + additionalProperties: false + properties: + name: + type: string + path: + type: string + required: + type: boolean + type: + type: string + required: + - name + - path + - type + - required + type: object MemoryNote: additionalProperties: false properties: @@ -2457,6 +2835,22 @@ components: required: - body type: object + PreviewInputBody: + additionalProperties: false + properties: + capture_id: + format: uuid + type: string + fields: + items: + $ref: "#/components/schemas/MappingField" + type: + - array + - "null" + required: + - capture_id + - fields + type: object PutDraftInputBody: additionalProperties: false properties: @@ -2498,7 +2892,7 @@ components: - sha256= type: string hmac_secret: - description: "The shared signing secret (hmac only). Write-only: it is sealed and never returned" + description: "The shared signing secret (hmac only). Write-only: it is sealed and never returned. When none is stored or sent, one is generated and returned once" type: string verification: description: "How deliveries are verified: a bearer token, an HMAC-SHA256 signature of the body, or no verification" @@ -2518,6 +2912,10 @@ components: event_type: description: The event type that fires it; fixed when every is set type: string + event_type_id: + description: Fire only for events identified as this event type of the source + format: uuid + type: string every: description: Fire on an interval instead of on events, such as 1h examples: @@ -2528,6 +2926,13 @@ components: type: string description: Event fields that must match exactly type: object + mapping_id: + description: A mapping of the event type that reads the payload into the run's input under mapped + format: uuid + type: string + pending: + description: Save the trigger waiting for approval; it fires once a person approves it + type: boolean source: description: The source whose events fire it; fixed when every is set type: string @@ -2610,6 +3015,18 @@ components: - name - expected_version type: object + ReplyStruct { Mappings ListMappingView "json:\"mappings\"" }Body: + additionalProperties: false + properties: + mappings: + items: + $ref: "#/components/schemas/EventsMappingView" + type: + - array + - "null" + required: + - mappings + type: object RetryRunInputBody: additionalProperties: false properties: @@ -3278,6 +3695,24 @@ components: - email - password type: object + SigningInputBody: + additionalProperties: false + properties: + signed: + description: True requires an HMAC signature; false asks to take deliveries without one, which a person must approve + type: boolean + required: + - signed + type: object + SourceNameInputBody: + additionalProperties: false + properties: + name: + maxLength: 100 + type: string + required: + - name + type: object SystemAdminToken: additionalProperties: false properties: @@ -3419,6 +3854,17 @@ components: required: - text type: object + UpdateEventTypeInputBody: + additionalProperties: false + properties: + name: + type: string + rule: + $ref: "#/components/schemas/EventtypeRule" + required: + - name + - rule + type: object Usage: additionalProperties: false properties: @@ -4257,6 +4703,65 @@ paths: summary: Report which dashboard sections this binary serves tags: - Accounts + /api/captures: + get: + operationId: list-captures + parameters: + - explode: false + in: query + name: limit + schema: + default: 100 + format: int64 + maximum: 500 + minimum: 1 + type: integer + responses: + "200": + content: + application/json: + schema: + $ref: "#/components/schemas/EventsCapturesBody" + description: OK + default: + content: + application/problem+json: + schema: + $ref: "#/components/schemas/ErrorModel" + description: Error + summary: List recent deliveries a source kept + tags: + - Events + /api/captures/{id}/dispatches: + get: + operationId: list-capture-dispatches + parameters: + - in: path + name: id + required: true + schema: + format: uuid + type: string + responses: + "200": + content: + application/json: + schema: + items: + $ref: "#/components/schemas/EventsDispatchView" + type: + - array + - "null" + description: OK + default: + content: + application/problem+json: + schema: + $ref: "#/components/schemas/ErrorModel" + description: Error + summary: List the runs a capture started + tags: + - Events /api/chat/sessions: get: operationId: list-chat-sessions @@ -4961,6 +5466,102 @@ paths: summary: Install the draft's workflow, policy and trigger tags: - Drafts + /api/event-types: + get: + operationId: list-event-types + responses: + "200": + content: + application/json: + schema: + $ref: "#/components/schemas/EventsEventTypesBody" + description: OK + default: + content: + application/problem+json: + schema: + $ref: "#/components/schemas/ErrorModel" + description: Error + summary: List event types and the types unidentified captures suggest + tags: + - Events + post: + operationId: create-event-type + requestBody: + content: + application/json: + schema: + $ref: "#/components/schemas/CreateEventTypeInputBody" + required: true + responses: + "200": + content: + application/json: + schema: + $ref: "#/components/schemas/EventsEventTypeView" + description: OK + default: + content: + application/problem+json: + schema: + $ref: "#/components/schemas/ErrorModel" + description: Error + summary: Name a kind of event on a source + tags: + - Events + /api/event-types/{id}: + delete: + operationId: delete-event-type + parameters: + - in: path + name: id + required: true + schema: + format: uuid + type: string + responses: + "204": + description: No Content + default: + content: + application/problem+json: + schema: + $ref: "#/components/schemas/ErrorModel" + description: Error + summary: Delete an event type + tags: + - Events + put: + operationId: update-event-type + parameters: + - in: path + name: id + required: true + schema: + format: uuid + type: string + requestBody: + content: + application/json: + schema: + $ref: "#/components/schemas/UpdateEventTypeInputBody" + required: true + responses: + "200": + content: + application/json: + schema: + $ref: "#/components/schemas/EventsSavedID" + description: OK + default: + content: + application/problem+json: + schema: + $ref: "#/components/schemas/ErrorModel" + description: Error + summary: Rename an event type or change its rule + tags: + - Events /api/events: get: operationId: list-events @@ -5192,6 +5793,152 @@ paths: summary: Accept a package and start it tags: - Guards + /api/mappings: + get: + operationId: list-mappings + responses: + "200": + content: + application/json: + schema: + $ref: "#/components/schemas/ReplyStruct { Mappings ListMappingView \"json:\\\"mappings\\\"\" }Body" + description: OK + default: + content: + application/problem+json: + schema: + $ref: "#/components/schemas/ErrorModel" + description: Error + summary: List field mappings with how often each matched + tags: + - Events + post: + operationId: create-mapping + requestBody: + content: + application/json: + schema: + $ref: "#/components/schemas/EventsMappingBody" + required: true + responses: + "200": + content: + application/json: + schema: + $ref: "#/components/schemas/EventsMappingView" + description: OK + default: + content: + application/problem+json: + schema: + $ref: "#/components/schemas/ErrorModel" + description: Error + summary: Create a field mapping + tags: + - Events + /api/mappings/preview: + post: + operationId: preview-mapping + requestBody: + content: + application/json: + schema: + $ref: "#/components/schemas/PreviewInputBody" + required: true + responses: + "200": + content: + application/json: + schema: + $ref: "#/components/schemas/EventsPreviewBody" + description: OK + default: + content: + application/problem+json: + schema: + $ref: "#/components/schemas/ErrorModel" + description: Error + summary: Read a captured payload through candidate fields + tags: + - Events + /api/mappings/{id}: + delete: + operationId: delete-mapping + parameters: + - in: path + name: id + required: true + schema: + format: uuid + type: string + responses: + "204": + description: No Content + default: + content: + application/problem+json: + schema: + $ref: "#/components/schemas/ErrorModel" + description: Error + summary: Delete a field mapping + tags: + - Events + get: + operationId: get-mapping + parameters: + - in: path + name: id + required: true + schema: + format: uuid + type: string + responses: + "200": + content: + application/json: + schema: + $ref: "#/components/schemas/EventsMappingView" + description: OK + default: + content: + application/problem+json: + schema: + $ref: "#/components/schemas/ErrorModel" + description: Error + summary: Get a field mapping + tags: + - Events + patch: + operationId: update-mapping + parameters: + - in: path + name: id + required: true + schema: + format: uuid + type: string + requestBody: + content: + application/json: + schema: + $ref: "#/components/schemas/EventsMappingBody" + required: true + responses: + "200": + content: + application/json: + schema: + $ref: "#/components/schemas/EventsMappingView" + description: OK + default: + content: + application/problem+json: + schema: + $ref: "#/components/schemas/ErrorModel" + description: Error + summary: Change a field mapping + tags: + - Events /api/mcp-servers: get: operationId: list-mcp-servers @@ -6577,6 +7324,26 @@ paths: tags: - Events /api/sources/{name}: + delete: + operationId: delete-source + parameters: + - in: path + name: name + required: true + schema: + type: string + responses: + "204": + description: No Content + default: + content: + application/problem+json: + schema: + $ref: "#/components/schemas/ErrorModel" + description: Error + summary: Delete a source + tags: + - Events put: operationId: put-source parameters: @@ -6609,6 +7376,93 @@ paths: summary: Add or update a source, its verification scheme and its token tags: - Events + /api/sources/{name}/approve-unsigned: + post: + operationId: approve-source-unsigned + parameters: + - in: path + name: name + required: true + schema: + type: string + responses: + "200": + content: + application/json: + schema: + $ref: "#/components/schemas/EventsSourceView" + description: OK + default: + content: + application/problem+json: + schema: + $ref: "#/components/schemas/ErrorModel" + description: Error + summary: Accept deliveries that carry no signature + tags: + - Events + /api/sources/{name}/capture: + put: + operationId: set-source-capture + parameters: + - in: path + name: name + required: true + schema: + type: string + requestBody: + content: + application/json: + schema: + $ref: "#/components/schemas/CaptureInputBody" + required: true + responses: + "200": + content: + application/json: + schema: + $ref: "#/components/schemas/EventsSourceView" + description: OK + default: + content: + application/problem+json: + schema: + $ref: "#/components/schemas/ErrorModel" + description: Error + summary: Keep or stop keeping the headers and payloads a source receives + tags: + - Events + /api/sources/{name}/delivery-header: + put: + operationId: set-source-delivery-header + parameters: + - in: path + name: name + required: true + schema: + type: string + requestBody: + content: + application/json: + schema: + $ref: "#/components/schemas/DeliveryHeaderInputBody" + required: true + responses: + "200": + content: + application/json: + schema: + $ref: "#/components/schemas/EventsSourceView" + description: OK + default: + content: + application/problem+json: + schema: + $ref: "#/components/schemas/ErrorModel" + description: Error + summary: Name the header a sender repeats when it retries a delivery + tags: + - Events /api/sources/{name}/key-fields: put: operationId: put-source-key-fields @@ -6640,6 +7494,93 @@ paths: summary: Set the fields a source's events are grouped by tags: - Events + /api/sources/{name}/name: + put: + operationId: set-source-name + parameters: + - in: path + name: name + required: true + schema: + type: string + requestBody: + content: + application/json: + schema: + $ref: "#/components/schemas/SourceNameInputBody" + required: true + responses: + "200": + content: + application/json: + schema: + $ref: "#/components/schemas/EventsSourceView" + description: OK + default: + content: + application/problem+json: + schema: + $ref: "#/components/schemas/ErrorModel" + description: Error + summary: Label a source + tags: + - Events + /api/sources/{name}/secret: + post: + operationId: rotate-source-secret + parameters: + - in: path + name: name + required: true + schema: + type: string + responses: + "200": + content: + application/json: + schema: + $ref: "#/components/schemas/EventsSourceView" + description: OK + default: + content: + application/problem+json: + schema: + $ref: "#/components/schemas/ErrorModel" + description: Error + summary: Replace a signed source's secret and return it once + tags: + - Events + /api/sources/{name}/signing: + post: + operationId: set-source-signing + parameters: + - in: path + name: name + required: true + schema: + type: string + requestBody: + content: + application/json: + schema: + $ref: "#/components/schemas/SigningInputBody" + required: true + responses: + "200": + content: + application/json: + schema: + $ref: "#/components/schemas/EventsSourceView" + description: OK + default: + content: + application/problem+json: + schema: + $ref: "#/components/schemas/ErrorModel" + description: Error + summary: Require an HMAC signature, or ask to take unsigned deliveries + tags: + - Events /api/stream: get: operationId: stream-dashboard @@ -6789,6 +7730,26 @@ paths: tags: - Events /api/triggers/{name}: + delete: + operationId: delete-trigger + parameters: + - in: path + name: name + required: true + schema: + type: string + responses: + "204": + description: No Content + default: + content: + application/problem+json: + schema: + $ref: "#/components/schemas/ErrorModel" + description: Error + summary: Delete a trigger + tags: + - Events put: operationId: put-trigger parameters: @@ -6821,6 +7782,119 @@ paths: summary: Add or change a trigger tags: - Events + /api/triggers/{name}/approve: + post: + operationId: approve-trigger + parameters: + - in: path + name: name + required: true + schema: + type: string + responses: + "200": + content: + application/json: + schema: + $ref: "#/components/schemas/EventsTriggerView" + description: OK + default: + content: + application/problem+json: + schema: + $ref: "#/components/schemas/ErrorModel" + description: Error + summary: Arm a trigger that waits for approval + tags: + - Events + /api/triggers/{name}/dispatches: + get: + operationId: list-trigger-dispatches + parameters: + - in: path + name: name + required: true + schema: + type: string + - explode: false + in: query + name: limit + schema: + default: 50 + format: int64 + maximum: 200 + minimum: 1 + type: integer + responses: + "200": + content: + application/json: + schema: + items: + $ref: "#/components/schemas/EventsDispatchView" + type: + - array + - "null" + description: OK + default: + content: + application/problem+json: + schema: + $ref: "#/components/schemas/ErrorModel" + description: Error + summary: List the runs a trigger started + tags: + - Events + /api/triggers/{name}/pause: + post: + operationId: pause-trigger + parameters: + - in: path + name: name + required: true + schema: + type: string + responses: + "200": + content: + application/json: + schema: + $ref: "#/components/schemas/EventsTriggerView" + description: OK + default: + content: + application/problem+json: + schema: + $ref: "#/components/schemas/ErrorModel" + description: Error + summary: Stop a trigger firing + tags: + - Events + /api/triggers/{name}/resume: + post: + operationId: resume-trigger + parameters: + - in: path + name: name + required: true + schema: + type: string + responses: + "200": + content: + application/json: + schema: + $ref: "#/components/schemas/EventsTriggerView" + description: OK + default: + content: + application/problem+json: + schema: + $ref: "#/components/schemas/ErrorModel" + description: Error + summary: Return a paused trigger to waiting for approval + tags: + - Events /api/workflow-drafts: get: operationId: list-workflow-drafts diff --git a/internal/platform/pg/migrations/00052_events_captures_mappings_bindings.sql b/internal/platform/pg/migrations/00052_events_captures_mappings_bindings.sql new file mode 100644 index 000000000..cb600aaeb --- /dev/null +++ b/internal/platform/pg/migrations/00052_events_captures_mappings_bindings.sql @@ -0,0 +1,67 @@ +-- +goose Up +-- A source keeps a display name and the header its sender repeats on every +-- retry of one delivery. A source that does no verification refuses deliveries +-- until a person approves it, and may stop keeping the raw payload. +ALTER TABLE events.sources + ADD COLUMN display_name text NOT NULL DEFAULT '', + ADD COLUMN delivery_header text NOT NULL DEFAULT '', + ADD COLUMN unsigned_approved boolean NOT NULL DEFAULT false, + ADD COLUMN capture boolean NOT NULL DEFAULT true; +UPDATE events.sources SET unsigned_approved = true WHERE verification = 'none'; + +-- A person names a kind of event on a source by the headers and payload paths +-- that identify it. Its schema is the JSON paths of the example it came from. +CREATE TABLE events.event_types ( + id uuid PRIMARY KEY, + org_id text NOT NULL, + source text NOT NULL, + name text NOT NULL, + rule jsonb NOT NULL, + schema jsonb NOT NULL DEFAULT '{}', + created_at timestamptz NOT NULL DEFAULT now(), + updated_at timestamptz NOT NULL DEFAULT now(), + UNIQUE (org_id, source, name) +); + +-- A mapping names the payload paths a workflow reads from one event type. +CREATE TABLE events.mappings ( + id uuid PRIMARY KEY, + org_id text NOT NULL, + name text NOT NULL, + source_hint text NOT NULL DEFAULT '', + event_type_id uuid NOT NULL REFERENCES events.event_types (id) ON DELETE RESTRICT, + fields jsonb NOT NULL, + match_count bigint NOT NULL DEFAULT 0, + last_matched_at timestamptz, + created_at timestamptz NOT NULL DEFAULT now(), + updated_at timestamptz NOT NULL DEFAULT now() +); + +-- Each event records how its delivery was verified and the headers and sender +-- address it arrived with. The headers and body are null when the source does +-- not keep payloads. event_type_id is the type its headers and payload matched +-- on arrival. +ALTER TABLE events.events + ADD COLUMN verification text NOT NULL DEFAULT '' + CHECK (verification IN ('', 'signed', 'token', 'unsigned')), + ADD COLUMN headers jsonb, + ADD COLUMN remote_addr text NOT NULL DEFAULT '', + ADD COLUMN event_type_id uuid REFERENCES events.event_types (id) ON DELETE SET NULL; + +-- A trigger bound to an event type fires only for events identified as it; a +-- bound mapping reads the event's payload for the run. Triggers +-- saved before approval existed are approved. +ALTER TABLE events.triggers + ADD COLUMN approved boolean NOT NULL DEFAULT true, + ADD COLUMN event_type_id uuid REFERENCES events.event_types (id) ON DELETE RESTRICT, + ADD COLUMN mapping_id uuid REFERENCES events.mappings (id) ON DELETE RESTRICT; + +-- +goose Down +ALTER TABLE events.triggers + DROP COLUMN mapping_id, DROP COLUMN event_type_id, DROP COLUMN approved; +ALTER TABLE events.events + DROP COLUMN event_type_id, DROP COLUMN remote_addr, DROP COLUMN headers, DROP COLUMN verification; +DROP TABLE events.mappings; +DROP TABLE events.event_types; +ALTER TABLE events.sources + DROP COLUMN capture, DROP COLUMN unsigned_approved, DROP COLUMN delivery_header, DROP COLUMN display_name; diff --git a/internal/platform/pg/migrations/00052_runs_workflow_drafts.sql b/internal/platform/pg/migrations/00053_runs_workflow_drafts.sql similarity index 100% rename from internal/platform/pg/migrations/00052_runs_workflow_drafts.sql rename to internal/platform/pg/migrations/00053_runs_workflow_drafts.sql diff --git a/modules/events/access_test.go b/modules/events/access_test.go index eee06a705..e48ec71cd 100644 --- a/modules/events/access_test.go +++ b/modules/events/access_test.go @@ -150,7 +150,7 @@ func TestTriggersStartRunsAsTheWorkload(t *testing.T) { t.Run(tt.name, func(t *testing.T) { starts := &startRecorder{err: tt.err} m := &Module{runs: starts, log: slog.New(slog.DiscardHandler)} - _, started, err := m.startRun(t.Context(), store.Event{ID: uuid.New(), OrgID: "default"}, store.Trigger{Name: "t", Workflow: "triage"}) + _, started, err := m.startRun(t.Context(), store.Event{ID: uuid.New(), OrgID: "default"}, store.Trigger{Name: "t", Workflow: "triage"}, nil) if started != tt.wantStarted { t.Fatalf("started = %v, want %v", started, tt.wantStarted) } diff --git a/modules/events/api.go b/modules/events/api.go index 9e02ccd57..82f69148a 100644 --- a/modules/events/api.go +++ b/modules/events/api.go @@ -24,6 +24,8 @@ import ( // Register adds the events API: sources, triggers, groups of unmatched events // and the events themselves. func (m *Module) Register(api huma.API) { + m.registerCapture(api) + m.registerSourceSettings(api) tags := []string{"Events"} httpapi.Register(api, httpapi.AdminAuthorized(governance.ActionRead, governance.KindSource), huma.Operation{ OperationID: "list-sources", Method: http.MethodGet, Path: "/api/sources", @@ -68,13 +70,14 @@ type putSourceInput struct { Verification string `json:"verification,omitzero" enum:"token,hmac,none" doc:"How deliveries are verified: a bearer token, an HMAC-SHA256 signature of the body, or no verification"` HMACHeader string `json:"hmac_header,omitzero" doc:"The header that carries the signature (hmac only)"` HMACPrefix string `json:"hmac_prefix,omitzero" doc:"Text before the hex digest in the signature header, such as sha256= (hmac only)" example:"sha256="` - HMACSecret string `json:"hmac_secret,omitzero" doc:"The shared signing secret (hmac only). Write-only: it is sealed and never returned"` + HMACSecret string `json:"hmac_secret,omitzero" doc:"The shared signing secret (hmac only). Write-only: it is sealed and never returned. When none is stored or sent, one is generated and returned once"` } } type sourceCreated struct { Source string `json:"source"` Token string `json:"token,omitzero" doc:"Returned once for a token source; only its hash is stored"` + Secret string `json:"secret,omitzero" doc:"A signing secret generated for an HMAC source, returned once"` URL string `json:"url" doc:"Where the sender posts its events"` } @@ -95,19 +98,22 @@ func (m *Module) putSource(ctx context.Context, in *putSourceInput) (*httpapi.Re if err == nil { stored = &existing } - src, token, err := m.sourceToSave(in, stored) + src, issued, err := m.sourceToSave(in, stored) if err != nil { return nil, err } if err := m.store.UpsertSource(ctx, src); err != nil { return nil, huma.Error500InternalServerError("couldn't save the source", err) } - return httpapi.OK(sourceCreated{Source: in.Name, Token: token, URL: "/hooks/" + in.Name}), nil + return httpapi.OK(sourceCreated{Source: in.Name, Token: issued.token, Secret: issued.secret, URL: "/hooks/" + in.Name}), nil } +// issued is a credential generated for a saved source, returned once. +type issued struct{ token, secret string } + // sourceToSave builds the row to store from the request and the stored source, -// if there is one. It returns the new token when one was issued. -func (m *Module) sourceToSave(in *putSourceInput, stored *store.Source) (store.Source, string, error) { +// if there is one. It returns the token or secret it generated, if any. +func (m *Module) sourceToSave(in *putSourceInput, stored *store.Source) (store.Source, issued, error) { explicit := in.Body.Verification != "" // The secret column is NOT NULL, so a source without a secret carries an // empty value: the driver writes a nil slice as NULL. @@ -127,46 +133,56 @@ func (m *Module) sourceToSave(in *putSourceInput, stored *store.Source) (store.S switch src.Verification { case store.Token: if stored != nil && !explicit { - return src, "", nil + return src, issued{}, nil } token, err := newToken() if err != nil { - return src, "", huma.Error500InternalServerError("couldn't generate a token", err) + return src, issued{}, huma.Error500InternalServerError("couldn't generate a token", err) } src.TokenSHA256 = httpjson.TokenHash(token) - return src, token, nil + return src, issued{token: token}, nil case store.None: - return src, "", nil + return src, issued{}, nil case store.HMAC: + var secret string if explicit { - if err := m.setHMAC(&src, in, stored); err != nil { - return src, "", err + var err error + if secret, err = m.setHMAC(&src, in, stored); err != nil { + return src, issued{}, err } } if src.HMACHeader == "" || len(src.HMACSecretSealed) == 0 { - return src, "", huma.Error400BadRequest("an HMAC source needs a signature header and a secret") + return src, issued{}, huma.Error400BadRequest("an HMAC source needs a signature header") } - return src, "", nil + return src, issued{secret: secret}, nil default: - return src, "", huma.Error400BadRequest("verification must be token, hmac or none") + return src, issued{}, huma.Error400BadRequest("verification must be token, hmac or none") } } // setHMAC applies the requested HMAC settings. A blank secret keeps the sealed -// secret of a source that already uses HMAC. -func (m *Module) setHMAC(src *store.Source, in *putSourceInput, stored *store.Source) error { +// secret a source already has, and a source that has none gets a generated one, +// which is returned so it can be shown once. +func (m *Module) setHMAC(src *store.Source, in *putSourceInput, stored *store.Source) (generated string, err error) { src.HMACHeader, src.HMACPrefix = in.Body.HMACHeader, in.Body.HMACPrefix + secret := in.Body.HMACSecret switch { - case in.Body.HMACSecret != "": - sealed, err := m.sealer.Seal(secretLabel(m.orgID, in.Name), []byte(in.Body.HMACSecret)) - if err != nil { - return huma.Error500InternalServerError("couldn't seal the secret", err) - } - src.HMACSecretSealed = sealed - case stored != nil && stored.Verification == store.HMAC: + case secret != "": + case stored != nil && len(stored.HMACSecretSealed) > 0: src.HMACSecretSealed = stored.HMACSecretSealed + return "", nil + default: + if secret, err = newSecret(); err != nil { + return "", huma.Error500InternalServerError("couldn't generate a secret", err) + } + generated = secret } - return nil + sealed, err := m.sealer.Seal(secretLabel(m.orgID, in.Name), []byte(secret)) + if err != nil { + return "", huma.Error500InternalServerError("couldn't seal the secret", err) + } + src.HMACSecretSealed = sealed + return generated, nil } type putTriggerInput struct { @@ -178,6 +194,11 @@ type putTriggerInput struct { Workflow string `json:"workflow" doc:"The workflow each match runs"` Disabled bool `json:"disabled,omitzero"` Every string `json:"every,omitzero" doc:"Fire on an interval instead of on events, such as 1h" example:"1h"` + // EventTypeID restricts the trigger to events identified as that event + // type, and MappingID reads their payload for the run. + EventTypeID string `json:"event_type_id,omitzero" format:"uuid" doc:"Fire only for events identified as this event type of the source"` + MappingID string `json:"mapping_id,omitzero" format:"uuid" doc:"A mapping of the event type that reads the payload into the run's input under mapped"` + Pending bool `json:"pending,omitzero" doc:"Save the trigger waiting for approval; it fires once a person approves it"` } } @@ -199,31 +220,103 @@ func (m *Module) putTrigger(ctx context.Context, in *putTriggerInput) (*httpapi. // this trigger to its own. req.Filter = map[string]string{"trigger": in.Name} } - if !validName.MatchString(in.Name) || req.Source == "" || req.EventType == "" || req.Workflow == "" { + if !validName.MatchString(in.Name) || req.Source == "" || (req.EventType == "" && req.EventTypeID == "") || req.Workflow == "" { return nil, huma.Error400BadRequest("a trigger needs a lowercase name, a workflow, and either a source and event type or an every interval") } - if err := m.runs.CheckTrigger(ctx, m.orgID, req.Workflow); err != nil { - if refused, ok := errors.AsType[*runs.InvalidError](err); ok { - return nil, huma.Error400BadRequest(refused.Problems[0].Message, &huma.ErrorDetail{Location: "body.workflow", Message: refused.Problems[0].Message}) - } - return nil, huma.Error500InternalServerError("couldn't check the workflow", err) - } - if err := m.checkEventType(ctx, req.Source, req.EventType); err != nil { + typeID, mappingID, err := m.checkTargets(ctx, req.Source, req.EventType, req.Workflow, req.EventTypeID, req.MappingID) + if err != nil { return nil, err } filter, err := json.Marshal(req.Filter) if err != nil { return nil, huma.Error400BadRequest(err.Error()) } + approved, err := m.approvedOnSave(ctx, in.Name, req.Pending) + if err != nil { + return nil, err + } if err := m.store.UpsertTrigger(ctx, store.Trigger{ OrgID: m.orgID, Name: in.Name, Source: req.Source, EventType: req.EventType, - Filter: filter, Workflow: req.Workflow, Enabled: !req.Disabled, ScheduleEverySeconds: every, + Filter: filter, Workflow: req.Workflow, Enabled: !req.Disabled, Approved: approved, + EventTypeID: typeID, MappingID: mappingID, ScheduleEverySeconds: every, }); err != nil { return nil, huma.Error500InternalServerError("couldn't save the trigger", err) } return httpapi.OK(triggerSaved{Trigger: in.Name}), nil } +// checkTargets refuses a trigger whose workflow can't be started or whose +// event type, mapping or connector event type doesn't exist. +func (m *Module) checkTargets(ctx context.Context, source, eventType, workflow, eventTypeID, mappingID string) (typeID, mapID uuid.NullUUID, err error) { + if typeID, mapID, err = m.checkBinding(ctx, source, eventTypeID, mappingID); err != nil { + return typeID, mapID, err + } + if err := m.runs.CheckTrigger(ctx, m.orgID, workflow); err != nil { + if refused, ok := errors.AsType[*runs.InvalidError](err); ok { + return typeID, mapID, huma.Error400BadRequest(refused.Problems[0].Message, &huma.ErrorDetail{Location: "body.workflow", Message: refused.Problems[0].Message}) + } + return typeID, mapID, huma.Error500InternalServerError("couldn't check the workflow", err) + } + if eventType != "" { + err = m.checkEventType(ctx, source, eventType) + } + return typeID, mapID, err +} + +// approvedOnSave decides whether a saved trigger is approved. A new trigger is +// unless it asks to wait; a saved one keeps its approval unless the change asks +// to wait for a new approval. +func (m *Module) approvedOnSave(ctx context.Context, name string, pending bool) (bool, error) { + if pending { + return false, nil + } + t, err := m.store.Trigger(ctx, m.orgID, name) + if errors.Is(err, store.ErrNotFound) { + return true, nil + } + if err != nil { + return false, huma.Error500InternalServerError("couldn't read the trigger", err) + } + return t.Approved, nil +} + +// checkBinding resolves the event type and mapping a trigger binds to. The +// event type must be the source's, and the mapping the event type's. +func (m *Module) checkBinding(ctx context.Context, source, eventTypeID, mappingID string) (typeID, mapID uuid.NullUUID, err error) { + refuse := func(location, message string) error { + return huma.Error400BadRequest(message, &huma.ErrorDetail{Location: location, Message: message}) + } + if eventTypeID != "" { + id, err := uuid.Parse(eventTypeID) + if err != nil { + return typeID, mapID, refuse("body.event_type_id", "an event type id is a UUID") + } + t, err := m.store.EventType(ctx, m.orgID, id) + if errors.Is(err, store.ErrNotFound) || (err == nil && t.Source != source) { + return typeID, mapID, refuse("body.event_type_id", "the source has no such event type") + } + if err != nil { + return typeID, mapID, huma.Error500InternalServerError("couldn't read the event type", err) + } + typeID = uuid.NullUUID{UUID: id, Valid: true} + } + if mappingID != "" { + id, err := uuid.Parse(mappingID) + if err != nil { + return typeID, mapID, refuse("body.mapping_id", "a mapping id is a UUID") + } + mp, err := m.store.Mapping(ctx, m.orgID, id) + if errors.Is(err, store.ErrNotFound) || (err == nil && (!typeID.Valid || mp.EventTypeID != typeID.UUID)) { + return typeID, mapID, refuse("body.mapping_id", "the mapping must belong to the trigger's event type") + } + if err != nil { + return typeID, mapID, huma.Error500InternalServerError("couldn't read the mapping", err) + } + mapID = uuid.NullUUID{UUID: id, Valid: true} + } + return typeID, mapID, nil +} + // checkEventType refuses a trigger on an event type that the source's // connector doesn't declare, so the trigger can't be saved when it could never // fire. Schedule triggers have no connector. @@ -332,6 +425,16 @@ type sourceView struct { HMACHeader string `json:"hmac_header,omitzero"` HMACPrefix string `json:"hmac_prefix,omitzero"` HMACSecretSet bool `json:"hmac_secret_set"` + // DisplayName labels the source, and DeliveryHeader names the header a + // sender repeats on every retry of one delivery. + DisplayName string `json:"display_name,omitzero"` + DeliveryHeader string `json:"delivery_header,omitzero"` + // Signing is signed (HMAC), token, unsigned_pending_approval or unsigned. + Signing string `json:"signing" enum:"signed,token,unsigned_pending_approval,unsigned"` + // Capture says the source keeps each delivery's headers and payload. + Capture bool `json:"capture"` + // Secret is a signing secret the request issued, returned once. + Secret string `json:"secret,omitzero"` } func (m *Module) sources(ctx context.Context, _ *struct{}) (*httpapi.Reply[[]sourceView], error) { @@ -342,21 +445,30 @@ func (m *Module) sources(ctx context.Context, _ *struct{}) (*httpapi.Reply[[]sou } out := make([]sourceView, 0, len(sources)) for _, src := range sources { - view := sourceView{ - Name: src.Name, Connector: src.Connector, TokenSet: src.TokenSHA256 != "", KeyFields: []string{}, - Verification: src.Verification, HMACHeader: src.HMACHeader, HMACPrefix: src.HMACPrefix, HMACSecretSet: len(src.HMACSecretSealed) > 0, - } - if len(src.KeyFields) > 0 { - if err := json.Unmarshal(src.KeyFields, &view.KeyFields); err != nil { - m.log.ErrorContext(ctx, "read source key fields", "source", src.Name, "error", err) - return nil, huma.Error500InternalServerError("couldn't read the key fields of " + src.Name) - } + view, err := viewOfSource(src) + if err != nil { + m.log.ErrorContext(ctx, "read source key fields", "source", src.Name, "error", err) + return nil, huma.Error500InternalServerError("couldn't read the key fields of " + src.Name) } out = append(out, view) } return httpapi.OK(out), nil } +func viewOfSource(src store.Source) (sourceView, error) { + view := sourceView{ + Name: src.Name, Connector: src.Connector, TokenSet: src.TokenSHA256 != "", KeyFields: []string{}, + Verification: src.Verification, HMACHeader: src.HMACHeader, HMACPrefix: src.HMACPrefix, HMACSecretSet: len(src.HMACSecretSealed) > 0, + DisplayName: src.DisplayName, DeliveryHeader: src.DeliveryHeader, Signing: signingOf(src), Capture: src.Capture, + } + if len(src.KeyFields) > 0 { + if err := json.Unmarshal(src.KeyFields, &view.KeyFields); err != nil { + return view, err + } + } + return view, nil +} + // triggerView is a trigger as the API returns it. type triggerView struct { Name string `json:"name"` @@ -366,6 +478,56 @@ type triggerView struct { Workflow string `json:"workflow"` Enabled bool `json:"enabled"` Every string `json:"every,omitempty"` + // Status is pending_approval, armed or paused. + Status string `json:"status" enum:"pending_approval,armed,paused"` + EventTypeID string `json:"event_type_id,omitempty"` + MappingID string `json:"mapping_id,omitempty"` + // Unsigned marks a trigger whose source takes deliveries with no + // verification. + Unsigned bool `json:"unsigned"` +} + +func statusOf(t store.Trigger) string { + switch { + case !t.Approved: + return "pending_approval" + case !t.Enabled: + return "paused" + } + return "armed" +} + +// viewOfTrigger is the API's form of a stored trigger. Unsigned is set by the +// caller, which knows the sources. +func viewOfTrigger(t store.Trigger) (triggerView, error) { + view := triggerView{Name: t.Name, Source: t.Source, EventType: t.EventType, Workflow: t.Workflow, Enabled: t.Enabled, Filter: map[string]string{}, Status: statusOf(t)} + if len(t.Filter) > 0 { + if err := json.Unmarshal(t.Filter, &view.Filter); err != nil { + return view, fmt.Errorf("filter of %s: %w", t.Name, err) + } + } + if t.ScheduleEverySeconds.Valid { + view.Every = (time.Duration(t.ScheduleEverySeconds.Int64) * time.Second).String() + } + if t.EventTypeID.Valid { + view.EventTypeID = t.EventTypeID.UUID.String() + } + if t.MappingID.Valid { + view.MappingID = t.MappingID.UUID.String() + } + return view, nil +} + +// viewOfTrigger returns a trigger with whether its source is unsigned. +func (m *Module) viewOfTrigger(ctx context.Context, t store.Trigger) (triggerView, error) { + view, err := viewOfTrigger(t) + if err != nil { + return view, err + } + if src, err := m.store.Source(ctx, m.orgID, t.Source); err == nil { + view.Unsigned = src.Verification == store.None + } + return view, nil } func (m *Module) triggers(ctx context.Context, _ *struct{}) (*httpapi.Reply[[]triggerView], error) { @@ -374,18 +536,23 @@ func (m *Module) triggers(ctx context.Context, _ *struct{}) (*httpapi.Reply[[]tr m.log.ErrorContext(ctx, "list triggers", "error", err) return nil, huma.Error500InternalServerError("couldn't list triggers") } + sources, err := m.store.Sources(ctx, m.orgID) + if err != nil { + m.log.ErrorContext(ctx, "list sources", "error", err) + return nil, huma.Error500InternalServerError("couldn't list triggers") + } + unsigned := map[string]bool{} + for _, src := range sources { + unsigned[src.Name] = src.Verification == store.None + } out := make([]triggerView, 0, len(triggers)) for _, t := range triggers { - view := triggerView{Name: t.Name, Source: t.Source, EventType: t.EventType, Workflow: t.Workflow, Enabled: t.Enabled, Filter: map[string]string{}} - if len(t.Filter) > 0 { - if err := json.Unmarshal(t.Filter, &view.Filter); err != nil { - m.log.ErrorContext(ctx, "read trigger filter", "trigger", t.Name, "error", err) - return nil, huma.Error500InternalServerError("couldn't read the filter of " + t.Name) - } - } - if t.ScheduleEverySeconds.Valid { - view.Every = (time.Duration(t.ScheduleEverySeconds.Int64) * time.Second).String() + view, err := viewOfTrigger(t) + if err != nil { + m.log.ErrorContext(ctx, "read trigger", "trigger", t.Name, "error", err) + return nil, huma.Error500InternalServerError("couldn't read the filter of " + t.Name) } + view.Unsigned = unsigned[t.Source] out = append(out, view) } return httpapi.OK(out), nil @@ -474,11 +641,15 @@ type eventView struct { ReceivedAt time.Time `json:"received_at"` Matched bool `json:"matched"` DispatchedAt *time.Time `json:"dispatched_at,omitempty"` + // Verification is how the delivery was verified: signed, token or unsigned. + // It is empty for an event a connector streamed. + Verification string `json:"verification,omitempty"` } func viewOf(e store.Event) eventView { v := eventView{ - ID: e.ID, Source: e.Source, Type: e.Type, Fields: e.Fields, LinkedTo: e.LinkedTo, + Verification: e.Verification, + ID: e.ID, Source: e.Source, Type: e.Type, Fields: e.Fields, LinkedTo: e.LinkedTo, ReceivedAt: e.ReceivedAt, Matched: e.Matched, } if e.OccurredAt.Valid { diff --git a/modules/events/api_capture.go b/modules/events/api_capture.go new file mode 100644 index 000000000..a8380420c --- /dev/null +++ b/modules/events/api_capture.go @@ -0,0 +1,527 @@ +package events + +import ( + "context" + json "encoding/json/v2" + "errors" + "net/http" + "slices" + "time" + + "github.com/danielgtaylor/huma/v2" + "github.com/google/uuid" + + "github.com/samcharles93/archie-core/internal/platform/httpapi" + "github.com/samcharles93/archie-core/modules/events/internal/eventtype" + "github.com/samcharles93/archie-core/modules/events/internal/mapping" + "github.com/samcharles93/archie-core/modules/events/internal/store" + "github.com/samcharles93/archie-core/modules/governance" +) + +// registerCapture adds the operations for captured deliveries, event types and +// mappings. Every one is administrator-only, like the rest of the events API. +func (m *Module) registerCapture(api huma.API) { + tags := []string{"Events"} + op := func(id, method, path, summary string) huma.Operation { + return huma.Operation{OperationID: id, Method: method, Path: path, Summary: summary, Tags: tags} + } + capture := func(action governance.Action) httpapi.Access { + return httpapi.AdminAuthorized(action, governance.KindCapture) + } + httpapi.Register(api, capture(governance.ActionRead), op("list-captures", http.MethodGet, "/api/captures", "List recent deliveries a source kept"), m.captures) + httpapi.Register(api, capture(governance.ActionRead), op("list-capture-dispatches", http.MethodGet, "/api/captures/{id}/dispatches", "List the runs a capture started"), m.captureDispatches) + + eventType := func(action governance.Action) httpapi.Access { + return httpapi.AdminAuthorized(action, governance.KindEventType) + } + httpapi.Register(api, eventType(governance.ActionRead), op("list-event-types", http.MethodGet, "/api/event-types", "List event types and the types unidentified captures suggest"), m.eventTypes) + httpapi.Register(api, eventType(governance.ActionCreate), op("create-event-type", http.MethodPost, "/api/event-types", "Name a kind of event on a source"), m.createEventType) + httpapi.Register(api, eventType(governance.ActionUpdate), op("update-event-type", http.MethodPut, "/api/event-types/{id}", "Rename an event type or change its rule"), m.updateEventType) + httpapi.Register(api, eventType(governance.ActionDelete), op("delete-event-type", http.MethodDelete, "/api/event-types/{id}", "Delete an event type"), m.deleteEventType) + + mapped := func(action governance.Action) httpapi.Access { + return httpapi.AdminAuthorized(action, governance.KindMapping) + } + httpapi.Register(api, mapped(governance.ActionRead), op("list-mappings", http.MethodGet, "/api/mappings", "List field mappings with how often each matched"), m.mappings) + httpapi.Register(api, mapped(governance.ActionRead), op("get-mapping", http.MethodGet, "/api/mappings/{id}", "Get a field mapping"), m.mapping) + httpapi.Register(api, mapped(governance.ActionCreate), op("create-mapping", http.MethodPost, "/api/mappings", "Create a field mapping"), m.createMapping) + httpapi.Register(api, mapped(governance.ActionUpdate), op("update-mapping", http.MethodPatch, "/api/mappings/{id}", "Change a field mapping"), m.updateMapping) + httpapi.Register(api, mapped(governance.ActionDelete), op("delete-mapping", http.MethodDelete, "/api/mappings/{id}", "Delete a field mapping"), m.deleteMapping) + httpapi.Register(api, mapped(governance.ActionRead), op("preview-mapping", http.MethodPost, "/api/mappings/preview", "Read a captured payload through candidate fields"), m.previewMapping) +} + +// captureView is a stored delivery as the inspector shows it. +type captureView struct { + ID uuid.UUID `json:"id"` + ReceivedAt time.Time `json:"received_at"` + Source string `json:"source"` + RemoteAddr string `json:"remote_addr,omitempty"` + ContentType string `json:"content_type,omitempty"` + Headers string `json:"headers"` + Body string `json:"body"` + // Verification is how the delivery was verified: signed, token or unsigned. + Verification string `json:"verification" enum:"signed,token,unsigned,"` + // EventType is the id of the event type the delivery was identified as. + EventType string `json:"event_type,omitempty"` +} + +type capturesInput struct { + Limit int `query:"limit" minimum:"1" maximum:"500" default:"100"` +} + +type capturesBody struct { + Captures []captureView `json:"captures"` + // Enabled is false when no source keeps deliveries. + Enabled bool `json:"enabled"` +} + +func (m *Module) captures(ctx context.Context, in *capturesInput) (*httpapi.Reply[capturesBody], error) { + rows, err := m.store.Captures(ctx, m.orgID, in.Limit) + if err != nil { + return nil, huma.Error500InternalServerError("couldn't list captures", err) + } + sources, err := m.store.Sources(ctx, m.orgID) + if err != nil { + return nil, huma.Error500InternalServerError("couldn't list sources", err) + } + enabled := len(sources) == 0 || slices.ContainsFunc(sources, func(s store.Source) bool { return s.Capture }) + out := capturesBody{Captures: make([]captureView, 0, len(rows)), Enabled: enabled} + for _, e := range rows { + out.Captures = append(out.Captures, captureOf(e)) + } + return httpapi.OK(out), nil +} + +func captureOf(e store.Event) captureView { + v := captureView{ + ID: e.ID, ReceivedAt: e.ReceivedAt, Source: e.Source, RemoteAddr: e.RemoteAddr, + Headers: string(e.Headers), Body: string(e.Body), Verification: e.Verification, + } + v.ContentType = eventtype.ParseHeaders(v.Headers)["content-type"] + if e.EventTypeID.Valid { + v.EventType = e.EventTypeID.UUID.String() + } + return v +} + +type dispatchView struct { + EventID uuid.UUID `json:"event_id"` + RunID uuid.UUID `json:"run_id"` + Trigger string `json:"trigger,omitempty"` + CreatedAt time.Time `json:"created_at,omitzero"` +} + +type idInput struct { + ID string `path:"id" format:"uuid"` +} + +func (m *Module) captureDispatches(ctx context.Context, in *idInput) (*httpapi.Reply[[]dispatchView], error) { + id, err := uuid.Parse(in.ID) + if err != nil { + return nil, huma.Error400BadRequest("a capture id is a UUID") + } + runs, err := m.store.RunsForEvent(ctx, m.orgID, id) + if err != nil { + return nil, huma.Error500InternalServerError("couldn't read the runs of the capture", err) + } + out := make([]dispatchView, 0, len(runs)) + for _, r := range runs { + out = append(out, dispatchView{EventID: id, RunID: r.RunID, Trigger: r.Trigger}) + } + return httpapi.OK(out), nil +} + +// eventTypeView is an event type as the inspector reads it. +type eventTypeView struct { + ID uuid.UUID `json:"id"` + Source string `json:"source"` + Name string `json:"name"` + Rule eventtype.Rule `json:"rule"` + Schema map[string]eventtype.ValueType `json:"schema"` +} + +type eventTypesBody struct { + Enabled bool `json:"enabled"` + EventTypes []eventTypeView `json:"event_types"` + Proposals []eventtype.Proposal `json:"proposals"` +} + +// proposalScan bounds how many recent captures are grouped into proposals. +const proposalScan = 500 + +func (m *Module) eventTypes(ctx context.Context, _ *struct{}) (*httpapi.Reply[eventTypesBody], error) { + rows, err := m.store.EventTypes(ctx, m.orgID) + if err != nil { + return nil, huma.Error500InternalServerError("couldn't list event types", err) + } + types, err := typesOf(rows) + if err != nil { + return nil, huma.Error500InternalServerError("couldn't read the event types", err) + } + out := eventTypesBody{Enabled: true, EventTypes: make([]eventTypeView, 0, len(types))} + for i, t := range types { + out.EventTypes = append(out.EventTypes, eventTypeView{ID: rows[i].ID, Source: t.Source, Name: t.Name, Rule: t.Rule, Schema: t.Schema}) + } + captured, err := m.store.Captures(ctx, m.orgID, proposalScan) + if err != nil { + return nil, huma.Error500InternalServerError("couldn't list captures", err) + } + out.Proposals = eventtype.Propose(unidentified(types, captured)) + if out.Proposals == nil { + out.Proposals = []eventtype.Proposal{} + } + return httpapi.OK(out), nil +} + +// unidentified keeps the captures that no event type was named for on arrival +// and that no current type would identify either. +func unidentified(types []eventtype.EventType, captured []store.Event) []eventtype.Capture { + var out []eventtype.Capture + for _, e := range captured { + if e.EventTypeID.Valid { + continue + } + sample := eventtype.Sample{Headers: eventtype.ParseHeaders(string(e.Headers)), Body: e.Body} + if _, ok := eventtype.Identify(types, e.Source, sample); ok { + continue + } + out = append(out, eventtype.Capture{ID: e.ID.String(), Source: e.Source, ReceivedAt: e.ReceivedAt, Sample: sample}) + } + return out +} + +type createEventTypeInput struct { + Body struct { + Source string `json:"source"` + Name string `json:"name"` + Rule *eventtype.Rule `json:"rule,omitzero" doc:"The conditions that identify the type; inferred from the example when absent"` + Example struct { + Headers map[string]string `json:"headers,omitzero"` + Body string `json:"body,omitzero"` + } `json:"example,omitzero"` + } +} + +func (m *Module) createEventType(ctx context.Context, in *createEventTypeInput) (*httpapi.Reply[eventTypeView], error) { + req := in.Body + t := eventtype.FromExample(req.Source, req.Name, eventtype.Sample{Headers: req.Example.Headers, Body: []byte(req.Example.Body)}) + if req.Rule != nil { + t.Rule = *req.Rule + } + if err := t.Validate(); err != nil { + return nil, huma.Error400BadRequest(err.Error()) + } + if _, err := m.store.Source(ctx, m.orgID, t.Source); errors.Is(err, store.ErrNotFound) { + return nil, huma.Error400BadRequest("no source named " + t.Source) + } else if err != nil { + return nil, huma.Error500InternalServerError("couldn't read the source", err) + } + if err := m.checkOverlap(ctx, t); err != nil { + return nil, err + } + rule, err := json.Marshal(t.Rule) + if err != nil { + return nil, huma.Error400BadRequest(err.Error()) + } + schema, err := json.Marshal(t.Schema) + if err != nil { + return nil, huma.Error400BadRequest(err.Error()) + } + id := uuid.New() + err = m.store.InsertEventType(ctx, store.EventType{ID: id, OrgID: m.orgID, Source: t.Source, Name: t.Name, Rule: rule, Schema: schema}) + if errors.Is(err, store.ErrExists) { + return nil, huma.Error409Conflict("the source already has an event type named " + t.Name) + } + if err != nil { + return nil, huma.Error500InternalServerError("couldn't save the event type", err) + } + return httpapi.OK(eventTypeView{ID: id, Source: t.Source, Name: t.Name, Rule: t.Rule, Schema: t.Schema}), nil +} + +// checkOverlap refuses a type whose rule can match events another type on the +// source already matches, since an event is identified only when one type +// matches it. +func (m *Module) checkOverlap(ctx context.Context, candidate eventtype.EventType) error { + rows, err := m.store.EventTypesOf(ctx, m.orgID, candidate.Source) + if err != nil { + return huma.Error500InternalServerError("couldn't list event types", err) + } + existing, err := typesOf(rows) + if err != nil { + return huma.Error500InternalServerError("couldn't read the event types", err) + } + if err := eventtype.CheckOverlap(existing, candidate); err != nil { + return huma.Error409Conflict("this rule can match events another type on the source already matches") + } + return nil +} + +type updateEventTypeInput struct { + ID string `path:"id" format:"uuid"` + Body struct { + Name string `json:"name"` + Rule eventtype.Rule `json:"rule"` + } +} + +type savedID struct { + ID string `json:"id"` +} + +func (m *Module) updateEventType(ctx context.Context, in *updateEventTypeInput) (*httpapi.Reply[savedID], error) { + id, err := uuid.Parse(in.ID) + if err != nil { + return nil, huma.Error400BadRequest("an event type id is a UUID") + } + stored, err := m.store.EventType(ctx, m.orgID, id) + if errors.Is(err, store.ErrNotFound) { + return nil, huma.Error404NotFound("no such event type") + } + if err != nil { + return nil, huma.Error500InternalServerError("couldn't read the event type", err) + } + t := eventtype.EventType{ID: in.ID, Source: stored.Source, Name: in.Body.Name, Rule: in.Body.Rule} + if err := t.Validate(); err != nil { + return nil, huma.Error400BadRequest(err.Error()) + } + if err := m.checkOverlap(ctx, t); err != nil { + return nil, err + } + rule, err := json.Marshal(t.Rule) + if err != nil { + return nil, huma.Error400BadRequest(err.Error()) + } + err = m.store.UpdateEventType(ctx, m.orgID, id, t.Name, rule) + if errors.Is(err, store.ErrExists) { + return nil, huma.Error409Conflict("the source already has an event type named " + t.Name) + } + if err != nil { + return nil, huma.Error500InternalServerError("couldn't save the event type", err) + } + return httpapi.OK(savedID{ID: in.ID}), nil +} + +func (m *Module) deleteEventType(ctx context.Context, in *idInput) (*struct{}, error) { + id, err := uuid.Parse(in.ID) + if err != nil { + return nil, huma.Error400BadRequest("an event type id is a UUID") + } + err = m.store.DeleteEventType(ctx, m.orgID, id) + switch { + case errors.Is(err, store.ErrNotFound): + return nil, huma.Error404NotFound("no such event type") + case errors.Is(err, store.ErrInUse): + return nil, huma.Error409Conflict("a mapping or trigger uses this event type") + case err != nil: + return nil, huma.Error500InternalServerError("couldn't delete the event type", err) + } + return nil, nil +} + +// mappingView is a mapping as the page lists it. +type mappingView struct { + ID uuid.UUID `json:"id"` + Name string `json:"name"` + SourceHint string `json:"source_hint"` + EventTypeID uuid.UUID `json:"event_type_id"` + Fields []mapping.Field `json:"fields"` + MatchCount int64 `json:"match_count"` + LastMatchedAt *time.Time `json:"last_matched_at,omitempty"` + CreatedAt time.Time `json:"created_at"` + UpdatedAt time.Time `json:"updated_at"` +} + +func mappingOf(r store.Mapping) (mappingView, error) { + v := mappingView{ + ID: r.ID, Name: r.Name, SourceHint: r.SourceHint, EventTypeID: r.EventTypeID, + MatchCount: r.MatchCount, CreatedAt: r.CreatedAt, UpdatedAt: r.UpdatedAt, + } + if err := json.Unmarshal(r.Fields, &v.Fields); err != nil { + return v, err + } + if r.LastMatchedAt.Valid { + v.LastMatchedAt = &r.LastMatchedAt.Time + } + return v, nil +} + +func (m *Module) mappings(ctx context.Context, _ *struct{}) (*httpapi.Reply[struct { + Mappings []mappingView `json:"mappings"` +}], error, +) { + rows, err := m.store.Mappings(ctx, m.orgID) + if err != nil { + return nil, huma.Error500InternalServerError("couldn't list mappings", err) + } + var out struct { + Mappings []mappingView `json:"mappings"` + } + out.Mappings = make([]mappingView, 0, len(rows)) + for _, r := range rows { + v, err := mappingOf(r) + if err != nil { + return nil, huma.Error500InternalServerError("couldn't read mapping "+r.Name, err) + } + out.Mappings = append(out.Mappings, v) + } + return httpapi.OK(out), nil +} + +func (m *Module) mapping(ctx context.Context, in *idInput) (*httpapi.Reply[mappingView], error) { + id, err := uuid.Parse(in.ID) + if err != nil { + return nil, huma.Error400BadRequest("a mapping id is a UUID") + } + row, err := m.store.Mapping(ctx, m.orgID, id) + if errors.Is(err, store.ErrNotFound) { + return nil, huma.Error404NotFound("no such mapping") + } + if err != nil { + return nil, huma.Error500InternalServerError("couldn't read the mapping", err) + } + v, err := mappingOf(row) + if err != nil { + return nil, huma.Error500InternalServerError("couldn't read the mapping", err) + } + return httpapi.OK(v), nil +} + +type mappingBody struct { + Name string `json:"name"` + SourceHint string `json:"source_hint,omitzero"` + EventTypeID string `json:"event_type_id" format:"uuid"` + Fields []mapping.Field `json:"fields"` +} + +type createMappingInput struct{ Body mappingBody } + +type updateMappingInput struct { + ID string `path:"id" format:"uuid"` + Body mappingBody +} + +// checkedMapping validates a request and fits its fields to the event type's +// schema. +func (m *Module) checkedMapping(ctx context.Context, b mappingBody) (store.Mapping, error) { + typeID, err := uuid.Parse(b.EventTypeID) + if err != nil { + return store.Mapping{}, huma.Error400BadRequest("an event type id is a UUID") + } + candidate := mapping.Mapping{Name: b.Name, Fields: b.Fields} + if err := candidate.Validate(); err != nil { + return store.Mapping{}, huma.Error400BadRequest(err.Error()) + } + stored, err := m.store.EventType(ctx, m.orgID, typeID) + if errors.Is(err, store.ErrNotFound) { + return store.Mapping{}, huma.Error400BadRequest("mapping: event type not found: " + b.EventTypeID) + } + if err != nil { + return store.Mapping{}, huma.Error500InternalServerError("couldn't read the event type", err) + } + t, err := typeOf(stored) + if err != nil { + return store.Mapping{}, huma.Error500InternalServerError("couldn't read the event type", err) + } + if err := mapping.CheckSchema(b.Fields, t.Schema); err != nil { + return store.Mapping{}, huma.Error400BadRequest(err.Error()) + } + fields, err := json.Marshal(b.Fields) + if err != nil { + return store.Mapping{}, huma.Error400BadRequest(err.Error()) + } + return store.Mapping{OrgID: m.orgID, Name: b.Name, SourceHint: b.SourceHint, EventTypeID: typeID, Fields: fields}, nil +} + +func (m *Module) createMapping(ctx context.Context, in *createMappingInput) (*httpapi.Reply[mappingView], error) { + row, err := m.checkedMapping(ctx, in.Body) + if err != nil { + return nil, err + } + row.ID = uuid.New() + if err := m.store.InsertMapping(ctx, row); err != nil { + return nil, huma.Error500InternalServerError("couldn't save the mapping", err) + } + return m.mappingReply(ctx, row.ID) +} + +func (m *Module) updateMapping(ctx context.Context, in *updateMappingInput) (*httpapi.Reply[mappingView], error) { + id, err := uuid.Parse(in.ID) + if err != nil { + return nil, huma.Error400BadRequest("a mapping id is a UUID") + } + row, err := m.checkedMapping(ctx, in.Body) + if err != nil { + return nil, err + } + row.ID = id + err = m.store.UpdateMapping(ctx, row) + if errors.Is(err, store.ErrNotFound) { + return nil, huma.Error404NotFound("no such mapping") + } + if err != nil { + return nil, huma.Error500InternalServerError("couldn't save the mapping", err) + } + return m.mappingReply(ctx, id) +} + +func (m *Module) mappingReply(ctx context.Context, id uuid.UUID) (*httpapi.Reply[mappingView], error) { + row, err := m.store.Mapping(ctx, m.orgID, id) + if err != nil { + return nil, huma.Error500InternalServerError("couldn't read the mapping", err) + } + v, err := mappingOf(row) + if err != nil { + return nil, huma.Error500InternalServerError("couldn't read the mapping", err) + } + return httpapi.OK(v), nil +} + +func (m *Module) deleteMapping(ctx context.Context, in *idInput) (*struct{}, error) { + id, err := uuid.Parse(in.ID) + if err != nil { + return nil, huma.Error400BadRequest("a mapping id is a UUID") + } + err = m.store.DeleteMapping(ctx, m.orgID, id) + switch { + case errors.Is(err, store.ErrNotFound): + return nil, huma.Error404NotFound("no such mapping") + case errors.Is(err, store.ErrInUse): + return nil, huma.Error409Conflict("a trigger uses this mapping") + case err != nil: + return nil, huma.Error500InternalServerError("couldn't delete the mapping", err) + } + return nil, nil +} + +type previewInput struct { + Body struct { + CaptureID string `json:"capture_id" format:"uuid"` + Fields []mapping.Field `json:"fields"` + } +} + +type previewBody struct { + Values map[string]any `json:"values"` + Failures []mapping.Failure `json:"failures"` +} + +// previewMapping reads a stored payload through candidate fields, so a mapping +// can be checked before it is saved. +func (m *Module) previewMapping(ctx context.Context, in *previewInput) (*httpapi.Reply[previewBody], error) { + id, err := uuid.Parse(in.Body.CaptureID) + if err != nil { + return nil, huma.Error400BadRequest("a capture id is a UUID") + } + e, err := m.store.Event(ctx, m.orgID, id) + if errors.Is(err, store.ErrNotFound) { + return nil, huma.Error404NotFound("no such capture") + } + if err != nil { + return nil, huma.Error500InternalServerError("couldn't read the capture", err) + } + values, failures := mapping.Resolve(in.Body.Fields, e.Body) + if failures == nil { + failures = []mapping.Failure{} + } + return httpapi.OK(previewBody{Values: values, Failures: failures}), nil +} diff --git a/modules/events/api_sources.go b/modules/events/api_sources.go new file mode 100644 index 000000000..d63060b3c --- /dev/null +++ b/modules/events/api_sources.go @@ -0,0 +1,341 @@ +package events + +import ( + "context" + "crypto/rand" + "encoding/hex" + "errors" + "fmt" + "net/http" + "net/textproto" + "strings" + + "github.com/danielgtaylor/huma/v2" + "golang.org/x/net/http/httpguts" + + "github.com/samcharles93/archie-core/internal/platform/httpapi" + "github.com/samcharles93/archie-core/modules/events/internal/store" + "github.com/samcharles93/archie-core/modules/governance" +) + +// registerSourceSettings adds the operations that change one setting of a +// source, and the trigger lifecycle. Approving an unsigned source or a +// trigger has its own grant, so a role that may edit them cannot also approve. +func (m *Module) registerSourceSettings(api huma.API) { + tags := []string{"Events"} + op := func(id, method, path, summary string) huma.Operation { + return huma.Operation{OperationID: id, Method: method, Path: path, Summary: summary, Tags: tags} + } + admin := httpapi.AdminAuthorized[governance.Action, governance.ResourceKind] + httpapi.Register(api, admin(governance.ActionUpdate, governance.KindSource), op("set-source-name", http.MethodPut, "/api/sources/{name}/name", "Label a source"), m.setSourceName) + httpapi.Register(api, admin(governance.ActionUpdate, governance.KindSource), op("set-source-delivery-header", http.MethodPut, "/api/sources/{name}/delivery-header", "Name the header a sender repeats when it retries a delivery"), m.setDeliveryHeader) + httpapi.Register(api, admin(governance.ActionUpdate, governance.KindSource), op("set-source-capture", http.MethodPut, "/api/sources/{name}/capture", "Keep or stop keeping the headers and payloads a source receives"), m.setCapture) + httpapi.Register(api, admin(governance.ActionUpdate, governance.KindSource), op("set-source-signing", http.MethodPost, "/api/sources/{name}/signing", "Require an HMAC signature, or ask to take unsigned deliveries"), m.setSigning) + httpapi.Register(api, admin(governance.ActionApprove, governance.KindSource), op("approve-source-unsigned", http.MethodPost, "/api/sources/{name}/approve-unsigned", "Accept deliveries that carry no signature"), m.approveUnsigned) + httpapi.Register(api, admin(governance.ActionReadSecret, governance.KindSecret), op("rotate-source-secret", http.MethodPost, "/api/sources/{name}/secret", "Replace a signed source's secret and return it once"), m.rotateSecret) + httpapi.Register(api, admin(governance.ActionDelete, governance.KindSource), op("delete-source", http.MethodDelete, "/api/sources/{name}", "Delete a source"), m.deleteSource) + + httpapi.Register(api, admin(governance.ActionApprove, governance.KindTrigger), op("approve-trigger", http.MethodPost, "/api/triggers/{name}/approve", "Arm a trigger that waits for approval"), m.approveTrigger) + httpapi.Register(api, admin(governance.ActionUpdate, governance.KindTrigger), op("pause-trigger", http.MethodPost, "/api/triggers/{name}/pause", "Stop a trigger firing"), m.pauseTrigger) + httpapi.Register(api, admin(governance.ActionUpdate, governance.KindTrigger), op("resume-trigger", http.MethodPost, "/api/triggers/{name}/resume", "Return a paused trigger to waiting for approval"), m.resumeTrigger) + httpapi.Register(api, admin(governance.ActionDelete, governance.KindTrigger), op("delete-trigger", http.MethodDelete, "/api/triggers/{name}", "Delete a trigger"), m.deleteTrigger) + httpapi.Register(api, admin(governance.ActionRead, governance.KindTrigger), op("list-trigger-dispatches", http.MethodGet, "/api/triggers/{name}/dispatches", "List the runs a trigger started"), m.triggerDispatches) +} + +type sourceInput struct { + Name string `path:"name"` +} + +// Signing states a source reports: an HMAC signature, a bearer token, or no +// verification, which waits for approval before it takes deliveries. +const ( + signingSigned = "signed" + signingToken = "token" + signingPending = "unsigned_pending_approval" + signingUnsigned = "unsigned" +) + +func signingOf(src store.Source) string { + switch { + case src.Verification == store.HMAC: + return signingSigned + case src.Verification == store.Token: + return signingToken + case src.UnsignedApproved: + return signingUnsigned + } + return signingPending +} + +// readSource loads a source for a settings change. +func (m *Module) readSource(ctx context.Context, name string) (store.Source, error) { + src, err := m.store.Source(ctx, m.orgID, name) + if errors.Is(err, store.ErrNotFound) { + return src, huma.Error404NotFound("no such source") + } + if err != nil { + return src, huma.Error500InternalServerError("couldn't read the source", err) + } + return src, nil +} + +// sourceReply reads a source back after a change. secret is a signing secret +// issued by the change, returned once. +func (m *Module) sourceReply(ctx context.Context, name, secret string) (*httpapi.Reply[sourceView], error) { + src, err := m.readSource(ctx, name) + if err != nil { + return nil, err + } + view, err := viewOfSource(src) + if err != nil { + return nil, huma.Error500InternalServerError("couldn't read the key fields of "+name, err) + } + view.Secret = secret + return httpapi.OK(view), nil +} + +type sourceNameInput struct { + Name string `path:"name"` + Body struct { + Name string `json:"name" maxLength:"100"` + } +} + +func (m *Module) setSourceName(ctx context.Context, in *sourceNameInput) (*httpapi.Reply[sourceView], error) { + err := m.store.SetDisplayName(ctx, m.orgID, in.Name, strings.TrimSpace(in.Body.Name)) + if err := sourceWriteError(err); err != nil { + return nil, err + } + return m.sourceReply(ctx, in.Name, "") +} + +type deliveryHeaderInput struct { + Name string `path:"name"` + Body struct { + Header string `json:"header" doc:"A header name; empty stops de-duplicating retries by header"` + } +} + +func (m *Module) setDeliveryHeader(ctx context.Context, in *deliveryHeaderInput) (*httpapi.Reply[sourceView], error) { + header := strings.TrimSpace(in.Body.Header) + if header != "" { + if len(header) > 128 || !httpguts.ValidHeaderFieldName(header) { + return nil, huma.Error400BadRequest("the delivery header must be an HTTP header name") + } + header = textproto.CanonicalMIMEHeaderKey(header) + } + err := m.store.SetDeliveryHeader(ctx, m.orgID, in.Name, header) + if err := sourceWriteError(err); err != nil { + return nil, err + } + return m.sourceReply(ctx, in.Name, "") +} + +type captureInput struct { + Name string `path:"name"` + Body struct { + Capture bool `json:"capture"` + } +} + +// setCapture turns the keeping of a source's payloads on or off. A trigger +// that maps the source's payload needs the stored payload, so it blocks +// turning capture off. +func (m *Module) setCapture(ctx context.Context, in *captureInput) (*httpapi.Reply[sourceView], error) { + if !in.Body.Capture { + n, err := m.store.MappedTriggers(ctx, m.orgID, in.Name) + if err != nil { + return nil, huma.Error500InternalServerError("couldn't check the triggers of the source", err) + } + if n > 0 { + return nil, huma.Error409Conflict(fmt.Sprintf("%d trigger(s) of this source read its payload through a mapping, which needs it kept", n)) + } + } + err := m.store.SetCapture(ctx, m.orgID, in.Name, in.Body.Capture) + if err := sourceWriteError(err); err != nil { + return nil, err + } + return m.sourceReply(ctx, in.Name, "") +} + +type signingInput struct { + Name string `path:"name"` + Body struct { + Signed bool `json:"signed" doc:"True requires an HMAC signature; false asks to take deliveries without one, which a person must approve"` + } +} + +func (m *Module) setSigning(ctx context.Context, in *signingInput) (*httpapi.Reply[sourceView], error) { + src, err := m.readSource(ctx, in.Name) + if err != nil { + return nil, err + } + secret := "" + if in.Body.Signed { + if src.HMACHeader == "" { + return nil, huma.Error409Conflict("set the signature header first: edit the source and choose HMAC signature") + } + if len(src.HMACSecretSealed) == 0 { + if secret, err = m.issueSecret(ctx, src); err != nil { + return nil, err + } + } + } + if err := sourceWriteError(m.store.SetSigning(ctx, m.orgID, in.Name, in.Body.Signed)); err != nil { + return nil, err + } + return m.sourceReply(ctx, in.Name, secret) +} + +// approveUnsigned accepts that a source takes deliveries with no verification. +func (m *Module) approveUnsigned(ctx context.Context, in *sourceInput) (*httpapi.Reply[sourceView], error) { + approved, err := m.store.ApproveUnsigned(ctx, m.orgID, in.Name) + if err != nil { + return nil, huma.Error500InternalServerError("couldn't approve the source", err) + } + if !approved { + if _, err := m.readSource(ctx, in.Name); err != nil { + return nil, err + } + return nil, huma.Error409Conflict("the source isn't waiting for approval of unsigned deliveries") + } + return m.sourceReply(ctx, in.Name, "") +} + +func (m *Module) rotateSecret(ctx context.Context, in *sourceInput) (*httpapi.Reply[sourceView], error) { + src, err := m.readSource(ctx, in.Name) + if err != nil { + return nil, err + } + if src.HMACHeader == "" { + return nil, huma.Error409Conflict("only a source with an HMAC signature header has a secret") + } + secret, err := m.issueSecret(ctx, src) + if err != nil { + return nil, err + } + return m.sourceReply(ctx, in.Name, secret) +} + +// issueSecret generates a signing secret, stores it sealed and returns it. +func (m *Module) issueSecret(ctx context.Context, src store.Source) (string, error) { + secret, err := newSecret() + if err != nil { + return "", huma.Error500InternalServerError("couldn't generate a secret", err) + } + sealed, err := m.sealer.Seal(secretLabel(m.orgID, src.Name), []byte(secret)) + if err != nil { + return "", huma.Error500InternalServerError("couldn't seal the secret", err) + } + if err := m.store.SetSecret(ctx, m.orgID, src.Name, sealed); err != nil { + return "", huma.Error500InternalServerError("couldn't save the secret", err) + } + return secret, nil +} + +// newSecret returns a 256-bit signing secret in hex. +func newSecret() (string, error) { + b := make([]byte, 32) + if _, err := rand.Read(b); err != nil { + return "", fmt.Errorf("generate secret: %w", err) + } + return hex.EncodeToString(b), nil +} + +func sourceWriteError(err error) error { + switch { + case err == nil: + return nil + case errors.Is(err, store.ErrNotFound): + return huma.Error404NotFound("no such source") + } + return huma.Error500InternalServerError("couldn't save the source", err) +} + +// deleteSource removes a source. A trigger that names it blocks the delete; +// its event types go with it, and the events it stored stay. +func (m *Module) deleteSource(ctx context.Context, in *sourceInput) (*struct{}, error) { + err := m.store.DeleteSource(ctx, m.orgID, in.Name) + switch { + case errors.Is(err, store.ErrNotFound): + return nil, huma.Error404NotFound("no such source") + case errors.Is(err, store.ErrInUse): + return nil, huma.Error409Conflict("the source is still in use: " + err.Error()) + case err != nil: + return nil, huma.Error500InternalServerError("couldn't delete the source", err) + } + return nil, nil +} + +type triggerInput struct { + Name string `path:"name"` +} + +// moveTrigger changes a trigger's state when it is in one of from, and refuses +// otherwise. +func (m *Module) moveTrigger(ctx context.Context, name string, allowed func(store.Trigger) bool, refusal string, approved, enabled bool) (*httpapi.Reply[triggerView], error) { + t, err := m.store.Trigger(ctx, m.orgID, name) + if errors.Is(err, store.ErrNotFound) { + return nil, huma.Error404NotFound("no such trigger") + } + if err != nil { + return nil, huma.Error500InternalServerError("couldn't read the trigger", err) + } + if !allowed(t) { + return nil, huma.Error409Conflict(refusal) + } + if _, err := m.store.SetTriggerState(ctx, m.orgID, name, approved, enabled); err != nil { + return nil, huma.Error500InternalServerError("couldn't save the trigger", err) + } + t.Approved, t.Enabled = approved, enabled + view, err := m.viewOfTrigger(ctx, t) + if err != nil { + return nil, huma.Error500InternalServerError("couldn't read the trigger", err) + } + return httpapi.OK(view), nil +} + +func (m *Module) approveTrigger(ctx context.Context, in *triggerInput) (*httpapi.Reply[triggerView], error) { + return m.moveTrigger(ctx, in.Name, func(t store.Trigger) bool { return !t.Approved }, "the trigger isn't waiting for approval", true, true) +} + +func (m *Module) pauseTrigger(ctx context.Context, in *triggerInput) (*httpapi.Reply[triggerView], error) { + return m.moveTrigger(ctx, in.Name, func(t store.Trigger) bool { return t.Approved && t.Enabled }, "only an armed trigger can be paused", true, false) +} + +// resumeTrigger returns a paused trigger to waiting for approval, so a person +// approves it again before it fires. +func (m *Module) resumeTrigger(ctx context.Context, in *triggerInput) (*httpapi.Reply[triggerView], error) { + return m.moveTrigger(ctx, in.Name, func(t store.Trigger) bool { return t.Approved && !t.Enabled }, "only a paused trigger can be resumed", false, true) +} + +// deleteTrigger removes a trigger. It cascades to nothing: the runs it already +// started continue, and the dispatch records that link them to their events stay +// under the trigger's name. +func (m *Module) deleteTrigger(ctx context.Context, in *triggerInput) (*struct{}, error) { + err := m.store.DeleteTrigger(ctx, m.orgID, in.Name) + switch { + case errors.Is(err, store.ErrNotFound): + return nil, huma.Error404NotFound("no such trigger") + case err != nil: + return nil, huma.Error500InternalServerError("couldn't delete the trigger", err) + } + return nil, nil +} + +type triggerDispatchesInput struct { + Name string `path:"name"` + Limit int `query:"limit" minimum:"1" maximum:"200" default:"50"` +} + +func (m *Module) triggerDispatches(ctx context.Context, in *triggerDispatchesInput) (*httpapi.Reply[[]dispatchView], error) { + rows, err := m.store.DispatchesOf(ctx, m.orgID, in.Name, in.Limit) + if err != nil { + return nil, huma.Error500InternalServerError("couldn't list the runs of the trigger", err) + } + out := make([]dispatchView, 0, len(rows)) + for _, r := range rows { + out = append(out, dispatchView{EventID: r.EventID, RunID: r.RunID, Trigger: in.Name, CreatedAt: r.CreatedAt}) + } + return httpapi.OK(out), nil +} diff --git a/modules/events/capture.go b/modules/events/capture.go new file mode 100644 index 000000000..0de76c7db --- /dev/null +++ b/modules/events/capture.go @@ -0,0 +1,147 @@ +package events + +import ( + "context" + json "encoding/json/v2" + "fmt" + "net/http" + "net/textproto" + "strconv" + "strings" + + "github.com/google/uuid" + + "github.com/samcharles93/archie-core/modules/events/internal/eventtype" + "github.com/samcharles93/archie-core/modules/events/internal/store" +) + +// arrival is what a webhook request adds to the events it carried: how it was +// verified, who sent it and the headers it came with. A listening connector's +// events have none of it. +type arrival struct { + verification string + addr string + headers http.Header +} + +// typeReader lists a source's event types. +type typeReader interface { + EventTypesOf(ctx context.Context, orgID, source string) ([]store.EventType, error) +} + +// credentialWords mark a header as carrying a credential or a signature, which +// are never stored. +var credentialWords = []string{"authorization", "cookie", "token", "secret", "password", "key", "signature"} + +// storable returns the headers worth keeping with a capture: everything except +// credentials and the source's own signature header. +func storable(h http.Header, src store.Source) http.Header { + out := http.Header{} + own := textproto.CanonicalMIMEHeaderKey(src.HMACHeader) + for name, values := range h { + lower := strings.ToLower(name) + if name == own || containsAny(lower, credentialWords) { + continue + } + out[name] = values + } + return out +} + +func containsAny(s string, words []string) bool { + for _, w := range words { + if strings.Contains(s, w) { + return true + } + } + return false +} + +// decorate fills in what a webhook request adds to an event row. The raw +// headers and body are kept only when the source captures them. The event +// type is identified from the headers and body before either is dropped. +func (m *Module) decorate(ctx context.Context, row *store.Event, src store.Source, a arrival, index int) error { + row.Verification, row.RemoteAddr = a.verification, a.addr + if a.headers == nil { + return nil + } + if id := strings.TrimSpace(a.headers.Get(src.DeliveryHeader)); src.DeliveryHeader != "" && id != "" { + // A retry repeats the delivery id, so its events are the ones already stored. + row.SourceEventID = "delivery:" + id + ":" + strconv.Itoa(index) + } + if err := m.identify(ctx, row, src, a.headers); err != nil { + return err + } + if !src.Capture { + row.Body = nil + return nil + } + raw, err := json.Marshal(storable(a.headers, src)) + if err != nil { + return fmt.Errorf("encode headers: %w", err) + } + row.Headers = raw + return nil +} + +// identify names the event type that the delivery matches. It names none when +// no type or several types match. +func (m *Module) identify(ctx context.Context, row *store.Event, src store.Source, h http.Header) error { + if m.types == nil { + return nil + } + rows, err := m.types.EventTypesOf(ctx, row.OrgID, src.Name) + if err != nil { + return err + } + types, err := typesOf(rows) + if err != nil { + return err + } + sample := eventtype.Sample{Headers: lowered(h), Body: row.Body} + if found, ok := eventtype.Identify(types, src.Name, sample); ok { + id, err := uuid.Parse(found.ID) + if err != nil { + return fmt.Errorf("event type id %q: %w", found.ID, err) + } + row.EventTypeID = uuid.NullUUID{UUID: id, Valid: true} + } + return nil +} + +// lowered returns each header's first value under its lower-case name. +func lowered(h http.Header) map[string]string { + out := make(map[string]string, len(h)) + for name, values := range h { + if len(values) > 0 { + out[strings.ToLower(name)] = values[0] + } + } + return out +} + +// typesOf converts stored event types to the form rules are evaluated in. +func typesOf(rows []store.EventType) ([]eventtype.EventType, error) { + out := make([]eventtype.EventType, 0, len(rows)) + for _, r := range rows { + t, err := typeOf(r) + if err != nil { + return nil, err + } + out = append(out, t) + } + return out, nil +} + +func typeOf(r store.EventType) (eventtype.EventType, error) { + t := eventtype.EventType{ID: r.ID.String(), Source: r.Source, Name: r.Name} + if err := json.Unmarshal(r.Rule, &t.Rule); err != nil { + return t, fmt.Errorf("event type %q rule: %w", r.Name, err) + } + if len(r.Schema) > 0 { + if err := json.Unmarshal(r.Schema, &t.Schema); err != nil { + return t, fmt.Errorf("event type %q schema: %w", r.Name, err) + } + } + return t, nil +} diff --git a/modules/events/capture_test.go b/modules/events/capture_test.go new file mode 100644 index 000000000..57db3ecb2 --- /dev/null +++ b/modules/events/capture_test.go @@ -0,0 +1,149 @@ +package events + +import ( + "context" + "log/slog" + "maps" + "net/http" + "net/http/httptest" + "strings" + "testing" + + "filippo.io/age" + + "github.com/samcharles93/archie-core/internal/platform/httpapi" + "github.com/samcharles93/archie-core/internal/platform/httpjson" + "github.com/samcharles93/archie-core/internal/platform/seal" + "github.com/samcharles93/archie-core/modules/connectors" + "github.com/samcharles93/archie-core/modules/events/internal/store" + "github.com/samcharles93/archie-core/modules/governance" +) + +// rowRecorder keeps every event row intake stores. +type rowRecorder struct{ rows []store.Event } + +func (r *rowRecorder) Insert(_ context.Context, e store.Event) (bool, error) { + r.rows = append(r.rows, e) + return true, nil +} + +// TestIntakeRecordsHowEachDeliveryWasVerified guards three things about a +// webhook delivery: each stored event records the scheme that verified it, a +// source that checks nothing takes no deliveries until a person approved that, +// and the kept headers never include a credential or the signature itself. +func TestIntakeRecordsHowEachDeliveryWasVerified(t *testing.T) { + const secret = "not-a-real-secret" //nolint:gosec // fixture signing secret the verifier must accept + const token = "not-a-real-token" //nolint:gosec // fixture bearer token the verifier must accept + const body = `{"id":"1"}` + id, err := age.GenerateX25519Identity() + if err != nil { + t.Fatal(err) + } + sealer, err := seal.New(id.String(), nil) + if err != nil { + t.Fatal(err) + } + sealed, err := sealer.Seal(secretLabel("org", "hook"), []byte(secret)) + if err != nil { + t.Fatal(err) + } + base := store.Source{OrgID: "org", Name: "hook", Connector: "c", Capture: true} + hmacSource := base + hmacSource.Verification, hmacSource.HMACHeader, hmacSource.HMACPrefix, hmacSource.HMACSecretSealed = store.HMAC, "X-Signature", "sha256=", sealed + tokenSource := base + tokenSource.Verification, tokenSource.TokenSHA256 = store.Token, httpjson.TokenHash(token) + pending, approved, unkept := base, base, hmacSource + pending.Verification, approved.Verification, approved.UnsignedApproved = store.None, store.None, true + unkept.Capture = false + + tests := []struct { + name string + src store.Source + header http.Header + wantCode int + wantVerdict string + wantKept bool + }{ + {"a signed delivery", hmacSource, http.Header{"X-Signature": {"sha256=" + sign(secret, body)}}, http.StatusAccepted, store.Signed, true}, + {"a token delivery", tokenSource, http.Header{"Authorization": {"Bearer " + token}}, http.StatusAccepted, store.TokenVerified, true}, + {"an approved unsigned source", approved, http.Header{}, http.StatusAccepted, store.Unsigned, true}, + {"an unsigned source nobody approved", pending, http.Header{}, http.StatusUnauthorized, "", false}, + {"a source that keeps no payloads", unkept, http.Header{"X-Signature": {"sha256=" + sign(secret, body)}}, http.StatusAccepted, store.Signed, false}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + rows := &rowRecorder{} + parsed := false + m := &Module{ + orgID: "org", reader: fakeSources{tt.src}, inserter: rows, sealer: sealer, delivery: permitAll{}, + deliveries: noEcho{}, log: slog.New(slog.DiscardHandler), + parse: func(context.Context, string, []byte, map[string]string) ([]connectors.Event, error) { + parsed = true + return []connectors.Event{{Type: "t", SourceEventID: "e1"}}, nil + }, + } + req := httptest.NewRequestWithContext(t.Context(), http.MethodPost, "/hooks/hook", strings.NewReader(body)) + req.SetPathValue("source", "hook") + maps.Copy(req.Header, tt.header) + req.Header.Set("Cookie", "session=not-a-real-cookie") + rec := httptest.NewRecorder() + m.intake(rec, req) + if rec.Code != tt.wantCode { + t.Fatalf("status %d, want %d", rec.Code, tt.wantCode) + } + if tt.wantCode != http.StatusAccepted { + if parsed || len(rows.rows) != 0 { + t.Fatalf("a refused delivery was parsed (%v) or stored (%d rows)", parsed, len(rows.rows)) + } + return + } + if len(rows.rows) != 1 { + t.Fatalf("%d rows stored, want 1", len(rows.rows)) + } + row := rows.rows[0] + if row.Verification != tt.wantVerdict { + t.Fatalf("recorded verification %q, want %q", row.Verification, tt.wantVerdict) + } + if kept := row.Headers != nil && row.Body != nil; kept != tt.wantKept { + t.Fatalf("headers and body kept = %v, want %v", kept, tt.wantKept) + } + for _, secretHeader := range []string{"Authorization", "Cookie", "X-Signature", "not-a-real"} { + if strings.Contains(string(row.Headers), secretHeader) { + t.Fatalf("kept headers %s contain %q", row.Headers, secretHeader) + } + } + }) + } +} + +type adminCaller struct{} + +func (adminCaller) Authenticate(ctx context.Context, _ *http.Request) (context.Context, bool, error) { + return ctx, true, nil +} + +// TestApprovalsNeedTheirOwnGrant guards the separation between asking and +// approving: a role that may edit a source or trigger can't also approve what +// it asked for, because approving unsigned deliveries and arming a trigger are +// declared under the approve action and turning signing off is not. +func TestApprovalsNeedTheirOwnGrant(t *testing.T) { + api := httpapi.New(http.NewServeMux(), "test", adminCaller{}, nil) + (&Module{}).Register(api) + got := map[string][2]string{} + for _, r := range httpapi.Routes(api) { + if action, kind, ok := r.Access.Declared(); ok { + got[r.ID] = [2]string{action, kind} + } + } + for id, want := range map[string][2]string{ + "approve-source-unsigned": {string(governance.ActionApprove), string(governance.KindSource)}, + "approve-trigger": {string(governance.ActionApprove), string(governance.KindTrigger)}, + "set-source-signing": {string(governance.ActionUpdate), string(governance.KindSource)}, + "resume-trigger": {string(governance.ActionUpdate), string(governance.KindTrigger)}, + "rotate-source-secret": {string(governance.ActionReadSecret), string(governance.KindSecret)}, + } { + if got[id] != want { + t.Errorf("%s declares %v, want %v", id, got[id], want) + } + } +} diff --git a/modules/events/drafting.go b/modules/events/drafting.go index 88b418d51..b63386a68 100644 --- a/modules/events/drafting.go +++ b/modules/events/drafting.go @@ -90,6 +90,6 @@ func (m *Module) SaveTrigger(ctx context.Context, t TriggerSpec) error { } return m.store.UpsertTrigger(ctx, store.Trigger{ OrgID: m.orgID, Name: t.Name, Source: t.Source, EventType: t.EventType, - Filter: filter, Workflow: t.Workflow, Enabled: true, + Filter: filter, Workflow: t.Workflow, Enabled: true, Approved: true, }) } diff --git a/modules/events/events.go b/modules/events/events.go index c4ca4065b..8efbee4c4 100644 --- a/modules/events/events.go +++ b/modules/events/events.go @@ -29,6 +29,7 @@ import ( "github.com/samcharles93/archie-core/internal/platform/httpjson" "github.com/samcharles93/archie-core/modules/approvals" "github.com/samcharles93/archie-core/modules/connectors" + "github.com/samcharles93/archie-core/modules/events/internal/mapping" "github.com/samcharles93/archie-core/modules/events/internal/match" "github.com/samcharles93/archie-core/modules/events/internal/store" "github.com/samcharles93/archie-core/modules/governance" @@ -91,6 +92,7 @@ type Module struct { store *store.Store reader sourceReader inserter eventInserter + types typeReader delivery governance.DeliveryAuthorizer sealer Sealer parse func(ctx context.Context, connector string, body []byte, headers map[string]string) ([]connectors.Event, error) @@ -106,7 +108,7 @@ type Module struct { // NewModule returns the events module for orgID. func NewModule(db *sqlx.DB, log *slog.Logger, orgID string, sealer Sealer, c Connectors, r Runs, a Answers, l Links, d Deliveries) *Module { st := store.New(db) - m := &Module{orgID: orgID, store: st, reader: st, inserter: st, sealer: sealer, connectors: c, runs: r, answers: a, links: l, deliveries: d, log: log, poll: time.Second} + m := &Module{orgID: orgID, store: st, reader: st, inserter: st, types: st, sealer: sealer, connectors: c, runs: r, answers: a, links: l, deliveries: d, log: log, poll: time.Second} m.parse = m.parseWith return m } @@ -137,12 +139,12 @@ func (m *Module) Routes(mux *http.ServeMux) { // redelivered event violates a uniqueness constraint, is counted as a // duplicate and is not dispatched again. func (m *Module) intake(w http.ResponseWriter, r *http.Request) { - src, parsed, body, status, err := m.accept(r) + d, status, err := m.accept(r) if err != nil { httpjson.Error(w, status, err.Error()) return } - stored, duplicates, err := m.save(r.Context(), src, parsed, body) + stored, duplicates, err := m.save(r.Context(), d.src, d.events, d.body, d.arrival) if err != nil { httpjson.Error(w, http.StatusServiceUnavailable, err.Error()) return @@ -150,51 +152,70 @@ func (m *Module) intake(w http.ResponseWriter, r *http.Request) { httpjson.Write(w, http.StatusAccepted, map[string]int{"stored": stored, "duplicates": duplicates}) } +// delivery is a webhook delivery that passed verification and was parsed. +type delivery struct { + src store.Source + events []connectors.Event + body []byte + arrival arrival +} + // accept verifies a delivery against its source's scheme, then passes it to // the connector for parsing. A delivery that fails verification is rejected // with 401 before the connector sees its body, and the rejection is logged. // An unknown source and a failed verification get the same response, so a // caller cannot use it to discover source names. -func (m *Module) accept(r *http.Request) (src store.Source, parsed []connectors.Event, body []byte, status int, err error) { +func (m *Module) accept(r *http.Request) (d delivery, status int, err error) { ctx := r.Context() - src, err = m.reader.Source(ctx, m.orgID, r.PathValue("source")) + src, err := m.reader.Source(ctx, m.orgID, r.PathValue("source")) if errors.Is(err, store.ErrNotFound) { - return src, nil, nil, http.StatusUnauthorized, errUnknown + return d, http.StatusUnauthorized, errUnknown } if err != nil { - return src, nil, nil, http.StatusServiceUnavailable, errors.New("events store unavailable") + return d, http.StatusServiceUnavailable, errors.New("events store unavailable") } + d.src = src // A token is checked before the body is read. A signature covers the body, - // so it is checked on the raw bytes as soon as they are read. + // so it is checked on the raw bytes as soon as they are read. A source that + // checks nothing takes deliveries only once a person approved that. switch src.Verification { case store.Token: if err := checkToken(src, r); err != nil { - return m.refuse(ctx, src, err) + return m.refuse(ctx, d, err) } - case store.HMAC, store.None: + d.arrival.verification = store.TokenVerified + case store.HMAC: + d.arrival.verification = store.Signed + case store.None: + if !src.UnsignedApproved { + return m.refuse(ctx, d, fmt.Errorf("%w: the source isn't approved for unsigned deliveries", errUnverified)) + } + d.arrival.verification = store.Unsigned default: - return m.refuse(ctx, src, fmt.Errorf("unknown verification %q", src.Verification)) + return m.refuse(ctx, d, fmt.Errorf("unknown verification %q", src.Verification)) } - body, err = httpjson.Body(r) + d.body, err = httpjson.Body(r) if err != nil { - return src, nil, nil, http.StatusRequestEntityTooLarge, err + return d, http.StatusRequestEntityTooLarge, err } if src.Verification == store.HMAC { - if err := m.verifyHMAC(src, r.Header, body); err != nil { - return m.refuse(ctx, src, err) + if err := m.verifyHMAC(src, r.Header, d.body); err != nil { + return m.refuse(ctx, d, err) } } if status, err := m.deliveryAllowed(ctx, src, r); err != nil { - return src, nil, nil, status, err + return d, status, err } - parsed, err = m.parse(ctx, src.Connector, body, map[string]string{"Content-Type": r.Header.Get("Content-Type")}) + d.events, err = m.parse(ctx, src.Connector, d.body, map[string]string{"Content-Type": r.Header.Get("Content-Type")}) if errors.Is(err, errConnectorDown) { - return src, nil, nil, http.StatusServiceUnavailable, err + return d, http.StatusServiceUnavailable, err } if err != nil { - return src, nil, nil, http.StatusBadRequest, err + return d, http.StatusBadRequest, err } - return src, parsed, body, 0, nil + d.arrival.addr = senderAddr(r) + d.arrival.headers = r.Header + return d, 0, nil } var ( @@ -204,12 +225,12 @@ var ( // refuse logs why a delivery was rejected and chooses the status: 401 when // verification failed, 503 when verification could not run. -func (m *Module) refuse(ctx context.Context, src store.Source, err error) (store.Source, []connectors.Event, []byte, int, error) { - m.log.WarnContext(ctx, "refused webhook delivery", "source", src.Name, "verification", src.Verification, "reason", err.Error()) +func (m *Module) refuse(ctx context.Context, d delivery, err error) (delivery, int, error) { + m.log.WarnContext(ctx, "refused webhook delivery", "source", d.src.Name, "verification", d.src.Verification, "reason", err.Error()) if errors.Is(err, errUnverified) { - return src, nil, nil, http.StatusUnauthorized, errUnknown + return d, http.StatusUnauthorized, errUnknown } - return src, nil, nil, http.StatusServiceUnavailable, errors.New("couldn't verify the delivery") + return d, http.StatusServiceUnavailable, errors.New("couldn't verify the delivery") } var ( @@ -224,10 +245,7 @@ func (m *Module) deliveryAllowed(ctx context.Context, src store.Source, r *http. if m.delivery == nil { return http.StatusServiceUnavailable, errDeliveryUnavailable } - addr, _, err := net.SplitHostPort(r.RemoteAddr) - if err != nil { - addr = r.RemoteAddr - } + addr := senderAddr(r) // Delivery policies belong to the operating organisation, whatever the // source's own org id says. d := m.delivery.AuthorizeDelivery(ctx, governance.DefaultOrgID, src.Name, addr) @@ -242,6 +260,15 @@ func (m *Module) deliveryAllowed(ctx context.Context, src store.Source, r *http. return http.StatusForbidden, errDeliveryDenied } +// senderAddr is the address a request came from, without its port. +func senderAddr(r *http.Request) string { + addr, _, err := net.SplitHostPort(r.RemoteAddr) + if err != nil { + return r.RemoteAddr + } + return addr +} + // parseWith has a running connector parse a delivery. func (m *Module) parseWith(ctx context.Context, connector string, body []byte, headers map[string]string) ([]connectors.Event, error) { conn, ok := m.connectors.Get(connector) @@ -253,13 +280,17 @@ func (m *Module) parseWith(ctx context.Context, connector string, body []byte, h // save stores each event with the payload it arrived in, recording at arrival // whether it echoes a delivery Archie sent, so those never reach grouping. -func (m *Module) save(ctx context.Context, src store.Source, parsed []connectors.Event, body []byte) (stored, duplicates int, err error) { - for _, e := range parsed { +func (m *Module) save(ctx context.Context, src store.Source, parsed []connectors.Event, body []byte, a arrival) (stored, duplicates int, err error) { + for i, e := range parsed { run, own, err := m.deliveries.SentBy(ctx, src.Connector, e.SourceEventID, e.LinkKey) if err != nil { return stored, duplicates, fmt.Errorf("check delivery provenance: %w", err) } - isNew, err := m.inserter.Insert(ctx, newRow(m.orgID, src, e, body, own, run)) + row := newRow(m.orgID, src, e, body, own, run) + if err := m.decorate(ctx, &row, src, a, i); err != nil { + return stored, duplicates, err + } + isNew, err := m.inserter.Insert(ctx, row) if err != nil { return stored, duplicates, err } @@ -334,7 +365,7 @@ func (m *Module) listen(ctx context.Context, src store.Source) { err = fmt.Errorf("connector %q isn't running", src.Connector) } else { err = conn.Listen(ctx, cursor, func(events []connectors.Event, next string) error { - if _, _, err := m.save(ctx, src, events, received(events)); err != nil { + if _, _, err := m.save(ctx, src, events, received(events), arrival{}); err != nil { return err } if err := m.store.SetCursor(ctx, m.orgID, src.Name, next); err != nil { @@ -477,32 +508,22 @@ func (m *Module) dispatch(tx *sqlx.Tx, e store.Event) (bool, error) { if err != nil || linked { return false, err } - triggers, err := store.Triggers(ctx, tx, e.OrgID, e.Source, e.Type) + triggers, err := store.Triggers(ctx, tx, e.OrgID, e.Source, e.Type, e.EventTypeID) if err != nil { return false, err } matched, refused := false, false for _, t := range triggers { - var filter map[string]string - if err := json.Unmarshal(t.Filter, &filter); err != nil { - return false, fmt.Errorf("trigger %q filter: %w", t.Name, err) - } - if !match.Fields(e.Fields, filter) { - continue - } - runID, started, err := m.startRun(ctx, e, t) + outcome, err := m.fire(ctx, tx, e, t) if err != nil { return false, err } - if !started { + switch outcome { + case fired: + matched = true + case refusedRun: refused = true - continue } - if err := store.RecordDispatch(ctx, tx, e.ID, t.Name, runID); err != nil { - return false, err - } - m.log.InfoContext(ctx, "trigger started run", "event", e.ID, "trigger", t.Name, "run", runID) - matched = true } if !matched && !refused { return false, m.noteUnmatched(ctx, tx, e) @@ -510,6 +531,46 @@ func (m *Module) dispatch(tx *sqlx.Tx, e store.Event) (bool, error) { return matched, nil } +// What a trigger did with an event: left it (its filter didn't match), had its +// run refused, or started a run. +type outcome int + +const ( + skipped outcome = iota + refusedRun + fired +) + +// fire starts the run trigger t asks for when e matches its filter, and +// records the dispatch. +func (m *Module) fire(ctx context.Context, tx *sqlx.Tx, e store.Event, t store.Trigger) (outcome, error) { + var filter map[string]string + if err := json.Unmarshal(t.Filter, &filter); err != nil { + return skipped, fmt.Errorf("trigger %q filter: %w", t.Name, err) + } + if !match.Fields(e.Fields, filter) { + return skipped, nil + } + input, mapped, ok, err := m.inputFor(ctx, tx, e, t) + if err != nil || !ok { + return refusedRun, err + } + runID, started, err := m.startRun(ctx, e, t, input) + if err != nil || !started { + return refusedRun, err + } + if mapped { + if err := store.CountMatch(ctx, tx, t.MappingID.UUID); err != nil { + return skipped, err + } + } + if err := store.RecordDispatch(ctx, tx, e.ID, t.Name, runID); err != nil { + return skipped, err + } + m.log.InfoContext(ctx, "trigger started run", "event", e.ID, "trigger", t.Name, "run", runID) + return fired, nil +} + // noteUnmatched files an event no trigger took into a group for review. func (m *Module) noteUnmatched(ctx context.Context, tx *sqlx.Tx, e store.Event) error { created, err := store.NoteUnmatched(ctx, tx, e) @@ -527,10 +588,10 @@ func (m *Module) noteUnmatched(ctx context.Context, tx *sqlx.Tx, e store.Event) // is final: it reports no run and no error, and the trigger is skipped. A policy // chain that can't decide returns an error instead, which rolls the dispatch // back so the event stays queued and is dispatched again. -func (m *Module) startRun(ctx context.Context, e store.Event, t store.Trigger) (runID uuid.UUID, started bool, err error) { +func (m *Module) startRun(ctx context.Context, e store.Event, t store.Trigger, input jsontext.Value) (runID uuid.UUID, started bool, err error) { ctx = governance.WithPrincipal(ctx, governance.WorkloadPrincipal(governance.DefaultOrgID)) runID, err = m.runs.Start(ctx, runs.StartRequest{ - OrgID: e.OrgID, Workflow: t.Workflow, Key: e.ID.String() + ":" + t.Name, Input: runInput(e), + OrgID: e.OrgID, Workflow: t.Workflow, Key: e.ID.String() + ":" + t.Name, Input: input, }) if errors.Is(err, governance.ErrDenied) { m.log.WarnContext(ctx, "trigger not permitted to start its workflow", "event", e.ID, "trigger", t.Name, "error", err) @@ -559,15 +620,48 @@ func (m *Module) Origin(ctx context.Context, runID uuid.UUID) (runs.Origin, bool } func runInput(e store.Event) jsontext.Value { - input, err := json.Marshal(map[string]any{"event": map[string]any{ + return runInputWith(e, nil) +} + +// runInputWith is the run input for e, with the values a mapping read from its +// payload under "mapped" when there are any. +func runInputWith(e store.Event, mapped map[string]any) jsontext.Value { + doc := map[string]any{"event": map[string]any{ "id": e.ID, "source": e.Source, "type": e.Type, "fields": e.Fields, "occurred_at": e.OccurredAt.Time, - }}) + }} + if mapped != nil { + doc["mapped"] = mapped + } + input, err := json.Marshal(doc) if err != nil { return jsontext.Value("null") } return input } +// inputFor builds the run input for trigger t. A trigger with a mapping reads +// the event's payload through it; when a field the mapping requires is missing +// or has the wrong shape no run starts, and the reason is logged. +func (m *Module) inputFor(ctx context.Context, tx *sqlx.Tx, e store.Event, t store.Trigger) (input jsontext.Value, mapped, ok bool, err error) { + if !t.MappingID.Valid { + return runInput(e), false, true, nil + } + row, err := store.MappingIn(ctx, tx, e.OrgID, t.MappingID.UUID) + if err != nil { + return nil, false, false, err + } + var fields []mapping.Field + if err := json.Unmarshal(row.Fields, &fields); err != nil { + return nil, false, false, fmt.Errorf("mapping %q fields: %w", row.Name, err) + } + values, failures := mapping.Resolve(fields, e.Body) + if len(failures) > 0 { + m.log.WarnContext(ctx, "mapping didn't fit the event; trigger skipped", "event", e.ID, "trigger", t.Name, "mapping", row.Name, "failures", failures) + return nil, false, false, nil + } + return runInputWith(e, values), true, true, nil +} + func newToken() (string, error) { b := make([]byte, 32) if _, err := rand.Read(b); err != nil { diff --git a/modules/events/internal/eventtype/eventtype.go b/modules/events/internal/eventtype/eventtype.go new file mode 100644 index 000000000..b5ecb18e0 --- /dev/null +++ b/modules/events/internal/eventtype/eventtype.go @@ -0,0 +1,205 @@ +// Package eventtype defines event types: match rules over a source's headers +// and payload. It infers signatures, proposes types and refuses overlapping +// types on one source. +package eventtype + +import ( + "errors" + "fmt" + "maps" + "slices" + "strings" +) + +var ( + // ErrInvalid marks an event type that cannot be saved as written. + ErrInvalid = errors.New("eventtype: invalid event type") + // ErrOverlap marks an event type whose rule can match an event another + // type on the same source also matches. + ErrOverlap = errors.New("eventtype: rule overlaps another event type on the source") +) + +// Operator is a payload condition's test. +type Operator string + +const ( + OpEquals Operator = "equals" + OpPresent Operator = "present" +) + +// HeaderCondition holds when the named header (case-insensitive) has Value. +type HeaderCondition struct { + Name string `json:"name"` + Value string `json:"value"` +} + +// PayloadCondition holds when Path is present or, for OpEquals, when a value +// at Path renders as Value ("7", "true", "null", or the string itself). +type PayloadCondition struct { + Path string `json:"path"` + Op Operator `json:"op"` + Value string `json:"value,omitempty"` +} + +// Rule is a conjunction: every condition must hold. An empty rule matches +// every event from its source. +type Rule struct { + Headers []HeaderCondition `json:"headers"` + Payload []PayloadCondition `json:"payload"` +} + +// EventType is a named kind of event from one source. Schema is the inferred +// structure (path to type) of the payload it was created from. +type EventType struct { + ID string `json:"id"` + Source string `json:"source"` + Name string `json:"name"` + Rule Rule `json:"rule"` + Schema map[string]ValueType `json:"schema"` +} + +// Validate refuses an event type that has no name or source or carries a +// malformed condition. +func (t EventType) Validate() error { + if strings.TrimSpace(t.Name) == "" { + return fmt.Errorf("%w: name is required", ErrInvalid) + } + if strings.TrimSpace(t.Source) == "" { + return fmt.Errorf("%w: source is required", ErrInvalid) + } + for _, h := range t.Rule.Headers { + if strings.TrimSpace(h.Name) == "" { + return fmt.Errorf("%w: header condition has no name", ErrInvalid) + } + } + for _, p := range t.Rule.Payload { + if strings.TrimSpace(p.Path) == "" { + return fmt.Errorf("%w: payload condition has no path", ErrInvalid) + } + if p.Op != OpEquals && p.Op != OpPresent { + return fmt.Errorf("%w: unknown operator %q", ErrInvalid, p.Op) + } + } + return nil +} + +// Matches reports whether every condition of r holds for s. +func (r Rule) Matches(s Sample) bool { + for _, h := range r.Headers { + got, ok := headerLookup(s.Headers, h.Name) + if !ok || headerValue(h.Name, got) != headerValue(h.Name, h.Value) { + return false + } + } + if len(r.Payload) == 0 { + return true + } + root, ok := decode(s.Body) + if !ok { + return false + } + for _, p := range r.Payload { + if !payloadHolds(root, p) { + return false + } + } + return true +} + +func headerLookup(headers map[string]string, name string) (string, bool) { + for k, v := range headers { + if strings.EqualFold(k, name) { + return v, true + } + } + return "", false +} + +func payloadHolds(root any, p PayloadCondition) bool { + values := lookup(root, p.Path) + if p.Op == OpPresent { + return len(values) > 0 + } + for _, v := range values { + if text, ok := scalarText(v); ok && text == p.Value { + return true + } + } + return false +} + +// Overlaps reports whether an event could satisfy both rules. Rules are +// disjoint only when they require different values for one header or a +// non-array payload path. +func Overlaps(a, b Rule) bool { + for _, ha := range a.Headers { + for _, hb := range b.Headers { + if strings.EqualFold(ha.Name, hb.Name) && headerValue(ha.Name, ha.Value) != headerValue(hb.Name, hb.Value) { + return false + } + } + } + for _, pa := range a.Payload { + for _, pb := range b.Payload { + if pa.Op == OpEquals && pb.Op == OpEquals && pa.Path == pb.Path && + !strings.Contains(pa.Path, "[]") && pa.Value != pb.Value { + return false + } + } + } + return true +} + +// CheckOverlap refuses candidate when its rule overlaps another type on the +// same source. A type is not compared with itself, so an update passes. +func CheckOverlap(existing []EventType, candidate EventType) error { + for _, t := range existing { + if t.Source != candidate.Source || (candidate.ID != "" && t.ID == candidate.ID) { + continue + } + if Overlaps(t.Rule, candidate.Rule) { + return fmt.Errorf("%w: %q", ErrOverlap, t.Name) + } + } + return nil +} + +// Identify returns the one type on source whose rule matches s. No match, or +// more than one (which overlap refusal prevents), is unidentified. +func Identify(types []EventType, source string, s Sample) (EventType, bool) { + var found []EventType + for _, t := range types { + if t.Source == source && t.Rule.Matches(s) { + found = append(found, t) + } + } + if len(found) != 1 { + return EventType{}, false + } + return found[0], true +} + +// FromExample builds an event type from a pasted payload: its schema is the +// example's structure, and its rule is the example's discriminator headers or, +// when it has none, the presence of each top-level payload key. +func FromExample(source, name string, s Sample) EventType { + sig := Sign(s) + t := EventType{Source: source, Name: name, Schema: sig.Paths} + t.Rule.Headers = headerConditions(sig.Headers) + if len(t.Rule.Headers) == 0 { + for _, path := range slices.Sorted(maps.Keys(sig.Paths)) { + if !strings.ContainsAny(path, ".[") { + t.Rule.Payload = append(t.Rule.Payload, PayloadCondition{Path: path, Op: OpPresent}) + } + } + } + return t +} + +func headerConditions(headers map[string]string) []HeaderCondition { + var out []HeaderCondition + for _, name := range slices.Sorted(maps.Keys(headers)) { + out = append(out, HeaderCondition{Name: name, Value: headers[name]}) + } + return out +} diff --git a/modules/events/internal/eventtype/propose.go b/modules/events/internal/eventtype/propose.go new file mode 100644 index 000000000..4153fecf4 --- /dev/null +++ b/modules/events/internal/eventtype/propose.go @@ -0,0 +1,106 @@ +package eventtype + +import ( + "maps" + "slices" + "strings" + "time" +) + +// Capture is one stored event as proposal grouping reads it. +type Capture struct { + ID string + Source string + ReceivedAt time.Time + Sample Sample +} + +// Proposal is a group of one source's captures sharing a signature, offered +// to the operator as an event type to name. Rule is the set of +// discriminators that separates the group from its siblings on the source. +type Proposal struct { + Source string `json:"source"` + Key string `json:"key"` + Schema map[string]ValueType `json:"schema"` + Headers map[string]string `json:"headers"` + Count int `json:"count"` + SampleID string `json:"sample_id"` + Rule Rule `json:"rule"` +} + +// Propose groups captures by source and signature. Groups are ordered by +// source, then largest first, then first seen; the sample is the group's first +// capture in input order. +func Propose(captures []Capture) []Proposal { + var groups []*Proposal + sigs := map[*Proposal]Signature{} + index := map[string]*Proposal{} + for _, c := range captures { + sig := Sign(c.Sample) + key := sig.Key() + id := c.Source + "\x00" + key + if g, ok := index[id]; ok { + g.Count++ + continue + } + g := &Proposal{Source: c.Source, Key: key, Schema: sig.Paths, Headers: sig.Headers, Count: 1, SampleID: c.ID} + index[id] = g + sigs[g] = sig + groups = append(groups, g) + } + slices.SortStableFunc(groups, func(a, b *Proposal) int { + if c := strings.Compare(a.Source, b.Source); c != 0 { + return c + } + return b.Count - a.Count + }) + out := make([]Proposal, 0, len(groups)) + for _, g := range groups { + var siblings []Signature + for _, s := range groups { + if s != g && s.Source == g.Source { + siblings = append(siblings, sigs[s]) + } + } + g.Rule = separatingRule(sigs[g], siblings) + out = append(out, *g) + } + return out +} + +// separatingRule requires the group's discriminator headers, plus a path +// present in the group but absent from each remaining sibling. +func separatingRule(g Signature, siblings []Signature) Rule { + rule := Rule{Headers: headerConditions(g.Headers)} + paths := slices.Sorted(maps.Keys(g.Paths)) + for _, s := range siblings { + if headersSeparate(g.Headers, s.Headers) || rulePathsSeparate(rule, s) { + continue + } + for _, path := range paths { + if _, ok := s.Paths[path]; !ok { + rule.Payload = append(rule.Payload, PayloadCondition{Path: path, Op: OpPresent}) + break + } + } + } + return rule +} + +func headersSeparate(g, s map[string]string) bool { + for name, value := range g { + if other, ok := s[name]; !ok || other != value { + return true + } + } + return false +} + +func rulePathsSeparate(rule Rule, s Signature) bool { + for _, p := range rule.Payload { + if _, ok := s.Paths[p.Path]; !ok { + return true + } + } + return false +} diff --git a/modules/events/internal/eventtype/signature.go b/modules/events/internal/eventtype/signature.go new file mode 100644 index 000000000..676f627f4 --- /dev/null +++ b/modules/events/internal/eventtype/signature.go @@ -0,0 +1,202 @@ +package eventtype + +import ( + json "encoding/json/v2" + "maps" + "mime" + "slices" + "strconv" + "strings" +) + +// ValueType is the JSON shape of one payload path. The names match +// mapping.FieldType so a mapping can be checked against a type's schema. +type ValueType string + +const ( + TypeString ValueType = "string" + TypeNumber ValueType = "number" + TypeBool ValueType = "bool" + TypeObject ValueType = "object" + TypeArray ValueType = "array" + TypeNull ValueType = "null" +) + +// Sample is one event as the rules see it: headers keyed by lower-case name +// (first value only) and the raw body. +type Sample struct { + Headers map[string]string + Body []byte +} + +// Signature is the structure of one event: every JSON path with its value +// type, plus the discriminator headers with their values. Array elements share +// one path segment, "[]", so two payloads differing only in list length have +// the same signature. +type Signature struct { + Paths map[string]ValueType `json:"paths"` + Headers map[string]string `json:"headers"` +} + +// ParseHeaders reads a capture's stored headers (an encoded http.Header) into +// the lower-case, first-value form a Sample carries. Anything unreadable is no +// headers, not an error: a capture's headers are evidence, not input. +func ParseHeaders(raw string) map[string]string { + out := map[string]string{} + var h map[string][]string + if json.Unmarshal([]byte(raw), &h) != nil { + return out + } + for name, values := range h { + if len(values) > 0 { + out[strings.ToLower(name)] = values[0] + } + } + return out +} + +// isDiscriminator reports whether a header names the kind of event rather than +// describing its transport: X-GitHub-Event, X-Gitea-Event-Type, Content-Type. +func isDiscriminator(name string) bool { + name = strings.ToLower(name) + return strings.HasSuffix(name, "-event") || strings.HasSuffix(name, "-type") +} + +// headerValue normalises a header value for comparison. Content-Type compares +// by media type, so a charset parameter does not split one kind of event in two. +func headerValue(name, value string) string { + value = strings.TrimSpace(value) + if strings.EqualFold(name, "content-type") { + if mt, _, err := mime.ParseMediaType(value); err == nil { + return mt + } + } + return value +} + +// Sign computes an event's structural signature. A body that is not JSON has +// no paths; its headers still discriminate it. +func Sign(s Sample) Signature { + sig := Signature{Paths: map[string]ValueType{}, Headers: map[string]string{}} + for name, value := range s.Headers { + if isDiscriminator(name) { + sig.Headers[strings.ToLower(name)] = headerValue(name, value) + } + } + if root, ok := decode(s.Body); ok { + collectPaths(root, "", sig.Paths) + } + return sig +} + +// Key is a stable string that is equal for two signatures exactly when their +// paths, path types and discriminator headers are equal. +func (s Signature) Key() string { + var b strings.Builder + for _, name := range slices.Sorted(maps.Keys(s.Headers)) { + b.WriteString("h:" + name + "=" + s.Headers[name] + "\n") + } + for _, path := range slices.Sorted(maps.Keys(s.Paths)) { + b.WriteString("p:" + path + "=" + string(s.Paths[path]) + "\n") + } + return b.String() +} + +func decode(body []byte) (any, bool) { + var v any + if err := json.Unmarshal(body, &v); err != nil { + return nil, false + } + return v, true +} + +func valueType(v any) ValueType { + switch v.(type) { + case string: + return TypeString + case float64: + return TypeNumber + case bool: + return TypeBool + case map[string]any: + return TypeObject + case []any: + return TypeArray + default: + return TypeNull + } +} + +// collectPaths records every path below v. The root itself has no path. +func collectPaths(v any, prefix string, out map[string]ValueType) { + switch typed := v.(type) { + case map[string]any: + for key, child := range typed { + path := key + if prefix != "" { + path = prefix + "." + key + } + out[path] = valueType(child) + collectPaths(child, path, out) + } + case []any: + for _, child := range typed { + path := prefix + "[]" + out[path] = valueType(child) + collectPaths(child, path, out) + } + } +} + +// lookup returns every value at path; "[]" segments fan out over array +// elements, so a path inside a list yields one value per element that has it. +func lookup(root any, path string) []any { + current := []any{root} + for seg := range strings.SplitSeq(path, ".") { + name, arrays := seg, 0 + for strings.HasSuffix(name, "[]") { + name, arrays = strings.TrimSuffix(name, "[]"), arrays+1 + } + var next []any + for _, node := range current { + obj, ok := node.(map[string]any) + if !ok { + continue + } + if child, ok := obj[name]; ok { + next = append(next, child) + } + } + for range arrays { + var elems []any + for _, node := range next { + if arr, ok := node.([]any); ok { + elems = append(elems, arr...) + } + } + next = elems + } + current = next + } + return current +} + +// scalarText renders a scalar the way an operator types it into an equals +// condition; objects and arrays have no text and never equal anything. +func scalarText(v any) (string, bool) { + switch typed := v.(type) { + case string: + return typed, true + case float64: + return strconv.FormatFloat(typed, 'f', -1, 64), true + case bool: + if typed { + return "true", true + } + return "false", true + case nil: + return "null", true + default: + return "", false + } +} diff --git a/modules/events/internal/mapping/mapping.go b/modules/events/internal/mapping/mapping.go new file mode 100644 index 000000000..09a3c8304 --- /dev/null +++ b/modules/events/internal/mapping/mapping.go @@ -0,0 +1,290 @@ +// Package mapping owns the payload field-mapping vocabulary and its +// resolve/preview rule: binding named fields to JSON paths inside a +// captured webhook payload, and deciding -- loudly -- when a path a mapping +// depends on has drifted out from under it. See +package mapping + +import ( + json "encoding/json/v2" + "errors" + "fmt" + "strconv" + "strings" + "time" + + "github.com/samcharles93/archie-core/modules/events/internal/eventtype" +) + +// FieldType is the expected JSON shape of a mapped field, checked at +// resolve time. +type FieldType string + +const ( + TypeString FieldType = "string" + TypeNumber FieldType = "number" + TypeBool FieldType = "bool" + TypeObject FieldType = "object" + TypeArray FieldType = "array" + TypeAny FieldType = "any" +) + +// Failure reasons. A field either resolves to a value or fails for exactly +// one of these -- never both. +const ( + ReasonMissing = "missing" + ReasonNull = "null" + ReasonTypeMismatch = "type_mismatch" +) + +// Field binds one named field the workflow receives to a path into a +// captured payload. Required=false suppresses ReasonMissing only -- a +// present-but-wrong-shape value always fails, so a mapping never silently +// hands an agent a value it didn't expect. +type Field struct { + Name string `json:"name"` + Path string `json:"path"` + Type FieldType `json:"type"` + Required bool `json:"required"` +} + +// Mapping is a named set of field bindings belonging to one event type and +// checked against its schema (CheckSchema). Any number of bindings may reuse +// it. MatchCount and LastMatchedAt are read-only: the store counts each +// capture the mapping resolved at dispatch. +type Mapping struct { + ID string `json:"id"` + Name string `json:"name"` + SourceHint string `json:"source_hint"` + EventTypeID string `json:"event_type_id"` + Fields []Field `json:"fields"` + MatchCount int64 `json:"match_count"` + LastMatchedAt time.Time `json:"last_matched_at"` + CreatedAt time.Time `json:"created_at"` + UpdatedAt time.Time `json:"updated_at"` +} + +// validTypes is the set of FieldType values Validate accepts. +var validTypes = map[FieldType]bool{ + TypeString: true, TypeNumber: true, TypeBool: true, + TypeObject: true, TypeArray: true, TypeAny: true, +} + +// Validate checks a mapping is well-formed before it is persisted or +// previewed: a name, at least one field, and every field naming a +// non-empty field name, a non-empty path, and a recognised type. It does +// not check the mapping against any payload -- that is Resolve's job. +func (m Mapping) Validate() error { + if strings.TrimSpace(m.Name) == "" { + return fmt.Errorf("mapping: name is required") + } + if len(m.Fields) == 0 { + return fmt.Errorf("mapping: at least one field is required") + } + seen := make(map[string]bool, len(m.Fields)) + for i, f := range m.Fields { + if strings.TrimSpace(f.Name) == "" { + return fmt.Errorf("mapping: field %d: name is required", i) + } + if strings.TrimSpace(f.Path) == "" { + return fmt.Errorf("mapping: field %d (%s): path is required", i, f.Name) + } + if !validTypes[f.Type] { + return fmt.Errorf("mapping: field %d (%s): unrecognised type %q", i, f.Name, f.Type) + } + // A duplicate name would let one field silently overwrite another's + // resolved value in Resolve's values map -- reject it here instead + // of losing data silently at resolve time. + if seen[f.Name] { + return fmt.Errorf("mapping: field %d: duplicate field name %q", i, f.Name) + } + seen[f.Name] = true + } + return nil +} + +// Failure describes one field that did not resolve cleanly against a payload -- +// named by field and path so the operator knows exactly what drifted. +type Failure struct { + FieldName string `json:"field_name"` + Path string `json:"path"` + Reason string `json:"reason"` +} + +// Resolve walks payload independently for each field, so one field's +// failure never stops the others from resolving -- preview must show every +// problem at once, not the first one. A field that fails never appears in +// values. +func Resolve(fields []Field, payload []byte) (values map[string]any, failures []Failure) { + values = make(map[string]any, len(fields)) + + var root any + if err := json.Unmarshal(payload, &root); err != nil { + for _, f := range fields { + failures = append(failures, Failure{FieldName: f.Name, Path: f.Path, Reason: ReasonMissing}) + } + return values, failures + } + + for _, f := range fields { + v, found := walk(root, parsePath(f.Path)) + if !found { + if f.Required { + failures = append(failures, Failure{FieldName: f.Name, Path: f.Path, Reason: ReasonMissing}) + } + continue + } + if v == nil { + failures = append(failures, Failure{FieldName: f.Name, Path: f.Path, Reason: ReasonNull}) + continue + } + if !typeMatches(v, f.Type) { + failures = append(failures, Failure{FieldName: f.Name, Path: f.Path, Reason: ReasonTypeMismatch}) + continue + } + values[f.Name] = v + } + + return values, failures +} + +// pathSegment is a map key, an array index, or Invalid. walk always fails on +// Invalid. +type pathSegment struct { + Key string + Index int + Invalid bool +} + +// parsePath splits "data.items[0].id" into +// [{Key:"data"} {Key:"items"} {Index:0} {Key:"id"}]. No wildcards, no +// filters -- the grammar is exactly what a click-through UI generates. +func parsePath(path string) []pathSegment { + var segs []pathSegment + for part := range strings.SplitSeq(path, ".") { + if part == "" { + continue + } + for { + open := strings.IndexByte(part, '[') + if open < 0 { + if part != "" { + segs = append(segs, pathSegment{Key: part, Index: -1}) + } + break + } + if open > 0 { + segs = append(segs, pathSegment{Key: part[:open], Index: -1}) + } + closeIdx := strings.IndexByte(part[open:], ']') + if closeIdx < 0 { + segs = append(segs, pathSegment{Invalid: true}) + break + } + closeIdx += open + idx, err := strconv.Atoi(part[open+1 : closeIdx]) + if err != nil { + segs = append(segs, pathSegment{Invalid: true}) + } else { + segs = append(segs, pathSegment{Index: idx}) + } + part = part[closeIdx+1:] + if part == "" { + break + } + } + } + return segs +} + +// walk descends node by path. found=false means the path does not exist in +// the payload at all (missing), which is distinct from resolving to a JSON +// null (node == nil, found == true). +func walk(node any, path []pathSegment) (value any, found bool) { + cur := node + for _, seg := range path { + if seg.Invalid { + return nil, false + } + if cur == nil { + return nil, false + } + if seg.Index >= 0 { + arr, ok := cur.([]any) + if !ok || seg.Index >= len(arr) { + return nil, false + } + cur = arr[seg.Index] + continue + } + obj, ok := cur.(map[string]any) + if !ok { + return nil, false + } + v, ok := obj[seg.Key] + if !ok { + return nil, false + } + cur = v + } + return cur, true +} + +// ErrSchema marks a mapping whose field paths or types disagree with its +// event type's inferred schema. +var ErrSchema = errors.New("mapping: field does not fit the event type's schema") + +// CheckSchema refuses fields whose path the event type's schema lacks, or +// whose declared type differs from the schema's. Array indices match the +// schema's "[]" segment, and a path the example held as null accepts any type. +func CheckSchema(fields []Field, schema map[string]eventtype.ValueType) error { + for _, f := range fields { + got, ok := schema[schemaPath(f.Path)] + if !ok { + return fmt.Errorf("%w: %s: path %q is not in the event type", ErrSchema, f.Name, f.Path) + } + if f.Type != TypeAny && got != eventtype.TypeNull && string(got) != string(f.Type) { + return fmt.Errorf("%w: %s: path %q is %s, not %s", ErrSchema, f.Name, f.Path, got, f.Type) + } + } + return nil +} + +// schemaPath rewrites a resolve path ("items[0].id") into the schema's form +// ("items[].id"). A malformed path has no schema form. +func schemaPath(path string) string { + var b strings.Builder + for _, seg := range parsePath(path) { + switch { + case seg.Invalid: + return "" + case seg.Index >= 0: + b.WriteString("[]") + default: + if b.Len() > 0 { + b.WriteByte('.') + } + b.WriteString(seg.Key) + } + } + return b.String() +} + +func typeMatches(v any, want FieldType) bool { + if want == TypeAny { + return true + } + switch v.(type) { + case string: + return want == TypeString + case float64: + return want == TypeNumber + case bool: + return want == TypeBool + case map[string]any: + return want == TypeObject + case []any: + return want == TypeArray + default: + return false + } +} diff --git a/modules/events/internal/store/capture.go b/modules/events/internal/store/capture.go new file mode 100644 index 000000000..f20f27c54 --- /dev/null +++ b/modules/events/internal/store/capture.go @@ -0,0 +1,419 @@ +package store + +import ( + "context" + "database/sql" + "encoding/json/jsontext" + "errors" + "fmt" + "time" + + "github.com/google/uuid" + "github.com/jackc/pgx/v5/pgconn" + "github.com/jmoiron/sqlx" +) + +// ErrInUse means other rows still refer to the row being deleted. +var ErrInUse = errors.New("in use") + +// ErrExists means a row with that name already exists. +var ErrExists = errors.New("already exists") + +// Postgres reports a delete blocked by ON DELETE RESTRICT as 23001 and a +// reference to a missing row as 23503. +const ( + restrictViolation = "23001" + foreignKeyViolation = "23503" + uniqueViolation = "23505" +) + +// classify turns the constraint errors callers act on into the sentinels +// above. +func classify(err error) error { + if pqErr, ok := errors.AsType[*pgconn.PgError](err); ok { + switch pqErr.Code { + case restrictViolation, foreignKeyViolation: + return fmt.Errorf("%w: %s", ErrInUse, pqErr.Detail) + case uniqueViolation: + return ErrExists + } + } + return err +} + +// Captures returns the events whose source kept their headers and payload, +// newest first. +func (s *Store) Captures(ctx context.Context, orgID string, limit int) ([]Event, error) { + var out []Event + if err := s.db.SelectContext(ctx, &out, ` + SELECT id, org_id, source, type, source_event_id, fields, body, headers, remote_addr, verification, + event_type_id, received_at, occurred_at + FROM events.events + WHERE org_id = $1 AND headers IS NOT NULL + ORDER BY received_at DESC, id LIMIT $2`, orgID, limit); err != nil { + return nil, fmt.Errorf("list captures: %w", err) + } + return out, nil +} + +// EventType is a person's name for one kind of event on a source. +type EventType struct { + ID uuid.UUID `db:"id"` + OrgID string `db:"org_id"` + Source string `db:"source"` + Name string `db:"name"` + Rule jsontext.Value `db:"rule"` + Schema jsontext.Value `db:"schema"` + CreatedAt time.Time `db:"created_at"` + UpdatedAt time.Time `db:"updated_at"` +} + +// EventTypes returns the organisation's event types, by source and name. +func (s *Store) EventTypes(ctx context.Context, orgID string) ([]EventType, error) { + var out []EventType + if err := s.db.SelectContext(ctx, &out, ` + SELECT id, org_id, source, name, rule, schema, created_at, updated_at + FROM events.event_types WHERE org_id = $1 ORDER BY source, name`, orgID); err != nil { + return nil, fmt.Errorf("list event types: %w", err) + } + return out, nil +} + +// EventTypesOf returns the event types of one source. +func (s *Store) EventTypesOf(ctx context.Context, orgID, source string) ([]EventType, error) { + var out []EventType + if err := s.db.SelectContext(ctx, &out, ` + SELECT id, org_id, source, name, rule, schema, created_at, updated_at + FROM events.event_types WHERE org_id = $1 AND source = $2 ORDER BY name`, orgID, source); err != nil { + return nil, fmt.Errorf("list event types of source: %w", err) + } + return out, nil +} + +// EventType returns one event type. +func (s *Store) EventType(ctx context.Context, orgID string, id uuid.UUID) (EventType, error) { + var t EventType + err := s.db.GetContext(ctx, &t, ` + SELECT id, org_id, source, name, rule, schema, created_at, updated_at + FROM events.event_types WHERE org_id = $1 AND id = $2`, orgID, id) + if errors.Is(err, sql.ErrNoRows) { + return EventType{}, ErrNotFound + } + if err != nil { + return EventType{}, fmt.Errorf("read event type: %w", err) + } + return t, nil +} + +// InsertEventType stores a new event type. +func (s *Store) InsertEventType(ctx context.Context, t EventType) error { + if _, err := s.db.NamedExecContext(ctx, ` + INSERT INTO events.event_types (id, org_id, source, name, rule, schema) + VALUES (:id, :org_id, :source, :name, :rule, :schema)`, t); err != nil { + return fmt.Errorf("save event type: %w", classify(err)) + } + return nil +} + +// UpdateEventType changes an event type's name and rule. Its source and +// schema are fixed. +func (s *Store) UpdateEventType(ctx context.Context, orgID string, id uuid.UUID, name string, rule jsontext.Value) error { + res, err := s.db.ExecContext(ctx, ` + UPDATE events.event_types SET name = $3, rule = $4, updated_at = now() + WHERE org_id = $1 AND id = $2`, orgID, id, name, rule) + if err != nil { + return fmt.Errorf("save event type: %w", classify(err)) + } + if n, err := res.RowsAffected(); err != nil || n == 0 { + return ErrNotFound + } + return nil +} + +// DeleteEventType removes an event type. Events identified as it become +// unidentified; a mapping or trigger that uses it blocks the delete. +func (s *Store) DeleteEventType(ctx context.Context, orgID string, id uuid.UUID) error { + res, err := s.db.ExecContext(ctx, `DELETE FROM events.event_types WHERE org_id = $1 AND id = $2`, orgID, id) + if err != nil { + return fmt.Errorf("delete event type: %w", classify(err)) + } + if n, err := res.RowsAffected(); err != nil || n == 0 { + return ErrNotFound + } + return nil +} + +// Mapping names the payload paths a workflow reads from one event type. +type Mapping struct { + ID uuid.UUID `db:"id"` + OrgID string `db:"org_id"` + Name string `db:"name"` + SourceHint string `db:"source_hint"` + EventTypeID uuid.UUID `db:"event_type_id"` + Fields jsontext.Value `db:"fields"` + MatchCount int64 `db:"match_count"` + LastMatchedAt sql.NullTime `db:"last_matched_at"` + CreatedAt time.Time `db:"created_at"` + UpdatedAt time.Time `db:"updated_at"` +} + +const mappingColumns = `id, org_id, name, source_hint, event_type_id, fields, match_count, last_matched_at, created_at, updated_at` + +// Mappings returns the organisation's mappings, by name. +func (s *Store) Mappings(ctx context.Context, orgID string) ([]Mapping, error) { + var out []Mapping + if err := s.db.SelectContext(ctx, &out, `SELECT `+mappingColumns+` FROM events.mappings WHERE org_id = $1 ORDER BY name`, orgID); err != nil { + return nil, fmt.Errorf("list mappings: %w", err) + } + return out, nil +} + +// Mapping returns one mapping. +func (s *Store) Mapping(ctx context.Context, orgID string, id uuid.UUID) (Mapping, error) { + var m Mapping + err := s.db.GetContext(ctx, &m, `SELECT `+mappingColumns+` FROM events.mappings WHERE org_id = $1 AND id = $2`, orgID, id) + if errors.Is(err, sql.ErrNoRows) { + return Mapping{}, ErrNotFound + } + if err != nil { + return Mapping{}, fmt.Errorf("read mapping: %w", err) + } + return m, nil +} + +// MappingIn reads a mapping inside a dispatch transaction. +func MappingIn(ctx context.Context, tx *sqlx.Tx, orgID string, id uuid.UUID) (Mapping, error) { + var m Mapping + if err := tx.GetContext(ctx, &m, `SELECT `+mappingColumns+` FROM events.mappings WHERE org_id = $1 AND id = $2`, orgID, id); err != nil { + return Mapping{}, fmt.Errorf("read mapping: %w", err) + } + return m, nil +} + +// InsertMapping stores a new mapping. +func (s *Store) InsertMapping(ctx context.Context, m Mapping) error { + if _, err := s.db.NamedExecContext(ctx, ` + INSERT INTO events.mappings (id, org_id, name, source_hint, event_type_id, fields) + VALUES (:id, :org_id, :name, :source_hint, :event_type_id, :fields)`, m); err != nil { + return fmt.Errorf("save mapping: %w", classify(err)) + } + return nil +} + +// UpdateMapping replaces a mapping's name, event type and fields. Its match +// count is kept. +func (s *Store) UpdateMapping(ctx context.Context, m Mapping) error { + res, err := s.db.NamedExecContext(ctx, ` + UPDATE events.mappings SET name = :name, source_hint = :source_hint, event_type_id = :event_type_id, + fields = :fields, updated_at = now() + WHERE org_id = :org_id AND id = :id`, m) + if err != nil { + return fmt.Errorf("save mapping: %w", classify(err)) + } + if n, err := res.RowsAffected(); err != nil || n == 0 { + return ErrNotFound + } + return nil +} + +// DeleteMapping removes a mapping that no trigger uses. +func (s *Store) DeleteMapping(ctx context.Context, orgID string, id uuid.UUID) error { + res, err := s.db.ExecContext(ctx, `DELETE FROM events.mappings WHERE org_id = $1 AND id = $2`, orgID, id) + if err != nil { + return fmt.Errorf("delete mapping: %w", classify(err)) + } + if n, err := res.RowsAffected(); err != nil || n == 0 { + return ErrNotFound + } + return nil +} + +// CountMatch notes that a mapping read an event a trigger started a run for. +func CountMatch(ctx context.Context, tx *sqlx.Tx, id uuid.UUID) error { + if _, err := tx.ExecContext(ctx, ` + UPDATE events.mappings SET match_count = match_count + 1, last_matched_at = now() WHERE id = $1`, id); err != nil { + return fmt.Errorf("count mapping match: %w", err) + } + return nil +} + +// SetEventType records which event type an event was identified as. +func (s *Store) SetEventType(ctx context.Context, id, typeID uuid.UUID) error { + if _, err := s.db.ExecContext(ctx, `UPDATE events.events SET event_type_id = $2 WHERE id = $1`, id, typeID); err != nil { + return fmt.Errorf("identify event: %w", err) + } + return nil +} + +// SetSourceColumn changes one of a source's single-value settings. column is +// always a constant of this package. +func (s *Store) setSource(ctx context.Context, orgID, name, column string, value any) error { + res, err := s.db.ExecContext(ctx, ` + UPDATE events.sources SET `+column+` = $3, updated_at = now() WHERE org_id = $1 AND name = $2`, + orgID, name, value) + if err != nil { + return fmt.Errorf("save source: %w", err) + } + if n, err := res.RowsAffected(); err != nil || n == 0 { + return ErrNotFound + } + return nil +} + +// SetDisplayName labels a source. +func (s *Store) SetDisplayName(ctx context.Context, orgID, name, display string) error { + return s.setSource(ctx, orgID, name, "display_name", display) +} + +// SetDeliveryHeader names the header a sender repeats on a delivery's retries. +func (s *Store) SetDeliveryHeader(ctx context.Context, orgID, name, header string) error { + return s.setSource(ctx, orgID, name, "delivery_header", header) +} + +// SetCapture turns the keeping of a source's headers and payloads on or off. +func (s *Store) SetCapture(ctx context.Context, orgID, name string, capture bool) error { + return s.setSource(ctx, orgID, name, "capture", capture) +} + +// SetSecret replaces a source's sealed signing secret. +func (s *Store) SetSecret(ctx context.Context, orgID, name string, sealed []byte) error { + return s.setSource(ctx, orgID, name, "hmac_secret_sealed", sealed) +} + +// SetSigning switches a source between an HMAC signature and no verification. +// Turning signing off leaves the source unapproved, so it refuses deliveries +// until a person approves it; the signature header and secret are kept so +// signing can be required again. +func (s *Store) SetSigning(ctx context.Context, orgID, name string, signed bool) error { + verification := None + if signed { + verification = HMAC + } + res, err := s.db.ExecContext(ctx, ` + UPDATE events.sources SET verification = $3, unsigned_approved = false, updated_at = now() + WHERE org_id = $1 AND name = $2`, orgID, name, verification) + if err != nil { + return fmt.Errorf("save source: %w", err) + } + if n, err := res.RowsAffected(); err != nil || n == 0 { + return ErrNotFound + } + return nil +} + +// ApproveUnsigned accepts that a source with no verification takes +// deliveries. It reports false when the source isn't waiting for approval. +func (s *Store) ApproveUnsigned(ctx context.Context, orgID, name string) (bool, error) { + res, err := s.db.ExecContext(ctx, ` + UPDATE events.sources SET unsigned_approved = true, updated_at = now() + WHERE org_id = $1 AND name = $2 AND verification = 'none' AND NOT unsigned_approved`, orgID, name) + if err != nil { + return false, fmt.Errorf("approve source: %w", err) + } + n, err := res.RowsAffected() + return n == 1, err +} + +// DeleteSource removes a source and its event types. A trigger or a mapping +// that still uses it blocks the delete; stored events and their dispatch +// history stay. +func (s *Store) DeleteSource(ctx context.Context, orgID, name string) error { + tx, err := s.db.BeginTxx(ctx, nil) + if err != nil { + return fmt.Errorf("begin delete source: %w", err) + } + defer func() { _ = tx.Rollback() }() + var triggers int + if err := tx.GetContext(ctx, &triggers, `SELECT count(*) FROM events.triggers WHERE org_id = $1 AND source = $2`, orgID, name); err != nil { + return fmt.Errorf("check triggers of source: %w", err) + } + if triggers > 0 { + return fmt.Errorf("%w: %d trigger(s) use it", ErrInUse, triggers) + } + if _, err := tx.ExecContext(ctx, `DELETE FROM events.event_types WHERE org_id = $1 AND source = $2`, orgID, name); err != nil { + return fmt.Errorf("delete event types of source: %w", classify(err)) + } + res, err := tx.ExecContext(ctx, `DELETE FROM events.sources WHERE org_id = $1 AND name = $2`, orgID, name) + if err != nil { + return fmt.Errorf("delete source: %w", err) + } + if n, err := res.RowsAffected(); err != nil || n == 0 { + return ErrNotFound + } + if err := tx.Commit(); err != nil { + return fmt.Errorf("commit delete source: %w", err) + } + return nil +} + +// MappedTriggers counts the triggers of a source that read events with a +// mapping, which needs the stored payload. +func (s *Store) MappedTriggers(ctx context.Context, orgID, source string) (int, error) { + var n int + if err := s.db.GetContext(ctx, &n, ` + SELECT count(*) FROM events.triggers WHERE org_id = $1 AND source = $2 AND mapping_id IS NOT NULL`, orgID, source); err != nil { + return 0, fmt.Errorf("count mapped triggers: %w", err) + } + return n, nil +} + +// SetTriggerState changes whether a trigger is approved and enabled, and +// reports whether it exists. +func (s *Store) SetTriggerState(ctx context.Context, orgID, name string, approved, enabled bool) (bool, error) { + res, err := s.db.ExecContext(ctx, ` + UPDATE events.triggers SET approved = $3, enabled = $4, updated_at = now() + WHERE org_id = $1 AND name = $2`, orgID, name, approved, enabled) + if err != nil { + return false, fmt.Errorf("save trigger: %w", err) + } + n, err := res.RowsAffected() + return n == 1, err +} + +// Trigger returns one trigger. +func (s *Store) Trigger(ctx context.Context, orgID, name string) (Trigger, error) { + var t Trigger + err := s.db.GetContext(ctx, &t, ` + SELECT org_id, name, source, event_type, filter, workflow, enabled, approved, event_type_id, mapping_id, + schedule_every_s, schedule_since + FROM events.triggers WHERE org_id = $1 AND name = $2`, orgID, name) + if errors.Is(err, sql.ErrNoRows) { + return Trigger{}, ErrNotFound + } + if err != nil { + return Trigger{}, fmt.Errorf("read trigger: %w", err) + } + return t, nil +} + +// DeleteTrigger removes a trigger. Runs it already started and their dispatch +// records stay. +func (s *Store) DeleteTrigger(ctx context.Context, orgID, name string) error { + res, err := s.db.ExecContext(ctx, `DELETE FROM events.triggers WHERE org_id = $1 AND name = $2`, orgID, name) + if err != nil { + return fmt.Errorf("delete trigger: %w", err) + } + if n, err := res.RowsAffected(); err != nil || n == 0 { + return ErrNotFound + } + return nil +} + +// Dispatch is one run a trigger started for an event. +type Dispatch struct { + EventID uuid.UUID `db:"event_id"` + RunID uuid.UUID `db:"run_id"` + CreatedAt time.Time `db:"created_at"` +} + +// DispatchesOf returns the runs a trigger started, newest first. +func (s *Store) DispatchesOf(ctx context.Context, orgID, trigger string, limit int) ([]Dispatch, error) { + var out []Dispatch + if err := s.db.SelectContext(ctx, &out, ` + SELECT d.event_id, d.run_id, d.created_at FROM events.dispatches d + JOIN events.events e ON e.id = d.event_id + WHERE e.org_id = $1 AND d.trigger = $2 ORDER BY d.created_at DESC LIMIT $3`, orgID, trigger, limit); err != nil { + return nil, fmt.Errorf("list dispatches: %w", err) + } + return out, nil +} diff --git a/modules/events/internal/store/store.go b/modules/events/internal/store/store.go index fb9f1b7f3..a4f2bdb79 100644 --- a/modules/events/internal/store/store.go +++ b/modules/events/internal/store/store.go @@ -43,6 +43,15 @@ type Source struct { Cursor string `db:"cursor"` // KeyFields are the paths its events are grouped by; none means the default. KeyFields jsontext.Value `db:"key_fields"` + // DisplayName labels the source for people. DeliveryHeader names the header + // a sender repeats on every retry of one delivery. + DisplayName string `db:"display_name"` + DeliveryHeader string `db:"delivery_header"` + // UnsignedApproved says a person accepted that a None source does no + // verification. A None source refuses deliveries until then. + UnsignedApproved bool `db:"unsigned_approved"` + // Capture says the source keeps each delivery's headers and raw payload. + Capture bool `db:"capture"` } // Event is one stored event. @@ -70,8 +79,25 @@ type Event struct { GroupKey sql.NullString `db:"group_key"` // Scan into []byte to preserve SQL NULL for events with no group. GroupFilter []byte `db:"group_filter"` + // Verification is how the delivery was verified: Signed (an HMAC + // signature), TokenVerified (a bearer token) or Unsigned (an approved + // source that checks nothing). It is empty for an event that arrived + // through a connector's own stream. + Verification string `db:"verification"` + // Headers are the request's headers without credentials, and RemoteAddr is + // the sender's address. Headers is null when the source keeps no payloads. + Headers []byte `db:"headers"` + RemoteAddr string `db:"remote_addr"` + EventTypeID uuid.NullUUID `db:"event_type_id"` } +// How an event's delivery was verified. +const ( + Signed = "signed" + TokenVerified = "token" + Unsigned = "unsigned" +) + // Trigger starts a workflow for events of one type from one source whose // fields match its filter. type Trigger struct { @@ -87,6 +113,12 @@ type Trigger struct { ScheduleEverySeconds sql.NullInt64 `db:"schedule_every_s"` // ScheduleSince is when the schedule began or last changed. ScheduleSince sql.NullTime `db:"schedule_since"` + // Approved says a person accepted the trigger; it fires only once approved + // and enabled. EventTypeID restricts it to events identified as that type, + // MappingID reads the event's payload for the run. + Approved bool `db:"approved"` + EventTypeID uuid.NullUUID `db:"event_type_id"` + MappingID uuid.NullUUID `db:"mapping_id"` } // Store is the events module's Postgres store. @@ -101,7 +133,8 @@ func New(db *sqlx.DB) *Store { return &Store{db: db} } func (s *Store) Source(ctx context.Context, orgID, name string) (Source, error) { var src Source err := s.db.GetContext(ctx, &src, ` - SELECT org_id, name, connector, token_sha256, verification, hmac_header, hmac_prefix, hmac_secret_sealed, key_fields FROM events.sources + SELECT org_id, name, connector, token_sha256, verification, hmac_header, hmac_prefix, hmac_secret_sealed, key_fields, + display_name, delivery_header, unsigned_approved, capture FROM events.sources WHERE org_id = $1 AND name = $2`, orgID, name) if errors.Is(err, sql.ErrNoRows) { return Source{}, ErrNotFound @@ -116,7 +149,8 @@ func (s *Store) Source(ctx context.Context, orgID, name string) (Source, error) func (s *Store) Sources(ctx context.Context, orgID string) ([]Source, error) { var out []Source if err := s.db.SelectContext(ctx, &out, ` - SELECT org_id, name, connector, token_sha256, verification, hmac_header, hmac_prefix, hmac_secret_sealed, cursor, key_fields FROM events.sources + SELECT org_id, name, connector, token_sha256, verification, hmac_header, hmac_prefix, hmac_secret_sealed, cursor, key_fields, + display_name, delivery_header, unsigned_approved, capture FROM events.sources WHERE org_id = $1 ORDER BY name`, orgID); err != nil { return nil, fmt.Errorf("list sources: %w", err) } @@ -142,6 +176,8 @@ func (s *Store) UpsertSource(ctx context.Context, src Source) error { SET connector = EXCLUDED.connector, token_sha256 = EXCLUDED.token_sha256, verification = EXCLUDED.verification, hmac_header = EXCLUDED.hmac_header, hmac_prefix = EXCLUDED.hmac_prefix, hmac_secret_sealed = EXCLUDED.hmac_secret_sealed, + unsigned_approved = EXCLUDED.verification = 'none' AND events.sources.verification = 'none' + AND events.sources.unsigned_approved, updated_at = now()`, src); err != nil { return fmt.Errorf("save source: %w", err) } @@ -153,9 +189,11 @@ func (s *Store) UpsertSource(ctx context.Context, src Source) error { func (s *Store) Insert(ctx context.Context, e Event) (bool, error) { res, err := s.db.NamedExecContext(ctx, ` INSERT INTO events.events (id, org_id, source, type, source_event_id, link_key, fields, body, occurred_at, - own, origin_run_id, group_key, group_filter) + own, origin_run_id, group_key, group_filter, verification, headers, remote_addr, + event_type_id) VALUES (:id, :org_id, :source, :type, :source_event_id, :link_key, :fields, :body, :occurred_at, - :own, :origin_run_id, :group_key, :group_filter) + :own, :origin_run_id, :group_key, :group_filter, :verification, :headers, :remote_addr, + :event_type_id) ON CONFLICT (org_id, source, source_event_id) DO NOTHING`, e) if err != nil { return false, fmt.Errorf("store event: %w", err) @@ -179,8 +217,8 @@ func (s *Store) Dispatch(ctx context.Context, fn func(tx *sqlx.Tx, e Event) (mat defer func() { _ = tx.Rollback() }() var e Event err = tx.GetContext(ctx, &e, ` - SELECT id, org_id, source, type, source_event_id, link_key, linked_to, fields, occurred_at, - received_at, matched, dispatched_at, own, origin_run_id, group_key, group_filter + SELECT id, org_id, source, type, source_event_id, link_key, linked_to, fields, body, occurred_at, + received_at, matched, dispatched_at, own, origin_run_id, group_key, group_filter, event_type_id FROM events.events WHERE dispatched_at IS NULL ORDER BY received_at FOR UPDATE SKIP LOCKED LIMIT 1`) if errors.Is(err, sql.ErrNoRows) { @@ -225,13 +263,17 @@ func Link(ctx context.Context, tx *sqlx.Tx, e Event) (bool, error) { return n == 1, err } -// Triggers returns the enabled triggers for one source and event type. -func Triggers(ctx context.Context, tx *sqlx.Tx, orgID, source, eventType string) ([]Trigger, error) { +// Triggers returns the armed triggers that fire for one source and event type, +// and for the custom type the event was identified as, if any. +func Triggers(ctx context.Context, tx *sqlx.Tx, orgID, source, eventType string, typeID uuid.NullUUID) ([]Trigger, error) { var out []Trigger if err := tx.SelectContext(ctx, &out, ` - SELECT org_id, name, source, event_type, filter, workflow, enabled FROM events.triggers - WHERE org_id = $1 AND source = $2 AND event_type = $3 AND enabled ORDER BY name`, - orgID, source, eventType); err != nil { + SELECT org_id, name, source, event_type, filter, workflow, enabled, approved, event_type_id, mapping_id + FROM events.triggers + WHERE org_id = $1 AND source = $2 AND (event_type = $3 OR event_type = '') AND enabled AND approved + AND (event_type_id IS NULL OR event_type_id = $4) + ORDER BY name`, + orgID, source, eventType, typeID); err != nil { return nil, fmt.Errorf("read triggers: %w", err) } return out, nil @@ -250,13 +292,15 @@ func RecordDispatch(ctx context.Context, tx *sqlx.Tx, eventID uuid.UUID, trigger // UpsertTrigger creates or replaces a trigger. func (s *Store) UpsertTrigger(ctx context.Context, t Trigger) error { if _, err := s.db.NamedExecContext(ctx, ` - INSERT INTO events.triggers (org_id, name, source, event_type, filter, workflow, enabled, - schedule_every_s, schedule_since) - VALUES (:org_id, :name, :source, :event_type, :filter, :workflow, :enabled, + INSERT INTO events.triggers (org_id, name, source, event_type, filter, workflow, enabled, approved, + event_type_id, mapping_id, schedule_every_s, schedule_since) + VALUES (:org_id, :name, :source, :event_type, :filter, :workflow, :enabled, :approved, + :event_type_id, :mapping_id, CAST(:schedule_every_s AS integer), CASE WHEN CAST(:schedule_every_s AS integer) IS NULL THEN NULL ELSE now() END) ON CONFLICT (org_id, name) DO UPDATE SET source = EXCLUDED.source, event_type = EXCLUDED.event_type, filter = EXCLUDED.filter, - workflow = EXCLUDED.workflow, enabled = EXCLUDED.enabled, + workflow = EXCLUDED.workflow, enabled = EXCLUDED.enabled, approved = EXCLUDED.approved, + event_type_id = EXCLUDED.event_type_id, mapping_id = EXCLUDED.mapping_id, schedule_since = CASE WHEN EXCLUDED.schedule_every_s IS NULL THEN NULL WHEN events.triggers.schedule_every_s IS NOT DISTINCT FROM EXCLUDED.schedule_every_s @@ -272,9 +316,10 @@ func (s *Store) UpsertTrigger(ctx context.Context, t Trigger) error { func (s *Store) ScheduledTriggers(ctx context.Context, orgID string) ([]Trigger, error) { var out []Trigger if err := s.db.SelectContext(ctx, &out, ` - SELECT org_id, name, source, event_type, filter, workflow, enabled, schedule_every_s, schedule_since + SELECT org_id, name, source, event_type, filter, workflow, enabled, approved, event_type_id, mapping_id, + schedule_every_s, schedule_since FROM events.triggers - WHERE org_id = $1 AND enabled AND schedule_every_s IS NOT NULL ORDER BY name`, orgID); err != nil { + WHERE org_id = $1 AND enabled AND approved AND schedule_every_s IS NOT NULL ORDER BY name`, orgID); err != nil { return nil, fmt.Errorf("read schedules: %w", err) } return out, nil @@ -311,7 +356,8 @@ func (s *Store) Event(ctx context.Context, orgID string, id uuid.UUID) (Event, e var e Event err := s.db.GetContext(ctx, &e, ` SELECT id, org_id, source, type, source_event_id, link_key, linked_to, fields, body, occurred_at, - received_at, matched, dispatched_at, own, origin_run_id, group_key, group_filter + received_at, matched, dispatched_at, own, origin_run_id, group_key, group_filter, + verification, headers, remote_addr, event_type_id FROM events.events WHERE org_id = $1 AND id = $2`, orgID, id) if errors.Is(err, sql.ErrNoRows) { return Event{}, ErrNotFound @@ -344,7 +390,8 @@ func (s *Store) RunsForEvent(ctx context.Context, orgID string, id uuid.UUID) ([ func (s *Store) AllTriggers(ctx context.Context, orgID string) ([]Trigger, error) { var out []Trigger if err := s.db.SelectContext(ctx, &out, ` - SELECT org_id, name, source, event_type, filter, workflow, enabled, schedule_every_s, schedule_since + SELECT org_id, name, source, event_type, filter, workflow, enabled, approved, event_type_id, mapping_id, + schedule_every_s, schedule_since FROM events.triggers WHERE org_id = $1 ORDER BY name`, orgID); err != nil { return nil, fmt.Errorf("list triggers: %w", err) } diff --git a/modules/events/verify_test.go b/modules/events/verify_test.go index 5568a95f6..527338c6a 100644 --- a/modules/events/verify_test.go +++ b/modules/events/verify_test.go @@ -84,7 +84,7 @@ func TestHMACSourceVerifiesBeforeParsing(t *testing.T) { if tt.header != "" { req.Header.Set("X-Signature", tt.header) } - _, _, _, code, err := m.accept(req) + _, code, err := m.accept(req) if code != tt.wantCode || (err != nil) != (tt.wantCode != 0) { t.Fatalf("status %d, error %v; want status %d", code, err, tt.wantCode) } diff --git a/modules/governance/accounts_api.go b/modules/governance/accounts_api.go index 75b127476..4ea093ff7 100644 --- a/modules/governance/accounts_api.go +++ b/modules/governance/accounts_api.go @@ -608,13 +608,13 @@ func (m *Module) capabilities(ctx context.Context, _ *struct{}) (*httpapi.Reply[ sections := map[string]bool{"chat": signedIn, "memory": signedIn} for _, name := range []string{ "settings", "events", "workflows", "runs", "credentials", "extensions", "secret_engines", "models", "controls", "memory_admin", - "home", "system_status", "org_config", "workflow_editor", + "home", "system_status", "org_config", "captures", "mappings", "bindings", "workflow_editor", } { sections[name] = p.InstanceAdmin() } for _, name := range []string{ "tasks", "work_requests", "logs", "skills", "curators", "channels", - "captures", "mappings", "bindings", "extension_catalogue", "model_aliases", + "extension_catalogue", "model_aliases", "task_execution", "organisations", "tools", "plugins", "container_runtime", "agent_profiles", "org_agents", "org_audit", "org_usage", "org_work_intake", } { diff --git a/ui/src/bindings/BindingEditor.vue b/ui/src/bindings/BindingEditor.vue index cbc341925..fec0e1e26 100644 --- a/ui/src/bindings/BindingEditor.vue +++ b/ui/src/bindings/BindingEditor.vue @@ -1,6 +1,6 @@