Files
archie/modules/events/api_capture.go
T

528 lines
20 KiB
Go

package events
import (
"context"
json "encoding/json/v2"
"errors"
"net/http"
"slices"
"time"
"github.com/danielgtaylor/huma/v2"
"github.com/google/uuid"
"github.com/samcharles93/archie-core/internal/platform/httpapi"
"github.com/samcharles93/archie-core/modules/events/internal/eventtype"
"github.com/samcharles93/archie-core/modules/events/internal/mapping"
"github.com/samcharles93/archie-core/modules/events/internal/store"
"github.com/samcharles93/archie-core/modules/governance"
)
// registerCapture adds the operations for captured deliveries, event types and
// mappings. Every one is administrator-only, like the rest of the events API.
func (m *Module) registerCapture(api huma.API) {
tags := []string{"Events"}
op := func(id, method, path, summary string) huma.Operation {
return huma.Operation{OperationID: id, Method: method, Path: path, Summary: summary, Tags: tags}
}
capture := func(action governance.Action) httpapi.Access {
return httpapi.AdminAuthorized(action, governance.KindCapture)
}
httpapi.Register(api, capture(governance.ActionRead), op("list-captures", http.MethodGet, "/api/captures", "List recent deliveries a source kept"), m.captures)
httpapi.Register(api, capture(governance.ActionRead), op("list-capture-dispatches", http.MethodGet, "/api/captures/{id}/dispatches", "List the runs a capture started"), m.captureDispatches)
eventType := func(action governance.Action) httpapi.Access {
return httpapi.AdminAuthorized(action, governance.KindEventType)
}
httpapi.Register(api, eventType(governance.ActionRead), op("list-event-types", http.MethodGet, "/api/event-types", "List event types and the types unidentified captures suggest"), m.eventTypes)
httpapi.Register(api, eventType(governance.ActionCreate), op("create-event-type", http.MethodPost, "/api/event-types", "Name a kind of event on a source"), m.createEventType)
httpapi.Register(api, eventType(governance.ActionUpdate), op("update-event-type", http.MethodPut, "/api/event-types/{id}", "Rename an event type or change its rule"), m.updateEventType)
httpapi.Register(api, eventType(governance.ActionDelete), op("delete-event-type", http.MethodDelete, "/api/event-types/{id}", "Delete an event type"), m.deleteEventType)
mapped := func(action governance.Action) httpapi.Access {
return httpapi.AdminAuthorized(action, governance.KindMapping)
}
httpapi.Register(api, mapped(governance.ActionRead), op("list-mappings", http.MethodGet, "/api/mappings", "List field mappings with how often each matched"), m.mappings)
httpapi.Register(api, mapped(governance.ActionRead), op("get-mapping", http.MethodGet, "/api/mappings/{id}", "Get a field mapping"), m.mapping)
httpapi.Register(api, mapped(governance.ActionCreate), op("create-mapping", http.MethodPost, "/api/mappings", "Create a field mapping"), m.createMapping)
httpapi.Register(api, mapped(governance.ActionUpdate), op("update-mapping", http.MethodPatch, "/api/mappings/{id}", "Change a field mapping"), m.updateMapping)
httpapi.Register(api, mapped(governance.ActionDelete), op("delete-mapping", http.MethodDelete, "/api/mappings/{id}", "Delete a field mapping"), m.deleteMapping)
httpapi.Register(api, mapped(governance.ActionRead), op("preview-mapping", http.MethodPost, "/api/mappings/preview", "Read a captured payload through candidate fields"), m.previewMapping)
}
// captureView is a stored delivery as the inspector shows it.
type captureView struct {
ID uuid.UUID `json:"id"`
ReceivedAt time.Time `json:"received_at"`
Source string `json:"source"`
RemoteAddr string `json:"remote_addr,omitempty"`
ContentType string `json:"content_type,omitempty"`
Headers string `json:"headers"`
Body string `json:"body"`
// Verification is how the delivery was verified: signed, token or unsigned.
Verification string `json:"verification" enum:"signed,token,unsigned,"`
// EventType is the id of the event type the delivery was identified as.
EventType string `json:"event_type,omitempty"`
}
type capturesInput struct {
Limit int `query:"limit" minimum:"1" maximum:"500" default:"100"`
}
type capturesBody struct {
Captures []captureView `json:"captures"`
// Enabled is false when no source keeps deliveries.
Enabled bool `json:"enabled"`
}
func (m *Module) captures(ctx context.Context, in *capturesInput) (*httpapi.Reply[capturesBody], error) {
rows, err := m.store.Captures(ctx, m.orgID, in.Limit)
if err != nil {
return nil, huma.Error500InternalServerError("couldn't list captures", err)
}
sources, err := m.store.Sources(ctx, m.orgID)
if err != nil {
return nil, huma.Error500InternalServerError("couldn't list sources", err)
}
enabled := len(sources) == 0 || slices.ContainsFunc(sources, func(s store.Source) bool { return s.Capture })
out := capturesBody{Captures: make([]captureView, 0, len(rows)), Enabled: enabled}
for _, e := range rows {
out.Captures = append(out.Captures, captureOf(e))
}
return httpapi.OK(out), nil
}
func captureOf(e store.Event) captureView {
v := captureView{
ID: e.ID, ReceivedAt: e.ReceivedAt, Source: e.Source, RemoteAddr: e.RemoteAddr,
Headers: string(e.Headers), Body: string(e.Body), Verification: e.Verification,
}
v.ContentType = eventtype.ParseHeaders(v.Headers)["content-type"]
if e.EventTypeID.Valid {
v.EventType = e.EventTypeID.UUID.String()
}
return v
}
type dispatchView struct {
EventID uuid.UUID `json:"event_id"`
RunID uuid.UUID `json:"run_id"`
Trigger string `json:"trigger,omitempty"`
CreatedAt time.Time `json:"created_at,omitzero"`
}
type idInput struct {
ID string `path:"id" format:"uuid"`
}
func (m *Module) captureDispatches(ctx context.Context, in *idInput) (*httpapi.Reply[[]dispatchView], error) {
id, err := uuid.Parse(in.ID)
if err != nil {
return nil, huma.Error400BadRequest("a capture id is a UUID")
}
runs, err := m.store.RunsForEvent(ctx, m.orgID, id)
if err != nil {
return nil, huma.Error500InternalServerError("couldn't read the runs of the capture", err)
}
out := make([]dispatchView, 0, len(runs))
for _, r := range runs {
out = append(out, dispatchView{EventID: id, RunID: r.RunID, Trigger: r.Trigger})
}
return httpapi.OK(out), nil
}
// eventTypeView is an event type as the inspector reads it.
type eventTypeView struct {
ID uuid.UUID `json:"id"`
Source string `json:"source"`
Name string `json:"name"`
Rule eventtype.Rule `json:"rule"`
Schema map[string]eventtype.ValueType `json:"schema"`
}
type eventTypesBody struct {
Enabled bool `json:"enabled"`
EventTypes []eventTypeView `json:"event_types"`
Proposals []eventtype.Proposal `json:"proposals"`
}
// proposalScan bounds how many recent captures are grouped into proposals.
const proposalScan = 500
func (m *Module) eventTypes(ctx context.Context, _ *struct{}) (*httpapi.Reply[eventTypesBody], error) {
rows, err := m.store.EventTypes(ctx, m.orgID)
if err != nil {
return nil, huma.Error500InternalServerError("couldn't list event types", err)
}
types, err := typesOf(rows)
if err != nil {
return nil, huma.Error500InternalServerError("couldn't read the event types", err)
}
out := eventTypesBody{Enabled: true, EventTypes: make([]eventTypeView, 0, len(types))}
for i, t := range types {
out.EventTypes = append(out.EventTypes, eventTypeView{ID: rows[i].ID, Source: t.Source, Name: t.Name, Rule: t.Rule, Schema: t.Schema})
}
captured, err := m.store.Captures(ctx, m.orgID, proposalScan)
if err != nil {
return nil, huma.Error500InternalServerError("couldn't list captures", err)
}
out.Proposals = eventtype.Propose(unidentified(types, captured))
if out.Proposals == nil {
out.Proposals = []eventtype.Proposal{}
}
return httpapi.OK(out), nil
}
// unidentified keeps the captures that no event type was named for on arrival
// and that no current type would identify either.
func unidentified(types []eventtype.EventType, captured []store.Event) []eventtype.Capture {
var out []eventtype.Capture
for _, e := range captured {
if e.EventTypeID.Valid {
continue
}
sample := eventtype.Sample{Headers: eventtype.ParseHeaders(string(e.Headers)), Body: e.Body}
if _, ok := eventtype.Identify(types, e.Source, sample); ok {
continue
}
out = append(out, eventtype.Capture{ID: e.ID.String(), Source: e.Source, ReceivedAt: e.ReceivedAt, Sample: sample})
}
return out
}
type createEventTypeInput struct {
Body struct {
Source string `json:"source"`
Name string `json:"name"`
Rule *eventtype.Rule `json:"rule,omitzero" doc:"The conditions that identify the type; inferred from the example when absent"`
Example struct {
Headers map[string]string `json:"headers,omitzero"`
Body string `json:"body,omitzero"`
} `json:"example,omitzero"`
}
}
func (m *Module) createEventType(ctx context.Context, in *createEventTypeInput) (*httpapi.Reply[eventTypeView], error) {
req := in.Body
t := eventtype.FromExample(req.Source, req.Name, eventtype.Sample{Headers: req.Example.Headers, Body: []byte(req.Example.Body)})
if req.Rule != nil {
t.Rule = *req.Rule
}
if err := t.Validate(); err != nil {
return nil, huma.Error400BadRequest(err.Error())
}
if _, err := m.store.Source(ctx, m.orgID, t.Source); errors.Is(err, store.ErrNotFound) {
return nil, huma.Error400BadRequest("no source named " + t.Source)
} else if err != nil {
return nil, huma.Error500InternalServerError("couldn't read the source", err)
}
if err := m.checkOverlap(ctx, t); err != nil {
return nil, err
}
rule, err := json.Marshal(t.Rule)
if err != nil {
return nil, huma.Error400BadRequest(err.Error())
}
schema, err := json.Marshal(t.Schema)
if err != nil {
return nil, huma.Error400BadRequest(err.Error())
}
id := uuid.New()
err = m.store.InsertEventType(ctx, store.EventType{ID: id, OrgID: m.orgID, Source: t.Source, Name: t.Name, Rule: rule, Schema: schema})
if errors.Is(err, store.ErrExists) {
return nil, huma.Error409Conflict("the source already has an event type named " + t.Name)
}
if err != nil {
return nil, huma.Error500InternalServerError("couldn't save the event type", err)
}
return httpapi.OK(eventTypeView{ID: id, Source: t.Source, Name: t.Name, Rule: t.Rule, Schema: t.Schema}), nil
}
// checkOverlap refuses a type whose rule can match events another type on the
// source already matches, since an event is identified only when one type
// matches it.
func (m *Module) checkOverlap(ctx context.Context, candidate eventtype.EventType) error {
rows, err := m.store.EventTypesOf(ctx, m.orgID, candidate.Source)
if err != nil {
return huma.Error500InternalServerError("couldn't list event types", err)
}
existing, err := typesOf(rows)
if err != nil {
return huma.Error500InternalServerError("couldn't read the event types", err)
}
if err := eventtype.CheckOverlap(existing, candidate); err != nil {
return huma.Error409Conflict("this rule can match events another type on the source already matches")
}
return nil
}
type updateEventTypeInput struct {
ID string `path:"id" format:"uuid"`
Body struct {
Name string `json:"name"`
Rule eventtype.Rule `json:"rule"`
}
}
type savedID struct {
ID string `json:"id"`
}
func (m *Module) updateEventType(ctx context.Context, in *updateEventTypeInput) (*httpapi.Reply[savedID], error) {
id, err := uuid.Parse(in.ID)
if err != nil {
return nil, huma.Error400BadRequest("an event type id is a UUID")
}
stored, err := m.store.EventType(ctx, m.orgID, id)
if errors.Is(err, store.ErrNotFound) {
return nil, huma.Error404NotFound("no such event type")
}
if err != nil {
return nil, huma.Error500InternalServerError("couldn't read the event type", err)
}
t := eventtype.EventType{ID: in.ID, Source: stored.Source, Name: in.Body.Name, Rule: in.Body.Rule}
if err := t.Validate(); err != nil {
return nil, huma.Error400BadRequest(err.Error())
}
if err := m.checkOverlap(ctx, t); err != nil {
return nil, err
}
rule, err := json.Marshal(t.Rule)
if err != nil {
return nil, huma.Error400BadRequest(err.Error())
}
err = m.store.UpdateEventType(ctx, m.orgID, id, t.Name, rule)
if errors.Is(err, store.ErrExists) {
return nil, huma.Error409Conflict("the source already has an event type named " + t.Name)
}
if err != nil {
return nil, huma.Error500InternalServerError("couldn't save the event type", err)
}
return httpapi.OK(savedID{ID: in.ID}), nil
}
func (m *Module) deleteEventType(ctx context.Context, in *idInput) (*struct{}, error) {
id, err := uuid.Parse(in.ID)
if err != nil {
return nil, huma.Error400BadRequest("an event type id is a UUID")
}
err = m.store.DeleteEventType(ctx, m.orgID, id)
switch {
case errors.Is(err, store.ErrNotFound):
return nil, huma.Error404NotFound("no such event type")
case errors.Is(err, store.ErrInUse):
return nil, huma.Error409Conflict("a mapping or trigger uses this event type")
case err != nil:
return nil, huma.Error500InternalServerError("couldn't delete the event type", err)
}
return nil, nil
}
// mappingView is a mapping as the page lists it.
type mappingView struct {
ID uuid.UUID `json:"id"`
Name string `json:"name"`
SourceHint string `json:"source_hint"`
EventTypeID uuid.UUID `json:"event_type_id"`
Fields []mapping.Field `json:"fields"`
MatchCount int64 `json:"match_count"`
LastMatchedAt *time.Time `json:"last_matched_at,omitempty"`
CreatedAt time.Time `json:"created_at"`
UpdatedAt time.Time `json:"updated_at"`
}
func mappingOf(r store.Mapping) (mappingView, error) {
v := mappingView{
ID: r.ID, Name: r.Name, SourceHint: r.SourceHint, EventTypeID: r.EventTypeID,
MatchCount: r.MatchCount, CreatedAt: r.CreatedAt, UpdatedAt: r.UpdatedAt,
}
if err := json.Unmarshal(r.Fields, &v.Fields); err != nil {
return v, err
}
if r.LastMatchedAt.Valid {
v.LastMatchedAt = &r.LastMatchedAt.Time
}
return v, nil
}
func (m *Module) mappings(ctx context.Context, _ *struct{}) (*httpapi.Reply[struct {
Mappings []mappingView `json:"mappings"`
}], error,
) {
rows, err := m.store.Mappings(ctx, m.orgID)
if err != nil {
return nil, huma.Error500InternalServerError("couldn't list mappings", err)
}
var out struct {
Mappings []mappingView `json:"mappings"`
}
out.Mappings = make([]mappingView, 0, len(rows))
for _, r := range rows {
v, err := mappingOf(r)
if err != nil {
return nil, huma.Error500InternalServerError("couldn't read mapping "+r.Name, err)
}
out.Mappings = append(out.Mappings, v)
}
return httpapi.OK(out), nil
}
func (m *Module) mapping(ctx context.Context, in *idInput) (*httpapi.Reply[mappingView], error) {
id, err := uuid.Parse(in.ID)
if err != nil {
return nil, huma.Error400BadRequest("a mapping id is a UUID")
}
row, err := m.store.Mapping(ctx, m.orgID, id)
if errors.Is(err, store.ErrNotFound) {
return nil, huma.Error404NotFound("no such mapping")
}
if err != nil {
return nil, huma.Error500InternalServerError("couldn't read the mapping", err)
}
v, err := mappingOf(row)
if err != nil {
return nil, huma.Error500InternalServerError("couldn't read the mapping", err)
}
return httpapi.OK(v), nil
}
type mappingBody struct {
Name string `json:"name"`
SourceHint string `json:"source_hint,omitzero"`
EventTypeID string `json:"event_type_id" format:"uuid"`
Fields []mapping.Field `json:"fields"`
}
type createMappingInput struct{ Body mappingBody }
type updateMappingInput struct {
ID string `path:"id" format:"uuid"`
Body mappingBody
}
// checkedMapping validates a request and fits its fields to the event type's
// schema.
func (m *Module) checkedMapping(ctx context.Context, b mappingBody) (store.Mapping, error) {
typeID, err := uuid.Parse(b.EventTypeID)
if err != nil {
return store.Mapping{}, huma.Error400BadRequest("an event type id is a UUID")
}
candidate := mapping.Mapping{Name: b.Name, Fields: b.Fields}
if err := candidate.Validate(); err != nil {
return store.Mapping{}, huma.Error400BadRequest(err.Error())
}
stored, err := m.store.EventType(ctx, m.orgID, typeID)
if errors.Is(err, store.ErrNotFound) {
return store.Mapping{}, huma.Error400BadRequest("mapping: event type not found: " + b.EventTypeID)
}
if err != nil {
return store.Mapping{}, huma.Error500InternalServerError("couldn't read the event type", err)
}
t, err := typeOf(stored)
if err != nil {
return store.Mapping{}, huma.Error500InternalServerError("couldn't read the event type", err)
}
if err := mapping.CheckSchema(b.Fields, t.Schema); err != nil {
return store.Mapping{}, huma.Error400BadRequest(err.Error())
}
fields, err := json.Marshal(b.Fields)
if err != nil {
return store.Mapping{}, huma.Error400BadRequest(err.Error())
}
return store.Mapping{OrgID: m.orgID, Name: b.Name, SourceHint: b.SourceHint, EventTypeID: typeID, Fields: fields}, nil
}
func (m *Module) createMapping(ctx context.Context, in *createMappingInput) (*httpapi.Reply[mappingView], error) {
row, err := m.checkedMapping(ctx, in.Body)
if err != nil {
return nil, err
}
row.ID = uuid.New()
if err := m.store.InsertMapping(ctx, row); err != nil {
return nil, huma.Error500InternalServerError("couldn't save the mapping", err)
}
return m.mappingReply(ctx, row.ID)
}
func (m *Module) updateMapping(ctx context.Context, in *updateMappingInput) (*httpapi.Reply[mappingView], error) {
id, err := uuid.Parse(in.ID)
if err != nil {
return nil, huma.Error400BadRequest("a mapping id is a UUID")
}
row, err := m.checkedMapping(ctx, in.Body)
if err != nil {
return nil, err
}
row.ID = id
err = m.store.UpdateMapping(ctx, row)
if errors.Is(err, store.ErrNotFound) {
return nil, huma.Error404NotFound("no such mapping")
}
if err != nil {
return nil, huma.Error500InternalServerError("couldn't save the mapping", err)
}
return m.mappingReply(ctx, id)
}
func (m *Module) mappingReply(ctx context.Context, id uuid.UUID) (*httpapi.Reply[mappingView], error) {
row, err := m.store.Mapping(ctx, m.orgID, id)
if err != nil {
return nil, huma.Error500InternalServerError("couldn't read the mapping", err)
}
v, err := mappingOf(row)
if err != nil {
return nil, huma.Error500InternalServerError("couldn't read the mapping", err)
}
return httpapi.OK(v), nil
}
func (m *Module) deleteMapping(ctx context.Context, in *idInput) (*struct{}, error) {
id, err := uuid.Parse(in.ID)
if err != nil {
return nil, huma.Error400BadRequest("a mapping id is a UUID")
}
err = m.store.DeleteMapping(ctx, m.orgID, id)
switch {
case errors.Is(err, store.ErrNotFound):
return nil, huma.Error404NotFound("no such mapping")
case errors.Is(err, store.ErrInUse):
return nil, huma.Error409Conflict("a trigger uses this mapping")
case err != nil:
return nil, huma.Error500InternalServerError("couldn't delete the mapping", err)
}
return nil, nil
}
type previewInput struct {
Body struct {
CaptureID string `json:"capture_id" format:"uuid"`
Fields []mapping.Field `json:"fields"`
}
}
type previewBody struct {
Values map[string]any `json:"values"`
Failures []mapping.Failure `json:"failures"`
}
// previewMapping reads a stored payload through candidate fields, so a mapping
// can be checked before it is saved.
func (m *Module) previewMapping(ctx context.Context, in *previewInput) (*httpapi.Reply[previewBody], error) {
id, err := uuid.Parse(in.Body.CaptureID)
if err != nil {
return nil, huma.Error400BadRequest("a capture id is a UUID")
}
e, err := m.store.Event(ctx, m.orgID, id)
if errors.Is(err, store.ErrNotFound) {
return nil, huma.Error404NotFound("no such capture")
}
if err != nil {
return nil, huma.Error500InternalServerError("couldn't read the capture", err)
}
values, failures := mapping.Resolve(in.Body.Fields, e.Body)
if failures == nil {
failures = []mapping.Failure{}
}
return httpapi.OK(previewBody{Values: values, Failures: failures}), nil
}