package events import ( "context" "database/sql" "encoding/json/jsontext" json "encoding/json/v2" "errors" "fmt" "net/http" "strconv" "time" "github.com/danielgtaylor/huma/v2" "github.com/google/uuid" "github.com/samcharles93/archie-core/internal/platform/httpapi" "github.com/samcharles93/archie-core/internal/platform/httpjson" "github.com/samcharles93/archie-core/modules/events/internal/store" "github.com/samcharles93/archie-core/modules/governance" "github.com/samcharles93/archie-core/modules/runs" ) // Register adds the events API: sources, triggers, groups of unmatched events // and the events themselves. func (m *Module) Register(api huma.API) { tags := []string{"Events"} httpapi.Register(api, httpapi.AdminAuthorized(governance.ActionRead, governance.KindSource), huma.Operation{ OperationID: "list-sources", Method: http.MethodGet, Path: "/api/sources", Summary: "List sources", Tags: tags, }, m.sources) httpapi.Register(api, httpapi.AdminAuthorized(governance.ActionUpdate, governance.KindSource), huma.Operation{ OperationID: "put-source", Method: http.MethodPut, Path: "/api/sources/{name}", Summary: "Add or update a source, its verification scheme and its token", Tags: tags, }, m.putSource) httpapi.Register(api, httpapi.AdminAuthorized(governance.ActionUpdate, governance.KindSource), huma.Operation{ OperationID: "put-source-key-fields", Method: http.MethodPut, Path: "/api/sources/{name}/key-fields", Summary: "Set the fields a source's events are grouped by", Tags: tags, }, m.putKeyFields) httpapi.Register(api, httpapi.AdminAuthorized(governance.ActionRead, governance.KindTrigger), huma.Operation{ OperationID: "list-triggers", Method: http.MethodGet, Path: "/api/triggers", Summary: "List triggers", Tags: tags, }, m.triggers) httpapi.Register(api, httpapi.AdminAuthorized(governance.ActionUpdate, governance.KindTrigger), huma.Operation{ OperationID: "put-trigger", Method: http.MethodPut, Path: "/api/triggers/{name}", Summary: "Add or change a trigger", Tags: tags, }, m.putTrigger) httpapi.Register(api, httpapi.AdminAuthorized(governance.ActionRead, governance.KindEventIngest), huma.Operation{ OperationID: "list-groups", Method: http.MethodGet, Path: "/api/groups", Summary: "List groups of unmatched events", Tags: tags, }, m.groups) httpapi.Register(api, httpapi.AdminAuthorized(governance.ActionRead, governance.KindEventIngest), huma.Operation{ OperationID: "list-events", Method: http.MethodGet, Path: "/api/events", Summary: "List recent events", Tags: tags, }, m.recent) httpapi.Register(api, httpapi.AdminAuthorized(governance.ActionRead, governance.KindEventIngest), huma.Operation{ OperationID: "get-event", Method: http.MethodGet, Path: "/api/events/{id}", Summary: "Get an event with its payload and the runs it started", Tags: tags, }, m.event) } type putSourceInput struct { Name string `path:"name" doc:"Lowercase letters, digits and hyphens; the webhook URL ends in it"` Body struct { Connector string `json:"connector" doc:"The connector that parses this source's events"` // A new source without a scheme uses a bearer token, so existing // senders keep working. Verification string `json:"verification,omitzero" enum:"token,hmac,none" doc:"How deliveries are verified: a bearer token, an HMAC-SHA256 signature of the body, or no verification"` HMACHeader string `json:"hmac_header,omitzero" doc:"The header that carries the signature (hmac only)"` HMACPrefix string `json:"hmac_prefix,omitzero" doc:"Text before the hex digest in the signature header, such as sha256= (hmac only)" example:"sha256="` HMACSecret string `json:"hmac_secret,omitzero" doc:"The shared signing secret (hmac only). Write-only: it is sealed and never returned"` } } type sourceCreated struct { Source string `json:"source"` Token string `json:"token,omitzero" doc:"Returned once for a token source; only its hash is stored"` URL string `json:"url" doc:"Where the sender posts its events"` } // putSource creates or updates a source. A save that omits verification keeps // the stored scheme and its credentials. A token is returned only when the // token scheme is chosen explicitly or the source is new, and only its hash is // stored. An HMAC secret is sealed and never returned; a blank secret keeps the // sealed one. func (m *Module) putSource(ctx context.Context, in *putSourceInput) (*httpapi.Reply[sourceCreated], error) { if !validName.MatchString(in.Name) || !validName.MatchString(in.Body.Connector) { return nil, huma.Error400BadRequest("a source needs a lowercase name and a connector") } existing, err := m.store.Source(ctx, m.orgID, in.Name) if err != nil && !errors.Is(err, store.ErrNotFound) { return nil, huma.Error500InternalServerError("couldn't read the source", err) } var stored *store.Source if err == nil { stored = &existing } src, token, err := m.sourceToSave(in, stored) if err != nil { return nil, err } if err := m.store.UpsertSource(ctx, src); err != nil { return nil, huma.Error500InternalServerError("couldn't save the source", err) } return httpapi.OK(sourceCreated{Source: in.Name, Token: token, URL: "/hooks/" + in.Name}), nil } // sourceToSave builds the row to store from the request and the stored source, // if there is one. It returns the new token when one was issued. func (m *Module) sourceToSave(in *putSourceInput, stored *store.Source) (store.Source, string, error) { explicit := in.Body.Verification != "" src := store.Source{OrgID: m.orgID, Name: in.Name, Connector: in.Body.Connector, Verification: in.Body.Verification} switch { case !explicit && stored != nil: // Omitting the scheme keeps the stored one with its token or secret, so // a save cannot downgrade a source. src = *stored src.Connector = in.Body.Connector case !explicit: src.Verification = store.Token } switch src.Verification { case store.Token: if stored != nil && !explicit { return src, "", nil } token, err := newToken() if err != nil { return src, "", huma.Error500InternalServerError("couldn't generate a token", err) } src.TokenSHA256 = httpjson.TokenHash(token) return src, token, nil case store.None: return src, "", nil case store.HMAC: if explicit { if err := m.setHMAC(&src, in, stored); err != nil { return src, "", err } } if src.HMACHeader == "" || len(src.HMACSecretSealed) == 0 { return src, "", huma.Error400BadRequest("an HMAC source needs a signature header and a secret") } return src, "", nil default: return src, "", huma.Error400BadRequest("verification must be token, hmac or none") } } // setHMAC applies the requested HMAC settings. A blank secret keeps the sealed // secret of a source that already uses HMAC. func (m *Module) setHMAC(src *store.Source, in *putSourceInput, stored *store.Source) error { src.HMACHeader, src.HMACPrefix = in.Body.HMACHeader, in.Body.HMACPrefix switch { case in.Body.HMACSecret != "": sealed, err := m.sealer.Seal(secretLabel(m.orgID, in.Name), []byte(in.Body.HMACSecret)) if err != nil { return huma.Error500InternalServerError("couldn't seal the secret", err) } src.HMACSecretSealed = sealed case stored != nil && stored.Verification == store.HMAC: src.HMACSecretSealed = stored.HMACSecretSealed } return nil } type putTriggerInput struct { Name string `path:"name" doc:"Lowercase letters, digits and hyphens"` Body struct { Source string `json:"source,omitzero" doc:"The source whose events fire it; fixed when every is set"` EventType string `json:"event_type,omitzero" doc:"The event type that fires it; fixed when every is set"` Filter map[string]string `json:"filter,omitzero" doc:"Event fields that must match exactly"` Workflow string `json:"workflow" doc:"The workflow each match runs"` Disabled bool `json:"disabled,omitzero"` Every string `json:"every,omitzero" doc:"Fire on an interval instead of on events, such as 1h" example:"1h"` } } type triggerSaved struct { Trigger string `json:"trigger"` } func (m *Module) putTrigger(ctx context.Context, in *putTriggerInput) (*httpapi.Reply[triggerSaved], error) { req := in.Body var every sql.NullInt64 if req.Every != "" { d, err := time.ParseDuration(req.Every) if err != nil || d < minInterval || d%time.Second != 0 { return nil, huma.Error400BadRequest("every is a whole number of seconds of at least one minute, such as 1h") } every = sql.NullInt64{Int64: int64(d / time.Second), Valid: true} req.Source, req.EventType = scheduleSource, scheduleType // Every schedule's ticks share a source and type, so the filter keeps // this trigger to its own. req.Filter = map[string]string{"trigger": in.Name} } if !validName.MatchString(in.Name) || req.Source == "" || req.EventType == "" || req.Workflow == "" { return nil, huma.Error400BadRequest("a trigger needs a lowercase name, a workflow, and either a source and event type or an every interval") } if err := m.runs.CheckTrigger(ctx, m.orgID, req.Workflow); err != nil { if refused, ok := errors.AsType[*runs.InvalidError](err); ok { return nil, huma.Error400BadRequest(refused.Problems[0].Message, &huma.ErrorDetail{Location: "body.workflow", Message: refused.Problems[0].Message}) } return nil, huma.Error500InternalServerError("couldn't check the workflow", err) } if err := m.checkEventType(ctx, req.Source, req.EventType); err != nil { return nil, err } filter, err := json.Marshal(req.Filter) if err != nil { return nil, huma.Error400BadRequest(err.Error()) } if err := m.store.UpsertTrigger(ctx, store.Trigger{ OrgID: m.orgID, Name: in.Name, Source: req.Source, EventType: req.EventType, Filter: filter, Workflow: req.Workflow, Enabled: !req.Disabled, ScheduleEverySeconds: every, }); err != nil { return nil, huma.Error500InternalServerError("couldn't save the trigger", err) } return httpapi.OK(triggerSaved{Trigger: in.Name}), nil } // checkEventType refuses a trigger on an event type that the source's // connector doesn't declare, so the trigger can't be saved when it could never // fire. Schedule triggers have no connector. func (m *Module) checkEventType(ctx context.Context, source, eventType string) error { if source == scheduleSource { return nil } refuse := func(location, message string) error { return huma.Error400BadRequest(message, &huma.ErrorDetail{Location: location, Message: message}) } src, err := m.reader.Source(ctx, m.orgID, source) if errors.Is(err, store.ErrNotFound) { return refuse("body.source", fmt.Sprintf("no source named %q", source)) } if err != nil { return huma.Error500InternalServerError("couldn't read the source", err) } caps, ok, err := m.connectors.Capabilities(ctx, src.Connector) if err != nil { return huma.Error500InternalServerError("couldn't read what the connector declares", err) } if !ok { return refuse("body.event_type", fmt.Sprintf("connector %q isn't accepted or has never loaded, so its event types aren't known", src.Connector)) } if _, declared := caps.Event(eventType); !declared { return refuse("body.event_type", fmt.Sprintf("connector %q declares no event type %q", src.Connector, eventType)) } return nil } type putKeyFieldsInput struct { Name string `path:"name"` Body struct { Paths []string `json:"paths,omitzero" doc:"Paths into the event's fields; none uses the connector's default"` } } type keyFieldsSaved struct { KeyFields []string `json:"key_fields"` } // putKeyFields sets the fields a source's events are grouped by. It applies to // events that arrive after it. func (m *Module) putKeyFields(ctx context.Context, in *putKeyFieldsInput) (*httpapi.Reply[keyFieldsSaved], error) { paths := in.Body.Paths if paths == nil { paths = []string{} } raw, err := json.Marshal(paths) if err != nil { return nil, huma.Error400BadRequest(err.Error()) } err = m.store.SetKeyFields(ctx, m.orgID, in.Name, raw) if errors.Is(err, store.ErrNotFound) { return nil, huma.Error404NotFound("no such source") } if err != nil { return nil, huma.Error500InternalServerError("couldn't save the key fields", err) } return httpapi.OK(keyFieldsSaved{KeyFields: paths}), nil } // groupView is a group of unmatched events as the API returns it. Proposed is // the trigger it suggests, with the workflow left for the person to choose. type groupView struct { ID uuid.UUID `json:"id"` Count int64 `json:"count"` FirstSeen time.Time `json:"first_seen"` LastSeen time.Time `json:"last_seen"` Proposed proposedTrigger `json:"proposed"` } type proposedTrigger struct { Source string `json:"source"` EventType string `json:"event_type"` Filter jsontext.Value `json:"filter"` Workflow string `json:"workflow"` } func (m *Module) groups(ctx context.Context, _ *struct{}) (*httpapi.Reply[[]groupView], error) { groups, err := m.store.Groups(ctx, m.orgID, 200) if err != nil { m.log.ErrorContext(ctx, "list groups", "error", err) return nil, huma.Error500InternalServerError("couldn't list groups") } out := make([]groupView, 0, len(groups)) for _, g := range groups { out = append(out, groupView{ ID: g.ID, Count: g.Count, FirstSeen: g.FirstSeen, LastSeen: g.LastSeen, Proposed: proposedTrigger{Source: g.Source, EventType: g.EventType, Filter: g.Filter}, }) } return httpapi.OK(out), nil } // sourceView is a source as the API returns it. The token is returned only // when it is created or rotated, never here. type sourceView struct { Name string `json:"name"` Connector string `json:"connector"` KeyFields []string `json:"key_fields"` TokenSet bool `json:"token_set"` // Verification and the HMAC header and prefix are shown; the secret is // only reported as set. Verification string `json:"verification"` HMACHeader string `json:"hmac_header,omitzero"` HMACPrefix string `json:"hmac_prefix,omitzero"` HMACSecretSet bool `json:"hmac_secret_set"` } func (m *Module) sources(ctx context.Context, _ *struct{}) (*httpapi.Reply[[]sourceView], error) { sources, err := m.store.Sources(ctx, m.orgID) if err != nil { m.log.ErrorContext(ctx, "list sources", "error", err) return nil, huma.Error500InternalServerError("couldn't list sources") } out := make([]sourceView, 0, len(sources)) for _, src := range sources { view := sourceView{ Name: src.Name, Connector: src.Connector, TokenSet: src.TokenSHA256 != "", KeyFields: []string{}, Verification: src.Verification, HMACHeader: src.HMACHeader, HMACPrefix: src.HMACPrefix, HMACSecretSet: len(src.HMACSecretSealed) > 0, } if len(src.KeyFields) > 0 { if err := json.Unmarshal(src.KeyFields, &view.KeyFields); err != nil { m.log.ErrorContext(ctx, "read source key fields", "source", src.Name, "error", err) return nil, huma.Error500InternalServerError("couldn't read the key fields of " + src.Name) } } out = append(out, view) } return httpapi.OK(out), nil } // triggerView is a trigger as the API returns it. type triggerView struct { Name string `json:"name"` Source string `json:"source"` EventType string `json:"event_type"` Filter map[string]string `json:"filter"` Workflow string `json:"workflow"` Enabled bool `json:"enabled"` Every string `json:"every,omitempty"` } func (m *Module) triggers(ctx context.Context, _ *struct{}) (*httpapi.Reply[[]triggerView], error) { triggers, err := m.store.AllTriggers(ctx, m.orgID) if err != nil { m.log.ErrorContext(ctx, "list triggers", "error", err) return nil, huma.Error500InternalServerError("couldn't list triggers") } out := make([]triggerView, 0, len(triggers)) for _, t := range triggers { view := triggerView{Name: t.Name, Source: t.Source, EventType: t.EventType, Workflow: t.Workflow, Enabled: t.Enabled, Filter: map[string]string{}} if len(t.Filter) > 0 { if err := json.Unmarshal(t.Filter, &view.Filter); err != nil { m.log.ErrorContext(ctx, "read trigger filter", "trigger", t.Name, "error", err) return nil, huma.Error500InternalServerError("couldn't read the filter of " + t.Name) } } if t.ScheduleEverySeconds.Valid { view.Every = (time.Duration(t.ScheduleEverySeconds.Int64) * time.Second).String() } out = append(out, view) } return httpapi.OK(out), nil } type recentInput struct { Limit int `query:"limit" minimum:"1" maximum:"500" default:"50"` Source string `query:"source"` Type string `query:"type"` Matched string `query:"matched" enum:"true,false" doc:"Only events a trigger did or didn't match"` } func (m *Module) recent(ctx context.Context, in *recentInput) (*httpapi.Reply[[]eventView], error) { filter := store.EventFilter{Source: in.Source, Type: in.Type, Limit: in.Limit} if in.Matched != "" { matched, err := strconv.ParseBool(in.Matched) if err != nil { return nil, huma.Error400BadRequest("matched is true or false") } filter.Matched = &matched } events, err := m.store.Recent(ctx, m.orgID, filter) if err != nil { return nil, huma.Error500InternalServerError("couldn't list events", err) } out := make([]eventView, 0, len(events)) for _, e := range events { out = append(out, viewOf(e)) } return httpapi.OK(out), nil } // eventDetail is one event with the payload it arrived in and the runs it // started. type eventDetail struct { eventView Body jsontext.Value `json:"body"` Runs []triggerRun `json:"runs"` } type triggerRun struct { Trigger string `json:"trigger"` RunID uuid.UUID `json:"run_id"` } type eventInput struct { ID string `path:"id" format:"uuid"` } func (m *Module) event(ctx context.Context, in *eventInput) (*httpapi.Reply[eventDetail], error) { id, err := uuid.Parse(in.ID) if err != nil { return nil, huma.Error400BadRequest("an event id is a UUID") } e, err := m.store.Event(ctx, m.orgID, id) if errors.Is(err, store.ErrNotFound) { return nil, huma.Error404NotFound("no such event") } if err != nil { m.log.ErrorContext(ctx, "read event", "event", id, "error", err) return nil, huma.Error500InternalServerError("couldn't read the event") } runs, err := m.store.RunsForEvent(ctx, m.orgID, id) if err != nil { m.log.ErrorContext(ctx, "read event runs", "event", id, "error", err) return nil, huma.Error500InternalServerError("couldn't read the runs of the event") } view := eventDetail{eventView: viewOf(e), Body: jsontext.Value(e.Body), Runs: make([]triggerRun, 0, len(runs))} if len(view.Body) == 0 || !view.Body.IsValid() { view.Body = jsontext.Value("null") } for _, run := range runs { view.Runs = append(view.Runs, triggerRun{Trigger: run.Trigger, RunID: run.RunID}) } return httpapi.OK(view), nil } // eventView is an event as the API returns it. type eventView struct { ID uuid.UUID `json:"id"` Source string `json:"source"` Type string `json:"type"` Fields jsontext.Value `json:"fields"` LinkedTo *uuid.UUID `json:"linked_to,omitempty"` OccurredAt *time.Time `json:"occurred_at,omitempty"` ReceivedAt time.Time `json:"received_at"` Matched bool `json:"matched"` DispatchedAt *time.Time `json:"dispatched_at,omitempty"` } func viewOf(e store.Event) eventView { v := eventView{ ID: e.ID, Source: e.Source, Type: e.Type, Fields: e.Fields, LinkedTo: e.LinkedTo, ReceivedAt: e.ReceivedAt, Matched: e.Matched, } if e.OccurredAt.Valid { v.OccurredAt = &e.OccurredAt.Time } if e.DispatchedAt.Valid { v.DispatchedAt = &e.DispatchedAt.Time } return v }