Files
archie/modules/events/api.go
T

452 lines
17 KiB
Go

package events
import (
"context"
"database/sql"
"encoding/json/jsontext"
json "encoding/json/v2"
"errors"
"net/http"
"strconv"
"time"
"github.com/danielgtaylor/huma/v2"
"github.com/google/uuid"
"github.com/samcharles93/archie-core/internal/platform/httpapi"
"github.com/samcharles93/archie-core/internal/platform/httpjson"
"github.com/samcharles93/archie-core/modules/events/internal/store"
"github.com/samcharles93/archie-core/modules/runs"
)
// Register adds the events API: sources, triggers, groups of unmatched events
// and the events themselves.
func (m *Module) Register(api huma.API) {
tags := []string{"Events"}
huma.Register(api, huma.Operation{
OperationID: "list-sources", Method: http.MethodGet, Path: "/api/sources",
Summary: "List sources", Tags: tags,
}, m.sources)
huma.Register(api, huma.Operation{
OperationID: "put-source", Method: http.MethodPut, Path: "/api/sources/{name}",
Summary: "Add or update a source, its verification scheme and its token", Tags: tags,
}, m.putSource)
huma.Register(api, huma.Operation{
OperationID: "put-source-key-fields", Method: http.MethodPut, Path: "/api/sources/{name}/key-fields",
Summary: "Set the fields a source's events are grouped by", Tags: tags,
}, m.putKeyFields)
huma.Register(api, huma.Operation{
OperationID: "list-triggers", Method: http.MethodGet, Path: "/api/triggers",
Summary: "List triggers", Tags: tags,
}, m.triggers)
huma.Register(api, huma.Operation{
OperationID: "put-trigger", Method: http.MethodPut, Path: "/api/triggers/{name}",
Summary: "Add or change a trigger", Tags: tags,
}, m.putTrigger)
huma.Register(api, huma.Operation{
OperationID: "list-groups", Method: http.MethodGet, Path: "/api/groups",
Summary: "List groups of unmatched events", Tags: tags,
}, m.groups)
huma.Register(api, huma.Operation{
OperationID: "list-events", Method: http.MethodGet, Path: "/api/events",
Summary: "List recent events", Tags: tags,
}, m.recent)
huma.Register(api, huma.Operation{
OperationID: "get-event", Method: http.MethodGet, Path: "/api/events/{id}",
Summary: "Get an event with its payload and the runs it started", Tags: tags,
}, m.event)
}
type putSourceInput struct {
Name string `path:"name" doc:"Lowercase letters, digits and hyphens; the webhook URL ends in it"`
Body struct {
Connector string `json:"connector" doc:"The connector that parses this source's events"`
// A new source without a scheme uses a bearer token, so existing
// senders keep working.
Verification string `json:"verification,omitzero" enum:"token,hmac,none" doc:"How deliveries are verified: a bearer token, an HMAC-SHA256 signature of the body, or no verification"`
HMACHeader string `json:"hmac_header,omitzero" doc:"The header that carries the signature (hmac only)"`
HMACPrefix string `json:"hmac_prefix,omitzero" doc:"Text before the hex digest in the signature header, such as sha256= (hmac only)" example:"sha256="`
HMACSecret string `json:"hmac_secret,omitzero" doc:"The shared signing secret (hmac only). Write-only: it is sealed and never returned"`
}
}
type sourceCreated struct {
Source string `json:"source"`
Token string `json:"token,omitzero" doc:"Returned once for a token source; only its hash is stored"`
URL string `json:"url" doc:"Where the sender posts its events"`
}
// putSource creates or updates a source. A save that omits verification keeps
// the stored scheme and its credentials. A token is returned only when the
// token scheme is chosen explicitly or the source is new, and only its hash is
// stored. An HMAC secret is sealed and never returned; a blank secret keeps the
// sealed one.
func (m *Module) putSource(ctx context.Context, in *putSourceInput) (*httpapi.Reply[sourceCreated], error) {
if !validName.MatchString(in.Name) || !validName.MatchString(in.Body.Connector) {
return nil, huma.Error400BadRequest("a source needs a lowercase name and a connector")
}
existing, err := m.store.Source(ctx, m.orgID, in.Name)
if err != nil && !errors.Is(err, store.ErrNotFound) {
return nil, huma.Error500InternalServerError("couldn't read the source", err)
}
var stored *store.Source
if err == nil {
stored = &existing
}
src, token, err := m.sourceToSave(in, stored)
if err != nil {
return nil, err
}
if err := m.store.UpsertSource(ctx, src); err != nil {
return nil, huma.Error500InternalServerError("couldn't save the source", err)
}
return httpapi.OK(sourceCreated{Source: in.Name, Token: token, URL: "/hooks/" + in.Name}), nil
}
// sourceToSave builds the row to store from the request and the stored source,
// if there is one. It returns the new token when one was issued.
func (m *Module) sourceToSave(in *putSourceInput, stored *store.Source) (store.Source, string, error) {
explicit := in.Body.Verification != ""
src := store.Source{OrgID: m.orgID, Name: in.Name, Connector: in.Body.Connector, Verification: in.Body.Verification}
switch {
case !explicit && stored != nil:
// Omitting the scheme keeps the stored one with its token or secret, so
// a save cannot downgrade a source.
src = *stored
src.Connector = in.Body.Connector
case !explicit:
src.Verification = store.Token
}
switch src.Verification {
case store.Token:
if stored != nil && !explicit {
return src, "", nil
}
token, err := newToken()
if err != nil {
return src, "", huma.Error500InternalServerError("couldn't generate a token", err)
}
src.TokenSHA256 = httpjson.TokenHash(token)
return src, token, nil
case store.None:
return src, "", nil
case store.HMAC:
if explicit {
if err := m.setHMAC(&src, in, stored); err != nil {
return src, "", err
}
}
if src.HMACHeader == "" || len(src.HMACSecretSealed) == 0 {
return src, "", huma.Error400BadRequest("an HMAC source needs a signature header and a secret")
}
return src, "", nil
default:
return src, "", huma.Error400BadRequest("verification must be token, hmac or none")
}
}
// setHMAC applies the requested HMAC settings. A blank secret keeps the sealed
// secret of a source that already uses HMAC.
func (m *Module) setHMAC(src *store.Source, in *putSourceInput, stored *store.Source) error {
src.HMACHeader, src.HMACPrefix = in.Body.HMACHeader, in.Body.HMACPrefix
switch {
case in.Body.HMACSecret != "":
sealed, err := m.sealer.Seal(secretLabel(m.orgID, in.Name), []byte(in.Body.HMACSecret))
if err != nil {
return huma.Error500InternalServerError("couldn't seal the secret", err)
}
src.HMACSecretSealed = sealed
case stored != nil && stored.Verification == store.HMAC:
src.HMACSecretSealed = stored.HMACSecretSealed
}
return nil
}
type putTriggerInput struct {
Name string `path:"name" doc:"Lowercase letters, digits and hyphens"`
Body struct {
Source string `json:"source,omitzero" doc:"The source whose events fire it; fixed when every is set"`
EventType string `json:"event_type,omitzero" doc:"The event type that fires it; fixed when every is set"`
Filter map[string]string `json:"filter,omitzero" doc:"Event fields that must match exactly"`
Workflow string `json:"workflow" doc:"The workflow each match runs"`
Disabled bool `json:"disabled,omitzero"`
Every string `json:"every,omitzero" doc:"Fire on an interval instead of on events, such as 1h" example:"1h"`
}
}
type triggerSaved struct {
Trigger string `json:"trigger"`
}
func (m *Module) putTrigger(ctx context.Context, in *putTriggerInput) (*httpapi.Reply[triggerSaved], error) {
req := in.Body
var every sql.NullInt64
if req.Every != "" {
d, err := time.ParseDuration(req.Every)
if err != nil || d < minInterval || d%time.Second != 0 {
return nil, huma.Error400BadRequest("every is a whole number of seconds of at least one minute, such as 1h")
}
every = sql.NullInt64{Int64: int64(d / time.Second), Valid: true}
req.Source, req.EventType = scheduleSource, scheduleType
// Every schedule's ticks share a source and type, so the filter keeps
// this trigger to its own.
req.Filter = map[string]string{"trigger": in.Name}
}
if !validName.MatchString(in.Name) || req.Source == "" || req.EventType == "" || req.Workflow == "" {
return nil, huma.Error400BadRequest("a trigger needs a lowercase name, a workflow, and either a source and event type or an every interval")
}
if err := m.runs.CheckTrigger(ctx, m.orgID, req.Workflow); err != nil {
if refused, ok := errors.AsType[*runs.InvalidError](err); ok {
return nil, huma.Error400BadRequest(refused.Problems[0].Message, &huma.ErrorDetail{Location: "body.workflow", Message: refused.Problems[0].Message})
}
return nil, huma.Error500InternalServerError("couldn't check the workflow", err)
}
filter, err := json.Marshal(req.Filter)
if err != nil {
return nil, huma.Error400BadRequest(err.Error())
}
if err := m.store.UpsertTrigger(ctx, store.Trigger{
OrgID: m.orgID, Name: in.Name, Source: req.Source, EventType: req.EventType,
Filter: filter, Workflow: req.Workflow, Enabled: !req.Disabled, ScheduleEverySeconds: every,
}); err != nil {
return nil, huma.Error500InternalServerError("couldn't save the trigger", err)
}
return httpapi.OK(triggerSaved{Trigger: in.Name}), nil
}
type putKeyFieldsInput struct {
Name string `path:"name"`
Body struct {
Paths []string `json:"paths,omitzero" doc:"Paths into the event's fields; none uses the connector's default"`
}
}
type keyFieldsSaved struct {
KeyFields []string `json:"key_fields"`
}
// putKeyFields sets the fields a source's events are grouped by. It applies to
// events that arrive after it.
func (m *Module) putKeyFields(ctx context.Context, in *putKeyFieldsInput) (*httpapi.Reply[keyFieldsSaved], error) {
paths := in.Body.Paths
if paths == nil {
paths = []string{}
}
raw, err := json.Marshal(paths)
if err != nil {
return nil, huma.Error400BadRequest(err.Error())
}
err = m.store.SetKeyFields(ctx, m.orgID, in.Name, raw)
if errors.Is(err, store.ErrNotFound) {
return nil, huma.Error404NotFound("no such source")
}
if err != nil {
return nil, huma.Error500InternalServerError("couldn't save the key fields", err)
}
return httpapi.OK(keyFieldsSaved{KeyFields: paths}), nil
}
// groupView is a group of unmatched events as the API returns it. Proposed is
// the trigger it suggests, with the workflow left for the person to choose.
type groupView struct {
ID uuid.UUID `json:"id"`
Count int64 `json:"count"`
FirstSeen time.Time `json:"first_seen"`
LastSeen time.Time `json:"last_seen"`
Proposed proposedTrigger `json:"proposed"`
}
type proposedTrigger struct {
Source string `json:"source"`
EventType string `json:"event_type"`
Filter jsontext.Value `json:"filter"`
Workflow string `json:"workflow"`
}
func (m *Module) groups(ctx context.Context, _ *struct{}) (*httpapi.Reply[[]groupView], error) {
groups, err := m.store.Groups(ctx, m.orgID, 200)
if err != nil {
m.log.ErrorContext(ctx, "list groups", "error", err)
return nil, huma.Error500InternalServerError("couldn't list groups")
}
out := make([]groupView, 0, len(groups))
for _, g := range groups {
out = append(out, groupView{
ID: g.ID, Count: g.Count, FirstSeen: g.FirstSeen, LastSeen: g.LastSeen,
Proposed: proposedTrigger{Source: g.Source, EventType: g.EventType, Filter: g.Filter},
})
}
return httpapi.OK(out), nil
}
// sourceView is a source as the API returns it. The token is returned only
// when it is created or rotated, never here.
type sourceView struct {
Name string `json:"name"`
Connector string `json:"connector"`
KeyFields []string `json:"key_fields"`
TokenSet bool `json:"token_set"`
// Verification and the HMAC header and prefix are shown; the secret is
// only reported as set.
Verification string `json:"verification"`
HMACHeader string `json:"hmac_header,omitzero"`
HMACPrefix string `json:"hmac_prefix,omitzero"`
HMACSecretSet bool `json:"hmac_secret_set"`
}
func (m *Module) sources(ctx context.Context, _ *struct{}) (*httpapi.Reply[[]sourceView], error) {
sources, err := m.store.Sources(ctx, m.orgID)
if err != nil {
m.log.ErrorContext(ctx, "list sources", "error", err)
return nil, huma.Error500InternalServerError("couldn't list sources")
}
out := make([]sourceView, 0, len(sources))
for _, src := range sources {
view := sourceView{
Name: src.Name, Connector: src.Connector, TokenSet: src.TokenSHA256 != "", KeyFields: []string{},
Verification: src.Verification, HMACHeader: src.HMACHeader, HMACPrefix: src.HMACPrefix, HMACSecretSet: len(src.HMACSecretSealed) > 0,
}
if len(src.KeyFields) > 0 {
if err := json.Unmarshal(src.KeyFields, &view.KeyFields); err != nil {
m.log.ErrorContext(ctx, "read source key fields", "source", src.Name, "error", err)
return nil, huma.Error500InternalServerError("couldn't read the key fields of " + src.Name)
}
}
out = append(out, view)
}
return httpapi.OK(out), nil
}
// triggerView is a trigger as the API returns it.
type triggerView struct {
Name string `json:"name"`
Source string `json:"source"`
EventType string `json:"event_type"`
Filter map[string]string `json:"filter"`
Workflow string `json:"workflow"`
Enabled bool `json:"enabled"`
Every string `json:"every,omitempty"`
}
func (m *Module) triggers(ctx context.Context, _ *struct{}) (*httpapi.Reply[[]triggerView], error) {
triggers, err := m.store.AllTriggers(ctx, m.orgID)
if err != nil {
m.log.ErrorContext(ctx, "list triggers", "error", err)
return nil, huma.Error500InternalServerError("couldn't list triggers")
}
out := make([]triggerView, 0, len(triggers))
for _, t := range triggers {
view := triggerView{Name: t.Name, Source: t.Source, EventType: t.EventType, Workflow: t.Workflow, Enabled: t.Enabled, Filter: map[string]string{}}
if len(t.Filter) > 0 {
if err := json.Unmarshal(t.Filter, &view.Filter); err != nil {
m.log.ErrorContext(ctx, "read trigger filter", "trigger", t.Name, "error", err)
return nil, huma.Error500InternalServerError("couldn't read the filter of " + t.Name)
}
}
if t.ScheduleEverySeconds.Valid {
view.Every = (time.Duration(t.ScheduleEverySeconds.Int64) * time.Second).String()
}
out = append(out, view)
}
return httpapi.OK(out), nil
}
type recentInput struct {
Limit int `query:"limit" minimum:"1" maximum:"500" default:"50"`
Source string `query:"source"`
Type string `query:"type"`
Matched string `query:"matched" enum:"true,false" doc:"Only events a trigger did or didn't match"`
}
func (m *Module) recent(ctx context.Context, in *recentInput) (*httpapi.Reply[[]eventView], error) {
filter := store.EventFilter{Source: in.Source, Type: in.Type, Limit: in.Limit}
if in.Matched != "" {
matched, err := strconv.ParseBool(in.Matched)
if err != nil {
return nil, huma.Error400BadRequest("matched is true or false")
}
filter.Matched = &matched
}
events, err := m.store.Recent(ctx, m.orgID, filter)
if err != nil {
return nil, huma.Error500InternalServerError("couldn't list events", err)
}
out := make([]eventView, 0, len(events))
for _, e := range events {
out = append(out, viewOf(e))
}
return httpapi.OK(out), nil
}
// eventDetail is one event with the payload it arrived in and the runs it
// started.
type eventDetail struct {
eventView
Body jsontext.Value `json:"body"`
Runs []triggerRun `json:"runs"`
}
type triggerRun struct {
Trigger string `json:"trigger"`
RunID uuid.UUID `json:"run_id"`
}
type eventInput struct {
ID string `path:"id" format:"uuid"`
}
func (m *Module) event(ctx context.Context, in *eventInput) (*httpapi.Reply[eventDetail], error) {
id, err := uuid.Parse(in.ID)
if err != nil {
return nil, huma.Error400BadRequest("an event id is a UUID")
}
e, err := m.store.Event(ctx, m.orgID, id)
if errors.Is(err, store.ErrNotFound) {
return nil, huma.Error404NotFound("no such event")
}
if err != nil {
m.log.ErrorContext(ctx, "read event", "event", id, "error", err)
return nil, huma.Error500InternalServerError("couldn't read the event")
}
runs, err := m.store.RunsForEvent(ctx, m.orgID, id)
if err != nil {
m.log.ErrorContext(ctx, "read event runs", "event", id, "error", err)
return nil, huma.Error500InternalServerError("couldn't read the runs of the event")
}
view := eventDetail{eventView: viewOf(e), Body: jsontext.Value(e.Body), Runs: make([]triggerRun, 0, len(runs))}
if len(view.Body) == 0 || !view.Body.IsValid() {
view.Body = jsontext.Value("null")
}
for _, run := range runs {
view.Runs = append(view.Runs, triggerRun{Trigger: run.Trigger, RunID: run.RunID})
}
return httpapi.OK(view), nil
}
// eventView is an event as the API returns it.
type eventView struct {
ID uuid.UUID `json:"id"`
Source string `json:"source"`
Type string `json:"type"`
Fields jsontext.Value `json:"fields"`
LinkedTo *uuid.UUID `json:"linked_to,omitempty"`
OccurredAt *time.Time `json:"occurred_at,omitempty"`
ReceivedAt time.Time `json:"received_at"`
Matched bool `json:"matched"`
DispatchedAt *time.Time `json:"dispatched_at,omitempty"`
}
func viewOf(e store.Event) eventView {
v := eventView{
ID: e.ID, Source: e.Source, Type: e.Type, Fields: e.Fields, LinkedTo: e.LinkedTo,
ReceivedAt: e.ReceivedAt, Matched: e.Matched,
}
if e.OccurredAt.Valid {
v.OccurredAt = &e.OccurredAt.Time
}
if e.DispatchedAt.Valid {
v.DispatchedAt = &e.DispatchedAt.Time
}
return v
}