Files

405 lines
13 KiB
Go

// Package events is how Archie hears: sources deliver payloads, a connector
// turns each verified payload into typed events, and every event is stored
// before it is acknowledged. A source whose connector holds its own connection
// (a chat long poll) streams its events instead, resuming from a stored
// cursor. A dispatcher then claims each new event, settles an approval it
// answers, links a resolve to what it resolves, and starts a run for every
// trigger it matches.
package events
import (
"context"
"crypto/rand"
"encoding/hex"
"encoding/json/jsontext"
json "encoding/json/v2"
"errors"
"fmt"
"log/slog"
"net/http"
"regexp"
"sync"
"time"
"github.com/google/uuid"
"github.com/jmoiron/sqlx"
"github.com/samcharles93/archie-core/internal/platform/httpjson"
"github.com/samcharles93/archie-core/modules/connectors"
"github.com/samcharles93/archie-core/modules/events/internal/match"
"github.com/samcharles93/archie-core/modules/events/internal/store"
"github.com/samcharles93/archie-core/modules/runs"
)
var validName = regexp.MustCompile(`^[a-z][a-z0-9-]{0,62}$`)
// Connectors finds the running connector a source reads with.
type Connectors interface {
Get(name string) (*connectors.Connector, bool)
}
// Runs starts runs for matched triggers, and says whether a trigger could
// start its workflow.
type Runs interface {
Start(ctx context.Context, req runs.StartRequest) (uuid.UUID, error)
CheckTrigger(ctx context.Context, orgID, workflow string) error
}
// Deliveries recognises the echo of a message Archie sent. It returns the run
// that sent the message the ids name on channel, which is nil when the message
// belonged to no run.
type Deliveries interface {
SentBy(ctx context.Context, channel string, ids ...string) (run uuid.UUID, found bool, err error)
}
// Answers settles the approval requests answer events name.
type Answers interface {
Settle(ctx context.Context, id uuid.UUID, identity, via string, approved bool) (bool, error)
}
// answerType is the event a channel emits when a person answers an approval
// request.
const answerType = "approval.answer"
// listenRetry is how long a listener waits after its stream breaks.
const listenRetry = 5 * time.Second
// Module hears events for one organisation.
type Module struct {
orgID string
store *store.Store
reader sourceReader
sealer Sealer
parse func(ctx context.Context, connector string, body []byte, headers map[string]string) ([]connectors.Event, error)
connectors Connectors
runs Runs
answers Answers
deliveries Deliveries
log *slog.Logger
poll time.Duration
}
// NewModule returns the events module for orgID.
func NewModule(db *sqlx.DB, log *slog.Logger, orgID string, sealer Sealer, c Connectors, r Runs, a Answers, d Deliveries) *Module {
st := store.New(db)
m := &Module{orgID: orgID, store: st, reader: st, sealer: sealer, connectors: c, runs: r, answers: a, deliveries: d, log: log, poll: time.Second}
m.parse = m.parseWith
return m
}
// sourceReader is the store method that intake uses to look up a source.
type sourceReader interface {
Source(ctx context.Context, orgID, name string) (store.Source, error)
}
// Routes adds the webhook intake. It stays outside the API because each
// source verifies its own deliveries.
func (m *Module) Routes(mux *http.ServeMux) {
mux.HandleFunc("POST /hooks/{source}", m.intake)
}
// intake verifies a delivery, parses it with the source's connector and stores
// each event before responding. A redelivered event violates a uniqueness
// constraint and is counted as a duplicate.
func (m *Module) intake(w http.ResponseWriter, r *http.Request) {
src, parsed, body, status, err := m.accept(r)
if err != nil {
httpjson.Error(w, status, err.Error())
return
}
stored, duplicates, err := m.save(r.Context(), src, parsed, body)
if err != nil {
httpjson.Error(w, http.StatusServiceUnavailable, err.Error())
return
}
httpjson.Write(w, http.StatusAccepted, map[string]int{"stored": stored, "duplicates": duplicates})
}
// accept verifies a delivery against its source's scheme, then passes it to
// the connector for parsing. A delivery that fails verification is rejected
// with 401 before the connector sees its body, and the rejection is logged.
// An unknown source and a failed verification get the same response, so a
// caller cannot use it to discover source names.
func (m *Module) accept(r *http.Request) (src store.Source, parsed []connectors.Event, body []byte, status int, err error) {
ctx := r.Context()
src, err = m.reader.Source(ctx, m.orgID, r.PathValue("source"))
if errors.Is(err, store.ErrNotFound) {
return src, nil, nil, http.StatusUnauthorized, errUnknown
}
if err != nil {
return src, nil, nil, http.StatusServiceUnavailable, errors.New("events store unavailable")
}
// A token is checked before the body is read. A signature covers the body,
// so it is checked on the raw bytes as soon as they are read.
switch src.Verification {
case store.Token:
if err := checkToken(src, r); err != nil {
return m.refuse(ctx, src, err)
}
case store.HMAC, store.None:
default:
return m.refuse(ctx, src, fmt.Errorf("unknown verification %q", src.Verification))
}
body, err = httpjson.Body(r)
if err != nil {
return src, nil, nil, http.StatusRequestEntityTooLarge, err
}
if src.Verification == store.HMAC {
if err := m.verifyHMAC(src, r.Header, body); err != nil {
return m.refuse(ctx, src, err)
}
}
parsed, err = m.parse(ctx, src.Connector, body, map[string]string{"Content-Type": r.Header.Get("Content-Type")})
if errors.Is(err, errConnectorDown) {
return src, nil, nil, http.StatusServiceUnavailable, err
}
if err != nil {
return src, nil, nil, http.StatusBadRequest, err
}
return src, parsed, body, 0, nil
}
var (
errUnknown = errors.New("unknown source or failed verification")
errConnectorDown = errors.New("connector isn't running")
)
// refuse logs why a delivery was rejected and chooses the status: 401 when
// verification failed, 503 when verification could not run.
func (m *Module) refuse(ctx context.Context, src store.Source, err error) (store.Source, []connectors.Event, []byte, int, error) {
m.log.WarnContext(ctx, "refused webhook delivery", "source", src.Name, "verification", src.Verification, "reason", err.Error())
if errors.Is(err, errUnverified) {
return src, nil, nil, http.StatusUnauthorized, errUnknown
}
return src, nil, nil, http.StatusServiceUnavailable, errors.New("couldn't verify the delivery")
}
// parseWith has a running connector parse a delivery.
func (m *Module) parseWith(ctx context.Context, connector string, body []byte, headers map[string]string) ([]connectors.Event, error) {
conn, ok := m.connectors.Get(connector)
if !ok {
return nil, fmt.Errorf("connector %s: %w", connector, errConnectorDown)
}
return conn.Parse(ctx, body, headers)
}
// save stores each event with the payload it arrived in, recording at arrival
// whether it echoes a delivery Archie sent, so those never reach grouping.
func (m *Module) save(ctx context.Context, src store.Source, parsed []connectors.Event, body []byte) (stored, duplicates int, err error) {
for _, e := range parsed {
run, own, err := m.deliveries.SentBy(ctx, src.Connector, e.SourceEventID, e.LinkKey)
if err != nil {
return stored, duplicates, fmt.Errorf("check delivery provenance: %w", err)
}
isNew, err := m.store.Insert(ctx, newRow(m.orgID, src, e, body, own, run))
if err != nil {
return stored, duplicates, err
}
if isNew {
stored++
} else {
duplicates++
}
}
return stored, duplicates, nil
}
// Work listens to every listening source and dispatches stored events until
// ctx ends.
func (m *Module) Work(ctx context.Context) {
var listeners sync.WaitGroup
defer listeners.Wait()
listening := map[string]bool{}
nextCheck, nextTick := time.Time{}, time.Time{}
for ctx.Err() == nil {
if now := time.Now(); now.After(nextTick) {
if err := m.tick(ctx, now); err != nil && ctx.Err() == nil {
m.log.ErrorContext(ctx, "fire schedules", "error", err)
}
nextTick = now.Add(scheduleCheck)
}
if time.Now().After(nextCheck) {
m.startListeners(ctx, &listeners, listening)
nextCheck = time.Now().Add(listenRetry)
}
found, err := m.store.Dispatch(ctx, m.dispatch)
if err != nil {
m.log.ErrorContext(ctx, "dispatch event", "error", err)
}
if found && err == nil {
continue
}
select {
case <-ctx.Done():
case <-time.After(m.poll):
}
}
}
// startListeners starts one listener for each source whose connector listens
// and has none yet. A listener lives until ctx ends.
func (m *Module) startListeners(ctx context.Context, wg *sync.WaitGroup, listening map[string]bool) {
sources, err := m.store.Sources(ctx, m.orgID)
if err != nil {
m.log.ErrorContext(ctx, "list sources", "error", err)
return
}
for _, src := range sources {
conn, ok := m.connectors.Get(src.Connector)
if listening[src.Name] || !ok || !conn.Listens() {
continue
}
listening[src.Name] = true
wg.Go(func() { m.listen(ctx, src) })
}
}
// listen streams one source's events, storing each batch before its cursor,
// so a restart fetches again anything not yet stored and dedupe drops what
// was. A broken stream is opened again from the stored cursor.
func (m *Module) listen(ctx context.Context, src store.Source) {
cursor := src.Cursor
for ctx.Err() == nil {
conn, ok := m.connectors.Get(src.Connector)
var err error
if !ok {
err = fmt.Errorf("connector %q isn't running", src.Connector)
} else {
err = conn.Listen(ctx, cursor, func(events []connectors.Event, next string) error {
if _, _, err := m.save(ctx, src, events, received(events)); err != nil {
return err
}
if err := m.store.SetCursor(ctx, m.orgID, src.Name, next); err != nil {
return err
}
cursor = next
return nil
})
}
if ctx.Err() != nil {
return
}
m.log.WarnContext(ctx, "listening stopped", "source", src.Name, "error", err)
select {
case <-ctx.Done():
case <-time.After(listenRetry):
}
}
}
// answer settles the approval request an answer event names. An answer that
// can't settle it is logged and done with: the request's own record already
// says why.
func (m *Module) answer(ctx context.Context, e store.Event) error {
var a struct {
RequestID uuid.UUID `json:"request_id"`
Approved bool `json:"approved"`
By string `json:"by"`
}
if err := json.Unmarshal(e.Fields, &a); err != nil {
m.log.WarnContext(ctx, "unreadable approval answer", "event", e.ID, "error", err)
return nil
}
settled, err := m.answers.Settle(ctx, a.RequestID, a.By, e.Source, a.Approved)
if err != nil {
return fmt.Errorf("settle approval %s: %w", a.RequestID, err)
}
if !settled {
m.log.InfoContext(ctx, "approval answer changed nothing", "event", e.ID, "request", a.RequestID)
}
return nil
}
// dispatch links a resolve to the event it resolves and starts no work for
// it. Any other event starts one run per matching trigger, keyed so a retried
// dispatch never starts a second run.
func (m *Module) dispatch(tx *sqlx.Tx, e store.Event) (bool, error) {
ctx := context.Background()
if e.Own {
// Archie's own message coming back starts nothing and proposes nothing.
return false, nil
}
if e.Type == answerType {
return false, m.answer(ctx, e)
}
linked, err := store.Link(ctx, tx, e)
if err != nil || linked {
return false, err
}
triggers, err := store.Triggers(ctx, tx, e.OrgID, e.Source, e.Type)
if err != nil {
return false, err
}
matched := false
for _, t := range triggers {
var filter map[string]string
if err := json.Unmarshal(t.Filter, &filter); err != nil {
return false, fmt.Errorf("trigger %q filter: %w", t.Name, err)
}
if !match.Fields(e.Fields, filter) {
continue
}
runID, err := m.runs.Start(ctx, runs.StartRequest{
OrgID: e.OrgID, Workflow: t.Workflow, Key: e.ID.String() + ":" + t.Name, Input: runInput(e),
})
if err != nil {
return false, fmt.Errorf("start run for trigger %q: %w", t.Name, err)
}
if err := store.RecordDispatch(ctx, tx, e.ID, t.Name, runID); err != nil {
return false, err
}
m.log.InfoContext(ctx, "trigger started run", "event", e.ID, "trigger", t.Name, "run", runID)
matched = true
}
if !matched {
created, err := store.NoteUnmatched(ctx, tx, e)
if err != nil {
return false, err
}
if created {
m.log.InfoContext(ctx, "unmatched events started a group", "source", e.Source, "type", e.Type)
}
}
return matched, nil
}
// Origin returns the event and trigger that started a run.
func (m *Module) Origin(ctx context.Context, runID uuid.UUID) (runs.Origin, bool, error) {
o, ok, err := m.store.OriginOf(ctx, m.orgID, runID)
if err != nil || !ok {
return runs.Origin{}, ok, err
}
return runs.Origin{
EventID: o.EventID, Source: o.Source, Type: o.Type, Fields: o.Fields,
ReceivedAt: o.ReceivedAt, Trigger: o.Trigger, Filter: o.Filter,
}, true, nil
}
func runInput(e store.Event) jsontext.Value {
input, err := json.Marshal(map[string]any{"event": map[string]any{
"id": e.ID, "source": e.Source, "type": e.Type, "fields": e.Fields, "occurred_at": e.OccurredAt.Time,
}})
if err != nil {
return jsontext.Value("null")
}
return input
}
func newToken() (string, error) {
b := make([]byte, 32)
if _, err := rand.Read(b); err != nil {
return "", fmt.Errorf("generate token: %w", err)
}
return hex.EncodeToString(b), nil
}
// received renders a batch a listening connector streamed, so the stored body
// is what reached the run rather than only the fields parsed from it.
func received(events []connectors.Event) []byte {
body, err := json.Marshal(events)
if err != nil {
return nil
}
return body
}