Files
archie/modules/deliveries/deliveries.go
T

409 lines
14 KiB
Go

// Package deliveries is the transport for what Archie tells people, on each
// channel. It holds no approval state: approvals own the request and its
// answer, and use deliveries to carry them. The composer, the chat box in the
// dashboard, is the first channel: its messages are rows a client reads from a
// cursor, with a live stream that only says when to read. Every accepted
// connector that delivers messages is another: each gets its own row, retried
// until the connector confirms it, rejects it permanently, or the retry limit
// is reached.
package deliveries
import (
"context"
"database/sql"
json "encoding/json/v2"
"errors"
"fmt"
"log/slog"
"net/http"
"slices"
"strconv"
"time"
"github.com/danielgtaylor/huma/v2"
"github.com/google/uuid"
"github.com/jmoiron/sqlx"
"github.com/samcharles93/archie-core/internal/platform/httpapi"
"github.com/samcharles93/archie-core/internal/platform/hub"
"github.com/samcharles93/archie-core/modules/connectors"
"github.com/samcharles93/archie-core/modules/deliveries/internal/store"
)
const (
channelComposer = "composer"
topicComposer = "composer"
pageSize = 100
heartbeat = 25 * time.Second
// deliveryMessage is the delivery a connector declares to carry messages.
deliveryMessage = "message"
// deliveryChat is the delivery a connector declares to send text to one
// chat it can address.
deliveryChat = "chat_message"
maxBackoff = 10 * time.Minute
// maxAttempts ends a delivery that keeps failing: with the backoff capped
// at maxBackoff it is about a day of trying.
maxAttempts = 150
)
// The kinds of message a channel carries.
const (
// KindNotification tells people what happened or was done.
KindNotification = "notification"
// KindApprovalRequest is an action waiting for someone's answer.
KindApprovalRequest = "approval_request"
// KindApprovalAnswer records what they answered.
KindApprovalAnswer = "approval_answer"
)
// Notification is something a run tells people: what happened, what was done,
// what is needed.
type Notification struct {
RunID uuid.UUID
// Key makes the notification idempotent; a workflow passes its step key so
// a replayed step doesn't notify twice.
Key string
Title string
Body string
}
// ApprovalRequest is one judged action waiting for a person's answer. It is
// carried like a notification, with the request an answer names and the time
// it runs out.
type ApprovalRequest struct {
RunID uuid.UUID
RequestID uuid.UUID
// Key makes the request idempotent, as a notification's does.
Key string
Title string
Body string
ExpiresAt time.Time
}
// Message is a composer message as the API shows it. Attempts are the delivery
// attempts of the connector rows that carry the same message, one per channel.
type Message struct {
Seq int64 `json:"seq"`
ID uuid.UUID `json:"id"`
Kind string `json:"kind"`
RunID *uuid.UUID `json:"run_id,omitzero"`
RequestID *uuid.UUID `json:"request_id,omitzero"`
Title string `json:"title"`
Body string `json:"body"`
ExpiresAt *time.Time `json:"expires_at,omitzero"`
CreatedAt time.Time `json:"created_at"`
Attempts []Attempt `json:"attempts,omitzero"`
}
// Attempt is one delivery attempt on one channel.
type Attempt struct {
Channel string `json:"channel"`
Attempt int `json:"attempt"`
At time.Time `json:"at"`
OK bool `json:"ok"`
Response string `json:"response,omitzero"`
Error string `json:"error,omitzero"`
DurationMS int64 `json:"duration_ms"`
}
// Channels are the running connectors a message can go out on.
type Channels interface {
Delivering(delivery string) []*connectors.Connector
Get(name string) (*connectors.Connector, bool)
}
// Module delivers one organisation's notifications.
type Module struct {
orgID string
store *store.Store
hub *hub.Hub
channels Channels
log *slog.Logger
poll time.Duration
}
// NewModule returns the deliveries module for orgID, announcing new messages
// on h and sending them on every connector in channels that delivers them.
func NewModule(db *sqlx.DB, log *slog.Logger, orgID string, h *hub.Hub, channels Channels) *Module {
return &Module{orgID: orgID, store: store.New(db), hub: h, channels: channels, log: log, poll: time.Second}
}
// Notify delivers n on every channel. Delivering the same key again changes
// nothing.
func (m *Module) Notify(ctx context.Context, n Notification) error {
if n.Key == "" {
return errors.New("a notification needs a key")
}
return m.send(ctx, store.Message{
ID: uuid.New(), Kind: KindNotification, RunID: uuid.NullUUID{UUID: n.RunID, Valid: n.RunID != uuid.Nil},
Title: n.Title, Body: n.Body, Key: n.Key,
})
}
// Send delivers text to one chat on the named channel, and returns when the
// channel has accepted it. The key makes the delivery idempotent on the
// channel, so a step that is replayed after a crash does not send twice.
func (m *Module) Send(ctx context.Context, channel, chatID, text, key string) error {
c, ok := m.channels.Get(channel)
if !ok {
return fmt.Errorf("connector %q isn't running", channel)
}
raw, err := json.Marshal(map[string]string{"chat_id": chatID, "text": text})
if err != nil {
return err
}
if _, err := c.Deliver(ctx, deliveryChat, raw, key); err != nil {
return fmt.Errorf("send to %s chat %s: %w", channel, chatID, err)
}
return nil
}
// Ask delivers an approval request on every channel. Asking again for the same
// key changes nothing.
func (m *Module) Ask(ctx context.Context, a ApprovalRequest) error {
if a.Key == "" {
return errors.New("an approval request needs a key")
}
return m.send(ctx, store.Message{
ID: uuid.New(), Kind: KindApprovalRequest, RunID: uuid.NullUUID{UUID: a.RunID, Valid: a.RunID != uuid.Nil},
RequestID: uuid.NullUUID{UUID: a.RequestID, Valid: a.RequestID != uuid.Nil},
Title: a.Title, Body: a.Body, Key: a.Key, ExpiresAt: sql.NullTime{Time: a.ExpiresAt, Valid: !a.ExpiresAt.IsZero()},
})
}
// Told records what someone answered, so the composer shows the answer beside
// the request it settles.
func (m *Module) Told(ctx context.Context, a ApprovalRequest, answer string) error {
return m.send(ctx, store.Message{
ID: uuid.New(), Kind: KindApprovalAnswer, RunID: uuid.NullUUID{UUID: a.RunID, Valid: a.RunID != uuid.Nil},
RequestID: uuid.NullUUID{UUID: a.RequestID, Valid: a.RequestID != uuid.Nil},
Title: a.Title, Body: answer, Key: a.Key + ":answer",
})
}
// send writes msg to the composer, which wakes its readers, and queues it for
// every connector that delivers messages. Work sends the queued rows.
func (m *Module) send(ctx context.Context, msg store.Message) error {
msg.OrgID, msg.Channel, msg.Sent = m.orgID, channelComposer, true
seq, created, err := m.store.Insert(ctx, msg)
if err != nil {
return err
}
if created {
m.hub.Publish(hub.Note{Topic: topicComposer, Seq: seq})
}
for _, c := range m.channels.Delivering(deliveryMessage) {
// Each channel keeps its own row, so each carries its own id.
row := msg
row.ID, row.Channel, row.Sent = uuid.New(), c.Name(), false
if _, _, err := m.store.Insert(ctx, row); err != nil {
return err
}
}
return nil
}
// SentBy reports whether channel confirmed a message under one of ids, the id
// a source's event carries when it echoes what Archie sent, and the run that
// sent it (nil when the message belonged to none).
func (m *Module) SentBy(ctx context.Context, channel string, ids ...string) (uuid.UUID, bool, error) {
ids = slices.DeleteFunc(slices.Clone(ids), func(id string) bool { return id == "" })
if len(ids) == 0 {
return uuid.Nil, false, nil
}
run, found, err := m.store.SentBy(ctx, m.orgID, channel, ids)
return run.UUID, found, err
}
// Work sends queued messages to their connectors until ctx ends. A failed send
// is tried again after a backoff that grows to maxBackoff. A connector's
// permanent refusal, or the maxAttempts-th failure, marks the message failed
// with its reason.
func (m *Module) Work(ctx context.Context) {
for ctx.Err() == nil {
msg, found, err := m.store.Claim(ctx, backoff)
if err != nil {
m.log.ErrorContext(ctx, "claim delivery", "error", err)
}
if found {
m.deliver(ctx, msg)
continue
}
select {
case <-ctx.Done():
case <-time.After(m.poll):
}
}
}
func (m *Module) deliver(ctx context.Context, msg store.Message) {
start := time.Now()
external, err := m.sendTo(ctx, msg)
took := time.Since(start)
if err != nil {
m.log.WarnContext(ctx, "delivery failed", "channel", msg.Channel, "message", msg.ID, "attempt", msg.Attempts, "error", err)
m.recordFailure(ctx, msg, err, took)
return
}
if err := m.store.Sent(ctx, msg.Seq, msg.Attempts, external, took); err != nil {
m.log.ErrorContext(ctx, "record delivery", "error", err)
}
}
// recordFailure records why an attempt failed. It marks the delivery failed
// when the channel rejected it permanently or the retry limit is reached.
func (m *Module) recordFailure(ctx context.Context, msg store.Message, cause error, took time.Duration) {
record := m.store.Failed
reason := cause.Error()
switch {
case errors.Is(cause, connectors.ErrRejected):
record = m.store.GaveUp
case msg.Attempts >= maxAttempts:
record = m.store.GaveUp
reason = fmt.Sprintf("gave up after %d attempts: %s", msg.Attempts, reason)
}
if err := record(ctx, msg.Seq, msg.Attempts, reason, took); err != nil {
m.log.ErrorContext(ctx, "record failed delivery", "error", err)
}
}
func (m *Module) sendTo(ctx context.Context, msg store.Message) (string, error) {
c, ok := m.channels.Get(msg.Channel)
if !ok {
return "", fmt.Errorf("connector %q isn't running", msg.Channel)
}
input := map[string]string{"title": msg.Title}
if msg.Body != "" {
input["body"] = msg.Body
}
if msg.Kind == KindApprovalRequest && msg.RequestID.Valid {
input["approval_request_id"] = msg.RequestID.UUID.String()
}
raw, err := json.Marshal(input)
if err != nil {
return "", err
}
return c.Deliver(ctx, deliveryMessage, raw, msg.Key)
}
// backoff is how long to wait after the attempts so far before the next.
func backoff(attempts int) time.Duration {
if attempts >= 10 {
return maxBackoff
}
return min(time.Duration(1<<attempts)*time.Second, maxBackoff)
}
// Register adds the composer API.
func (m *Module) Register(api huma.API) {
tags := []string{"Composer"}
huma.Register(api, huma.Operation{
OperationID: "list-composer-messages", Method: http.MethodGet, Path: "/api/composer/messages",
Summary: "List composer messages with their delivery attempts", Tags: tags,
}, m.messages)
huma.Register(api, huma.Operation{
OperationID: "stream-composer", Method: http.MethodGet, Path: "/api/composer/stream",
Summary: "Stream the seq of each new composer message", Tags: tags,
Responses: map[string]*huma.Response{"200": {
Description: "Server-sent events: a message event carrying {\"seq\": n} for each new message, and a keepalive comment while idle",
Content: map[string]*huma.MediaType{"text/event-stream": {Schema: &huma.Schema{Type: huma.TypeString}}},
}},
}, m.stream)
}
type messagesInput struct {
After string `query:"after" doc:"Only messages after this seq; the latest page by default"`
}
// messages returns the newest page, or the messages after ?after=seq.
func (m *Module) messages(ctx context.Context, in *messagesInput) (*httpapi.Reply[[]Message], error) {
var (
rows []store.Message
err error
)
if in.After != "" {
seq, perr := strconv.ParseInt(in.After, 10, 64)
if perr != nil || seq < 0 {
return nil, huma.NewError(http.StatusBadRequest, "after is a message seq")
}
rows, err = m.store.After(ctx, m.orgID, channelComposer, seq, pageSize)
} else {
rows, err = m.store.Latest(ctx, m.orgID, channelComposer, pageSize)
}
if err != nil {
m.log.ErrorContext(ctx, "list composer messages", "error", err)
return nil, huma.NewError(http.StatusInternalServerError, "couldn't list messages")
}
keys := make([]string, 0, len(rows))
for _, row := range rows {
keys = append(keys, row.Key)
}
attempts, err := m.store.AttemptsFor(ctx, m.orgID, keys)
if err != nil {
m.log.ErrorContext(ctx, "list delivery attempts", "error", err)
return nil, huma.NewError(http.StatusInternalServerError, "couldn't list messages")
}
byKey := make(map[string][]Attempt, len(rows))
for _, a := range attempts {
byKey[a.Key] = append(byKey[a.Key], Attempt{
Channel: a.Channel, Attempt: a.Attempt, At: a.At, OK: a.OK,
Response: a.Response, Error: a.Error, DurationMS: a.DurationMS,
})
}
out := make([]Message, 0, len(rows))
for _, row := range rows {
msg := Message{Seq: row.Seq, ID: row.ID, Kind: row.Kind, Title: row.Title, Body: row.Body, CreatedAt: row.CreatedAt, Attempts: byKey[row.Key]}
if row.RunID.Valid {
msg.RunID = &row.RunID.UUID
}
if row.RequestID.Valid {
msg.RequestID = &row.RequestID.UUID
}
if row.ExpiresAt.Valid {
msg.ExpiresAt = &row.ExpiresAt.Time
}
out = append(out, msg)
}
return httpapi.OK(out), nil
}
// stream sends a server-sent event each time a composer message lands. It
// carries only the new seq: the client reads the messages themselves. A
// client that falls behind is cut off and reconnects with its cursor.
func (m *Module) stream(_ context.Context, _ *struct{}) (*huma.StreamResponse, error) {
return &huma.StreamResponse{Body: func(hc huma.Context) {
w := hc.BodyWriter()
flusher, ok := w.(http.Flusher)
if !ok {
hc.SetStatus(http.StatusInternalServerError)
return
}
notes, stop := m.hub.Subscribe(topicComposer)
defer stop()
hc.SetHeader("Content-Type", "text/event-stream")
hc.SetHeader("Cache-Control", "no-store")
hc.SetStatus(http.StatusOK)
flusher.Flush()
tick := time.NewTicker(heartbeat)
defer tick.Stop()
for {
select {
case <-hc.Context().Done():
return
case <-tick.C:
if _, err := fmt.Fprint(w, ": keepalive\n\n"); err != nil {
return
}
case n, ok := <-notes:
if !ok {
return
}
if _, err := fmt.Fprintf(w, "event: message\ndata: {\"seq\":%d}\n\n", n.Seq); err != nil {
return
}
}
flusher.Flush()
}
}}, nil
}