388 lines
13 KiB
Go
388 lines
13 KiB
Go
// Package deliveries is the transport for what Archie tells people, on each
|
|
// channel. It holds no approval state: approvals own the request and its
|
|
// answer, and use deliveries to carry them. The composer, the chat box in the
|
|
// dashboard, is the first channel: its messages are rows a client reads from a
|
|
// cursor, with a live stream that only says when to read. Every accepted
|
|
// connector that delivers messages is another: each gets its own row, retried
|
|
// until the connector confirms it, rejects it permanently, or the retry limit
|
|
// is reached.
|
|
package deliveries
|
|
|
|
import (
|
|
"context"
|
|
"database/sql"
|
|
json "encoding/json/v2"
|
|
"errors"
|
|
"fmt"
|
|
"log/slog"
|
|
"net/http"
|
|
"slices"
|
|
"strconv"
|
|
"time"
|
|
|
|
"github.com/danielgtaylor/huma/v2"
|
|
"github.com/google/uuid"
|
|
"github.com/jmoiron/sqlx"
|
|
|
|
"github.com/samcharles93/archie-core/internal/platform/httpapi"
|
|
"github.com/samcharles93/archie-core/internal/platform/hub"
|
|
"github.com/samcharles93/archie-core/modules/connectors"
|
|
"github.com/samcharles93/archie-core/modules/deliveries/internal/store"
|
|
)
|
|
|
|
const (
|
|
channelComposer = "composer"
|
|
topicComposer = "composer"
|
|
pageSize = 100
|
|
heartbeat = 25 * time.Second
|
|
// deliveryMessage is the delivery a connector declares to carry messages.
|
|
deliveryMessage = "message"
|
|
maxBackoff = 10 * time.Minute
|
|
// maxAttempts ends a delivery that keeps failing: with the backoff capped
|
|
// at maxBackoff it is about a day of trying.
|
|
maxAttempts = 150
|
|
)
|
|
|
|
// The kinds of message a channel carries.
|
|
const (
|
|
// KindNotification tells people what happened or was done.
|
|
KindNotification = "notification"
|
|
// KindApprovalRequest is an action waiting for someone's answer.
|
|
KindApprovalRequest = "approval_request"
|
|
// KindApprovalAnswer records what they answered.
|
|
KindApprovalAnswer = "approval_answer"
|
|
)
|
|
|
|
// Notification is something a run tells people: what happened, what was done,
|
|
// what is needed.
|
|
type Notification struct {
|
|
RunID uuid.UUID
|
|
// Key makes the notification idempotent; a workflow passes its step key so
|
|
// a replayed step doesn't notify twice.
|
|
Key string
|
|
Title string
|
|
Body string
|
|
}
|
|
|
|
// ApprovalRequest is one judged action waiting for a person's answer. It is
|
|
// carried like a notification, with the request an answer names and the time
|
|
// it runs out.
|
|
type ApprovalRequest struct {
|
|
RunID uuid.UUID
|
|
RequestID uuid.UUID
|
|
// Key makes the request idempotent, as a notification's does.
|
|
Key string
|
|
Title string
|
|
Body string
|
|
ExpiresAt time.Time
|
|
}
|
|
|
|
// Message is a composer message as the API shows it. Attempts are the delivery
|
|
// attempts of the connector rows that carry the same message, one per channel.
|
|
type Message struct {
|
|
Seq int64 `json:"seq"`
|
|
ID uuid.UUID `json:"id"`
|
|
Kind string `json:"kind"`
|
|
RunID *uuid.UUID `json:"run_id,omitzero"`
|
|
RequestID *uuid.UUID `json:"request_id,omitzero"`
|
|
Title string `json:"title"`
|
|
Body string `json:"body"`
|
|
ExpiresAt *time.Time `json:"expires_at,omitzero"`
|
|
CreatedAt time.Time `json:"created_at"`
|
|
Attempts []Attempt `json:"attempts,omitzero"`
|
|
}
|
|
|
|
// Attempt is one delivery attempt on one channel.
|
|
type Attempt struct {
|
|
Channel string `json:"channel"`
|
|
Attempt int `json:"attempt"`
|
|
At time.Time `json:"at"`
|
|
OK bool `json:"ok"`
|
|
Response string `json:"response,omitzero"`
|
|
Error string `json:"error,omitzero"`
|
|
DurationMS int64 `json:"duration_ms"`
|
|
}
|
|
|
|
// Channels are the running connectors a message can go out on.
|
|
type Channels interface {
|
|
Delivering(delivery string) []*connectors.Connector
|
|
Get(name string) (*connectors.Connector, bool)
|
|
}
|
|
|
|
// Module delivers one organisation's notifications.
|
|
type Module struct {
|
|
orgID string
|
|
store *store.Store
|
|
hub *hub.Hub
|
|
channels Channels
|
|
log *slog.Logger
|
|
poll time.Duration
|
|
}
|
|
|
|
// NewModule returns the deliveries module for orgID, announcing new messages
|
|
// on h and sending them on every connector in channels that delivers them.
|
|
func NewModule(db *sqlx.DB, log *slog.Logger, orgID string, h *hub.Hub, channels Channels) *Module {
|
|
return &Module{orgID: orgID, store: store.New(db), hub: h, channels: channels, log: log, poll: time.Second}
|
|
}
|
|
|
|
// Notify delivers n on every channel. Delivering the same key again changes
|
|
// nothing.
|
|
func (m *Module) Notify(ctx context.Context, n Notification) error {
|
|
if n.Key == "" {
|
|
return errors.New("a notification needs a key")
|
|
}
|
|
return m.send(ctx, store.Message{
|
|
ID: uuid.New(), Kind: KindNotification, RunID: uuid.NullUUID{UUID: n.RunID, Valid: n.RunID != uuid.Nil},
|
|
Title: n.Title, Body: n.Body, Key: n.Key,
|
|
})
|
|
}
|
|
|
|
// Ask delivers an approval request on every channel. Asking again for the same
|
|
// key changes nothing.
|
|
func (m *Module) Ask(ctx context.Context, a ApprovalRequest) error {
|
|
if a.Key == "" {
|
|
return errors.New("an approval request needs a key")
|
|
}
|
|
return m.send(ctx, store.Message{
|
|
ID: uuid.New(), Kind: KindApprovalRequest, RunID: uuid.NullUUID{UUID: a.RunID, Valid: a.RunID != uuid.Nil},
|
|
RequestID: uuid.NullUUID{UUID: a.RequestID, Valid: a.RequestID != uuid.Nil},
|
|
Title: a.Title, Body: a.Body, Key: a.Key, ExpiresAt: sql.NullTime{Time: a.ExpiresAt, Valid: !a.ExpiresAt.IsZero()},
|
|
})
|
|
}
|
|
|
|
// Told records what someone answered, so the composer shows the answer beside
|
|
// the request it settles.
|
|
func (m *Module) Told(ctx context.Context, a ApprovalRequest, answer string) error {
|
|
return m.send(ctx, store.Message{
|
|
ID: uuid.New(), Kind: KindApprovalAnswer, RunID: uuid.NullUUID{UUID: a.RunID, Valid: a.RunID != uuid.Nil},
|
|
RequestID: uuid.NullUUID{UUID: a.RequestID, Valid: a.RequestID != uuid.Nil},
|
|
Title: a.Title, Body: answer, Key: a.Key + ":answer",
|
|
})
|
|
}
|
|
|
|
// send writes msg to the composer, which wakes its readers, and queues it for
|
|
// every connector that delivers messages. Work sends the queued rows.
|
|
func (m *Module) send(ctx context.Context, msg store.Message) error {
|
|
msg.OrgID, msg.Channel, msg.Sent = m.orgID, channelComposer, true
|
|
seq, created, err := m.store.Insert(ctx, msg)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
if created {
|
|
m.hub.Publish(hub.Note{Topic: topicComposer, Seq: seq})
|
|
}
|
|
for _, c := range m.channels.Delivering(deliveryMessage) {
|
|
// Each channel keeps its own row, so each carries its own id.
|
|
row := msg
|
|
row.ID, row.Channel, row.Sent = uuid.New(), c.Name(), false
|
|
if _, _, err := m.store.Insert(ctx, row); err != nil {
|
|
return err
|
|
}
|
|
}
|
|
return nil
|
|
}
|
|
|
|
// SentBy reports whether channel confirmed a message under one of ids, the id
|
|
// a source's event carries when it echoes what Archie sent, and the run that
|
|
// sent it (nil when the message belonged to none).
|
|
func (m *Module) SentBy(ctx context.Context, channel string, ids ...string) (uuid.UUID, bool, error) {
|
|
ids = slices.DeleteFunc(slices.Clone(ids), func(id string) bool { return id == "" })
|
|
if len(ids) == 0 {
|
|
return uuid.Nil, false, nil
|
|
}
|
|
run, found, err := m.store.SentBy(ctx, m.orgID, channel, ids)
|
|
return run.UUID, found, err
|
|
}
|
|
|
|
// Work sends queued messages to their connectors until ctx ends. A failed send
|
|
// is tried again after a backoff that grows to maxBackoff. A connector's
|
|
// permanent refusal, or the maxAttempts-th failure, marks the message failed
|
|
// with its reason.
|
|
func (m *Module) Work(ctx context.Context) {
|
|
for ctx.Err() == nil {
|
|
msg, found, err := m.store.Claim(ctx, backoff)
|
|
if err != nil {
|
|
m.log.ErrorContext(ctx, "claim delivery", "error", err)
|
|
}
|
|
if found {
|
|
m.deliver(ctx, msg)
|
|
continue
|
|
}
|
|
select {
|
|
case <-ctx.Done():
|
|
case <-time.After(m.poll):
|
|
}
|
|
}
|
|
}
|
|
|
|
func (m *Module) deliver(ctx context.Context, msg store.Message) {
|
|
start := time.Now()
|
|
external, err := m.sendTo(ctx, msg)
|
|
took := time.Since(start)
|
|
if err != nil {
|
|
m.log.WarnContext(ctx, "delivery failed", "channel", msg.Channel, "message", msg.ID, "attempt", msg.Attempts, "error", err)
|
|
m.recordFailure(ctx, msg, err, took)
|
|
return
|
|
}
|
|
if err := m.store.Sent(ctx, msg.Seq, msg.Attempts, external, took); err != nil {
|
|
m.log.ErrorContext(ctx, "record delivery", "error", err)
|
|
}
|
|
}
|
|
|
|
// recordFailure records why an attempt failed. It marks the delivery failed
|
|
// when the channel rejected it permanently or the retry limit is reached.
|
|
func (m *Module) recordFailure(ctx context.Context, msg store.Message, cause error, took time.Duration) {
|
|
record := m.store.Failed
|
|
reason := cause.Error()
|
|
switch {
|
|
case errors.Is(cause, connectors.ErrRejected):
|
|
record = m.store.GaveUp
|
|
case msg.Attempts >= maxAttempts:
|
|
record = m.store.GaveUp
|
|
reason = fmt.Sprintf("gave up after %d attempts: %s", msg.Attempts, reason)
|
|
}
|
|
if err := record(ctx, msg.Seq, msg.Attempts, reason, took); err != nil {
|
|
m.log.ErrorContext(ctx, "record failed delivery", "error", err)
|
|
}
|
|
}
|
|
|
|
func (m *Module) sendTo(ctx context.Context, msg store.Message) (string, error) {
|
|
c, ok := m.channels.Get(msg.Channel)
|
|
if !ok {
|
|
return "", fmt.Errorf("connector %q isn't running", msg.Channel)
|
|
}
|
|
input := map[string]string{"title": msg.Title}
|
|
if msg.Body != "" {
|
|
input["body"] = msg.Body
|
|
}
|
|
if msg.Kind == KindApprovalRequest && msg.RequestID.Valid {
|
|
input["approval_request_id"] = msg.RequestID.UUID.String()
|
|
}
|
|
raw, err := json.Marshal(input)
|
|
if err != nil {
|
|
return "", err
|
|
}
|
|
return c.Deliver(ctx, deliveryMessage, raw, msg.Key)
|
|
}
|
|
|
|
// backoff is how long to wait after the attempts so far before the next.
|
|
func backoff(attempts int) time.Duration {
|
|
if attempts >= 10 {
|
|
return maxBackoff
|
|
}
|
|
return min(time.Duration(1<<attempts)*time.Second, maxBackoff)
|
|
}
|
|
|
|
// Register adds the composer API.
|
|
func (m *Module) Register(api huma.API) {
|
|
tags := []string{"Composer"}
|
|
huma.Register(api, huma.Operation{
|
|
OperationID: "list-composer-messages", Method: http.MethodGet, Path: "/api/composer/messages",
|
|
Summary: "List composer messages with their delivery attempts", Tags: tags,
|
|
}, m.messages)
|
|
huma.Register(api, huma.Operation{
|
|
OperationID: "stream-composer", Method: http.MethodGet, Path: "/api/composer/stream",
|
|
Summary: "Stream the seq of each new composer message", Tags: tags,
|
|
Responses: map[string]*huma.Response{"200": {
|
|
Description: "Server-sent events: a message event carrying {\"seq\": n} for each new message, and a keepalive comment while idle",
|
|
Content: map[string]*huma.MediaType{"text/event-stream": {Schema: &huma.Schema{Type: huma.TypeString}}},
|
|
}},
|
|
}, m.stream)
|
|
}
|
|
|
|
type messagesInput struct {
|
|
After string `query:"after" doc:"Only messages after this seq; the latest page by default"`
|
|
}
|
|
|
|
// messages returns the newest page, or the messages after ?after=seq.
|
|
func (m *Module) messages(ctx context.Context, in *messagesInput) (*httpapi.Reply[[]Message], error) {
|
|
var (
|
|
rows []store.Message
|
|
err error
|
|
)
|
|
if in.After != "" {
|
|
seq, perr := strconv.ParseInt(in.After, 10, 64)
|
|
if perr != nil || seq < 0 {
|
|
return nil, huma.NewError(http.StatusBadRequest, "after is a message seq")
|
|
}
|
|
rows, err = m.store.After(ctx, m.orgID, channelComposer, seq, pageSize)
|
|
} else {
|
|
rows, err = m.store.Latest(ctx, m.orgID, channelComposer, pageSize)
|
|
}
|
|
if err != nil {
|
|
m.log.ErrorContext(ctx, "list composer messages", "error", err)
|
|
return nil, huma.NewError(http.StatusInternalServerError, "couldn't list messages")
|
|
}
|
|
keys := make([]string, 0, len(rows))
|
|
for _, row := range rows {
|
|
keys = append(keys, row.Key)
|
|
}
|
|
attempts, err := m.store.AttemptsFor(ctx, m.orgID, keys)
|
|
if err != nil {
|
|
m.log.ErrorContext(ctx, "list delivery attempts", "error", err)
|
|
return nil, huma.NewError(http.StatusInternalServerError, "couldn't list messages")
|
|
}
|
|
byKey := make(map[string][]Attempt, len(rows))
|
|
for _, a := range attempts {
|
|
byKey[a.Key] = append(byKey[a.Key], Attempt{
|
|
Channel: a.Channel, Attempt: a.Attempt, At: a.At, OK: a.OK,
|
|
Response: a.Response, Error: a.Error, DurationMS: a.DurationMS,
|
|
})
|
|
}
|
|
out := make([]Message, 0, len(rows))
|
|
for _, row := range rows {
|
|
msg := Message{Seq: row.Seq, ID: row.ID, Kind: row.Kind, Title: row.Title, Body: row.Body, CreatedAt: row.CreatedAt, Attempts: byKey[row.Key]}
|
|
if row.RunID.Valid {
|
|
msg.RunID = &row.RunID.UUID
|
|
}
|
|
if row.RequestID.Valid {
|
|
msg.RequestID = &row.RequestID.UUID
|
|
}
|
|
if row.ExpiresAt.Valid {
|
|
msg.ExpiresAt = &row.ExpiresAt.Time
|
|
}
|
|
out = append(out, msg)
|
|
}
|
|
return httpapi.OK(out), nil
|
|
}
|
|
|
|
// stream sends a server-sent event each time a composer message lands. It
|
|
// carries only the new seq: the client reads the messages themselves. A
|
|
// client that falls behind is cut off and reconnects with its cursor.
|
|
func (m *Module) stream(_ context.Context, _ *struct{}) (*huma.StreamResponse, error) {
|
|
return &huma.StreamResponse{Body: func(hc huma.Context) {
|
|
w := hc.BodyWriter()
|
|
flusher, ok := w.(http.Flusher)
|
|
if !ok {
|
|
hc.SetStatus(http.StatusInternalServerError)
|
|
return
|
|
}
|
|
notes, stop := m.hub.Subscribe(topicComposer)
|
|
defer stop()
|
|
hc.SetHeader("Content-Type", "text/event-stream")
|
|
hc.SetHeader("Cache-Control", "no-store")
|
|
hc.SetStatus(http.StatusOK)
|
|
flusher.Flush()
|
|
tick := time.NewTicker(heartbeat)
|
|
defer tick.Stop()
|
|
for {
|
|
select {
|
|
case <-hc.Context().Done():
|
|
return
|
|
case <-tick.C:
|
|
if _, err := fmt.Fprint(w, ": keepalive\n\n"); err != nil {
|
|
return
|
|
}
|
|
case n, ok := <-notes:
|
|
if !ok {
|
|
return
|
|
}
|
|
if _, err := fmt.Fprintf(w, "event: message\ndata: {\"seq\":%d}\n\n", n.Seq); err != nil {
|
|
return
|
|
}
|
|
}
|
|
flusher.Flush()
|
|
}
|
|
}}, nil
|
|
}
|