From db36d853cedd106ca786a04794f0779738289a23 Mon Sep 17 00:00:00 2001 From: Sam Catlow Date: Mon, 16 Nov 2026 12:04:39 +1100 Subject: [PATCH] feat(deliveries): every attempt keeps its response, its error and how long it took A delivery.attempts row is written with the success or failure the store already records. The composer API returns the connector attempts that carry each message, so the dashboard can show each attempt's outcome and response. --- .../pg/migrations/00033_delivery_attempts.sql | 17 ++++ modules/deliveries/deliveries.go | 40 ++++++++- modules/deliveries/internal/store/store.go | 86 ++++++++++++++++--- 3 files changed, 125 insertions(+), 18 deletions(-) create mode 100644 internal/platform/pg/migrations/00033_delivery_attempts.sql diff --git a/internal/platform/pg/migrations/00033_delivery_attempts.sql b/internal/platform/pg/migrations/00033_delivery_attempts.sql new file mode 100644 index 000000000..333eb083d --- /dev/null +++ b/internal/platform/pg/migrations/00033_delivery_attempts.sql @@ -0,0 +1,17 @@ +-- +goose Up +-- One row per delivery attempt, so a channel's response and its failures are +-- kept rather than only the last error. Nothing is pruned: the volume is low +-- and the record is what makes a delivery auditable. +CREATE TABLE deliveries.attempts ( + message_seq bigint NOT NULL REFERENCES deliveries.messages (seq) ON DELETE CASCADE, + attempt int NOT NULL, + at timestamptz NOT NULL DEFAULT now(), + ok boolean NOT NULL, + response text NOT NULL DEFAULT '', + error text NOT NULL DEFAULT '', + duration_ms bigint NOT NULL, + PRIMARY KEY (message_seq, attempt) +); + +-- +goose Down +DROP TABLE deliveries.attempts; diff --git a/modules/deliveries/deliveries.go b/modules/deliveries/deliveries.go index d1b0c574b..21bbd4958 100644 --- a/modules/deliveries/deliveries.go +++ b/modules/deliveries/deliveries.go @@ -72,7 +72,8 @@ type ApprovalRequest struct { ExpiresAt time.Time } -// Message is a composer message as the API shows it. +// Message is a composer message as the API shows it. Attempts are the delivery +// attempts of the connector rows that carry the same message, one per channel. type Message struct { Seq int64 `json:"seq"` ID uuid.UUID `json:"id"` @@ -83,6 +84,18 @@ type Message struct { Body string `json:"body"` ExpiresAt *time.Time `json:"expires_at,omitzero"` CreatedAt time.Time `json:"created_at"` + Attempts []Attempt `json:"attempts,omitzero"` +} + +// Attempt is one delivery attempt on one channel. +type Attempt struct { + Channel string `json:"channel"` + Attempt int `json:"attempt"` + At time.Time `json:"at"` + OK bool `json:"ok"` + Response string `json:"response,omitzero"` + Error string `json:"error,omitzero"` + DurationMS int64 `json:"duration_ms"` } // Channels are the running connectors a message can go out on. @@ -196,15 +209,17 @@ func (m *Module) Work(ctx context.Context) { } func (m *Module) deliver(ctx context.Context, msg store.Message) { + start := time.Now() external, err := m.sendTo(ctx, msg) + took := time.Since(start) if err != nil { m.log.WarnContext(ctx, "delivery failed", "channel", msg.Channel, "message", msg.ID, "attempt", msg.Attempts, "error", err) - if err := m.store.Failed(ctx, msg.Seq, err.Error()); err != nil { + if err := m.store.Failed(ctx, msg.Seq, msg.Attempts, err.Error(), took); err != nil { m.log.ErrorContext(ctx, "record failed delivery", "error", err) } return } - if err := m.store.Sent(ctx, msg.Seq, external); err != nil { + if err := m.store.Sent(ctx, msg.Seq, msg.Attempts, external, took); err != nil { m.log.ErrorContext(ctx, "record delivery", "error", err) } } @@ -263,9 +278,26 @@ func (m *Module) messages(w http.ResponseWriter, r *http.Request) { httpjson.Error(w, http.StatusInternalServerError, "couldn't list messages") return } + keys := make([]string, 0, len(rows)) + for _, row := range rows { + keys = append(keys, row.Key) + } + attempts, err := m.store.AttemptsFor(r.Context(), m.orgID, keys) + if err != nil { + m.log.ErrorContext(r.Context(), "list delivery attempts", "error", err) + httpjson.Error(w, http.StatusInternalServerError, "couldn't list messages") + return + } + byKey := make(map[string][]Attempt, len(rows)) + for _, a := range attempts { + byKey[a.Key] = append(byKey[a.Key], Attempt{ + Channel: a.Channel, Attempt: a.Attempt, At: a.At, OK: a.OK, + Response: a.Response, Error: a.Error, DurationMS: a.DurationMS, + }) + } out := make([]Message, 0, len(rows)) for _, row := range rows { - msg := Message{Seq: row.Seq, ID: row.ID, Kind: row.Kind, Title: row.Title, Body: row.Body, CreatedAt: row.CreatedAt} + msg := Message{Seq: row.Seq, ID: row.ID, Kind: row.Kind, Title: row.Title, Body: row.Body, CreatedAt: row.CreatedAt, Attempts: byKey[row.Key]} if row.RunID.Valid { msg.RunID = &row.RunID.UUID } diff --git a/modules/deliveries/internal/store/store.go b/modules/deliveries/internal/store/store.go index b95ca7838..43622d3dc 100644 --- a/modules/deliveries/internal/store/store.go +++ b/modules/deliveries/internal/store/store.go @@ -126,14 +126,26 @@ func (s *Store) Claim(ctx context.Context, backoff func(attempts int) time.Durat return m, true, nil } -// Sent records that a channel confirmed a message. -func (s *Store) Sent(ctx context.Context, seq int64, externalID string) error { - if _, err := s.db.ExecContext(ctx, ` - UPDATE deliveries.messages SET sent_at = now(), external_id = $2, last_error = NULL WHERE seq = $1`, - seq, externalID); err != nil { - return fmt.Errorf("mark message sent: %w", err) - } - return nil +// Attempt is one delivery attempt as the dashboard shows it. +type Attempt struct { + // Key is the message key the attempt's channel row shares with the + // composer message. + Key string `db:"key"` + Channel string `db:"channel"` + MessageSeq int64 `db:"message_seq"` + Attempt int `db:"attempt"` + At time.Time `db:"at"` + OK bool `db:"ok"` + Response string `db:"response"` + Error string `db:"error"` + DurationMS int64 `db:"duration_ms"` +} + +// Sent records that a channel confirmed a message, with the attempt that +// confirmed it. +func (s *Store) Sent(ctx context.Context, seq int64, attempt int, externalID string, took time.Duration) error { + return s.attempt(ctx, seq, attempt, attemptOutcome{ok: true, response: externalID, took: took}, + `UPDATE deliveries.messages SET sent_at = now(), external_id = $2, last_error = NULL WHERE seq = $1`, externalID) } // SentBy finds a message its channel confirmed under one of ids. The run is @@ -152,12 +164,58 @@ func (s *Store) SentBy(ctx context.Context, orgID, channel string, ids []string) return run, true, nil } -// Failed records why an attempt failed; the lease already set when it is -// tried again. -func (s *Store) Failed(ctx context.Context, seq int64, reason string) error { - if _, err := s.db.ExecContext(ctx, ` - UPDATE deliveries.messages SET last_error = $2 WHERE seq = $1`, seq, reason); err != nil { - return fmt.Errorf("record failed delivery: %w", err) +// Failed records why an attempt failed, with the attempt that failed; the +// lease already set when it is tried again. +func (s *Store) Failed(ctx context.Context, seq int64, attempt int, reason string, took time.Duration) error { + return s.attempt(ctx, seq, attempt, attemptOutcome{failure: reason, took: took}, + `UPDATE deliveries.messages SET last_error = $2 WHERE seq = $1`, reason) +} + +// attemptOutcome is one attempt's result. +type attemptOutcome struct { + ok bool + response string + failure string + took time.Duration +} + +// attempt marks the message row and stores the attempt in one transaction, so +// a retried message never loses the reason it was retried. +func (s *Store) attempt(ctx context.Context, seq int64, attempt int, outcome attemptOutcome, mark, value string) error { + tx, err := s.db.BeginTxx(ctx, nil) + if err != nil { + return err + } + defer func() { _ = tx.Rollback() }() + if _, err := tx.ExecContext(ctx, mark, seq, value); err != nil { + return fmt.Errorf("record delivery: %w", err) + } + if _, err := tx.ExecContext(ctx, ` + INSERT INTO deliveries.attempts (message_seq, attempt, ok, response, error, duration_ms) + VALUES ($1, $2, $3, $4, $5, $6)`, + seq, attempt, outcome.ok, outcome.response, outcome.failure, outcome.took.Milliseconds()); err != nil { + return fmt.Errorf("record delivery attempt: %w", err) + } + if err := tx.Commit(); err != nil { + return fmt.Errorf("record delivery attempt: %w", err) } return nil } + +// AttemptsFor returns the attempts of every connector delivery carrying one of +// keys. Composer rows have none: they are stored already sent. +func (s *Store) AttemptsFor(ctx context.Context, orgID string, keys []string) ([]Attempt, error) { + if len(keys) == 0 { + return nil, nil + } + var out []Attempt + if err := s.db.SelectContext(ctx, &out, ` + SELECT m.key, m.channel, a.message_seq, a.attempt, a.at, a.ok, a.response, a.error, a.duration_ms + FROM deliveries.attempts a + JOIN deliveries.messages m ON m.seq = a.message_seq + WHERE m.org_id = $1 AND m.key = ANY($2) AND m.channel <> 'composer' + ORDER BY m.key, m.channel, a.attempt`, orgID, keys); err != nil { + return nil, fmt.Errorf("list delivery attempts: %w", err) + } + return out, nil +}