feat(deliveries): every attempt keeps its response, its error and how long it took

A delivery.attempts row is written with the success or failure the store already records. The composer API returns the connector attempts that carry each message, so the dashboard can show each attempt's outcome and response.
This commit is contained in:
sam committed 2026-11-16 12:06:05 +11:00
1 parent 4b97ac5841
commit db36d853ce
3 files changed
+125 -18

No files matched your search

@@ -0,0 +1,17 @@
-- +goose Up
-- One row per delivery attempt, so a channel's response and its failures are
-- kept rather than only the last error. Nothing is pruned: the volume is low
-- and the record is what makes a delivery auditable.
CREATE TABLE deliveries.attempts (
message_seq bigint NOT NULL REFERENCES deliveries.messages (seq) ON DELETE CASCADE,
attempt int NOT NULL,
at timestamptz NOT NULL DEFAULT now(),
ok boolean NOT NULL,
response text NOT NULL DEFAULT '',
error text NOT NULL DEFAULT '',
duration_ms bigint NOT NULL,
PRIMARY KEY (message_seq, attempt)
);
-- +goose Down
DROP TABLE deliveries.attempts;
+36 -4
View File
@@ -72,7 +72,8 @@ type ApprovalRequest struct {
ExpiresAt time.Time
}
// Message is a composer message as the API shows it.
// Message is a composer message as the API shows it. Attempts are the delivery
// attempts of the connector rows that carry the same message, one per channel.
type Message struct {
Seq int64 `json:"seq"`
ID uuid.UUID `json:"id"`
@@ -83,6 +84,18 @@ type Message struct {
Body string `json:"body"`
ExpiresAt *time.Time `json:"expires_at,omitzero"`
CreatedAt time.Time `json:"created_at"`
Attempts []Attempt `json:"attempts,omitzero"`
}
// Attempt is one delivery attempt on one channel.
type Attempt struct {
Channel string `json:"channel"`
Attempt int `json:"attempt"`
At time.Time `json:"at"`
OK bool `json:"ok"`
Response string `json:"response,omitzero"`
Error string `json:"error,omitzero"`
DurationMS int64 `json:"duration_ms"`
}
// Channels are the running connectors a message can go out on.
@@ -196,15 +209,17 @@ func (m *Module) Work(ctx context.Context) {
}
func (m *Module) deliver(ctx context.Context, msg store.Message) {
start := time.Now()
external, err := m.sendTo(ctx, msg)
took := time.Since(start)
if err != nil {
m.log.WarnContext(ctx, "delivery failed", "channel", msg.Channel, "message", msg.ID, "attempt", msg.Attempts, "error", err)
if err := m.store.Failed(ctx, msg.Seq, err.Error()); err != nil {
if err := m.store.Failed(ctx, msg.Seq, msg.Attempts, err.Error(), took); err != nil {
m.log.ErrorContext(ctx, "record failed delivery", "error", err)
}
return
}
if err := m.store.Sent(ctx, msg.Seq, external); err != nil {
if err := m.store.Sent(ctx, msg.Seq, msg.Attempts, external, took); err != nil {
m.log.ErrorContext(ctx, "record delivery", "error", err)
}
}
@@ -263,9 +278,26 @@ func (m *Module) messages(w http.ResponseWriter, r *http.Request) {
httpjson.Error(w, http.StatusInternalServerError, "couldn't list messages")
return
}
keys := make([]string, 0, len(rows))
for _, row := range rows {
keys = append(keys, row.Key)
}
attempts, err := m.store.AttemptsFor(r.Context(), m.orgID, keys)
if err != nil {
m.log.ErrorContext(r.Context(), "list delivery attempts", "error", err)
httpjson.Error(w, http.StatusInternalServerError, "couldn't list messages")
return
}
byKey := make(map[string][]Attempt, len(rows))
for _, a := range attempts {
byKey[a.Key] = append(byKey[a.Key], Attempt{
Channel: a.Channel, Attempt: a.Attempt, At: a.At, OK: a.OK,
Response: a.Response, Error: a.Error, DurationMS: a.DurationMS,
})
}
out := make([]Message, 0, len(rows))
for _, row := range rows {
msg := Message{Seq: row.Seq, ID: row.ID, Kind: row.Kind, Title: row.Title, Body: row.Body, CreatedAt: row.CreatedAt}
msg := Message{Seq: row.Seq, ID: row.ID, Kind: row.Kind, Title: row.Title, Body: row.Body, CreatedAt: row.CreatedAt, Attempts: byKey[row.Key]}
if row.RunID.Valid {
msg.RunID = &row.RunID.UUID
}
+72 -14
View File
@@ -126,14 +126,26 @@ func (s *Store) Claim(ctx context.Context, backoff func(attempts int) time.Durat
return m, true, nil
}
// Sent records that a channel confirmed a message.
func (s *Store) Sent(ctx context.Context, seq int64, externalID string) error {
if _, err := s.db.ExecContext(ctx, `
UPDATE deliveries.messages SET sent_at = now(), external_id = $2, last_error = NULL WHERE seq = $1`,
seq, externalID); err != nil {
return fmt.Errorf("mark message sent: %w", err)
}
return nil
// Attempt is one delivery attempt as the dashboard shows it.
type Attempt struct {
// Key is the message key the attempt's channel row shares with the
// composer message.
Key string `db:"key"`
Channel string `db:"channel"`
MessageSeq int64 `db:"message_seq"`
Attempt int `db:"attempt"`
At time.Time `db:"at"`
OK bool `db:"ok"`
Response string `db:"response"`
Error string `db:"error"`
DurationMS int64 `db:"duration_ms"`
}
// Sent records that a channel confirmed a message, with the attempt that
// confirmed it.
func (s *Store) Sent(ctx context.Context, seq int64, attempt int, externalID string, took time.Duration) error {
return s.attempt(ctx, seq, attempt, attemptOutcome{ok: true, response: externalID, took: took},
`UPDATE deliveries.messages SET sent_at = now(), external_id = $2, last_error = NULL WHERE seq = $1`, externalID)
}
// SentBy finds a message its channel confirmed under one of ids. The run is
@@ -152,12 +164,58 @@ func (s *Store) SentBy(ctx context.Context, orgID, channel string, ids []string)
return run, true, nil
}
// Failed records why an attempt failed; the lease already set when it is
// tried again.
func (s *Store) Failed(ctx context.Context, seq int64, reason string) error {
if _, err := s.db.ExecContext(ctx, `
UPDATE deliveries.messages SET last_error = $2 WHERE seq = $1`, seq, reason); err != nil {
return fmt.Errorf("record failed delivery: %w", err)
// Failed records why an attempt failed, with the attempt that failed; the
// lease already set when it is tried again.
func (s *Store) Failed(ctx context.Context, seq int64, attempt int, reason string, took time.Duration) error {
return s.attempt(ctx, seq, attempt, attemptOutcome{failure: reason, took: took},
`UPDATE deliveries.messages SET last_error = $2 WHERE seq = $1`, reason)
}
// attemptOutcome is one attempt's result.
type attemptOutcome struct {
ok bool
response string
failure string
took time.Duration
}
// attempt marks the message row and stores the attempt in one transaction, so
// a retried message never loses the reason it was retried.
func (s *Store) attempt(ctx context.Context, seq int64, attempt int, outcome attemptOutcome, mark, value string) error {
tx, err := s.db.BeginTxx(ctx, nil)
if err != nil {
return err
}
defer func() { _ = tx.Rollback() }()
if _, err := tx.ExecContext(ctx, mark, seq, value); err != nil {
return fmt.Errorf("record delivery: %w", err)
}
if _, err := tx.ExecContext(ctx, `
INSERT INTO deliveries.attempts (message_seq, attempt, ok, response, error, duration_ms)
VALUES ($1, $2, $3, $4, $5, $6)`,
seq, attempt, outcome.ok, outcome.response, outcome.failure, outcome.took.Milliseconds()); err != nil {
return fmt.Errorf("record delivery attempt: %w", err)
}
if err := tx.Commit(); err != nil {
return fmt.Errorf("record delivery attempt: %w", err)
}
return nil
}
// AttemptsFor returns the attempts of every connector delivery carrying one of
// keys. Composer rows have none: they are stored already sent.
func (s *Store) AttemptsFor(ctx context.Context, orgID string, keys []string) ([]Attempt, error) {
if len(keys) == 0 {
return nil, nil
}
var out []Attempt
if err := s.db.SelectContext(ctx, &out, `
SELECT m.key, m.channel, a.message_seq, a.attempt, a.at, a.ok, a.response, a.error, a.duration_ms
FROM deliveries.attempts a
JOIN deliveries.messages m ON m.seq = a.message_seq
WHERE m.org_id = $1 AND m.key = ANY($2) AND m.channel <> 'composer'
ORDER BY m.key, m.channel, a.attempt`, orgID, keys); err != nil {
return nil, fmt.Errorf("list delivery attempts: %w", err)
}
return out, nil
}