diff --git a/internal/platform/pg/migrations/00033_delivery_attempts.sql b/internal/platform/pg/migrations/00033_delivery_attempts.sql new file mode 100644 index 000000000..333eb083d --- /dev/null +++ b/internal/platform/pg/migrations/00033_delivery_attempts.sql @@ -0,0 +1,17 @@ +-- +goose Up +-- One row per delivery attempt, so a channel's response and its failures are +-- kept rather than only the last error. Nothing is pruned: the volume is low +-- and the record is what makes a delivery auditable. +CREATE TABLE deliveries.attempts ( + message_seq bigint NOT NULL REFERENCES deliveries.messages (seq) ON DELETE CASCADE, + attempt int NOT NULL, + at timestamptz NOT NULL DEFAULT now(), + ok boolean NOT NULL, + response text NOT NULL DEFAULT '', + error text NOT NULL DEFAULT '', + duration_ms bigint NOT NULL, + PRIMARY KEY (message_seq, attempt) +); + +-- +goose Down +DROP TABLE deliveries.attempts; diff --git a/modules/deliveries/deliveries.go b/modules/deliveries/deliveries.go index d1b0c574b..21bbd4958 100644 --- a/modules/deliveries/deliveries.go +++ b/modules/deliveries/deliveries.go @@ -72,7 +72,8 @@ type ApprovalRequest struct { ExpiresAt time.Time } -// Message is a composer message as the API shows it. +// Message is a composer message as the API shows it. Attempts are the delivery +// attempts of the connector rows that carry the same message, one per channel. type Message struct { Seq int64 `json:"seq"` ID uuid.UUID `json:"id"` @@ -83,6 +84,18 @@ type Message struct { Body string `json:"body"` ExpiresAt *time.Time `json:"expires_at,omitzero"` CreatedAt time.Time `json:"created_at"` + Attempts []Attempt `json:"attempts,omitzero"` +} + +// Attempt is one delivery attempt on one channel. +type Attempt struct { + Channel string `json:"channel"` + Attempt int `json:"attempt"` + At time.Time `json:"at"` + OK bool `json:"ok"` + Response string `json:"response,omitzero"` + Error string `json:"error,omitzero"` + DurationMS int64 `json:"duration_ms"` } // Channels are the running connectors a message can go out on. @@ -196,15 +209,17 @@ func (m *Module) Work(ctx context.Context) { } func (m *Module) deliver(ctx context.Context, msg store.Message) { + start := time.Now() external, err := m.sendTo(ctx, msg) + took := time.Since(start) if err != nil { m.log.WarnContext(ctx, "delivery failed", "channel", msg.Channel, "message", msg.ID, "attempt", msg.Attempts, "error", err) - if err := m.store.Failed(ctx, msg.Seq, err.Error()); err != nil { + if err := m.store.Failed(ctx, msg.Seq, msg.Attempts, err.Error(), took); err != nil { m.log.ErrorContext(ctx, "record failed delivery", "error", err) } return } - if err := m.store.Sent(ctx, msg.Seq, external); err != nil { + if err := m.store.Sent(ctx, msg.Seq, msg.Attempts, external, took); err != nil { m.log.ErrorContext(ctx, "record delivery", "error", err) } } @@ -263,9 +278,26 @@ func (m *Module) messages(w http.ResponseWriter, r *http.Request) { httpjson.Error(w, http.StatusInternalServerError, "couldn't list messages") return } + keys := make([]string, 0, len(rows)) + for _, row := range rows { + keys = append(keys, row.Key) + } + attempts, err := m.store.AttemptsFor(r.Context(), m.orgID, keys) + if err != nil { + m.log.ErrorContext(r.Context(), "list delivery attempts", "error", err) + httpjson.Error(w, http.StatusInternalServerError, "couldn't list messages") + return + } + byKey := make(map[string][]Attempt, len(rows)) + for _, a := range attempts { + byKey[a.Key] = append(byKey[a.Key], Attempt{ + Channel: a.Channel, Attempt: a.Attempt, At: a.At, OK: a.OK, + Response: a.Response, Error: a.Error, DurationMS: a.DurationMS, + }) + } out := make([]Message, 0, len(rows)) for _, row := range rows { - msg := Message{Seq: row.Seq, ID: row.ID, Kind: row.Kind, Title: row.Title, Body: row.Body, CreatedAt: row.CreatedAt} + msg := Message{Seq: row.Seq, ID: row.ID, Kind: row.Kind, Title: row.Title, Body: row.Body, CreatedAt: row.CreatedAt, Attempts: byKey[row.Key]} if row.RunID.Valid { msg.RunID = &row.RunID.UUID } diff --git a/modules/deliveries/internal/store/store.go b/modules/deliveries/internal/store/store.go index b95ca7838..43622d3dc 100644 --- a/modules/deliveries/internal/store/store.go +++ b/modules/deliveries/internal/store/store.go @@ -126,14 +126,26 @@ func (s *Store) Claim(ctx context.Context, backoff func(attempts int) time.Durat return m, true, nil } -// Sent records that a channel confirmed a message. -func (s *Store) Sent(ctx context.Context, seq int64, externalID string) error { - if _, err := s.db.ExecContext(ctx, ` - UPDATE deliveries.messages SET sent_at = now(), external_id = $2, last_error = NULL WHERE seq = $1`, - seq, externalID); err != nil { - return fmt.Errorf("mark message sent: %w", err) - } - return nil +// Attempt is one delivery attempt as the dashboard shows it. +type Attempt struct { + // Key is the message key the attempt's channel row shares with the + // composer message. + Key string `db:"key"` + Channel string `db:"channel"` + MessageSeq int64 `db:"message_seq"` + Attempt int `db:"attempt"` + At time.Time `db:"at"` + OK bool `db:"ok"` + Response string `db:"response"` + Error string `db:"error"` + DurationMS int64 `db:"duration_ms"` +} + +// Sent records that a channel confirmed a message, with the attempt that +// confirmed it. +func (s *Store) Sent(ctx context.Context, seq int64, attempt int, externalID string, took time.Duration) error { + return s.attempt(ctx, seq, attempt, attemptOutcome{ok: true, response: externalID, took: took}, + `UPDATE deliveries.messages SET sent_at = now(), external_id = $2, last_error = NULL WHERE seq = $1`, externalID) } // SentBy finds a message its channel confirmed under one of ids. The run is @@ -152,12 +164,58 @@ func (s *Store) SentBy(ctx context.Context, orgID, channel string, ids []string) return run, true, nil } -// Failed records why an attempt failed; the lease already set when it is -// tried again. -func (s *Store) Failed(ctx context.Context, seq int64, reason string) error { - if _, err := s.db.ExecContext(ctx, ` - UPDATE deliveries.messages SET last_error = $2 WHERE seq = $1`, seq, reason); err != nil { - return fmt.Errorf("record failed delivery: %w", err) +// Failed records why an attempt failed, with the attempt that failed; the +// lease already set when it is tried again. +func (s *Store) Failed(ctx context.Context, seq int64, attempt int, reason string, took time.Duration) error { + return s.attempt(ctx, seq, attempt, attemptOutcome{failure: reason, took: took}, + `UPDATE deliveries.messages SET last_error = $2 WHERE seq = $1`, reason) +} + +// attemptOutcome is one attempt's result. +type attemptOutcome struct { + ok bool + response string + failure string + took time.Duration +} + +// attempt marks the message row and stores the attempt in one transaction, so +// a retried message never loses the reason it was retried. +func (s *Store) attempt(ctx context.Context, seq int64, attempt int, outcome attemptOutcome, mark, value string) error { + tx, err := s.db.BeginTxx(ctx, nil) + if err != nil { + return err + } + defer func() { _ = tx.Rollback() }() + if _, err := tx.ExecContext(ctx, mark, seq, value); err != nil { + return fmt.Errorf("record delivery: %w", err) + } + if _, err := tx.ExecContext(ctx, ` + INSERT INTO deliveries.attempts (message_seq, attempt, ok, response, error, duration_ms) + VALUES ($1, $2, $3, $4, $5, $6)`, + seq, attempt, outcome.ok, outcome.response, outcome.failure, outcome.took.Milliseconds()); err != nil { + return fmt.Errorf("record delivery attempt: %w", err) + } + if err := tx.Commit(); err != nil { + return fmt.Errorf("record delivery attempt: %w", err) } return nil } + +// AttemptsFor returns the attempts of every connector delivery carrying one of +// keys. Composer rows have none: they are stored already sent. +func (s *Store) AttemptsFor(ctx context.Context, orgID string, keys []string) ([]Attempt, error) { + if len(keys) == 0 { + return nil, nil + } + var out []Attempt + if err := s.db.SelectContext(ctx, &out, ` + SELECT m.key, m.channel, a.message_seq, a.attempt, a.at, a.ok, a.response, a.error, a.duration_ms + FROM deliveries.attempts a + JOIN deliveries.messages m ON m.seq = a.message_seq + WHERE m.org_id = $1 AND m.key = ANY($2) AND m.channel <> 'composer' + ORDER BY m.key, m.channel, a.attempt`, orgID, keys); err != nil { + return nil, fmt.Errorf("list delivery attempts: %w", err) + } + return out, nil +}