diff --git a/internal/contracts/connector/v1/connector_grpc.pb.go b/internal/contracts/connector/v1/connector_grpc.pb.go index 2b1db229e..79bc67f3b 100644 --- a/internal/contracts/connector/v1/connector_grpc.pb.go +++ b/internal/contracts/connector/v1/connector_grpc.pb.go @@ -60,6 +60,9 @@ type ConnectorServiceClient interface { // delivery is never offered to an agent: hard rules apply to it, guards // don't. The host retries a failed delivery, so a connector should make a // repeat with the same idempotency key harmless where its service allows. + // A connector answers InvalidArgument, FailedPrecondition, PermissionDenied + // or NotFound when the service refuses the delivery for good: the host + // records the reason and stops retrying. Any other status is retried. Deliver(ctx context.Context, in *DeliverRequest, opts ...grpc.CallOption) (*DeliverResponse, error) // Listen streams the events of a source the connector holds its own // connection to, such as a chat long poll, from cursor on. Each response @@ -210,6 +213,9 @@ type ConnectorServiceServer interface { // delivery is never offered to an agent: hard rules apply to it, guards // don't. The host retries a failed delivery, so a connector should make a // repeat with the same idempotency key harmless where its service allows. + // A connector answers InvalidArgument, FailedPrecondition, PermissionDenied + // or NotFound when the service refuses the delivery for good: the host + // records the reason and stops retrying. Any other status is retried. Deliver(context.Context, *DeliverRequest) (*DeliverResponse, error) // Listen streams the events of a source the connector holds its own // connection to, such as a chat long poll, from cursor on. Each response diff --git a/internal/platform/pg/migrations/00043_deliveries_failed.sql b/internal/platform/pg/migrations/00043_deliveries_failed.sql new file mode 100644 index 000000000..c1a7c39b9 --- /dev/null +++ b/internal/platform/pg/migrations/00043_deliveries_failed.sql @@ -0,0 +1,11 @@ +-- +goose Up +-- A delivery the channel refuses for good, or that keeps failing past the retry +-- limit, is failed rather than retried: failed_at stops the worker claiming it. +ALTER TABLE deliveries.messages ADD COLUMN failed_at timestamptz; +DROP INDEX deliveries.messages_due; +CREATE INDEX messages_due ON deliveries.messages (next_attempt_at) WHERE sent_at IS NULL AND failed_at IS NULL; + +-- +goose Down +DROP INDEX deliveries.messages_due; +CREATE INDEX messages_due ON deliveries.messages (next_attempt_at) WHERE sent_at IS NULL; +ALTER TABLE deliveries.messages DROP COLUMN failed_at; diff --git a/modules/connectors/connector.go b/modules/connectors/connector.go index 0c98486ce..ac5c08dbf 100644 --- a/modules/connectors/connector.go +++ b/modules/connectors/connector.go @@ -13,6 +13,8 @@ import ( "time" "github.com/google/jsonschema-go/jsonschema" + "google.golang.org/grpc/codes" + "google.golang.org/grpc/status" pb "github.com/samcharles93/archie-core/internal/contracts/connector/v1" ) @@ -29,6 +31,10 @@ var ( // ErrUnknownName names an action, lookup or event type the connector // never declared. ErrUnknownName = errors.New("not declared by the connector") + // ErrRejected means a delivery was refused in a way a retry can't change: + // the connector answered InvalidArgument, FailedPrecondition, + // PermissionDenied or NotFound, or the input broke the declared schema. + ErrRejected = errors.New("delivery rejected for good") ) // Reversibility says whether and how an action can be undone. @@ -229,17 +235,31 @@ func (c *Connector) Deliver(ctx context.Context, name string, input jsontext.Val return "", fmt.Errorf("connector %q: delivery %q: %w", c.name, name, ErrUnknownName) } if err := validate(schema, input); err != nil { - return "", fmt.Errorf("connector %q: delivery %q: %w", c.name, name, err) + return "", fmt.Errorf("connector %q: delivery %q: %w: %w", c.name, name, ErrRejected, err) } req := &pb.DeliverRequest{Delivery: name, Input: input, IdempotencyKey: key} resp, err := c.client.Deliver(ctx, req) recordProto(ctx, c.name, "deliver", req, resp, err) if err != nil { + if rejectedForGood(err) { + return "", fmt.Errorf("connector %q: deliver %q: %w: %w", c.name, name, ErrRejected, err) + } return "", fmt.Errorf("connector %q: deliver %q: %w", c.name, name, err) } return resp.GetExternalId(), nil } +// rejectedForGood reports whether a connector's error says the same delivery +// can never succeed. +func rejectedForGood(err error) bool { + switch status.Code(err) { + case codes.InvalidArgument, codes.FailedPrecondition, codes.PermissionDenied, codes.NotFound: + return true + default: + return false + } +} + // Listens reports whether the connector holds its own connection to a source // and streams its events. func (c *Connector) Listens() bool { return c.listens } diff --git a/modules/deliveries/deliveries.go b/modules/deliveries/deliveries.go index 6ba94b7e5..6544df363 100644 --- a/modules/deliveries/deliveries.go +++ b/modules/deliveries/deliveries.go @@ -4,7 +4,7 @@ // dashboard, is the first channel: its messages are rows a client reads from a // cursor, with a live stream that only says when to read. Every accepted // connector that delivers messages is another: each gets its own row, retried -// until the connector confirms it. +// until the connector confirms it, refuses it for good, or the attempts run out. package deliveries import ( @@ -37,6 +37,9 @@ const ( // deliveryMessage is the delivery a connector declares to carry messages. deliveryMessage = "message" maxBackoff = 10 * time.Minute + // maxAttempts ends a delivery that keeps failing: with the backoff capped + // at maxBackoff it is about a day of trying. + maxAttempts = 150 ) // The kinds of message a channel carries. @@ -191,7 +194,9 @@ func (m *Module) SentBy(ctx context.Context, channel string, ids ...string) (uui } // Work sends queued messages to their connectors until ctx ends. A failed send -// is tried again after a backoff that grows to maxBackoff, never dropped. +// is tried again after a backoff that grows to maxBackoff. A connector's +// permanent refusal, or the maxAttempts-th failure, marks the message failed +// with its reason. func (m *Module) Work(ctx context.Context) { for ctx.Err() == nil { msg, found, err := m.store.Claim(ctx, backoff) @@ -215,9 +220,7 @@ func (m *Module) deliver(ctx context.Context, msg store.Message) { took := time.Since(start) if err != nil { m.log.WarnContext(ctx, "delivery failed", "channel", msg.Channel, "message", msg.ID, "attempt", msg.Attempts, "error", err) - if err := m.store.Failed(ctx, msg.Seq, msg.Attempts, err.Error(), took); err != nil { - m.log.ErrorContext(ctx, "record failed delivery", "error", err) - } + m.recordFailure(ctx, msg, err, took) return } if err := m.store.Sent(ctx, msg.Seq, msg.Attempts, external, took); err != nil { @@ -225,6 +228,23 @@ func (m *Module) deliver(ctx context.Context, msg store.Message) { } } +// recordFailure stores why an attempt failed, and ends the delivery when the +// channel refused it for good or the attempts are spent. +func (m *Module) recordFailure(ctx context.Context, msg store.Message, cause error, took time.Duration) { + record := m.store.Failed + reason := cause.Error() + switch { + case errors.Is(cause, connectors.ErrRejected): + record = m.store.GaveUp + case msg.Attempts >= maxAttempts: + record = m.store.GaveUp + reason = fmt.Sprintf("gave up after %d attempts: %s", msg.Attempts, reason) + } + if err := record(ctx, msg.Seq, msg.Attempts, reason, took); err != nil { + m.log.ErrorContext(ctx, "record failed delivery", "error", err) + } +} + func (m *Module) sendTo(ctx context.Context, msg store.Message) (string, error) { c, ok := m.channels.Get(msg.Channel) if !ok { diff --git a/modules/deliveries/internal/store/store.go b/modules/deliveries/internal/store/store.go index 43622d3dc..34f81efae 100644 --- a/modules/deliveries/internal/store/store.go +++ b/modules/deliveries/internal/store/store.go @@ -106,7 +106,7 @@ func (s *Store) Claim(ctx context.Context, backoff func(attempts int) time.Durat err = tx.GetContext(ctx, &m, ` SELECT seq, id, org_id, channel, kind, run_id, title, body, key, request_id, expires_at, created_at, attempts FROM deliveries.messages - WHERE sent_at IS NULL AND next_attempt_at <= now() + WHERE sent_at IS NULL AND failed_at IS NULL AND next_attempt_at <= now() ORDER BY seq LIMIT 1 FOR UPDATE SKIP LOCKED`) if errors.Is(err, sql.ErrNoRows) { return Message{}, false, nil @@ -171,6 +171,13 @@ func (s *Store) Failed(ctx context.Context, seq int64, attempt int, reason strin `UPDATE deliveries.messages SET last_error = $2 WHERE seq = $1`, reason) } +// GaveUp records the attempt that ended a delivery for good: the message is +// marked failed and no worker claims it again. +func (s *Store) GaveUp(ctx context.Context, seq int64, attempt int, reason string, took time.Duration) error { + return s.attempt(ctx, seq, attempt, attemptOutcome{failure: reason, took: took}, + `UPDATE deliveries.messages SET failed_at = now(), last_error = $2 WHERE seq = $1`, reason) +} + // attemptOutcome is one attempt's result. type attemptOutcome struct { ok bool diff --git a/proto/connector/v1/connector.proto b/proto/connector/v1/connector.proto index 0f2437edb..4991b0da5 100644 --- a/proto/connector/v1/connector.proto +++ b/proto/connector/v1/connector.proto @@ -30,6 +30,9 @@ service ConnectorService { // delivery is never offered to an agent: hard rules apply to it, guards // don't. The host retries a failed delivery, so a connector should make a // repeat with the same idempotency key harmless where its service allows. + // A connector answers InvalidArgument, FailedPrecondition, PermissionDenied + // or NotFound when the service refuses the delivery for good: the host + // records the reason and stops retrying. Any other status is retried. rpc Deliver(DeliverRequest) returns (DeliverResponse); // Listen streams the events of a source the connector holds its own // connection to, such as a chat long poll, from cursor on. Each response