Merge branch 'worktree-agent-a2d0eb6dd8a9d5b66'
This commit is contained in:
6 files changed
+74
-7
No files matched your search
@@ -60,6 +60,9 @@ type ConnectorServiceClient interface {
|
||||
// delivery is never offered to an agent: hard rules apply to it, guards
|
||||
// don't. The host retries a failed delivery, so a connector should make a
|
||||
// repeat with the same idempotency key harmless where its service allows.
|
||||
// A connector answers InvalidArgument, FailedPrecondition, PermissionDenied
|
||||
// or NotFound when the service refuses the delivery for good: the host
|
||||
// records the reason and stops retrying. Any other status is retried.
|
||||
Deliver(ctx context.Context, in *DeliverRequest, opts ...grpc.CallOption) (*DeliverResponse, error)
|
||||
// Listen streams the events of a source the connector holds its own
|
||||
// connection to, such as a chat long poll, from cursor on. Each response
|
||||
@@ -210,6 +213,9 @@ type ConnectorServiceServer interface {
|
||||
// delivery is never offered to an agent: hard rules apply to it, guards
|
||||
// don't. The host retries a failed delivery, so a connector should make a
|
||||
// repeat with the same idempotency key harmless where its service allows.
|
||||
// A connector answers InvalidArgument, FailedPrecondition, PermissionDenied
|
||||
// or NotFound when the service refuses the delivery for good: the host
|
||||
// records the reason and stops retrying. Any other status is retried.
|
||||
Deliver(context.Context, *DeliverRequest) (*DeliverResponse, error)
|
||||
// Listen streams the events of a source the connector holds its own
|
||||
// connection to, such as a chat long poll, from cursor on. Each response
|
||||
|
||||
@@ -0,0 +1,11 @@
|
||||
-- +goose Up
|
||||
-- A delivery the channel refuses for good, or that keeps failing past the retry
|
||||
-- limit, is failed rather than retried: failed_at stops the worker claiming it.
|
||||
ALTER TABLE deliveries.messages ADD COLUMN failed_at timestamptz;
|
||||
DROP INDEX deliveries.messages_due;
|
||||
CREATE INDEX messages_due ON deliveries.messages (next_attempt_at) WHERE sent_at IS NULL AND failed_at IS NULL;
|
||||
|
||||
-- +goose Down
|
||||
DROP INDEX deliveries.messages_due;
|
||||
CREATE INDEX messages_due ON deliveries.messages (next_attempt_at) WHERE sent_at IS NULL;
|
||||
ALTER TABLE deliveries.messages DROP COLUMN failed_at;
|
||||
@@ -13,6 +13,8 @@ import (
|
||||
"time"
|
||||
|
||||
"github.com/google/jsonschema-go/jsonschema"
|
||||
"google.golang.org/grpc/codes"
|
||||
"google.golang.org/grpc/status"
|
||||
|
||||
pb "github.com/samcharles93/archie-core/internal/contracts/connector/v1"
|
||||
)
|
||||
@@ -29,6 +31,10 @@ var (
|
||||
// ErrUnknownName names an action, lookup or event type the connector
|
||||
// never declared.
|
||||
ErrUnknownName = errors.New("not declared by the connector")
|
||||
// ErrRejected means a delivery was refused in a way a retry can't change:
|
||||
// the connector answered InvalidArgument, FailedPrecondition,
|
||||
// PermissionDenied or NotFound, or the input broke the declared schema.
|
||||
ErrRejected = errors.New("delivery rejected for good")
|
||||
)
|
||||
|
||||
// Reversibility says whether and how an action can be undone.
|
||||
@@ -229,17 +235,31 @@ func (c *Connector) Deliver(ctx context.Context, name string, input jsontext.Val
|
||||
return "", fmt.Errorf("connector %q: delivery %q: %w", c.name, name, ErrUnknownName)
|
||||
}
|
||||
if err := validate(schema, input); err != nil {
|
||||
return "", fmt.Errorf("connector %q: delivery %q: %w", c.name, name, err)
|
||||
return "", fmt.Errorf("connector %q: delivery %q: %w: %w", c.name, name, ErrRejected, err)
|
||||
}
|
||||
req := &pb.DeliverRequest{Delivery: name, Input: input, IdempotencyKey: key}
|
||||
resp, err := c.client.Deliver(ctx, req)
|
||||
recordProto(ctx, c.name, "deliver", req, resp, err)
|
||||
if err != nil {
|
||||
if rejectedForGood(err) {
|
||||
return "", fmt.Errorf("connector %q: deliver %q: %w: %w", c.name, name, ErrRejected, err)
|
||||
}
|
||||
return "", fmt.Errorf("connector %q: deliver %q: %w", c.name, name, err)
|
||||
}
|
||||
return resp.GetExternalId(), nil
|
||||
}
|
||||
|
||||
// rejectedForGood reports whether a connector's error says the same delivery
|
||||
// can never succeed.
|
||||
func rejectedForGood(err error) bool {
|
||||
switch status.Code(err) {
|
||||
case codes.InvalidArgument, codes.FailedPrecondition, codes.PermissionDenied, codes.NotFound:
|
||||
return true
|
||||
default:
|
||||
return false
|
||||
}
|
||||
}
|
||||
|
||||
// Listens reports whether the connector holds its own connection to a source
|
||||
// and streams its events.
|
||||
func (c *Connector) Listens() bool { return c.listens }
|
||||
|
||||
@@ -4,7 +4,7 @@
|
||||
// dashboard, is the first channel: its messages are rows a client reads from a
|
||||
// cursor, with a live stream that only says when to read. Every accepted
|
||||
// connector that delivers messages is another: each gets its own row, retried
|
||||
// until the connector confirms it.
|
||||
// until the connector confirms it, refuses it for good, or the attempts run out.
|
||||
package deliveries
|
||||
|
||||
import (
|
||||
@@ -37,6 +37,9 @@ const (
|
||||
// deliveryMessage is the delivery a connector declares to carry messages.
|
||||
deliveryMessage = "message"
|
||||
maxBackoff = 10 * time.Minute
|
||||
// maxAttempts ends a delivery that keeps failing: with the backoff capped
|
||||
// at maxBackoff it is about a day of trying.
|
||||
maxAttempts = 150
|
||||
)
|
||||
|
||||
// The kinds of message a channel carries.
|
||||
@@ -191,7 +194,9 @@ func (m *Module) SentBy(ctx context.Context, channel string, ids ...string) (uui
|
||||
}
|
||||
|
||||
// Work sends queued messages to their connectors until ctx ends. A failed send
|
||||
// is tried again after a backoff that grows to maxBackoff, never dropped.
|
||||
// is tried again after a backoff that grows to maxBackoff. A connector's
|
||||
// permanent refusal, or the maxAttempts-th failure, marks the message failed
|
||||
// with its reason.
|
||||
func (m *Module) Work(ctx context.Context) {
|
||||
for ctx.Err() == nil {
|
||||
msg, found, err := m.store.Claim(ctx, backoff)
|
||||
@@ -215,9 +220,7 @@ func (m *Module) deliver(ctx context.Context, msg store.Message) {
|
||||
took := time.Since(start)
|
||||
if err != nil {
|
||||
m.log.WarnContext(ctx, "delivery failed", "channel", msg.Channel, "message", msg.ID, "attempt", msg.Attempts, "error", err)
|
||||
if err := m.store.Failed(ctx, msg.Seq, msg.Attempts, err.Error(), took); err != nil {
|
||||
m.log.ErrorContext(ctx, "record failed delivery", "error", err)
|
||||
}
|
||||
m.recordFailure(ctx, msg, err, took)
|
||||
return
|
||||
}
|
||||
if err := m.store.Sent(ctx, msg.Seq, msg.Attempts, external, took); err != nil {
|
||||
@@ -225,6 +228,23 @@ func (m *Module) deliver(ctx context.Context, msg store.Message) {
|
||||
}
|
||||
}
|
||||
|
||||
// recordFailure stores why an attempt failed, and ends the delivery when the
|
||||
// channel refused it for good or the attempts are spent.
|
||||
func (m *Module) recordFailure(ctx context.Context, msg store.Message, cause error, took time.Duration) {
|
||||
record := m.store.Failed
|
||||
reason := cause.Error()
|
||||
switch {
|
||||
case errors.Is(cause, connectors.ErrRejected):
|
||||
record = m.store.GaveUp
|
||||
case msg.Attempts >= maxAttempts:
|
||||
record = m.store.GaveUp
|
||||
reason = fmt.Sprintf("gave up after %d attempts: %s", msg.Attempts, reason)
|
||||
}
|
||||
if err := record(ctx, msg.Seq, msg.Attempts, reason, took); err != nil {
|
||||
m.log.ErrorContext(ctx, "record failed delivery", "error", err)
|
||||
}
|
||||
}
|
||||
|
||||
func (m *Module) sendTo(ctx context.Context, msg store.Message) (string, error) {
|
||||
c, ok := m.channels.Get(msg.Channel)
|
||||
if !ok {
|
||||
|
||||
@@ -106,7 +106,7 @@ func (s *Store) Claim(ctx context.Context, backoff func(attempts int) time.Durat
|
||||
err = tx.GetContext(ctx, &m, `
|
||||
SELECT seq, id, org_id, channel, kind, run_id, title, body, key, request_id, expires_at, created_at, attempts
|
||||
FROM deliveries.messages
|
||||
WHERE sent_at IS NULL AND next_attempt_at <= now()
|
||||
WHERE sent_at IS NULL AND failed_at IS NULL AND next_attempt_at <= now()
|
||||
ORDER BY seq LIMIT 1 FOR UPDATE SKIP LOCKED`)
|
||||
if errors.Is(err, sql.ErrNoRows) {
|
||||
return Message{}, false, nil
|
||||
@@ -171,6 +171,13 @@ func (s *Store) Failed(ctx context.Context, seq int64, attempt int, reason strin
|
||||
`UPDATE deliveries.messages SET last_error = $2 WHERE seq = $1`, reason)
|
||||
}
|
||||
|
||||
// GaveUp records the attempt that ended a delivery for good: the message is
|
||||
// marked failed and no worker claims it again.
|
||||
func (s *Store) GaveUp(ctx context.Context, seq int64, attempt int, reason string, took time.Duration) error {
|
||||
return s.attempt(ctx, seq, attempt, attemptOutcome{failure: reason, took: took},
|
||||
`UPDATE deliveries.messages SET failed_at = now(), last_error = $2 WHERE seq = $1`, reason)
|
||||
}
|
||||
|
||||
// attemptOutcome is one attempt's result.
|
||||
type attemptOutcome struct {
|
||||
ok bool
|
||||
|
||||
@@ -30,6 +30,9 @@ service ConnectorService {
|
||||
// delivery is never offered to an agent: hard rules apply to it, guards
|
||||
// don't. The host retries a failed delivery, so a connector should make a
|
||||
// repeat with the same idempotency key harmless where its service allows.
|
||||
// A connector answers InvalidArgument, FailedPrecondition, PermissionDenied
|
||||
// or NotFound when the service refuses the delivery for good: the host
|
||||
// records the reason and stops retrying. Any other status is retried.
|
||||
rpc Deliver(DeliverRequest) returns (DeliverResponse);
|
||||
// Listen streams the events of a source the connector holds its own
|
||||
// connection to, such as a chat long poll, from cursor on. Each response
|
||||
|
||||
Reference in new issue
Block a user