Merge branch 'worktree-agent-a2d0eb6dd8a9d5b66'

This commit is contained in:
sam committed 2026-11-21 13:47:46 +11:00
commit 29a06f7c54
6 files changed
+74 -7

No files matched your search

@@ -60,6 +60,9 @@ type ConnectorServiceClient interface {
// delivery is never offered to an agent: hard rules apply to it, guards
// don't. The host retries a failed delivery, so a connector should make a
// repeat with the same idempotency key harmless where its service allows.
// A connector answers InvalidArgument, FailedPrecondition, PermissionDenied
// or NotFound when the service refuses the delivery for good: the host
// records the reason and stops retrying. Any other status is retried.
Deliver(ctx context.Context, in *DeliverRequest, opts ...grpc.CallOption) (*DeliverResponse, error)
// Listen streams the events of a source the connector holds its own
// connection to, such as a chat long poll, from cursor on. Each response
@@ -210,6 +213,9 @@ type ConnectorServiceServer interface {
// delivery is never offered to an agent: hard rules apply to it, guards
// don't. The host retries a failed delivery, so a connector should make a
// repeat with the same idempotency key harmless where its service allows.
// A connector answers InvalidArgument, FailedPrecondition, PermissionDenied
// or NotFound when the service refuses the delivery for good: the host
// records the reason and stops retrying. Any other status is retried.
Deliver(context.Context, *DeliverRequest) (*DeliverResponse, error)
// Listen streams the events of a source the connector holds its own
// connection to, such as a chat long poll, from cursor on. Each response
@@ -0,0 +1,11 @@
-- +goose Up
-- A delivery the channel refuses for good, or that keeps failing past the retry
-- limit, is failed rather than retried: failed_at stops the worker claiming it.
ALTER TABLE deliveries.messages ADD COLUMN failed_at timestamptz;
DROP INDEX deliveries.messages_due;
CREATE INDEX messages_due ON deliveries.messages (next_attempt_at) WHERE sent_at IS NULL AND failed_at IS NULL;
-- +goose Down
DROP INDEX deliveries.messages_due;
CREATE INDEX messages_due ON deliveries.messages (next_attempt_at) WHERE sent_at IS NULL;
ALTER TABLE deliveries.messages DROP COLUMN failed_at;
+21 -1
View File
@@ -13,6 +13,8 @@ import (
"time"
"github.com/google/jsonschema-go/jsonschema"
"google.golang.org/grpc/codes"
"google.golang.org/grpc/status"
pb "github.com/samcharles93/archie-core/internal/contracts/connector/v1"
)
@@ -29,6 +31,10 @@ var (
// ErrUnknownName names an action, lookup or event type the connector
// never declared.
ErrUnknownName = errors.New("not declared by the connector")
// ErrRejected means a delivery was refused in a way a retry can't change:
// the connector answered InvalidArgument, FailedPrecondition,
// PermissionDenied or NotFound, or the input broke the declared schema.
ErrRejected = errors.New("delivery rejected for good")
)
// Reversibility says whether and how an action can be undone.
@@ -229,17 +235,31 @@ func (c *Connector) Deliver(ctx context.Context, name string, input jsontext.Val
return "", fmt.Errorf("connector %q: delivery %q: %w", c.name, name, ErrUnknownName)
}
if err := validate(schema, input); err != nil {
return "", fmt.Errorf("connector %q: delivery %q: %w", c.name, name, err)
return "", fmt.Errorf("connector %q: delivery %q: %w: %w", c.name, name, ErrRejected, err)
}
req := &pb.DeliverRequest{Delivery: name, Input: input, IdempotencyKey: key}
resp, err := c.client.Deliver(ctx, req)
recordProto(ctx, c.name, "deliver", req, resp, err)
if err != nil {
if rejectedForGood(err) {
return "", fmt.Errorf("connector %q: deliver %q: %w: %w", c.name, name, ErrRejected, err)
}
return "", fmt.Errorf("connector %q: deliver %q: %w", c.name, name, err)
}
return resp.GetExternalId(), nil
}
// rejectedForGood reports whether a connector's error says the same delivery
// can never succeed.
func rejectedForGood(err error) bool {
switch status.Code(err) {
case codes.InvalidArgument, codes.FailedPrecondition, codes.PermissionDenied, codes.NotFound:
return true
default:
return false
}
}
// Listens reports whether the connector holds its own connection to a source
// and streams its events.
func (c *Connector) Listens() bool { return c.listens }
+25 -5
View File
@@ -4,7 +4,7 @@
// dashboard, is the first channel: its messages are rows a client reads from a
// cursor, with a live stream that only says when to read. Every accepted
// connector that delivers messages is another: each gets its own row, retried
// until the connector confirms it.
// until the connector confirms it, refuses it for good, or the attempts run out.
package deliveries
import (
@@ -37,6 +37,9 @@ const (
// deliveryMessage is the delivery a connector declares to carry messages.
deliveryMessage = "message"
maxBackoff = 10 * time.Minute
// maxAttempts ends a delivery that keeps failing: with the backoff capped
// at maxBackoff it is about a day of trying.
maxAttempts = 150
)
// The kinds of message a channel carries.
@@ -191,7 +194,9 @@ func (m *Module) SentBy(ctx context.Context, channel string, ids ...string) (uui
}
// Work sends queued messages to their connectors until ctx ends. A failed send
// is tried again after a backoff that grows to maxBackoff, never dropped.
// is tried again after a backoff that grows to maxBackoff. A connector's
// permanent refusal, or the maxAttempts-th failure, marks the message failed
// with its reason.
func (m *Module) Work(ctx context.Context) {
for ctx.Err() == nil {
msg, found, err := m.store.Claim(ctx, backoff)
@@ -215,9 +220,7 @@ func (m *Module) deliver(ctx context.Context, msg store.Message) {
took := time.Since(start)
if err != nil {
m.log.WarnContext(ctx, "delivery failed", "channel", msg.Channel, "message", msg.ID, "attempt", msg.Attempts, "error", err)
if err := m.store.Failed(ctx, msg.Seq, msg.Attempts, err.Error(), took); err != nil {
m.log.ErrorContext(ctx, "record failed delivery", "error", err)
}
m.recordFailure(ctx, msg, err, took)
return
}
if err := m.store.Sent(ctx, msg.Seq, msg.Attempts, external, took); err != nil {
@@ -225,6 +228,23 @@ func (m *Module) deliver(ctx context.Context, msg store.Message) {
}
}
// recordFailure stores why an attempt failed, and ends the delivery when the
// channel refused it for good or the attempts are spent.
func (m *Module) recordFailure(ctx context.Context, msg store.Message, cause error, took time.Duration) {
record := m.store.Failed
reason := cause.Error()
switch {
case errors.Is(cause, connectors.ErrRejected):
record = m.store.GaveUp
case msg.Attempts >= maxAttempts:
record = m.store.GaveUp
reason = fmt.Sprintf("gave up after %d attempts: %s", msg.Attempts, reason)
}
if err := record(ctx, msg.Seq, msg.Attempts, reason, took); err != nil {
m.log.ErrorContext(ctx, "record failed delivery", "error", err)
}
}
func (m *Module) sendTo(ctx context.Context, msg store.Message) (string, error) {
c, ok := m.channels.Get(msg.Channel)
if !ok {
+8 -1
View File
@@ -106,7 +106,7 @@ func (s *Store) Claim(ctx context.Context, backoff func(attempts int) time.Durat
err = tx.GetContext(ctx, &m, `
SELECT seq, id, org_id, channel, kind, run_id, title, body, key, request_id, expires_at, created_at, attempts
FROM deliveries.messages
WHERE sent_at IS NULL AND next_attempt_at <= now()
WHERE sent_at IS NULL AND failed_at IS NULL AND next_attempt_at <= now()
ORDER BY seq LIMIT 1 FOR UPDATE SKIP LOCKED`)
if errors.Is(err, sql.ErrNoRows) {
return Message{}, false, nil
@@ -171,6 +171,13 @@ func (s *Store) Failed(ctx context.Context, seq int64, attempt int, reason strin
`UPDATE deliveries.messages SET last_error = $2 WHERE seq = $1`, reason)
}
// GaveUp records the attempt that ended a delivery for good: the message is
// marked failed and no worker claims it again.
func (s *Store) GaveUp(ctx context.Context, seq int64, attempt int, reason string, took time.Duration) error {
return s.attempt(ctx, seq, attempt, attemptOutcome{failure: reason, took: took},
`UPDATE deliveries.messages SET failed_at = now(), last_error = $2 WHERE seq = $1`, reason)
}
// attemptOutcome is one attempt's result.
type attemptOutcome struct {
ok bool
+3
View File
@@ -30,6 +30,9 @@ service ConnectorService {
// delivery is never offered to an agent: hard rules apply to it, guards
// don't. The host retries a failed delivery, so a connector should make a
// repeat with the same idempotency key harmless where its service allows.
// A connector answers InvalidArgument, FailedPrecondition, PermissionDenied
// or NotFound when the service refuses the delivery for good: the host
// records the reason and stops retrying. Any other status is retried.
rpc Deliver(DeliverRequest) returns (DeliverResponse);
// Listen streams the events of a source the connector holds its own
// connection to, such as a chat long poll, from cursor on. Each response