diff --git a/internal/daemon/reactions.go b/internal/daemon/reactions.go index 566838727..eb54cd242 100644 --- a/internal/daemon/reactions.go +++ b/internal/daemon/reactions.go @@ -91,27 +91,27 @@ func (c *reactionConsumer) handle(ctx context.Context, msg eventbus.Message) { envelope, err := workintake.DecodeReviewComment(msg.Data()) if err != nil { c.log.ErrorContext(ctx, "review reaction decode failed", "subject", msg.Subject(), "err", err) - _ = msg.Ack() // it will never decode on redelivery + c.settle(ctx, "ack", msg.Ack) // it will never decode on redelivery return } task, err := c.lookup.OpenTaskByPR(ctx, envelope.Owner, envelope.Repo, envelope.PRNumber) if err != nil { c.log.ErrorContext(ctx, "review reaction lookup failed", "owner", envelope.Owner, "repo", envelope.Repo, "pr", envelope.PRNumber, "err", err) - _ = msg.Nak() + c.settle(ctx, "nak", msg.Nak) return } if task == nil { // The resolver counts this: a reaction that does not resolve to a // live task Archie owns is dropped, never work. - c.drop(msg, "no live task owns this PR") + c.drop(ctx, msg, "no live task owns this PR") return } if c.botUser(task) != "" && envelope.Author == c.botUser(task) { // Decision 5: only Archie's own comments are excluded, so its // ReplyToReview replies never retrigger it. Every other author -- // human or bot -- is actionable. - c.drop(msg, "reaction is archie's own comment") + c.drop(ctx, msg, "reaction is archie's own comment") return } @@ -121,7 +121,7 @@ func (c *reactionConsumer) handle(ctx context.Context, msg eventbus.Message) { case workintake.ReviewReactionComment: c.handleComment(ctx, msg, task, envelope) default: - c.drop(msg, "unknown reaction kind") + c.drop(ctx, msg, "unknown reaction kind") } } @@ -133,7 +133,7 @@ func (c *reactionConsumer) handleReview(ctx context.Context, msg eventbus.Messag return } if !remediableReviewStates[e.State] { - c.drop(msg, fmt.Sprintf("review state %q does not trigger remediation", e.State)) + c.drop(ctx, msg, fmt.Sprintf("review state %q does not trigger remediation", e.State)) return } @@ -141,7 +141,7 @@ func (c *reactionConsumer) handleReview(ctx context.Context, msg eventbus.Messag payload, err := workflow.EncodeReviewUnit(unit) if err != nil { c.log.ErrorContext(ctx, "review unit encode failed", "task", task.ID, "err", err) - _ = msg.Nak() + c.settle(ctx, "nak", msg.Nak) return } if err := c.remediations.BeginRemediation(ctx, task.ID, payload); err != nil { @@ -150,15 +150,15 @@ func (c *reactionConsumer) handleReview(ctx context.Context, msg eventbus.Messag // behind, so the reaction resolves to no work. A review that // arrives mid-run is absorbed by the store, not dropped here. c.log.DebugContext(ctx, "task does not accept remediation", "task", task.ID, "review", e.ReviewID) - _ = msg.Ack() + c.settle(ctx, "ack", msg.Ack) return } c.log.ErrorContext(ctx, "BeginRemediation failed", "task", task.ID, "err", err) - _ = msg.Nak() + c.settle(ctx, "nak", msg.Nak) return } c.log.InfoContext(ctx, "review reaction queued a remediation", "task", task.ID, "pr", fmt.Sprintf("%s/%s#%d", e.Owner, e.Repo, e.PRNumber), "review", e.ReviewID, "state", e.State) - _ = msg.Ack() + c.settle(ctx, "ack", msg.Ack) } func (c *reactionConsumer) handleComment(ctx context.Context, msg eventbus.Message, task *workflowtask.Task, e workintake.ReviewCommentEnvelope) { @@ -171,19 +171,19 @@ func (c *reactionConsumer) handleComment(ctx context.Context, msg eventbus.Messa payload, err := workflow.EncodeReviewUnit(unit) if err != nil { c.log.ErrorContext(ctx, "review unit encode failed", "task", task.ID, "err", err) - _ = msg.Nak() + c.settle(ctx, "nak", msg.Nak) return } if err := c.remediations.BeginRemediation(ctx, task.ID, payload); err != nil { if errors.Is(err, storecontract.ErrStaleTransition) { - _ = msg.Ack() + c.settle(ctx, "ack", msg.Ack) return } c.log.ErrorContext(ctx, "BeginRemediation failed", "task", task.ID, "err", err) - _ = msg.Nak() + c.settle(ctx, "nak", msg.Nak) return } - _ = msg.Ack() + c.settle(ctx, "ack", msg.Ack) return } @@ -198,19 +198,19 @@ func (c *reactionConsumer) handleComment(ctx context.Context, msg eventbus.Messa }) if err != nil { c.log.ErrorContext(ctx, "review unit encode failed", "task", task.ID, "err", err) - _ = msg.Nak() + c.settle(ctx, "nak", msg.Nak) return } if err := c.remediations.UpdateReviewPayload(ctx, task.ID, payload); err != nil { if errors.Is(err, storecontract.ErrStaleTransition) { - c.drop(msg, "parent review's unit is no longer pending") + c.drop(ctx, msg, "parent review's unit is no longer pending") return } c.log.ErrorContext(ctx, "UpdateReviewPayload failed", "task", task.ID, "err", err) - _ = msg.Nak() + c.settle(ctx, "nak", msg.Nak) return } - _ = msg.Ack() + c.settle(ctx, "ack", msg.Ack) } func (c *reactionConsumer) recordApproval(ctx context.Context, msg eventbus.Message, task *workflowtask.Task, e workintake.ReviewCommentEnvelope) { @@ -220,14 +220,22 @@ func (c *reactionConsumer) recordApproval(ctx context.Context, msg eventbus.Mess Data: map[string]any{"review_id": e.ReviewID, "author": e.Author, "pr_number": e.PRNumber}, }); err != nil { c.log.ErrorContext(ctx, "record approval failed", "task", task.ID, "err", err) - _ = msg.Nak() + c.settle(ctx, "nak", msg.Nak) return } - _ = msg.Ack() + c.settle(ctx, "ack", msg.Ack) } -func (c *reactionConsumer) drop(msg eventbus.Message, reason string) { +func (c *reactionConsumer) drop(ctx context.Context, msg eventbus.Message, reason string) { c.dropped++ - c.log.Info("review reaction dropped", "subject", msg.Subject(), "reason", reason) - _ = msg.Ack() + c.log.InfoContext(ctx, "review reaction dropped", "subject", msg.Subject(), "reason", reason) + c.settle(ctx, "ack", msg.Ack) +} + +// settle runs one ack or nak. A failure leaves the message for JetStream to +// redeliver, so it is logged here rather than returned to the drain loop. +func (c *reactionConsumer) settle(ctx context.Context, how string, fn func() error) { + if err := fn(); err != nil { + c.log.WarnContext(ctx, "settle reaction message", "how", how, "err", err) + } } diff --git a/internal/infrastructure/eventbus/nats/consumer.go b/internal/infrastructure/eventbus/nats/consumer.go index 11c7b943e..2334651f5 100644 --- a/internal/infrastructure/eventbus/nats/consumer.go +++ b/internal/infrastructure/eventbus/nats/consumer.go @@ -29,7 +29,7 @@ func (c *Client) Fetch(ctx context.Context) (eventbus.Message, error) { // Honour cancellation: return the message unacked so JetStream // redelivers it rather than losing it to a shutting-down daemon. if err := ctx.Err(); err != nil { - _ = msg.Nak() + c.settle(ctx, "nak", msg.Nak) return nil, fmt.Errorf("fetch from %s: %w", c.cfg.ConsumerName, err) } return message{msg: msg}, nil @@ -41,6 +41,14 @@ func (c *Client) Fetch(ctx context.Context) (eventbus.Message, error) { return nil, eventbus.ErrNoMessage } +// settle runs one ack or nak. A failure leaves the message for JetStream to +// redeliver, so it is logged rather than returned. +func (c *Client) settle(ctx context.Context, how string, fn func() error) { + if err := fn(); err != nil { + c.log.WarnContext(ctx, "settle message", "how", how, "err", err) + } +} + // Subscribe delivers messages matching filterSubject to handle until ctx is // done. It exists so callers do not need the SDK's consume machinery; handle // receives the same boundary type as Fetch. @@ -66,7 +74,7 @@ func (c *Client) Subscribe(ctx context.Context, filterSubject string, handle fun span.RecordError(err) c.log.ErrorContext(ctx, "message handler failed", "subject", msg.Subject(), "error", err) - _ = msg.Nak() + c.settle(ctx, "nak", msg.Nak) return } if err := msg.Ack(); err != nil { diff --git a/internal/infrastructure/spanrelay/spanrelay.go b/internal/infrastructure/spanrelay/spanrelay.go index 21601f3fc..df776862b 100644 --- a/internal/infrastructure/spanrelay/spanrelay.go +++ b/internal/infrastructure/spanrelay/spanrelay.go @@ -120,18 +120,26 @@ func forward(ctx context.Context, msg eventbus.Message, upstream otlptrace.Clien var req coltracepb.ExportTraceServiceRequest if err := proto.Unmarshal(msg.Data(), &req); err != nil { log.WarnContext(ctx, "span relay dropped undecodable batch", "subject", msg.Subject(), "err", err) - _ = msg.Ack() + settle(ctx, log, "ack", msg.Ack) return } upload, cancel := context.WithTimeout(context.WithoutCancel(ctx), uploadTimeout) defer cancel() if err := upstream.UploadTraces(upload, req.GetResourceSpans()); err != nil { log.WarnContext(ctx, "span relay upload failed", "err", err) - _ = msg.Nak() + settle(ctx, log, "nak", msg.Nak) sleep(ctx, retryDelay) return } - _ = msg.Ack() + settle(ctx, log, "ack", msg.Ack) +} + +// settle runs one ack or nak. A failure leaves the batch for JetStream to +// redeliver, so it is logged rather than returned. +func settle(ctx context.Context, log *slog.Logger, how string, fn func() error) { + if err := fn(); err != nil { + log.WarnContext(ctx, "span relay settle failed", "how", how, "err", err) + } } func sleep(ctx context.Context, d time.Duration) {