fix: log NATS ack and nak failures instead of discarding them (#1924)

This commit is contained in:
sam committed 2026-11-13 03:52:38 +11:00
1 parent b25676b612
commit 68e8552f0d
3 files changed
+52 -28

No files matched your search

+31 -23
View File
@@ -91,27 +91,27 @@ func (c *reactionConsumer) handle(ctx context.Context, msg eventbus.Message) {
envelope, err := workintake.DecodeReviewComment(msg.Data())
if err != nil {
c.log.ErrorContext(ctx, "review reaction decode failed", "subject", msg.Subject(), "err", err)
_ = msg.Ack() // it will never decode on redelivery
c.settle(ctx, "ack", msg.Ack) // it will never decode on redelivery
return
}
task, err := c.lookup.OpenTaskByPR(ctx, envelope.Owner, envelope.Repo, envelope.PRNumber)
if err != nil {
c.log.ErrorContext(ctx, "review reaction lookup failed", "owner", envelope.Owner, "repo", envelope.Repo, "pr", envelope.PRNumber, "err", err)
_ = msg.Nak()
c.settle(ctx, "nak", msg.Nak)
return
}
if task == nil {
// The resolver counts this: a reaction that does not resolve to a
// live task Archie owns is dropped, never work.
c.drop(msg, "no live task owns this PR")
c.drop(ctx, msg, "no live task owns this PR")
return
}
if c.botUser(task) != "" && envelope.Author == c.botUser(task) {
// Decision 5: only Archie's own comments are excluded, so its
// ReplyToReview replies never retrigger it. Every other author --
// human or bot -- is actionable.
c.drop(msg, "reaction is archie's own comment")
c.drop(ctx, msg, "reaction is archie's own comment")
return
}
@@ -121,7 +121,7 @@ func (c *reactionConsumer) handle(ctx context.Context, msg eventbus.Message) {
case workintake.ReviewReactionComment:
c.handleComment(ctx, msg, task, envelope)
default:
c.drop(msg, "unknown reaction kind")
c.drop(ctx, msg, "unknown reaction kind")
}
}
@@ -133,7 +133,7 @@ func (c *reactionConsumer) handleReview(ctx context.Context, msg eventbus.Messag
return
}
if !remediableReviewStates[e.State] {
c.drop(msg, fmt.Sprintf("review state %q does not trigger remediation", e.State))
c.drop(ctx, msg, fmt.Sprintf("review state %q does not trigger remediation", e.State))
return
}
@@ -141,7 +141,7 @@ func (c *reactionConsumer) handleReview(ctx context.Context, msg eventbus.Messag
payload, err := workflow.EncodeReviewUnit(unit)
if err != nil {
c.log.ErrorContext(ctx, "review unit encode failed", "task", task.ID, "err", err)
_ = msg.Nak()
c.settle(ctx, "nak", msg.Nak)
return
}
if err := c.remediations.BeginRemediation(ctx, task.ID, payload); err != nil {
@@ -150,15 +150,15 @@ func (c *reactionConsumer) handleReview(ctx context.Context, msg eventbus.Messag
// behind, so the reaction resolves to no work. A review that
// arrives mid-run is absorbed by the store, not dropped here.
c.log.DebugContext(ctx, "task does not accept remediation", "task", task.ID, "review", e.ReviewID)
_ = msg.Ack()
c.settle(ctx, "ack", msg.Ack)
return
}
c.log.ErrorContext(ctx, "BeginRemediation failed", "task", task.ID, "err", err)
_ = msg.Nak()
c.settle(ctx, "nak", msg.Nak)
return
}
c.log.InfoContext(ctx, "review reaction queued a remediation", "task", task.ID, "pr", fmt.Sprintf("%s/%s#%d", e.Owner, e.Repo, e.PRNumber), "review", e.ReviewID, "state", e.State)
_ = msg.Ack()
c.settle(ctx, "ack", msg.Ack)
}
func (c *reactionConsumer) handleComment(ctx context.Context, msg eventbus.Message, task *workflowtask.Task, e workintake.ReviewCommentEnvelope) {
@@ -171,19 +171,19 @@ func (c *reactionConsumer) handleComment(ctx context.Context, msg eventbus.Messa
payload, err := workflow.EncodeReviewUnit(unit)
if err != nil {
c.log.ErrorContext(ctx, "review unit encode failed", "task", task.ID, "err", err)
_ = msg.Nak()
c.settle(ctx, "nak", msg.Nak)
return
}
if err := c.remediations.BeginRemediation(ctx, task.ID, payload); err != nil {
if errors.Is(err, storecontract.ErrStaleTransition) {
_ = msg.Ack()
c.settle(ctx, "ack", msg.Ack)
return
}
c.log.ErrorContext(ctx, "BeginRemediation failed", "task", task.ID, "err", err)
_ = msg.Nak()
c.settle(ctx, "nak", msg.Nak)
return
}
_ = msg.Ack()
c.settle(ctx, "ack", msg.Ack)
return
}
@@ -198,19 +198,19 @@ func (c *reactionConsumer) handleComment(ctx context.Context, msg eventbus.Messa
})
if err != nil {
c.log.ErrorContext(ctx, "review unit encode failed", "task", task.ID, "err", err)
_ = msg.Nak()
c.settle(ctx, "nak", msg.Nak)
return
}
if err := c.remediations.UpdateReviewPayload(ctx, task.ID, payload); err != nil {
if errors.Is(err, storecontract.ErrStaleTransition) {
c.drop(msg, "parent review's unit is no longer pending")
c.drop(ctx, msg, "parent review's unit is no longer pending")
return
}
c.log.ErrorContext(ctx, "UpdateReviewPayload failed", "task", task.ID, "err", err)
_ = msg.Nak()
c.settle(ctx, "nak", msg.Nak)
return
}
_ = msg.Ack()
c.settle(ctx, "ack", msg.Ack)
}
func (c *reactionConsumer) recordApproval(ctx context.Context, msg eventbus.Message, task *workflowtask.Task, e workintake.ReviewCommentEnvelope) {
@@ -220,14 +220,22 @@ func (c *reactionConsumer) recordApproval(ctx context.Context, msg eventbus.Mess
Data: map[string]any{"review_id": e.ReviewID, "author": e.Author, "pr_number": e.PRNumber},
}); err != nil {
c.log.ErrorContext(ctx, "record approval failed", "task", task.ID, "err", err)
_ = msg.Nak()
c.settle(ctx, "nak", msg.Nak)
return
}
_ = msg.Ack()
c.settle(ctx, "ack", msg.Ack)
}
func (c *reactionConsumer) drop(msg eventbus.Message, reason string) {
func (c *reactionConsumer) drop(ctx context.Context, msg eventbus.Message, reason string) {
c.dropped++
c.log.Info("review reaction dropped", "subject", msg.Subject(), "reason", reason)
_ = msg.Ack()
c.log.InfoContext(ctx, "review reaction dropped", "subject", msg.Subject(), "reason", reason)
c.settle(ctx, "ack", msg.Ack)
}
// settle runs one ack or nak. A failure leaves the message for JetStream to
// redeliver, so it is logged here rather than returned to the drain loop.
func (c *reactionConsumer) settle(ctx context.Context, how string, fn func() error) {
if err := fn(); err != nil {
c.log.WarnContext(ctx, "settle reaction message", "how", how, "err", err)
}
}
@@ -29,7 +29,7 @@ func (c *Client) Fetch(ctx context.Context) (eventbus.Message, error) {
// Honour cancellation: return the message unacked so JetStream
// redelivers it rather than losing it to a shutting-down daemon.
if err := ctx.Err(); err != nil {
_ = msg.Nak()
c.settle(ctx, "nak", msg.Nak)
return nil, fmt.Errorf("fetch from %s: %w", c.cfg.ConsumerName, err)
}
return message{msg: msg}, nil
@@ -41,6 +41,14 @@ func (c *Client) Fetch(ctx context.Context) (eventbus.Message, error) {
return nil, eventbus.ErrNoMessage
}
// settle runs one ack or nak. A failure leaves the message for JetStream to
// redeliver, so it is logged rather than returned.
func (c *Client) settle(ctx context.Context, how string, fn func() error) {
if err := fn(); err != nil {
c.log.WarnContext(ctx, "settle message", "how", how, "err", err)
}
}
// Subscribe delivers messages matching filterSubject to handle until ctx is
// done. It exists so callers do not need the SDK's consume machinery; handle
// receives the same boundary type as Fetch.
@@ -66,7 +74,7 @@ func (c *Client) Subscribe(ctx context.Context, filterSubject string, handle fun
span.RecordError(err)
c.log.ErrorContext(ctx, "message handler failed",
"subject", msg.Subject(), "error", err)
_ = msg.Nak()
c.settle(ctx, "nak", msg.Nak)
return
}
if err := msg.Ack(); err != nil {
+11 -3
View File
@@ -120,18 +120,26 @@ func forward(ctx context.Context, msg eventbus.Message, upstream otlptrace.Clien
var req coltracepb.ExportTraceServiceRequest
if err := proto.Unmarshal(msg.Data(), &req); err != nil {
log.WarnContext(ctx, "span relay dropped undecodable batch", "subject", msg.Subject(), "err", err)
_ = msg.Ack()
settle(ctx, log, "ack", msg.Ack)
return
}
upload, cancel := context.WithTimeout(context.WithoutCancel(ctx), uploadTimeout)
defer cancel()
if err := upstream.UploadTraces(upload, req.GetResourceSpans()); err != nil {
log.WarnContext(ctx, "span relay upload failed", "err", err)
_ = msg.Nak()
settle(ctx, log, "nak", msg.Nak)
sleep(ctx, retryDelay)
return
}
_ = msg.Ack()
settle(ctx, log, "ack", msg.Ack)
}
// settle runs one ack or nak. A failure leaves the batch for JetStream to
// redeliver, so it is logged rather than returned.
func settle(ctx context.Context, log *slog.Logger, how string, fn func() error) {
if err := fn(); err != nil {
log.WarnContext(ctx, "span relay settle failed", "how", how, "err", err)
}
}
func sleep(ctx context.Context, d time.Duration) {