fix: log NATS ack and nak failures instead of discarding them (#1924)
This commit is contained in:
3 files changed
+52
-28
No files matched your search
@@ -91,27 +91,27 @@ func (c *reactionConsumer) handle(ctx context.Context, msg eventbus.Message) {
|
||||
envelope, err := workintake.DecodeReviewComment(msg.Data())
|
||||
if err != nil {
|
||||
c.log.ErrorContext(ctx, "review reaction decode failed", "subject", msg.Subject(), "err", err)
|
||||
_ = msg.Ack() // it will never decode on redelivery
|
||||
c.settle(ctx, "ack", msg.Ack) // it will never decode on redelivery
|
||||
return
|
||||
}
|
||||
|
||||
task, err := c.lookup.OpenTaskByPR(ctx, envelope.Owner, envelope.Repo, envelope.PRNumber)
|
||||
if err != nil {
|
||||
c.log.ErrorContext(ctx, "review reaction lookup failed", "owner", envelope.Owner, "repo", envelope.Repo, "pr", envelope.PRNumber, "err", err)
|
||||
_ = msg.Nak()
|
||||
c.settle(ctx, "nak", msg.Nak)
|
||||
return
|
||||
}
|
||||
if task == nil {
|
||||
// The resolver counts this: a reaction that does not resolve to a
|
||||
// live task Archie owns is dropped, never work.
|
||||
c.drop(msg, "no live task owns this PR")
|
||||
c.drop(ctx, msg, "no live task owns this PR")
|
||||
return
|
||||
}
|
||||
if c.botUser(task) != "" && envelope.Author == c.botUser(task) {
|
||||
// Decision 5: only Archie's own comments are excluded, so its
|
||||
// ReplyToReview replies never retrigger it. Every other author --
|
||||
// human or bot -- is actionable.
|
||||
c.drop(msg, "reaction is archie's own comment")
|
||||
c.drop(ctx, msg, "reaction is archie's own comment")
|
||||
return
|
||||
}
|
||||
|
||||
@@ -121,7 +121,7 @@ func (c *reactionConsumer) handle(ctx context.Context, msg eventbus.Message) {
|
||||
case workintake.ReviewReactionComment:
|
||||
c.handleComment(ctx, msg, task, envelope)
|
||||
default:
|
||||
c.drop(msg, "unknown reaction kind")
|
||||
c.drop(ctx, msg, "unknown reaction kind")
|
||||
}
|
||||
}
|
||||
|
||||
@@ -133,7 +133,7 @@ func (c *reactionConsumer) handleReview(ctx context.Context, msg eventbus.Messag
|
||||
return
|
||||
}
|
||||
if !remediableReviewStates[e.State] {
|
||||
c.drop(msg, fmt.Sprintf("review state %q does not trigger remediation", e.State))
|
||||
c.drop(ctx, msg, fmt.Sprintf("review state %q does not trigger remediation", e.State))
|
||||
return
|
||||
}
|
||||
|
||||
@@ -141,7 +141,7 @@ func (c *reactionConsumer) handleReview(ctx context.Context, msg eventbus.Messag
|
||||
payload, err := workflow.EncodeReviewUnit(unit)
|
||||
if err != nil {
|
||||
c.log.ErrorContext(ctx, "review unit encode failed", "task", task.ID, "err", err)
|
||||
_ = msg.Nak()
|
||||
c.settle(ctx, "nak", msg.Nak)
|
||||
return
|
||||
}
|
||||
if err := c.remediations.BeginRemediation(ctx, task.ID, payload); err != nil {
|
||||
@@ -150,15 +150,15 @@ func (c *reactionConsumer) handleReview(ctx context.Context, msg eventbus.Messag
|
||||
// behind, so the reaction resolves to no work. A review that
|
||||
// arrives mid-run is absorbed by the store, not dropped here.
|
||||
c.log.DebugContext(ctx, "task does not accept remediation", "task", task.ID, "review", e.ReviewID)
|
||||
_ = msg.Ack()
|
||||
c.settle(ctx, "ack", msg.Ack)
|
||||
return
|
||||
}
|
||||
c.log.ErrorContext(ctx, "BeginRemediation failed", "task", task.ID, "err", err)
|
||||
_ = msg.Nak()
|
||||
c.settle(ctx, "nak", msg.Nak)
|
||||
return
|
||||
}
|
||||
c.log.InfoContext(ctx, "review reaction queued a remediation", "task", task.ID, "pr", fmt.Sprintf("%s/%s#%d", e.Owner, e.Repo, e.PRNumber), "review", e.ReviewID, "state", e.State)
|
||||
_ = msg.Ack()
|
||||
c.settle(ctx, "ack", msg.Ack)
|
||||
}
|
||||
|
||||
func (c *reactionConsumer) handleComment(ctx context.Context, msg eventbus.Message, task *workflowtask.Task, e workintake.ReviewCommentEnvelope) {
|
||||
@@ -171,19 +171,19 @@ func (c *reactionConsumer) handleComment(ctx context.Context, msg eventbus.Messa
|
||||
payload, err := workflow.EncodeReviewUnit(unit)
|
||||
if err != nil {
|
||||
c.log.ErrorContext(ctx, "review unit encode failed", "task", task.ID, "err", err)
|
||||
_ = msg.Nak()
|
||||
c.settle(ctx, "nak", msg.Nak)
|
||||
return
|
||||
}
|
||||
if err := c.remediations.BeginRemediation(ctx, task.ID, payload); err != nil {
|
||||
if errors.Is(err, storecontract.ErrStaleTransition) {
|
||||
_ = msg.Ack()
|
||||
c.settle(ctx, "ack", msg.Ack)
|
||||
return
|
||||
}
|
||||
c.log.ErrorContext(ctx, "BeginRemediation failed", "task", task.ID, "err", err)
|
||||
_ = msg.Nak()
|
||||
c.settle(ctx, "nak", msg.Nak)
|
||||
return
|
||||
}
|
||||
_ = msg.Ack()
|
||||
c.settle(ctx, "ack", msg.Ack)
|
||||
return
|
||||
}
|
||||
|
||||
@@ -198,19 +198,19 @@ func (c *reactionConsumer) handleComment(ctx context.Context, msg eventbus.Messa
|
||||
})
|
||||
if err != nil {
|
||||
c.log.ErrorContext(ctx, "review unit encode failed", "task", task.ID, "err", err)
|
||||
_ = msg.Nak()
|
||||
c.settle(ctx, "nak", msg.Nak)
|
||||
return
|
||||
}
|
||||
if err := c.remediations.UpdateReviewPayload(ctx, task.ID, payload); err != nil {
|
||||
if errors.Is(err, storecontract.ErrStaleTransition) {
|
||||
c.drop(msg, "parent review's unit is no longer pending")
|
||||
c.drop(ctx, msg, "parent review's unit is no longer pending")
|
||||
return
|
||||
}
|
||||
c.log.ErrorContext(ctx, "UpdateReviewPayload failed", "task", task.ID, "err", err)
|
||||
_ = msg.Nak()
|
||||
c.settle(ctx, "nak", msg.Nak)
|
||||
return
|
||||
}
|
||||
_ = msg.Ack()
|
||||
c.settle(ctx, "ack", msg.Ack)
|
||||
}
|
||||
|
||||
func (c *reactionConsumer) recordApproval(ctx context.Context, msg eventbus.Message, task *workflowtask.Task, e workintake.ReviewCommentEnvelope) {
|
||||
@@ -220,14 +220,22 @@ func (c *reactionConsumer) recordApproval(ctx context.Context, msg eventbus.Mess
|
||||
Data: map[string]any{"review_id": e.ReviewID, "author": e.Author, "pr_number": e.PRNumber},
|
||||
}); err != nil {
|
||||
c.log.ErrorContext(ctx, "record approval failed", "task", task.ID, "err", err)
|
||||
_ = msg.Nak()
|
||||
c.settle(ctx, "nak", msg.Nak)
|
||||
return
|
||||
}
|
||||
_ = msg.Ack()
|
||||
c.settle(ctx, "ack", msg.Ack)
|
||||
}
|
||||
|
||||
func (c *reactionConsumer) drop(msg eventbus.Message, reason string) {
|
||||
func (c *reactionConsumer) drop(ctx context.Context, msg eventbus.Message, reason string) {
|
||||
c.dropped++
|
||||
c.log.Info("review reaction dropped", "subject", msg.Subject(), "reason", reason)
|
||||
_ = msg.Ack()
|
||||
c.log.InfoContext(ctx, "review reaction dropped", "subject", msg.Subject(), "reason", reason)
|
||||
c.settle(ctx, "ack", msg.Ack)
|
||||
}
|
||||
|
||||
// settle runs one ack or nak. A failure leaves the message for JetStream to
|
||||
// redeliver, so it is logged here rather than returned to the drain loop.
|
||||
func (c *reactionConsumer) settle(ctx context.Context, how string, fn func() error) {
|
||||
if err := fn(); err != nil {
|
||||
c.log.WarnContext(ctx, "settle reaction message", "how", how, "err", err)
|
||||
}
|
||||
}
|
||||
@@ -29,7 +29,7 @@ func (c *Client) Fetch(ctx context.Context) (eventbus.Message, error) {
|
||||
// Honour cancellation: return the message unacked so JetStream
|
||||
// redelivers it rather than losing it to a shutting-down daemon.
|
||||
if err := ctx.Err(); err != nil {
|
||||
_ = msg.Nak()
|
||||
c.settle(ctx, "nak", msg.Nak)
|
||||
return nil, fmt.Errorf("fetch from %s: %w", c.cfg.ConsumerName, err)
|
||||
}
|
||||
return message{msg: msg}, nil
|
||||
@@ -41,6 +41,14 @@ func (c *Client) Fetch(ctx context.Context) (eventbus.Message, error) {
|
||||
return nil, eventbus.ErrNoMessage
|
||||
}
|
||||
|
||||
// settle runs one ack or nak. A failure leaves the message for JetStream to
|
||||
// redeliver, so it is logged rather than returned.
|
||||
func (c *Client) settle(ctx context.Context, how string, fn func() error) {
|
||||
if err := fn(); err != nil {
|
||||
c.log.WarnContext(ctx, "settle message", "how", how, "err", err)
|
||||
}
|
||||
}
|
||||
|
||||
// Subscribe delivers messages matching filterSubject to handle until ctx is
|
||||
// done. It exists so callers do not need the SDK's consume machinery; handle
|
||||
// receives the same boundary type as Fetch.
|
||||
@@ -66,7 +74,7 @@ func (c *Client) Subscribe(ctx context.Context, filterSubject string, handle fun
|
||||
span.RecordError(err)
|
||||
c.log.ErrorContext(ctx, "message handler failed",
|
||||
"subject", msg.Subject(), "error", err)
|
||||
_ = msg.Nak()
|
||||
c.settle(ctx, "nak", msg.Nak)
|
||||
return
|
||||
}
|
||||
if err := msg.Ack(); err != nil {
|
||||
|
||||
@@ -120,18 +120,26 @@ func forward(ctx context.Context, msg eventbus.Message, upstream otlptrace.Clien
|
||||
var req coltracepb.ExportTraceServiceRequest
|
||||
if err := proto.Unmarshal(msg.Data(), &req); err != nil {
|
||||
log.WarnContext(ctx, "span relay dropped undecodable batch", "subject", msg.Subject(), "err", err)
|
||||
_ = msg.Ack()
|
||||
settle(ctx, log, "ack", msg.Ack)
|
||||
return
|
||||
}
|
||||
upload, cancel := context.WithTimeout(context.WithoutCancel(ctx), uploadTimeout)
|
||||
defer cancel()
|
||||
if err := upstream.UploadTraces(upload, req.GetResourceSpans()); err != nil {
|
||||
log.WarnContext(ctx, "span relay upload failed", "err", err)
|
||||
_ = msg.Nak()
|
||||
settle(ctx, log, "nak", msg.Nak)
|
||||
sleep(ctx, retryDelay)
|
||||
return
|
||||
}
|
||||
_ = msg.Ack()
|
||||
settle(ctx, log, "ack", msg.Ack)
|
||||
}
|
||||
|
||||
// settle runs one ack or nak. A failure leaves the batch for JetStream to
|
||||
// redeliver, so it is logged rather than returned.
|
||||
func settle(ctx context.Context, log *slog.Logger, how string, fn func() error) {
|
||||
if err := fn(); err != nil {
|
||||
log.WarnContext(ctx, "span relay settle failed", "how", how, "err", err)
|
||||
}
|
||||
}
|
||||
|
||||
func sleep(ctx context.Context, d time.Duration) {
|
||||
|
||||
Reference in new issue
Block a user