fix: log NATS ack and nak failures instead of discarding them (#1924)
This commit is contained in:
3 files changed
+52
-28
No files matched your search
@@ -91,27 +91,27 @@ func (c *reactionConsumer) handle(ctx context.Context, msg eventbus.Message) {
|
|||||||
envelope, err := workintake.DecodeReviewComment(msg.Data())
|
envelope, err := workintake.DecodeReviewComment(msg.Data())
|
||||||
if err != nil {
|
if err != nil {
|
||||||
c.log.ErrorContext(ctx, "review reaction decode failed", "subject", msg.Subject(), "err", err)
|
c.log.ErrorContext(ctx, "review reaction decode failed", "subject", msg.Subject(), "err", err)
|
||||||
_ = msg.Ack() // it will never decode on redelivery
|
c.settle(ctx, "ack", msg.Ack) // it will never decode on redelivery
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
|
|
||||||
task, err := c.lookup.OpenTaskByPR(ctx, envelope.Owner, envelope.Repo, envelope.PRNumber)
|
task, err := c.lookup.OpenTaskByPR(ctx, envelope.Owner, envelope.Repo, envelope.PRNumber)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
c.log.ErrorContext(ctx, "review reaction lookup failed", "owner", envelope.Owner, "repo", envelope.Repo, "pr", envelope.PRNumber, "err", err)
|
c.log.ErrorContext(ctx, "review reaction lookup failed", "owner", envelope.Owner, "repo", envelope.Repo, "pr", envelope.PRNumber, "err", err)
|
||||||
_ = msg.Nak()
|
c.settle(ctx, "nak", msg.Nak)
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
if task == nil {
|
if task == nil {
|
||||||
// The resolver counts this: a reaction that does not resolve to a
|
// The resolver counts this: a reaction that does not resolve to a
|
||||||
// live task Archie owns is dropped, never work.
|
// live task Archie owns is dropped, never work.
|
||||||
c.drop(msg, "no live task owns this PR")
|
c.drop(ctx, msg, "no live task owns this PR")
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
if c.botUser(task) != "" && envelope.Author == c.botUser(task) {
|
if c.botUser(task) != "" && envelope.Author == c.botUser(task) {
|
||||||
// Decision 5: only Archie's own comments are excluded, so its
|
// Decision 5: only Archie's own comments are excluded, so its
|
||||||
// ReplyToReview replies never retrigger it. Every other author --
|
// ReplyToReview replies never retrigger it. Every other author --
|
||||||
// human or bot -- is actionable.
|
// human or bot -- is actionable.
|
||||||
c.drop(msg, "reaction is archie's own comment")
|
c.drop(ctx, msg, "reaction is archie's own comment")
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -121,7 +121,7 @@ func (c *reactionConsumer) handle(ctx context.Context, msg eventbus.Message) {
|
|||||||
case workintake.ReviewReactionComment:
|
case workintake.ReviewReactionComment:
|
||||||
c.handleComment(ctx, msg, task, envelope)
|
c.handleComment(ctx, msg, task, envelope)
|
||||||
default:
|
default:
|
||||||
c.drop(msg, "unknown reaction kind")
|
c.drop(ctx, msg, "unknown reaction kind")
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -133,7 +133,7 @@ func (c *reactionConsumer) handleReview(ctx context.Context, msg eventbus.Messag
|
|||||||
return
|
return
|
||||||
}
|
}
|
||||||
if !remediableReviewStates[e.State] {
|
if !remediableReviewStates[e.State] {
|
||||||
c.drop(msg, fmt.Sprintf("review state %q does not trigger remediation", e.State))
|
c.drop(ctx, msg, fmt.Sprintf("review state %q does not trigger remediation", e.State))
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -141,7 +141,7 @@ func (c *reactionConsumer) handleReview(ctx context.Context, msg eventbus.Messag
|
|||||||
payload, err := workflow.EncodeReviewUnit(unit)
|
payload, err := workflow.EncodeReviewUnit(unit)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
c.log.ErrorContext(ctx, "review unit encode failed", "task", task.ID, "err", err)
|
c.log.ErrorContext(ctx, "review unit encode failed", "task", task.ID, "err", err)
|
||||||
_ = msg.Nak()
|
c.settle(ctx, "nak", msg.Nak)
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
if err := c.remediations.BeginRemediation(ctx, task.ID, payload); err != nil {
|
if err := c.remediations.BeginRemediation(ctx, task.ID, payload); err != nil {
|
||||||
@@ -150,15 +150,15 @@ func (c *reactionConsumer) handleReview(ctx context.Context, msg eventbus.Messag
|
|||||||
// behind, so the reaction resolves to no work. A review that
|
// behind, so the reaction resolves to no work. A review that
|
||||||
// arrives mid-run is absorbed by the store, not dropped here.
|
// arrives mid-run is absorbed by the store, not dropped here.
|
||||||
c.log.DebugContext(ctx, "task does not accept remediation", "task", task.ID, "review", e.ReviewID)
|
c.log.DebugContext(ctx, "task does not accept remediation", "task", task.ID, "review", e.ReviewID)
|
||||||
_ = msg.Ack()
|
c.settle(ctx, "ack", msg.Ack)
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
c.log.ErrorContext(ctx, "BeginRemediation failed", "task", task.ID, "err", err)
|
c.log.ErrorContext(ctx, "BeginRemediation failed", "task", task.ID, "err", err)
|
||||||
_ = msg.Nak()
|
c.settle(ctx, "nak", msg.Nak)
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
c.log.InfoContext(ctx, "review reaction queued a remediation", "task", task.ID, "pr", fmt.Sprintf("%s/%s#%d", e.Owner, e.Repo, e.PRNumber), "review", e.ReviewID, "state", e.State)
|
c.log.InfoContext(ctx, "review reaction queued a remediation", "task", task.ID, "pr", fmt.Sprintf("%s/%s#%d", e.Owner, e.Repo, e.PRNumber), "review", e.ReviewID, "state", e.State)
|
||||||
_ = msg.Ack()
|
c.settle(ctx, "ack", msg.Ack)
|
||||||
}
|
}
|
||||||
|
|
||||||
func (c *reactionConsumer) handleComment(ctx context.Context, msg eventbus.Message, task *workflowtask.Task, e workintake.ReviewCommentEnvelope) {
|
func (c *reactionConsumer) handleComment(ctx context.Context, msg eventbus.Message, task *workflowtask.Task, e workintake.ReviewCommentEnvelope) {
|
||||||
@@ -171,19 +171,19 @@ func (c *reactionConsumer) handleComment(ctx context.Context, msg eventbus.Messa
|
|||||||
payload, err := workflow.EncodeReviewUnit(unit)
|
payload, err := workflow.EncodeReviewUnit(unit)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
c.log.ErrorContext(ctx, "review unit encode failed", "task", task.ID, "err", err)
|
c.log.ErrorContext(ctx, "review unit encode failed", "task", task.ID, "err", err)
|
||||||
_ = msg.Nak()
|
c.settle(ctx, "nak", msg.Nak)
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
if err := c.remediations.BeginRemediation(ctx, task.ID, payload); err != nil {
|
if err := c.remediations.BeginRemediation(ctx, task.ID, payload); err != nil {
|
||||||
if errors.Is(err, storecontract.ErrStaleTransition) {
|
if errors.Is(err, storecontract.ErrStaleTransition) {
|
||||||
_ = msg.Ack()
|
c.settle(ctx, "ack", msg.Ack)
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
c.log.ErrorContext(ctx, "BeginRemediation failed", "task", task.ID, "err", err)
|
c.log.ErrorContext(ctx, "BeginRemediation failed", "task", task.ID, "err", err)
|
||||||
_ = msg.Nak()
|
c.settle(ctx, "nak", msg.Nak)
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
_ = msg.Ack()
|
c.settle(ctx, "ack", msg.Ack)
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -198,19 +198,19 @@ func (c *reactionConsumer) handleComment(ctx context.Context, msg eventbus.Messa
|
|||||||
})
|
})
|
||||||
if err != nil {
|
if err != nil {
|
||||||
c.log.ErrorContext(ctx, "review unit encode failed", "task", task.ID, "err", err)
|
c.log.ErrorContext(ctx, "review unit encode failed", "task", task.ID, "err", err)
|
||||||
_ = msg.Nak()
|
c.settle(ctx, "nak", msg.Nak)
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
if err := c.remediations.UpdateReviewPayload(ctx, task.ID, payload); err != nil {
|
if err := c.remediations.UpdateReviewPayload(ctx, task.ID, payload); err != nil {
|
||||||
if errors.Is(err, storecontract.ErrStaleTransition) {
|
if errors.Is(err, storecontract.ErrStaleTransition) {
|
||||||
c.drop(msg, "parent review's unit is no longer pending")
|
c.drop(ctx, msg, "parent review's unit is no longer pending")
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
c.log.ErrorContext(ctx, "UpdateReviewPayload failed", "task", task.ID, "err", err)
|
c.log.ErrorContext(ctx, "UpdateReviewPayload failed", "task", task.ID, "err", err)
|
||||||
_ = msg.Nak()
|
c.settle(ctx, "nak", msg.Nak)
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
_ = msg.Ack()
|
c.settle(ctx, "ack", msg.Ack)
|
||||||
}
|
}
|
||||||
|
|
||||||
func (c *reactionConsumer) recordApproval(ctx context.Context, msg eventbus.Message, task *workflowtask.Task, e workintake.ReviewCommentEnvelope) {
|
func (c *reactionConsumer) recordApproval(ctx context.Context, msg eventbus.Message, task *workflowtask.Task, e workintake.ReviewCommentEnvelope) {
|
||||||
@@ -220,14 +220,22 @@ func (c *reactionConsumer) recordApproval(ctx context.Context, msg eventbus.Mess
|
|||||||
Data: map[string]any{"review_id": e.ReviewID, "author": e.Author, "pr_number": e.PRNumber},
|
Data: map[string]any{"review_id": e.ReviewID, "author": e.Author, "pr_number": e.PRNumber},
|
||||||
}); err != nil {
|
}); err != nil {
|
||||||
c.log.ErrorContext(ctx, "record approval failed", "task", task.ID, "err", err)
|
c.log.ErrorContext(ctx, "record approval failed", "task", task.ID, "err", err)
|
||||||
_ = msg.Nak()
|
c.settle(ctx, "nak", msg.Nak)
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
_ = msg.Ack()
|
c.settle(ctx, "ack", msg.Ack)
|
||||||
}
|
}
|
||||||
|
|
||||||
func (c *reactionConsumer) drop(msg eventbus.Message, reason string) {
|
func (c *reactionConsumer) drop(ctx context.Context, msg eventbus.Message, reason string) {
|
||||||
c.dropped++
|
c.dropped++
|
||||||
c.log.Info("review reaction dropped", "subject", msg.Subject(), "reason", reason)
|
c.log.InfoContext(ctx, "review reaction dropped", "subject", msg.Subject(), "reason", reason)
|
||||||
_ = msg.Ack()
|
c.settle(ctx, "ack", msg.Ack)
|
||||||
|
}
|
||||||
|
|
||||||
|
// settle runs one ack or nak. A failure leaves the message for JetStream to
|
||||||
|
// redeliver, so it is logged here rather than returned to the drain loop.
|
||||||
|
func (c *reactionConsumer) settle(ctx context.Context, how string, fn func() error) {
|
||||||
|
if err := fn(); err != nil {
|
||||||
|
c.log.WarnContext(ctx, "settle reaction message", "how", how, "err", err)
|
||||||
|
}
|
||||||
}
|
}
|
||||||
@@ -29,7 +29,7 @@ func (c *Client) Fetch(ctx context.Context) (eventbus.Message, error) {
|
|||||||
// Honour cancellation: return the message unacked so JetStream
|
// Honour cancellation: return the message unacked so JetStream
|
||||||
// redelivers it rather than losing it to a shutting-down daemon.
|
// redelivers it rather than losing it to a shutting-down daemon.
|
||||||
if err := ctx.Err(); err != nil {
|
if err := ctx.Err(); err != nil {
|
||||||
_ = msg.Nak()
|
c.settle(ctx, "nak", msg.Nak)
|
||||||
return nil, fmt.Errorf("fetch from %s: %w", c.cfg.ConsumerName, err)
|
return nil, fmt.Errorf("fetch from %s: %w", c.cfg.ConsumerName, err)
|
||||||
}
|
}
|
||||||
return message{msg: msg}, nil
|
return message{msg: msg}, nil
|
||||||
@@ -41,6 +41,14 @@ func (c *Client) Fetch(ctx context.Context) (eventbus.Message, error) {
|
|||||||
return nil, eventbus.ErrNoMessage
|
return nil, eventbus.ErrNoMessage
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// settle runs one ack or nak. A failure leaves the message for JetStream to
|
||||||
|
// redeliver, so it is logged rather than returned.
|
||||||
|
func (c *Client) settle(ctx context.Context, how string, fn func() error) {
|
||||||
|
if err := fn(); err != nil {
|
||||||
|
c.log.WarnContext(ctx, "settle message", "how", how, "err", err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
// Subscribe delivers messages matching filterSubject to handle until ctx is
|
// Subscribe delivers messages matching filterSubject to handle until ctx is
|
||||||
// done. It exists so callers do not need the SDK's consume machinery; handle
|
// done. It exists so callers do not need the SDK's consume machinery; handle
|
||||||
// receives the same boundary type as Fetch.
|
// receives the same boundary type as Fetch.
|
||||||
@@ -66,7 +74,7 @@ func (c *Client) Subscribe(ctx context.Context, filterSubject string, handle fun
|
|||||||
span.RecordError(err)
|
span.RecordError(err)
|
||||||
c.log.ErrorContext(ctx, "message handler failed",
|
c.log.ErrorContext(ctx, "message handler failed",
|
||||||
"subject", msg.Subject(), "error", err)
|
"subject", msg.Subject(), "error", err)
|
||||||
_ = msg.Nak()
|
c.settle(ctx, "nak", msg.Nak)
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
if err := msg.Ack(); err != nil {
|
if err := msg.Ack(); err != nil {
|
||||||
|
|||||||
@@ -120,18 +120,26 @@ func forward(ctx context.Context, msg eventbus.Message, upstream otlptrace.Clien
|
|||||||
var req coltracepb.ExportTraceServiceRequest
|
var req coltracepb.ExportTraceServiceRequest
|
||||||
if err := proto.Unmarshal(msg.Data(), &req); err != nil {
|
if err := proto.Unmarshal(msg.Data(), &req); err != nil {
|
||||||
log.WarnContext(ctx, "span relay dropped undecodable batch", "subject", msg.Subject(), "err", err)
|
log.WarnContext(ctx, "span relay dropped undecodable batch", "subject", msg.Subject(), "err", err)
|
||||||
_ = msg.Ack()
|
settle(ctx, log, "ack", msg.Ack)
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
upload, cancel := context.WithTimeout(context.WithoutCancel(ctx), uploadTimeout)
|
upload, cancel := context.WithTimeout(context.WithoutCancel(ctx), uploadTimeout)
|
||||||
defer cancel()
|
defer cancel()
|
||||||
if err := upstream.UploadTraces(upload, req.GetResourceSpans()); err != nil {
|
if err := upstream.UploadTraces(upload, req.GetResourceSpans()); err != nil {
|
||||||
log.WarnContext(ctx, "span relay upload failed", "err", err)
|
log.WarnContext(ctx, "span relay upload failed", "err", err)
|
||||||
_ = msg.Nak()
|
settle(ctx, log, "nak", msg.Nak)
|
||||||
sleep(ctx, retryDelay)
|
sleep(ctx, retryDelay)
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
_ = msg.Ack()
|
settle(ctx, log, "ack", msg.Ack)
|
||||||
|
}
|
||||||
|
|
||||||
|
// settle runs one ack or nak. A failure leaves the batch for JetStream to
|
||||||
|
// redeliver, so it is logged rather than returned.
|
||||||
|
func settle(ctx context.Context, log *slog.Logger, how string, fn func() error) {
|
||||||
|
if err := fn(); err != nil {
|
||||||
|
log.WarnContext(ctx, "span relay settle failed", "how", how, "err", err)
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
func sleep(ctx context.Context, d time.Duration) {
|
func sleep(ctx context.Context, d time.Duration) {
|
||||||
|
|||||||
Reference in new issue
Block a user