fix(nats): bound the ARCHIE_REACTIONS stream by age
Review fix for PR #875. ARCHIE_REACTIONS is provisioned under LimitsPolicy, which retains every message until a limit is reached. With no limit configured, acknowledged reactions accumulated without bound and continuous traffic could grow the stream until it exhausted disk. nats.Config gains MaxAge *time.Duration, using the same pointer form as Retention so "unset" (nil) is distinguishable from an explicit zero. It is applied to the stream config only when set, so every existing caller keeps its current behaviour. ARCHIE_REACTIONS sets MaxAge = 24h. A reaction is a producer-only wake event whose authoritative state is re-read at pass time, so a dropped reaction delays work until the next poll (default 60s) rather than losing it; 24h tolerates a consumer down for a full maintenance window and matches the package's existing DefaultInactiveTTL. ARCHIE_TASKS is deliberately left unbounded. It uses WorkQueuePolicy, which discards a message on acknowledgement, so its size is bounded by outstanding work; a limit there could discard a task that has not been delivered yet, which is a worse failure than the disk risk. The table asserts that non-change.
This commit is contained in:
4 files changed
+100
-32
No files matched your search
@@ -453,6 +453,15 @@ func (b *boot) setupObservability(ctx context.Context) {
|
||||
go persistEvents(ctx, sink, b.stateStore, log)
|
||||
}
|
||||
|
||||
// reactionStreamMaxAge bounds how long a reaction survives in the fan-out
|
||||
// stream before JetStream discards it. Reactions are producer-only wake
|
||||
// events (docs/prds/event-sources-and-reactions.md): a dropped reaction only
|
||||
// delays work until the next poll, never loses it, because the authoritative
|
||||
// state is re-read at pass time. One day therefore tolerates a consumer that
|
||||
// is down or lagging for a full maintenance window without letting
|
||||
// acknowledged reactions accumulate without bound.
|
||||
const reactionStreamMaxAge = 24 * time.Hour
|
||||
|
||||
// connectNATS opens the NATS client. External mode dials cfg.NATS.URL;
|
||||
// embedded mode starts an in-process nats-server and dials it, so single-
|
||||
// process deployments get task distribution and reaction delivery without a
|
||||
@@ -512,6 +521,10 @@ func (b *boot) connectNATS(ctx context.Context) error { //nolint:nestif // embed
|
||||
// must see each one, so they get their own stream under LimitsPolicy
|
||||
// rather than the work-queue policy ARCHIE_TASKS uses (which lets a
|
||||
// second consumer on an overlapping filter silently receive nothing).
|
||||
// LimitsPolicy retains every message until a limit is reached, so without
|
||||
// a finite limit acknowledged reactions would accumulate forever; the
|
||||
// MaxAge cap bounds that by time.
|
||||
reactionMaxAge := reactionStreamMaxAge
|
||||
reactionClient, err := nats.Connect(ctx, nats.Config{
|
||||
URL: url,
|
||||
Token: natsToken,
|
||||
@@ -519,6 +532,7 @@ func (b *boot) connectNATS(ctx context.Context) error { //nolint:nestif // embed
|
||||
Subjects: []string{workintake.SubjectReactionWildcard},
|
||||
FilterSubject: workintake.SubjectReactionWildcard,
|
||||
Retention: nats.FanOutRetention(),
|
||||
MaxAge: &reactionMaxAge,
|
||||
}, log)
|
||||
if err != nil {
|
||||
log.Error("nats reaction stream connect failed", "err", err)
|
||||
|
||||
@@ -13,15 +13,19 @@ import (
|
||||
"github.com/samcharles93/archie-core/internal/config"
|
||||
)
|
||||
|
||||
// TestConnectNATSCreatesReactionStream pins the ARCHIE_REACTIONS stream the
|
||||
// daemon must provision alongside ARCHIE_TASKS. A publish to archie.reaction.*
|
||||
// must land in a fan-out (LimitsPolicy) stream: two consumers on overlapping
|
||||
// filter subjects both receive it. Under the WorkQueuePolicy ARCHIE_TASKS
|
||||
// keeps, the second consumer would silently receive nothing (the trap
|
||||
// documented in CLAUDE.md).
|
||||
// TestConnectNATSCreatesReactionStream pins the two JetStream streams the
|
||||
// daemon must provision alongside each other: ARCHIE_TASKS (work-queue task
|
||||
// distribution) and ARCHIE_REACTIONS (fan-out reaction delivery). Each case
|
||||
// asserts the ACTUAL stream configuration the embedded broker reports, not
|
||||
// just the Config struct that composed it.
|
||||
//
|
||||
// Before this stream existed the publish below failed with "no matching
|
||||
// stream" because connectNATS bound only archie.task.>.
|
||||
// ARCHIE_REACTIONS must be a LimitsPolicy stream so two consumers on
|
||||
// overlapping filter subjects both receive a reaction (the trap documented in
|
||||
// CLAUDE.md), and it must carry a finite retention limit so acknowledged
|
||||
// reactions do not accumulate without bound. ARCHIE_TASKS must remain a
|
||||
// WorkQueuePolicy stream with no age/count/byte limit: acked messages are
|
||||
// discarded, so its only bound is outstanding work, and imposing a limit
|
||||
// could silently drop undelivered tasks.
|
||||
func TestConnectNATSCreatesReactionStream(t *testing.T) {
|
||||
b := &boot{
|
||||
cfg: config.Config{
|
||||
@@ -46,8 +50,59 @@ func TestConnectNATSCreatesReactionStream(t *testing.T) {
|
||||
t.Fatalf("jetstream.New = %v", err)
|
||||
}
|
||||
|
||||
// Criterion 1: a publish to archie.reaction.* lands in ARCHIE_REACTIONS.
|
||||
// Before the stream existed this returned "no matching stream".
|
||||
streamCases := []struct {
|
||||
name string
|
||||
stream string
|
||||
retention jetstream.RetentionPolicy
|
||||
subjects []string
|
||||
wantMaxAge time.Duration
|
||||
}{
|
||||
{
|
||||
name: "reactions fan-out with finite retention",
|
||||
stream: "ARCHIE_REACTIONS",
|
||||
retention: jetstream.LimitsPolicy,
|
||||
subjects: []string{"archie.reaction.>"},
|
||||
wantMaxAge: reactionStreamMaxAge,
|
||||
},
|
||||
{
|
||||
name: "tasks work-queue unchanged and unbounded",
|
||||
stream: "ARCHIE_TASKS",
|
||||
retention: jetstream.WorkQueuePolicy,
|
||||
subjects: []string{"archie.task.>"},
|
||||
// wantMaxAge zero asserts ARCHIE_TASKS stays unbounded by age.
|
||||
},
|
||||
}
|
||||
for _, tc := range streamCases {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
stream, err := js.Stream(t.Context(), tc.stream)
|
||||
if err != nil {
|
||||
t.Fatalf("js.Stream(%s) = %v", tc.stream, err)
|
||||
}
|
||||
info, err := stream.Info(t.Context())
|
||||
if err != nil {
|
||||
t.Fatalf("stream.Info(%s) = %v", tc.stream, err)
|
||||
}
|
||||
if info.Config.Retention != tc.retention {
|
||||
t.Fatalf("%s retention = %v, want %v", tc.stream, info.Config.Retention, tc.retention)
|
||||
}
|
||||
for _, subject := range tc.subjects {
|
||||
if !slices.Contains(info.Config.Subjects, subject) {
|
||||
t.Errorf("%s subjects = %v, want to contain %s", tc.stream, info.Config.Subjects, subject)
|
||||
}
|
||||
}
|
||||
if info.Config.MaxAge != tc.wantMaxAge {
|
||||
t.Fatalf("%s MaxAge = %v, want %v", tc.stream, info.Config.MaxAge, tc.wantMaxAge)
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
// Fan-out proof: two consumers on the same overlapping filter subject
|
||||
// both receive the one published message. A work-queue stream would
|
||||
// deliver it to only one of them.
|
||||
reactionStream, err := js.Stream(t.Context(), "ARCHIE_REACTIONS")
|
||||
if err != nil {
|
||||
t.Fatalf("js.Stream(ARCHIE_REACTIONS) = %v", err)
|
||||
}
|
||||
if _, err := js.PublishMsg(t.Context(), &natsio.Msg{
|
||||
Subject: "archie.reaction.test",
|
||||
Data: []byte("x"),
|
||||
@@ -55,27 +110,9 @@ func TestConnectNATSCreatesReactionStream(t *testing.T) {
|
||||
t.Fatalf("publish to archie.reaction.* = %v, want it to land in ARCHIE_REACTIONS", err)
|
||||
}
|
||||
|
||||
stream, err := js.Stream(t.Context(), "ARCHIE_REACTIONS")
|
||||
if err != nil {
|
||||
t.Fatalf("js.Stream(ARCHIE_REACTIONS) = %v", err)
|
||||
}
|
||||
info, err := stream.Info(t.Context())
|
||||
if err != nil {
|
||||
t.Fatalf("stream.Info = %v", err)
|
||||
}
|
||||
if info.Config.Retention != jetstream.LimitsPolicy {
|
||||
t.Fatalf("ARCHIE_REACTIONS retention = %v, want %v (fan-out)", info.Config.Retention, jetstream.LimitsPolicy)
|
||||
}
|
||||
if !slices.Contains(info.Config.Subjects, "archie.reaction.>") {
|
||||
t.Fatalf("ARCHIE_REACTIONS subjects = %v, want to contain archie.reaction.>", info.Config.Subjects)
|
||||
}
|
||||
|
||||
// Criterion 2: two consumers on the same overlapping filter subject both
|
||||
// receive the one published message. A work-queue stream would deliver it
|
||||
// to only one of them.
|
||||
got := make(chan string, 2)
|
||||
fetch := func() {
|
||||
consumer, err := stream.CreateOrUpdateConsumer(t.Context(), jetstream.ConsumerConfig{
|
||||
consumer, err := reactionStream.CreateOrUpdateConsumer(t.Context(), jetstream.ConsumerConfig{
|
||||
FilterSubject: "archie.reaction.>",
|
||||
AckPolicy: jetstream.AckExplicitPolicy,
|
||||
})
|
||||
|
||||
@@ -59,13 +59,20 @@ func Connect(ctx context.Context, cfg Config, log *slog.Logger) (*Client, error)
|
||||
return nil, fmt.Errorf("nats jetstream init: %w", err)
|
||||
}
|
||||
|
||||
stream, err := js.CreateOrUpdateStream(ctx, jetstream.StreamConfig{
|
||||
streamConfig := jetstream.StreamConfig{
|
||||
Name: cfg.StreamName,
|
||||
Subjects: cfg.Subjects,
|
||||
Storage: jetstream.FileStorage,
|
||||
Retention: *cfg.Retention,
|
||||
Duplicates: cfg.DedupWindow,
|
||||
})
|
||||
}
|
||||
// Limits are applied only when the composition explicitly sets them;
|
||||
// leaving MaxAge nil keeps the stream unbounded exactly as before.
|
||||
if cfg.MaxAge != nil {
|
||||
streamConfig.MaxAge = *cfg.MaxAge
|
||||
}
|
||||
|
||||
stream, err := js.CreateOrUpdateStream(ctx, streamConfig)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("nats create stream %q: %w", cfg.StreamName, err)
|
||||
}
|
||||
|
||||
@@ -63,6 +63,16 @@ type Config struct {
|
||||
// (LimitsPolicy) is a valid explicit choice.
|
||||
Retention *jetstream.RetentionPolicy
|
||||
|
||||
// MaxAge is the maximum age of messages the stream retains. Nil leaves
|
||||
// the stream unbounded by age, which is correct for ARCHIE_TASKS: under
|
||||
// WorkQueuePolicy acked messages are discarded, so its only bound is
|
||||
// outstanding work. The pointer form distinguishes "unset" (nil,
|
||||
// unbounded, today's behaviour) from an explicit zero (also unbounded,
|
||||
// but expressed on purpose), exactly like Retention. The fan-out
|
||||
// reaction stream sets a finite MaxAge so acknowledged reactions cannot
|
||||
// accumulate forever.
|
||||
MaxAge *time.Duration
|
||||
|
||||
// DedupWindow is how long JetStream remembers a Nats-Msg-Id, suppressing
|
||||
// republished duplicates of the same issue within the window.
|
||||
DedupWindow time.Duration
|
||||
@@ -92,7 +102,7 @@ func (c Config) Validate() error {
|
||||
if c.FilterSubject == "" {
|
||||
return fmt.Errorf("%w: FilterSubject is required", ErrInvalidConfig)
|
||||
}
|
||||
if c.PollTimeout < 0 || c.AckWait < 0 || c.DedupWindow < 0 {
|
||||
if c.PollTimeout < 0 || c.AckWait < 0 || c.DedupWindow < 0 || (c.MaxAge != nil && *c.MaxAge < 0) {
|
||||
return fmt.Errorf("%w: durations must not be negative", ErrInvalidConfig)
|
||||
}
|
||||
if c.MaxDeliver < 0 {
|
||||
|
||||
Reference in new issue
Block a user