From 0bc65b680dde09c060fba1d7602155e0bf6f0c1b Mon Sep 17 00:00:00 2001 From: Sam Catlow Date: Sat, 5 Sep 2026 03:11:43 +1000 Subject: [PATCH] feat(servicediscovery): scaffold ServiceRegistry contract and NATS KV implementation Add a broker-neutral service registry contract (Endpoint, EventKind, Event, ServiceRegistry, ErrNotInstalled) and a NATS JetStream KV implementation, plus the registration side a service uses to announce itself. The contract's governing semantic -- NotInstalled is not Down -- is modeled by two buckets: a durable installed marker (never expires) distinct from expiring heartbeat keys. Instances heartbeat their key on an interval; a stopped process leaves via TTL expiry. Register writes the marker once then heartbeats; Resolve returns ErrNotInstalled only when the marker is absent, and an empty roster for installed-but-down. Watch dedupes heartbeat refreshes so it emits membership changes, not liveness pings. Unwired: no daemon/bootstrap path consumes it yet. Deliberately uses a dot key separator instead of the research sketch's slash so per-service watch wildcards match. --- .../servicediscovery/nats/config.go | 84 +++++ .../servicediscovery/nats/config_test.go | 49 +++ .../servicediscovery/nats/doc.go | 50 +++ .../servicediscovery/nats/errors.go | 11 + .../servicediscovery/nats/keys.go | 31 ++ .../servicediscovery/nats/keys_test.go | 49 +++ .../servicediscovery/nats/registrar.go | 151 +++++++++ .../servicediscovery/nats/registry.go | 314 ++++++++++++++++++ .../servicediscovery/nats/registry_test.go | 299 +++++++++++++++++ .../servicediscovery/nats/value.go | 29 ++ internal/servicediscovery/doc.go | 36 ++ internal/servicediscovery/servicediscovery.go | 86 +++++ .../servicediscovery/servicediscovery_test.go | 39 +++ 13 files changed, 1228 insertions(+) create mode 100644 internal/infrastructure/servicediscovery/nats/config.go create mode 100644 internal/infrastructure/servicediscovery/nats/config_test.go create mode 100644 internal/infrastructure/servicediscovery/nats/doc.go create mode 100644 internal/infrastructure/servicediscovery/nats/errors.go create mode 100644 internal/infrastructure/servicediscovery/nats/keys.go create mode 100644 internal/infrastructure/servicediscovery/nats/keys_test.go create mode 100644 internal/infrastructure/servicediscovery/nats/registrar.go create mode 100644 internal/infrastructure/servicediscovery/nats/registry.go create mode 100644 internal/infrastructure/servicediscovery/nats/registry_test.go create mode 100644 internal/infrastructure/servicediscovery/nats/value.go create mode 100644 internal/servicediscovery/doc.go create mode 100644 internal/servicediscovery/servicediscovery.go create mode 100644 internal/servicediscovery/servicediscovery_test.go diff --git a/internal/infrastructure/servicediscovery/nats/config.go b/internal/infrastructure/servicediscovery/nats/config.go new file mode 100644 index 000000000..dd3f4bee4 --- /dev/null +++ b/internal/infrastructure/servicediscovery/nats/config.go @@ -0,0 +1,84 @@ +package nats + +import ( + "fmt" + "time" +) + +// Defaults for Config. Previously these would have been unexported package +// constants, which made bucket naming and timeouts untunable per deployment. +const ( + // DefaultBucket is the heartbeat bucket: one live endpoint per instance. + DefaultBucket = "ARCHIE_SERVICE_REGISTRY" + + // DefaultInstalledBucket is the durable "installed" bucket: one marker per + // service that has ever registered, never expiring. + DefaultInstalledBucket = "ARCHIE_SERVICE_INSTALLED" + + // DefaultTTL is how long a heartbeat entry survives without being + // refreshed. It must comfortably exceed DefaultHeartbeat so a transient + // pause in a live service's refresh does not drop it. + DefaultTTL = 15 * time.Second + + // DefaultHeartbeat is the interval at which a registrar refreshes its + // endpoint entry. + DefaultHeartbeat = 5 * time.Second +) + +// Config describes how to reach NATS and how the registry buckets behave. +// The zero value is not usable; call [Config.withDefaults] via [Connect], which +// fills every unset field. +type Config struct { + // URL is the NATS server address. Required. + URL string + + // Token authenticates the connection. Empty means no token auth. + Token string + + // Bucket is the heartbeat bucket name. + Bucket string + + // InstalledBucket is the durable "installed" bucket name. + InstalledBucket string + + // TTL is how long a heartbeat entry survives without being refreshed. It + // is the difference between a service that is installed-but-down (its + // entries expired) and one that is live (its entries are fresh). + TTL time.Duration + + // Heartbeat is the interval at which a registrar refreshes its entry. It + // must be less than TTL, or a live service would expire between refreshes. + Heartbeat time.Duration +} + +// Validate reports whether the config can produce a usable client. +func (c Config) Validate() error { + if c.URL == "" { + return fmt.Errorf("%w: URL is required", ErrInvalidConfig) + } + if c.TTL < 0 || c.Heartbeat < 0 { + return fmt.Errorf("%w: durations must not be negative", ErrInvalidConfig) + } + if c.Heartbeat > 0 && c.TTL > 0 && c.Heartbeat >= c.TTL { + return fmt.Errorf("%w: Heartbeat must be less than TTL", ErrInvalidConfig) + } + return nil +} + +// withDefaults returns a copy with every unset field populated, so the rest of +// the package never has to test for zero values. +func (c Config) withDefaults() Config { + if c.Bucket == "" { + c.Bucket = DefaultBucket + } + if c.InstalledBucket == "" { + c.InstalledBucket = DefaultInstalledBucket + } + if c.TTL == 0 { + c.TTL = DefaultTTL + } + if c.Heartbeat == 0 { + c.Heartbeat = DefaultHeartbeat + } + return c +} diff --git a/internal/infrastructure/servicediscovery/nats/config_test.go b/internal/infrastructure/servicediscovery/nats/config_test.go new file mode 100644 index 000000000..7dd76723f --- /dev/null +++ b/internal/infrastructure/servicediscovery/nats/config_test.go @@ -0,0 +1,49 @@ +package nats + +import ( + "errors" + "testing" + "time" +) + +func TestWithDefaults(t *testing.T) { + cfg := (Config{}).withDefaults() + if cfg.Bucket != DefaultBucket { + t.Errorf("withDefaults().Bucket = %q, want %q", cfg.Bucket, DefaultBucket) + } + if cfg.InstalledBucket != DefaultInstalledBucket { + t.Errorf("withDefaults().InstalledBucket = %q, want %q", cfg.InstalledBucket, DefaultInstalledBucket) + } + if cfg.TTL != DefaultTTL { + t.Errorf("withDefaults().TTL = %v, want %v", cfg.TTL, DefaultTTL) + } + if cfg.Heartbeat != DefaultHeartbeat { + t.Errorf("withDefaults().Heartbeat = %v, want %v", cfg.Heartbeat, DefaultHeartbeat) + } +} + +func TestValidate(t *testing.T) { + tests := []struct { + name string + cfg Config + wantErr bool + }{ + {name: "valid", cfg: Config{URL: "nats://x", TTL: 15 * time.Second, Heartbeat: 5 * time.Second}, wantErr: false}, + {name: "missing url", cfg: Config{TTL: time.Second, Heartbeat: 200 * time.Millisecond}, wantErr: true}, + {name: "heartbeat equals ttl", cfg: Config{URL: "nats://x", TTL: time.Second, Heartbeat: time.Second}, wantErr: true}, + {name: "heartbeat exceeds ttl", cfg: Config{URL: "nats://x", TTL: time.Second, Heartbeat: 2 * time.Second}, wantErr: true}, + {name: "negative durations", cfg: Config{URL: "nats://x", TTL: -time.Second, Heartbeat: time.Second}, wantErr: true}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + err := tt.cfg.Validate() + if tt.wantErr { + if !errors.Is(err, ErrInvalidConfig) { + t.Fatalf("Validate() = %v, want ErrInvalidConfig", err) + } + } else if err != nil { + t.Fatalf("Validate() = %v, want nil", err) + } + }) + } +} diff --git a/internal/infrastructure/servicediscovery/nats/doc.go b/internal/infrastructure/servicediscovery/nats/doc.go new file mode 100644 index 000000000..8b1f3ea39 --- /dev/null +++ b/internal/infrastructure/servicediscovery/nats/doc.go @@ -0,0 +1,50 @@ +// Package nats is the NATS JetStream KV transport for service discovery. +// +// It implements the broker-neutral +// [servicediscovery.ServiceRegistry] contract against a JetStream KeyValue +// store, and additionally provides the registration side a service uses to +// announce itself. It is the fallback discovery mechanism for single-host, +// no-Kubernetes installs (Path B of +// docs/inspiration/service-discovery-research-2026-09-05.md); the +// Kubernetes-native primary is a sibling implementation of the same contract. +// +// # Two buckets: presence versus liveness +// +// The NotInstalled/Down distinction (the governing semantic of +// [servicediscovery]) requires modelling a service's presence separately from +// any instance's liveness. NATS KV cannot express that with one bucket, +// because a bucket-wide TTL (needed so stale heartbeats expire) would also +// expire the "installed" record, collapsing Down into NotInstalled. So the +// implementation uses two buckets: +// +// - Registry bucket (heartbeats): one key per live instance, refreshed on a +// heartbeat interval and expired by a bucket-wide TTL. Presence of a key +// here means the instance is live. +// - Installed bucket (marker): one key per service, written once at first +// registration with no TTL, never expires. Presence of this marker means +// the service was installed; absence means never installed => NotInstalled. +// +// A service that registered once but whose instances have all gone is still +// installed (its marker persists) and simply has zero live endpoints, so +// [Client.Resolve] returns an empty slice with no error. Only a service that +// never wrote a marker resolves to [servicediscovery.ErrNotInstalled]. +// +// # Key scheme +// +// Instance heartbeat keys are "." (dot-separated), and +// the installed marker key is "". The dot separator is a deliberate +// deviation from the research sketch's "/": NATS KV +// watch filters match on dot-separated subject tokens, so a per-service watch +// of "curator.*" must see "curator.inst-1" as two tokens. A slash would make +// the whole key a single opaque token that no per-service wildcard can match. +// Consequently both service names and instance IDs must avoid the dot +// character; standard slugs and UUIDs satisfy that. +// +// Layout: +// +// config.go connection and bucket tuning, defaults and validation +// errors.go sentinel errors callers match on +// keys.go key scheme, pure and dependency-free +// registry.go the ServiceRegistry client (Resolve/Watch) +// registrar.go the registration side (announce + heartbeat) +package nats diff --git a/internal/infrastructure/servicediscovery/nats/errors.go b/internal/infrastructure/servicediscovery/nats/errors.go new file mode 100644 index 000000000..4f1d0d3fa --- /dev/null +++ b/internal/infrastructure/servicediscovery/nats/errors.go @@ -0,0 +1,11 @@ +package nats + +import "errors" + +// ErrInvalidConfig reports a Config that cannot produce a usable client. +// +// The delivery-outcome sentinel a caller matches on -- NotInstalled -- belongs +// to the broker-neutral contract in internal/servicediscovery, so a caller can +// handle it without knowing which registry is in use. This package returns +// that, not a copy of it. +var ErrInvalidConfig = errors.New("nats: invalid configuration") diff --git a/internal/infrastructure/servicediscovery/nats/keys.go b/internal/infrastructure/servicediscovery/nats/keys.go new file mode 100644 index 000000000..d698f8388 --- /dev/null +++ b/internal/infrastructure/servicediscovery/nats/keys.go @@ -0,0 +1,31 @@ +package nats + +import "strings" + +// keySeparator joins a service and an instance ID in a registry key. It is a +// dot rather than the research sketch's slash so that a per-service watch +// filter ("curator.*") matches "curator.inst-1" as two NATS subject tokens; +// a slash would make the whole key one opaque token no wildcard could match. +// See the package doc comment for the full reasoning. +const keySeparator = "." + +// endpointKey is the heartbeat bucket key for one live instance. +func endpointKey(service, id string) string { + return service + keySeparator + id +} + +// endpointKeyPrefix is the watch/list filter matching every instance of a +// service. It is a single-token wildcard over the instance ID. +func endpointKeyPrefix(service string) string { + return service + keySeparator + "*" +} + +// idFromEndpointKey returns the instance ID carried by a registry key, or "" if +// the key does not belong to the given service (or carries an empty ID). +func idFromEndpointKey(service, key string) string { + prefix := service + keySeparator + if !strings.HasPrefix(key, prefix) { + return "" + } + return strings.TrimPrefix(key, prefix) +} diff --git a/internal/infrastructure/servicediscovery/nats/keys_test.go b/internal/infrastructure/servicediscovery/nats/keys_test.go new file mode 100644 index 000000000..7b0a2d019 --- /dev/null +++ b/internal/infrastructure/servicediscovery/nats/keys_test.go @@ -0,0 +1,49 @@ +package nats + +import "testing" + +func TestEndpointKey(t *testing.T) { + tests := []struct { + name string + service string + id string + want string + }{ + {name: "simple", service: "curator", id: "inst-1", want: "curator.inst-1"}, + {name: "single-char id", service: "scheduler", id: "a", want: "scheduler.a"}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + if got := endpointKey(tt.service, tt.id); got != tt.want { + t.Errorf("endpointKey(%q, %q) = %q, want %q", tt.service, tt.id, got, tt.want) + } + }) + } +} + +func TestEndpointKeyPrefix(t *testing.T) { + if got := endpointKeyPrefix("curator"); got != "curator.*" { + t.Errorf("endpointKeyPrefix(curator) = %q, want %q", got, "curator.*") + } +} + +func TestIDFromEndpointKey(t *testing.T) { + tests := []struct { + name string + service string + key string + want string + }{ + {name: "own key returns id", service: "curator", key: "curator.inst-1", want: "inst-1"}, + {name: "other service", service: "curator", key: "scheduler.inst", want: ""}, + {name: "empty id", service: "curator", key: "curator.", want: ""}, + {name: "key equals prefix", service: "curator", key: "curator", want: ""}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + if got := idFromEndpointKey(tt.service, tt.key); got != tt.want { + t.Errorf("idFromEndpointKey(%q, %q) = %q, want %q", tt.service, tt.key, got, tt.want) + } + }) + } +} diff --git a/internal/infrastructure/servicediscovery/nats/registrar.go b/internal/infrastructure/servicediscovery/nats/registrar.go new file mode 100644 index 000000000..51a9091f3 --- /dev/null +++ b/internal/infrastructure/servicediscovery/nats/registrar.go @@ -0,0 +1,151 @@ +package nats + +import ( + "context" + "errors" + "fmt" + "log/slog" + "strings" + "sync" + "time" + + "github.com/nats-io/nats.go/jetstream" + + "github.com/samcharles93/archie-core/internal/servicediscovery" +) + +// Register announces service's instance ep with the registry and keeps its +// heartbeat fresh until the returned Registration is unregistered or ctx is +// cancelled. +// +// It writes the service's durable installed marker once (idempotent: an +// existing marker is left alone) -- that marker is what makes the service +// "installed" rather than NotInstalled -- then writes a live heartbeat entry +// for ep and refreshes it on an interval. If the service's process stops +// without calling Unregister (its ctx is cancelled, or it crashes), the entry +// expires via the bucket TTL and watchers observe a Leave; only an explicit +// uninstall of the whole service removes the installed marker. +// +// ep.ID must be non-empty and must not contain the registry key separator; the +// service name must likewise avoid the separator (see the package doc). +func (c *Client) Register(ctx context.Context, service string, ep servicediscovery.Endpoint) (*Registration, error) { + if ep.ID == "" { + return nil, fmt.Errorf("register %s: %w: endpoint ID is required", service, ErrInvalidConfig) + } + if strings.Contains(ep.ID, keySeparator) || strings.Contains(service, keySeparator) { + return nil, fmt.Errorf("register %s: %w: service and instance ID must not contain %q", service, ErrInvalidConfig, keySeparator) + } + + if err := c.ensureInstalled(ctx, service); err != nil { + return nil, err + } + + key := endpointKey(service, ep.ID) + value, err := encodeEndpoint(ep) + if err != nil { + return nil, err + } + if _, err := c.kv.Put(ctx, key, value); err != nil { + return nil, fmt.Errorf("register %s/%s: %w", service, ep.ID, err) + } + + hbCtx, cancel := context.WithCancel(ctx) + reg := &Registration{ + service: service, + key: key, + value: value, + kv: c.kv, + heartbeat: c.cfg.Heartbeat, + log: c.log, + cancel: cancel, + done: make(chan struct{}), + } + go reg.heartbeatLoop(hbCtx) + return reg, nil +} + +// ensureInstalled writes the service's installed marker if it is absent. The +// marker has no TTL, so it persists for the life of the install; a service that +// registered once but has no live instance is still installed (and resolves to +// an empty, non-error roster), not NotInstalled. +func (c *Client) ensureInstalled(ctx context.Context, service string) error { + if _, err := c.installed.Create(ctx, service, installedMarkerValue); err != nil { + if errors.Is(err, jetstream.ErrKeyExists) { + return nil + } + return fmt.Errorf("mark %s installed: %w", service, err) + } + return nil +} + +// Registration is one live announcement of a service instance. It owns the +// background heartbeat that keeps the instance's entry fresh; stopping it is +// the caller's responsibility via Unregister (or by cancelling the ctx passed +// to Register, which stops the heartbeat and lets the entry expire). +type Registration struct { + service string + key string + value []byte + kv jetstream.KeyValue + heartbeat time.Duration + log *slog.Logger + cancel context.CancelFunc + done chan struct{} + once sync.Once +} + +// heartbeatLoop refreshes the instance's live entry on an interval until ctx is +// cancelled. It never stops on a single refresh failure -- a transient broker +// blip must not unregister a healthy instance -- only on cancellation. +func (rg *Registration) heartbeatLoop(ctx context.Context) { + defer close(rg.done) + ticker := time.NewTicker(rg.heartbeat) + defer ticker.Stop() + for { + select { + case <-ctx.Done(): + return + case <-ticker.C: + if err := rg.refresh(ctx); err != nil { + if ctx.Err() != nil { + return + } + rg.log.Warn("heartbeat refresh failed", "service", rg.service, "key", rg.key, "error", err) + } + } + } +} + +// refresh re-publishes the instance's entry, resetting its bucket-TTL window so +// it stays live. +func (rg *Registration) refresh(ctx context.Context) error { + if _, err := rg.kv.Put(ctx, rg.key, rg.value); err != nil { + return fmt.Errorf("refresh %s: %w", rg.key, err) + } + return nil +} + +// Unregister removes the instance's live entry and stops its heartbeat. It is +// idempotent: a second call is a no-op that returns nil. The service's durable +// installed marker is left in place, so the service remains "installed" but +// down until an explicit uninstall. +func (rg *Registration) Unregister(ctx context.Context) error { + var unregErr error + rg.once.Do(func() { + // Stop the heartbeat first and wait for it to finish, so no in-flight + // refresh can re-create the entry after we delete it. + rg.cancel() + <-rg.done + + if err := rg.kv.Delete(ctx, rg.key); err != nil { + // An already-expired entry is a normal leave, not a failure. + if !errors.Is(err, jetstream.ErrKeyNotFound) && !errors.Is(err, jetstream.ErrKeyDeleted) { + unregErr = err + } + } + }) + if unregErr != nil { + return fmt.Errorf("unregister %s: %w", rg.key, unregErr) + } + return nil +} diff --git a/internal/infrastructure/servicediscovery/nats/registry.go b/internal/infrastructure/servicediscovery/nats/registry.go new file mode 100644 index 000000000..f5d6e8873 --- /dev/null +++ b/internal/infrastructure/servicediscovery/nats/registry.go @@ -0,0 +1,314 @@ +package nats + +import ( + "context" + "errors" + "fmt" + "log/slog" + "slices" + "strings" + "time" + + "github.com/nats-io/nats.go" + "github.com/nats-io/nats.go/jetstream" + + "github.com/samcharles93/archie-core/internal/servicediscovery" +) + +// Compile-time proof that Client satisfies the broker-neutral contract. The +// interface lives in internal/servicediscovery so domains can depend on the +// behaviour without importing this package. +var _ servicediscovery.ServiceRegistry = (*Client)(nil) + +// installedMarkerValue is the value stored at an installed-bucket key. Only its +// presence matters; the value exists so the bucket is self-describing. +var installedMarkerValue = []byte("installed") + +// Client owns the NATS connection and the two registry buckets: a heartbeat +// bucket (one live endpoint per instance) and a durable installed bucket (one +// marker per service that has ever registered). It implements +// [servicediscovery.ServiceRegistry] and provides the registration side a +// service uses to announce itself. +type Client struct { + conn *nats.Conn + kv jetstream.KeyValue // heartbeat bucket + installed jetstream.KeyValue // installed bucket + cfg Config + log *slog.Logger +} + +// Connect dials the broker and provisions the two registry buckets. The bucket +// that carries heartbeats is configured with a TTL so stale entries expire +// (with a marker TTL so watchers observe the expiry as a leave); the installed +// bucket has no TTL so a marker persists for the life of the install. +// +// On any failure the partially-built connection is closed before returning, so +// a failed Connect leaks nothing. +func Connect(ctx context.Context, cfg Config, log *slog.Logger) (*Client, error) { + if err := cfg.Validate(); err != nil { + return nil, err + } + cfg = cfg.withDefaults() + if log == nil { + log = slog.New(slog.DiscardHandler) + } + + conn, err := dial(cfg) + if err != nil { + return nil, err + } + + // Every failure past this point must close conn. Cleared on success. + closeOnFail := conn.Close + defer func() { + if closeOnFail != nil { + closeOnFail() + } + }() + + js, err := jetstream.New(conn) + if err != nil { + return nil, fmt.Errorf("nats jetstream init: %w", err) + } + + kv, err := js.CreateOrUpdateKeyValue(ctx, jetstream.KeyValueConfig{ + Bucket: cfg.Bucket, + History: 1, + TTL: cfg.TTL, + LimitMarkerTTL: markerTTL(cfg.TTL), + }) + if err != nil { + return nil, fmt.Errorf("nats create registry bucket %q: %w", cfg.Bucket, err) + } + + installed, err := js.CreateOrUpdateKeyValue(ctx, jetstream.KeyValueConfig{ + Bucket: cfg.InstalledBucket, + History: 1, + }) + if err != nil { + return nil, fmt.Errorf("nats create installed bucket %q: %w", cfg.InstalledBucket, err) + } + + closeOnFail = nil + log.Info("service registry connected", + "url", conn.ConnectedUrl(), + "bucket", cfg.Bucket, + "installed_bucket", cfg.InstalledBucket) + + return &Client{ + conn: conn, + kv: kv, + installed: installed, + cfg: cfg, + log: log, + }, nil +} + +// markerTTL returns the subject-delete-marker TTL for the registry bucket. A +// heartbeat expiry is surfaced to watchers as a delete marker, but NATS rejects +// a marker TTL below one second; clamp the configured TTL up to that floor so a +// fast-expiring bucket still lets watchers observe the leave. +func markerTTL(ttl time.Duration) time.Duration { + if ttl < time.Second { + return time.Second + } + return ttl +} + +// dial opens the raw connection, applying auth options from cfg. +func dial(cfg Config) (*nats.Conn, error) { + var opts []nats.Option + if cfg.Token != "" { + opts = append(opts, nats.Token(cfg.Token)) + } + conn, err := nats.Connect(cfg.URL, opts...) + if err != nil { + return nil, fmt.Errorf("nats connect %s: %w", cfg.URL, err) + } + return conn, nil +} + +// Close shuts the connection down. It is safe to call on a nil client so +// callers holding an optional registry can defer it unconditionally. +func (c *Client) Close() { + if c == nil || c.conn == nil { + return + } + c.conn.Close() + c.log.Debug("service registry connection closed") +} + +// Resolve returns the current healthy endpoints for service. +// +// It returns servicediscovery.ErrNotInstalled when the service's installed +// marker is absent (the service was never installed). When the service is +// installed but has no live endpoint right now -- all its heartbeats expired or +// were unregistered -- it returns a possibly-empty slice and a nil error. That +// distinction is the contract's governing semantic; see the package doc. +func (c *Client) Resolve(ctx context.Context, service string) ([]servicediscovery.Endpoint, error) { + if err := c.requireInstalled(ctx, service); err != nil { + return nil, err + } + + lister, err := c.kv.ListKeysFiltered(ctx, endpointKeyPrefix(service)) + if err != nil { + return nil, fmt.Errorf("resolve %s: %w", service, err) + } + defer lister.Stop() + + // A lister may report duplicate keys; dedupe by ID and sort for a stable + // order so callers get a deterministic roster. + byID := map[string]servicediscovery.Endpoint{} + for key := range lister.Keys() { + entry, err := c.kv.Get(ctx, key) + if err != nil { + // A key may expire between listing and Get; that is a healthy + // leave, not a failure. + if errors.Is(err, jetstream.ErrKeyNotFound) || errors.Is(err, jetstream.ErrKeyDeleted) { + continue + } + return nil, fmt.Errorf("resolve %s: %w", service, err) + } + if entry.Operation() == jetstream.KeyValueDelete || entry.Operation() == jetstream.KeyValuePurge { + continue + } + ep, ok := decodeEndpoint(entry.Value()) + if !ok { + c.log.Warn("discarding undecodable endpoint", "service", service, "key", key) + continue + } + ep.Service = service + byID[ep.ID] = ep + } + + eps := make([]servicediscovery.Endpoint, 0, len(byID)) + for _, ep := range byID { + eps = append(eps, ep) + } + slices.SortFunc(eps, func(a, b servicediscovery.Endpoint) int { + return strings.Compare(a.ID, b.ID) + }) + return eps, nil +} + +// Watch returns a channel that emits Join and Leave events as service's +// membership changes, until ctx is cancelled. The channel is closed when ctx +// is cancelled. +// +// It returns servicediscovery.ErrNotInstalled when the service's installed +// marker is absent at call time. A service that is installed but has no live +// endpoint returns a live channel that simply does not emit until an endpoint +// joins. Watching a service that becomes installed only after this call returns +// requires re-calling Watch. +func (c *Client) Watch(ctx context.Context, service string) (<-chan servicediscovery.Event, error) { + if err := c.requireInstalled(ctx, service); err != nil { + return nil, err + } + + watcher, err := c.kv.Watch(ctx, endpointKeyPrefix(service)) + if err != nil { + return nil, fmt.Errorf("watch %s: %w", service, err) + } + + out := make(chan servicediscovery.Event) + go c.fanWatch(ctx, service, watcher, out) + return out, nil +} + +// fanWatch translates KV updates into contract Events until ctx is cancelled or +// the underlying watcher closes. +// +// A live instance heartbeats by re-Putting its key, which the KV watch reports +// as another Put. That must not be re-emitted as a Join: only a genuine +// appearance of an ID (including its first appearance in the initial snapshot) +// is a Join, and only its genuine disappearance is a Leave. The known set +// deduplicates refreshes so the stream carries membership changes, not +// liveness pings. +func (c *Client) fanWatch(ctx context.Context, service string, watcher jetstream.KeyWatcher, out chan<- servicediscovery.Event) { + defer close(out) + defer watcher.Stop() + + known := map[string]struct{}{} + for { + select { + case <-ctx.Done(): + return + case entry, ok := <-watcher.Updates(): + if !ok { + return + } + // A nil entry marks the end of the initial snapshot; membership + // events continue from there. + if entry == nil { + continue + } + ev, ok := eventFromEntry(service, entry) + if !ok { + c.log.Warn("discarding unrecognised registry entry", + "service", service, "key", entry.Key(), "op", entry.Operation()) + continue + } + id := ev.Endpoint.ID + switch ev.Kind { + case servicediscovery.Join: + if _, seen := known[id]; seen { + continue + } + known[id] = struct{}{} + case servicediscovery.Leave: + if _, seen := known[id]; !seen { + continue + } + delete(known, id) + } + select { + case out <- ev: + case <-ctx.Done(): + return + } + } + } +} + +// requireInstalled checks the installed marker for service, translating absence +// into the contract's ErrNotInstalled so the NotInstalled/Down distinction is +// expressed by both Resolve and Watch. +func (c *Client) requireInstalled(ctx context.Context, service string) error { + if _, err := c.installed.Get(ctx, service); err != nil { + if errors.Is(err, jetstream.ErrKeyNotFound) { + return fmt.Errorf("%s: %w", service, servicediscovery.ErrNotInstalled) + } + return fmt.Errorf("%s: %w", service, err) + } + return nil +} + +// eventFromEntry converts one KV entry into a contract Event. +// +// A Put is a Join, and its value carries the full endpoint. A delete or purge +// marker -- including one left by a heartbeat TTL expiry -- is a Leave; the +// value is gone, so it reconstructs the instance from the key with an empty +// address (the ID still identifies the instance). It reports false for an entry +// it cannot interpret. +func eventFromEntry(service string, entry jetstream.KeyValueEntry) (servicediscovery.Event, bool) { + switch entry.Operation() { + case jetstream.KeyValuePut: + ep, ok := decodeEndpoint(entry.Value()) + if !ok { + return servicediscovery.Event{}, false + } + ep.Service = service + return servicediscovery.Event{Kind: servicediscovery.Join, Endpoint: ep}, true + case jetstream.KeyValueDelete, jetstream.KeyValuePurge: + id := idFromEndpointKey(service, entry.Key()) + if id == "" { + return servicediscovery.Event{}, false + } + return servicediscovery.Event{ + Kind: servicediscovery.Leave, + Endpoint: servicediscovery.Endpoint{Service: service, ID: id}, + }, true + default: + return servicediscovery.Event{}, false + } +} diff --git a/internal/infrastructure/servicediscovery/nats/registry_test.go b/internal/infrastructure/servicediscovery/nats/registry_test.go new file mode 100644 index 000000000..1b7f56b1a --- /dev/null +++ b/internal/infrastructure/servicediscovery/nats/registry_test.go @@ -0,0 +1,299 @@ +package nats + +import ( + "context" + "errors" + "log/slog" + "testing" + "time" + + "github.com/nats-io/nats-server/v2/server" + natstest "github.com/nats-io/nats-server/v2/test" + + "github.com/samcharles93/archie-core/internal/servicediscovery" +) + +// discardLogger silences the registry logger in tests, matching the sibling +// eventbus/nats test convention. +func discardLogger() *slog.Logger { return slog.New(slog.DiscardHandler) } + +// testConfig returns a discovery Config with a short TTL/heartbeat so tests +// exercise expiry quickly without relying on a fixture elsewhere. +func testConfig() Config { + return Config{TTL: time.Second, Heartbeat: 200 * time.Millisecond} +} + +// newServer starts a real in-process NATS server with JetStream enabled. This +// is the established test technique for the NATS infrastructure (see the +// sibling eventbus/nats retention test): a real server, not a mocked client. +func newServer(t *testing.T) *server.Server { + t.Helper() + srv := natstest.RunServer(&server.Options{ + Port: -1, + JetStream: true, + StoreDir: t.TempDir(), + }) + t.Cleanup(srv.Shutdown) + return srv +} + +// newClient connects a registry client to a fresh server with the given config. +// Cleanup closes the connection after the caller's own cleanups (registered +// later) have run, so a caller can cancel registration contexts first. +func newClient(t *testing.T, cfg Config) *Client { + t.Helper() + cfg.URL = newServer(t).ClientURL() + c, err := Connect(context.Background(), cfg, discardLogger()) + if err != nil { + t.Fatalf("Connect = %v", err) + } + t.Cleanup(c.Close) + return c +} + +// expectEvent reads from a Watch channel until it observes the wanted event, +// discarding unrelated early events (e.g. the rest of an initial snapshot). +func expectEvent(t *testing.T, ch <-chan servicediscovery.Event, want servicediscovery.Event) { + t.Helper() + deadline := time.NewTimer(10 * time.Second) + defer deadline.Stop() + for { + select { + case ev, ok := <-ch: + if !ok { + t.Fatalf("watch channel closed before %s for %s", want.Kind, want.Endpoint.ID) + } + if ev == want { + return + } + case <-deadline.C: + t.Fatalf("timed out waiting for %s for %s", want.Kind, want.Endpoint.ID) + } + } +} + +// TestRegistryResolveNotInstalled pins that a service that was never registered +// resolves to ErrNotInstalled, not an empty roster. +func TestRegistryResolveNotInstalled(t *testing.T) { + client := newClient(t, testConfig()) + _, err := client.Resolve(context.Background(), "curator") + if !errors.Is(err, servicediscovery.ErrNotInstalled) { + t.Fatalf("Resolve(unregistered) = %v, want ErrNotInstalled", err) + } +} + +// TestRegistryWatchNotInstalled pins that watching a never-registered service +// returns ErrNotInstalled at call time. +func TestRegistryWatchNotInstalled(t *testing.T) { + client := newClient(t, testConfig()) + _, err := client.Watch(context.Background(), "curator") + if !errors.Is(err, servicediscovery.ErrNotInstalled) { + t.Fatalf("Watch(unregistered) = %v, want ErrNotInstalled", err) + } +} + +// TestRegistryRegisterResolve covers registration and a single-instance resolve. +func TestRegistryRegisterResolve(t *testing.T) { + client := newClient(t, testConfig()) + regCtx, regCancel := context.WithCancel(context.Background()) + t.Cleanup(regCancel) + + reg, err := client.Register(regCtx, "curator", servicediscovery.Endpoint{ + ID: "inst-1", + Address: "127.0.0.1:5001", + }) + if err != nil { + t.Fatalf("Register = %v", err) + } + t.Cleanup(func() { _ = reg.Unregister(context.Background()) }) + + eps, err := client.Resolve(context.Background(), "curator") + if err != nil { + t.Fatalf("Resolve after Register = %v", err) + } + if len(eps) != 1 { + t.Fatalf("Resolve length = %d, want 1: %+v", len(eps), eps) + } + got := eps[0] + want := servicediscovery.Endpoint{Service: "curator", ID: "inst-1", Address: "127.0.0.1:5001"} + if got != want { + t.Errorf("Resolve endpoint = %+v, want %+v", got, want) + } +} + +// TestRegistryRegisterResolveMultiple proves multiple instances of one service +// resolve together, ordered deterministically by ID. +func TestRegistryRegisterResolveMultiple(t *testing.T) { + client := newClient(t, testConfig()) + regCtx, regCancel := context.WithCancel(context.Background()) + t.Cleanup(regCancel) + + instances := []servicediscovery.Endpoint{ + {ID: "b", Address: "127.0.0.1:5002"}, + {ID: "a", Address: "127.0.0.1:5001"}, + } + var regs []*Registration + for _, ep := range instances { + reg, err := client.Register(regCtx, "curator", ep) + if err != nil { + t.Fatalf("Register %s = %v", ep.ID, err) + } + regs = append(regs, reg) + } + t.Cleanup(func() { + for _, r := range regs { + _ = r.Unregister(context.Background()) + } + }) + + eps, err := client.Resolve(context.Background(), "curator") + if err != nil { + t.Fatalf("Resolve = %v", err) + } + if len(eps) != 2 { + t.Fatalf("Resolve length = %d, want 2: %+v", len(eps), eps) + } + if eps[0].ID != "a" || eps[1].ID != "b" { + t.Errorf("Resolve order = %s, %s; want a, b", eps[0].ID, eps[1].ID) + } +} + +// TestRegistryResolveInstalledButDown is the contract's governing semantic: a +// service that registered but has no live endpoint resolves to an empty, +// non-error roster, distinct from ErrNotInstalled. +func TestRegistryResolveInstalledButDown(t *testing.T) { + client := newClient(t, testConfig()) + regCtx, regCancel := context.WithCancel(context.Background()) + t.Cleanup(regCancel) + + reg, err := client.Register(regCtx, "curator", servicediscovery.Endpoint{ + ID: "inst-1", + Address: "127.0.0.1:5001", + }) + if err != nil { + t.Fatalf("Register = %v", err) + } + if err := reg.Unregister(context.Background()); err != nil { + t.Fatalf("Unregister = %v", err) + } + + // The installed marker persists, so the service is still installed; it just + // has no live endpoints. + eps, err := client.Resolve(context.Background(), "curator") + if err != nil { + t.Fatalf("Resolve after Unregister = %v, want nil (installed but down)", err) + } + if len(eps) != 0 { + t.Fatalf("Resolve length = %d, want 0 for installed-but-down service", len(eps)) + } + if eps == nil { + t.Fatal("Resolve returned nil slice, want a non-nil empty slice") + } + + _, err = client.Resolve(context.Background(), "never-installed") + if !errors.Is(err, servicediscovery.ErrNotInstalled) { + t.Fatalf("Resolve(never-installed) = %v, want ErrNotInstalled", err) + } +} + +// TestRegistryWatchJoinLeave covers Join and Leave emission, including the +// initial snapshot and a heartbeat refresh not being re-emitted as a Join. +func TestRegistryWatchJoinLeave(t *testing.T) { + client := newClient(t, testConfig()) + + // Register A first so its entry is part of the initial watch snapshot and + // the installed marker is present before Watch. + regCtx, regCancel := context.WithCancel(context.Background()) + t.Cleanup(regCancel) + regA, err := client.Register(regCtx, "curator", servicediscovery.Endpoint{ID: "a", Address: "127.0.0.1:5001"}) + if err != nil { + t.Fatalf("Register a = %v", err) + } + t.Cleanup(func() { _ = regA.Unregister(context.Background()) }) + + watchCtx, watchCancel := context.WithCancel(context.Background()) + t.Cleanup(watchCancel) + events, err := client.Watch(watchCtx, "curator") + if err != nil { + t.Fatalf("Watch = %v", err) + } + + // Initial snapshot delivers the pre-existing instance as a Join. + expectEvent(t, events, servicediscovery.Event{ + Kind: servicediscovery.Join, + Endpoint: servicediscovery.Endpoint{Service: "curator", ID: "a", Address: "127.0.0.1:5001"}, + }) + + // A new instance joining is a Join. + regB, err := client.Register(regCtx, "curator", servicediscovery.Endpoint{ID: "b", Address: "127.0.0.1:5002"}) + if err != nil { + t.Fatalf("Register b = %v", err) + } + expectEvent(t, events, servicediscovery.Event{ + Kind: servicediscovery.Join, + Endpoint: servicediscovery.Endpoint{Service: "curator", ID: "b", Address: "127.0.0.1:5002"}, + }) + + // An explicit unregister is a Leave. + if err := regB.Unregister(context.Background()); err != nil { + t.Fatalf("Unregister b = %v", err) + } + expectEvent(t, events, servicediscovery.Event{ + Kind: servicediscovery.Leave, + Endpoint: servicediscovery.Endpoint{Service: "curator", ID: "b"}, + }) +} + +// TestRegistryWatchHeartbeatExpiry proves that a stopped heartbeat leads to a +// Leave via the bucket TTL (installed-but-down), not to a flapping re-Join of +// the same ID. +func TestRegistryWatchHeartbeatExpiry(t *testing.T) { + cfg := testConfig() + cfg.TTL = 600 * time.Millisecond + cfg.Heartbeat = 100 * time.Millisecond + client := newClient(t, cfg) + + regCtx, regCancel := context.WithCancel(context.Background()) + t.Cleanup(regCancel) + reg, err := client.Register(regCtx, "curator", servicediscovery.Endpoint{ID: "exp", Address: "127.0.0.1:5001"}) + if err != nil { + t.Fatalf("Register = %v", err) + } + t.Cleanup(func() { _ = reg.Unregister(context.Background()) }) + + watchCtx, watchCancel := context.WithCancel(context.Background()) + t.Cleanup(watchCancel) + events, err := client.Watch(watchCtx, "curator") + if err != nil { + t.Fatalf("Watch = %v", err) + } + expectEvent(t, events, servicediscovery.Event{ + Kind: servicediscovery.Join, + Endpoint: servicediscovery.Endpoint{Service: "curator", ID: "exp", Address: "127.0.0.1:5001"}, + }) + + // Stop the heartbeat; the entry is no longer refreshed and expires via TTL. + regCancel() + + expectEvent(t, events, servicediscovery.Event{ + Kind: servicediscovery.Leave, + Endpoint: servicediscovery.Endpoint{Service: "curator", ID: "exp"}, + }) + + // The service is still installed (its marker persists), so Resolve is an + // empty, non-error roster -- not ErrNotInstalled. + deadline := time.Now().Add(3 * time.Second) + for { + eps, err := client.Resolve(context.Background(), "curator") + if err != nil { + t.Fatalf("Resolve after expiry = %v, want nil (installed but down)", err) + } + if len(eps) == 0 { + break + } + if time.Now().After(deadline) { + t.Fatalf("Resolve still reports %d live endpoints after heartbeat expiry", len(eps)) + } + time.Sleep(50 * time.Millisecond) + } +} diff --git a/internal/infrastructure/servicediscovery/nats/value.go b/internal/infrastructure/servicediscovery/nats/value.go new file mode 100644 index 000000000..01d5c838b --- /dev/null +++ b/internal/infrastructure/servicediscovery/nats/value.go @@ -0,0 +1,29 @@ +package nats + +import ( + "encoding/json" + "fmt" + + "github.com/samcharles93/archie-core/internal/servicediscovery" +) + +// endpointJSON is the on-disk (KV value) encoding of a registered endpoint. It +// is deliberately the contract type so a consumer reading raw records gets the +// same shape ServiceRegistry hands back. +func encodeEndpoint(ep servicediscovery.Endpoint) ([]byte, error) { + b, err := json.Marshal(ep) + if err != nil { + return nil, fmt.Errorf("encode endpoint %s/%s: %w", ep.Service, ep.ID, err) + } + return b, nil +} + +// decodeEndpoint parses a KV value back into an Endpoint. It reports false for +// a value it cannot parse. +func decodeEndpoint(value []byte) (servicediscovery.Endpoint, bool) { + var ep servicediscovery.Endpoint + if err := json.Unmarshal(value, &ep); err != nil { + return servicediscovery.Endpoint{}, false + } + return ep, true +} diff --git a/internal/servicediscovery/doc.go b/internal/servicediscovery/doc.go new file mode 100644 index 000000000..b6d3f05ee --- /dev/null +++ b/internal/servicediscovery/doc.go @@ -0,0 +1,36 @@ +// Package servicediscovery is the broker-neutral service registry contract. +// +// It defines only the shape archie-core needs to discover and watch the +// services of its decomposed fleet: a way to resolve a named service to its +// live endpoints, and a way to watch that membership change over time. It +// names no registry. A NATS JetStream KV implementation lives in +// internal/infrastructure/servicediscovery/nats; the Kubernetes-native +// primary (Service/EndpointSlice + CoreDNS, per +// docs/inspiration/service-discovery-research-2026-09-05.md) is the sibling +// that serves the same contract. +// +// Per docs/architecture/dependencies-and-contracts.md, consumers depend on +// this package and never on an infrastructure implementation; only +// application composition chooses a concrete registry. +// +// # NotInstalled is not Down +// +// The single most important semantic in this package is the distinction +// between a service that was never installed and a service that is installed +// but currently has no healthy endpoint. +// +// - [ErrNotInstalled] reports that a service has never been installed. It +// is a distinct outcome, not a failure. An optional service (Curator +// today, others later) that is not present must resolve to "absent", +// never "broken": callers match it with errors.Is and treat it as "this +// capability is disabled", not "this capability is unhealthy". +// - A service that IS installed but has zero healthy endpoints right now is +// a normal, non-error outcome. [ServiceRegistry.Resolve] returns a +// possibly-empty slice with a nil error; [ServiceRegistry.Watch] returns +// a live channel that simply does not emit until an endpoint joins. +// +// This is why a registry implementation must model presence separately from +// liveness. The NATS implementation keeps a durable "installed" marker that +// never expires, distinct from expiring heartbeat keys: the marker's presence +// means NotInstalled=false, while heartbeat expiry is Down/Unhealthy. +package servicediscovery diff --git a/internal/servicediscovery/servicediscovery.go b/internal/servicediscovery/servicediscovery.go new file mode 100644 index 000000000..80bde4730 --- /dev/null +++ b/internal/servicediscovery/servicediscovery.go @@ -0,0 +1,86 @@ +package servicediscovery + +import ( + "context" + "errors" +) + +// ErrNotInstalled reports that a service has never been installed. It is +// distinct from a service that is installed but currently has no healthy +// endpoint: that case is an empty, non-error result, not ErrNotInstalled. +// Implementations of [ServiceRegistry.Resolve] and [ServiceRegistry.Watch] +// return it (wrapped, matched with errors.Is) when a service's "installed" +// marker is absent. Callers match it and treat the capability as disabled. +var ErrNotInstalled = errors.New("servicediscovery: service not installed") + +// Endpoint is one resolved instance of a service. +type Endpoint struct { + // Service is the logical name the endpoint was registered under. It is + // redundant within the scope of a single Resolve/Watch call, but it makes + // the value self-describing and directly usable by a consumer that reads + // raw registry records (e.g. a custom gRPC resolver) rather than going + // through ServiceRegistry. + Service string `json:"service,omitempty"` + + // ID distinguishes multiple instances of the same service. It is the + // per-instance identifier; two endpoints with the same Service and ID are + // the same instance. An ID must not contain the registry's key separator + // (see the NATS implementation's key scheme). + ID string `json:"id"` + + // Address is the dialable host:port for the instance. + Address string `json:"address"` +} + +// EventKind is the kind of membership change an [Event] describes. +type EventKind int + +const ( + // Join reports that an endpoint became live. + Join EventKind = iota + + // Leave reports that an endpoint left: it was unregistered, its heartbeat + // expired, or its process stopped. + Leave +) + +// String returns a human-readable kind, for logging and tests. +func (k EventKind) String() string { + switch k { + case Join: + return "join" + case Leave: + return "leave" + default: + return "unknown" + } +} + +// Event is a single membership change for one endpoint. +type Event struct { + // Endpoint is the instance that joined or left. On a [Leave] the Address + // may be empty: the registry observing an entry disappear can recover the + // instance's ID from the key but not the address it carried. The ID + // uniquely identifies the instance. + Endpoint Endpoint + + // Kind is [Join] or [Leave]. + Kind EventKind +} + +// ServiceRegistry resolves and watches the live endpoints of a named service. +type ServiceRegistry interface { + // Resolve returns the current healthy endpoints for service. It returns + // [ErrNotInstalled] if service has never been installed. If service is + // installed but currently has no healthy endpoint, it returns a + // possibly-empty slice with a nil error -- callers distinguish that from + // NotInstalled to avoid treating an optional, disabled service as broken. + Resolve(ctx context.Context, service string) ([]Endpoint, error) + + // Watch returns a channel that emits [Join] and [Leave] events as service's + // membership changes, until ctx is cancelled. It returns [ErrNotInstalled] + // if service has never been installed at the time of the call. A service + // that is installed but currently has no healthy endpoint returns a live + // channel that simply does not emit until an endpoint joins. + Watch(ctx context.Context, service string) (<-chan Event, error) +} diff --git a/internal/servicediscovery/servicediscovery_test.go b/internal/servicediscovery/servicediscovery_test.go new file mode 100644 index 000000000..7c87fad14 --- /dev/null +++ b/internal/servicediscovery/servicediscovery_test.go @@ -0,0 +1,39 @@ +package servicediscovery + +import ( + "errors" + "fmt" + "testing" +) + +// TestEventKindString pins the human-readable form of each kind, including the +// zero value (unset/unknown), so logging and diagnostics never render a magic +// number. +func TestEventKindString(t *testing.T) { + tests := []struct { + name string + kind EventKind + want string + }{ + {name: "join", kind: Join, want: "join"}, + {name: "leave", kind: Leave, want: "leave"}, + {name: "unknown out-of-range", kind: EventKind(99), want: "unknown"}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + if got := tt.kind.String(); got != tt.want { + t.Errorf("EventKind(%d).String() = %q, want %q", tt.kind, got, tt.want) + } + }) + } +} + +// TestErrNotInstalledWraps pins that implementations can wrap the sentinel and +// callers can still match it with errors.Is, which is the contract's intended +// use (a broker or registry specific error may be joined around it). +func TestErrNotInstalledWraps(t *testing.T) { + wrapped := fmt.Errorf("resolve %s: %w", "curator", ErrNotInstalled) + if !errors.Is(wrapped, ErrNotInstalled) { + t.Fatalf("wrapped ErrNotInstalled does not match errors.Is: %v", wrapped) + } +}