feat(servicediscovery): scaffold ServiceRegistry contract and NATS KV implementation

Add a broker-neutral service registry contract (Endpoint, EventKind,
Event, ServiceRegistry, ErrNotInstalled) and a NATS JetStream KV
implementation, plus the registration side a service uses to announce itself.

The contract's governing semantic -- NotInstalled is not Down -- is modeled
by two buckets: a durable installed marker (never expires) distinct from
expiring heartbeat keys. Instances heartbeat their key on an interval; a
stopped process leaves via TTL expiry. Register writes the marker once then
heartbeats; Resolve returns ErrNotInstalled only when the marker is absent,
and an empty roster for installed-but-down. Watch dedupes heartbeat refreshes
so it emits membership changes, not liveness pings.

Unwired: no daemon/bootstrap path consumes it yet. Deliberately uses a dot
key separator instead of the research sketch's slash so per-service watch
wildcards match.
This commit is contained in:
sam committed 2026-09-05 03:11:43 +10:00
1 parent 8715fed701
commit 0adc4f4599
13 files changed
+1228

No files matched your search

@@ -0,0 +1,84 @@
package nats
import (
"fmt"
"time"
)
// Defaults for Config. Previously these would have been unexported package
// constants, which made bucket naming and timeouts untunable per deployment.
const (
// DefaultBucket is the heartbeat bucket: one live endpoint per instance.
DefaultBucket = "ARCHIE_SERVICE_REGISTRY"
// DefaultInstalledBucket is the durable "installed" bucket: one marker per
// service that has ever registered, never expiring.
DefaultInstalledBucket = "ARCHIE_SERVICE_INSTALLED"
// DefaultTTL is how long a heartbeat entry survives without being
// refreshed. It must comfortably exceed DefaultHeartbeat so a transient
// pause in a live service's refresh does not drop it.
DefaultTTL = 15 * time.Second
// DefaultHeartbeat is the interval at which a registrar refreshes its
// endpoint entry.
DefaultHeartbeat = 5 * time.Second
)
// Config describes how to reach NATS and how the registry buckets behave.
// The zero value is not usable; call [Config.withDefaults] via [Connect], which
// fills every unset field.
type Config struct {
// URL is the NATS server address. Required.
URL string
// Token authenticates the connection. Empty means no token auth.
Token string
// Bucket is the heartbeat bucket name.
Bucket string
// InstalledBucket is the durable "installed" bucket name.
InstalledBucket string
// TTL is how long a heartbeat entry survives without being refreshed. It
// is the difference between a service that is installed-but-down (its
// entries expired) and one that is live (its entries are fresh).
TTL time.Duration
// Heartbeat is the interval at which a registrar refreshes its entry. It
// must be less than TTL, or a live service would expire between refreshes.
Heartbeat time.Duration
}
// Validate reports whether the config can produce a usable client.
func (c Config) Validate() error {
if c.URL == "" {
return fmt.Errorf("%w: URL is required", ErrInvalidConfig)
}
if c.TTL < 0 || c.Heartbeat < 0 {
return fmt.Errorf("%w: durations must not be negative", ErrInvalidConfig)
}
if c.Heartbeat > 0 && c.TTL > 0 && c.Heartbeat >= c.TTL {
return fmt.Errorf("%w: Heartbeat must be less than TTL", ErrInvalidConfig)
}
return nil
}
// withDefaults returns a copy with every unset field populated, so the rest of
// the package never has to test for zero values.
func (c Config) withDefaults() Config {
if c.Bucket == "" {
c.Bucket = DefaultBucket
}
if c.InstalledBucket == "" {
c.InstalledBucket = DefaultInstalledBucket
}
if c.TTL == 0 {
c.TTL = DefaultTTL
}
if c.Heartbeat == 0 {
c.Heartbeat = DefaultHeartbeat
}
return c
}
@@ -0,0 +1,49 @@
package nats
import (
"errors"
"testing"
"time"
)
func TestWithDefaults(t *testing.T) {
cfg := (Config{}).withDefaults()
if cfg.Bucket != DefaultBucket {
t.Errorf("withDefaults().Bucket = %q, want %q", cfg.Bucket, DefaultBucket)
}
if cfg.InstalledBucket != DefaultInstalledBucket {
t.Errorf("withDefaults().InstalledBucket = %q, want %q", cfg.InstalledBucket, DefaultInstalledBucket)
}
if cfg.TTL != DefaultTTL {
t.Errorf("withDefaults().TTL = %v, want %v", cfg.TTL, DefaultTTL)
}
if cfg.Heartbeat != DefaultHeartbeat {
t.Errorf("withDefaults().Heartbeat = %v, want %v", cfg.Heartbeat, DefaultHeartbeat)
}
}
func TestValidate(t *testing.T) {
tests := []struct {
name string
cfg Config
wantErr bool
}{
{name: "valid", cfg: Config{URL: "nats://x", TTL: 15 * time.Second, Heartbeat: 5 * time.Second}, wantErr: false},
{name: "missing url", cfg: Config{TTL: time.Second, Heartbeat: 200 * time.Millisecond}, wantErr: true},
{name: "heartbeat equals ttl", cfg: Config{URL: "nats://x", TTL: time.Second, Heartbeat: time.Second}, wantErr: true},
{name: "heartbeat exceeds ttl", cfg: Config{URL: "nats://x", TTL: time.Second, Heartbeat: 2 * time.Second}, wantErr: true},
{name: "negative durations", cfg: Config{URL: "nats://x", TTL: -time.Second, Heartbeat: time.Second}, wantErr: true},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
err := tt.cfg.Validate()
if tt.wantErr {
if !errors.Is(err, ErrInvalidConfig) {
t.Fatalf("Validate() = %v, want ErrInvalidConfig", err)
}
} else if err != nil {
t.Fatalf("Validate() = %v, want nil", err)
}
})
}
}
@@ -0,0 +1,50 @@
// Package nats is the NATS JetStream KV transport for service discovery.
//
// It implements the broker-neutral
// [servicediscovery.ServiceRegistry] contract against a JetStream KeyValue
// store, and additionally provides the registration side a service uses to
// announce itself. It is the fallback discovery mechanism for single-host,
// no-Kubernetes installs (Path B of
// docs/inspiration/service-discovery-research-2026-09-05.md); the
// Kubernetes-native primary is a sibling implementation of the same contract.
//
// # Two buckets: presence versus liveness
//
// The NotInstalled/Down distinction (the governing semantic of
// [servicediscovery]) requires modelling a service's presence separately from
// any instance's liveness. NATS KV cannot express that with one bucket,
// because a bucket-wide TTL (needed so stale heartbeats expire) would also
// expire the "installed" record, collapsing Down into NotInstalled. So the
// implementation uses two buckets:
//
// - Registry bucket (heartbeats): one key per live instance, refreshed on a
// heartbeat interval and expired by a bucket-wide TTL. Presence of a key
// here means the instance is live.
// - Installed bucket (marker): one key per service, written once at first
// registration with no TTL, never expires. Presence of this marker means
// the service was installed; absence means never installed => NotInstalled.
//
// A service that registered once but whose instances have all gone is still
// installed (its marker persists) and simply has zero live endpoints, so
// [Client.Resolve] returns an empty slice with no error. Only a service that
// never wrote a marker resolves to [servicediscovery.ErrNotInstalled].
//
// # Key scheme
//
// Instance heartbeat keys are "<service>.<instance-id>" (dot-separated), and
// the installed marker key is "<service>". The dot separator is a deliberate
// deviation from the research sketch's "<service>/<instance-id>": NATS KV
// watch filters match on dot-separated subject tokens, so a per-service watch
// of "curator.*" must see "curator.inst-1" as two tokens. A slash would make
// the whole key a single opaque token that no per-service wildcard can match.
// Consequently both service names and instance IDs must avoid the dot
// character; standard slugs and UUIDs satisfy that.
//
// Layout:
//
// config.go connection and bucket tuning, defaults and validation
// errors.go sentinel errors callers match on
// keys.go key scheme, pure and dependency-free
// registry.go the ServiceRegistry client (Resolve/Watch)
// registrar.go the registration side (announce + heartbeat)
package nats
@@ -0,0 +1,11 @@
package nats
import "errors"
// ErrInvalidConfig reports a Config that cannot produce a usable client.
//
// The delivery-outcome sentinel a caller matches on -- NotInstalled -- belongs
// to the broker-neutral contract in internal/servicediscovery, so a caller can
// handle it without knowing which registry is in use. This package returns
// that, not a copy of it.
var ErrInvalidConfig = errors.New("nats: invalid configuration")
@@ -0,0 +1,31 @@
package nats
import "strings"
// keySeparator joins a service and an instance ID in a registry key. It is a
// dot rather than the research sketch's slash so that a per-service watch
// filter ("curator.*") matches "curator.inst-1" as two NATS subject tokens;
// a slash would make the whole key one opaque token no wildcard could match.
// See the package doc comment for the full reasoning.
const keySeparator = "."
// endpointKey is the heartbeat bucket key for one live instance.
func endpointKey(service, id string) string {
return service + keySeparator + id
}
// endpointKeyPrefix is the watch/list filter matching every instance of a
// service. It is a single-token wildcard over the instance ID.
func endpointKeyPrefix(service string) string {
return service + keySeparator + "*"
}
// idFromEndpointKey returns the instance ID carried by a registry key, or "" if
// the key does not belong to the given service (or carries an empty ID).
func idFromEndpointKey(service, key string) string {
prefix := service + keySeparator
if !strings.HasPrefix(key, prefix) {
return ""
}
return strings.TrimPrefix(key, prefix)
}
@@ -0,0 +1,49 @@
package nats
import "testing"
func TestEndpointKey(t *testing.T) {
tests := []struct {
name string
service string
id string
want string
}{
{name: "simple", service: "curator", id: "inst-1", want: "curator.inst-1"},
{name: "single-char id", service: "scheduler", id: "a", want: "scheduler.a"},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
if got := endpointKey(tt.service, tt.id); got != tt.want {
t.Errorf("endpointKey(%q, %q) = %q, want %q", tt.service, tt.id, got, tt.want)
}
})
}
}
func TestEndpointKeyPrefix(t *testing.T) {
if got := endpointKeyPrefix("curator"); got != "curator.*" {
t.Errorf("endpointKeyPrefix(curator) = %q, want %q", got, "curator.*")
}
}
func TestIDFromEndpointKey(t *testing.T) {
tests := []struct {
name string
service string
key string
want string
}{
{name: "own key returns id", service: "curator", key: "curator.inst-1", want: "inst-1"},
{name: "other service", service: "curator", key: "scheduler.inst", want: ""},
{name: "empty id", service: "curator", key: "curator.", want: ""},
{name: "key equals prefix", service: "curator", key: "curator", want: ""},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
if got := idFromEndpointKey(tt.service, tt.key); got != tt.want {
t.Errorf("idFromEndpointKey(%q, %q) = %q, want %q", tt.service, tt.key, got, tt.want)
}
})
}
}
@@ -0,0 +1,151 @@
package nats
import (
"context"
"errors"
"fmt"
"log/slog"
"strings"
"sync"
"time"
"github.com/nats-io/nats.go/jetstream"
"github.com/samcharles93/archie-core/internal/servicediscovery"
)
// Register announces service's instance ep with the registry and keeps its
// heartbeat fresh until the returned Registration is unregistered or ctx is
// cancelled.
//
// It writes the service's durable installed marker once (idempotent: an
// existing marker is left alone) -- that marker is what makes the service
// "installed" rather than NotInstalled -- then writes a live heartbeat entry
// for ep and refreshes it on an interval. If the service's process stops
// without calling Unregister (its ctx is cancelled, or it crashes), the entry
// expires via the bucket TTL and watchers observe a Leave; only an explicit
// uninstall of the whole service removes the installed marker.
//
// ep.ID must be non-empty and must not contain the registry key separator; the
// service name must likewise avoid the separator (see the package doc).
func (c *Client) Register(ctx context.Context, service string, ep servicediscovery.Endpoint) (*Registration, error) {
if ep.ID == "" {
return nil, fmt.Errorf("register %s: %w: endpoint ID is required", service, ErrInvalidConfig)
}
if strings.Contains(ep.ID, keySeparator) || strings.Contains(service, keySeparator) {
return nil, fmt.Errorf("register %s: %w: service and instance ID must not contain %q", service, ErrInvalidConfig, keySeparator)
}
if err := c.ensureInstalled(ctx, service); err != nil {
return nil, err
}
key := endpointKey(service, ep.ID)
value, err := encodeEndpoint(ep)
if err != nil {
return nil, err
}
if _, err := c.kv.Put(ctx, key, value); err != nil {
return nil, fmt.Errorf("register %s/%s: %w", service, ep.ID, err)
}
hbCtx, cancel := context.WithCancel(ctx)
reg := &Registration{
service: service,
key: key,
value: value,
kv: c.kv,
heartbeat: c.cfg.Heartbeat,
log: c.log,
cancel: cancel,
done: make(chan struct{}),
}
go reg.heartbeatLoop(hbCtx)
return reg, nil
}
// ensureInstalled writes the service's installed marker if it is absent. The
// marker has no TTL, so it persists for the life of the install; a service that
// registered once but has no live instance is still installed (and resolves to
// an empty, non-error roster), not NotInstalled.
func (c *Client) ensureInstalled(ctx context.Context, service string) error {
if _, err := c.installed.Create(ctx, service, installedMarkerValue); err != nil {
if errors.Is(err, jetstream.ErrKeyExists) {
return nil
}
return fmt.Errorf("mark %s installed: %w", service, err)
}
return nil
}
// Registration is one live announcement of a service instance. It owns the
// background heartbeat that keeps the instance's entry fresh; stopping it is
// the caller's responsibility via Unregister (or by cancelling the ctx passed
// to Register, which stops the heartbeat and lets the entry expire).
type Registration struct {
service string
key string
value []byte
kv jetstream.KeyValue
heartbeat time.Duration
log *slog.Logger
cancel context.CancelFunc
done chan struct{}
once sync.Once
}
// heartbeatLoop refreshes the instance's live entry on an interval until ctx is
// cancelled. It never stops on a single refresh failure -- a transient broker
// blip must not unregister a healthy instance -- only on cancellation.
func (rg *Registration) heartbeatLoop(ctx context.Context) {
defer close(rg.done)
ticker := time.NewTicker(rg.heartbeat)
defer ticker.Stop()
for {
select {
case <-ctx.Done():
return
case <-ticker.C:
if err := rg.refresh(ctx); err != nil {
if ctx.Err() != nil {
return
}
rg.log.Warn("heartbeat refresh failed", "service", rg.service, "key", rg.key, "error", err)
}
}
}
}
// refresh re-publishes the instance's entry, resetting its bucket-TTL window so
// it stays live.
func (rg *Registration) refresh(ctx context.Context) error {
if _, err := rg.kv.Put(ctx, rg.key, rg.value); err != nil {
return fmt.Errorf("refresh %s: %w", rg.key, err)
}
return nil
}
// Unregister removes the instance's live entry and stops its heartbeat. It is
// idempotent: a second call is a no-op that returns nil. The service's durable
// installed marker is left in place, so the service remains "installed" but
// down until an explicit uninstall.
func (rg *Registration) Unregister(ctx context.Context) error {
var unregErr error
rg.once.Do(func() {
// Stop the heartbeat first and wait for it to finish, so no in-flight
// refresh can re-create the entry after we delete it.
rg.cancel()
<-rg.done
if err := rg.kv.Delete(ctx, rg.key); err != nil {
// An already-expired entry is a normal leave, not a failure.
if !errors.Is(err, jetstream.ErrKeyNotFound) && !errors.Is(err, jetstream.ErrKeyDeleted) {
unregErr = err
}
}
})
if unregErr != nil {
return fmt.Errorf("unregister %s: %w", rg.key, unregErr)
}
return nil
}
@@ -0,0 +1,314 @@
package nats
import (
"context"
"errors"
"fmt"
"log/slog"
"slices"
"strings"
"time"
"github.com/nats-io/nats.go"
"github.com/nats-io/nats.go/jetstream"
"github.com/samcharles93/archie-core/internal/servicediscovery"
)
// Compile-time proof that Client satisfies the broker-neutral contract. The
// interface lives in internal/servicediscovery so domains can depend on the
// behaviour without importing this package.
var _ servicediscovery.ServiceRegistry = (*Client)(nil)
// installedMarkerValue is the value stored at an installed-bucket key. Only its
// presence matters; the value exists so the bucket is self-describing.
var installedMarkerValue = []byte("installed")
// Client owns the NATS connection and the two registry buckets: a heartbeat
// bucket (one live endpoint per instance) and a durable installed bucket (one
// marker per service that has ever registered). It implements
// [servicediscovery.ServiceRegistry] and provides the registration side a
// service uses to announce itself.
type Client struct {
conn *nats.Conn
kv jetstream.KeyValue // heartbeat bucket
installed jetstream.KeyValue // installed bucket
cfg Config
log *slog.Logger
}
// Connect dials the broker and provisions the two registry buckets. The bucket
// that carries heartbeats is configured with a TTL so stale entries expire
// (with a marker TTL so watchers observe the expiry as a leave); the installed
// bucket has no TTL so a marker persists for the life of the install.
//
// On any failure the partially-built connection is closed before returning, so
// a failed Connect leaks nothing.
func Connect(ctx context.Context, cfg Config, log *slog.Logger) (*Client, error) {
if err := cfg.Validate(); err != nil {
return nil, err
}
cfg = cfg.withDefaults()
if log == nil {
log = slog.New(slog.DiscardHandler)
}
conn, err := dial(cfg)
if err != nil {
return nil, err
}
// Every failure past this point must close conn. Cleared on success.
closeOnFail := conn.Close
defer func() {
if closeOnFail != nil {
closeOnFail()
}
}()
js, err := jetstream.New(conn)
if err != nil {
return nil, fmt.Errorf("nats jetstream init: %w", err)
}
kv, err := js.CreateOrUpdateKeyValue(ctx, jetstream.KeyValueConfig{
Bucket: cfg.Bucket,
History: 1,
TTL: cfg.TTL,
LimitMarkerTTL: markerTTL(cfg.TTL),
})
if err != nil {
return nil, fmt.Errorf("nats create registry bucket %q: %w", cfg.Bucket, err)
}
installed, err := js.CreateOrUpdateKeyValue(ctx, jetstream.KeyValueConfig{
Bucket: cfg.InstalledBucket,
History: 1,
})
if err != nil {
return nil, fmt.Errorf("nats create installed bucket %q: %w", cfg.InstalledBucket, err)
}
closeOnFail = nil
log.Info("service registry connected",
"url", conn.ConnectedUrl(),
"bucket", cfg.Bucket,
"installed_bucket", cfg.InstalledBucket)
return &Client{
conn: conn,
kv: kv,
installed: installed,
cfg: cfg,
log: log,
}, nil
}
// markerTTL returns the subject-delete-marker TTL for the registry bucket. A
// heartbeat expiry is surfaced to watchers as a delete marker, but NATS rejects
// a marker TTL below one second; clamp the configured TTL up to that floor so a
// fast-expiring bucket still lets watchers observe the leave.
func markerTTL(ttl time.Duration) time.Duration {
if ttl < time.Second {
return time.Second
}
return ttl
}
// dial opens the raw connection, applying auth options from cfg.
func dial(cfg Config) (*nats.Conn, error) {
var opts []nats.Option
if cfg.Token != "" {
opts = append(opts, nats.Token(cfg.Token))
}
conn, err := nats.Connect(cfg.URL, opts...)
if err != nil {
return nil, fmt.Errorf("nats connect %s: %w", cfg.URL, err)
}
return conn, nil
}
// Close shuts the connection down. It is safe to call on a nil client so
// callers holding an optional registry can defer it unconditionally.
func (c *Client) Close() {
if c == nil || c.conn == nil {
return
}
c.conn.Close()
c.log.Debug("service registry connection closed")
}
// Resolve returns the current healthy endpoints for service.
//
// It returns servicediscovery.ErrNotInstalled when the service's installed
// marker is absent (the service was never installed). When the service is
// installed but has no live endpoint right now -- all its heartbeats expired or
// were unregistered -- it returns a possibly-empty slice and a nil error. That
// distinction is the contract's governing semantic; see the package doc.
func (c *Client) Resolve(ctx context.Context, service string) ([]servicediscovery.Endpoint, error) {
if err := c.requireInstalled(ctx, service); err != nil {
return nil, err
}
lister, err := c.kv.ListKeysFiltered(ctx, endpointKeyPrefix(service))
if err != nil {
return nil, fmt.Errorf("resolve %s: %w", service, err)
}
defer lister.Stop()
// A lister may report duplicate keys; dedupe by ID and sort for a stable
// order so callers get a deterministic roster.
byID := map[string]servicediscovery.Endpoint{}
for key := range lister.Keys() {
entry, err := c.kv.Get(ctx, key)
if err != nil {
// A key may expire between listing and Get; that is a healthy
// leave, not a failure.
if errors.Is(err, jetstream.ErrKeyNotFound) || errors.Is(err, jetstream.ErrKeyDeleted) {
continue
}
return nil, fmt.Errorf("resolve %s: %w", service, err)
}
if entry.Operation() == jetstream.KeyValueDelete || entry.Operation() == jetstream.KeyValuePurge {
continue
}
ep, ok := decodeEndpoint(entry.Value())
if !ok {
c.log.Warn("discarding undecodable endpoint", "service", service, "key", key)
continue
}
ep.Service = service
byID[ep.ID] = ep
}
eps := make([]servicediscovery.Endpoint, 0, len(byID))
for _, ep := range byID {
eps = append(eps, ep)
}
slices.SortFunc(eps, func(a, b servicediscovery.Endpoint) int {
return strings.Compare(a.ID, b.ID)
})
return eps, nil
}
// Watch returns a channel that emits Join and Leave events as service's
// membership changes, until ctx is cancelled. The channel is closed when ctx
// is cancelled.
//
// It returns servicediscovery.ErrNotInstalled when the service's installed
// marker is absent at call time. A service that is installed but has no live
// endpoint returns a live channel that simply does not emit until an endpoint
// joins. Watching a service that becomes installed only after this call returns
// requires re-calling Watch.
func (c *Client) Watch(ctx context.Context, service string) (<-chan servicediscovery.Event, error) {
if err := c.requireInstalled(ctx, service); err != nil {
return nil, err
}
watcher, err := c.kv.Watch(ctx, endpointKeyPrefix(service))
if err != nil {
return nil, fmt.Errorf("watch %s: %w", service, err)
}
out := make(chan servicediscovery.Event)
go c.fanWatch(ctx, service, watcher, out)
return out, nil
}
// fanWatch translates KV updates into contract Events until ctx is cancelled or
// the underlying watcher closes.
//
// A live instance heartbeats by re-Putting its key, which the KV watch reports
// as another Put. That must not be re-emitted as a Join: only a genuine
// appearance of an ID (including its first appearance in the initial snapshot)
// is a Join, and only its genuine disappearance is a Leave. The known set
// deduplicates refreshes so the stream carries membership changes, not
// liveness pings.
func (c *Client) fanWatch(ctx context.Context, service string, watcher jetstream.KeyWatcher, out chan<- servicediscovery.Event) {
defer close(out)
defer watcher.Stop()
known := map[string]struct{}{}
for {
select {
case <-ctx.Done():
return
case entry, ok := <-watcher.Updates():
if !ok {
return
}
// A nil entry marks the end of the initial snapshot; membership
// events continue from there.
if entry == nil {
continue
}
ev, ok := eventFromEntry(service, entry)
if !ok {
c.log.Warn("discarding unrecognised registry entry",
"service", service, "key", entry.Key(), "op", entry.Operation())
continue
}
id := ev.Endpoint.ID
switch ev.Kind {
case servicediscovery.Join:
if _, seen := known[id]; seen {
continue
}
known[id] = struct{}{}
case servicediscovery.Leave:
if _, seen := known[id]; !seen {
continue
}
delete(known, id)
}
select {
case out <- ev:
case <-ctx.Done():
return
}
}
}
}
// requireInstalled checks the installed marker for service, translating absence
// into the contract's ErrNotInstalled so the NotInstalled/Down distinction is
// expressed by both Resolve and Watch.
func (c *Client) requireInstalled(ctx context.Context, service string) error {
if _, err := c.installed.Get(ctx, service); err != nil {
if errors.Is(err, jetstream.ErrKeyNotFound) {
return fmt.Errorf("%s: %w", service, servicediscovery.ErrNotInstalled)
}
return fmt.Errorf("%s: %w", service, err)
}
return nil
}
// eventFromEntry converts one KV entry into a contract Event.
//
// A Put is a Join, and its value carries the full endpoint. A delete or purge
// marker -- including one left by a heartbeat TTL expiry -- is a Leave; the
// value is gone, so it reconstructs the instance from the key with an empty
// address (the ID still identifies the instance). It reports false for an entry
// it cannot interpret.
func eventFromEntry(service string, entry jetstream.KeyValueEntry) (servicediscovery.Event, bool) {
switch entry.Operation() {
case jetstream.KeyValuePut:
ep, ok := decodeEndpoint(entry.Value())
if !ok {
return servicediscovery.Event{}, false
}
ep.Service = service
return servicediscovery.Event{Kind: servicediscovery.Join, Endpoint: ep}, true
case jetstream.KeyValueDelete, jetstream.KeyValuePurge:
id := idFromEndpointKey(service, entry.Key())
if id == "" {
return servicediscovery.Event{}, false
}
return servicediscovery.Event{
Kind: servicediscovery.Leave,
Endpoint: servicediscovery.Endpoint{Service: service, ID: id},
}, true
default:
return servicediscovery.Event{}, false
}
}
@@ -0,0 +1,299 @@
package nats
import (
"context"
"errors"
"log/slog"
"testing"
"time"
"github.com/nats-io/nats-server/v2/server"
natstest "github.com/nats-io/nats-server/v2/test"
"github.com/samcharles93/archie-core/internal/servicediscovery"
)
// discardLogger silences the registry logger in tests, matching the sibling
// eventbus/nats test convention.
func discardLogger() *slog.Logger { return slog.New(slog.DiscardHandler) }
// testConfig returns a discovery Config with a short TTL/heartbeat so tests
// exercise expiry quickly without relying on a fixture elsewhere.
func testConfig() Config {
return Config{TTL: time.Second, Heartbeat: 200 * time.Millisecond}
}
// newServer starts a real in-process NATS server with JetStream enabled. This
// is the established test technique for the NATS infrastructure (see the
// sibling eventbus/nats retention test): a real server, not a mocked client.
func newServer(t *testing.T) *server.Server {
t.Helper()
srv := natstest.RunServer(&server.Options{
Port: -1,
JetStream: true,
StoreDir: t.TempDir(),
})
t.Cleanup(srv.Shutdown)
return srv
}
// newClient connects a registry client to a fresh server with the given config.
// Cleanup closes the connection after the caller's own cleanups (registered
// later) have run, so a caller can cancel registration contexts first.
func newClient(t *testing.T, cfg Config) *Client {
t.Helper()
cfg.URL = newServer(t).ClientURL()
c, err := Connect(context.Background(), cfg, discardLogger())
if err != nil {
t.Fatalf("Connect = %v", err)
}
t.Cleanup(c.Close)
return c
}
// expectEvent reads from a Watch channel until it observes the wanted event,
// discarding unrelated early events (e.g. the rest of an initial snapshot).
func expectEvent(t *testing.T, ch <-chan servicediscovery.Event, want servicediscovery.Event) {
t.Helper()
deadline := time.NewTimer(10 * time.Second)
defer deadline.Stop()
for {
select {
case ev, ok := <-ch:
if !ok {
t.Fatalf("watch channel closed before %s for %s", want.Kind, want.Endpoint.ID)
}
if ev == want {
return
}
case <-deadline.C:
t.Fatalf("timed out waiting for %s for %s", want.Kind, want.Endpoint.ID)
}
}
}
// TestRegistryResolveNotInstalled pins that a service that was never registered
// resolves to ErrNotInstalled, not an empty roster.
func TestRegistryResolveNotInstalled(t *testing.T) {
client := newClient(t, testConfig())
_, err := client.Resolve(context.Background(), "curator")
if !errors.Is(err, servicediscovery.ErrNotInstalled) {
t.Fatalf("Resolve(unregistered) = %v, want ErrNotInstalled", err)
}
}
// TestRegistryWatchNotInstalled pins that watching a never-registered service
// returns ErrNotInstalled at call time.
func TestRegistryWatchNotInstalled(t *testing.T) {
client := newClient(t, testConfig())
_, err := client.Watch(context.Background(), "curator")
if !errors.Is(err, servicediscovery.ErrNotInstalled) {
t.Fatalf("Watch(unregistered) = %v, want ErrNotInstalled", err)
}
}
// TestRegistryRegisterResolve covers registration and a single-instance resolve.
func TestRegistryRegisterResolve(t *testing.T) {
client := newClient(t, testConfig())
regCtx, regCancel := context.WithCancel(context.Background())
t.Cleanup(regCancel)
reg, err := client.Register(regCtx, "curator", servicediscovery.Endpoint{
ID: "inst-1",
Address: "127.0.0.1:5001",
})
if err != nil {
t.Fatalf("Register = %v", err)
}
t.Cleanup(func() { _ = reg.Unregister(context.Background()) })
eps, err := client.Resolve(context.Background(), "curator")
if err != nil {
t.Fatalf("Resolve after Register = %v", err)
}
if len(eps) != 1 {
t.Fatalf("Resolve length = %d, want 1: %+v", len(eps), eps)
}
got := eps[0]
want := servicediscovery.Endpoint{Service: "curator", ID: "inst-1", Address: "127.0.0.1:5001"}
if got != want {
t.Errorf("Resolve endpoint = %+v, want %+v", got, want)
}
}
// TestRegistryRegisterResolveMultiple proves multiple instances of one service
// resolve together, ordered deterministically by ID.
func TestRegistryRegisterResolveMultiple(t *testing.T) {
client := newClient(t, testConfig())
regCtx, regCancel := context.WithCancel(context.Background())
t.Cleanup(regCancel)
instances := []servicediscovery.Endpoint{
{ID: "b", Address: "127.0.0.1:5002"},
{ID: "a", Address: "127.0.0.1:5001"},
}
var regs []*Registration
for _, ep := range instances {
reg, err := client.Register(regCtx, "curator", ep)
if err != nil {
t.Fatalf("Register %s = %v", ep.ID, err)
}
regs = append(regs, reg)
}
t.Cleanup(func() {
for _, r := range regs {
_ = r.Unregister(context.Background())
}
})
eps, err := client.Resolve(context.Background(), "curator")
if err != nil {
t.Fatalf("Resolve = %v", err)
}
if len(eps) != 2 {
t.Fatalf("Resolve length = %d, want 2: %+v", len(eps), eps)
}
if eps[0].ID != "a" || eps[1].ID != "b" {
t.Errorf("Resolve order = %s, %s; want a, b", eps[0].ID, eps[1].ID)
}
}
// TestRegistryResolveInstalledButDown is the contract's governing semantic: a
// service that registered but has no live endpoint resolves to an empty,
// non-error roster, distinct from ErrNotInstalled.
func TestRegistryResolveInstalledButDown(t *testing.T) {
client := newClient(t, testConfig())
regCtx, regCancel := context.WithCancel(context.Background())
t.Cleanup(regCancel)
reg, err := client.Register(regCtx, "curator", servicediscovery.Endpoint{
ID: "inst-1",
Address: "127.0.0.1:5001",
})
if err != nil {
t.Fatalf("Register = %v", err)
}
if err := reg.Unregister(context.Background()); err != nil {
t.Fatalf("Unregister = %v", err)
}
// The installed marker persists, so the service is still installed; it just
// has no live endpoints.
eps, err := client.Resolve(context.Background(), "curator")
if err != nil {
t.Fatalf("Resolve after Unregister = %v, want nil (installed but down)", err)
}
if len(eps) != 0 {
t.Fatalf("Resolve length = %d, want 0 for installed-but-down service", len(eps))
}
if eps == nil {
t.Fatal("Resolve returned nil slice, want a non-nil empty slice")
}
_, err = client.Resolve(context.Background(), "never-installed")
if !errors.Is(err, servicediscovery.ErrNotInstalled) {
t.Fatalf("Resolve(never-installed) = %v, want ErrNotInstalled", err)
}
}
// TestRegistryWatchJoinLeave covers Join and Leave emission, including the
// initial snapshot and a heartbeat refresh not being re-emitted as a Join.
func TestRegistryWatchJoinLeave(t *testing.T) {
client := newClient(t, testConfig())
// Register A first so its entry is part of the initial watch snapshot and
// the installed marker is present before Watch.
regCtx, regCancel := context.WithCancel(context.Background())
t.Cleanup(regCancel)
regA, err := client.Register(regCtx, "curator", servicediscovery.Endpoint{ID: "a", Address: "127.0.0.1:5001"})
if err != nil {
t.Fatalf("Register a = %v", err)
}
t.Cleanup(func() { _ = regA.Unregister(context.Background()) })
watchCtx, watchCancel := context.WithCancel(context.Background())
t.Cleanup(watchCancel)
events, err := client.Watch(watchCtx, "curator")
if err != nil {
t.Fatalf("Watch = %v", err)
}
// Initial snapshot delivers the pre-existing instance as a Join.
expectEvent(t, events, servicediscovery.Event{
Kind: servicediscovery.Join,
Endpoint: servicediscovery.Endpoint{Service: "curator", ID: "a", Address: "127.0.0.1:5001"},
})
// A new instance joining is a Join.
regB, err := client.Register(regCtx, "curator", servicediscovery.Endpoint{ID: "b", Address: "127.0.0.1:5002"})
if err != nil {
t.Fatalf("Register b = %v", err)
}
expectEvent(t, events, servicediscovery.Event{
Kind: servicediscovery.Join,
Endpoint: servicediscovery.Endpoint{Service: "curator", ID: "b", Address: "127.0.0.1:5002"},
})
// An explicit unregister is a Leave.
if err := regB.Unregister(context.Background()); err != nil {
t.Fatalf("Unregister b = %v", err)
}
expectEvent(t, events, servicediscovery.Event{
Kind: servicediscovery.Leave,
Endpoint: servicediscovery.Endpoint{Service: "curator", ID: "b"},
})
}
// TestRegistryWatchHeartbeatExpiry proves that a stopped heartbeat leads to a
// Leave via the bucket TTL (installed-but-down), not to a flapping re-Join of
// the same ID.
func TestRegistryWatchHeartbeatExpiry(t *testing.T) {
cfg := testConfig()
cfg.TTL = 600 * time.Millisecond
cfg.Heartbeat = 100 * time.Millisecond
client := newClient(t, cfg)
regCtx, regCancel := context.WithCancel(context.Background())
t.Cleanup(regCancel)
reg, err := client.Register(regCtx, "curator", servicediscovery.Endpoint{ID: "exp", Address: "127.0.0.1:5001"})
if err != nil {
t.Fatalf("Register = %v", err)
}
t.Cleanup(func() { _ = reg.Unregister(context.Background()) })
watchCtx, watchCancel := context.WithCancel(context.Background())
t.Cleanup(watchCancel)
events, err := client.Watch(watchCtx, "curator")
if err != nil {
t.Fatalf("Watch = %v", err)
}
expectEvent(t, events, servicediscovery.Event{
Kind: servicediscovery.Join,
Endpoint: servicediscovery.Endpoint{Service: "curator", ID: "exp", Address: "127.0.0.1:5001"},
})
// Stop the heartbeat; the entry is no longer refreshed and expires via TTL.
regCancel()
expectEvent(t, events, servicediscovery.Event{
Kind: servicediscovery.Leave,
Endpoint: servicediscovery.Endpoint{Service: "curator", ID: "exp"},
})
// The service is still installed (its marker persists), so Resolve is an
// empty, non-error roster -- not ErrNotInstalled.
deadline := time.Now().Add(3 * time.Second)
for {
eps, err := client.Resolve(context.Background(), "curator")
if err != nil {
t.Fatalf("Resolve after expiry = %v, want nil (installed but down)", err)
}
if len(eps) == 0 {
break
}
if time.Now().After(deadline) {
t.Fatalf("Resolve still reports %d live endpoints after heartbeat expiry", len(eps))
}
time.Sleep(50 * time.Millisecond)
}
}
@@ -0,0 +1,29 @@
package nats
import (
"encoding/json"
"fmt"
"github.com/samcharles93/archie-core/internal/servicediscovery"
)
// endpointJSON is the on-disk (KV value) encoding of a registered endpoint. It
// is deliberately the contract type so a consumer reading raw records gets the
// same shape ServiceRegistry hands back.
func encodeEndpoint(ep servicediscovery.Endpoint) ([]byte, error) {
b, err := json.Marshal(ep)
if err != nil {
return nil, fmt.Errorf("encode endpoint %s/%s: %w", ep.Service, ep.ID, err)
}
return b, nil
}
// decodeEndpoint parses a KV value back into an Endpoint. It reports false for
// a value it cannot parse.
func decodeEndpoint(value []byte) (servicediscovery.Endpoint, bool) {
var ep servicediscovery.Endpoint
if err := json.Unmarshal(value, &ep); err != nil {
return servicediscovery.Endpoint{}, false
}
return ep, true
}
+36
View File
@@ -0,0 +1,36 @@
// Package servicediscovery is the broker-neutral service registry contract.
//
// It defines only the shape archie-core needs to discover and watch the
// services of its decomposed fleet: a way to resolve a named service to its
// live endpoints, and a way to watch that membership change over time. It
// names no registry. A NATS JetStream KV implementation lives in
// internal/infrastructure/servicediscovery/nats; the Kubernetes-native
// primary (Service/EndpointSlice + CoreDNS, per
// docs/inspiration/service-discovery-research-2026-09-05.md) is the sibling
// that serves the same contract.
//
// Per docs/architecture/dependencies-and-contracts.md, consumers depend on
// this package and never on an infrastructure implementation; only
// application composition chooses a concrete registry.
//
// # NotInstalled is not Down
//
// The single most important semantic in this package is the distinction
// between a service that was never installed and a service that is installed
// but currently has no healthy endpoint.
//
// - [ErrNotInstalled] reports that a service has never been installed. It
// is a distinct outcome, not a failure. An optional service (Curator
// today, others later) that is not present must resolve to "absent",
// never "broken": callers match it with errors.Is and treat it as "this
// capability is disabled", not "this capability is unhealthy".
// - A service that IS installed but has zero healthy endpoints right now is
// a normal, non-error outcome. [ServiceRegistry.Resolve] returns a
// possibly-empty slice with a nil error; [ServiceRegistry.Watch] returns
// a live channel that simply does not emit until an endpoint joins.
//
// This is why a registry implementation must model presence separately from
// liveness. The NATS implementation keeps a durable "installed" marker that
// never expires, distinct from expiring heartbeat keys: the marker's presence
// means NotInstalled=false, while heartbeat expiry is Down/Unhealthy.
package servicediscovery
@@ -0,0 +1,86 @@
package servicediscovery
import (
"context"
"errors"
)
// ErrNotInstalled reports that a service has never been installed. It is
// distinct from a service that is installed but currently has no healthy
// endpoint: that case is an empty, non-error result, not ErrNotInstalled.
// Implementations of [ServiceRegistry.Resolve] and [ServiceRegistry.Watch]
// return it (wrapped, matched with errors.Is) when a service's "installed"
// marker is absent. Callers match it and treat the capability as disabled.
var ErrNotInstalled = errors.New("servicediscovery: service not installed")
// Endpoint is one resolved instance of a service.
type Endpoint struct {
// Service is the logical name the endpoint was registered under. It is
// redundant within the scope of a single Resolve/Watch call, but it makes
// the value self-describing and directly usable by a consumer that reads
// raw registry records (e.g. a custom gRPC resolver) rather than going
// through ServiceRegistry.
Service string `json:"service,omitempty"`
// ID distinguishes multiple instances of the same service. It is the
// per-instance identifier; two endpoints with the same Service and ID are
// the same instance. An ID must not contain the registry's key separator
// (see the NATS implementation's key scheme).
ID string `json:"id"`
// Address is the dialable host:port for the instance.
Address string `json:"address"`
}
// EventKind is the kind of membership change an [Event] describes.
type EventKind int
const (
// Join reports that an endpoint became live.
Join EventKind = iota
// Leave reports that an endpoint left: it was unregistered, its heartbeat
// expired, or its process stopped.
Leave
)
// String returns a human-readable kind, for logging and tests.
func (k EventKind) String() string {
switch k {
case Join:
return "join"
case Leave:
return "leave"
default:
return "unknown"
}
}
// Event is a single membership change for one endpoint.
type Event struct {
// Endpoint is the instance that joined or left. On a [Leave] the Address
// may be empty: the registry observing an entry disappear can recover the
// instance's ID from the key but not the address it carried. The ID
// uniquely identifies the instance.
Endpoint Endpoint
// Kind is [Join] or [Leave].
Kind EventKind
}
// ServiceRegistry resolves and watches the live endpoints of a named service.
type ServiceRegistry interface {
// Resolve returns the current healthy endpoints for service. It returns
// [ErrNotInstalled] if service has never been installed. If service is
// installed but currently has no healthy endpoint, it returns a
// possibly-empty slice with a nil error -- callers distinguish that from
// NotInstalled to avoid treating an optional, disabled service as broken.
Resolve(ctx context.Context, service string) ([]Endpoint, error)
// Watch returns a channel that emits [Join] and [Leave] events as service's
// membership changes, until ctx is cancelled. It returns [ErrNotInstalled]
// if service has never been installed at the time of the call. A service
// that is installed but currently has no healthy endpoint returns a live
// channel that simply does not emit until an endpoint joins.
Watch(ctx context.Context, service string) (<-chan Event, error)
}
@@ -0,0 +1,39 @@
package servicediscovery
import (
"errors"
"fmt"
"testing"
)
// TestEventKindString pins the human-readable form of each kind, including the
// zero value (unset/unknown), so logging and diagnostics never render a magic
// number.
func TestEventKindString(t *testing.T) {
tests := []struct {
name string
kind EventKind
want string
}{
{name: "join", kind: Join, want: "join"},
{name: "leave", kind: Leave, want: "leave"},
{name: "unknown out-of-range", kind: EventKind(99), want: "unknown"},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
if got := tt.kind.String(); got != tt.want {
t.Errorf("EventKind(%d).String() = %q, want %q", tt.kind, got, tt.want)
}
})
}
}
// TestErrNotInstalledWraps pins that implementations can wrap the sentinel and
// callers can still match it with errors.Is, which is the contract's intended
// use (a broker or registry specific error may be joined around it).
func TestErrNotInstalledWraps(t *testing.T) {
wrapped := fmt.Errorf("resolve %s: %w", "curator", ErrNotInstalled)
if !errors.Is(wrapped, ErrNotInstalled) {
t.Fatalf("wrapped ErrNotInstalled does not match errors.Is: %v", wrapped)
}
}