Merge branch 'work/53jc'

This commit is contained in:
sam committed 2026-10-11 04:11:43 +11:00
commit 7d1f2d73ea
9 files changed
-656

No files matched your search

@@ -1,84 +0,0 @@
package nats
import (
"fmt"
"time"
)
// Defaults for Config. Previously these would have been unexported package
// constants, which made bucket naming and timeouts untunable per deployment.
const (
// DefaultBucket is the heartbeat bucket: one live endpoint per instance.
DefaultBucket = "ARCHIE_SERVICE_REGISTRY"
// DefaultInstalledBucket is the durable "installed" bucket: one marker per
// service that has ever registered, never expiring.
DefaultInstalledBucket = "ARCHIE_SERVICE_INSTALLED"
// DefaultTTL is how long a heartbeat entry survives without being
// refreshed. It must comfortably exceed DefaultHeartbeat so a transient
// pause in a live service's refresh does not drop it.
DefaultTTL = 15 * time.Second
// DefaultHeartbeat is the interval at which a registrar refreshes its
// endpoint entry.
DefaultHeartbeat = 5 * time.Second
)
// Config describes how to reach NATS and how the registry buckets behave.
// The zero value is not usable; call [Config.withDefaults] via [Connect], which
// fills every unset field.
type Config struct {
// URL is the NATS server address. Required.
URL string
// Token authenticates the connection. Empty means no token auth.
Token string
// Bucket is the heartbeat bucket name.
Bucket string
// InstalledBucket is the durable "installed" bucket name.
InstalledBucket string
// TTL is how long a heartbeat entry survives without being refreshed. It
// is the difference between a service that is installed-but-down (its
// entries expired) and one that is live (its entries are fresh).
TTL time.Duration
// Heartbeat is the interval at which a registrar refreshes its entry. It
// must be less than TTL, or a live service would expire between refreshes.
Heartbeat time.Duration
}
// Validate reports whether the config can produce a usable client.
func (c Config) Validate() error {
if c.URL == "" {
return fmt.Errorf("%w: URL is required", ErrInvalidConfig)
}
if c.TTL < 0 || c.Heartbeat < 0 {
return fmt.Errorf("%w: durations must not be negative", ErrInvalidConfig)
}
if c.Heartbeat > 0 && c.TTL > 0 && c.Heartbeat >= c.TTL {
return fmt.Errorf("%w: Heartbeat must be less than TTL", ErrInvalidConfig)
}
return nil
}
// withDefaults returns a copy with every unset field populated, so the rest of
// the package never has to test for zero values.
func (c Config) withDefaults() Config {
if c.Bucket == "" {
c.Bucket = DefaultBucket
}
if c.InstalledBucket == "" {
c.InstalledBucket = DefaultInstalledBucket
}
if c.TTL == 0 {
c.TTL = DefaultTTL
}
if c.Heartbeat == 0 {
c.Heartbeat = DefaultHeartbeat
}
return c
}
@@ -1,8 +0,0 @@
// Package nats implements service discovery on NATS JetStream KV.
//
// A heartbeat bucket holds one TTL'd key per live instance,
// "<service>.<instance-id>". An installed bucket holds one permanent marker
// per service, "<service>". A service with a marker and no live instances is
// installed with no endpoints; one with no marker is not installed. Names
// and IDs must not contain dots.
package nats
@@ -1,6 +0,0 @@
package nats
import "errors"
// ErrInvalidConfig reports a Config that cannot produce a usable client.
var ErrInvalidConfig = errors.New("nats: invalid configuration")
@@ -1,27 +0,0 @@
package nats
import "strings"
// keySeparator joins a service and an instance ID in a registry key.
const keySeparator = "."
// endpointKey is the heartbeat bucket key for one live instance.
func endpointKey(service, id string) string {
return service + keySeparator + id
}
// endpointKeyPrefix is the watch/list filter matching every instance of a
// service. It is a single-token wildcard over the instance ID.
func endpointKeyPrefix(service string) string {
return service + keySeparator + "*"
}
// idFromEndpointKey returns the instance ID carried by a registry key, or "" if
// the key does not belong to the given service (or carries an empty ID).
func idFromEndpointKey(service, key string) string {
prefix := service + keySeparator
if !strings.HasPrefix(key, prefix) {
return ""
}
return strings.TrimPrefix(key, prefix)
}
@@ -1,140 +0,0 @@
package nats
import (
"context"
"errors"
"fmt"
"log/slog"
"strings"
"sync"
"time"
"github.com/nats-io/nats.go/jetstream"
"github.com/samcharles93/archie-core/internal/servicediscovery"
)
// Register writes service's installed marker if missing and heartbeats ep
// until it is unregistered or ctx ends. ep.ID and service must not contain
// the key separator.
func (c *Client) Register(ctx context.Context, service string, ep servicediscovery.Endpoint) (*Registration, error) {
if ep.ID == "" {
return nil, fmt.Errorf("register %s: %w: endpoint ID is required", service, ErrInvalidConfig)
}
if strings.Contains(ep.ID, keySeparator) || strings.Contains(service, keySeparator) {
return nil, fmt.Errorf("register %s: %w: service and instance ID must not contain %q", service, ErrInvalidConfig, keySeparator)
}
if err := c.ensureInstalled(ctx, service); err != nil {
return nil, err
}
key := endpointKey(service, ep.ID)
value, err := encodeEndpoint(ep)
if err != nil {
return nil, err
}
if _, err := c.kv.Put(ctx, key, value); err != nil {
return nil, fmt.Errorf("register %s/%s: %w", service, ep.ID, err)
}
hbCtx, cancel := context.WithCancel(ctx)
reg := &Registration{
service: service,
key: key,
value: value,
kv: c.kv,
heartbeat: c.cfg.Heartbeat,
log: c.log,
cancel: cancel,
done: make(chan struct{}),
}
go reg.heartbeatLoop(hbCtx)
return reg, nil
}
// ensureInstalled writes the service's installed marker if it is absent. The
// marker has no TTL, so it persists for the life of the install; a service that
// registered once but has no live instance is still installed (and resolves to
// an empty, non-error roster), not NotInstalled.
func (c *Client) ensureInstalled(ctx context.Context, service string) error {
if _, err := c.installed.Create(ctx, service, installedMarkerValue); err != nil {
if errors.Is(err, jetstream.ErrKeyExists) {
return nil
}
return fmt.Errorf("mark %s installed: %w", service, err)
}
return nil
}
// Registration is one live announcement of a service instance. It owns the
// background heartbeat that keeps the instance's entry fresh; stopping it is
// the caller's responsibility via Unregister (or by cancelling the ctx passed
// to Register, which stops the heartbeat and lets the entry expire).
type Registration struct {
service string
key string
value []byte
kv jetstream.KeyValue
heartbeat time.Duration
log *slog.Logger
cancel context.CancelFunc
done chan struct{}
once sync.Once
}
// heartbeatLoop refreshes the instance's live entry on an interval until ctx is
// cancelled. It never stops on a single refresh failure -- a transient broker
// blip must not unregister a healthy instance -- only on cancellation.
func (rg *Registration) heartbeatLoop(ctx context.Context) {
defer close(rg.done)
ticker := time.NewTicker(rg.heartbeat)
defer ticker.Stop()
for {
select {
case <-ctx.Done():
return
case <-ticker.C:
if err := rg.refresh(ctx); err != nil {
if ctx.Err() != nil {
return
}
rg.log.Warn("heartbeat refresh failed", "service", rg.service, "key", rg.key, "error", err)
}
}
}
}
// refresh re-publishes the instance's entry, resetting its bucket-TTL window so
// it stays live.
func (rg *Registration) refresh(ctx context.Context) error {
if _, err := rg.kv.Put(ctx, rg.key, rg.value); err != nil {
return fmt.Errorf("refresh %s: %w", rg.key, err)
}
return nil
}
// Unregister removes the instance's live entry and stops its heartbeat. It is
// idempotent: a second call is a no-op that returns nil. The service's durable
// installed marker is left in place, so the service remains "installed" but
// down until an explicit uninstall.
func (rg *Registration) Unregister(ctx context.Context) error {
var unregErr error
rg.once.Do(func() {
// Stop the heartbeat first and wait for it to finish, so no in-flight
// refresh can re-create the entry after we delete it.
rg.cancel()
<-rg.done
if err := rg.kv.Delete(ctx, rg.key); err != nil {
// An already-expired entry is a normal leave, not a failure.
if !errors.Is(err, jetstream.ErrKeyNotFound) && !errors.Is(err, jetstream.ErrKeyDeleted) {
unregErr = err
}
}
})
if unregErr != nil {
return fmt.Errorf("unregister %s: %w", rg.key, unregErr)
}
return nil
}
@@ -1,283 +0,0 @@
package nats
import (
"context"
"errors"
"fmt"
"log/slog"
"slices"
"strings"
"time"
"github.com/nats-io/nats.go"
"github.com/nats-io/nats.go/jetstream"
"github.com/samcharles93/archie-core/internal/servicediscovery"
)
// Compile-time proof that Client satisfies the broker-neutral contract. The
// interface lives in internal/servicediscovery so domains can depend on the
// behaviour without importing this package.
var _ servicediscovery.ServiceRegistry = (*Client)(nil)
// installedMarkerValue is the value stored at an installed-bucket key. Only its
// presence matters; the value exists so the bucket is self-describing.
var installedMarkerValue = []byte("installed")
// Client implements servicediscovery.ServiceRegistry and registration over
// the heartbeat and installed buckets.
type Client struct {
conn *nats.Conn
kv jetstream.KeyValue // heartbeat bucket
installed jetstream.KeyValue // installed bucket
cfg Config
log *slog.Logger
}
// Connect dials NATS and creates the heartbeat bucket (with TTL) and the
// installed bucket (without). On failure it closes the connection.
func Connect(ctx context.Context, cfg Config, log *slog.Logger) (*Client, error) {
cfg = cfg.withDefaults()
if err := cfg.Validate(); err != nil {
return nil, err
}
if log == nil {
log = slog.New(slog.DiscardHandler)
}
conn, err := dial(cfg)
if err != nil {
return nil, err
}
// Every failure past this point must close conn. Cleared on success.
closeOnFail := conn.Close
defer func() {
if closeOnFail != nil {
closeOnFail()
}
}()
js, err := jetstream.New(conn)
if err != nil {
return nil, fmt.Errorf("nats jetstream init: %w", err)
}
kv, err := js.CreateOrUpdateKeyValue(ctx, jetstream.KeyValueConfig{
Bucket: cfg.Bucket,
History: 1,
TTL: cfg.TTL,
LimitMarkerTTL: markerTTL(cfg.TTL),
})
if err != nil {
return nil, fmt.Errorf("nats create registry bucket %q: %w", cfg.Bucket, err)
}
installed, err := js.CreateOrUpdateKeyValue(ctx, jetstream.KeyValueConfig{
Bucket: cfg.InstalledBucket,
History: 1,
})
if err != nil {
return nil, fmt.Errorf("nats create installed bucket %q: %w", cfg.InstalledBucket, err)
}
closeOnFail = nil
log.Info("service registry connected",
"url", conn.ConnectedUrl(),
"bucket", cfg.Bucket,
"installed_bucket", cfg.InstalledBucket)
return &Client{
conn: conn,
kv: kv,
installed: installed,
cfg: cfg,
log: log,
}, nil
}
// markerTTL returns the subject-delete-marker TTL for the registry bucket. A
// heartbeat expiry is surfaced to watchers as a delete marker, but NATS rejects
// a marker TTL below one second; clamp the configured TTL up to that floor so a
// fast-expiring bucket still lets watchers observe the leave.
func markerTTL(ttl time.Duration) time.Duration {
if ttl < time.Second {
return time.Second
}
return ttl
}
// dial opens the raw connection, applying auth options from cfg.
func dial(cfg Config) (*nats.Conn, error) {
var opts []nats.Option
if cfg.Token != "" {
opts = append(opts, nats.Token(cfg.Token))
}
conn, err := nats.Connect(cfg.URL, opts...)
if err != nil {
return nil, fmt.Errorf("nats connect %s: %w", cfg.URL, err)
}
return conn, nil
}
// Close shuts the connection down. It is safe to call on a nil client so
// callers holding an optional registry can defer it unconditionally.
func (c *Client) Close() {
if c == nil || c.conn == nil {
return
}
c.conn.Close()
c.log.Debug("service registry connection closed")
}
// Resolve returns service's live endpoints, possibly none. It returns
// servicediscovery.ErrNotInstalled when the service has no marker.
func (c *Client) Resolve(ctx context.Context, service string) ([]servicediscovery.Endpoint, error) {
if err := c.requireInstalled(ctx, service); err != nil {
return nil, err
}
lister, err := c.kv.ListKeysFiltered(ctx, endpointKeyPrefix(service))
if err != nil {
return nil, fmt.Errorf("resolve %s: %w", service, err)
}
defer func() { _ = lister.Stop() }()
// A lister may report duplicate keys; dedupe by ID and sort for a stable
// order so callers get a deterministic roster.
byID := map[string]servicediscovery.Endpoint{}
for key := range lister.Keys() {
entry, err := c.kv.Get(ctx, key)
if err != nil {
// A key may expire between listing and Get; that is a healthy
// leave, not a failure.
if errors.Is(err, jetstream.ErrKeyNotFound) || errors.Is(err, jetstream.ErrKeyDeleted) {
continue
}
return nil, fmt.Errorf("resolve %s: %w", service, err)
}
if entry.Operation() == jetstream.KeyValueDelete || entry.Operation() == jetstream.KeyValuePurge {
continue
}
ep, ok := decodeEndpoint(entry.Value())
if !ok {
c.log.Warn("discarding undecodable endpoint", "service", service, "key", key)
continue
}
ep.Service = service
byID[ep.ID] = ep
}
eps := make([]servicediscovery.Endpoint, 0, len(byID))
for _, ep := range byID {
eps = append(eps, ep)
}
slices.SortFunc(eps, func(a, b servicediscovery.Endpoint) int {
return strings.Compare(a.ID, b.ID)
})
return eps, nil
}
// Watch emits Join and Leave events for service until ctx ends, then closes
// the channel. It returns servicediscovery.ErrNotInstalled when the service
// has no marker.
func (c *Client) Watch(ctx context.Context, service string) (<-chan servicediscovery.Event, error) {
if err := c.requireInstalled(ctx, service); err != nil {
return nil, err
}
watcher, err := c.kv.Watch(ctx, endpointKeyPrefix(service))
if err != nil {
return nil, fmt.Errorf("watch %s: %w", service, err)
}
out := make(chan servicediscovery.Event)
go c.fanWatch(ctx, service, watcher, out)
return out, nil
}
// fanWatch converts KV updates to Join and Leave events until ctx ends,
// ignoring heartbeat refreshes.
func (c *Client) fanWatch(ctx context.Context, service string, watcher jetstream.KeyWatcher, out chan<- servicediscovery.Event) {
defer close(out)
defer func() { _ = watcher.Stop() }()
known := map[string]struct{}{}
for {
select {
case <-ctx.Done():
return
case entry, ok := <-watcher.Updates():
if !ok {
return
}
// A nil entry marks the end of the initial snapshot; membership
// events continue from there.
if entry == nil {
continue
}
ev, ok := eventFromEntry(service, entry)
if !ok {
c.log.Warn("discarding unrecognised registry entry",
"service", service, "key", entry.Key(), "op", entry.Operation())
continue
}
id := ev.Endpoint.ID
switch ev.Kind {
case servicediscovery.Join:
if _, seen := known[id]; seen {
continue
}
known[id] = struct{}{}
case servicediscovery.Leave:
if _, seen := known[id]; !seen {
continue
}
delete(known, id)
}
select {
case out <- ev:
case <-ctx.Done():
return
}
}
}
}
// requireInstalled checks the installed marker for service, translating absence
// into the contract's ErrNotInstalled so the NotInstalled/Down distinction is
// expressed by both Resolve and Watch.
func (c *Client) requireInstalled(ctx context.Context, service string) error {
if _, err := c.installed.Get(ctx, service); err != nil {
if errors.Is(err, jetstream.ErrKeyNotFound) {
return fmt.Errorf("%s: %w", service, servicediscovery.ErrNotInstalled)
}
return fmt.Errorf("%s: %w", service, err)
}
return nil
}
// eventFromEntry converts a KV entry to an event: a Put is a Join, a delete
// or expiry a Leave. It reports false for an entry it cannot read.
func eventFromEntry(service string, entry jetstream.KeyValueEntry) (servicediscovery.Event, bool) {
switch entry.Operation() {
case jetstream.KeyValuePut:
ep, ok := decodeEndpoint(entry.Value())
if !ok {
return servicediscovery.Event{}, false
}
ep.Service = service
return servicediscovery.Event{Kind: servicediscovery.Join, Endpoint: ep}, true
case jetstream.KeyValueDelete, jetstream.KeyValuePurge:
id := idFromEndpointKey(service, entry.Key())
if id == "" {
return servicediscovery.Event{}, false
}
return servicediscovery.Event{
Kind: servicediscovery.Leave,
Endpoint: servicediscovery.Endpoint{Service: service, ID: id},
}, true
default:
return servicediscovery.Event{}, false
}
}
@@ -1,29 +0,0 @@
package nats
import (
"encoding/json"
"fmt"
"github.com/samcharles93/archie-core/internal/servicediscovery"
)
// endpointJSON is the on-disk (KV value) encoding of a registered endpoint. It
// is deliberately the contract type so a consumer reading raw records gets the
// same shape ServiceRegistry hands back.
func encodeEndpoint(ep servicediscovery.Endpoint) ([]byte, error) {
b, err := json.Marshal(ep)
if err != nil {
return nil, fmt.Errorf("encode endpoint %s/%s: %w", ep.Service, ep.ID, err)
}
return b, nil
}
// decodeEndpoint parses a KV value back into an Endpoint. It reports false for
// a value it cannot parse.
func decodeEndpoint(value []byte) (servicediscovery.Endpoint, bool) {
var ep servicediscovery.Endpoint
if err := json.Unmarshal(value, &ep); err != nil {
return servicediscovery.Endpoint{}, false
}
return ep, true
}
-5
View File
@@ -1,5 +0,0 @@
// Package servicediscovery is the broker-neutral service registry contract:
// resolve a service's live endpoints and watch membership changes. A service
// never installed returns ErrNotInstalled; an installed service with no live
// endpoints returns no endpoints and no error.
package servicediscovery
@@ -1,74 +0,0 @@
package servicediscovery
import (
"context"
"errors"
)
// ErrNotInstalled reports that a service was never installed. Callers treat
// the capability as disabled.
var ErrNotInstalled = errors.New("servicediscovery: service not installed")
// Endpoint is one resolved instance of a service.
type Endpoint struct {
// Service is the name the endpoint registered under.
Service string `json:"service,omitempty"`
// ID distinguishes multiple instances of the same service. It is the
// per-instance identifier; two endpoints with the same Service and ID are
// the same instance. An ID must not contain the registry's key separator
// (see the NATS implementation's key scheme).
ID string `json:"id"`
// Address is the dialable host:port for the instance.
Address string `json:"address"`
}
// EventKind is the kind of membership change an [Event] describes.
type EventKind int
const (
// Join reports that an endpoint became live.
Join EventKind = iota
// Leave reports that an endpoint left: it was unregistered, its heartbeat
// expired, or its process stopped.
Leave
)
// String returns a human-readable kind, for logging and tests.
func (k EventKind) String() string {
switch k {
case Join:
return "join"
case Leave:
return "leave"
default:
return "unknown"
}
}
// Event is a single membership change for one endpoint.
type Event struct {
// Endpoint is the instance that joined or left. On a [Leave] the Address
// may be empty: the registry observing an entry disappear can recover the
// instance's ID from the key but not the address it carried. The ID
// uniquely identifies the instance.
Endpoint Endpoint
// Kind is [Join] or [Leave].
Kind EventKind
}
// ServiceRegistry resolves and watches the live endpoints of a named service.
// It models network endpoints only. In-process services do not register here;
// application composition selects local adapters independently of discovery.
type ServiceRegistry interface {
// Resolve returns service's live endpoints, possibly none, or
// ErrNotInstalled.
Resolve(ctx context.Context, service string) ([]Endpoint, error)
// Watch emits Join and Leave events for service until ctx ends, or returns
// ErrNotInstalled.
Watch(ctx context.Context, service string) (<-chan Event, error)
}