fix(services): services wait for peers instead of exiting; restarted broker keeps its endpoint

State Store calls from the daemon, Gateway and Messaging wait for the store to
come up, so start order no longer matters; the daemon's state_db probe carries
its own deadline so an outage reports degraded instead of hanging. The
Gateway retries its first NATS connect. A restarted daemon reuses the embedded
broker's previous port and token, so the Gateway reconnects instead of
holding a dead address. Fixes a nil catalog the catalog move introduced.
This commit is contained in:
sam committed 2026-10-04 01:15:11 +10:00
1 parent cd3c317af0
commit 7fd092ac77
7 files changed
+47 -16

No files matched your search

+19 -6
View File
@@ -12,10 +12,12 @@ import (
"fmt"
"log/slog"
"net/http"
"net/url"
"os"
"os/signal"
"path/filepath"
"slices"
"strconv"
"strings"
"sync/atomic"
"syscall"
@@ -233,6 +235,7 @@ func (b *boot) loadConfig(_ context.Context, cfgPath, overlayPath string) error
return err
}
b.loader, b.doc = loader, doc
b.catalog = servicekit.NewCatalog(cfgPath)
b.cfg = b.doc.Config
b.cfgHolder = config.NewHolder(b.cfg)
b.currentProvenance.Store(&b.doc.Provenance)
@@ -439,10 +442,21 @@ func (b *boot) startEmbeddedNATS(ctx context.Context) (string, string, error) {
if b.containerPool != nil {
host = b.containerPool.HostGateway()
}
srv, err := nats.StartEmbedded(ctx, nats.EmbeddedOptions{
Host: host,
StoreDir: filepath.Join(cfg.StateDir, "nats"),
}, log)
// A restarted broker keeps the previous endpoint, so the Gateway and
// running agents reconnect to it instead of a broker that no longer exists.
opts := nats.EmbeddedOptions{Host: host, StoreDir: filepath.Join(cfg.StateDir, "nats")}
if previous, err := servicekit.ReadNATSEndpoint(cfg.StateDir); err == nil {
opts.Token = previous.Token
if u, err := url.Parse(previous.URL); err == nil {
opts.Port, _ = strconv.Atoi(u.Port())
}
}
srv, err := nats.StartEmbedded(ctx, opts, log)
if err != nil && opts.Port != 0 {
log.Warn("embedded nats: previous port unavailable; taking a new one", "port", opts.Port, "err", err)
opts.Port = 0
srv, err = nats.StartEmbedded(ctx, opts, log)
}
if err != nil {
log.Error("embedded nats start failed", "err", err)
return "", "", err
@@ -961,8 +975,7 @@ func (b *boot) catalogState() (modelcatalog.Snapshot, []string) { return b.catal
// loadCatalog layers the model catalog under the file config. A catalog that
// cannot be read leaves the configured providers and models in effect.
func (b *boot) loadCatalog(ctx context.Context, cfgPath string) {
b.catalog = servicekit.NewCatalog(cfgPath)
func (b *boot) loadCatalog(ctx context.Context) {
snapshot, err := b.catalog.Fetch(ctx, b.secrets.Getenv, b.cfgHolder.Get().Providers)
if err != nil {
b.log.Warn("model catalog unavailable; using configured providers and models", "err", err)
+1 -1
View File
@@ -167,7 +167,7 @@ func Run() int { //nolint:cyclop,funlen // the composition root's setup sequence
} else if exit {
return 0
}
b.loadCatalog(ctx, args.cfgPath)
b.loadCatalog(ctx)
b.seedSoul(args.cfgPath)
b.setupObservability(ctx)
+6 -1
View File
@@ -28,7 +28,12 @@ import (
func (b *boot) setupReadinessProbes() {
cfg := b.cfg
probes := []health.Probe{
readiness.NewStoreProbe(b.stateStore),
// The State Store client waits for its peer, so the probe carries its
// own deadline to report an outage instead of waiting it out.
readiness.NewContractProbe("state_db", time.Duration(cfg.Health.DependencyTimeout), func(ctx context.Context) error {
_, err := b.stateStore.StatusCounts(ctx)
return err
}),
readiness.NewConfigProbe(b.cfgHolder.Get, configuration.Validate),
readiness.NewDiskProbeTargets(diskProbeTargets(cfg)),
// The daemon no longer runs a chat channel, so it cannot observe one's
+4 -4
View File
@@ -104,7 +104,7 @@ func Run(ctx context.Context, options Options) error {
if err := b.openState(ctx); err != nil {
return err
}
b.loadCatalog(ctx, options.Config)
b.loadCatalog(ctx)
nc, err := b.connectNATS(ctx)
if err != nil {
return err
@@ -151,6 +151,7 @@ func (b *server) loadConfig(cfgPath, overlayPath string) error {
return err
}
b.cfg = doc.Config
b.catalog = servicekit.NewCatalog(cfgPath)
b.cfgHolder = config.NewHolder(b.cfg)
logs := servicekit.Logging(b.cfg, "gateway")
b.log, b.taskLogs = logs.Log, logs.TaskLogs
@@ -212,7 +213,7 @@ func (b *server) connectNATS(ctx context.Context) (*natsio.Conn, error) {
}
url, token = endpoint.URL, endpoint.Token
}
nc, err := natsio.Connect(url, natsio.Token(token), natsio.MaxReconnects(-1))
nc, err := natsio.Connect(url, natsio.Token(token), natsio.MaxReconnects(-1), natsio.RetryOnFailedConnect(true))
if err != nil {
return nil, fmt.Errorf("connect gateway task actions: %w", err)
}
@@ -238,8 +239,7 @@ func (b *server) catalogState() (modelcatalog.Snapshot, []string) { return b.cat
// loadCatalog layers the model catalog under the running config. A catalog
// that cannot be read leaves the configured providers and models in effect.
func (b *server) loadCatalog(ctx context.Context, cfgPath string) {
b.catalog = servicekit.NewCatalog(cfgPath)
func (b *server) loadCatalog(ctx context.Context) {
snapshot, err := b.catalog.Fetch(ctx, b.secrets.Getenv, b.cfg.Providers)
if err != nil {
b.log.Warn("model catalog unavailable; using configured providers and models", "err", err)
+2 -1
View File
@@ -5,6 +5,7 @@ import (
"fmt"
"log/slog"
"github.com/samcharles93/archie-core/internal/app/servicekit"
"github.com/samcharles93/archie-core/internal/config"
"github.com/samcharles93/archie-core/internal/domain/applystatus"
"github.com/samcharles93/archie-core/internal/domain/messaging"
@@ -46,7 +47,7 @@ func Run(ctx context.Context, o Options) error {
var applyReporter *applystatus.Reporter
var appliedVersion int64
if cfg.Options.StateStore.Target != "" {
stateStore, closeClient, dialErr := staterpc.Dial(cfg.Options.StateStore.Target, cfg.Options.StateStore.Token)
stateStore, closeClient, dialErr := staterpc.Dial(cfg.Options.StateStore.Target, cfg.Options.StateStore.Token, servicekit.WaitForPeer)
if dialErr != nil {
return fmt.Errorf("dial state store control plane (%s): %w", cfg.Options.StateStore.Target, dialErr)
}
+9 -2
View File
@@ -4,17 +4,24 @@ import (
"github.com/samcharles93/archie-core/internal/config"
"github.com/samcharles93/archie-core/internal/infrastructure/staterpc"
"github.com/samcharles93/archie-core/internal/secret"
"google.golang.org/grpc"
)
// StateStoreClient dials the State Store gRPC service the daemon's own
// capture/mapping/binding consumers call. Token resolution is the daemon's
// (it owns the config keys and the secret registry); the dial itself and the
// fail-closed non-loopback rule belong to the transport, so they come
// from staterpc.Dial.
// from staterpc.Dial. Calls wait for the State Store rather than fail while it
// is down, so boot order between services does not matter.
func StateStoreClient(services config.Services, secrets *secret.Registry) (*staterpc.Client, func(), error) {
target, err := services.RequireTarget(config.ServiceNameState)
if err != nil {
return nil, nil, err
}
return staterpc.Dial(target, services.ResolvedToken(config.ServiceNameState, secrets.Getenv))
return staterpc.Dial(target, services.ResolvedToken(config.ServiceNameState, secrets.Getenv), WaitForPeer)
}
// WaitForPeer makes a service's calls to another service wait for it to come
// up instead of failing while it is down or restarting. A call that must not
// wait carries its own deadline.
var WaitForPeer = grpc.WithDefaultCallOptions(grpc.WaitForReady(true))
@@ -26,6 +26,8 @@ type EmbeddedOptions struct {
Host string
// Port is the listen port. Zero means a random port.
Port int
// Token is the client token. Empty generates one.
Token string
// StoreDir is where JetStream persists streams. Empty means a temporary
// directory (streams do not survive a reboot). The daemon passes a dir
// under its data directory so the ARCHIE_TASKS and reaction streams are
@@ -46,7 +48,10 @@ func StartEmbedded(ctx context.Context, opts EmbeddedOptions, log *slog.Logger)
if log == nil {
log = slog.New(slog.DiscardHandler)
}
token := rand.Text()
token := opts.Token
if token == "" {
token = rand.Text()
}
srv, err := server.NewServer(&server.Options{
Host: opts.Host,