From 7fd092ac7750df2405468917024a281dac9ade08 Mon Sep 17 00:00:00 2001 From: Sam Catlow Date: Sun, 4 Oct 2026 01:15:11 +1000 Subject: [PATCH] fix(services): services wait for peers instead of exiting; restarted broker keeps its endpoint State Store calls from the daemon, Gateway and Messaging wait for the store to come up, so start order no longer matters; the daemon's state_db probe carries its own deadline so an outage reports degraded instead of hanging. The Gateway retries its first NATS connect. A restarted daemon reuses the embedded broker's previous port and token, so the Gateway reconnects instead of holding a dead address. Fixes a nil catalog the catalog move introduced. --- internal/app/archied/bootstrap.go | 25 ++++++++++++++----- internal/app/archied/main.go | 2 +- internal/app/archied/readiness.go | 7 +++++- internal/app/archiegateway/server.go | 8 +++--- internal/app/archiemessaging/run.go | 3 ++- internal/app/servicekit/statestore.go | 11 ++++++-- .../infrastructure/eventbus/nats/embedded.go | 7 +++++- 7 files changed, 47 insertions(+), 16 deletions(-) diff --git a/internal/app/archied/bootstrap.go b/internal/app/archied/bootstrap.go index 62c8c2f7a..8e7295e28 100644 --- a/internal/app/archied/bootstrap.go +++ b/internal/app/archied/bootstrap.go @@ -12,10 +12,12 @@ import ( "fmt" "log/slog" "net/http" + "net/url" "os" "os/signal" "path/filepath" "slices" + "strconv" "strings" "sync/atomic" "syscall" @@ -233,6 +235,7 @@ func (b *boot) loadConfig(_ context.Context, cfgPath, overlayPath string) error return err } b.loader, b.doc = loader, doc + b.catalog = servicekit.NewCatalog(cfgPath) b.cfg = b.doc.Config b.cfgHolder = config.NewHolder(b.cfg) b.currentProvenance.Store(&b.doc.Provenance) @@ -439,10 +442,21 @@ func (b *boot) startEmbeddedNATS(ctx context.Context) (string, string, error) { if b.containerPool != nil { host = b.containerPool.HostGateway() } - srv, err := nats.StartEmbedded(ctx, nats.EmbeddedOptions{ - Host: host, - StoreDir: filepath.Join(cfg.StateDir, "nats"), - }, log) + // A restarted broker keeps the previous endpoint, so the Gateway and + // running agents reconnect to it instead of a broker that no longer exists. + opts := nats.EmbeddedOptions{Host: host, StoreDir: filepath.Join(cfg.StateDir, "nats")} + if previous, err := servicekit.ReadNATSEndpoint(cfg.StateDir); err == nil { + opts.Token = previous.Token + if u, err := url.Parse(previous.URL); err == nil { + opts.Port, _ = strconv.Atoi(u.Port()) + } + } + srv, err := nats.StartEmbedded(ctx, opts, log) + if err != nil && opts.Port != 0 { + log.Warn("embedded nats: previous port unavailable; taking a new one", "port", opts.Port, "err", err) + opts.Port = 0 + srv, err = nats.StartEmbedded(ctx, opts, log) + } if err != nil { log.Error("embedded nats start failed", "err", err) return "", "", err @@ -961,8 +975,7 @@ func (b *boot) catalogState() (modelcatalog.Snapshot, []string) { return b.catal // loadCatalog layers the model catalog under the file config. A catalog that // cannot be read leaves the configured providers and models in effect. -func (b *boot) loadCatalog(ctx context.Context, cfgPath string) { - b.catalog = servicekit.NewCatalog(cfgPath) +func (b *boot) loadCatalog(ctx context.Context) { snapshot, err := b.catalog.Fetch(ctx, b.secrets.Getenv, b.cfgHolder.Get().Providers) if err != nil { b.log.Warn("model catalog unavailable; using configured providers and models", "err", err) diff --git a/internal/app/archied/main.go b/internal/app/archied/main.go index 50c1c177a..2525a34d7 100644 --- a/internal/app/archied/main.go +++ b/internal/app/archied/main.go @@ -167,7 +167,7 @@ func Run() int { //nolint:cyclop,funlen // the composition root's setup sequence } else if exit { return 0 } - b.loadCatalog(ctx, args.cfgPath) + b.loadCatalog(ctx) b.seedSoul(args.cfgPath) b.setupObservability(ctx) diff --git a/internal/app/archied/readiness.go b/internal/app/archied/readiness.go index 0c710fe85..0a393956d 100644 --- a/internal/app/archied/readiness.go +++ b/internal/app/archied/readiness.go @@ -28,7 +28,12 @@ import ( func (b *boot) setupReadinessProbes() { cfg := b.cfg probes := []health.Probe{ - readiness.NewStoreProbe(b.stateStore), + // The State Store client waits for its peer, so the probe carries its + // own deadline to report an outage instead of waiting it out. + readiness.NewContractProbe("state_db", time.Duration(cfg.Health.DependencyTimeout), func(ctx context.Context) error { + _, err := b.stateStore.StatusCounts(ctx) + return err + }), readiness.NewConfigProbe(b.cfgHolder.Get, configuration.Validate), readiness.NewDiskProbeTargets(diskProbeTargets(cfg)), // The daemon no longer runs a chat channel, so it cannot observe one's diff --git a/internal/app/archiegateway/server.go b/internal/app/archiegateway/server.go index 758b5cc84..4aa02902a 100644 --- a/internal/app/archiegateway/server.go +++ b/internal/app/archiegateway/server.go @@ -104,7 +104,7 @@ func Run(ctx context.Context, options Options) error { if err := b.openState(ctx); err != nil { return err } - b.loadCatalog(ctx, options.Config) + b.loadCatalog(ctx) nc, err := b.connectNATS(ctx) if err != nil { return err @@ -151,6 +151,7 @@ func (b *server) loadConfig(cfgPath, overlayPath string) error { return err } b.cfg = doc.Config + b.catalog = servicekit.NewCatalog(cfgPath) b.cfgHolder = config.NewHolder(b.cfg) logs := servicekit.Logging(b.cfg, "gateway") b.log, b.taskLogs = logs.Log, logs.TaskLogs @@ -212,7 +213,7 @@ func (b *server) connectNATS(ctx context.Context) (*natsio.Conn, error) { } url, token = endpoint.URL, endpoint.Token } - nc, err := natsio.Connect(url, natsio.Token(token), natsio.MaxReconnects(-1)) + nc, err := natsio.Connect(url, natsio.Token(token), natsio.MaxReconnects(-1), natsio.RetryOnFailedConnect(true)) if err != nil { return nil, fmt.Errorf("connect gateway task actions: %w", err) } @@ -238,8 +239,7 @@ func (b *server) catalogState() (modelcatalog.Snapshot, []string) { return b.cat // loadCatalog layers the model catalog under the running config. A catalog // that cannot be read leaves the configured providers and models in effect. -func (b *server) loadCatalog(ctx context.Context, cfgPath string) { - b.catalog = servicekit.NewCatalog(cfgPath) +func (b *server) loadCatalog(ctx context.Context) { snapshot, err := b.catalog.Fetch(ctx, b.secrets.Getenv, b.cfg.Providers) if err != nil { b.log.Warn("model catalog unavailable; using configured providers and models", "err", err) diff --git a/internal/app/archiemessaging/run.go b/internal/app/archiemessaging/run.go index 49833381f..d00381237 100644 --- a/internal/app/archiemessaging/run.go +++ b/internal/app/archiemessaging/run.go @@ -5,6 +5,7 @@ import ( "fmt" "log/slog" + "github.com/samcharles93/archie-core/internal/app/servicekit" "github.com/samcharles93/archie-core/internal/config" "github.com/samcharles93/archie-core/internal/domain/applystatus" "github.com/samcharles93/archie-core/internal/domain/messaging" @@ -46,7 +47,7 @@ func Run(ctx context.Context, o Options) error { var applyReporter *applystatus.Reporter var appliedVersion int64 if cfg.Options.StateStore.Target != "" { - stateStore, closeClient, dialErr := staterpc.Dial(cfg.Options.StateStore.Target, cfg.Options.StateStore.Token) + stateStore, closeClient, dialErr := staterpc.Dial(cfg.Options.StateStore.Target, cfg.Options.StateStore.Token, servicekit.WaitForPeer) if dialErr != nil { return fmt.Errorf("dial state store control plane (%s): %w", cfg.Options.StateStore.Target, dialErr) } diff --git a/internal/app/servicekit/statestore.go b/internal/app/servicekit/statestore.go index 4c9805b40..ab23f2fc8 100644 --- a/internal/app/servicekit/statestore.go +++ b/internal/app/servicekit/statestore.go @@ -4,17 +4,24 @@ import ( "github.com/samcharles93/archie-core/internal/config" "github.com/samcharles93/archie-core/internal/infrastructure/staterpc" "github.com/samcharles93/archie-core/internal/secret" + "google.golang.org/grpc" ) // StateStoreClient dials the State Store gRPC service the daemon's own // capture/mapping/binding consumers call. Token resolution is the daemon's // (it owns the config keys and the secret registry); the dial itself and the // fail-closed non-loopback rule belong to the transport, so they come -// from staterpc.Dial. +// from staterpc.Dial. Calls wait for the State Store rather than fail while it +// is down, so boot order between services does not matter. func StateStoreClient(services config.Services, secrets *secret.Registry) (*staterpc.Client, func(), error) { target, err := services.RequireTarget(config.ServiceNameState) if err != nil { return nil, nil, err } - return staterpc.Dial(target, services.ResolvedToken(config.ServiceNameState, secrets.Getenv)) + return staterpc.Dial(target, services.ResolvedToken(config.ServiceNameState, secrets.Getenv), WaitForPeer) } + +// WaitForPeer makes a service's calls to another service wait for it to come +// up instead of failing while it is down or restarting. A call that must not +// wait carries its own deadline. +var WaitForPeer = grpc.WithDefaultCallOptions(grpc.WaitForReady(true)) diff --git a/internal/infrastructure/eventbus/nats/embedded.go b/internal/infrastructure/eventbus/nats/embedded.go index 0f849dfaf..307e68e92 100644 --- a/internal/infrastructure/eventbus/nats/embedded.go +++ b/internal/infrastructure/eventbus/nats/embedded.go @@ -26,6 +26,8 @@ type EmbeddedOptions struct { Host string // Port is the listen port. Zero means a random port. Port int + // Token is the client token. Empty generates one. + Token string // StoreDir is where JetStream persists streams. Empty means a temporary // directory (streams do not survive a reboot). The daemon passes a dir // under its data directory so the ARCHIE_TASKS and reaction streams are @@ -46,7 +48,10 @@ func StartEmbedded(ctx context.Context, opts EmbeddedOptions, log *slog.Logger) if log == nil { log = slog.New(slog.DiscardHandler) } - token := rand.Text() + token := opts.Token + if token == "" { + token = rand.Text() + } srv, err := server.NewServer(&server.Options{ Host: opts.Host,