fix(services): services wait for peers instead of exiting; restarted broker keeps its endpoint
State Store calls from the daemon, Gateway and Messaging wait for the store to come up, so start order no longer matters; the daemon's state_db probe carries its own deadline so an outage reports degraded instead of hanging. The Gateway retries its first NATS connect. A restarted daemon reuses the embedded broker's previous port and token, so the Gateway reconnects instead of holding a dead address. Fixes a nil catalog the catalog move introduced.
This commit is contained in:
7 files changed
+47
-16
No files matched your search
@@ -12,10 +12,12 @@ import (
|
||||
"fmt"
|
||||
"log/slog"
|
||||
"net/http"
|
||||
"net/url"
|
||||
"os"
|
||||
"os/signal"
|
||||
"path/filepath"
|
||||
"slices"
|
||||
"strconv"
|
||||
"strings"
|
||||
"sync/atomic"
|
||||
"syscall"
|
||||
@@ -233,6 +235,7 @@ func (b *boot) loadConfig(_ context.Context, cfgPath, overlayPath string) error
|
||||
return err
|
||||
}
|
||||
b.loader, b.doc = loader, doc
|
||||
b.catalog = servicekit.NewCatalog(cfgPath)
|
||||
b.cfg = b.doc.Config
|
||||
b.cfgHolder = config.NewHolder(b.cfg)
|
||||
b.currentProvenance.Store(&b.doc.Provenance)
|
||||
@@ -439,10 +442,21 @@ func (b *boot) startEmbeddedNATS(ctx context.Context) (string, string, error) {
|
||||
if b.containerPool != nil {
|
||||
host = b.containerPool.HostGateway()
|
||||
}
|
||||
srv, err := nats.StartEmbedded(ctx, nats.EmbeddedOptions{
|
||||
Host: host,
|
||||
StoreDir: filepath.Join(cfg.StateDir, "nats"),
|
||||
}, log)
|
||||
// A restarted broker keeps the previous endpoint, so the Gateway and
|
||||
// running agents reconnect to it instead of a broker that no longer exists.
|
||||
opts := nats.EmbeddedOptions{Host: host, StoreDir: filepath.Join(cfg.StateDir, "nats")}
|
||||
if previous, err := servicekit.ReadNATSEndpoint(cfg.StateDir); err == nil {
|
||||
opts.Token = previous.Token
|
||||
if u, err := url.Parse(previous.URL); err == nil {
|
||||
opts.Port, _ = strconv.Atoi(u.Port())
|
||||
}
|
||||
}
|
||||
srv, err := nats.StartEmbedded(ctx, opts, log)
|
||||
if err != nil && opts.Port != 0 {
|
||||
log.Warn("embedded nats: previous port unavailable; taking a new one", "port", opts.Port, "err", err)
|
||||
opts.Port = 0
|
||||
srv, err = nats.StartEmbedded(ctx, opts, log)
|
||||
}
|
||||
if err != nil {
|
||||
log.Error("embedded nats start failed", "err", err)
|
||||
return "", "", err
|
||||
@@ -961,8 +975,7 @@ func (b *boot) catalogState() (modelcatalog.Snapshot, []string) { return b.catal
|
||||
|
||||
// loadCatalog layers the model catalog under the file config. A catalog that
|
||||
// cannot be read leaves the configured providers and models in effect.
|
||||
func (b *boot) loadCatalog(ctx context.Context, cfgPath string) {
|
||||
b.catalog = servicekit.NewCatalog(cfgPath)
|
||||
func (b *boot) loadCatalog(ctx context.Context) {
|
||||
snapshot, err := b.catalog.Fetch(ctx, b.secrets.Getenv, b.cfgHolder.Get().Providers)
|
||||
if err != nil {
|
||||
b.log.Warn("model catalog unavailable; using configured providers and models", "err", err)
|
||||
|
||||
@@ -167,7 +167,7 @@ func Run() int { //nolint:cyclop,funlen // the composition root's setup sequence
|
||||
} else if exit {
|
||||
return 0
|
||||
}
|
||||
b.loadCatalog(ctx, args.cfgPath)
|
||||
b.loadCatalog(ctx)
|
||||
b.seedSoul(args.cfgPath)
|
||||
b.setupObservability(ctx)
|
||||
|
||||
|
||||
@@ -28,7 +28,12 @@ import (
|
||||
func (b *boot) setupReadinessProbes() {
|
||||
cfg := b.cfg
|
||||
probes := []health.Probe{
|
||||
readiness.NewStoreProbe(b.stateStore),
|
||||
// The State Store client waits for its peer, so the probe carries its
|
||||
// own deadline to report an outage instead of waiting it out.
|
||||
readiness.NewContractProbe("state_db", time.Duration(cfg.Health.DependencyTimeout), func(ctx context.Context) error {
|
||||
_, err := b.stateStore.StatusCounts(ctx)
|
||||
return err
|
||||
}),
|
||||
readiness.NewConfigProbe(b.cfgHolder.Get, configuration.Validate),
|
||||
readiness.NewDiskProbeTargets(diskProbeTargets(cfg)),
|
||||
// The daemon no longer runs a chat channel, so it cannot observe one's
|
||||
|
||||
@@ -104,7 +104,7 @@ func Run(ctx context.Context, options Options) error {
|
||||
if err := b.openState(ctx); err != nil {
|
||||
return err
|
||||
}
|
||||
b.loadCatalog(ctx, options.Config)
|
||||
b.loadCatalog(ctx)
|
||||
nc, err := b.connectNATS(ctx)
|
||||
if err != nil {
|
||||
return err
|
||||
@@ -151,6 +151,7 @@ func (b *server) loadConfig(cfgPath, overlayPath string) error {
|
||||
return err
|
||||
}
|
||||
b.cfg = doc.Config
|
||||
b.catalog = servicekit.NewCatalog(cfgPath)
|
||||
b.cfgHolder = config.NewHolder(b.cfg)
|
||||
logs := servicekit.Logging(b.cfg, "gateway")
|
||||
b.log, b.taskLogs = logs.Log, logs.TaskLogs
|
||||
@@ -212,7 +213,7 @@ func (b *server) connectNATS(ctx context.Context) (*natsio.Conn, error) {
|
||||
}
|
||||
url, token = endpoint.URL, endpoint.Token
|
||||
}
|
||||
nc, err := natsio.Connect(url, natsio.Token(token), natsio.MaxReconnects(-1))
|
||||
nc, err := natsio.Connect(url, natsio.Token(token), natsio.MaxReconnects(-1), natsio.RetryOnFailedConnect(true))
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("connect gateway task actions: %w", err)
|
||||
}
|
||||
@@ -238,8 +239,7 @@ func (b *server) catalogState() (modelcatalog.Snapshot, []string) { return b.cat
|
||||
|
||||
// loadCatalog layers the model catalog under the running config. A catalog
|
||||
// that cannot be read leaves the configured providers and models in effect.
|
||||
func (b *server) loadCatalog(ctx context.Context, cfgPath string) {
|
||||
b.catalog = servicekit.NewCatalog(cfgPath)
|
||||
func (b *server) loadCatalog(ctx context.Context) {
|
||||
snapshot, err := b.catalog.Fetch(ctx, b.secrets.Getenv, b.cfg.Providers)
|
||||
if err != nil {
|
||||
b.log.Warn("model catalog unavailable; using configured providers and models", "err", err)
|
||||
|
||||
@@ -5,6 +5,7 @@ import (
|
||||
"fmt"
|
||||
"log/slog"
|
||||
|
||||
"github.com/samcharles93/archie-core/internal/app/servicekit"
|
||||
"github.com/samcharles93/archie-core/internal/config"
|
||||
"github.com/samcharles93/archie-core/internal/domain/applystatus"
|
||||
"github.com/samcharles93/archie-core/internal/domain/messaging"
|
||||
@@ -46,7 +47,7 @@ func Run(ctx context.Context, o Options) error {
|
||||
var applyReporter *applystatus.Reporter
|
||||
var appliedVersion int64
|
||||
if cfg.Options.StateStore.Target != "" {
|
||||
stateStore, closeClient, dialErr := staterpc.Dial(cfg.Options.StateStore.Target, cfg.Options.StateStore.Token)
|
||||
stateStore, closeClient, dialErr := staterpc.Dial(cfg.Options.StateStore.Target, cfg.Options.StateStore.Token, servicekit.WaitForPeer)
|
||||
if dialErr != nil {
|
||||
return fmt.Errorf("dial state store control plane (%s): %w", cfg.Options.StateStore.Target, dialErr)
|
||||
}
|
||||
|
||||
@@ -4,17 +4,24 @@ import (
|
||||
"github.com/samcharles93/archie-core/internal/config"
|
||||
"github.com/samcharles93/archie-core/internal/infrastructure/staterpc"
|
||||
"github.com/samcharles93/archie-core/internal/secret"
|
||||
"google.golang.org/grpc"
|
||||
)
|
||||
|
||||
// StateStoreClient dials the State Store gRPC service the daemon's own
|
||||
// capture/mapping/binding consumers call. Token resolution is the daemon's
|
||||
// (it owns the config keys and the secret registry); the dial itself and the
|
||||
// fail-closed non-loopback rule belong to the transport, so they come
|
||||
// from staterpc.Dial.
|
||||
// from staterpc.Dial. Calls wait for the State Store rather than fail while it
|
||||
// is down, so boot order between services does not matter.
|
||||
func StateStoreClient(services config.Services, secrets *secret.Registry) (*staterpc.Client, func(), error) {
|
||||
target, err := services.RequireTarget(config.ServiceNameState)
|
||||
if err != nil {
|
||||
return nil, nil, err
|
||||
}
|
||||
return staterpc.Dial(target, services.ResolvedToken(config.ServiceNameState, secrets.Getenv))
|
||||
return staterpc.Dial(target, services.ResolvedToken(config.ServiceNameState, secrets.Getenv), WaitForPeer)
|
||||
}
|
||||
|
||||
// WaitForPeer makes a service's calls to another service wait for it to come
|
||||
// up instead of failing while it is down or restarting. A call that must not
|
||||
// wait carries its own deadline.
|
||||
var WaitForPeer = grpc.WithDefaultCallOptions(grpc.WaitForReady(true))
|
||||
@@ -26,6 +26,8 @@ type EmbeddedOptions struct {
|
||||
Host string
|
||||
// Port is the listen port. Zero means a random port.
|
||||
Port int
|
||||
// Token is the client token. Empty generates one.
|
||||
Token string
|
||||
// StoreDir is where JetStream persists streams. Empty means a temporary
|
||||
// directory (streams do not survive a reboot). The daemon passes a dir
|
||||
// under its data directory so the ARCHIE_TASKS and reaction streams are
|
||||
@@ -46,7 +48,10 @@ func StartEmbedded(ctx context.Context, opts EmbeddedOptions, log *slog.Logger)
|
||||
if log == nil {
|
||||
log = slog.New(slog.DiscardHandler)
|
||||
}
|
||||
token := rand.Text()
|
||||
token := opts.Token
|
||||
if token == "" {
|
||||
token = rand.Text()
|
||||
}
|
||||
|
||||
srv, err := server.NewServer(&server.Options{
|
||||
Host: opts.Host,
|
||||
|
||||
Reference in new issue
Block a user