fix(services): services wait for peers instead of exiting; restarted broker keeps its endpoint
State Store calls from the daemon, Gateway and Messaging wait for the store to come up, so start order no longer matters; the daemon's state_db probe carries its own deadline so an outage reports degraded instead of hanging. The Gateway retries its first NATS connect. A restarted daemon reuses the embedded broker's previous port and token, so the Gateway reconnects instead of holding a dead address. Fixes a nil catalog the catalog move introduced.
This commit is contained in:
7 files changed
+47
-16
No files matched your search
@@ -12,10 +12,12 @@ import (
|
|||||||
"fmt"
|
"fmt"
|
||||||
"log/slog"
|
"log/slog"
|
||||||
"net/http"
|
"net/http"
|
||||||
|
"net/url"
|
||||||
"os"
|
"os"
|
||||||
"os/signal"
|
"os/signal"
|
||||||
"path/filepath"
|
"path/filepath"
|
||||||
"slices"
|
"slices"
|
||||||
|
"strconv"
|
||||||
"strings"
|
"strings"
|
||||||
"sync/atomic"
|
"sync/atomic"
|
||||||
"syscall"
|
"syscall"
|
||||||
@@ -233,6 +235,7 @@ func (b *boot) loadConfig(_ context.Context, cfgPath, overlayPath string) error
|
|||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
b.loader, b.doc = loader, doc
|
b.loader, b.doc = loader, doc
|
||||||
|
b.catalog = servicekit.NewCatalog(cfgPath)
|
||||||
b.cfg = b.doc.Config
|
b.cfg = b.doc.Config
|
||||||
b.cfgHolder = config.NewHolder(b.cfg)
|
b.cfgHolder = config.NewHolder(b.cfg)
|
||||||
b.currentProvenance.Store(&b.doc.Provenance)
|
b.currentProvenance.Store(&b.doc.Provenance)
|
||||||
@@ -439,10 +442,21 @@ func (b *boot) startEmbeddedNATS(ctx context.Context) (string, string, error) {
|
|||||||
if b.containerPool != nil {
|
if b.containerPool != nil {
|
||||||
host = b.containerPool.HostGateway()
|
host = b.containerPool.HostGateway()
|
||||||
}
|
}
|
||||||
srv, err := nats.StartEmbedded(ctx, nats.EmbeddedOptions{
|
// A restarted broker keeps the previous endpoint, so the Gateway and
|
||||||
Host: host,
|
// running agents reconnect to it instead of a broker that no longer exists.
|
||||||
StoreDir: filepath.Join(cfg.StateDir, "nats"),
|
opts := nats.EmbeddedOptions{Host: host, StoreDir: filepath.Join(cfg.StateDir, "nats")}
|
||||||
}, log)
|
if previous, err := servicekit.ReadNATSEndpoint(cfg.StateDir); err == nil {
|
||||||
|
opts.Token = previous.Token
|
||||||
|
if u, err := url.Parse(previous.URL); err == nil {
|
||||||
|
opts.Port, _ = strconv.Atoi(u.Port())
|
||||||
|
}
|
||||||
|
}
|
||||||
|
srv, err := nats.StartEmbedded(ctx, opts, log)
|
||||||
|
if err != nil && opts.Port != 0 {
|
||||||
|
log.Warn("embedded nats: previous port unavailable; taking a new one", "port", opts.Port, "err", err)
|
||||||
|
opts.Port = 0
|
||||||
|
srv, err = nats.StartEmbedded(ctx, opts, log)
|
||||||
|
}
|
||||||
if err != nil {
|
if err != nil {
|
||||||
log.Error("embedded nats start failed", "err", err)
|
log.Error("embedded nats start failed", "err", err)
|
||||||
return "", "", err
|
return "", "", err
|
||||||
@@ -961,8 +975,7 @@ func (b *boot) catalogState() (modelcatalog.Snapshot, []string) { return b.catal
|
|||||||
|
|
||||||
// loadCatalog layers the model catalog under the file config. A catalog that
|
// loadCatalog layers the model catalog under the file config. A catalog that
|
||||||
// cannot be read leaves the configured providers and models in effect.
|
// cannot be read leaves the configured providers and models in effect.
|
||||||
func (b *boot) loadCatalog(ctx context.Context, cfgPath string) {
|
func (b *boot) loadCatalog(ctx context.Context) {
|
||||||
b.catalog = servicekit.NewCatalog(cfgPath)
|
|
||||||
snapshot, err := b.catalog.Fetch(ctx, b.secrets.Getenv, b.cfgHolder.Get().Providers)
|
snapshot, err := b.catalog.Fetch(ctx, b.secrets.Getenv, b.cfgHolder.Get().Providers)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
b.log.Warn("model catalog unavailable; using configured providers and models", "err", err)
|
b.log.Warn("model catalog unavailable; using configured providers and models", "err", err)
|
||||||
|
|||||||
@@ -167,7 +167,7 @@ func Run() int { //nolint:cyclop,funlen // the composition root's setup sequence
|
|||||||
} else if exit {
|
} else if exit {
|
||||||
return 0
|
return 0
|
||||||
}
|
}
|
||||||
b.loadCatalog(ctx, args.cfgPath)
|
b.loadCatalog(ctx)
|
||||||
b.seedSoul(args.cfgPath)
|
b.seedSoul(args.cfgPath)
|
||||||
b.setupObservability(ctx)
|
b.setupObservability(ctx)
|
||||||
|
|
||||||
|
|||||||
@@ -28,7 +28,12 @@ import (
|
|||||||
func (b *boot) setupReadinessProbes() {
|
func (b *boot) setupReadinessProbes() {
|
||||||
cfg := b.cfg
|
cfg := b.cfg
|
||||||
probes := []health.Probe{
|
probes := []health.Probe{
|
||||||
readiness.NewStoreProbe(b.stateStore),
|
// The State Store client waits for its peer, so the probe carries its
|
||||||
|
// own deadline to report an outage instead of waiting it out.
|
||||||
|
readiness.NewContractProbe("state_db", time.Duration(cfg.Health.DependencyTimeout), func(ctx context.Context) error {
|
||||||
|
_, err := b.stateStore.StatusCounts(ctx)
|
||||||
|
return err
|
||||||
|
}),
|
||||||
readiness.NewConfigProbe(b.cfgHolder.Get, configuration.Validate),
|
readiness.NewConfigProbe(b.cfgHolder.Get, configuration.Validate),
|
||||||
readiness.NewDiskProbeTargets(diskProbeTargets(cfg)),
|
readiness.NewDiskProbeTargets(diskProbeTargets(cfg)),
|
||||||
// The daemon no longer runs a chat channel, so it cannot observe one's
|
// The daemon no longer runs a chat channel, so it cannot observe one's
|
||||||
|
|||||||
@@ -104,7 +104,7 @@ func Run(ctx context.Context, options Options) error {
|
|||||||
if err := b.openState(ctx); err != nil {
|
if err := b.openState(ctx); err != nil {
|
||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
b.loadCatalog(ctx, options.Config)
|
b.loadCatalog(ctx)
|
||||||
nc, err := b.connectNATS(ctx)
|
nc, err := b.connectNATS(ctx)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return err
|
return err
|
||||||
@@ -151,6 +151,7 @@ func (b *server) loadConfig(cfgPath, overlayPath string) error {
|
|||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
b.cfg = doc.Config
|
b.cfg = doc.Config
|
||||||
|
b.catalog = servicekit.NewCatalog(cfgPath)
|
||||||
b.cfgHolder = config.NewHolder(b.cfg)
|
b.cfgHolder = config.NewHolder(b.cfg)
|
||||||
logs := servicekit.Logging(b.cfg, "gateway")
|
logs := servicekit.Logging(b.cfg, "gateway")
|
||||||
b.log, b.taskLogs = logs.Log, logs.TaskLogs
|
b.log, b.taskLogs = logs.Log, logs.TaskLogs
|
||||||
@@ -212,7 +213,7 @@ func (b *server) connectNATS(ctx context.Context) (*natsio.Conn, error) {
|
|||||||
}
|
}
|
||||||
url, token = endpoint.URL, endpoint.Token
|
url, token = endpoint.URL, endpoint.Token
|
||||||
}
|
}
|
||||||
nc, err := natsio.Connect(url, natsio.Token(token), natsio.MaxReconnects(-1))
|
nc, err := natsio.Connect(url, natsio.Token(token), natsio.MaxReconnects(-1), natsio.RetryOnFailedConnect(true))
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, fmt.Errorf("connect gateway task actions: %w", err)
|
return nil, fmt.Errorf("connect gateway task actions: %w", err)
|
||||||
}
|
}
|
||||||
@@ -238,8 +239,7 @@ func (b *server) catalogState() (modelcatalog.Snapshot, []string) { return b.cat
|
|||||||
|
|
||||||
// loadCatalog layers the model catalog under the running config. A catalog
|
// loadCatalog layers the model catalog under the running config. A catalog
|
||||||
// that cannot be read leaves the configured providers and models in effect.
|
// that cannot be read leaves the configured providers and models in effect.
|
||||||
func (b *server) loadCatalog(ctx context.Context, cfgPath string) {
|
func (b *server) loadCatalog(ctx context.Context) {
|
||||||
b.catalog = servicekit.NewCatalog(cfgPath)
|
|
||||||
snapshot, err := b.catalog.Fetch(ctx, b.secrets.Getenv, b.cfg.Providers)
|
snapshot, err := b.catalog.Fetch(ctx, b.secrets.Getenv, b.cfg.Providers)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
b.log.Warn("model catalog unavailable; using configured providers and models", "err", err)
|
b.log.Warn("model catalog unavailable; using configured providers and models", "err", err)
|
||||||
|
|||||||
@@ -5,6 +5,7 @@ import (
|
|||||||
"fmt"
|
"fmt"
|
||||||
"log/slog"
|
"log/slog"
|
||||||
|
|
||||||
|
"github.com/samcharles93/archie-core/internal/app/servicekit"
|
||||||
"github.com/samcharles93/archie-core/internal/config"
|
"github.com/samcharles93/archie-core/internal/config"
|
||||||
"github.com/samcharles93/archie-core/internal/domain/applystatus"
|
"github.com/samcharles93/archie-core/internal/domain/applystatus"
|
||||||
"github.com/samcharles93/archie-core/internal/domain/messaging"
|
"github.com/samcharles93/archie-core/internal/domain/messaging"
|
||||||
@@ -46,7 +47,7 @@ func Run(ctx context.Context, o Options) error {
|
|||||||
var applyReporter *applystatus.Reporter
|
var applyReporter *applystatus.Reporter
|
||||||
var appliedVersion int64
|
var appliedVersion int64
|
||||||
if cfg.Options.StateStore.Target != "" {
|
if cfg.Options.StateStore.Target != "" {
|
||||||
stateStore, closeClient, dialErr := staterpc.Dial(cfg.Options.StateStore.Target, cfg.Options.StateStore.Token)
|
stateStore, closeClient, dialErr := staterpc.Dial(cfg.Options.StateStore.Target, cfg.Options.StateStore.Token, servicekit.WaitForPeer)
|
||||||
if dialErr != nil {
|
if dialErr != nil {
|
||||||
return fmt.Errorf("dial state store control plane (%s): %w", cfg.Options.StateStore.Target, dialErr)
|
return fmt.Errorf("dial state store control plane (%s): %w", cfg.Options.StateStore.Target, dialErr)
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -4,17 +4,24 @@ import (
|
|||||||
"github.com/samcharles93/archie-core/internal/config"
|
"github.com/samcharles93/archie-core/internal/config"
|
||||||
"github.com/samcharles93/archie-core/internal/infrastructure/staterpc"
|
"github.com/samcharles93/archie-core/internal/infrastructure/staterpc"
|
||||||
"github.com/samcharles93/archie-core/internal/secret"
|
"github.com/samcharles93/archie-core/internal/secret"
|
||||||
|
"google.golang.org/grpc"
|
||||||
)
|
)
|
||||||
|
|
||||||
// StateStoreClient dials the State Store gRPC service the daemon's own
|
// StateStoreClient dials the State Store gRPC service the daemon's own
|
||||||
// capture/mapping/binding consumers call. Token resolution is the daemon's
|
// capture/mapping/binding consumers call. Token resolution is the daemon's
|
||||||
// (it owns the config keys and the secret registry); the dial itself and the
|
// (it owns the config keys and the secret registry); the dial itself and the
|
||||||
// fail-closed non-loopback rule belong to the transport, so they come
|
// fail-closed non-loopback rule belong to the transport, so they come
|
||||||
// from staterpc.Dial.
|
// from staterpc.Dial. Calls wait for the State Store rather than fail while it
|
||||||
|
// is down, so boot order between services does not matter.
|
||||||
func StateStoreClient(services config.Services, secrets *secret.Registry) (*staterpc.Client, func(), error) {
|
func StateStoreClient(services config.Services, secrets *secret.Registry) (*staterpc.Client, func(), error) {
|
||||||
target, err := services.RequireTarget(config.ServiceNameState)
|
target, err := services.RequireTarget(config.ServiceNameState)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, nil, err
|
return nil, nil, err
|
||||||
}
|
}
|
||||||
return staterpc.Dial(target, services.ResolvedToken(config.ServiceNameState, secrets.Getenv))
|
return staterpc.Dial(target, services.ResolvedToken(config.ServiceNameState, secrets.Getenv), WaitForPeer)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// WaitForPeer makes a service's calls to another service wait for it to come
|
||||||
|
// up instead of failing while it is down or restarting. A call that must not
|
||||||
|
// wait carries its own deadline.
|
||||||
|
var WaitForPeer = grpc.WithDefaultCallOptions(grpc.WaitForReady(true))
|
||||||
@@ -26,6 +26,8 @@ type EmbeddedOptions struct {
|
|||||||
Host string
|
Host string
|
||||||
// Port is the listen port. Zero means a random port.
|
// Port is the listen port. Zero means a random port.
|
||||||
Port int
|
Port int
|
||||||
|
// Token is the client token. Empty generates one.
|
||||||
|
Token string
|
||||||
// StoreDir is where JetStream persists streams. Empty means a temporary
|
// StoreDir is where JetStream persists streams. Empty means a temporary
|
||||||
// directory (streams do not survive a reboot). The daemon passes a dir
|
// directory (streams do not survive a reboot). The daemon passes a dir
|
||||||
// under its data directory so the ARCHIE_TASKS and reaction streams are
|
// under its data directory so the ARCHIE_TASKS and reaction streams are
|
||||||
@@ -46,7 +48,10 @@ func StartEmbedded(ctx context.Context, opts EmbeddedOptions, log *slog.Logger)
|
|||||||
if log == nil {
|
if log == nil {
|
||||||
log = slog.New(slog.DiscardHandler)
|
log = slog.New(slog.DiscardHandler)
|
||||||
}
|
}
|
||||||
token := rand.Text()
|
token := opts.Token
|
||||||
|
if token == "" {
|
||||||
|
token = rand.Text()
|
||||||
|
}
|
||||||
|
|
||||||
srv, err := server.NewServer(&server.Options{
|
srv, err := server.NewServer(&server.Options{
|
||||||
Host: opts.Host,
|
Host: opts.Host,
|
||||||
|
|||||||
Reference in new issue
Block a user