388 lines
14 KiB
Go
388 lines
14 KiB
Go
package archiegateway
|
|
|
|
import (
|
|
"context"
|
|
"errors"
|
|
"fmt"
|
|
"log/slog"
|
|
"net"
|
|
"os"
|
|
"slices"
|
|
"sync"
|
|
"sync/atomic"
|
|
"time"
|
|
|
|
"github.com/jackc/pgx/v5/pgxpool"
|
|
natsio "github.com/nats-io/nats.go"
|
|
"github.com/samcharles93/ai-sdk/runtime"
|
|
|
|
"github.com/samcharles93/archie-core/internal/app/servicekit"
|
|
"github.com/samcharles93/archie-core/internal/config"
|
|
"github.com/samcharles93/archie-core/internal/domain/applystatus"
|
|
"github.com/samcharles93/archie-core/internal/domain/curator"
|
|
"github.com/samcharles93/archie-core/internal/domain/health"
|
|
domainmemory "github.com/samcharles93/archie-core/internal/domain/memory"
|
|
"github.com/samcharles93/archie-core/internal/domain/messaging"
|
|
"github.com/samcharles93/archie-core/internal/domain/presence"
|
|
"github.com/samcharles93/archie-core/internal/events"
|
|
"github.com/samcharles93/archie-core/internal/gateway"
|
|
"github.com/samcharles93/archie-core/internal/infrastructure/controlplanerpc"
|
|
"github.com/samcharles93/archie-core/internal/infrastructure/controlplanestore"
|
|
"github.com/samcharles93/archie-core/internal/infrastructure/modelcatalog"
|
|
"github.com/samcharles93/archie-core/internal/infrastructure/readiness"
|
|
"github.com/samcharles93/archie-core/internal/infrastructure/rpctls"
|
|
"github.com/samcharles93/archie-core/internal/infrastructure/secretengine"
|
|
"github.com/samcharles93/archie-core/internal/infrastructure/staterpc"
|
|
"github.com/samcharles93/archie-core/internal/infrastructure/taskactions"
|
|
"github.com/samcharles93/archie-core/internal/logging"
|
|
"github.com/samcharles93/archie-core/internal/plugin"
|
|
"github.com/samcharles93/archie-core/internal/ratelimit"
|
|
"github.com/samcharles93/archie-core/internal/releaseupdate"
|
|
"github.com/samcharles93/archie-core/internal/sdnotify"
|
|
"github.com/samcharles93/archie-core/internal/secret"
|
|
"github.com/samcharles93/archie-core/internal/skill"
|
|
"github.com/samcharles93/archie-core/internal/tools"
|
|
toolprovider "github.com/samcharles93/archie-core/internal/tools/provider"
|
|
)
|
|
|
|
// Options contains process inputs for the standalone Gateway.
|
|
type Options struct {
|
|
Config string
|
|
Overlay string
|
|
// Listen is the gRPC listen address. Empty falls back to
|
|
// [services.gateway].listen.
|
|
Listen string
|
|
}
|
|
|
|
// server is the Gateway process: the owner of conversations and of every
|
|
// model call.
|
|
type server struct {
|
|
cfg config.Config
|
|
cfgHolder *config.Holder
|
|
cfgPath string
|
|
log *slog.Logger
|
|
|
|
secrets *secret.Registry
|
|
id *rpctls.Identity
|
|
engines *secretengine.Runner
|
|
stateStore *staterpc.Client
|
|
controlPlane *controlplanestore.Client
|
|
applyStatus *applystatus.Reporter
|
|
runtimeVersions map[string]int64
|
|
catalog *modelcatalog.Catalog
|
|
chatPool *pgxpool.Pool
|
|
chatSessionStore gateway.SessionStore
|
|
logFeed *logging.Feed
|
|
taskActionsConn *natsio.Conn
|
|
|
|
bus *events.Bus
|
|
capabilityHost *plugin.Host
|
|
rateLimiter *ratelimit.Limiter
|
|
llm atomic.Pointer[runtime.Runtime]
|
|
transcriber messaging.Transcriber
|
|
toolReg *tools.Registry
|
|
providerRegistry *toolprovider.Registry
|
|
mcpMu sync.Mutex
|
|
mcpApplied map[string]appliedMCPServer
|
|
chatModels *chatModelManager
|
|
soul *soulSource
|
|
memEngines *domainmemory.Registry
|
|
curatorRegistry *curator.Registry
|
|
curatorRuntime *curator.Runtime
|
|
providerOutcomes *providerOutcomeRecorder
|
|
usage *usageSink
|
|
statusHealth gateway.HealthSource
|
|
updateService *releaseupdate.Service
|
|
|
|
chatTasks gateway.TaskCreator
|
|
chatController *gateway.StoreTaskController
|
|
defaultChatIdentity string
|
|
|
|
// skillMu guards skillList, the catalogue refreshSkills last registered.
|
|
skillMu sync.Mutex
|
|
skillList []skill.CatalogEntry
|
|
|
|
cleanups []func()
|
|
}
|
|
|
|
// Run serves the Gateway's chat contract until ctx ends.
|
|
func Run(ctx context.Context, options Options) error {
|
|
b := &server{log: slog.New(slog.NewJSONHandler(os.Stderr, nil))}
|
|
defer b.cleanup()
|
|
if err := b.loadConfig(ctx, options.Config, options.Overlay); err != nil {
|
|
return err
|
|
}
|
|
if err := b.openState(ctx); err != nil {
|
|
return err
|
|
}
|
|
b.loadCatalog(ctx)
|
|
nc, err := b.connectNATS(ctx)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
contract, err := b.startGatewayRuntime(ctx, taskactions.Client{Conn: nc, Timeout: 30 * time.Second})
|
|
if err != nil {
|
|
return err
|
|
}
|
|
listen, err := servicekit.ResolveListen("gateway", options.Listen, b.cfg.Services.Get(config.ServiceNameGateway).Listen)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
opts, err := gatewayServerOpts(listen, b.id, b.stateStore)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
listener, err := (&net.ListenConfig{}).Listen(ctx, "tcp", listen)
|
|
if err != nil {
|
|
return fmt.Errorf("listen for gateway: %w", err)
|
|
}
|
|
return b.serveGatewayListener(ctx, listener, contract, opts)
|
|
}
|
|
|
|
func (b *server) addCleanup(fn func()) { b.cleanups = append(b.cleanups, fn) }
|
|
|
|
func (b *server) cleanup() {
|
|
for _, fn := range slices.Backward(b.cleanups) {
|
|
fn()
|
|
}
|
|
}
|
|
|
|
func (b *server) chatLLM() *runtime.Runtime { return b.llm.Load() }
|
|
func (b *server) setLLM(rt *runtime.Runtime) { b.llm.Store(rt) }
|
|
func (b *server) announceReady() { sdnotify.Ready(b.log) }
|
|
|
|
func (b *server) loadConfig(ctx context.Context, cfgPath, overlayPath string) error {
|
|
_, doc, err := servicekit.Resolve(b.log, cfgPath, overlayPath)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
b.cfg = doc.Config
|
|
b.cfgPath = cfgPath
|
|
b.catalog = modelcatalog.NewCatalog(cfgPath)
|
|
b.cfgHolder = config.NewHolder(b.cfg)
|
|
logs, err := servicekit.Logging(ctx, b.cfg, "gateway")
|
|
if err != nil {
|
|
return err
|
|
}
|
|
b.log, b.logFeed = logs.Log, logs.Feed
|
|
b.addCleanup(func() { _ = logs.Closer.Close() })
|
|
b.secrets = secret.NewRegistry()
|
|
b.id, err = servicekit.Identity(b.cfg.TLS, presence.Gateway)
|
|
return err
|
|
}
|
|
|
|
// openState claims the conversation store, dials the State Store, starts the
|
|
// enabled secret engines and layers the stored runtime settings.
|
|
func (b *server) openState(ctx context.Context) error {
|
|
if err := b.openChatSessions(ctx); err != nil {
|
|
return err
|
|
}
|
|
if err := b.claimGatewayOwnership(ctx); err != nil {
|
|
return err
|
|
}
|
|
client, cleanup, err := servicekit.StateStoreClient(presence.Gateway, b.cfg)
|
|
if err != nil {
|
|
return fmt.Errorf("state store client: %w", err)
|
|
}
|
|
b.addCleanup(cleanup)
|
|
b.stateStore = client
|
|
b.controlPlane = controlplanestore.NewRPCClient(client.ControlPlane())
|
|
b.applyStatus = applystatus.New(applystatus.Gateway, client, b.log)
|
|
source := secretengine.Source{Query: b.controlPlane, Packages: client}
|
|
report := func(ctx context.Context, version int64, err error) {
|
|
b.applyStatus.Report(ctx, controlplanerpc.ExtensionSettingsKind, version, err)
|
|
}
|
|
b.engines = secretengine.Supervise(ctx, b.secrets, source, applystatus.Gateway, report, b.log)
|
|
b.addCleanup(b.engines.Close)
|
|
cfg, versions, err := b.runtimeConfig(ctx, b.cfg)
|
|
if err != nil {
|
|
return fmt.Errorf("load runtime settings: %w", err)
|
|
}
|
|
b.cfg, b.runtimeVersions = cfg, versions
|
|
b.cfgHolder.Set(cfg)
|
|
go b.applyStatus.Run(ctx)
|
|
|
|
return nil
|
|
}
|
|
|
|
func (b *server) runtimeConfig(ctx context.Context, base config.Config) (config.Config, map[string]int64, error) {
|
|
return servicekit.RuntimeConfig(ctx, b.controlPlane, b.applyStatus, b.secrets, b.log, base, nil, gatewayRuntimeKinds)
|
|
}
|
|
|
|
// connectNATS joins the broker task actions travel over. The daemon owns the
|
|
// embedded broker, so the Gateway waits for its endpoint rather than starting
|
|
// a second one.
|
|
func (b *server) connectNATS(ctx context.Context) (*natsio.Conn, error) {
|
|
url, token := b.cfg.NATS.URL, ""
|
|
if b.cfg.NATS.Mode == config.NATSModeExternal {
|
|
var err error
|
|
if token, err = servicekit.NATSToken(b.cfg.NATS, os.Getenv); err != nil {
|
|
return nil, err
|
|
}
|
|
} else {
|
|
endpoint, err := b.waitForNATSEndpoint(ctx)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
url, token = endpoint.URL, endpoint.Token
|
|
}
|
|
nc, err := natsio.Connect(url, natsio.Token(token), natsio.MaxReconnects(-1), natsio.RetryOnFailedConnect(true))
|
|
if err != nil {
|
|
return nil, fmt.Errorf("connect gateway task actions: %w", err)
|
|
}
|
|
b.addCleanup(nc.Close)
|
|
b.taskActionsConn = nc
|
|
registry := health.NewRegistry(
|
|
readiness.NewContractProbe("state_store", b.cfg.Health.DependencyTimeout.Std(), func(ctx context.Context) error { _, err := b.stateStore.StatusCounts(ctx); return err }),
|
|
readiness.NewContractProbe("nats", b.cfg.Health.DependencyTimeout.Std(), func(context.Context) error {
|
|
if !nc.IsConnected() {
|
|
return fmt.Errorf("broker disconnected")
|
|
}
|
|
return nil
|
|
}),
|
|
)
|
|
go presence.Run(ctx, presence.Gateway, servicekit.Build(), b.stateStore, registry, b.log)
|
|
return nc, nil
|
|
}
|
|
|
|
func (b *server) waitForNATSEndpoint(ctx context.Context) (servicekit.NATSEndpoint, error) {
|
|
for {
|
|
endpoint, err := servicekit.ReadNATSEndpoint(b.cfg.StateDir)
|
|
if err == nil {
|
|
return endpoint, nil
|
|
}
|
|
b.log.WarnContext(ctx, "waiting for the daemon's NATS endpoint", "err", err)
|
|
if !servicekit.WaitFor(ctx, 2*time.Second) {
|
|
return servicekit.NATSEndpoint{}, ctx.Err()
|
|
}
|
|
}
|
|
}
|
|
|
|
func (b *server) catalogState() modelcatalog.Snapshot { return b.catalog.State() }
|
|
|
|
// loadCatalog layers the model catalog under the running config. A catalog
|
|
// that cannot be read leaves the configured providers and models in effect.
|
|
func (b *server) loadCatalog(ctx context.Context) {
|
|
snapshot, err := b.catalog.Fetch(ctx, b.secrets.Getenv, b.cfg.Providers)
|
|
if err != nil {
|
|
b.log.WarnContext(ctx, "model catalog unavailable; using configured providers and models", "err", err)
|
|
return
|
|
}
|
|
modelcatalog.Apply(&b.cfg, snapshot)
|
|
b.cfgHolder.Set(b.cfg.Clone())
|
|
b.catalog.Set(snapshot)
|
|
}
|
|
|
|
func (b *server) startModelCatalogRefresh(ctx context.Context) {
|
|
modelcatalog.Every(ctx, modelcatalog.RefreshInterval, func() {
|
|
snapshot, err := b.catalog.Fetch(ctx, b.secrets.Getenv, b.cfgHolder.Get().Providers)
|
|
if err != nil {
|
|
b.log.WarnContext(ctx, "model catalog refresh failed; keeping the loaded catalog", "err", err)
|
|
return
|
|
}
|
|
base := b.cfgHolder.Get().Clone()
|
|
modelcatalog.Apply(&base, snapshot)
|
|
b.catalog.Set(snapshot)
|
|
if err := b.relayer(ctx, base, ""); err != nil {
|
|
b.log.WarnContext(ctx, "model catalog refresh rejected with the settings; the running settings stay", "err", err)
|
|
}
|
|
})
|
|
}
|
|
|
|
// gatewayRuntimeKinds are the kinds the Gateway applies live: the settings its
|
|
// chat runtime and tool providers are built from.
|
|
var gatewayRuntimeKinds = []string{
|
|
controlplanestore.ProviderSettingsKind,
|
|
controlplanestore.ModelAliasesKind,
|
|
controlplanestore.ToolSettingsKind,
|
|
}
|
|
|
|
func (b *server) startRuntimeWatches(ctx context.Context) error {
|
|
// An engine that starts after boot answers references that failed then, so
|
|
// the providers resolve again.
|
|
b.engines.OnChange(func() {
|
|
base := b.cfgHolder.Get()
|
|
catalog := b.catalogState()
|
|
modelcatalog.Apply(&base, catalog)
|
|
_ = b.relayer(ctx, base, controlplanestore.ProviderSettingsKind)
|
|
})
|
|
for _, kind := range gatewayRuntimeKinds {
|
|
updates, err := b.controlPlane.WatchResource(ctx, kind, b.runtimeVersions[kind])
|
|
if err != nil {
|
|
return fmt.Errorf("watch %s: %w", kind, err)
|
|
}
|
|
var lastApplyErr, streamErr error
|
|
open := func(ctx context.Context, afterVersion int64) (<-chan controlplanestore.AppliedResource, error) {
|
|
next, err := b.controlPlane.WatchResource(ctx, kind, afterVersion)
|
|
if err == nil {
|
|
// A live watch is re-established: the outage clears, and an
|
|
// outstanding refusal stays the record's error.
|
|
b.applyStatus.Report(ctx, kind, afterVersion, lastApplyErr)
|
|
}
|
|
return next, err
|
|
}
|
|
go servicekit.KeepWatch(ctx, b.log, kind, b.runtimeVersions[kind], updates,
|
|
open,
|
|
servicekit.WaitFor,
|
|
func(update controlplanestore.AppliedResource) int64 { return update.Version },
|
|
func(update controlplanestore.AppliedResource) {
|
|
if update.Err != nil {
|
|
// A stream failure carries no version: the outage is what the
|
|
// end callback records. A refused document carries its version
|
|
// and becomes the record's error.
|
|
if update.Version > 0 {
|
|
lastApplyErr = update.Err
|
|
b.applyStatus.Report(ctx, kind, update.Version, update.Err)
|
|
} else {
|
|
streamErr = update.Err
|
|
}
|
|
b.log.ErrorContext(ctx, "runtime settings watch failed", "kind", kind, "err", update.Err)
|
|
return
|
|
}
|
|
base := b.cfgHolder.Get()
|
|
catalog := b.catalogState()
|
|
modelcatalog.Apply(&base, catalog)
|
|
lastApplyErr = b.relayer(ctx, base, kind)
|
|
},
|
|
func() {
|
|
// A refusal already on the record outranks the outage; it is the
|
|
// state the running settings are actually in. Otherwise the gap is
|
|
// reported so the record stops reading the last-applied version as
|
|
// current while no live updates can arrive.
|
|
if lastApplyErr != nil {
|
|
return
|
|
}
|
|
if streamErr == nil {
|
|
streamErr = fmt.Errorf("control-plane watch stream ended")
|
|
}
|
|
b.applyStatus.Report(ctx, kind, 0,
|
|
fmt.Errorf("runtime settings watch unavailable: %w", streamErr))
|
|
streamErr = nil
|
|
})
|
|
}
|
|
return nil
|
|
}
|
|
|
|
// relayer re-runs the runtime layering over base and applies the result to the
|
|
// chat runtime and, for tool settings, the tool providers. A refused document
|
|
// leaves the running settings in place and its error returned, so the watch
|
|
// keeps it as lastApplyErr rather than recording an outage over it.
|
|
func (b *server) relayer(ctx context.Context, base config.Config, kind string) error {
|
|
cfg, versions, err := b.runtimeConfig(ctx, base)
|
|
if err != nil {
|
|
b.log.ErrorContext(ctx, "runtime settings update rejected; the running settings stay", "kind", kind, "err", err)
|
|
return err
|
|
}
|
|
b.cfgHolder.Set(cfg)
|
|
b.rebuildChatModelRuntime(cfg)
|
|
var problems []error
|
|
if kind == controlplanestore.ToolSettingsKind {
|
|
if err := b.reconcileToolSettings(ctx, cfg); err != nil {
|
|
b.applyStatus.Report(ctx, kind, versions[kind], err)
|
|
b.log.ErrorContext(ctx, "tool settings reconciled with errors; the running servers stay", "err", err)
|
|
problems = append(problems, err)
|
|
}
|
|
}
|
|
return errors.Join(problems...)
|
|
}
|