feat(archied): refresh the model catalog at runtime
modelcatalog.Load ran once at boot, so a provider whose credential became resolvable later, or a model models.dev published since, needed a restart. The daemon now re-reads the catalog on an interval and republishes everything derived from it, without a restart. - `refreshModelCatalog` re-reads through the same options boot used, layers the result with the process's own runtimeConfig, and publishes. A failed read keeps the catalog already running: the model list must not go blank because the catalog service had a bad minute. - Boot now publishes the merged config into `config.Holder`. applyModelCatalog only ever wrote into the boot struct, so the catalog's model limits and discovered providers never reached the running config -- and therefore never reached `config.ForTask`, which is what a workflow stage resolves model limits from. The first live update was the first snapshot to carry them. - The chat model manager's catalog is a contribution that is replaced, not merged. A model the catalog no longer publishes stops being offered and described instead of lingering from an earlier snapshot. - The catalog snapshot and the refs it contributes live behind one lock: the refresh goroutine replaces them while the config paths read them. The daemon owns the published configuration, so the loop is the daemon's; the standalone Gateway reads the catalog once at boot, as it does for the other live-applied settings it does not watch. The interval is deliberately not an operator setting: the catalog is derived state rather than a stored resource, and a knob with no second consumer would be a setting that parses and does nothing. A key exported into the shell after the process started stays invisible to every refresh, and cannot be otherwise: the process environment is fixed at exec. A credential a secret engine starts resolving is visible, because the catalog's credential lookup is the secrets registry. Refs archie-core-zfb0.6
This commit is contained in:
1 parent
92ec539149
commit
639914c832
6 files changed
+439
-28
No files matched your search
@@ -17,6 +17,7 @@ import (
|
||||
"path/filepath"
|
||||
"slices"
|
||||
"strings"
|
||||
"sync"
|
||||
"sync/atomic"
|
||||
"syscall"
|
||||
"time"
|
||||
@@ -165,8 +166,22 @@ type boot struct {
|
||||
// closes; the Gateway also holds its serve claim on it.
|
||||
chatPool *pgxpool.Pool
|
||||
|
||||
// catalog, catalogModels and catalogMu are the model catalog the process
|
||||
// last loaded and the model references it contributes. The refresh loop
|
||||
// replaces both together while the config paths read them, so they are one
|
||||
// value behind one lock rather than two fields that can disagree.
|
||||
catalogMu sync.RWMutex
|
||||
catalog modelcatalog.Snapshot
|
||||
catalogModels []string
|
||||
// catalogCachePath is where the catalog's last download is cached, beside
|
||||
// the config file. Empty when boot had no file path to derive it from.
|
||||
catalogCachePath string
|
||||
// catalogURL overrides the catalog endpoint. Zero in production:
|
||||
// modelcatalog defaults to models.dev. Set only by tests.
|
||||
catalogURL string
|
||||
// catalogRefreshInterval is how often the running daemon re-reads the
|
||||
// catalog. The daemon's refresh loop reads it; zero means the default.
|
||||
catalogRefreshInterval time.Duration
|
||||
|
||||
bus *events.Bus
|
||||
// taskActionsConn is the standalone Gateway process's own NATS connection:
|
||||
@@ -530,19 +545,128 @@ func (b *boot) handleRequeue(ctx context.Context, requeue int64, once bool) (boo
|
||||
return !once, nil
|
||||
}
|
||||
|
||||
// catalogState returns the model catalog the process is running and the model
|
||||
// references it contributes. Both together, under one lock: the refresh loop
|
||||
// replaces them as a pair.
|
||||
func (b *boot) catalogState() (modelcatalog.Snapshot, []string) {
|
||||
b.catalogMu.RLock()
|
||||
defer b.catalogMu.RUnlock()
|
||||
return b.catalog, b.catalogModels
|
||||
}
|
||||
|
||||
// setCatalogState installs a loaded catalog. Called only after everything it
|
||||
// has to reach has been built, so a read that fails leaves the previous
|
||||
// snapshot running.
|
||||
func (b *boot) setCatalogState(snapshot modelcatalog.Snapshot, models []string) {
|
||||
b.catalogMu.Lock()
|
||||
defer b.catalogMu.Unlock()
|
||||
b.catalog = snapshot
|
||||
b.catalogModels = models
|
||||
}
|
||||
|
||||
// loadCatalog reads the model catalog once at boot and merges it into the
|
||||
// running config. A catalog that cannot be read is not fatal -- the configured
|
||||
// providers and models keep running, and the refresh loop retries the same
|
||||
// read -- but a catalog that *is* read must reach the config the process runs
|
||||
// on, not just the boot struct the chat runtime was built from.
|
||||
func (b *boot) loadCatalog(ctx context.Context, cfgPath string) {
|
||||
catalog, err := modelcatalog.Load(ctx, modelcatalog.Options{
|
||||
CachePath: filepath.Join(filepath.Dir(cfgPath), "models.json"),
|
||||
Getenv: b.secrets.Getenv,
|
||||
Configured: b.cfg.Providers,
|
||||
})
|
||||
b.catalog = catalog
|
||||
b.catalogCachePath = filepath.Join(filepath.Dir(cfgPath), "models.json")
|
||||
catalog, err := modelcatalog.Load(ctx, b.catalogOptions(b.catalogCachePath))
|
||||
if err != nil {
|
||||
b.log.Warn("model catalog unavailable; using configured providers and models", "err", err)
|
||||
return
|
||||
}
|
||||
b.catalogModels = applyModelCatalog(&b.cfg, catalog)
|
||||
b.log.Info("model catalog loaded", "providers", len(catalog.Providers), "models", len(b.catalogModels))
|
||||
models := applyModelCatalog(&b.cfg, catalog)
|
||||
// applyModelCatalog writes into the boot struct, while every consumer
|
||||
// reads config.Holder. Publishing the merged copy here is what puts the
|
||||
// catalog's model limits and discovered providers into the running config;
|
||||
// without it the first live update would be the first snapshot to carry
|
||||
// them.
|
||||
b.cfgHolder.Set(b.cfg.Clone())
|
||||
b.setCatalogState(catalog, models)
|
||||
b.log.Info("model catalog loaded", "providers", len(catalog.Providers), "models", len(models))
|
||||
}
|
||||
|
||||
// refreshModelCatalog re-reads the catalog and republishes everything derived
|
||||
// from it, so a credential that becomes resolvable after boot or a model added
|
||||
// upstream reaches the running config, the dashboard and the chat runtime
|
||||
// without a restart. It is the live half of loadCatalog.
|
||||
//
|
||||
// A failed read keeps the snapshot already running: the model list must not go
|
||||
// blank because the catalog service had a bad minute. The caller decides
|
||||
// whether the failure is fatal to it -- the refresh loop logs and retries.
|
||||
func (b *boot) refreshModelCatalog(ctx context.Context) error {
|
||||
catalog, err := modelcatalog.Load(ctx, b.catalogOptions(b.catalogCachePath))
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
base := b.cfgHolder.Get().Clone()
|
||||
models := applyModelCatalog(&base, catalog)
|
||||
cfg, _, err := b.runtimeConfig(ctx, base)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
// Committed only once the layered config passed validation, so a refused
|
||||
// layering leaves the catalog the process is running alone.
|
||||
b.setCatalogState(catalog, models)
|
||||
b.publishConfig(ctx, cfg)
|
||||
b.rebuildChatModelRuntime(cfg)
|
||||
b.log.Info("model catalog refreshed", "providers", len(catalog.Providers), "models", len(models))
|
||||
return nil
|
||||
}
|
||||
|
||||
// catalogOptions is the one place the catalog read options are built: boot and
|
||||
// every live refresh read the same endpoint, the same cache file and the same
|
||||
// credential lookup. Getenv is the secrets registry, not the raw process
|
||||
// environment, so a key a secret engine starts resolving after boot is visible
|
||||
// to the next refresh. A variable exported into the shell after the process
|
||||
// started is not, and cannot be: the process environment is fixed at exec.
|
||||
func (b *boot) catalogOptions(cachePath string) modelcatalog.Options {
|
||||
opts := modelcatalog.Options{
|
||||
URL: b.catalogURL,
|
||||
CachePath: cachePath,
|
||||
}
|
||||
if b.secrets != nil {
|
||||
opts.Getenv = b.secrets.Getenv
|
||||
}
|
||||
if b.cfgHolder != nil {
|
||||
opts.Configured = b.cfgHolder.Get().Providers
|
||||
}
|
||||
return opts
|
||||
}
|
||||
|
||||
// modelCatalogRefreshInterval is how often the running daemon re-reads the
|
||||
// model catalog. The document itself changes on the order of weeks, but a
|
||||
// credential a secret engine starts resolving can appear at any moment, so the
|
||||
// interval trades one cheap HTTP read against how long an operator waits for a
|
||||
// newly usable provider to show up. It is deliberately the daemon's own
|
||||
// cadence and not an operator setting: the catalog is derived state, not a
|
||||
// stored resource, and a knob with no second consumer would be a setting that
|
||||
// parses and does nothing.
|
||||
const modelCatalogRefreshInterval = time.Hour
|
||||
|
||||
// startModelCatalogRefresh re-reads the catalog on an interval for the life of
|
||||
// the process. It belongs to the daemon, which owns the published
|
||||
// configuration; the standalone Gateway reads the catalog once at boot.
|
||||
func (b *boot) startModelCatalogRefresh(ctx context.Context) {
|
||||
every := b.catalogRefreshInterval
|
||||
if every <= 0 {
|
||||
every = modelCatalogRefreshInterval
|
||||
}
|
||||
go func() {
|
||||
ticker := time.NewTicker(every)
|
||||
defer ticker.Stop()
|
||||
for {
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return
|
||||
case <-ticker.C:
|
||||
if err := b.refreshModelCatalog(ctx); err != nil {
|
||||
b.log.Warn("model catalog refresh failed; keeping the loaded catalog", "err", err)
|
||||
}
|
||||
}
|
||||
}
|
||||
}()
|
||||
}
|
||||
|
||||
// setupObservability builds the event bus and dashboard server. Every event is persisted (stamped with its row id) and
|
||||
@@ -1556,10 +1680,11 @@ func (b *boot) configOrigins() []webui.ConfigOrigin {
|
||||
// the daemon's own configuration state. The daemon is the configuration
|
||||
// owner, so it is the process that renders this view (archie-core-ml30).
|
||||
func (b *boot) configViewInput(ctx context.Context) webui.ConfigViewInput {
|
||||
catalog, _ := b.catalogState()
|
||||
in := webui.ConfigViewInput{
|
||||
Config: b.cfgHolder.Get(),
|
||||
Provenance: b.configOrigins(),
|
||||
Catalog: catalogView(b.catalog),
|
||||
Catalog: catalogView(catalog),
|
||||
}
|
||||
if b.lastReload != nil {
|
||||
status := b.lastReload()
|
||||
|
||||
@@ -18,10 +18,13 @@ type chatModelManager struct {
|
||||
mu sync.RWMutex
|
||||
models []string
|
||||
active string
|
||||
// catalogs are the non-role sources the model set is derived from, kept
|
||||
// so SetConfigured can rebuild it after a live settings update without a
|
||||
// caller re-passing them.
|
||||
catalogs [][]string
|
||||
// configured, explicit and catalog are the sources the offered set is
|
||||
// derived from: the role assignments (live via SetConfigured), the
|
||||
// [chat].models catalog (fixed for the process), and the model catalog's
|
||||
// own refs (replaced wholesale by SetModelCatalog).
|
||||
configured map[string]string
|
||||
explicit []string
|
||||
catalog []string
|
||||
details map[string]gateway.ModelDetails
|
||||
providerNames map[string]string
|
||||
}
|
||||
@@ -63,13 +66,29 @@ func mergeModelRefs(configured map[string]string, catalogs ...[]string) []string
|
||||
}
|
||||
|
||||
func newChatModelManager(configured map[string]string, catalogs ...[]string) *chatModelManager {
|
||||
return &chatModelManager{
|
||||
models: mergeModelRefs(configured, catalogs...),
|
||||
active: pickChatDefault(configured),
|
||||
catalogs: append([][]string(nil), catalogs...),
|
||||
explicit := make([]string, 0, len(catalogs))
|
||||
for _, catalog := range catalogs {
|
||||
explicit = append(explicit, catalog...)
|
||||
}
|
||||
m := &chatModelManager{
|
||||
configured: configured,
|
||||
explicit: explicit,
|
||||
details: make(map[string]gateway.ModelDetails),
|
||||
providerNames: make(map[string]string),
|
||||
}
|
||||
m.active = pickChatDefault(configured)
|
||||
m.setModelsLocked()
|
||||
return m
|
||||
}
|
||||
|
||||
// setModelsLocked re-derives the offered set from the three sources and keeps
|
||||
// the active selection when the new set still carries it, falling back the way
|
||||
// construction does when it does not.
|
||||
func (m *chatModelManager) setModelsLocked() {
|
||||
m.models = mergeModelRefs(m.configured, m.explicit, m.catalog)
|
||||
if !slices.Contains(m.models, m.active) {
|
||||
m.active = pickChatDefault(m.configured)
|
||||
}
|
||||
}
|
||||
|
||||
// SetConfigured re-derives the model set the manager offers after a live
|
||||
@@ -81,16 +100,22 @@ func newChatModelManager(configured map[string]string, catalogs ...[]string) *ch
|
||||
func (m *chatModelManager) SetConfigured(configured map[string]string) {
|
||||
m.mu.Lock()
|
||||
defer m.mu.Unlock()
|
||||
models := mergeModelRefs(configured, m.catalogs...)
|
||||
if !slices.Contains(models, m.active) {
|
||||
m.active = pickChatDefault(configured)
|
||||
}
|
||||
m.models = models
|
||||
m.configured = configured
|
||||
m.setModelsLocked()
|
||||
}
|
||||
|
||||
func (m *chatModelManager) ApplyModelCatalog(snapshot modelcatalog.Snapshot) {
|
||||
// SetModelCatalog replaces the model catalog's contribution to the manager:
|
||||
// the references it offers, the provider display names and the per-model
|
||||
// details. The previous snapshot's entries are dropped rather than merged --
|
||||
// a model or provider the catalog no longer publishes must stop being offered
|
||||
// and described, and these maps are the manager's own state rather than a
|
||||
// second catalogue to fall back on.
|
||||
func (m *chatModelManager) SetModelCatalog(snapshot modelcatalog.Snapshot, models []string) {
|
||||
m.mu.Lock()
|
||||
defer m.mu.Unlock()
|
||||
m.catalog = slices.Clone(models)
|
||||
m.details = make(map[string]gateway.ModelDetails)
|
||||
m.providerNames = make(map[string]string)
|
||||
for _, provider := range snapshot.Providers {
|
||||
m.providerNames[provider.ID] = provider.Name
|
||||
for _, model := range provider.Models {
|
||||
@@ -104,6 +129,7 @@ func (m *chatModelManager) ApplyModelCatalog(snapshot modelcatalog.Snapshot) {
|
||||
}
|
||||
}
|
||||
}
|
||||
m.setModelsLocked()
|
||||
}
|
||||
|
||||
func (m *chatModelManager) ProviderDisplayName(provider string) string {
|
||||
|
||||
@@ -75,7 +75,8 @@ func (b *boot) reloadConfig(ctx context.Context, doc *configuration.Document) er
|
||||
// drop them from the running config even though the file is
|
||||
// unchanged. Re-apply the same merge (idempotent: a catalog that
|
||||
// failed to load merges to identity).
|
||||
applyModelCatalog(&doc.Config, b.catalog)
|
||||
catalog, _ := b.catalogState()
|
||||
applyModelCatalog(&doc.Config, catalog)
|
||||
// Bounded: this runs on the signal loop, which handles nothing else
|
||||
// while it waits, and a State Store that has stopped answering must
|
||||
// surface as a failed reload rather than a SIGHUP that never returns.
|
||||
@@ -324,7 +325,8 @@ func (b *boot) applyRuntimeResourceUpdate(ctx context.Context, kind string, upda
|
||||
// this read returns can lack the catalog's model limits and discovered
|
||||
// providers. The merge is idempotent, and reloadConfig re-applies it
|
||||
// before layering for the same reason.
|
||||
applyModelCatalog(&base, b.catalog)
|
||||
catalog, _ := b.catalogState()
|
||||
applyModelCatalog(&base, catalog)
|
||||
cfg, _, err := b.runtimeConfig(ctx, base)
|
||||
if err != nil {
|
||||
// runtimeConfig reported the refusal through apply status for every
|
||||
@@ -344,12 +346,14 @@ func (b *boot) applyRuntimeResourceUpdate(ctx context.Context, kind string, upda
|
||||
// after a live change to provider-settings or model-role-assignments. The
|
||||
// turn runner reads the runtime through boot.chatLLM, so the swap reaches its
|
||||
// next turn without rebuilding the runner, and the model manager re-derives
|
||||
// the references it offers from the new role assignments.
|
||||
// the references it offers from the new role assignments and the catalog.
|
||||
func (b *boot) rebuildChatModelRuntime(cfg config.Config) {
|
||||
if b.chatModels == nil {
|
||||
return // this process serves no chat turns
|
||||
}
|
||||
b.setLLM(agentexec.NewRuntime(executionProviders(cfg)))
|
||||
b.chatModels.SetConfigured(cfg.Models)
|
||||
catalog, models := b.catalogState()
|
||||
b.chatModels.SetModelCatalog(catalog, models)
|
||||
b.log.Info("chat model runtime rebuilt", "providers", len(cfg.Providers), "models", len(cfg.Models))
|
||||
}
|
||||
@@ -26,8 +26,9 @@ func (b *boot) setupChatRuntime(ctx context.Context, cfg config.Config) error {
|
||||
providers := executionProviders(cfg)
|
||||
b.setLLM(agentexec.NewRuntime(providers))
|
||||
b.toolReg = tools.NewRegistry()
|
||||
chatModels := newChatModelManager(cfg.Models, cfg.Chat.Models, b.catalogModels)
|
||||
chatModels.ApplyModelCatalog(b.catalog)
|
||||
chatModels := newChatModelManager(cfg.Models, cfg.Chat.Models)
|
||||
catalog, catalogModels := b.catalogState()
|
||||
chatModels.SetModelCatalog(catalog, catalogModels)
|
||||
b.chatModels = chatModels
|
||||
|
||||
// ── Persona registry ─────────────────────────────────────────────
|
||||
|
||||
@@ -388,6 +388,11 @@ func Run() int { //nolint:cyclop,funlen // the composition root's setup sequence
|
||||
if err := b.setupLLMAndChat(ctx); err != nil {
|
||||
return 1
|
||||
}
|
||||
// The catalog's consumers exist from here on, and the daemon is the
|
||||
// process that owns the published configuration, so this is where the
|
||||
// live catalog refresh belongs. The standalone Gateway reads the catalog
|
||||
// once at boot and runs no control-plane watches (RunGateway).
|
||||
b.startModelCatalogRefresh(ctx)
|
||||
if err := b.buildTreesAndIdentities(ctx); err != nil {
|
||||
return 1
|
||||
}
|
||||
|
||||
@@ -0,0 +1,250 @@
|
||||
package archied
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"io/fs"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"slices"
|
||||
"strings"
|
||||
"sync/atomic"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/samcharles93/archie-core/internal/agentexec"
|
||||
"github.com/samcharles93/archie-core/internal/infrastructure/modelcatalog"
|
||||
"github.com/samcharles93/archie-core/internal/secret"
|
||||
"github.com/samcharles93/archie-core/internal/webui"
|
||||
)
|
||||
|
||||
// catalogServer answers the model catalog read with a models.dev-shaped
|
||||
// document. It is the outside dependency the refresh loop actually meets:
|
||||
// modelcatalog.Load fetches this and caches what it read.
|
||||
func catalogServer(t *testing.T, document func() map[string]any) *httptest.Server {
|
||||
t.Helper()
|
||||
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) {
|
||||
w.Header().Set("Content-Type", "application/json")
|
||||
_ = json.NewEncoder(w).Encode(document())
|
||||
}))
|
||||
t.Cleanup(srv.Close)
|
||||
return srv
|
||||
}
|
||||
|
||||
// catalogProvider is one usable provider entry: an environment variable that
|
||||
// resolves (so modelcatalog keeps it) and the tool-callable models it offers.
|
||||
func catalogProvider(id, env string, models ...string) map[string]any {
|
||||
entries := make(map[string]any, len(models))
|
||||
for _, model := range models {
|
||||
entries[model] = map[string]any{
|
||||
"id": model, "name": strings.ToUpper(model), "tool_call": true,
|
||||
"limit": map[string]any{"context": 128000, "output": 16000},
|
||||
}
|
||||
}
|
||||
return map[string]any{
|
||||
"id": id, "name": strings.ToUpper(id), "npm": "@ai-sdk/openai-compatible",
|
||||
"api": "https://" + id + ".test/v1", "env": []string{env}, "models": entries,
|
||||
}
|
||||
}
|
||||
|
||||
// newCatalogBoot builds the smallest boot the refresh path runs against: the
|
||||
// control plane reads behind runtimeConfig, the secrets registry the catalog
|
||||
// resolves keys through, the chat runtime a refresh must reach, and a catalog
|
||||
// endpoint a test controls.
|
||||
func newCatalogBoot(t *testing.T, url string) *boot {
|
||||
t.Helper()
|
||||
b := newReloadBoot(t, &controlPlaneStub{values: databaseOwnedResources()})
|
||||
b.secrets = secret.NewRegistry()
|
||||
b.chatModels = newChatModelManager(fileConfig().Models)
|
||||
b.setLLM(agentexec.NewRuntime(executionProviders(fileConfig())))
|
||||
b.catalogURL = url
|
||||
b.catalogCachePath = filepath.Join(t.TempDir(), "models.json")
|
||||
if err := b.loadRuntimeConfig(t.Context()); err != nil {
|
||||
t.Fatalf("loadRuntimeConfig: %v", err)
|
||||
}
|
||||
return b
|
||||
}
|
||||
|
||||
func awaitCatalog(t *testing.T, b *boot, ref string) {
|
||||
t.Helper()
|
||||
deadline := time.Now().Add(5 * time.Second)
|
||||
for !slices.Contains(b.chatModels.Models(), ref) {
|
||||
if time.Now().After(deadline) {
|
||||
t.Fatalf("chat models = %v, want %s to appear within the refresh interval", b.chatModels.Models(), ref)
|
||||
}
|
||||
time.Sleep(2 * time.Millisecond)
|
||||
}
|
||||
}
|
||||
|
||||
// TestRefreshModelCatalogRepublishesEveryProjectionSite: a catalog read after
|
||||
// boot must reach everything derived from it -- the running config a workflow
|
||||
// stage resolves model limits from, the dashboard's published catalog view,
|
||||
// and the chat runtime's model list and details. A value that reaches the
|
||||
// boot struct alone parses and does nothing.
|
||||
func TestRefreshModelCatalogRepublishesEveryProjectionSite(t *testing.T) {
|
||||
srv := catalogServer(t, func() map[string]any {
|
||||
return map[string]any{"acme": catalogProvider("acme", "ACME_API_KEY", "acme-tool")}
|
||||
})
|
||||
t.Setenv("ACME_API_KEY", "resolved-by-the-secret-registry")
|
||||
b := newCatalogBoot(t, srv.URL)
|
||||
|
||||
if snapshot, _ := b.catalogState(); len(snapshot.Providers) != 0 {
|
||||
t.Fatalf("boot catalog = %+v, want the empty snapshot this test starts from", snapshot.Providers)
|
||||
}
|
||||
|
||||
if err := b.refreshModelCatalog(t.Context()); err != nil {
|
||||
t.Fatalf("refreshModelCatalog: %v", err)
|
||||
}
|
||||
|
||||
if got := b.cfgHolder.Get().ModelLimits["acme/acme-tool"]; got.ContextWindow != 128000 || got.MaxOutputTokens != 16000 {
|
||||
t.Errorf("running model limits = %+v, want the refreshed catalog's capacity", got)
|
||||
}
|
||||
if _, models := b.catalogState(); !slices.Contains(models, "acme/acme-tool") {
|
||||
t.Errorf("catalog models = %v, want the refreshed reference", models)
|
||||
}
|
||||
|
||||
view := webui.BuildConfigView(b.configViewInput(t.Context()))
|
||||
if len(view.Catalog) != 1 || view.Catalog[0].ID != "acme" {
|
||||
t.Fatalf("published catalog = %+v, want the refreshed provider", view.Catalog)
|
||||
}
|
||||
if !slices.Contains(view.Catalog[0].Models, "acme-tool") {
|
||||
t.Errorf("published catalog models = %v, want the refreshed model", view.Catalog[0].Models)
|
||||
}
|
||||
|
||||
if !slices.Contains(b.chatModels.Models(), "acme/acme-tool") {
|
||||
t.Errorf("chat models = %v, want the refreshed reference offered", b.chatModels.Models())
|
||||
}
|
||||
if _, ok := b.chatModels.ModelDetails("acme/acme-tool"); !ok {
|
||||
t.Error("chat model details missing the refreshed model")
|
||||
}
|
||||
if got := b.chatModels.ProviderDisplayName("acme"); got != "ACME" {
|
||||
t.Errorf("provider display name = %q, want the refreshed catalog's name", got)
|
||||
}
|
||||
}
|
||||
|
||||
// TestModelCatalogRefreshLoopPicksUpANewModelWhileTheDaemonRuns is the
|
||||
// without-a-restart proof: the process is already running on the catalog it
|
||||
// booted with, and a model that appears upstream afterwards is offered by the
|
||||
// next tick.
|
||||
func TestModelCatalogRefreshLoopPicksUpANewModelWhileTheDaemonRuns(t *testing.T) {
|
||||
var calls atomic.Int64
|
||||
srv := catalogServer(t, func() map[string]any {
|
||||
if calls.Add(1) == 1 {
|
||||
return map[string]any{"acme": catalogProvider("acme", "ACME_API_KEY", "acme-one")}
|
||||
}
|
||||
return map[string]any{"acme": catalogProvider("acme", "ACME_API_KEY", "acme-one", "acme-two")}
|
||||
})
|
||||
t.Setenv("ACME_API_KEY", "resolved-by-the-secret-registry")
|
||||
b := newCatalogBoot(t, srv.URL)
|
||||
if err := b.refreshModelCatalog(t.Context()); err != nil {
|
||||
t.Fatalf("initial refresh: %v", err)
|
||||
}
|
||||
if slices.Contains(b.chatModels.Models(), "acme/acme-two") {
|
||||
t.Fatal("the second model was offered before upstream published it")
|
||||
}
|
||||
|
||||
b.catalogRefreshInterval = 5 * time.Millisecond
|
||||
ctx, cancel := context.WithCancel(t.Context())
|
||||
defer cancel()
|
||||
b.startModelCatalogRefresh(ctx)
|
||||
|
||||
awaitCatalog(t, b, "acme/acme-two")
|
||||
if got := b.cfgHolder.Get().ModelLimits["acme/acme-two"]; got.ContextWindow != 128000 {
|
||||
t.Errorf("running model limits for the new model = %+v, want the refreshed capacity", got)
|
||||
}
|
||||
}
|
||||
|
||||
// TestRefreshModelCatalogKeepsTheLoadedCatalogWhenTheReadFails: the catalog
|
||||
// service having a bad minute must not blank the model list the process is
|
||||
// already running on. The failed read is reported; the snapshot stays.
|
||||
func TestRefreshModelCatalogKeepsTheLoadedCatalogWhenTheReadFails(t *testing.T) {
|
||||
srv := catalogServer(t, func() map[string]any {
|
||||
return map[string]any{"acme": catalogProvider("acme", "ACME_API_KEY", "acme-one")}
|
||||
})
|
||||
t.Setenv("ACME_API_KEY", "resolved-by-the-secret-registry")
|
||||
b := newCatalogBoot(t, srv.URL)
|
||||
if err := b.refreshModelCatalog(t.Context()); err != nil {
|
||||
t.Fatalf("refreshModelCatalog: %v", err)
|
||||
}
|
||||
|
||||
// The endpoint stops answering and the cache is gone, so the read has no
|
||||
// fallback left.
|
||||
srv.Close()
|
||||
if err := os.Remove(b.catalogCachePath); err != nil && !errors.Is(err, fs.ErrNotExist) {
|
||||
t.Fatalf("remove cache: %v", err)
|
||||
}
|
||||
if err := b.refreshModelCatalog(t.Context()); err == nil {
|
||||
t.Fatal("refreshModelCatalog error = nil, want the failed read reported")
|
||||
}
|
||||
|
||||
if _, models := b.catalogState(); !slices.Contains(models, "acme/acme-one") {
|
||||
t.Errorf("catalog models = %v, want the loaded catalog kept", models)
|
||||
}
|
||||
if got := b.cfgHolder.Get().ModelLimits["acme/acme-one"]; got.ContextWindow != 128000 {
|
||||
t.Errorf("running model limits = %+v, want the last-known-good catalog", got)
|
||||
}
|
||||
}
|
||||
|
||||
// TestBootCatalogReachesTheRunningConfig is the projection-site check for the
|
||||
// boot half: the catalog boot read must be in the config the process runs on,
|
||||
// not only in the boot struct the chat runtime was built from. Otherwise the
|
||||
// first live update is the first snapshot to carry a catalog model limit.
|
||||
func TestBootCatalogReachesTheRunningConfig(t *testing.T) {
|
||||
srv := catalogServer(t, func() map[string]any {
|
||||
return map[string]any{"acme": catalogProvider("acme", "ACME_API_KEY", "acme-one")}
|
||||
})
|
||||
t.Setenv("ACME_API_KEY", "resolved-by-the-secret-registry")
|
||||
b := newCatalogBoot(t, srv.URL)
|
||||
|
||||
b.loadCatalog(t.Context(), filepath.Join(t.TempDir(), "config.toml"))
|
||||
|
||||
if got := b.cfgHolder.Get().ModelLimits["acme/acme-one"]; got.ContextWindow != 128000 {
|
||||
t.Errorf("running model limits = %+v, want the catalog boot loaded", got)
|
||||
}
|
||||
if _, models := b.catalogState(); !slices.Contains(models, "acme/acme-one") {
|
||||
t.Errorf("catalog models = %v, want the boot catalog", models)
|
||||
}
|
||||
}
|
||||
|
||||
// TestChatModelManagerReplacesItsCatalogRatherThanMerging: a refresh must
|
||||
// drop what the previous snapshot contributed. A manager that only merged
|
||||
// would keep offering a model the catalog no longer publishes, and would
|
||||
// describe a provider that no longer exists.
|
||||
func TestChatModelManagerReplacesItsCatalogRatherThanMerging(t *testing.T) {
|
||||
manager := newChatModelManager(map[string]string{"chat": "acme/one"})
|
||||
manager.SetModelCatalog(
|
||||
modelcatalog.Snapshot{Providers: []modelcatalog.Provider{
|
||||
{ID: "acme", Name: "Acme", Models: []modelcatalog.Model{{ID: "one"}, {ID: "two"}}},
|
||||
}},
|
||||
[]string{"acme/one", "acme/two"},
|
||||
)
|
||||
if !slices.Contains(manager.Models(), "acme/two") {
|
||||
t.Fatalf("chat models = %v, want the catalog's models offered", manager.Models())
|
||||
}
|
||||
if got := manager.ProviderDisplayName("acme"); got != "Acme" {
|
||||
t.Fatalf("provider display name = %q, want the catalog's name", got)
|
||||
}
|
||||
|
||||
manager.SetModelCatalog(
|
||||
modelcatalog.Snapshot{Providers: []modelcatalog.Provider{
|
||||
{ID: "acme", Name: "Acme", Models: []modelcatalog.Model{{ID: "one"}}},
|
||||
}},
|
||||
[]string{"acme/one"},
|
||||
)
|
||||
|
||||
if slices.Contains(manager.Models(), "acme/two") {
|
||||
t.Errorf("chat models = %v, want the withdrawn model dropped", manager.Models())
|
||||
}
|
||||
if _, ok := manager.ModelDetails("acme/two"); ok {
|
||||
t.Error("chat model details still carry the withdrawn model")
|
||||
}
|
||||
if got := manager.ProviderDisplayName("gone"); got != "" {
|
||||
t.Errorf("provider display name for an absent provider = %q, want empty", got)
|
||||
}
|
||||
if !slices.Contains(manager.Models(), "acme/one") {
|
||||
t.Errorf("chat models = %v, want the surviving model kept", manager.Models())
|
||||
}
|
||||
}
|
||||
Reference in new issue
Block a user