diff --git a/internal/app/archied/bootstrap.go b/internal/app/archied/bootstrap.go index 555d31610..1f682de55 100644 --- a/internal/app/archied/bootstrap.go +++ b/internal/app/archied/bootstrap.go @@ -261,8 +261,14 @@ type boot struct { defaultChatIdentity string updateService *releaseupdate.Service - startGateways []func() - capabilityHost *plugin.Host + startGateways []func() + capabilityHost *plugin.Host + // pluginReconciler loads plugin, module and secret-engine files dropped into + // the running config's directories without a restart. Nil before boot's + // startPluginReconcile, and in processes that load no directory (the + // Gateway), where a stored plugin-settings change still layers but loads + // nothing. + pluginReconciler *pluginReconciler trees *worktree.Manager worktreeGrants *worktreerpc.Grants identityRunners []*daemon.IdentityRunner diff --git a/internal/app/archied/control_plane.go b/internal/app/archied/control_plane.go index 600fbcdd1..aa9b8ad3d 100644 --- a/internal/app/archied/control_plane.go +++ b/internal/app/archied/control_plane.go @@ -2,12 +2,14 @@ package archied import ( "context" + "encoding/json" "fmt" "time" "github.com/samcharles93/archie-core/internal/agentexec" "github.com/samcharles93/archie-core/internal/app/controlplane" "github.com/samcharles93/archie-core/internal/config" + "github.com/samcharles93/archie-core/internal/domain/applystatus" "github.com/samcharles93/archie-core/internal/domain/workflow" "github.com/samcharles93/archie-core/internal/infrastructure/configuration" ) @@ -258,16 +260,21 @@ func (b *boot) startLiveSettings(ctx context.Context) error { // (b.runtimeConfig, which applies the execution budgets the settings watch // published) and republishes through config.Holder, so the change takes // effect without a restart. Kinds stay out of this list while a -// startup-built component still holds their value -- tool, plugin, container -// and channel settings are frozen in components built at boot and remain +// startup-built component still holds their value -- tool, container and +// channel settings are frozen in components built at boot and remain // restart-required -- and a kind joins it only with a consumer that re-reads -// it. Each kind is watched in its own goroutine, exactly the shape the -// workflow-execution-settings watch established. +// it. PluginSettingsKind is on the list because its only consumer is the +// directory reconciliation, which reads the running config each tick; a +// removed plugin or engine still cannot unload, so that kind reports the +// removal rather than requiring a restart. Each kind is watched in its own +// goroutine, exactly the shape the workflow-execution-settings watch +// established. var runtimeResourceKinds = []string{ controlplane.ProviderSettingsKind, controlplane.ModelRoleAssignmentsKind, controlplane.RepositoryPoliciesKind, controlplane.SchedulingPolicyKind, + controlplane.PluginSettingsKind, } // startRuntimeResourceWatches keeps a watch per live kind established for the @@ -320,6 +327,13 @@ func (b *boot) applyRuntimeResourceUpdate(ctx context.Context, kind string, upda b.log.Error("runtime settings watch failed", "kind", kind, "err", update.Err) return } + if kind == controlplane.PluginSettingsKind { + if err := b.refuseSkillsDirChange(ctx); err != nil { + b.applyStatus.Report(ctx, kind, update.Version, err) + b.log.Error("plugin settings refused; the running settings stay", "version", update.Version, "err", err) + return + } + } base := b.cfgHolder.Get() // Boot publishes before the model catalog is merged in, so the snapshot // this read returns can lack the catalog's model limits and discovered @@ -339,9 +353,77 @@ func (b *boot) applyRuntimeResourceUpdate(ctx context.Context, kind string, upda switch kind { case controlplane.ProviderSettingsKind, controlplane.ModelRoleAssignmentsKind: b.rebuildChatModelRuntime(cfg) + case controlplane.PluginSettingsKind: + // The document's directories are live now; load what they hold and + // report the reconciliation's own outcome, which is the one that knows + // whether a removal is outstanding. + b.reconcileRuntimePlugins(ctx) } } +// startPluginReconcile establishes the plugin, module and secret-engine +// directory reconciliation on the apply-status restamp interval. Boot calls it +// after loadPlugins, loadWorkflows and configuredSecretRegistry have loaded +// the directories, so the seed records what is already running and the first +// tick loads only what appears or changes afterwards. +func (b *boot) startPluginReconcile(ctx context.Context) { + r := newPluginReconciler(b.log, pluginReconcileTargets{ + host: b.capabilityHost, + secrets: b.secrets, + modules: b.modules, + dirs: func() config.Config { return b.cfgHolder.Get() }, + relayer: func(ctx context.Context) error { + cfg, _, err := b.runtimeConfig(ctx, b.cfgHolder.Get()) + if err != nil { + return err + } + b.publishConfig(ctx, cfg) + return nil + }, + report: func(ctx context.Context, err error) { + b.applyStatus.Report(ctx, controlplane.PluginSettingsKind, b.applyStatus.AppliedVersion(controlplane.PluginSettingsKind), err) + }, + }) + b.pluginReconciler = r + r.seed() + go r.run(ctx, applystatus.RestampInterval) +} + +// reconcileRuntimePlugins runs one directory reconciliation after a stored +// plugin-settings change, so a new directory loads on the tick the document +// lands rather than waiting for the poller. +func (b *boot) reconcileRuntimePlugins(ctx context.Context) { + if b.pluginReconciler == nil { + return + } + _ = b.pluginReconciler.reconcile(ctx) +} + +// refuseSkillsDirChange refuses a stored plugin-settings document whose +// skills_dir differs from the value in force. skills_dir is the one directory +// with no live consumer, so a change to it stays a restart-scoped edit rather +// than a value the daemon reports as applied. The check reads the running +// config, so it cannot live in validatePluginSettings, which is value-agnostic +// by the archie-core-1143 decision. +func (b *boot) refuseSkillsDirChange(ctx context.Context) error { + var stored struct { + SkillsDir string `json:"skills_dir"` + } + _, found, err := b.controlPlane.Query(ctx, controlplane.PluginSettingsKind, func(value []byte) error { + return json.Unmarshal(value, &stored) + }) + if err != nil { + return fmt.Errorf("read plugin settings: %w", err) + } + if !found { + return nil + } + if running := b.cfgHolder.Get().SkillsDir; stored.SkillsDir != running { + return fmt.Errorf("skills_dir applies on restart: stored %q, running %q", stored.SkillsDir, running) + } + return nil +} + // rebuildChatModelRuntime re-derives the gateway chat runtime's provider set // after a live change to provider-settings or model-role-assignments. The // turn runner reads the runtime through boot.chatLLM, so the swap reaches its diff --git a/internal/app/archied/main.go b/internal/app/archied/main.go index a95c1f6e7..e40ae037f 100644 --- a/internal/app/archied/main.go +++ b/internal/app/archied/main.go @@ -411,6 +411,9 @@ func Run() int { //nolint:cyclop,funlen // the composition root's setup sequence if err := b.loadPlugins(); err != nil { return 1 } + // Every directory loader has run by here, so the reconciliation seeds the + // running set and from its first tick loads only what appears or changes. + b.startPluginReconcile(ctx) if err := b.registerNATSRPC(); err != nil { return 1 diff --git a/internal/app/archied/plugin_reconcile.go b/internal/app/archied/plugin_reconcile.go new file mode 100644 index 000000000..cef081dd1 --- /dev/null +++ b/internal/app/archied/plugin_reconcile.go @@ -0,0 +1,306 @@ +package archied + +import ( + "context" + "crypto/sha256" + "encoding/hex" + "errors" + "fmt" + "io/fs" + "log/slog" + "os" + "path/filepath" + "slices" + "strings" + "sync" + "time" + + "github.com/samcharles93/archie-core/internal/config" + "github.com/samcharles93/archie-core/internal/domain/eda/module" + "github.com/samcharles93/archie-core/internal/plugin" + "github.com/samcharles93/archie-core/internal/plugin/pluginextract" + "github.com/samcharles93/archie-core/internal/secret" +) + +// The three directory families a plugin-settings document names and a +// reconciliation reads. +const ( + reconcilePlugins = "plugin" + reconcileModules = "module" + reconcileSecrets = "secret" +) + +// pluginReconciler loads plugin, module and secret-engine files that appear or +// change in the directories the running configuration names, so an operator can +// drop a file beside a running daemon and have it take effect without a +// restart. Yaegi cannot unload an interpreter, so a removed file's code keeps +// running and the reconciler reports it through apply status until the process +// restarts (docs/prds/plugin-settings-live.md). +// +// Files are content-hashed, so a tick loads only what appeared or changed: the +// boot load's files are seeded as already loaded. A file whose load fails is +// forgotten until its content changes, the same degrade-and-skip rule the boot +// loaders apply. +type pluginReconciler struct { + log *slog.Logger + pluginReconcileTargets + + // reconcileMu serializes reconciliations: the poller's tick and a stored + // plugin-settings update both run one, and they share the loaded set. + reconcileMu sync.Mutex + loaded map[string]loadedPluginFile +} + +type pluginReconcileTargets struct { + host *plugin.Host + secrets *secret.Registry + modules *module.ModuleRegistry + // dirs returns the running configuration, the source of the three + // directories. It is read each tick so a changed directory retargets the + // next reconciliation without a restart. + dirs func() config.Config + // relayer re-runs the configuration layering after a secret engine loads + // or changes, which re-resolves provider credentials against the new + // engine. Nil in tests that do not layer. + relayer func(ctx context.Context) error + // report publishes the reconciliation outcome, including a nil error on a + // clean pass so an outstanding removal clears. + report func(ctx context.Context, err error) +} + +type loadedPluginFile struct { + hash string + category string + // failed marks a file whose last load failed. It is not reported as a + // removal while it is still present, and a content change retries it. + failed bool +} + +func newPluginReconciler(log *slog.Logger, targets pluginReconcileTargets) *pluginReconciler { + if log == nil { + log = slog.New(slog.DiscardHandler) + } + return &pluginReconciler{ + log: log, + pluginReconcileTargets: targets, + loaded: make(map[string]loadedPluginFile), + } +} + +// seed records the files the boot load already evaluated, so the first tick +// loads only what appeared or changed since boot. The daemon calls it after +// loadPlugins, loadWorkflows and configuredSecretRegistry have run. +func (r *pluginReconciler) seed() { + next := make(map[string]loadedPluginFile) + for _, spec := range r.categories(r.dirs()) { + paths, err := goFiles(spec.dir) + if err != nil { + r.log.Warn("plugin directory not seeded", "category", spec.category, "dir", spec.dir, "err", err) + continue + } + for _, path := range paths { + hash, err := fileHash(path) + if err != nil { + continue + } + next[path] = loadedPluginFile{hash: hash, category: spec.category} + } + } + r.reconcileMu.Lock() + r.loaded = next + r.reconcileMu.Unlock() +} + +// run reconciles on interval until ctx ends. The interval is the apply-status +// restamp constant, so a dropped file needs no filesystem watch and a +// reconciliation report is refreshed at the same cadence as every other apply +// status record. +func (r *pluginReconciler) run(ctx context.Context, interval time.Duration) { + ticker := time.NewTicker(interval) + defer ticker.Stop() + for { + select { + case <-ctx.Done(): + return + case <-ticker.C: + _ = r.reconcile(ctx) + } + } +} + +type reconcileCategory struct { + category string + dir string +} + +func (r *pluginReconciler) categories(cfg config.Config) []reconcileCategory { + return []reconcileCategory{ + {reconcilePlugins, cfg.PluginDir}, + {reconcileModules, cfg.ModuleDir}, + {reconcileSecrets, cfg.SecretEngineDir}, + } +} + +// reconcile re-reads the three directories against the running config and +// loads every new or changed file. It returns the outstanding problems -- a +// removal, a failed load, an unreadable directory -- joined, and reports the +// same through apply status. A nil return is a clean pass and clears the +// record's error. +func (r *pluginReconciler) reconcile(ctx context.Context) error { + r.reconcileMu.Lock() + defer r.reconcileMu.Unlock() + + var problems []error + next := make(map[string]loadedPluginFile, len(r.loaded)) + + for _, spec := range r.categories(r.dirs()) { + problems = append(problems, r.reconcileCategory(ctx, spec, next)...) + } + + for path, file := range r.loaded { + if _, ok := next[path]; ok || file.failed { + continue + } + problems = append(problems, fmt.Errorf("%s file %s was removed; it keeps running until the process restarts", file.category, path)) + } + + r.loaded = next + + // The layering re-runs every tick, not only when a secret engine changed: + // it re-resolves provider credentials, so a provider disabled at boot for + // an unresolvable engine is re-resolved on the first tick that finds it, + // and a changed directory retargets the next tick without a restart + // (docs/prds/plugin-settings-live.md, "Triggers"). + if r.relayer != nil { + if err := r.relayer(ctx); err != nil { + problems = append(problems, fmt.Errorf("re-resolve provider credentials: %w", err)) + } + } + + err := errors.Join(problems...) + if r.report != nil { + r.report(ctx, err) + } + return err +} + +// reconcileCategory loads the new and changed files of one directory into +// next, returning the problems it met. A directory that became unreadable +// keeps the entries this category already had: it is not a removal of the +// files loaded before that. +func (r *pluginReconciler) reconcileCategory(ctx context.Context, spec reconcileCategory, next map[string]loadedPluginFile) []error { + var problems []error + paths, err := goFiles(spec.dir) + if err != nil { + problems = append(problems, fmt.Errorf("%s directory %q: %w", spec.category, spec.dir, err)) + for path, file := range r.loaded { + if file.category == spec.category { + next[path] = file + } + } + return problems + } + for _, path := range paths { + hash, err := fileHash(path) + if err != nil { + problems = append(problems, fmt.Errorf("read %s: %w", path, err)) + continue + } + if previous, ok := r.loaded[path]; ok && previous.hash == hash { + next[path] = previous + continue + } + if err := r.load(ctx, spec.category, path); err != nil { + problems = append(problems, fmt.Errorf("%s %s: %w", spec.category, path, err)) + // Record the failed content so the same broken file is not retried + // every tick; changing it retries the load. + next[path] = loadedPluginFile{hash: hash, category: spec.category, failed: true} + continue + } + next[path] = loadedPluginFile{hash: hash, category: spec.category} + r.log.Info("runtime directory file loaded", "category", spec.category, "path", path) + } + return problems +} + +func (r *pluginReconciler) load(ctx context.Context, category, path string) error { + switch category { + case reconcilePlugins: + return r.loadPlugin(ctx, path) + case reconcileModules: + return r.loadModule(path) + case reconcileSecrets: + if r.secrets == nil { + return errors.New("no secret registry to load into") + } + return r.secrets.LoadFile(path, secretEngineSymbols...) + default: + return fmt.Errorf("unknown reconcile category %q", category) + } +} + +func (r *pluginReconciler) loadPlugin(ctx context.Context, path string) error { + if r.host == nil { + return errors.New("no plugin host to load into") + } + loaded, err := plugin.LoadFile(path, pluginextract.Symbols) + if err != nil { + return err + } + module, err := plugin.AdaptLegacy(loaded) + if err != nil { + return err + } + // The manifest id is the plugin's name, so an edited file that keeps its + // name replaces the running module and a new name adds one. + if r.host.Has(module.Manifest().ID) { + return r.host.Replace(ctx, module) + } + return r.host.Add(ctx, module) +} + +func (r *pluginReconciler) loadModule(path string) error { + if r.modules == nil { + return errors.New("no module registry to load into") + } + kind := strings.TrimSuffix(filepath.Base(path), ".go") + if !slices.Contains(module.Kinds(), kind) { + // A .go file for a kind this build has no contract for; the boot load + // iterates the known kinds and ignores it too. + return nil + } + return r.modules.Register(kind, filepath.Dir(path)) +} + +// goFiles lists the .go files directly under dir, sorted. A missing directory +// reconciles to an empty set, the convention every boot loader already follows. +func goFiles(dir string) ([]string, error) { + if dir == "" { + return nil, nil + } + entries, err := os.ReadDir(dir) + if errors.Is(err, fs.ErrNotExist) { + return nil, nil + } + if err != nil { + return nil, err + } + paths := make([]string, 0, len(entries)) + for _, entry := range entries { + if entry.IsDir() || !strings.HasSuffix(entry.Name(), ".go") { + continue + } + paths = append(paths, filepath.Join(dir, entry.Name())) + } + slices.Sort(paths) + return paths, nil +} + +func fileHash(path string) (string, error) { + content, err := os.ReadFile(path) + if err != nil { + return "", err + } + sum := sha256.Sum256(content) + return hex.EncodeToString(sum[:]), nil +} diff --git a/internal/app/archied/plugin_reconcile_test.go b/internal/app/archied/plugin_reconcile_test.go new file mode 100644 index 000000000..87b9e294a --- /dev/null +++ b/internal/app/archied/plugin_reconcile_test.go @@ -0,0 +1,297 @@ +package archied + +import ( + "context" + "log/slog" + "os" + "path/filepath" + "strings" + "testing" + + "github.com/samcharles93/archie-core/internal/app/controlplane" + "github.com/samcharles93/archie-core/internal/config" + "github.com/samcharles93/archie-core/internal/domain/eda/module" + "github.com/samcharles93/archie-core/internal/plugin" + "github.com/samcharles93/archie-core/internal/secret" +) + +// docs/prds/plugin-settings-live.md: a plugin or secret-engine file dropped +// into the running configuration's directory loads without a restart, an +// edited file replaces what runs, and a removed file keeps running with the +// removal reported through apply status until the process restarts. + +const reconcilerPluginSource = `package main + +import "github.com/samcharles93/archie-core/internal/plugin" + +var Plugin = plugin._Plugin{ + WName: func() string { return "hello" }, + WVersion: func() string { return "1.0.0" }, +} +` + +const reconcilerSecretSource = `package main + +import "github.com/samcharles93/archie-core/internal/secret" + +var Engine = secret._Engine{ + WName: func() string { return "hello" }, + WVersion: func() string { return "1.0.0" }, + WResolve: func(key string) (string, error) { return "one:" + key, nil }, +} +` + +const reconcilerEditedSecretSource = `package main + +import "github.com/samcharles93/archie-core/internal/secret" + +var Engine = secret._Engine{ + WName: func() string { return "hello" }, + WVersion: func() string { return "2.0.0" }, + WResolve: func(key string) (string, error) { return "two:" + key, nil }, +} +` + +// reconcilerHarness is one running directory set: a started plugin host, a +// fresh secret registry, an empty module registry, and the config that names +// the two directories a test writes into. +type reconcilerHarness struct { + root *plugin.Host + secrets *secret.Registry + cfg config.Config + plugins string + secretsD string + mods *module.ModuleRegistry +} + +func newReconcilerHarness(t *testing.T) *reconcilerHarness { + t.Helper() + h := &reconcilerHarness{ + root: plugin.NewHost(), + secrets: secret.NewRegistry(), + mods: module.New(), + plugins: t.TempDir(), + secretsD: t.TempDir(), + } + if err := h.root.Start(t.Context()); err != nil { + t.Fatalf("start host: %v", err) + } + h.cfg = config.Config{PluginDir: h.plugins, SecretEngineDir: h.secretsD} + return h +} + +func (h *reconcilerHarness) reconciler() *pluginReconciler { + return newPluginReconciler(slog.New(slog.DiscardHandler), pluginReconcileTargets{ + host: h.root, + secrets: h.secrets, + modules: h.mods, + dirs: func() config.Config { return h.cfg }, + }) +} + +func writeReconcilerFile(t *testing.T, path, source string) { + t.Helper() + if err := os.WriteFile(path, []byte(source), 0o644); err != nil { + t.Fatal(err) + } +} + +func TestPluginReconcilerLoadsADroppedPluginAndSecretEngine(t *testing.T) { + h := newReconcilerHarness(t) + r := h.reconciler() + // Boot loaded the (empty) directories; seed is what the daemon does after + // its own boot load so the first tick does not re-load the boot set. + r.seed() + + writeReconcilerFile(t, filepath.Join(h.plugins, "hello.go"), reconcilerPluginSource) + writeReconcilerFile(t, filepath.Join(h.secretsD, "hello.go"), reconcilerSecretSource) + + if err := r.reconcile(t.Context()); err != nil { + t.Fatalf("reconcile: %v", err) + } + if !h.root.Has("hello") { + t.Error("dropped plugin was not loaded in place") + } + engine, ok := h.secrets.Get("hello") + if !ok { + t.Fatal("dropped secret engine was not loaded in place") + } + value, err := engine.Resolve("k") + if err != nil { + t.Fatalf("Resolve: %v", err) + } + if value != "one:k" { + t.Errorf("Resolve() = %q, want one:k", value) + } +} + +func TestPluginReconcilerReplacesAnEditedSecretEngine(t *testing.T) { + h := newReconcilerHarness(t) + r := h.reconciler() + r.seed() + + path := filepath.Join(h.secretsD, "hello.go") + writeReconcilerFile(t, path, reconcilerSecretSource) + if err := r.reconcile(t.Context()); err != nil { + t.Fatalf("first reconcile: %v", err) + } + + writeReconcilerFile(t, path, reconcilerEditedSecretSource) + if err := r.reconcile(t.Context()); err != nil { + t.Fatalf("second reconcile: %v", err) + } + engine, ok := h.secrets.Get("hello") + if !ok { + t.Fatal("edited secret engine disappeared") + } + value, err := engine.Resolve("k") + if err != nil { + t.Fatalf("Resolve: %v", err) + } + if value != "two:k" { + t.Errorf("Resolve() after edit = %q, want the replaced two:k", value) + } +} + +func TestPluginReconcilerReportsARemovedFileThatKeepsRunning(t *testing.T) { + h := newReconcilerHarness(t) + r := h.reconciler() + r.seed() + + pluginPath := filepath.Join(h.plugins, "hello.go") + secretPath := filepath.Join(h.secretsD, "hello.go") + writeReconcilerFile(t, pluginPath, reconcilerPluginSource) + writeReconcilerFile(t, secretPath, reconcilerSecretSource) + if err := r.reconcile(t.Context()); err != nil { + t.Fatalf("first reconcile: %v", err) + } + + if err := os.Remove(pluginPath); err != nil { + t.Fatal(err) + } + if err := os.Remove(secretPath); err != nil { + t.Fatal(err) + } + err := r.reconcile(t.Context()) + if err == nil { + t.Fatal("a reconcile after two removals reported no problem") + } + for _, want := range []string{"hello.go", "removed", "restart"} { + if !strings.Contains(err.Error(), want) { + t.Errorf("removal report %q does not mention %q", err, want) + } + } + // The running code is not unloaded: Yaegi has no unload path, so both the + // module and the engine keep serving until the process restarts. + if !h.root.Has("hello") { + t.Error("a removed plugin stopped running") + } + if _, ok := h.secrets.Get("hello"); !ok { + t.Error("a removed secret engine was unregistered") + } + + // Re-adding the file clears the removal: the replacement takes over and no + // old interpreter is left serving the path. + writeReconcilerFile(t, pluginPath, reconcilerPluginSource) + writeReconcilerFile(t, secretPath, reconcilerSecretSource) + if err := r.reconcile(t.Context()); err != nil { + t.Fatalf("reconcile after re-adding the files: %v", err) + } +} + +func TestPluginSettingsLiveUpdateLoadsTheNewDirectory(t *testing.T) { + dir := t.TempDir() + writeReconcilerFile(t, filepath.Join(dir, "hello.go"), reconcilerPluginSource) + + running := fileConfig() + b, status := newLiveApplyBoot(t, running) + resources := databaseOwnedResources() + resources[controlplane.PluginSettingsKind] = map[string]any{ + "plugin_dir": dir, "module_dir": "/db/modules", "secret_engine_dir": "/db/secrets", "skills_dir": "", + } + b.controlPlane = controlplane.NewRPCClient(&controlPlaneStub{values: resources}) + b.secrets = secret.NewRegistry() + host := plugin.NewHost() + if err := host.Start(t.Context()); err != nil { + t.Fatalf("start host: %v", err) + } + b.pluginReconciler = newPluginReconciler(b.log, pluginReconcileTargets{ + host: host, + secrets: b.secrets, + modules: module.New(), + dirs: func() config.Config { return b.cfgHolder.Get() }, + }) + b.pluginReconciler.seed() + + b.applyRuntimeResourceUpdate(t.Context(), controlplane.PluginSettingsKind, controlplane.AppliedResource{Version: 2}) + + if got := b.cfgHolder.Get().PluginDir; got != dir { + t.Fatalf("running plugin dir = %q, want the stored %q", got, dir) + } + if !host.Has("hello") { + t.Fatal("a stored plugin_dir change did not load its plugin without a restart") + } + last, ok := status.last(controlplane.PluginSettingsKind) + if !ok { + t.Fatal("the live plugin-settings update reported no apply status") + } + if last.Error != "" { + t.Errorf("apply status error = %q, want none for a live directory load", last.Error) + } +} + +func TestPluginSettingsSkillsDirChangeIsRefusedOnTheLivePath(t *testing.T) { + running := fileConfig() + b, status := newLiveApplyBoot(t, running) + // The stored document's skills_dir (/db/skills) differs from the running + // empty value, and skills_dir has no live consumer. + b.controlPlane = controlplane.NewRPCClient(&controlPlaneStub{values: databaseOwnedResources()}) + + b.applyRuntimeResourceUpdate(t.Context(), controlplane.PluginSettingsKind, controlplane.AppliedResource{Version: 2}) + + if got := b.cfgHolder.Get().PluginDir; got != fileConfig().PluginDir { + t.Errorf("a refused plugin-settings document published plugin_dir %q, want the file's %q", got, fileConfig().PluginDir) + } + last, ok := status.last(controlplane.PluginSettingsKind) + if !ok { + t.Fatal("the refused plugin-settings update reported no apply status") + } + if !strings.Contains(last.Error, "skills_dir") { + t.Errorf("apply status error = %q, want it to name skills_dir", last.Error) + } + if last.AppliedVersion != 2 { + t.Errorf("apply status version = %d, want the refused version 2", last.AppliedVersion) + } +} + +func TestPluginReconcilerReportsTheApplyStatusTruth(t *testing.T) { + h := newReconcilerHarness(t) + var reported []error + reported = nil + r := newPluginReconciler(slog.New(slog.DiscardHandler), pluginReconcileTargets{ + host: h.root, + secrets: h.secrets, + modules: h.mods, + dirs: func() config.Config { return h.cfg }, + report: func(_ context.Context, err error) { reported = append(reported, err) }, + }) + r.seed() + + writeReconcilerFile(t, filepath.Join(h.secretsD, "hello.go"), reconcilerSecretSource) + if err := r.reconcile(t.Context()); err != nil { + t.Fatalf("clean reconcile: %v", err) + } + if len(reported) != 1 || reported[0] != nil { + t.Fatalf("clean reconcile reported %v, want one nil (a live add is applied)", reported) + } + + if err := os.Remove(filepath.Join(h.secretsD, "hello.go")); err != nil { + t.Fatal(err) + } + if err := r.reconcile(t.Context()); err == nil { + t.Fatal("removal reconcile reported no error") + } + if len(reported) != 2 || reported[1] == nil { + t.Fatalf("removal reconcile reported %v, want the removal as the second record", reported) + } +} diff --git a/internal/app/archied/provider_secrets.go b/internal/app/archied/provider_secrets.go index fb32f2926..725cfd94d 100644 --- a/internal/app/archied/provider_secrets.go +++ b/internal/app/archied/provider_secrets.go @@ -5,6 +5,7 @@ import ( "fmt" "log/slog" "os" + "reflect" "strings" "github.com/traefik/yaegi/stdlib/unrestricted" @@ -15,6 +16,15 @@ import ( "github.com/samcharles93/archie-core/internal/secret/secretextract" ) +// secretEngineSymbols is the Yaegi symbol set every secret-engine load uses: +// the generated interface bridge, the host-side CLI helper, and unrestricted +// symbols for parity with skillscript interpretation. The boot load and a +// live directory reconciliation share it so an engine always loads the same +// way. +var secretEngineSymbols = []map[string]map[string]reflect.Value{ + secretextract.Symbols, enginehost.Symbols, unrestricted.Symbols, +} + func configuredSecretRegistry(cfg *config.Config, log *slog.Logger) (*secret.Registry, error) { registry := secret.NewRegistry() if cfg.SecretEngineDir != "" { @@ -24,7 +34,7 @@ func configuredSecretRegistry(cfg *config.Config, log *slog.Logger) (*secret.Reg // host-side helper) because yaegi's interpreted os/exec cannot // pass an environment to child processes; unrestricted symbols // are included for parity with skillscript interpretation. - loaded, err := registry.LoadDir(cfg.SecretEngineDir, secretextract.Symbols, enginehost.Symbols, unrestricted.Symbols) + loaded, err := registry.LoadDir(cfg.SecretEngineDir, secretEngineSymbols...) if err != nil { return nil, fmt.Errorf("load secret engines from %q: %w", cfg.SecretEngineDir, err) } diff --git a/internal/app/controlplane/operational_settings.go b/internal/app/controlplane/operational_settings.go index 6dd695663..69ba9f1c2 100644 --- a/internal/app/controlplane/operational_settings.go +++ b/internal/app/controlplane/operational_settings.go @@ -46,7 +46,12 @@ func operationalDefinitions() []Definition { return schedulingPolicy{PollInterval: cfg.PollInterval.Std().String(), MaxRetries: cfg.MaxRetries, Label: &cfg.Label, Dispatch: cfg.Dispatch} }, Validate: validateScheduling}, {Kind: ToolSettingsKind, Title: "Tool and MCP settings", ApplyMode: "restart-required", Document: toolSettings{}, Seed: seedTools, Validate: validateTools}, - {Kind: PluginSettingsKind, Title: "Plugin settings", ApplyMode: "restart-required", Document: pluginSettings{}, Seed: func(cfg config.Config) any { + // ApplyMode is live for additions: a stored directory change re-layers live + // and the daemon's reconciliation loads new and changed files without a + // restart. A removal cannot unload Yaegi's interpreter, so it stays an + // outstanding apply-status problem rather than a restart of the whole kind + // (docs/prds/plugin-settings-live.md). + {Kind: PluginSettingsKind, Title: "Plugin settings", ApplyMode: "live", Document: pluginSettings{}, Seed: func(cfg config.Config) any { return pluginSettings{cfg.PluginDir, cfg.ModuleDir, cfg.SecretEngineDir, cfg.SkillsDir} }, Validate: validatePluginSettings}, {Kind: ContainerRuntimePoliciesKind, Title: "Container runtime policies", ApplyMode: "restart-required", Document: containerRuntimePolicies{}, Seed: seedContainerPolicies, Validate: validateContainers, Normalize: normalizeContainerPolicies}, diff --git a/internal/app/controlplane/runtime_config_test.go b/internal/app/controlplane/runtime_config_test.go index 5e9b559bd..7f0f7b0a3 100644 --- a/internal/app/controlplane/runtime_config_test.go +++ b/internal/app/controlplane/runtime_config_test.go @@ -553,13 +553,13 @@ func TestRuntimeResourceKindsApplyLive(t *testing.T) { modes[definition.Kind] = definition.ApplyMode } for _, kind := range []string{ - ProviderSettingsKind, ModelRoleAssignmentsKind, RepositoryPoliciesKind, SchedulingPolicyKind, AgentProfileKind, CredentialBindingsKind, + ProviderSettingsKind, ModelRoleAssignmentsKind, RepositoryPoliciesKind, SchedulingPolicyKind, AgentProfileKind, CredentialBindingsKind, PluginSettingsKind, } { if modes[kind] != "live" { t.Errorf("%s applies %q, want live: the daemon re-layers this kind on a watch", kind, modes[kind]) } } - for _, kind := range []string{ToolSettingsKind, PluginSettingsKind, ContainerRuntimePoliciesKind, ChannelSettingsKind} { + for _, kind := range []string{ToolSettingsKind, ContainerRuntimePoliciesKind, ChannelSettingsKind} { if modes[kind] != "restart-required" { t.Errorf("%s applies %q, want restart-required: a startup-built component still holds it", kind, modes[kind]) } diff --git a/internal/domain/applystatus/applystatus.go b/internal/domain/applystatus/applystatus.go index 5a0a4c5d9..aa30cf604 100644 --- a/internal/domain/applystatus/applystatus.go +++ b/internal/domain/applystatus/applystatus.go @@ -107,6 +107,20 @@ func (r *Reporter) Report(ctx context.Context, kind string, version int64, apply r.write(ctx, status) } +// AppliedVersion returns the version last recorded for kind, or 0 when this +// process has reported none. A caller that reports an outcome rather than an +// application -- the plugin directory reconciliation, which reports against +// the plugin-settings version the process is already running -- reads it so a +// clean pass does not reset the version to zero. +func (r *Reporter) AppliedVersion(kind string) int64 { + if r == nil { + return 0 + } + r.mu.Lock() + defer r.mu.Unlock() + return r.applied[kind].AppliedVersion +} + // Run re-stamps this process's records until ctx ends. func (r *Reporter) Run(ctx context.Context) { if r == nil { diff --git a/internal/plugin/host.go b/internal/plugin/host.go index 7e2148b0e..ab93a34d6 100644 --- a/internal/plugin/host.go +++ b/internal/plugin/host.go @@ -263,17 +263,10 @@ func (h *Host) Register(module Module) error { h.opMu.Lock() defer h.opMu.Unlock() - if isNilModule(module) { - return errors.New("plugin module is nil") - } - manifest, err := safeManifest(module) + manifest, err := prepareModule(module) if err != nil { return err } - if err := manifest.Validate(); err != nil { - return err - } - manifest = manifest.clone() h.mu.Lock() defer h.mu.Unlock() @@ -501,3 +494,182 @@ func reverseClone(values []string) []string { } return out } + +// Add registers and starts a module on a host that is already running, the +// running-state half of a live plugin load (docs/prds/plugin-settings-live.md). +// The module starts on its own -- every module registered before it is already +// running -- and a failed start removes it again without disturbing them. On a +// host that has not started, Add behaves like Register and the module starts +// with the rest. +func (h *Host) Add(ctx context.Context, module Module) error { + h.opMu.Lock() + defer h.opMu.Unlock() + + manifest, err := prepareModule(module) + if err != nil { + return err + } + + h.mu.Lock() + h.ensureModulesLocked() + if _, exists := h.modules[manifest.ID]; exists { + h.mu.Unlock() + return fmt.Errorf("plugin module %q is already registered", manifest.ID) + } + h.modules[manifest.ID] = ®isteredModule{module: module, manifest: manifest} + h.registration = append(h.registration, manifest.ID) + state := h.state + h.mu.Unlock() + + switch state { + case hostIdle: + // Not started yet: Start will pick this module up with the rest. + return nil + case hostRunning: + default: + h.remove(manifest.ID) + return fmt.Errorf("plugin host cannot add a module from state %d", state) + } + + // Every registered module is running while the host is running, so a + // dependency that is not registered cannot be satisfied by a later module + // in this pass; refuse and let the next reconciliation retry once it is. + for _, dependency := range manifest.Dependencies { + if !h.registered(dependency) { + h.remove(manifest.ID) + return fmt.Errorf("plugin module %q depends on unregistered module %q", manifest.ID, dependency) + } + } + + if err := safeStart(ctx, manifest.ID, module); err != nil { + h.remove(manifest.ID) + return err + } + h.mu.Lock() + h.started = append(h.started, manifest.ID) + h.mu.Unlock() + return nil +} + +// Replace swaps the module whose manifest id matches for a new one while the +// host is running. The old module is stopped and the new one started; when +// the new start fails the old one is restarted, so a refused replacement +// leaves the running set as it was. Replacing a module another running module +// depends on is refused rather than swapped, because the dependent would keep +// calling a stopped implementation. +func (h *Host) Replace(ctx context.Context, module Module) error { + h.opMu.Lock() + defer h.opMu.Unlock() + + manifest, err := prepareModule(module) + if err != nil { + return err + } + + h.mu.Lock() + previous, exists := h.modules[manifest.ID] + if !exists { + h.mu.Unlock() + return fmt.Errorf("plugin module %q is not registered", manifest.ID) + } + if dependent, ok := h.runningDependentLocked(manifest.ID); ok { + h.mu.Unlock() + return fmt.Errorf("plugin module %q is required by running module %q and cannot be replaced", manifest.ID, dependent) + } + wasStarted := slices.Contains(h.started, manifest.ID) + h.mu.Unlock() + + if !wasStarted { + // Not running: swap the registration only, so a later Start uses the + // replacement. + h.mu.Lock() + h.modules[manifest.ID] = ®isteredModule{module: module, manifest: manifest} + h.mu.Unlock() + return nil + } + + if err := safeStop(ctx, manifest.ID, previous.module); err != nil { + return err + } + if err := safeStart(ctx, manifest.ID, module); err != nil { + rollbackCtx, cancel := context.WithTimeout(context.WithoutCancel(ctx), rollbackTimeout) + restartErr := safeStart(rollbackCtx, manifest.ID, previous.module) + cancel() + return errors.Join(err, restartErr) + } + h.mu.Lock() + h.modules[manifest.ID] = ®isteredModule{module: module, manifest: manifest} + h.mu.Unlock() + return nil +} + +// Has reports whether a module with id is registered, running or not. +func (h *Host) Has(id string) bool { + h.mu.RLock() + defer h.mu.RUnlock() + _, ok := h.modules[id] + return ok +} + +// Manifests returns the registered modules' manifests sorted by id. It is the +// read side of the running set: a live load reports what is loaded from here. +func (h *Host) Manifests() []Manifest { + h.mu.RLock() + defer h.mu.RUnlock() + out := make([]Manifest, 0, len(h.modules)) + for _, entry := range h.modules { + out = append(out, entry.manifest.clone()) + } + slices.SortFunc(out, func(a, b Manifest) int { return strings.Compare(a.ID, b.ID) }) + return out +} + +// prepareModule validates a module and returns its cloned manifest, the +// validation Register, Add and Replace all share. +func prepareModule(module Module) (Manifest, error) { + if isNilModule(module) { + return Manifest{}, errors.New("plugin module is nil") + } + manifest, err := safeManifest(module) + if err != nil { + return Manifest{}, err + } + if err := manifest.Validate(); err != nil { + return Manifest{}, err + } + return manifest.clone(), nil +} + +// registered reports whether id is registered. The caller holds no lock. +func (h *Host) registered(id string) bool { + h.mu.RLock() + defer h.mu.RUnlock() + _, ok := h.modules[id] + return ok +} + +// runningDependentLocked returns a started module other than id that depends +// on id, which is what makes replacing id unsafe. The caller holds h.mu. +func (h *Host) runningDependentLocked(id string) (string, bool) { + for _, startedID := range h.started { + if startedID == id { + continue + } + entry, ok := h.modules[startedID] + if !ok { + continue + } + if slices.Contains(entry.manifest.Dependencies, id) { + return startedID, true + } + } + return "", false +} + +// remove unregisters id, the rollback of a refused Add. +func (h *Host) remove(id string) { + h.mu.Lock() + defer h.mu.Unlock() + delete(h.modules, id) + h.registration = slices.DeleteFunc(h.registration, func(v string) bool { return v == id }) +} diff --git a/internal/plugin/host_running_test.go b/internal/plugin/host_running_test.go new file mode 100644 index 000000000..d56e92af2 --- /dev/null +++ b/internal/plugin/host_running_test.go @@ -0,0 +1,200 @@ +package plugin_test + +import ( + "context" + "errors" + "slices" + "strings" + "testing" + + "github.com/samcharles93/archie-core/internal/plugin" +) + +// The running-state operations a live plugin load needs: Add puts one new +// module beside the running set, Replace swaps one module's implementation, +// and neither disturbs a module it is not touching. +// docs/prds/plugin-settings-live.md ("The three registries"). + +func TestHostAddStartsANewModuleWhileRunning(t *testing.T) { + t.Parallel() + + events := &eventLog{} + host := plugin.NewHost() + base := &fakeModule{manifest: validManifest("base"), events: events} + if err := host.Register(base); err != nil { + t.Fatal(err) + } + if err := host.Start(context.Background()); err != nil { + t.Fatal(err) + } + + added := &fakeModule{manifest: validManifest("late", "base"), events: events} + if err := host.Add(context.Background(), added); err != nil { + t.Fatalf("Add() error = %v", err) + } + if !host.Has("late") { + t.Fatal("Add() did not register the module") + } + if got, want := events.snapshot(), []string{"start:base", "start:late"}; !slices.Equal(got, want) { + t.Fatalf("lifecycle events = %v, want %v", got, want) + } + // The already-running set is stopped after the added module, since the + // added module came later. + if err := host.Stop(context.Background()); err != nil { + t.Fatal(err) + } + if got, want := events.snapshot(), []string{"start:base", "start:late", "stop:late", "stop:base"}; !slices.Equal(got, want) { + t.Fatalf("lifecycle events = %v, want %v", got, want) + } +} + +func TestHostAddRefusesDuplicateAndUnknownDependency(t *testing.T) { + t.Parallel() + + host := plugin.NewHost() + if err := host.Register(&fakeModule{manifest: validManifest("base")}); err != nil { + t.Fatal(err) + } + if err := host.Start(context.Background()); err != nil { + t.Fatal(err) + } + + if err := host.Add(context.Background(), &fakeModule{manifest: validManifest("base")}); err == nil { + t.Fatal("Add() of a duplicate id succeeded") + } + if err := host.Add(context.Background(), &fakeModule{manifest: validManifest("dependent", "missing")}); err == nil { + t.Fatal("Add() with an unregistered dependency succeeded") + } + if host.Has("dependent") { + t.Error("a refused Add() left the module registered") + } +} + +func TestHostAddRollsBackItsOwnStartFailure(t *testing.T) { + t.Parallel() + + events := &eventLog{} + host := plugin.NewHost() + if err := host.Register(&fakeModule{manifest: validManifest("base"), events: events}); err != nil { + t.Fatal(err) + } + if err := host.Start(context.Background()); err != nil { + t.Fatal(err) + } + + broken := &fakeModule{manifest: validManifest("late"), events: events, startErr: errors.New("boom")} + if err := host.Add(context.Background(), broken); err == nil { + t.Fatal("Add() of a module whose Start fails succeeded") + } + if host.Has("late") { + t.Error("Add() left a module whose Start failed registered") + } + if got, want := events.snapshot(), []string{"start:base", "start:late"}; !slices.Equal(got, want) { + t.Fatalf("lifecycle events = %v, want the base module untouched by the failed add: %v", got, want) + } +} + +func TestHostReplaceStopsTheOldAndStartsTheNew(t *testing.T) { + t.Parallel() + + events := &eventLog{} + host := plugin.NewHost() + oldManifest := validManifest("module") + oldManifest.Version = "1.0.0" + if err := host.Register(&fakeModule{manifest: oldManifest, events: events}); err != nil { + t.Fatal(err) + } + if err := host.Start(context.Background()); err != nil { + t.Fatal(err) + } + + newManifest := validManifest("module") + newManifest.Version = "2.0.0" + if err := host.Replace(context.Background(), &fakeModule{manifest: newManifest, events: events}); err != nil { + t.Fatalf("Replace() error = %v", err) + } + if got, want := events.snapshot(), []string{"start:module", "stop:module", "start:module"}; !slices.Equal(got, want) { + t.Fatalf("lifecycle events = %v, want %v", got, want) + } + manifests := host.Manifests() + if len(manifests) != 1 || manifests[0].Version != "2.0.0" { + t.Fatalf("manifests after replace = %+v, want the new 2.0.0", manifests) + } +} + +func TestHostReplaceRestartsTheOldWhenTheNewStartFails(t *testing.T) { + t.Parallel() + + events := &eventLog{} + host := plugin.NewHost() + if err := host.Register(&fakeModule{manifest: validManifest("module"), events: events}); err != nil { + t.Fatal(err) + } + if err := host.Start(context.Background()); err != nil { + t.Fatal(err) + } + + broken := &fakeModule{manifest: validManifest("module"), events: events, startErr: errors.New("boom")} + err := host.Replace(context.Background(), broken) + if err == nil { + t.Fatal("Replace() with a failing new module succeeded") + } + if got, want := events.snapshot(), []string{"start:module", "stop:module", "start:module", "start:module"}; !slices.Equal(got, want) { + t.Fatalf("lifecycle events = %v, want the old module restarted after the failed start: %v", got, want) + } + if !host.Has("module") { + t.Error("a refused Replace() unregistered the module") + } +} + +func TestHostReplaceRefusesAModuleARunningDependentNeeds(t *testing.T) { + t.Parallel() + + events := &eventLog{} + host := plugin.NewHost() + if err := host.Register(&fakeModule{manifest: validManifest("base"), events: events}); err != nil { + t.Fatal(err) + } + if err := host.Register(&fakeModule{manifest: validManifest("dependent", "base"), events: events}); err != nil { + t.Fatal(err) + } + if err := host.Start(context.Background()); err != nil { + t.Fatal(err) + } + + err := host.Replace(context.Background(), &fakeModule{manifest: validManifest("base"), events: events}) + if err == nil || !strings.Contains(err.Error(), "dependent") { + t.Fatalf("Replace() of a depended-on module = %v, want a refusal naming the dependent", err) + } + if got, want := events.snapshot(), []string{"start:base", "start:dependent"}; !slices.Equal(got, want) { + t.Fatalf("lifecycle events = %v, want the dependent left running: %v", got, want) + } +} + +func TestHostReplaceRefusesAnUnregisteredModule(t *testing.T) { + t.Parallel() + + host := plugin.NewHost() + if err := host.Start(context.Background()); err != nil { + t.Fatal(err) + } + if err := host.Replace(context.Background(), &fakeModule{manifest: validManifest("absent")}); err == nil { + t.Fatal("Replace() of an unregistered module succeeded") + } +} + +func TestHostAddBeforeStartBehavesLikeRegister(t *testing.T) { + t.Parallel() + + events := &eventLog{} + host := plugin.NewHost() + if err := host.Add(context.Background(), &fakeModule{manifest: validManifest("early"), events: events}); err != nil { + t.Fatalf("Add() before Start error = %v", err) + } + if err := host.Start(context.Background()); err != nil { + t.Fatal(err) + } + if got, want := events.snapshot(), []string{"start:early"}; !slices.Equal(got, want) { + t.Fatalf("lifecycle events = %v, want the module started by Start(): %v", got, want) + } +} diff --git a/internal/plugin/plugin.go b/internal/plugin/plugin.go index 3a7162013..2e496e940 100644 --- a/internal/plugin/plugin.go +++ b/internal/plugin/plugin.go @@ -56,25 +56,32 @@ func LoadDir(dir string, extraSymbols ...map[string]map[string]reflect.Value) ([ plugins := make([]Plugin, 0) for _, name := range names { - path := filepath.Join(dir, name) - src, err := os.ReadFile(path) + p, err := LoadFile(filepath.Join(dir, name), extraSymbols...) if err != nil { - slog.Default().Warn("skipping unreadable daemon plugin", "file", name, "err", err) - continue - } - // Each plugin file is package main -- must use a fresh interpreter - // to avoid symbol collisions between files. - i, err := yaegiutil.New(interp.Options{}, extraSymbols...) - if err != nil { - slog.Default().Warn("skipping plugin -- interpreter setup failed", "file", name, "err", err) - continue - } - p, err := yaegiutil.Resolve[Plugin](i, string(src), "main.Plugin") - if err != nil { - slog.Default().Warn("skipping plugin", "file", name, "err", err) + slog.Default().Warn("skipping daemon plugin", "file", name, "err", err) continue } plugins = append(plugins, p) } return plugins, nil } + +// LoadFile evaluates one plugin file. Each file is package main and needs a +// fresh interpreter to avoid symbol collisions with the other files, so the +// boot load (LoadDir) and a live directory reconciliation both load through +// here rather than through two copies of the interpreter setup. +func LoadFile(path string, extraSymbols ...map[string]map[string]reflect.Value) (Plugin, error) { + src, err := os.ReadFile(path) + if err != nil { + return nil, fmt.Errorf("read plugin %s: %w", path, err) + } + i, err := yaegiutil.New(interp.Options{}, extraSymbols...) + if err != nil { + return nil, fmt.Errorf("plugin %s: interpreter setup: %w", path, err) + } + p, err := yaegiutil.Resolve[Plugin](i, string(src), "main.Plugin") + if err != nil { + return nil, fmt.Errorf("plugin %s: %w", path, err) + } + return p, nil +} diff --git a/internal/secret/engine.go b/internal/secret/engine.go index 9146f3e14..38cf5155e 100644 --- a/internal/secret/engine.go +++ b/internal/secret/engine.go @@ -157,34 +157,41 @@ func (r *Registry) LoadDir(dir string, extraSymbols ...map[string]map[string]ref loaded := 0 for _, name := range names { - path := filepath.Join(dir, name) - src, err := os.ReadFile(path) - if err != nil { - slog.Default().Warn("skipping unreadable secret engine", "file", name, "err", err) - continue - } - // Each file is package main -- must use a fresh interpreter to - // avoid symbol collisions between files. - // The interpreter is seeded with the host environment: yaegi's - // interpreted os package keeps a private env otherwise, so - // plugins reading SOPS_FILE/VAULT_*/AGE_* via os.Getenv would - // silently see nothing. - i, err := yaegiutil.New(interp.Options{Env: os.Environ()}, extraSymbols...) - if err != nil { - slog.Default().Warn("skipping secret engine -- interpreter setup failed", "file", name, "err", err) - continue - } - e, err := yaegiutil.Resolve[Engine](i, string(src), "main.Engine") - if err != nil { + if err := r.LoadFile(filepath.Join(dir, name), extraSymbols...); err != nil { slog.Default().Warn("skipping secret engine", "file", name, "err", err) continue } - r.Register(e) loaded++ } return loaded, nil } +// LoadFile evaluates one secret-engine file and registers the engine it +// exports. Each file is package main and needs a fresh interpreter to avoid +// symbol collisions, so the boot load (LoadDir) and a live directory +// reconciliation both load through here rather than through two copies of the +// interpreter setup. +// +// The interpreter is seeded with the host environment: yaegi's interpreted os +// package keeps a private env otherwise, so an engine reading +// SOPS_FILE/VAULT_*/AGE_* via os.Getenv would silently see nothing. +func (r *Registry) LoadFile(path string, extraSymbols ...map[string]map[string]reflect.Value) error { + src, err := os.ReadFile(path) + if err != nil { + return fmt.Errorf("secret: read %s: %w", path, err) + } + i, err := yaegiutil.New(interp.Options{Env: os.Environ()}, extraSymbols...) + if err != nil { + return fmt.Errorf("secret: %s: interpreter setup: %w", path, err) + } + e, err := yaegiutil.Resolve[Engine](i, string(src), "main.Engine") + if err != nil { + return fmt.Errorf("secret: %s: %w", path, err) + } + r.Register(e) + return nil +} + // DefaultRegistry returns a singleton pre-loaded with the built-in env engine. // Use this when you need a one-off secret resolution without threading a // Registry through the entire call stack.