fix(secrets): wire provider credentials through engines

This commit is contained in:
sam committed 2026-07-31 08:45:58 +10:00
1 parent 059eee412c
commit 3ae59c99c0
17 files changed
+567 -37

No files matched your search

+1
View File
@@ -226,3 +226,4 @@
{"id":"int-d72f7fa25fe6964e77322aefb6e51b92","kind":"field_change","created_at":"2026-07-29T16:19:22.842905807Z","actor":"Sam Catlow","issue_id":"archie-core-0d6","extra":{"field":"status","new_value":"closed","old_value":"open"}}
{"id":"int-708d599fdab7b76f04111619b278a3de","kind":"field_change","created_at":"2026-07-30T17:39:36.940566206Z","actor":"Sam Catlow","issue_id":"archie-core-v8x","extra":{"field":"status","new_value":"closed","old_value":"in_progress","reason":"Implemented SDK-backed provider discovery, unified Telegram model picker, safe catalog caching, and execution composition tests"}}
{"id":"int-f1b886821c69a4b52d422bc826c5c153","kind":"field_change","created_at":"2026-07-30T17:47:08.215418355Z","actor":"Sam Catlow","issue_id":"archie-core-6he","extra":{"field":"status","new_value":"closed","old_value":"in_progress","reason":"Telegram confirmation edits now omit nil reply_markup; regression test passes"}}
{"id":"int-a5d387a7885fbe49d1a2b6d910b26fcf","kind":"field_change","created_at":"2026-07-30T22:45:57.584919906Z","actor":"Sam Catlow","issue_id":"archie-core-u4v","extra":{"field":"status","new_value":"closed","old_value":"in_progress","reason":"Built-in BWS and custom secret engines are production-wired; provider SecretRefs resolve safely across root and identity execution paths; full gate and adversarial review pass"}}
+1
View File
@@ -53,6 +53,7 @@
{"_type":"issue","id":"archie-core-abg.18","title":"[EPIC] Memory: pluggable provider architecture with file-backed persistence","description":"Pluggable memory system to match Hermes parity:\\n\\n1. **MemoryProvider interface**: Abstract interface with methods for initialize(), get_tool_schemas(), prefetch(), sync_turn(), and optional lifecycle hooks (on_session_end, on_session_switch, on_pre_compress, on_memory_write).\\n\\n2. **Built-in file-backed memory**: MEMORY.md and USER.md stored per-session/profile. LLM-callable memory tool with add/replace/remove actions. Frozen snapshot at session start (preserves prefix cache), immediate disk write on updates. Section-sign delimited entries with substring matching.\\n\\n3. **MemoryManager**: Orchestrates built-in + at most one external provider. Background serialized write executor. Prefetch with timeout (8s). Streaming context scrubber that strips injected memory spans from model output.\\n\\nRef: docs/prds/hermes-gap-analysis.md section 4","acceptance_criteria":"An external memory provider (e.g. Honcho, Mem0) can be plugged in via config. The built-in memory tool persists across sessions. Conversation context is recalled on session resume.","status":"closed","priority":0,"issue_type":"epic","owner":"sam@catlow.tech","created_at":"2026-07-25T14:02:22Z","created_by":"Sam Catlow","updated_at":"2026-07-28T17:04:41Z","closed_at":"2026-07-28T17:04:41Z","close_reason":"Superseded by merged work: all 32 children complete (32/32, bd flags it 'eligible for close'). MemoryProvider interface, built-in file-backed memory, MemoryManager, scrubber, drain and threat scanning all landed in internal/memory. Nothing left in scope.","dependencies":[{"issue_id":"archie-core-abg.18","depends_on_id":"archie-core-abg","type":"parent-child","created_at":"2026-07-26T00:02:22Z","created_by":"Sam Catlow","metadata":"{}"}],"dependency_count":0,"dependent_count":0,"comment_count":0}
{"_type":"issue","id":"archie-core-abg.16","title":"[EPIC] Gateway: rich message model, adapter lifecycle, session management","description":"Foundational gateway infrastructure to match Hermes parity. Three interdependent pieces:\\n\\n1. **Message model**: Replace the 3-field Message struct with a rich MessageEvent (MessageType enum: text/photo/video/audio/voice/document/sticker/command, media_urls, reply_to context, metadata dict, source). This is the wire type every adapter produces.\\n\\n2. **Adapter lifecycle**: Replace the 3-method Gateway interface with a BasePlatformAdapter interface: connect()/disconnect()/send() as required methods, plus optional overrides for typing indicators, edit/delete, image/voice/video/document send, interactive UX (clarify buttons, approval dialogs). Capability flags (supports_code_blocks, splits_long_messages, supports_async_delivery).\\n\\n3. **Session management**: SessionSource struct (platform, chat_id, chat_type, user_id, thread_id, guild_id, profile), AsyncSessionStore for persistent session storage, SessionContext for system prompt injection.\\n\\nAll three blocks together enable adding new platforms as plugins without touching core gateway code.\\n\\nRef: docs/prds/hermes-gap-analysis.md sections 1-2","acceptance_criteria":"New adapters can be added by implementing a Go interface with connect/disconnect/send + optional media methods. Messages carry full metadata through the pipeline. Sessions persist across gateway restarts.","status":"open","priority":0,"issue_type":"epic","owner":"sam@catlow.tech","created_at":"2026-07-25T14:02:21Z","created_by":"Sam Catlow","updated_at":"2026-07-25T14:02:21Z","dependencies":[{"issue_id":"archie-core-abg.16","depends_on_id":"archie-core-abg","type":"parent-child","created_at":"2026-07-26T00:02:21Z","created_by":"Sam Catlow","metadata":"{}"}],"dependency_count":0,"dependent_count":0,"comment_count":0}
{"_type":"issue","id":"archie-core-jfg","title":"Spawned archie-agent containers not on daemon's docker network — can't reach NATS","description":"Root cause of 'agent still failing on the server': archied's container.Pool.Acquire creates archie-agent containers via the Docker SDK with no NetworkMode set, so Docker attaches them to the default bridge network. archied and nats run on the compose-created 'archie-core_default' network instead, so the spawned container's hostname resolution for 'nats' (from NATS_URL=nats://nats:4222) fails and archie-agent can't connect. Confirmed on carina host logs: archied logs show repeating 'taskrun request failed err=\"nats: no responders available for request\"' for every task, containers created and destroyed within ~1.5s (consistent with archie-agent's NATS client failing fast). Fixed by detecting the daemon's own container's network in NewPool (selfNetwork, via ContainerInspect on os.Hostname()) and setting HostConfig.NetworkMode on spawned containers to match.","status":"closed","priority":0,"issue_type":"bug","owner":"sam@catlow.tech","created_at":"2026-07-24T03:10:26Z","created_by":"Sam Catlow","updated_at":"2026-07-25T13:04:34Z","closed_at":"2026-07-25T13:04:34Z","close_reason":"Fix already implemented and verified: selfNetwork() in internal/container/pool.go detects the daemon's own container network via ContainerInspect(os.Hostname()) and Acquire() sets HostConfig.NetworkMode to match, so spawned archie-agent containers join the same compose network as nats. Build and internal/container tests pass.","dependency_count":0,"dependent_count":0,"comment_count":0}
{"_type":"issue","id":"archie-core-u4v","title":"Wire secret engines into provider discovery and execution","description":"Production creates only the env registry, never loads secret-engine plugins, and Provider lacks the documented api_key SecretRef. Implement startup plugin loading, backwards-compatible provider secret references, resolved provider discovery/runtime/container propagation, tests, release, and Carina deployment.","acceptance_criteria":"Provider api_key SecretRefs resolve through loaded engines at startup; existing api_key_env remains compatible; discovered providers, chat runtime, taskrun and containers receive resolved credentials without logging or serializing secret values; production composition tests and task check pass.","status":"closed","priority":1,"issue_type":"bug","assignee":"Sam Catlow","owner":"sam@catlow.tech","created_at":"2026-07-30T17:52:08Z","created_by":"Sam Catlow","updated_at":"2026-07-30T22:45:58Z","started_at":"2026-07-30T17:52:13Z","closed_at":"2026-07-30T22:45:58Z","close_reason":"Built-in BWS and custom secret engines are production-wired; provider SecretRefs resolve safely across root and identity execution paths; full gate and adversarial review pass","dependency_count":0,"dependent_count":0,"comment_count":0}
{"_type":"issue","id":"archie-core-6he","title":"Telegram rejects model confirmation when clearing inline keyboard","description":"Selecting a model calls editMessageText with a typed nil reply_markup; Telegram rejects it as object expected, leaving the picker visible. Omit reply_markup when no keyboard is required and cover the request shape.","acceptance_criteria":"Selecting any model replaces the picker with metadata confirmation and the Telegram edit request omits reply_markup when clearing controls.","status":"closed","priority":1,"issue_type":"bug","assignee":"Sam Catlow","owner":"sam@catlow.tech","created_at":"2026-07-30T17:46:25Z","created_by":"Sam Catlow","updated_at":"2026-07-30T17:47:08Z","started_at":"2026-07-30T17:46:29Z","closed_at":"2026-07-30T17:47:08Z","close_reason":"Telegram confirmation edits now omit nil reply_markup; regression test passes","dependency_count":0,"dependent_count":0,"comment_count":0}
{"_type":"issue","id":"archie-core-v8x","title":"Unify Telegram provider and model selection around SDK catalog","description":"Replace the separate /provider and static configured-model surfaces with one /model provider-to-model wizard. Build the usable provider catalog from the local ai-sdk models.dev catalog plus environment credentials and explicit overrides, cache the filtered tool-capable catalog under the active config directory, and use the same resolved provider/model state across chat and execution paths. Remove the handwritten WellKnown table and remove external-framework references from the Archie tree.","acceptance_criteria":"/model first shows usable providers, then paginated tool-capable models, then atomically switches provider/model and reports model metadata and scope. /provider is absent from executable/menu/help surfaces. Catalog refresh is downloaded at daemon startup and cached beside configuration with safe cached fallback. Environment discovery and explicit overrides flow through chat, identities, subprocess and container/taskrun paths. Focused and composition tests cover discovery, pagination, callbacks and execution parity. A case-insensitive repository scan outside the explicit .references source tree finds no forbidden external-framework name.","status":"closed","priority":1,"issue_type":"feature","assignee":"Sam Catlow","owner":"sam@catlow.tech","created_at":"2026-07-30T17:10:38Z","created_by":"Sam Catlow","updated_at":"2026-07-30T17:39:37Z","started_at":"2026-07-30T17:10:59Z","closed_at":"2026-07-30T17:39:37Z","close_reason":"Implemented SDK-backed provider discovery, unified Telegram model picker, safe catalog caching, and execution composition tests","dependency_count":0,"dependent_count":0,"comment_count":0}
{"_type":"issue","id":"archie-core-ky6","title":"One optional MCP provider failing takes down the whole daemon","description":"internal/tools/provider/provider.go Start() is atomic by design -- 'Any failure rolls the complete family back' -- so a single failing provider unregisters every other provider including the builtins, capabilityHost.Start returns the error, and cmd/archied/main.go:655 exits 1. Under compose restart or systemd Restart=on-failure that is a crash loop. Observed 2026-07-30 on carina: mcp.desktop-commander could not start (missing unzip in the runtime image, fixed in 455fd6f) and archied restarted continuously, losing chat, the gateway and every builtin tool over one third-party npm package.\\n\\nAtomicity is correct for a provider the daemon requires. It is wrong for an external MCP server pulled from a registry at runtime, which is exactly the category that should be allowed to be absent -- the same reasoning applied to the forge in 518a1a0, where a missing credential now disables the capability rather than the daemon.","design":"Needs a required/optional distinction on the registration or manifest, which does not exist today. Keep the all-or-nothing path for required providers; the atomic batch semantics for tool visibility (tools appear only after Start and Discover both succeed) should hold per-provider either way. Relates to the plugin engine rule in ARCHITECTURE.md: failure isolation is explicitly the owning manager's responsibility.","acceptance_criteria":"A provider can be declared optional. An optional provider that fails to start or discover is logged, excluded from the running set, and does not roll back the family or stop the daemon. A required provider keeps today's atomic rollback. Provider health reflects the degraded state so the failure is visible rather than silent. Regression test starts a family containing one failing optional provider and asserts the other providers' tools remain registered and Start returns nil.","status":"open","priority":1,"issue_type":"bug","owner":"sam@catlow.tech","created_at":"2026-07-30T06:49:26Z","created_by":"Sam Catlow","updated_at":"2026-07-30T06:49:26Z","dependency_count":0,"dependent_count":0,"comment_count":0}
+10 -9
View File
@@ -180,17 +180,18 @@ Daemon-level TOML (`~/.config/archie/config.toml`):
Secrets (forge token, provider API keys, channel tokens) are not embedded in
config files. `internal/secret` defines a pluggable `Engine` interface (`Name`,
`Version`, `Resolve(key) (string, error)`) resolved through a `Registry`; the
built-in `env` engine reads an environment variable by name. Custom engines
(sops, Bitwarden Secrets Manager, Vault/OpenBao) are Yaegi-interpreted `.go`
plugins loaded from a directory, following the same convention as
`internal/plugin`'s daemon plugins. Config fields reference a secret as
built-in `env` engine reads an environment variable by name and the built-in
`bws` engine caches the accessible Bitwarden project at startup. Custom engines
(sops, Vault/OpenBao) are Yaegi-interpreted `.go` plugins loaded from a
directory, following the same convention as `internal/plugin`'s daemon plugins.
Config fields reference a secret as
`{engine, key}` (`secret.SecretRef`), e.g.
`token = { engine = "env", key = "ARCHIE_GITHUB_TOKEN" }`. As of this writing
only `Forge.Token` has been migrated to `SecretRef`; `Provider.APIKeyEnv`,
`NATSConfig.TokenEnv`, and `TelegramConfig.TokenEnv` are still plain
env-var-name strings pending a follow-up (they're forwarded by name to
`ai-sdk/runtime` and subprocess workers rather than resolved centrally, which
needs its own design pass before migrating).
`Forge.Token` and `Provider.APIKey` use `SecretRef`; resolved provider values
are exported under private, identity-scoped environment names because the SDK,
subprocess, and container boundaries consume credential names rather than
plaintext wire fields. Legacy `Provider.APIKeyEnv`, `NATSConfig.TokenEnv`, and
`TelegramConfig.TokenEnv` remain supported.
Subprocess mode is a migration transport boundary, not a security sandbox: the
worker still runs under the daemon UID and can access daemon-readable host
+27 -2
View File
@@ -224,7 +224,11 @@ func run() int {
}
cfg := doc.Config
secrets := secret.NewRegistry()
secrets, err := configuredSecretRegistry(&cfg, log)
if err != nil {
log.Error("configure secrets", "err", err)
return 1
}
forgeClient, token := resolveForge(cfg.Forge, secrets, log)
st, err := nell.OpenStore(cfg.DBPath, cfg.BotUser)
@@ -254,7 +258,7 @@ func run() int {
catalog, err := modelcatalog.Load(ctx, modelcatalog.Options{
CachePath: filepath.Join(filepath.Dir(*cfgPath), "models.json"),
Getenv: os.Getenv,
Getenv: secrets.Getenv,
Configured: cfg.Providers,
})
var catalogModels []string
@@ -524,6 +528,27 @@ func run() int {
log.Error("identity construction failed", "identity", idCfg.Name, "err", err)
return 1
}
idProviders := agentexec.ProvidersFromConfig(idCfg.Providers)
idRuntime := agentexec.NewRuntime(idProviders)
if idRuntime != nil {
switch cfg.Agent.Mode {
case "subprocess":
runner.Agent = &agentexec.SubprocessRunner{
Command: cfg.Agent.Command,
Environ: os.Environ(),
AdditionalEnv: cfg.Agent.Env,
Diagnostics: os.Stderr,
Providers: idProviders,
}
case "inprocess":
runner.Agent = agentexec.NewInProcessRunner(idRuntime, log.With("identity", idCfg.Name), toolReg)
case "nats":
runner.Agent = &agentexec.NATSRunner{
Bus: natsClient, Providers: idProviders,
MCPServers: cfg.Tools.MCPServers, Log: log.With("identity", idCfg.Name),
}
}
}
identityRunners = append(identityRunners, runner)
log.Info("identity configured", "identity", idCfg.Name, "bot_user", idCfg.BotUser, "repos", len(idCfg.Repos))
}
+85
View File
@@ -6,6 +6,7 @@ import (
"fmt"
"io"
"log/slog"
"os"
"path/filepath"
"strings"
"testing"
@@ -105,6 +106,90 @@ func TestResolveForgeBuildsClientWhenTokenResolves(t *testing.T) {
}
}
func TestResolveProviderSecretSetsPrivateRuntimeEnvironment(t *testing.T) {
registry := secret.NewRegistry()
registry.Register(providerSecretEngine{value: "provider-secret"})
cfg := config.Config{Providers: map[string]config.Provider{
"openai": {
Class: "openai",
APIKey: secret.SecretRef{Engine: "provider-test", Key: "OPENAI_API_KEY"},
},
}}
if err := resolveProviderSecrets(&cfg, registry); err != nil {
t.Fatal(err)
}
provider := cfg.Providers["openai"]
if provider.APIKeyEnv == "" {
t.Fatal("resolved provider has no runtime environment name")
}
if provider.APIKey != (secret.SecretRef{}) {
t.Fatal("resolved provider retained serializable secret reference")
}
if got := os.Getenv(provider.APIKeyEnv); got != "provider-secret" {
t.Fatalf("runtime environment value = %q", got)
}
t.Cleanup(func() { _ = os.Unsetenv(provider.APIKeyEnv) })
}
func TestResolveProviderSecretsIsolatesIdentityCredentials(t *testing.T) {
registry := secret.NewRegistry()
registry.Register(mappedProviderSecretEngine{values: map[string]string{
"root-key": "root-secret", "one-key": "one-secret", "two-key": "two-secret",
}})
cfg := config.Config{
Providers: map[string]config.Provider{"openai": {
APIKey: secret.SecretRef{Engine: "provider-map", Key: "root-key"},
}},
Identities: []config.IdentityConfig{
{Name: "one", Providers: map[string]config.Provider{"openai": {
APIKey: secret.SecretRef{Engine: "provider-map", Key: "one-key"},
}}},
{Name: "two", Providers: map[string]config.Provider{"openai": {
APIKey: secret.SecretRef{Engine: "provider-map", Key: "two-key"},
}}},
},
}
if err := resolveProviderSecrets(&cfg, registry); err != nil {
t.Fatal(err)
}
names := []string{
cfg.Providers["openai"].APIKeyEnv,
cfg.Identities[0].Providers["openai"].APIKeyEnv,
cfg.Identities[1].Providers["openai"].APIKeyEnv,
}
if names[0] == names[1] || names[0] == names[2] || names[1] == names[2] {
t.Fatalf("credential environment names collide: %v", names)
}
for i, want := range []string{"root-secret", "one-secret", "two-secret"} {
if got := os.Getenv(names[i]); got != want {
t.Errorf("credential %d = %q, want %q", i, got, want)
}
t.Cleanup(func() { _ = os.Unsetenv(names[i]) })
}
}
type providerSecretEngine struct{ value string }
func (providerSecretEngine) Name() string { return "provider-test" }
func (providerSecretEngine) Version() string { return "test" }
func (e providerSecretEngine) Resolve(string) (string, error) {
return e.value, nil
}
type mappedProviderSecretEngine struct{ values map[string]string }
func (mappedProviderSecretEngine) Name() string { return "provider-map" }
func (mappedProviderSecretEngine) Version() string { return "test" }
func (e mappedProviderSecretEngine) Resolve(key string) (string, error) {
value, ok := e.values[key]
if !ok {
return "", fmt.Errorf("missing test secret")
}
return value, nil
}
func TestChatGenerateOptionsIncludesToolsAndMultipleSteps(t *testing.T) {
registry := tools.NewRegistry()
if err := registry.Register(tools.ToolEntry{
+4
View File
@@ -6,6 +6,7 @@ import (
"github.com/samcharles93/archie-core/internal/config"
"github.com/samcharles93/archie-core/internal/infrastructure/modelcatalog"
"github.com/samcharles93/archie-core/internal/secret"
)
func applyModelCatalog(cfg *config.Config, snapshot modelcatalog.Snapshot) []string {
@@ -38,6 +39,9 @@ func mergeProviders(base, overrides map[string]config.Provider) map[string]confi
if override.APIKeyEnv != "" {
resolved.APIKeyEnv = override.APIKeyEnv
}
if override.APIKey != (secret.SecretRef{}) {
resolved.APIKey = override.APIKey
}
if override.BaseURL != "" {
resolved.BaseURL = override.BaseURL
}
+69
View File
@@ -0,0 +1,69 @@
package main
import (
"crypto/sha256"
"fmt"
"log/slog"
"os"
"strings"
"github.com/samcharles93/archie-core/internal/config"
"github.com/samcharles93/archie-core/internal/secret"
"github.com/samcharles93/archie-core/internal/secret/secretextract"
)
func configuredSecretRegistry(cfg *config.Config, log *slog.Logger) (*secret.Registry, error) {
registry := secret.NewRegistry()
if cfg.SecretEngineDir != "" {
loaded, err := registry.LoadDir(cfg.SecretEngineDir, secretextract.Symbols)
if err != nil {
return nil, fmt.Errorf("load secret engines from %q: %w", cfg.SecretEngineDir, err)
}
log.Info("secret engines loaded", "count", loaded)
}
if err := resolveProviderSecrets(cfg, registry); err != nil {
return nil, fmt.Errorf("resolve provider secrets: %w", err)
}
return registry, nil
}
func resolveProviderSecrets(cfg *config.Config, registry *secret.Registry) error {
if err := resolveProviderMap("root", cfg.Providers, registry); err != nil {
return err
}
for i := range cfg.Identities {
if err := resolveProviderMap(cfg.Identities[i].Name, cfg.Identities[i].Providers, registry); err != nil {
return fmt.Errorf("identity %q: %w", cfg.Identities[i].Name, err)
}
}
return nil
}
func resolveProviderMap(scope string, providers map[string]config.Provider, registry *secret.Registry) error {
for id, provider := range providers {
if provider.APIKey == (secret.SecretRef{}) {
continue
}
value, err := provider.APIKey.Resolve(registry)
if err != nil {
return fmt.Errorf("provider %q: %w", id, err)
}
value = strings.TrimSpace(value)
if value == "" {
return fmt.Errorf("provider %q: resolved api_key is empty", id)
}
envName := providerSecretEnvName(scope, id)
if err := os.Setenv(envName, value); err != nil {
return fmt.Errorf("provider %q: export resolved api_key: %w", id, err)
}
provider.APIKey = secret.SecretRef{}
provider.APIKeyEnv = envName
providers[id] = provider
}
return nil
}
func providerSecretEnvName(scope, providerID string) string {
sum := sha256.Sum256([]byte(scope + "\x00" + providerID))
return fmt.Sprintf("ARCHIE_PROVIDER_%X_API_KEY", sum[:8])
}
+3 -1
View File
@@ -4,6 +4,7 @@
# "env" engine reads key as an environment variable name. Custom engines
# (sops, bws, vault/openbao, ...) are Yaegi plugins loaded from a
# directory -- see internal/secret.
# secret_engine_dir = "~/.config/archie/secret-engines"
# work_dir = "~/.local/share/archie/work" # default
# db_path = "~/.local/share/archie/archie.db"
@@ -37,7 +38,8 @@ builder = "openai/gpt-5.4"
[providers.openai]
class = "openai"
api_key_env = "OPENAI_API_KEY"
api_key = { engine = "bws", key = "OPENAI_API_KEY" }
# api_key_env = "OPENAI_API_KEY" # backwards-compatible environment form
[providers.ollama]
class = "ollama"
+10 -6
View File
@@ -171,9 +171,10 @@ type Budgets struct {
// Provider configures one LLM provider for the runtime catalog.
type Provider struct {
Class string `toml:"class" yaml:"class" json:"class"`
APIKeyEnv string `toml:"api_key_env" yaml:"api_key_env" json:"api_key_env"`
BaseURL string `toml:"base_url" yaml:"base_url" json:"base_url"`
Class string `toml:"class" yaml:"class" json:"class"`
APIKeyEnv string `toml:"api_key_env" yaml:"api_key_env" json:"api_key_env"`
APIKey secret.SecretRef `toml:"api_key" yaml:"api_key" json:"-"`
BaseURL string `toml:"base_url" yaml:"base_url" json:"base_url"`
}
// Agent configures how archied executes autonomous stages.
@@ -311,9 +312,12 @@ type Config struct {
// variable satisfying the plugin.Plugin interface. Failed plugins
// are skipped -- the daemon starts with the remaining set. When
// empty, no daemon plugins are loaded.
PluginDir string `toml:"plugin_dir" yaml:"plugin_dir"`
DBPath string `toml:"db_path" yaml:"db_path"`
PollInterval Duration `toml:"poll_interval" yaml:"poll_interval"`
PluginDir string `toml:"plugin_dir" yaml:"plugin_dir"`
// SecretEngineDir contains Yaegi secret-engine plugins. Built-in env and
// bws engines remain available when this is empty.
SecretEngineDir string `toml:"secret_engine_dir" yaml:"secret_engine_dir"`
DBPath string `toml:"db_path" yaml:"db_path"`
PollInterval Duration `toml:"poll_interval" yaml:"poll_interval"`
// Label marks issues archie should pick up.
Label string `toml:"label" yaml:"label"`
BotUser string `toml:"bot_user" yaml:"bot_user"`
+37 -8
View File
@@ -139,7 +139,9 @@ type IdentityRunner struct {
// Cfg is the identity-scoped config subset (forge, models, dispatch,
// budgets, etc.).
Cfg config.IdentityConfig
Log *slog.Logger
// Agent is built from this identity's provider credentials.
Agent agentexec.Runner
Log *slog.Logger
}
// NewIdentityRunner constructs an IdentityRunner from an identity config
@@ -913,6 +915,7 @@ func (d *Daemon) process(ctx context.Context, task *store.Task) {
fg := d.forgeFor(task)
trees := d.treesFor(task)
executionCfg, executionAgent := d.executionFor(task)
repo, ok := d.repoFor(task)
if !ok {
_ = d.Store.Transition(ctx, task.ID, store.StatusRunning, store.StatusParked, "repo no longer in config")
@@ -993,13 +996,13 @@ func (d *Daemon) process(ctx context.Context, task *store.Task) {
workflow.Run(ctx, wf, &workflow.TaskContext{
Task: task,
Repo: repo,
Cfg: d.Cfg,
Cfg: executionCfg,
Forge: fg,
Store: d.Store,
Trees: trees,
SystemPrompt: d.memoryPrompt,
Guardrails: d.Guardrails,
Agent: d.Agent,
Agent: executionAgent,
Bus: d.Bus,
Log: d.Log,
CustomStages: d.CustomStages,
@@ -1064,7 +1067,7 @@ func (d *Daemon) acquireTaskContainer(
return nil, false
}
ctr, err := d.ContainerPool.Acquire(ctx, mounts, d.containerEnv())
ctr, err := d.ContainerPool.Acquire(ctx, mounts, d.containerEnv(task))
if err != nil {
park("container acquire failed", err)
return nil, false
@@ -1086,8 +1089,8 @@ func (d *Daemon) runViaAgent(ctx context.Context, task *store.Task, repo config.
req := taskrun.Request{
Task: task,
Repo: repo,
Cfg: d.Cfg.ForTask(),
Providers: agentexec.ProvidersFromConfig(d.Cfg.Providers),
Cfg: d.configFor(task).ForTask(),
Providers: agentexec.ProvidersFromConfig(d.configFor(task).Providers),
}
data, err := json.Marshal(req)
if err != nil {
@@ -1172,7 +1175,7 @@ func (d *Daemon) requestTaskRun(ctx context.Context, taskID int64, data []byte)
}
}
func (d *Daemon) containerEnv() []string {
func (d *Daemon) containerEnv(task *store.Task) []string {
var env []string
env = append(env, "NATS_URL="+d.Cfg.NATS.URL)
if tokenEnv := d.Cfg.NATS.TokenEnv; tokenEnv != "" {
@@ -1180,7 +1183,7 @@ func (d *Daemon) containerEnv() []string {
env = append(env, "NATS_TOKEN="+token)
}
}
for _, p := range d.Cfg.Providers {
for _, p := range d.configFor(task).Providers {
if p.APIKeyEnv != "" {
if v := os.Getenv(p.APIKeyEnv); v != "" {
env = append(env, p.APIKeyEnv+"="+v)
@@ -1190,6 +1193,32 @@ func (d *Daemon) containerEnv() []string {
return env
}
func (d *Daemon) executionFor(task *store.Task) (config.Config, agentexec.Runner) {
if id := d.identityFor(task); id != nil {
return configForIdentity(d.Cfg, id.Cfg), id.Agent
}
return d.Cfg, d.Agent
}
func (d *Daemon) configFor(task *store.Task) config.Config {
cfg, _ := d.executionFor(task)
return cfg
}
func configForIdentity(root config.Config, identity config.IdentityConfig) config.Config {
root.BotUser = identity.BotUser
root.BotEmail = identity.BotEmail
root.DiffCapLines = identity.DiffCapLines
root.Forge = identity.Forge
root.Dispatch = identity.Dispatch
root.Models = identity.Models
root.Providers = identity.Providers
root.Budgets = identity.Budgets
root.Notify = identity.Notify
root.Repos = identity.Repos
return root
}
// identityFor resolves the IdentityRunner that owns task, or nil for
// legacy single-identity deployments and forge-sourced tasks recorded
// before multi-identity routing existed (task.Identity == ""). Callers
+68 -1
View File
@@ -31,6 +31,12 @@ type cleanupStorage struct {
ttl time.Duration
}
type identityRunnerStub struct{}
func (*identityRunnerStub) Run(context.Context, string, agentexec.Request) (agentexec.Result, error) {
return agentexec.Result{}, nil
}
func (s *cleanupStorage) Setup(context.Context, storage.TaskRef) ([]storage.Mount, error) {
return nil, nil
}
@@ -251,7 +257,7 @@ func TestContainerEnvIncludesConfiguredNATSCredentials(t *testing.T) {
},
}}
got := d.containerEnv()
got := d.containerEnv(nil)
for _, want := range []string{
"NATS_URL=nats://nats.example:4222",
"NATS_TOKEN=test-nats-token",
@@ -262,6 +268,30 @@ func TestContainerEnvIncludesConfiguredNATSCredentials(t *testing.T) {
}
}
func TestContainerEnvUsesOnlyOwningIdentityProviderCredential(t *testing.T) {
t.Setenv("ROOT_PROVIDER_KEY", "root-secret")
t.Setenv("WORKER_PROVIDER_KEY", "worker-secret")
d := &Daemon{
Cfg: config.Config{Providers: map[string]config.Provider{
"openai": {Class: "openai", APIKeyEnv: "ROOT_PROVIDER_KEY"},
}},
Identities: []*IdentityRunner{{
Name: "worker",
Cfg: config.IdentityConfig{Name: "worker", Providers: map[string]config.Provider{
"openai": {Class: "openai", APIKeyEnv: "WORKER_PROVIDER_KEY"},
}},
}},
}
got := d.containerEnv(&store.Task{Identity: "worker"})
if !slices.Contains(got, "WORKER_PROVIDER_KEY=worker-secret") {
t.Fatalf("identity credential missing from container env: %q", got)
}
if slices.Contains(got, "ROOT_PROVIDER_KEY=root-secret") {
t.Fatalf("root credential leaked into identity container env: %q", got)
}
}
func startEmbeddedNATSForDaemon(t *testing.T) *server.Server {
t.Helper()
srv := natssrv.RunRandClientPortServer()
@@ -764,6 +794,43 @@ func TestNewIdentityRunnerRejectsEmptyName(t *testing.T) {
}
}
func TestExecutionForUsesIdentityConfigAgentAndProvider(t *testing.T) {
rootAgent := &identityRunnerStub{}
identityAgent := &identityRunnerStub{}
d := &Daemon{
Cfg: config.Config{
Models: map[string]string{"builder": "root/model"},
Providers: map[string]config.Provider{
"root": {Class: "openai", APIKeyEnv: "ROOT_KEY"},
},
},
Agent: rootAgent,
Identities: []*IdentityRunner{{
Name: "worker", Agent: identityAgent,
Cfg: config.IdentityConfig{
Name: "worker", Models: map[string]string{"builder": "worker/model"},
Providers: map[string]config.Provider{
"worker": {Class: "anthropic", APIKeyEnv: "WORKER_KEY"},
},
},
}},
}
cfg, runner := d.executionFor(&store.Task{Identity: "worker"})
if runner != identityAgent {
t.Fatal("executionFor returned root agent for identity task")
}
if cfg.Models["builder"] != "worker/model" {
t.Fatalf("builder model = %q", cfg.Models["builder"])
}
if _, ok := cfg.Providers["worker"]; !ok {
t.Fatal("identity provider missing from execution config")
}
if _, ok := cfg.Providers["root"]; ok {
t.Fatal("root provider leaked into identity execution config")
}
}
func TestDaemonStoresIdentityRunners(t *testing.T) {
d := &Daemon{
Identities: []*IdentityRunner{
@@ -1,7 +1,9 @@
package configuration
import (
"os"
"path/filepath"
"strings"
"time"
"github.com/samcharles93/archie-core/internal/config"
@@ -46,6 +48,7 @@ func (l *Loader) applyDefaults(cfg *config.Config) {
// applyGeneralDefaults derives the paths and limits that have no owning
// section.
func (l *Loader) applyGeneralDefaults(cfg *config.Config) {
cfg.SecretEngineDir = expandHomePath(cfg.SecretEngineDir)
if cfg.WorkDir == "" {
cfg.WorkDir = filepath.Join(l.dataHome(), "archie", "work")
}
@@ -67,6 +70,20 @@ func (l *Loader) applyGeneralDefaults(cfg *config.Config) {
}
}
func expandHomePath(path string) string {
if path != "~" && !strings.HasPrefix(path, "~/") {
return path
}
home, err := os.UserHomeDir()
if err != nil {
return path
}
if path == "~" {
return home
}
return filepath.Join(home, strings.TrimPrefix(path, "~/"))
}
// applyForgeDefaults fills the forge section, promoting the legacy
// token_env field to a secret reference when no explicit one was given.
func applyForgeDefaults(cfg *config.Config) {
@@ -692,6 +692,40 @@ plugin_dir: /opt/archie/plugins
}
}
func TestLoadDirSecretEngineDir(t *testing.T) {
dir := tmpConfigDir(t)
writeFile(t, dir, "config.yaml", `
bot_user: secrettest
secret_engine_dir: /opt/archie/secret-engines
`)
cfg, err := loadDir(dir, "")
if err != nil {
t.Fatal(err)
}
if got := cfg.SecretEngineDir; got != "/opt/archie/secret-engines" {
t.Fatalf("SecretEngineDir = %q", got)
}
}
func TestLoadDirExpandsSecretEngineHome(t *testing.T) {
home := t.TempDir()
t.Setenv("HOME", home)
dir := tmpConfigDir(t)
writeFile(t, dir, "config.yaml", `
bot_user: secrettest
secret_engine_dir: ~/.config/archie/secret-engines
`)
cfg, err := loadDir(dir, "")
if err != nil {
t.Fatal(err)
}
want := filepath.Join(home, ".config", "archie", "secret-engines")
if cfg.SecretEngineDir != want {
t.Fatalf("SecretEngineDir = %q, want %q", cfg.SecretEngineDir, want)
}
}
// ── 18. API stability: ForTask must still work ─────────────────────
func TestLoadDirForTaskRoundTrip(t *testing.T) {
+86
View File
@@ -0,0 +1,86 @@
package secret
import (
"context"
"encoding/json"
"fmt"
"io"
"os"
"os/exec"
"path/filepath"
"sync"
"time"
)
const bwsTimeout = 30 * time.Second
type bwsEngine struct {
once sync.Once
secrets map[string]string
err error
}
type bwsSecret struct {
Key string `json:"key"`
Value string `json:"value"`
}
func newBWSEngine() *bwsEngine { return &bwsEngine{} }
func (*bwsEngine) Name() string { return "bws" }
func (*bwsEngine) Version() string { return "1.0.0" }
func (e *bwsEngine) Resolve(key string) (string, error) {
e.once.Do(e.load)
if e.err != nil {
return "", e.err
}
value, ok := e.secrets[key]
if !ok {
return "", fmt.Errorf("bws secret %q not found", key)
}
return value, nil
}
func (e *bwsEngine) load() {
command, err := findBWS()
if err != nil {
e.err = err
return
}
ctx, cancel := context.WithTimeout(context.Background(), bwsTimeout)
defer cancel()
cmd := exec.CommandContext(ctx, command, "secret", "list", "--output", "json")
cmd.Env = os.Environ()
cmd.Stderr = io.Discard
raw, err := cmd.Output()
if err != nil {
e.err = fmt.Errorf("list bws secrets: %w", err)
return
}
var records []bwsSecret
if err := json.Unmarshal(raw, &records); err != nil {
e.err = fmt.Errorf("decode bws secrets: %w", err)
return
}
e.secrets = make(map[string]string, len(records))
for _, record := range records {
if record.Key != "" && record.Value != "" {
e.secrets[record.Key] = record.Value
}
}
}
func findBWS() (string, error) {
if command, err := exec.LookPath("bws"); err == nil {
return command, nil
}
home, err := os.UserHomeDir()
if err == nil {
command := filepath.Join(home, ".local", "bin", "bws")
if info, statErr := os.Stat(command); statErr == nil && !info.IsDir() && info.Mode()&0o111 != 0 {
return command, nil
}
}
return "", fmt.Errorf("bws executable not found")
}
+46 -8
View File
@@ -1,12 +1,10 @@
// Package secret provides a pluggable secrets-engine registry. Engines
// resolve SecretRef values (engine + key) into strings at daemon startup,
// keeping credentials out of config files and environment variables.
// keeping credentials out of config files and serialized worker contracts.
//
// Only the "env" engine is compiled in (see env.go). Other backends
// (sops, Bitwarden Secrets Manager, Vault/OpenBao) ship as Yaegi-loaded
// .go plugins under a configurable directory via Registry.LoadDir,
// mirroring internal/plugin conventions -- they are not hard-coded
// into the binary.
// The "env" and "bws" engines are compiled in. Other backends (sops,
// Vault/OpenBao) may ship as Yaegi-loaded .go plugins under a configurable
// directory via Registry.LoadDir, mirroring internal/plugin conventions.
package secret
import (
@@ -67,6 +65,41 @@ func (r *Registry) Get(name string) (Engine, bool) {
return e, ok
}
// Getenv resolves an environment-shaped secret name. The real process
// environment wins; registered external engines are tried in stable name
// order. A resolved value is exported so SDKs and child processes that accept
// only environment-variable credentials observe the same secret.
func (r *Registry) Getenv(key string) string {
if value := os.Getenv(key); value != "" {
return value
}
r.mu.RLock()
names := make([]string, 0, len(r.engines))
for name := range r.engines {
if name != "env" {
names = append(names, name)
}
}
sort.Strings(names)
engines := make([]Engine, 0, len(names))
for _, name := range names {
engines = append(engines, r.engines[name])
}
r.mu.RUnlock()
for _, engine := range engines {
value, err := engine.Resolve(key)
value = strings.TrimSpace(value)
if err != nil || value == "" {
continue
}
if err := os.Setenv(key, value); err != nil {
return ""
}
return value
}
return ""
}
// Resolve resolves a SecretRef through the registered engine.
func (r *Registry) Resolve(ref SecretRef) (string, error) {
e, ok := r.Get(ref.Engine)
@@ -136,11 +169,16 @@ func DefaultRegistry() *Registry {
return defaultReg
}
var defaultReg = &Registry{engines: map[string]Engine{"env": &envEngine{}}}
var defaultReg = &Registry{engines: map[string]Engine{
"env": &envEngine{},
"bws": newBWSEngine(),
}}
// NewRegistry creates a Registry pre-loaded with the built-in env engine.
// NewRegistry creates a Registry pre-loaded with the built-in env and
// Bitwarden Secrets Manager engines.
func NewRegistry() *Registry {
r := &Registry{engines: make(map[string]Engine)}
r.Register(&envEngine{})
r.Register(newBWSEngine())
return r
}
+2 -2
View File
@@ -1,7 +1,7 @@
package secret
// SecretRef references a secret stored in a named engine. It replaces
// the previous TokenEnv/APIKeyEnv string fields throughout config.
// SecretRef references a secret stored in a named engine. Legacy environment
// variable fields remain supported where required for backwards compatibility.
//
// Format in TOML/YAML:
//
+67
View File
@@ -1,6 +1,10 @@
package secret
import (
"fmt"
"os"
"path/filepath"
"strings"
"testing"
)
@@ -83,6 +87,69 @@ func TestRegistryResolveUnknownEngine(t *testing.T) {
}
}
func TestRegistryGetenvFallsBackToNonEnvEngine(t *testing.T) {
t.Setenv("ARCHIE_TEST_BWS_PROVIDER_KEY", "")
r := NewRegistry()
r.Register(testEngine{name: "test-source", values: map[string]string{
"ARCHIE_TEST_BWS_PROVIDER_KEY": "resolved-value",
}})
if got := r.Getenv("ARCHIE_TEST_BWS_PROVIDER_KEY"); got != "resolved-value" {
t.Fatalf("Getenv() = %q, want resolved value", got)
}
if got := os.Getenv("ARCHIE_TEST_BWS_PROVIDER_KEY"); got != "resolved-value" {
t.Fatalf("resolved value was not exported for SDK/container consumers")
}
}
type testEngine struct {
name string
values map[string]string
}
func TestBWSEngineLoadsNamedSecretsOnce(t *testing.T) {
dir := t.TempDir()
command := filepath.Join(dir, "bws")
countPath := filepath.Join(dir, "count")
script := fmt.Sprintf(`#!/bin/sh
echo x >> %q
printf '%%s' '[{"key":"OPENAI_API_KEY","value":"from-bws"}]'
`, countPath)
if err := os.WriteFile(command, []byte(script), 0o700); err != nil {
t.Fatal(err)
}
t.Setenv("PATH", dir)
engine := newBWSEngine()
got, err := engine.Resolve("OPENAI_API_KEY")
if err != nil {
t.Fatal(err)
}
if got != "from-bws" {
t.Fatalf("Resolve() = %q", got)
}
if _, err := engine.Resolve("OPENAI_API_KEY"); err != nil {
t.Fatal(err)
}
count, err := os.ReadFile(countPath)
if err != nil {
t.Fatal(err)
}
if got := strings.Count(string(count), "x"); got != 1 {
t.Fatalf("bws invocation count = %d, want 1", got)
}
}
func (e testEngine) Name() string { return e.name }
func (e testEngine) Version() string { return "test" }
func (e testEngine) Resolve(key string) (string, error) {
value, ok := e.values[key]
if !ok {
return "", fmt.Errorf("not found")
}
return value, nil
}
func TestLoadDirNonexistent(t *testing.T) {
r := NewRegistry()
n, err := r.LoadDir("/nonexistent/path/12345")