From a2b2dc9ef2fca5c634016abf76339570e700036e Mon Sep 17 00:00:00 2001 From: Sam Catlow Date: Tue, 4 Aug 2026 23:32:47 +1000 Subject: [PATCH] fix(tools): let an optional provider fail without taking the daemon down provider.Registry.Start was atomic for every provider: one failure unregistered every other provider including the builtins, returned an error up through the capability host, and exited archied. Under Restart=on-failure that is a crash loop. Observed on carina 2026-07-30 -- mcp.desktop-commander could not start because the runtime image was missing unzip, and the daemon restarted continuously, losing chat, the gateway and every builtin tool over one third-party npm package. Atomicity is right for a provider archied requires. It is wrong for an external MCP server pulled from a registry at runtime, which is exactly the category allowed to be absent -- the same reasoning already applied to the forge, and to containers. RegisterOptional marks a provider the daemon can run without. Its failure to start, discover or index stops only itself and excludes it; Register keeps today's rollback. Required/optional is a deployment fact rather than a property of the code, so the registrar states it instead of the engine declaring it in its own manifest -- which also keeps plugin.Manifest metadata-only, per the plugin engine rule. A skipped provider's dependents cannot run either: an optional dependent is skipped with the reason, a required one is a hard failure, since it declared it cannot work without something now absent. Degradation is reported, not silent -- Health goes degraded naming each provider and cause, Skipped() exposes the set, and the daemon logs one error line per exclusion. Silence is how this went unnoticed until the crash loop. A provider that half-started before failing is still stopped, so no child process is left behind. The regression test drives the real capability host, the path main.go calls: with desktop-commander registered as required it fails with the verbatim carina error, and as optional the daemon starts with its builtins intact. Closes archie-core-ky6 --- CLAUDE.md | 19 +- cmd/archied/main.go | 18 +- internal/tools/provider/optional_test.go | 309 +++++++++++++++++++++++ internal/tools/provider/provider.go | 148 ++++++++++- 4 files changed, 472 insertions(+), 22 deletions(-) create mode 100644 internal/tools/provider/optional_test.go diff --git a/CLAUDE.md b/CLAUDE.md index 4fd074cc7..10ff389f3 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -150,12 +150,19 @@ boundaries. Summary of what's evolved since that doc was last fully accurate: `worktree.go:41-46` documents this reasoning for the prepared sentinel. If you change the path, update `cmd/archie-agent/main.go` `bootTaskID` (the reader) in lockstep. -- **`cmd/archied/main.go:588-598`** — the daemon registers MCP providers - itself and returns a hard failure if one cannot start. No npx-launched - servers are configured in production; the builtin tools (read, write, - edit, find, grep, shell, test) cover what desktop-commander did. Adding - an `[[tools.mcp_servers]]` entry to the config carries real blast radius: - a failed provider exits the daemon, taking Telegram down with it. +- **MCP providers are registered by the daemon as *optional*** (`cmd/archied/ + main.go`, `providerRegistry.RegisterOptional`). One that cannot start is + logged at error level, excluded from the running set, and reported through + `Registry.Health` as degraded — it does not roll back the other providers + or stop the daemon. That was previously a hard failure, so one broken npm + package unregistered every builtin tool and crash-looped archied under + `Restart=on-failure`, taking Telegram with it. Providers the daemon + genuinely requires still use `Register` and keep atomic rollback; check + `providerRegistry.Skipped()` when tools are mysteriously missing. No + npx-launched servers are configured in production anyway — the builtin + tools (read, write, edit, find, grep, shell, test) cover what + desktop-commander did. + **Task lifecycle** (from ARCHITECTURE.md, still current): `queued → running(workflow:stage) → pr_open → merged|rejected`, with `waiting_human` and `parked` side states. Crash recovery re-queues anything left diff --git a/cmd/archied/main.go b/cmd/archied/main.go index 3e51ad0f7..7adec1186 100644 --- a/cmd/archied/main.go +++ b/cmd/archied/main.go @@ -630,9 +630,15 @@ func run() int { log.Warn("mcp tool provider skipped", "name", srv.Name, "err", err) continue } - if err := providerRegistry.Register(provider); err != nil { - log.Error("mcp tool provider registration failed", "name", srv.Name, "err", err) - return 1 + // Optional: an MCP server is a third-party process pulled in at + // runtime, and it is exactly the category that is allowed to be + // absent. Registering it as required meant one failing npm package + // unregistered every builtin tool and exited the daemon, which under + // Restart=on-failure is a crash loop that takes chat and the gateway + // with it. + if err := providerRegistry.RegisterOptional(provider); err != nil { + log.Warn("mcp tool provider skipped", "name", srv.Name, "err", err) + continue } } if err := capabilityHost.Register(providerRegistry); err != nil { @@ -694,6 +700,12 @@ func run() int { log.Error("capability host startup", "err", err) return 1 } + // Optional providers that could not start are excluded rather than fatal, + // so the only way an operator learns about one is this line. + for _, skipped := range providerRegistry.Skipped() { + log.Error("tool provider unavailable; archie is running without its tools", + "provider", skipped.ID, "err", skipped.Err) + } if err := d.Startup(ctx); err != nil { log.Error("startup", "err", err) diff --git a/internal/tools/provider/optional_test.go b/internal/tools/provider/optional_test.go new file mode 100644 index 000000000..51c670bef --- /dev/null +++ b/internal/tools/provider/optional_test.go @@ -0,0 +1,309 @@ +package provider + +import ( + "context" + "errors" + "slices" + "strings" + "testing" + + "github.com/samcharles93/archie-core/internal/plugin" + "github.com/samcharles93/archie-core/internal/tools" +) + +// Atomic rollback is right for a provider the daemon requires. It is wrong +// for an external MCP server pulled from a registry at runtime -- exactly the +// category that is allowed to be absent. +// +// Observed on carina 2026-07-30: mcp.desktop-commander could not start +// (missing unzip in the runtime image) and archied crash-looped under +// Restart=on-failure, losing chat, the gateway and every builtin tool over +// one third-party npm package. +func TestOptionalProviderFailureDoesNotStopTheFamily(t *testing.T) { + failure := errors.New("npx: command not found") + + tests := []struct { + name string + // break is applied to the optional engine. + breakEngine func(*fakeEngine) + wantReason string + }{ + { + name: "start fails", + breakEngine: func(e *fakeEngine) { e.startErr = failure }, + wantReason: "npx", + }, + { + name: "discover fails", + breakEngine: func(e *fakeEngine) { e.discoverErr = failure }, + wantReason: "npx", + }, + { + name: "start panics", + breakEngine: func(e *fakeEngine) { e.startPanic = "boom" }, + wantReason: "boom", + }, + { + name: "discover panics", + breakEngine: func(e *fakeEngine) { e.discoverPanic = "boom" }, + wantReason: "boom", + }, + } + + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + index := tools.NewRegistry() + registry := NewRegistry(index) + + required := newFakeEngine("builtin") + required.tools = []tools.ToolEntry{testTool("read"), testTool("shell")} + if err := registry.Register(required); err != nil { + t.Fatalf("Register(required): %v", err) + } + + optional := newFakeEngine("mcp.desktop-commander") + optional.tools = []tools.ToolEntry{testTool("desktop")} + tc.breakEngine(optional) + if err := registry.RegisterOptional(optional); err != nil { + t.Fatalf("RegisterOptional: %v", err) + } + + if err := registry.Start(context.Background()); err != nil { + t.Fatalf("Start returned %v; an optional provider failing must not "+ + "fail the family, or the daemon exits and crash-loops", err) + } + + // The builtins must still be there. This is the whole point. + for _, name := range []string{"read", "shell"} { + if _, ok := index.Get(name); !ok { + t.Errorf("tool %q is missing: an optional provider's failure "+ + "rolled back the required providers", name) + } + } + // The failed provider's tools must not be half-registered. + if _, ok := index.Get("desktop"); ok { + t.Error("tool \"desktop\" is registered despite its provider failing") + } + + // Degraded, not healthy: a silent failure is how this went + // unnoticed until the daemon crash-looped. + health := registry.Health(context.Background()) + if health.Status != plugin.HealthDegraded { + t.Errorf("Health = %q, want %q", health.Status, plugin.HealthDegraded) + } + if !strings.Contains(health.Message, "mcp.desktop-commander") { + t.Errorf("Health message %q does not name the failed provider", health.Message) + } + + skipped := registry.Skipped() + if len(skipped) != 1 { + t.Fatalf("Skipped() = %v, want exactly the failed provider", skipped) + } + if skipped[0].ID != "mcp.desktop-commander" { + t.Errorf("Skipped()[0].ID = %q, want %q", skipped[0].ID, "mcp.desktop-commander") + } + if skipped[0].Err == nil { + t.Error("Skipped()[0].Err is nil: the operator has nothing to act on") + } else if !strings.Contains(skipped[0].Err.Error(), tc.wantReason) { + t.Errorf("Skipped()[0].Err = %v, want it to mention %q", skipped[0].Err, tc.wantReason) + } + }) + } +} + +// A required provider keeps today's all-or-nothing semantics. +func TestRequiredProviderFailureStillRollsBack(t *testing.T) { + index := tools.NewRegistry() + registry := NewRegistry(index) + + healthy := newFakeEngine("builtin") + healthy.tools = []tools.ToolEntry{testTool("read")} + if err := registry.Register(healthy); err != nil { + t.Fatal(err) + } + broken := newFakeEngine("required-thing") + broken.startErr = errors.New("nope") + if err := registry.Register(broken); err != nil { + t.Fatal(err) + } + + if err := registry.Start(context.Background()); err == nil { + t.Fatal("Start returned nil; a required provider failing must still fail the family") + } + if _, ok := index.Get("read"); ok { + t.Error("tool \"read\" survived: the required-provider rollback no longer rolls back") + } +} + +// Dependencies of a skipped provider cannot run either. An optional dependent +// is skipped with the reason; a required dependent is a hard failure, because +// it declared it cannot work without something that is now absent. +func TestSkippedProviderCascadesToDependents(t *testing.T) { + tests := []struct { + name string + dependentOptional bool + wantStartErr bool + wantSkippedIDs []string + }{ + { + name: "optional dependent is skipped too", + dependentOptional: true, + wantSkippedIDs: []string{"dependent", "flaky"}, + }, + { + name: "required dependent fails the family", + wantStartErr: true, + }, + } + + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + index := tools.NewRegistry() + registry := NewRegistry(index) + + flaky := newFakeEngine("flaky") + flaky.startErr = errors.New("unavailable") + if err := registry.RegisterOptional(flaky); err != nil { + t.Fatal(err) + } + + dependent := engineWithDependencies("dependent", "flaky") + dependent.tools = []tools.ToolEntry{testTool("dependent-tool")} + var err error + if tc.dependentOptional { + err = registry.RegisterOptional(dependent) + } else { + err = registry.Register(dependent) + } + if err != nil { + t.Fatal(err) + } + + startErr := registry.Start(context.Background()) + if tc.wantStartErr { + if startErr == nil { + t.Fatal("Start returned nil; a required provider depending on an " + + "unavailable one must fail rather than run without it") + } + return + } + if startErr != nil { + t.Fatalf("Start: %v", startErr) + } + if _, ok := index.Get("dependent-tool"); ok { + t.Error("a provider whose dependency was skipped was started anyway") + } + + got := make([]string, 0, len(registry.Skipped())) + for _, s := range registry.Skipped() { + got = append(got, s.ID) + } + slices.Sort(got) + slices.Sort(tc.wantSkippedIDs) + if !slices.Equal(got, tc.wantSkippedIDs) { + t.Errorf("Skipped() = %v, want %v", got, tc.wantSkippedIDs) + } + }) + } +} + +// All-optional-and-all-failing is still a successful start: the daemon runs +// with its builtins and nothing else, which is the degraded state we want +// rather than no daemon at all. +func TestEveryOptionalProviderFailing(t *testing.T) { + index := tools.NewRegistry() + registry := NewRegistry(index) + + for _, id := range []string{"mcp.one", "mcp.two"} { + engine := newFakeEngine(id) + engine.startErr = errors.New("gone") + if err := registry.RegisterOptional(engine); err != nil { + t.Fatal(err) + } + } + + if err := registry.Start(context.Background()); err != nil { + t.Fatalf("Start: %v", err) + } + if got := registry.Health(context.Background()).Status; got != plugin.HealthDegraded { + t.Errorf("Health = %q, want %q", got, plugin.HealthDegraded) + } + if len(registry.Skipped()) != 2 { + t.Errorf("Skipped() = %v, want both providers", registry.Skipped()) + } + if err := registry.Stop(context.Background()); err != nil { + t.Errorf("Stop: %v", err) + } +} + +// A failed optional provider must still be stopped, or a half-started child +// process is left behind for the daemon's lifetime. +func TestSkippedOptionalProviderIsCleanedUp(t *testing.T) { + registry := NewRegistry(tools.NewRegistry()) + + optional := newFakeEngine("mcp.leaky") + optional.discoverErr = errors.New("no tools") + if err := registry.RegisterOptional(optional); err != nil { + t.Fatal(err) + } + if err := registry.Start(context.Background()); err != nil { + t.Fatalf("Start: %v", err) + } + if optional.stopCount != 1 { + t.Errorf("stopCount = %d, want 1: a provider that started but failed to "+ + "discover must be stopped, not abandoned", optional.stopCount) + } +} + +// The carina failure, reproduced through the capability host -- the path +// main.go actually calls, and the one that returned the error that exited the +// daemon. Registering desktop-commander as required makes this fail with +// `start plugin module "archie.tools": ... unzip: not found`, which is +// verbatim what crash-looped archied on 2026-07-30. +func TestCarinaScenarioThroughCapabilityHost(t *testing.T) { + index := tools.NewRegistry() + registry := NewRegistry(index) + + builtins := newFakeEngine("archie.tools.builtin") + builtins.tools = []tools.ToolEntry{testTool("read"), testTool("shell"), testTool("edit")} + if err := registry.Register(builtins); err != nil { + t.Fatal(err) + } + memory := newFakeEngine("archie.tools.memory") + memory.tools = []tools.ToolEntry{testTool("memory_edit")} + if err := registry.Register(memory); err != nil { + t.Fatal(err) + } + desktop := newFakeEngine("mcp.desktop-commander") + desktop.startErr = errors.New("npx exec: unzip: not found") + if err := registry.RegisterOptional(desktop); err != nil { + t.Fatal(err) + } + + host := plugin.NewHost() + if err := host.Register(registry); err != nil { + t.Fatal(err) + } + if err := host.Start(context.Background()); err != nil { + t.Fatalf("capability host startup failed: %v\n"+ + "main.go logs 'capability host startup' and returns 1 here, which "+ + "under Restart=on-failure is the crash loop", err) + } + + // Everything the operator actually depends on is still there. + for _, name := range []string{"read", "shell", "edit", "memory_edit"} { + if _, ok := index.Get(name); !ok { + t.Errorf("tool %q was lost to a third-party npm package", name) + } + } + health := registry.Health(context.Background()) + if health.Status != plugin.HealthDegraded { + t.Errorf("health = %q (%s), want degraded", health.Status, health.Message) + } + if !strings.Contains(health.Message, "unzip") { + t.Errorf("health message %q does not carry the cause", health.Message) + } + if err := host.Stop(context.Background()); err != nil { + t.Errorf("Stop: %v", err) + } +} diff --git a/internal/tools/provider/provider.go b/internal/tools/provider/provider.go index d9805f608..2a80f7d1c 100644 --- a/internal/tools/provider/provider.go +++ b/internal/tools/provider/provider.go @@ -46,6 +46,17 @@ const ( type registration struct { engine Engine manifest plugin.Manifest + // optional marks a provider the daemon can run without. Atomicity is + // right for a provider archied requires; it is wrong for an external + // MCP server pulled from a registry at runtime, which is exactly the + // category that is allowed to be absent. + optional bool +} + +// SkippedProvider is an optional provider that could not be started, and why. +type SkippedProvider struct { + ID string + Err error } type runningProvider struct { @@ -63,7 +74,11 @@ type Registry struct { index *tools.Registry providers map[string]registration running []runningProvider - state registryState + // skipped records optional providers excluded from the running set. + // Health reports it: a silent degradation is how one broken npm package + // went unnoticed until the daemon crash-looped. + skipped []SkippedProvider + state registryState } // NewRegistry creates a provider registry backed by the central tool index. @@ -77,8 +92,24 @@ func NewRegistry(index *tools.Registry) *Registry { } } -// Register adds an engine to the family. +// Register adds an engine the daemon requires. Its failure to start or +// discover fails the whole family, as before. func (r *Registry) Register(engine Engine) error { + return r.register(engine, false) +} + +// RegisterOptional adds an engine the daemon can run without. +// +// An optional provider that fails to start or discover is logged, excluded +// from the running set, and does not roll the family back or stop the daemon. +// Required/optional is a deployment decision, not a property of the code, so +// it is stated by the registrar rather than declared in the engine's own +// manifest. +func (r *Registry) RegisterOptional(engine Engine) error { + return r.register(engine, true) +} + +func (r *Registry) register(engine Engine, optional bool) error { if isNilEngine(engine) { return errors.New("tool provider is nil") } @@ -102,10 +133,17 @@ func (r *Registry) Register(engine Engine) error { if _, exists := r.providers[manifest.ID]; exists { return fmt.Errorf("%w: %q", ErrDuplicateProvider, manifest.ID) } - r.providers[manifest.ID] = registration{engine: engine, manifest: manifest} + r.providers[manifest.ID] = registration{engine: engine, manifest: manifest, optional: optional} return nil } +// Skipped returns the optional providers excluded from the running set. +func (r *Registry) Skipped() []SkippedProvider { + r.mu.RLock() + defer r.mu.RUnlock() + return append([]SkippedProvider(nil), r.skipped...) +} + // Get returns a registered engine by manifest ID. func (r *Registry) Get(id string) (Engine, bool) { r.mu.RLock() @@ -154,27 +192,74 @@ func (r *Registry) Start(ctx context.Context) error { } started := make([]runningProvider, 0, len(order)) + var skipped []SkippedProvider + unavailable := make(map[string]struct{}) + for _, id := range order { reg := providers[id] + + // A provider whose dependency was skipped cannot run either. If it + // is required, that is a hard failure: it declared it cannot work + // without something now absent, so running it anyway would be worse + // than not starting. + if missing, blocked := firstUnavailable(reg, unavailable); blocked { + cause := fmt.Errorf("tool provider %q depends on unavailable provider %q", id, missing) + if !reg.optional { + return r.failFamily(ctx, started, cause) + } + unavailable[id] = struct{}{} + skipped = append(skipped, SkippedProvider{ID: id, Err: cause}) + continue + } + rp, err := r.startProvider(ctx, reg, started) - if err != nil { + if err == nil { + started = append(started, rp) + continue + } + if !reg.optional { + // failProvider has already rolled back and set the failed state. return err } - started = append(started, rp) + // Optional: the provider is out, the family carries on. + unavailable[id] = struct{}{} + skipped = append(skipped, SkippedProvider{ID: id, Err: err}) } r.mu.Lock() r.running = started + r.skipped = skipped r.state = stateRunning r.mu.Unlock() return nil } +// firstUnavailable reports the first dependency of reg that was skipped. +func firstUnavailable(reg registration, unavailable map[string]struct{}) (string, bool) { + for _, dependency := range reg.manifest.Dependencies { + if _, missing := unavailable[dependency]; missing { + return dependency, true + } + } + return "", false +} + +// failFamily rolls back everything started so far and marks the registry +// failed, for a failure that is not attributable to one provider's own Start. +func (r *Registry) failFamily(ctx context.Context, started []runningProvider, cause error) error { + cleanupCtx, cancel := context.WithTimeout(context.WithoutCancel(ctx), cleanupTimeout) + defer cancel() + rollbackFailed, rollbackErr := rollback(cleanupCtx, r.index, started) + r.setFailed(rollbackFailed) + return errors.Join(cause, rollbackErr) +} + // Health reports aggregate provider health. func (r *Registry) Health(ctx context.Context) plugin.Health { r.mu.RLock() state := r.state running := append([]runningProvider(nil), r.running...) + skipped := append([]SkippedProvider(nil), r.skipped...) r.mu.RUnlock() switch state { @@ -187,6 +272,19 @@ func (r *Registry) Health(ctx context.Context) plugin.Health { aggregate := plugin.Health{Status: plugin.HealthHealthy} var messages []string + + // An excluded optional provider is a degradation, not a clean start. + // Reporting healthy here is what let one broken npm package go unnoticed + // until the daemon crash-looped. + for _, s := range skipped { + aggregate.Status = plugin.HealthDegraded + message := s.ID + ": unavailable" + if s.Err != nil { + message = s.ID + ": " + s.Err.Error() + } + messages = append(messages, message) + } + for _, provider := range running { health := safeHealth(ctx, provider.registration.engine) switch health.Status { @@ -275,22 +373,35 @@ func (r *Registry) setFailed(running []runningProvider) { r.mu.Unlock() } -// startProvider starts, discovers, and indexes one provider. On failure -// it rolls back previously-started providers and returns a joined error. +// startProvider starts, discovers, and indexes one provider. +// +// A required provider's failure rolls back previously-started providers and +// returns a joined error. An optional provider's failure stops only itself +// and returns the cause, leaving the family running -- Start decides which, +// since only the registrar knows whether the daemon needs this provider. func (r *Registry) startProvider(ctx context.Context, reg registration, started []runningProvider) (runningProvider, error) { id := reg.manifest.ID + + fail := func(cause error, startedOK bool) (runningProvider, error) { + if reg.optional { + return runningProvider{}, r.abandonProvider(ctx, id, reg, cause, startedOK) + } + return runningProvider{}, r.failProvider(ctx, id, reg, started, cause) + } + if err := safeStart(ctx, reg.engine); err != nil { - return runningProvider{}, r.failProvider(ctx, id, reg, started, - fmt.Errorf("start tool provider %q: %w", id, err)) + // Start failed, so there may or may not be anything to stop. Stop is + // still attempted: a provider that half-started (spawned a child + // process, opened a pipe) and then errored would otherwise leak it + // for the daemon's lifetime. + return fail(fmt.Errorf("start tool provider %q: %w", id, err), true) } entries, err := safeDiscover(ctx, reg.engine) if err != nil { - return runningProvider{}, r.failProvider(ctx, id, reg, started, - fmt.Errorf("discover tools from provider %q: %w", id, err)) + return fail(fmt.Errorf("discover tools from provider %q: %w", id, err), true) } if err := r.index.RegisterBatch(entries); err != nil { - return runningProvider{}, r.failProvider(ctx, id, reg, started, - fmt.Errorf("index tools from provider %q: %w", id, err)) + return fail(fmt.Errorf("index tools from provider %q: %w", id, err), true) } return runningProvider{ registration: reg, @@ -298,6 +409,17 @@ func (r *Registry) startProvider(ctx context.Context, reg registration, started }, nil } +// abandonProvider stops one failed optional provider and returns why it was +// excluded. Nothing else is touched: the rest of the family keeps running. +func (r *Registry) abandonProvider(ctx context.Context, id string, reg registration, cause error, stop bool) error { + if !stop { + return cause + } + cleanupCtx, cancel := context.WithTimeout(context.WithoutCancel(ctx), cleanupTimeout) + defer cancel() + return errors.Join(cause, wrapCleanupError(id, safeStop(cleanupCtx, reg.engine))) +} + // failProvider stops the failing provider, rolls back previously-started // providers, and returns a joined error. func (r *Registry) failProvider(ctx context.Context, id string, reg registration, started []runningProvider, cause error) error {