diff --git a/CLAUDE.md b/CLAUDE.md index 4fd074cc7..10ff389f3 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -150,12 +150,19 @@ boundaries. Summary of what's evolved since that doc was last fully accurate: `worktree.go:41-46` documents this reasoning for the prepared sentinel. If you change the path, update `cmd/archie-agent/main.go` `bootTaskID` (the reader) in lockstep. -- **`cmd/archied/main.go:588-598`** — the daemon registers MCP providers - itself and returns a hard failure if one cannot start. No npx-launched - servers are configured in production; the builtin tools (read, write, - edit, find, grep, shell, test) cover what desktop-commander did. Adding - an `[[tools.mcp_servers]]` entry to the config carries real blast radius: - a failed provider exits the daemon, taking Telegram down with it. +- **MCP providers are registered by the daemon as *optional*** (`cmd/archied/ + main.go`, `providerRegistry.RegisterOptional`). One that cannot start is + logged at error level, excluded from the running set, and reported through + `Registry.Health` as degraded — it does not roll back the other providers + or stop the daemon. That was previously a hard failure, so one broken npm + package unregistered every builtin tool and crash-looped archied under + `Restart=on-failure`, taking Telegram with it. Providers the daemon + genuinely requires still use `Register` and keep atomic rollback; check + `providerRegistry.Skipped()` when tools are mysteriously missing. No + npx-launched servers are configured in production anyway — the builtin + tools (read, write, edit, find, grep, shell, test) cover what + desktop-commander did. + **Task lifecycle** (from ARCHITECTURE.md, still current): `queued → running(workflow:stage) → pr_open → merged|rejected`, with `waiting_human` and `parked` side states. Crash recovery re-queues anything left diff --git a/cmd/archied/main.go b/cmd/archied/main.go index 3e51ad0f7..7adec1186 100644 --- a/cmd/archied/main.go +++ b/cmd/archied/main.go @@ -630,9 +630,15 @@ func run() int { log.Warn("mcp tool provider skipped", "name", srv.Name, "err", err) continue } - if err := providerRegistry.Register(provider); err != nil { - log.Error("mcp tool provider registration failed", "name", srv.Name, "err", err) - return 1 + // Optional: an MCP server is a third-party process pulled in at + // runtime, and it is exactly the category that is allowed to be + // absent. Registering it as required meant one failing npm package + // unregistered every builtin tool and exited the daemon, which under + // Restart=on-failure is a crash loop that takes chat and the gateway + // with it. + if err := providerRegistry.RegisterOptional(provider); err != nil { + log.Warn("mcp tool provider skipped", "name", srv.Name, "err", err) + continue } } if err := capabilityHost.Register(providerRegistry); err != nil { @@ -694,6 +700,12 @@ func run() int { log.Error("capability host startup", "err", err) return 1 } + // Optional providers that could not start are excluded rather than fatal, + // so the only way an operator learns about one is this line. + for _, skipped := range providerRegistry.Skipped() { + log.Error("tool provider unavailable; archie is running without its tools", + "provider", skipped.ID, "err", skipped.Err) + } if err := d.Startup(ctx); err != nil { log.Error("startup", "err", err) diff --git a/internal/tools/provider/optional_test.go b/internal/tools/provider/optional_test.go new file mode 100644 index 000000000..51c670bef --- /dev/null +++ b/internal/tools/provider/optional_test.go @@ -0,0 +1,309 @@ +package provider + +import ( + "context" + "errors" + "slices" + "strings" + "testing" + + "github.com/samcharles93/archie-core/internal/plugin" + "github.com/samcharles93/archie-core/internal/tools" +) + +// Atomic rollback is right for a provider the daemon requires. It is wrong +// for an external MCP server pulled from a registry at runtime -- exactly the +// category that is allowed to be absent. +// +// Observed on carina 2026-07-30: mcp.desktop-commander could not start +// (missing unzip in the runtime image) and archied crash-looped under +// Restart=on-failure, losing chat, the gateway and every builtin tool over +// one third-party npm package. +func TestOptionalProviderFailureDoesNotStopTheFamily(t *testing.T) { + failure := errors.New("npx: command not found") + + tests := []struct { + name string + // break is applied to the optional engine. + breakEngine func(*fakeEngine) + wantReason string + }{ + { + name: "start fails", + breakEngine: func(e *fakeEngine) { e.startErr = failure }, + wantReason: "npx", + }, + { + name: "discover fails", + breakEngine: func(e *fakeEngine) { e.discoverErr = failure }, + wantReason: "npx", + }, + { + name: "start panics", + breakEngine: func(e *fakeEngine) { e.startPanic = "boom" }, + wantReason: "boom", + }, + { + name: "discover panics", + breakEngine: func(e *fakeEngine) { e.discoverPanic = "boom" }, + wantReason: "boom", + }, + } + + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + index := tools.NewRegistry() + registry := NewRegistry(index) + + required := newFakeEngine("builtin") + required.tools = []tools.ToolEntry{testTool("read"), testTool("shell")} + if err := registry.Register(required); err != nil { + t.Fatalf("Register(required): %v", err) + } + + optional := newFakeEngine("mcp.desktop-commander") + optional.tools = []tools.ToolEntry{testTool("desktop")} + tc.breakEngine(optional) + if err := registry.RegisterOptional(optional); err != nil { + t.Fatalf("RegisterOptional: %v", err) + } + + if err := registry.Start(context.Background()); err != nil { + t.Fatalf("Start returned %v; an optional provider failing must not "+ + "fail the family, or the daemon exits and crash-loops", err) + } + + // The builtins must still be there. This is the whole point. + for _, name := range []string{"read", "shell"} { + if _, ok := index.Get(name); !ok { + t.Errorf("tool %q is missing: an optional provider's failure "+ + "rolled back the required providers", name) + } + } + // The failed provider's tools must not be half-registered. + if _, ok := index.Get("desktop"); ok { + t.Error("tool \"desktop\" is registered despite its provider failing") + } + + // Degraded, not healthy: a silent failure is how this went + // unnoticed until the daemon crash-looped. + health := registry.Health(context.Background()) + if health.Status != plugin.HealthDegraded { + t.Errorf("Health = %q, want %q", health.Status, plugin.HealthDegraded) + } + if !strings.Contains(health.Message, "mcp.desktop-commander") { + t.Errorf("Health message %q does not name the failed provider", health.Message) + } + + skipped := registry.Skipped() + if len(skipped) != 1 { + t.Fatalf("Skipped() = %v, want exactly the failed provider", skipped) + } + if skipped[0].ID != "mcp.desktop-commander" { + t.Errorf("Skipped()[0].ID = %q, want %q", skipped[0].ID, "mcp.desktop-commander") + } + if skipped[0].Err == nil { + t.Error("Skipped()[0].Err is nil: the operator has nothing to act on") + } else if !strings.Contains(skipped[0].Err.Error(), tc.wantReason) { + t.Errorf("Skipped()[0].Err = %v, want it to mention %q", skipped[0].Err, tc.wantReason) + } + }) + } +} + +// A required provider keeps today's all-or-nothing semantics. +func TestRequiredProviderFailureStillRollsBack(t *testing.T) { + index := tools.NewRegistry() + registry := NewRegistry(index) + + healthy := newFakeEngine("builtin") + healthy.tools = []tools.ToolEntry{testTool("read")} + if err := registry.Register(healthy); err != nil { + t.Fatal(err) + } + broken := newFakeEngine("required-thing") + broken.startErr = errors.New("nope") + if err := registry.Register(broken); err != nil { + t.Fatal(err) + } + + if err := registry.Start(context.Background()); err == nil { + t.Fatal("Start returned nil; a required provider failing must still fail the family") + } + if _, ok := index.Get("read"); ok { + t.Error("tool \"read\" survived: the required-provider rollback no longer rolls back") + } +} + +// Dependencies of a skipped provider cannot run either. An optional dependent +// is skipped with the reason; a required dependent is a hard failure, because +// it declared it cannot work without something that is now absent. +func TestSkippedProviderCascadesToDependents(t *testing.T) { + tests := []struct { + name string + dependentOptional bool + wantStartErr bool + wantSkippedIDs []string + }{ + { + name: "optional dependent is skipped too", + dependentOptional: true, + wantSkippedIDs: []string{"dependent", "flaky"}, + }, + { + name: "required dependent fails the family", + wantStartErr: true, + }, + } + + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + index := tools.NewRegistry() + registry := NewRegistry(index) + + flaky := newFakeEngine("flaky") + flaky.startErr = errors.New("unavailable") + if err := registry.RegisterOptional(flaky); err != nil { + t.Fatal(err) + } + + dependent := engineWithDependencies("dependent", "flaky") + dependent.tools = []tools.ToolEntry{testTool("dependent-tool")} + var err error + if tc.dependentOptional { + err = registry.RegisterOptional(dependent) + } else { + err = registry.Register(dependent) + } + if err != nil { + t.Fatal(err) + } + + startErr := registry.Start(context.Background()) + if tc.wantStartErr { + if startErr == nil { + t.Fatal("Start returned nil; a required provider depending on an " + + "unavailable one must fail rather than run without it") + } + return + } + if startErr != nil { + t.Fatalf("Start: %v", startErr) + } + if _, ok := index.Get("dependent-tool"); ok { + t.Error("a provider whose dependency was skipped was started anyway") + } + + got := make([]string, 0, len(registry.Skipped())) + for _, s := range registry.Skipped() { + got = append(got, s.ID) + } + slices.Sort(got) + slices.Sort(tc.wantSkippedIDs) + if !slices.Equal(got, tc.wantSkippedIDs) { + t.Errorf("Skipped() = %v, want %v", got, tc.wantSkippedIDs) + } + }) + } +} + +// All-optional-and-all-failing is still a successful start: the daemon runs +// with its builtins and nothing else, which is the degraded state we want +// rather than no daemon at all. +func TestEveryOptionalProviderFailing(t *testing.T) { + index := tools.NewRegistry() + registry := NewRegistry(index) + + for _, id := range []string{"mcp.one", "mcp.two"} { + engine := newFakeEngine(id) + engine.startErr = errors.New("gone") + if err := registry.RegisterOptional(engine); err != nil { + t.Fatal(err) + } + } + + if err := registry.Start(context.Background()); err != nil { + t.Fatalf("Start: %v", err) + } + if got := registry.Health(context.Background()).Status; got != plugin.HealthDegraded { + t.Errorf("Health = %q, want %q", got, plugin.HealthDegraded) + } + if len(registry.Skipped()) != 2 { + t.Errorf("Skipped() = %v, want both providers", registry.Skipped()) + } + if err := registry.Stop(context.Background()); err != nil { + t.Errorf("Stop: %v", err) + } +} + +// A failed optional provider must still be stopped, or a half-started child +// process is left behind for the daemon's lifetime. +func TestSkippedOptionalProviderIsCleanedUp(t *testing.T) { + registry := NewRegistry(tools.NewRegistry()) + + optional := newFakeEngine("mcp.leaky") + optional.discoverErr = errors.New("no tools") + if err := registry.RegisterOptional(optional); err != nil { + t.Fatal(err) + } + if err := registry.Start(context.Background()); err != nil { + t.Fatalf("Start: %v", err) + } + if optional.stopCount != 1 { + t.Errorf("stopCount = %d, want 1: a provider that started but failed to "+ + "discover must be stopped, not abandoned", optional.stopCount) + } +} + +// The carina failure, reproduced through the capability host -- the path +// main.go actually calls, and the one that returned the error that exited the +// daemon. Registering desktop-commander as required makes this fail with +// `start plugin module "archie.tools": ... unzip: not found`, which is +// verbatim what crash-looped archied on 2026-07-30. +func TestCarinaScenarioThroughCapabilityHost(t *testing.T) { + index := tools.NewRegistry() + registry := NewRegistry(index) + + builtins := newFakeEngine("archie.tools.builtin") + builtins.tools = []tools.ToolEntry{testTool("read"), testTool("shell"), testTool("edit")} + if err := registry.Register(builtins); err != nil { + t.Fatal(err) + } + memory := newFakeEngine("archie.tools.memory") + memory.tools = []tools.ToolEntry{testTool("memory_edit")} + if err := registry.Register(memory); err != nil { + t.Fatal(err) + } + desktop := newFakeEngine("mcp.desktop-commander") + desktop.startErr = errors.New("npx exec: unzip: not found") + if err := registry.RegisterOptional(desktop); err != nil { + t.Fatal(err) + } + + host := plugin.NewHost() + if err := host.Register(registry); err != nil { + t.Fatal(err) + } + if err := host.Start(context.Background()); err != nil { + t.Fatalf("capability host startup failed: %v\n"+ + "main.go logs 'capability host startup' and returns 1 here, which "+ + "under Restart=on-failure is the crash loop", err) + } + + // Everything the operator actually depends on is still there. + for _, name := range []string{"read", "shell", "edit", "memory_edit"} { + if _, ok := index.Get(name); !ok { + t.Errorf("tool %q was lost to a third-party npm package", name) + } + } + health := registry.Health(context.Background()) + if health.Status != plugin.HealthDegraded { + t.Errorf("health = %q (%s), want degraded", health.Status, health.Message) + } + if !strings.Contains(health.Message, "unzip") { + t.Errorf("health message %q does not carry the cause", health.Message) + } + if err := host.Stop(context.Background()); err != nil { + t.Errorf("Stop: %v", err) + } +} diff --git a/internal/tools/provider/provider.go b/internal/tools/provider/provider.go index d9805f608..2a80f7d1c 100644 --- a/internal/tools/provider/provider.go +++ b/internal/tools/provider/provider.go @@ -46,6 +46,17 @@ const ( type registration struct { engine Engine manifest plugin.Manifest + // optional marks a provider the daemon can run without. Atomicity is + // right for a provider archied requires; it is wrong for an external + // MCP server pulled from a registry at runtime, which is exactly the + // category that is allowed to be absent. + optional bool +} + +// SkippedProvider is an optional provider that could not be started, and why. +type SkippedProvider struct { + ID string + Err error } type runningProvider struct { @@ -63,7 +74,11 @@ type Registry struct { index *tools.Registry providers map[string]registration running []runningProvider - state registryState + // skipped records optional providers excluded from the running set. + // Health reports it: a silent degradation is how one broken npm package + // went unnoticed until the daemon crash-looped. + skipped []SkippedProvider + state registryState } // NewRegistry creates a provider registry backed by the central tool index. @@ -77,8 +92,24 @@ func NewRegistry(index *tools.Registry) *Registry { } } -// Register adds an engine to the family. +// Register adds an engine the daemon requires. Its failure to start or +// discover fails the whole family, as before. func (r *Registry) Register(engine Engine) error { + return r.register(engine, false) +} + +// RegisterOptional adds an engine the daemon can run without. +// +// An optional provider that fails to start or discover is logged, excluded +// from the running set, and does not roll the family back or stop the daemon. +// Required/optional is a deployment decision, not a property of the code, so +// it is stated by the registrar rather than declared in the engine's own +// manifest. +func (r *Registry) RegisterOptional(engine Engine) error { + return r.register(engine, true) +} + +func (r *Registry) register(engine Engine, optional bool) error { if isNilEngine(engine) { return errors.New("tool provider is nil") } @@ -102,10 +133,17 @@ func (r *Registry) Register(engine Engine) error { if _, exists := r.providers[manifest.ID]; exists { return fmt.Errorf("%w: %q", ErrDuplicateProvider, manifest.ID) } - r.providers[manifest.ID] = registration{engine: engine, manifest: manifest} + r.providers[manifest.ID] = registration{engine: engine, manifest: manifest, optional: optional} return nil } +// Skipped returns the optional providers excluded from the running set. +func (r *Registry) Skipped() []SkippedProvider { + r.mu.RLock() + defer r.mu.RUnlock() + return append([]SkippedProvider(nil), r.skipped...) +} + // Get returns a registered engine by manifest ID. func (r *Registry) Get(id string) (Engine, bool) { r.mu.RLock() @@ -154,27 +192,74 @@ func (r *Registry) Start(ctx context.Context) error { } started := make([]runningProvider, 0, len(order)) + var skipped []SkippedProvider + unavailable := make(map[string]struct{}) + for _, id := range order { reg := providers[id] + + // A provider whose dependency was skipped cannot run either. If it + // is required, that is a hard failure: it declared it cannot work + // without something now absent, so running it anyway would be worse + // than not starting. + if missing, blocked := firstUnavailable(reg, unavailable); blocked { + cause := fmt.Errorf("tool provider %q depends on unavailable provider %q", id, missing) + if !reg.optional { + return r.failFamily(ctx, started, cause) + } + unavailable[id] = struct{}{} + skipped = append(skipped, SkippedProvider{ID: id, Err: cause}) + continue + } + rp, err := r.startProvider(ctx, reg, started) - if err != nil { + if err == nil { + started = append(started, rp) + continue + } + if !reg.optional { + // failProvider has already rolled back and set the failed state. return err } - started = append(started, rp) + // Optional: the provider is out, the family carries on. + unavailable[id] = struct{}{} + skipped = append(skipped, SkippedProvider{ID: id, Err: err}) } r.mu.Lock() r.running = started + r.skipped = skipped r.state = stateRunning r.mu.Unlock() return nil } +// firstUnavailable reports the first dependency of reg that was skipped. +func firstUnavailable(reg registration, unavailable map[string]struct{}) (string, bool) { + for _, dependency := range reg.manifest.Dependencies { + if _, missing := unavailable[dependency]; missing { + return dependency, true + } + } + return "", false +} + +// failFamily rolls back everything started so far and marks the registry +// failed, for a failure that is not attributable to one provider's own Start. +func (r *Registry) failFamily(ctx context.Context, started []runningProvider, cause error) error { + cleanupCtx, cancel := context.WithTimeout(context.WithoutCancel(ctx), cleanupTimeout) + defer cancel() + rollbackFailed, rollbackErr := rollback(cleanupCtx, r.index, started) + r.setFailed(rollbackFailed) + return errors.Join(cause, rollbackErr) +} + // Health reports aggregate provider health. func (r *Registry) Health(ctx context.Context) plugin.Health { r.mu.RLock() state := r.state running := append([]runningProvider(nil), r.running...) + skipped := append([]SkippedProvider(nil), r.skipped...) r.mu.RUnlock() switch state { @@ -187,6 +272,19 @@ func (r *Registry) Health(ctx context.Context) plugin.Health { aggregate := plugin.Health{Status: plugin.HealthHealthy} var messages []string + + // An excluded optional provider is a degradation, not a clean start. + // Reporting healthy here is what let one broken npm package go unnoticed + // until the daemon crash-looped. + for _, s := range skipped { + aggregate.Status = plugin.HealthDegraded + message := s.ID + ": unavailable" + if s.Err != nil { + message = s.ID + ": " + s.Err.Error() + } + messages = append(messages, message) + } + for _, provider := range running { health := safeHealth(ctx, provider.registration.engine) switch health.Status { @@ -275,22 +373,35 @@ func (r *Registry) setFailed(running []runningProvider) { r.mu.Unlock() } -// startProvider starts, discovers, and indexes one provider. On failure -// it rolls back previously-started providers and returns a joined error. +// startProvider starts, discovers, and indexes one provider. +// +// A required provider's failure rolls back previously-started providers and +// returns a joined error. An optional provider's failure stops only itself +// and returns the cause, leaving the family running -- Start decides which, +// since only the registrar knows whether the daemon needs this provider. func (r *Registry) startProvider(ctx context.Context, reg registration, started []runningProvider) (runningProvider, error) { id := reg.manifest.ID + + fail := func(cause error, startedOK bool) (runningProvider, error) { + if reg.optional { + return runningProvider{}, r.abandonProvider(ctx, id, reg, cause, startedOK) + } + return runningProvider{}, r.failProvider(ctx, id, reg, started, cause) + } + if err := safeStart(ctx, reg.engine); err != nil { - return runningProvider{}, r.failProvider(ctx, id, reg, started, - fmt.Errorf("start tool provider %q: %w", id, err)) + // Start failed, so there may or may not be anything to stop. Stop is + // still attempted: a provider that half-started (spawned a child + // process, opened a pipe) and then errored would otherwise leak it + // for the daemon's lifetime. + return fail(fmt.Errorf("start tool provider %q: %w", id, err), true) } entries, err := safeDiscover(ctx, reg.engine) if err != nil { - return runningProvider{}, r.failProvider(ctx, id, reg, started, - fmt.Errorf("discover tools from provider %q: %w", id, err)) + return fail(fmt.Errorf("discover tools from provider %q: %w", id, err), true) } if err := r.index.RegisterBatch(entries); err != nil { - return runningProvider{}, r.failProvider(ctx, id, reg, started, - fmt.Errorf("index tools from provider %q: %w", id, err)) + return fail(fmt.Errorf("index tools from provider %q: %w", id, err), true) } return runningProvider{ registration: reg, @@ -298,6 +409,17 @@ func (r *Registry) startProvider(ctx context.Context, reg registration, started }, nil } +// abandonProvider stops one failed optional provider and returns why it was +// excluded. Nothing else is touched: the rest of the family keeps running. +func (r *Registry) abandonProvider(ctx context.Context, id string, reg registration, cause error, stop bool) error { + if !stop { + return cause + } + cleanupCtx, cancel := context.WithTimeout(context.WithoutCancel(ctx), cleanupTimeout) + defer cancel() + return errors.Join(cause, wrapCleanupError(id, safeStop(cleanupCtx, reg.engine))) +} + // failProvider stops the failing provider, rolls back previously-started // providers, and returns a joined error. func (r *Registry) failProvider(ctx context.Context, id string, reg registration, started []runningProvider, cause error) error {