From 7d664d7efb15d306edf94b453d6b0c8d1da00782 Mon Sep 17 00:00:00 2001 From: Sam Catlow Date: Sat, 14 Nov 2026 04:00:51 +1100 Subject: [PATCH] fix(archied): label OpenShell gateway providers and sweep leftovers at boot (#1970) --- internal/app/archied/bootstrap.go | 8 ++++- .../app/archied/openshell_model_egress.go | 28 ++++++++++++++-- .../archied/openshell_model_egress_test.go | 33 +++++++++++++++++++ internal/infrastructure/openshell/runtime.go | 12 +++---- .../infrastructure/openshell/runtime_test.go | 4 +-- 5 files changed, 74 insertions(+), 11 deletions(-) diff --git a/internal/app/archied/bootstrap.go b/internal/app/archied/bootstrap.go index a21337a4e..9e828725b 100644 --- a/internal/app/archied/bootstrap.go +++ b/internal/app/archied/bootstrap.go @@ -511,7 +511,13 @@ func (b *boot) setupContainers(ctx context.Context) func() { b.taskRuntime = daemon.OpenShellRuntime(rt) settings := b.cfg.Containers.OpenShell bindings := runCredentialResolver{runs: b.stateStoreGrants.Client, models: b.taskModels, config: b.cfgHolder, secrets: b.secrets, credentials: b.credentialWatch} - b.modelEgress = newOpenShellModelEgress(client, settings.Workspace, b.cfg.Instance, b.secrets, bindings, b.log) + egress := newOpenShellModelEgress(client, settings.Workspace, b.cfg.Instance, b.secrets, bindings, b.log) + b.modelEgress = egress + sweepCtx, cancel := context.WithTimeout(ctx, 15*time.Second) + defer cancel() + if err := egress.Sweep(sweepCtx); err != nil { + b.log.WarnContext(ctx, "openshell: orphaned gateway providers remain", "err", err) + } return func() { closeOpenShell(); closeDocker() } } diff --git a/internal/app/archied/openshell_model_egress.go b/internal/app/archied/openshell_model_egress.go index 0f9aa0900..a43403609 100644 --- a/internal/app/archied/openshell_model_egress.go +++ b/internal/app/archied/openshell_model_egress.go @@ -4,6 +4,7 @@ import ( "context" "crypto/rand" "encoding/hex" + "errors" "fmt" "log/slog" "maps" @@ -20,6 +21,7 @@ import ( "github.com/samcharles93/archie-core/internal/config" "github.com/samcharles93/archie-core/internal/daemon" "github.com/samcharles93/archie-core/internal/infrastructure/egress" + "github.com/samcharles93/archie-core/internal/infrastructure/openshell" "github.com/samcharles93/archie-core/internal/secret" ) @@ -107,8 +109,10 @@ func (m *openShellModelEgress) openProvider(ctx context.Context, token, org, nam // accepted it is still deleted. m.own(token, gateway) if _, err := m.client.Providers().Create(ctx, m.workspace, &v1.Provider{ - Name: gateway, - Type: profile, + Name: gateway, + Type: profile, + Labels: map[string]string{openshell.InstanceLabel: m.instance}, + Spec: v1.ProviderSpec{Credentials: map[string]string{keyEnv: key}}, }); err != nil { return daemon.ModelProvider{}, fmt.Errorf("provider %q: create gateway provider: %w", name, err) @@ -219,6 +223,26 @@ func (m *openShellModelEgress) Close(ctx context.Context, token string) { } } +// Sweep deletes the gateway providers a previous daemon of this instance left +// holding keys. It runs at boot, before any task opens a session. +func (m *openShellModelEgress) Sweep(ctx context.Context) error { + all, err := m.client.Providers().ListAll(ctx, m.workspace) + if err != nil { + return fmt.Errorf("list gateway providers: %w", err) + } + var errs []error + for _, p := range all { + if p.Labels[openshell.InstanceLabel] != m.instance { + continue + } + m.log.InfoContext(ctx, "removing orphaned gateway provider", "provider", p.Name) + if err := m.deleteProvider(ctx, p.Name); err != nil { + errs = append(errs, fmt.Errorf("remove gateway provider %s: %w", p.Name, err)) + } + } + return errors.Join(errs...) +} + // deleteProvider retries while the gateway still has the provider attached to // a sandbox it is deleting. func (m *openShellModelEgress) deleteProvider(ctx context.Context, name string) error { diff --git a/internal/app/archied/openshell_model_egress_test.go b/internal/app/archied/openshell_model_egress_test.go index 4ac4a31bb..036074f70 100644 --- a/internal/app/archied/openshell_model_egress_test.go +++ b/internal/app/archied/openshell_model_egress_test.go @@ -12,6 +12,7 @@ import ( "github.com/ai-openshell/sdk-go/openshell/v1/types" "github.com/samcharles93/archie-core/internal/config" + "github.com/samcharles93/archie-core/internal/infrastructure/openshell" "github.com/samcharles93/archie-core/internal/secret" ) @@ -146,3 +147,35 @@ func TestOpenShellModelEgressHoldsKeysOnTheGateway(t *testing.T) { }) } } + +// A daemon that died mid-task leaves its providers, and their keys, on the +// gateway. The boot sweep removes this instance's and keeps other instances'. +// Guards keys outliving a crash, and one instance deleting another's providers. +func TestOpenShellModelEgressSweepsOnlyItsOwnLeftovers(t *testing.T) { + fc := fake.NewClient() + t.Cleanup(func() { _ = fc.Close() }) + gw := fakeGateway{stubProviders{ProviderInterface: fc.Providers(), profiles: stubProfiles{known: map[string]bool{}, items: &[]v1.ProviderProfile{}}}} + log := slog.New(slog.DiscardHandler) + providers := map[string]config.Provider{"claude": {Class: "anthropic", APIKeyEnv: "CLAUDE_KEY"}} + + other := newOpenShellModelEgress(gw, "ws", "other", secret.NewRegistry(), fixedBindings("fake-key"), log) + if _, err := other.Open(t.Context(), "run-other", "org-test", providers, nil); err != nil { + t.Fatal(err) + } + crashed := newOpenShellModelEgress(gw, "ws", "mine", secret.NewRegistry(), fixedBindings("fake-key"), log) + if _, err := crashed.Open(t.Context(), "run-mine", "org-test", providers, nil); err != nil { + t.Fatal(err) + } + + restarted := newOpenShellModelEgress(gw, "ws", "mine", secret.NewRegistry(), fixedBindings("fake-key"), log) + if err := restarted.Sweep(t.Context()); err != nil { + t.Fatal(err) + } + left, err := fc.Providers().ListAll(t.Context(), "ws") + if err != nil { + t.Fatal(err) + } + if len(left) != 1 || left[0].Labels[openshell.InstanceLabel] != "other" { + t.Fatalf("providers after sweep = %+v, want only the other instance's", left) + } +} diff --git a/internal/infrastructure/openshell/runtime.go b/internal/infrastructure/openshell/runtime.go index 082211d1a..4ba6ca3e6 100644 --- a/internal/infrastructure/openshell/runtime.go +++ b/internal/infrastructure/openshell/runtime.go @@ -23,10 +23,10 @@ import ( "github.com/samcharles93/archie-core/internal/storage" ) -// instanceLabel marks the sandboxes one archie instance owns; its value is -// the instance name, so instances sharing a gateway never remove each -// other's. -const instanceLabel = "archie.instance" +// InstanceLabel marks the sandboxes and gateway providers one archie instance +// owns; its value is the instance name, so instances sharing a gateway never +// remove each other's. +const InstanceLabel = "archie.instance" // agentBinary is where the agent image installs archie-agent. The sandbox's // main process names it, since OpenShell replaces the image's entrypoint, @@ -200,7 +200,7 @@ func (r *Runtime) Sweep(ctx context.Context) error { } var errs []error for _, sb := range all { - if sb.Spec.Template == nil || sb.Spec.Template.Labels[instanceLabel] != r.instance { + if sb.Spec.Template == nil || sb.Spec.Template.Labels[InstanceLabel] != r.instance { continue } r.log.InfoContext(ctx, "removing orphaned sandbox", "sandbox", sb.Name) @@ -277,7 +277,7 @@ func (r *Runtime) spec(l Launch) (*v1.SandboxSpec, error) { return &v1.SandboxSpec{ Template: &v1.SandboxTemplate{ Image: l.Image, - Labels: map[string]string{instanceLabel: r.instance}, + Labels: map[string]string{InstanceLabel: r.instance}, DriverConfig: map[string]any{"docker": map[string]any{"mounts": driverMounts}}, }, Environment: environment, diff --git a/internal/infrastructure/openshell/runtime_test.go b/internal/infrastructure/openshell/runtime_test.go index c353d38df..d4689f2ff 100644 --- a/internal/infrastructure/openshell/runtime_test.go +++ b/internal/infrastructure/openshell/runtime_test.go @@ -81,8 +81,8 @@ func TestSweepRemovesOnlyThisInstancesSandboxes(t *testing.T) { client := fake.NewClient() defer func() { _ = client.Close() }() for name, labels := range map[string]map[string]string{ - "archie-agent-1": {instanceLabel: "archie"}, - "staging-agent-1": {instanceLabel: "staging"}, + "archie-agent-1": {InstanceLabel: "archie"}, + "staging-agent-1": {InstanceLabel: "staging"}, "someone-else": nil, } { client.AddSandbox("default", &v1.Sandbox{Name: name, Spec: v1.SandboxSpec{Template: &v1.SandboxTemplate{Labels: labels}}})