fix(archied): label OpenShell gateway providers and sweep leftovers at boot (#1970)

This commit is contained in:
sam committed 2026-11-14 04:04:15 +11:00
1 parent 92fbcd7a28
commit 7d664d7efb
5 files changed
+74 -11

No files matched your search

+7 -1
View File
@@ -511,7 +511,13 @@ func (b *boot) setupContainers(ctx context.Context) func() {
b.taskRuntime = daemon.OpenShellRuntime(rt)
settings := b.cfg.Containers.OpenShell
bindings := runCredentialResolver{runs: b.stateStoreGrants.Client, models: b.taskModels, config: b.cfgHolder, secrets: b.secrets, credentials: b.credentialWatch}
b.modelEgress = newOpenShellModelEgress(client, settings.Workspace, b.cfg.Instance, b.secrets, bindings, b.log)
egress := newOpenShellModelEgress(client, settings.Workspace, b.cfg.Instance, b.secrets, bindings, b.log)
b.modelEgress = egress
sweepCtx, cancel := context.WithTimeout(ctx, 15*time.Second)
defer cancel()
if err := egress.Sweep(sweepCtx); err != nil {
b.log.WarnContext(ctx, "openshell: orphaned gateway providers remain", "err", err)
}
return func() { closeOpenShell(); closeDocker() }
}
+26 -2
View File
@@ -4,6 +4,7 @@ import (
"context"
"crypto/rand"
"encoding/hex"
"errors"
"fmt"
"log/slog"
"maps"
@@ -20,6 +21,7 @@ import (
"github.com/samcharles93/archie-core/internal/config"
"github.com/samcharles93/archie-core/internal/daemon"
"github.com/samcharles93/archie-core/internal/infrastructure/egress"
"github.com/samcharles93/archie-core/internal/infrastructure/openshell"
"github.com/samcharles93/archie-core/internal/secret"
)
@@ -107,8 +109,10 @@ func (m *openShellModelEgress) openProvider(ctx context.Context, token, org, nam
// accepted it is still deleted.
m.own(token, gateway)
if _, err := m.client.Providers().Create(ctx, m.workspace, &v1.Provider{
Name: gateway,
Type: profile,
Name: gateway,
Type: profile,
Labels: map[string]string{openshell.InstanceLabel: m.instance},
Spec: v1.ProviderSpec{Credentials: map[string]string{keyEnv: key}},
}); err != nil {
return daemon.ModelProvider{}, fmt.Errorf("provider %q: create gateway provider: %w", name, err)
@@ -219,6 +223,26 @@ func (m *openShellModelEgress) Close(ctx context.Context, token string) {
}
}
// Sweep deletes the gateway providers a previous daemon of this instance left
// holding keys. It runs at boot, before any task opens a session.
func (m *openShellModelEgress) Sweep(ctx context.Context) error {
all, err := m.client.Providers().ListAll(ctx, m.workspace)
if err != nil {
return fmt.Errorf("list gateway providers: %w", err)
}
var errs []error
for _, p := range all {
if p.Labels[openshell.InstanceLabel] != m.instance {
continue
}
m.log.InfoContext(ctx, "removing orphaned gateway provider", "provider", p.Name)
if err := m.deleteProvider(ctx, p.Name); err != nil {
errs = append(errs, fmt.Errorf("remove gateway provider %s: %w", p.Name, err))
}
}
return errors.Join(errs...)
}
// deleteProvider retries while the gateway still has the provider attached to
// a sandbox it is deleting.
func (m *openShellModelEgress) deleteProvider(ctx context.Context, name string) error {
@@ -12,6 +12,7 @@ import (
"github.com/ai-openshell/sdk-go/openshell/v1/types"
"github.com/samcharles93/archie-core/internal/config"
"github.com/samcharles93/archie-core/internal/infrastructure/openshell"
"github.com/samcharles93/archie-core/internal/secret"
)
@@ -146,3 +147,35 @@ func TestOpenShellModelEgressHoldsKeysOnTheGateway(t *testing.T) {
})
}
}
// A daemon that died mid-task leaves its providers, and their keys, on the
// gateway. The boot sweep removes this instance's and keeps other instances'.
// Guards keys outliving a crash, and one instance deleting another's providers.
func TestOpenShellModelEgressSweepsOnlyItsOwnLeftovers(t *testing.T) {
fc := fake.NewClient()
t.Cleanup(func() { _ = fc.Close() })
gw := fakeGateway{stubProviders{ProviderInterface: fc.Providers(), profiles: stubProfiles{known: map[string]bool{}, items: &[]v1.ProviderProfile{}}}}
log := slog.New(slog.DiscardHandler)
providers := map[string]config.Provider{"claude": {Class: "anthropic", APIKeyEnv: "CLAUDE_KEY"}}
other := newOpenShellModelEgress(gw, "ws", "other", secret.NewRegistry(), fixedBindings("fake-key"), log)
if _, err := other.Open(t.Context(), "run-other", "org-test", providers, nil); err != nil {
t.Fatal(err)
}
crashed := newOpenShellModelEgress(gw, "ws", "mine", secret.NewRegistry(), fixedBindings("fake-key"), log)
if _, err := crashed.Open(t.Context(), "run-mine", "org-test", providers, nil); err != nil {
t.Fatal(err)
}
restarted := newOpenShellModelEgress(gw, "ws", "mine", secret.NewRegistry(), fixedBindings("fake-key"), log)
if err := restarted.Sweep(t.Context()); err != nil {
t.Fatal(err)
}
left, err := fc.Providers().ListAll(t.Context(), "ws")
if err != nil {
t.Fatal(err)
}
if len(left) != 1 || left[0].Labels[openshell.InstanceLabel] != "other" {
t.Fatalf("providers after sweep = %+v, want only the other instance's", left)
}
}
+6 -6
View File
@@ -23,10 +23,10 @@ import (
"github.com/samcharles93/archie-core/internal/storage"
)
// instanceLabel marks the sandboxes one archie instance owns; its value is
// the instance name, so instances sharing a gateway never remove each
// other's.
const instanceLabel = "archie.instance"
// InstanceLabel marks the sandboxes and gateway providers one archie instance
// owns; its value is the instance name, so instances sharing a gateway never
// remove each other's.
const InstanceLabel = "archie.instance"
// agentBinary is where the agent image installs archie-agent. The sandbox's
// main process names it, since OpenShell replaces the image's entrypoint,
@@ -200,7 +200,7 @@ func (r *Runtime) Sweep(ctx context.Context) error {
}
var errs []error
for _, sb := range all {
if sb.Spec.Template == nil || sb.Spec.Template.Labels[instanceLabel] != r.instance {
if sb.Spec.Template == nil || sb.Spec.Template.Labels[InstanceLabel] != r.instance {
continue
}
r.log.InfoContext(ctx, "removing orphaned sandbox", "sandbox", sb.Name)
@@ -277,7 +277,7 @@ func (r *Runtime) spec(l Launch) (*v1.SandboxSpec, error) {
return &v1.SandboxSpec{
Template: &v1.SandboxTemplate{
Image: l.Image,
Labels: map[string]string{instanceLabel: r.instance},
Labels: map[string]string{InstanceLabel: r.instance},
DriverConfig: map[string]any{"docker": map[string]any{"mounts": driverMounts}},
},
Environment: environment,
@@ -81,8 +81,8 @@ func TestSweepRemovesOnlyThisInstancesSandboxes(t *testing.T) {
client := fake.NewClient()
defer func() { _ = client.Close() }()
for name, labels := range map[string]map[string]string{
"archie-agent-1": {instanceLabel: "archie"},
"staging-agent-1": {instanceLabel: "staging"},
"archie-agent-1": {InstanceLabel: "archie"},
"staging-agent-1": {InstanceLabel: "staging"},
"someone-else": nil,
} {
client.AddSandbox("default", &v1.Sandbox{Name: name, Spec: v1.SandboxSpec{Template: &v1.SandboxTemplate{Labels: labels}}})