fix(archied): label OpenShell gateway providers and sweep leftovers at boot (#1970)
This commit is contained in:
5 files changed
+74
-11
No files matched your search
@@ -511,7 +511,13 @@ func (b *boot) setupContainers(ctx context.Context) func() {
|
||||
b.taskRuntime = daemon.OpenShellRuntime(rt)
|
||||
settings := b.cfg.Containers.OpenShell
|
||||
bindings := runCredentialResolver{runs: b.stateStoreGrants.Client, models: b.taskModels, config: b.cfgHolder, secrets: b.secrets, credentials: b.credentialWatch}
|
||||
b.modelEgress = newOpenShellModelEgress(client, settings.Workspace, b.cfg.Instance, b.secrets, bindings, b.log)
|
||||
egress := newOpenShellModelEgress(client, settings.Workspace, b.cfg.Instance, b.secrets, bindings, b.log)
|
||||
b.modelEgress = egress
|
||||
sweepCtx, cancel := context.WithTimeout(ctx, 15*time.Second)
|
||||
defer cancel()
|
||||
if err := egress.Sweep(sweepCtx); err != nil {
|
||||
b.log.WarnContext(ctx, "openshell: orphaned gateway providers remain", "err", err)
|
||||
}
|
||||
return func() { closeOpenShell(); closeDocker() }
|
||||
}
|
||||
|
||||
|
||||
@@ -4,6 +4,7 @@ import (
|
||||
"context"
|
||||
"crypto/rand"
|
||||
"encoding/hex"
|
||||
"errors"
|
||||
"fmt"
|
||||
"log/slog"
|
||||
"maps"
|
||||
@@ -20,6 +21,7 @@ import (
|
||||
"github.com/samcharles93/archie-core/internal/config"
|
||||
"github.com/samcharles93/archie-core/internal/daemon"
|
||||
"github.com/samcharles93/archie-core/internal/infrastructure/egress"
|
||||
"github.com/samcharles93/archie-core/internal/infrastructure/openshell"
|
||||
"github.com/samcharles93/archie-core/internal/secret"
|
||||
)
|
||||
|
||||
@@ -107,8 +109,10 @@ func (m *openShellModelEgress) openProvider(ctx context.Context, token, org, nam
|
||||
// accepted it is still deleted.
|
||||
m.own(token, gateway)
|
||||
if _, err := m.client.Providers().Create(ctx, m.workspace, &v1.Provider{
|
||||
Name: gateway,
|
||||
Type: profile,
|
||||
Name: gateway,
|
||||
Type: profile,
|
||||
Labels: map[string]string{openshell.InstanceLabel: m.instance},
|
||||
|
||||
Spec: v1.ProviderSpec{Credentials: map[string]string{keyEnv: key}},
|
||||
}); err != nil {
|
||||
return daemon.ModelProvider{}, fmt.Errorf("provider %q: create gateway provider: %w", name, err)
|
||||
@@ -219,6 +223,26 @@ func (m *openShellModelEgress) Close(ctx context.Context, token string) {
|
||||
}
|
||||
}
|
||||
|
||||
// Sweep deletes the gateway providers a previous daemon of this instance left
|
||||
// holding keys. It runs at boot, before any task opens a session.
|
||||
func (m *openShellModelEgress) Sweep(ctx context.Context) error {
|
||||
all, err := m.client.Providers().ListAll(ctx, m.workspace)
|
||||
if err != nil {
|
||||
return fmt.Errorf("list gateway providers: %w", err)
|
||||
}
|
||||
var errs []error
|
||||
for _, p := range all {
|
||||
if p.Labels[openshell.InstanceLabel] != m.instance {
|
||||
continue
|
||||
}
|
||||
m.log.InfoContext(ctx, "removing orphaned gateway provider", "provider", p.Name)
|
||||
if err := m.deleteProvider(ctx, p.Name); err != nil {
|
||||
errs = append(errs, fmt.Errorf("remove gateway provider %s: %w", p.Name, err))
|
||||
}
|
||||
}
|
||||
return errors.Join(errs...)
|
||||
}
|
||||
|
||||
// deleteProvider retries while the gateway still has the provider attached to
|
||||
// a sandbox it is deleting.
|
||||
func (m *openShellModelEgress) deleteProvider(ctx context.Context, name string) error {
|
||||
|
||||
@@ -12,6 +12,7 @@ import (
|
||||
"github.com/ai-openshell/sdk-go/openshell/v1/types"
|
||||
|
||||
"github.com/samcharles93/archie-core/internal/config"
|
||||
"github.com/samcharles93/archie-core/internal/infrastructure/openshell"
|
||||
"github.com/samcharles93/archie-core/internal/secret"
|
||||
)
|
||||
|
||||
@@ -146,3 +147,35 @@ func TestOpenShellModelEgressHoldsKeysOnTheGateway(t *testing.T) {
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// A daemon that died mid-task leaves its providers, and their keys, on the
|
||||
// gateway. The boot sweep removes this instance's and keeps other instances'.
|
||||
// Guards keys outliving a crash, and one instance deleting another's providers.
|
||||
func TestOpenShellModelEgressSweepsOnlyItsOwnLeftovers(t *testing.T) {
|
||||
fc := fake.NewClient()
|
||||
t.Cleanup(func() { _ = fc.Close() })
|
||||
gw := fakeGateway{stubProviders{ProviderInterface: fc.Providers(), profiles: stubProfiles{known: map[string]bool{}, items: &[]v1.ProviderProfile{}}}}
|
||||
log := slog.New(slog.DiscardHandler)
|
||||
providers := map[string]config.Provider{"claude": {Class: "anthropic", APIKeyEnv: "CLAUDE_KEY"}}
|
||||
|
||||
other := newOpenShellModelEgress(gw, "ws", "other", secret.NewRegistry(), fixedBindings("fake-key"), log)
|
||||
if _, err := other.Open(t.Context(), "run-other", "org-test", providers, nil); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
crashed := newOpenShellModelEgress(gw, "ws", "mine", secret.NewRegistry(), fixedBindings("fake-key"), log)
|
||||
if _, err := crashed.Open(t.Context(), "run-mine", "org-test", providers, nil); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
restarted := newOpenShellModelEgress(gw, "ws", "mine", secret.NewRegistry(), fixedBindings("fake-key"), log)
|
||||
if err := restarted.Sweep(t.Context()); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
left, err := fc.Providers().ListAll(t.Context(), "ws")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if len(left) != 1 || left[0].Labels[openshell.InstanceLabel] != "other" {
|
||||
t.Fatalf("providers after sweep = %+v, want only the other instance's", left)
|
||||
}
|
||||
}
|
||||
@@ -23,10 +23,10 @@ import (
|
||||
"github.com/samcharles93/archie-core/internal/storage"
|
||||
)
|
||||
|
||||
// instanceLabel marks the sandboxes one archie instance owns; its value is
|
||||
// the instance name, so instances sharing a gateway never remove each
|
||||
// other's.
|
||||
const instanceLabel = "archie.instance"
|
||||
// InstanceLabel marks the sandboxes and gateway providers one archie instance
|
||||
// owns; its value is the instance name, so instances sharing a gateway never
|
||||
// remove each other's.
|
||||
const InstanceLabel = "archie.instance"
|
||||
|
||||
// agentBinary is where the agent image installs archie-agent. The sandbox's
|
||||
// main process names it, since OpenShell replaces the image's entrypoint,
|
||||
@@ -200,7 +200,7 @@ func (r *Runtime) Sweep(ctx context.Context) error {
|
||||
}
|
||||
var errs []error
|
||||
for _, sb := range all {
|
||||
if sb.Spec.Template == nil || sb.Spec.Template.Labels[instanceLabel] != r.instance {
|
||||
if sb.Spec.Template == nil || sb.Spec.Template.Labels[InstanceLabel] != r.instance {
|
||||
continue
|
||||
}
|
||||
r.log.InfoContext(ctx, "removing orphaned sandbox", "sandbox", sb.Name)
|
||||
@@ -277,7 +277,7 @@ func (r *Runtime) spec(l Launch) (*v1.SandboxSpec, error) {
|
||||
return &v1.SandboxSpec{
|
||||
Template: &v1.SandboxTemplate{
|
||||
Image: l.Image,
|
||||
Labels: map[string]string{instanceLabel: r.instance},
|
||||
Labels: map[string]string{InstanceLabel: r.instance},
|
||||
DriverConfig: map[string]any{"docker": map[string]any{"mounts": driverMounts}},
|
||||
},
|
||||
Environment: environment,
|
||||
|
||||
@@ -81,8 +81,8 @@ func TestSweepRemovesOnlyThisInstancesSandboxes(t *testing.T) {
|
||||
client := fake.NewClient()
|
||||
defer func() { _ = client.Close() }()
|
||||
for name, labels := range map[string]map[string]string{
|
||||
"archie-agent-1": {instanceLabel: "archie"},
|
||||
"staging-agent-1": {instanceLabel: "staging"},
|
||||
"archie-agent-1": {InstanceLabel: "archie"},
|
||||
"staging-agent-1": {InstanceLabel: "staging"},
|
||||
"someone-else": nil,
|
||||
} {
|
||||
client.AddSandbox("default", &v1.Sandbox{Name: name, Spec: v1.SandboxSpec{Template: &v1.SandboxTemplate{Labels: labels}}})
|
||||
|
||||
Reference in new issue
Block a user