Files
archie/internal/infrastructure/openshell/runtime.go
T
sam 2096ae8267 feat(openshell): each sandbox's network policy admits only what its task needs (#1864, #1967)
Runtimes now take a task launch: image, mounts, env, the daemon's NATS
and State Store with the task's credentials, its model providers and the
hosts its tools may reach. Docker addresses them through the egress relay
as before. An OpenShell sandbox reaches NATS and the State Store over
plain TCP, the model providers with their credential bound, and the
task's remote MCP hosts and the allowed package registries; services
addressed by IP or loopback are named by the configured services host,
since sandbox policy admits only DNS names.
2026-11-14 03:24:37 +11:00

426 lines
14 KiB
Go

// Package openshell runs task agents as NVIDIA OpenShell sandboxes. A sandbox
// has no network of its own: every connection it makes passes the gateway's
// policy, and credentials reach it only as placeholders.
package openshell
import (
"context"
"errors"
"fmt"
"log/slog"
"net"
"slices"
"strconv"
"strings"
"sync"
"time"
v1 "github.com/ai-openshell/sdk-go/openshell/v1"
"github.com/ai-openshell/sdk-go/openshell/v1/types"
"github.com/samcharles93/archie-core/internal/config"
"github.com/samcharles93/archie-core/internal/infrastructure/egress"
"github.com/samcharles93/archie-core/internal/storage"
)
// instanceLabel marks the sandboxes one archie instance owns; its value is
// the instance name, so instances sharing a gateway never remove each
// other's.
const instanceLabel = "archie.instance"
// agentBinary is where the agent image installs archie-agent. The sandbox's
// main process names it, since OpenShell replaces the image's entrypoint,
// and policy admits archie's own services and model providers to it alone.
const agentBinary = "/usr/local/bin/archie-agent"
// anyBinary admits every program in the sandbox to an endpoint.
const anyBinary = "/**"
// writable are container paths tools write to inside the sandbox. They are
// the container's own layer, so writing them exposes nothing on the host.
var writable = []string{"/tmp", "/var/tmp", "/root", "/home"}
// readOnly are the system paths the agent and its tools read.
var readOnly = []string{"/usr", "/lib", "/lib64", "/bin", "/sbin", "/etc", "/opt"}
// Options configure a Runtime.
type Options struct {
// Workspace is the gateway workspace; empty means "default".
Workspace string
// Instance names this archie instance. It prefixes sandbox names and is
// the instance label's value.
Instance string
// MaxUptime returns the current lifetime cap; zero or a nil func means
// none. It is read per acquire, so a stored change reaches the next task.
MaxUptime func() time.Duration
// Poll is how often a running sandbox is checked for an exit; zero means
// two seconds.
Poll time.Duration
// ServicesHost is the DNS name a sandbox reaches NATS and the State Store
// by when they are addressed by IP or loopback.
ServicesHost string
Log *slog.Logger
}
// Launch is one task's agent: what it runs, and everything it may reach.
type Launch struct {
Image string
Mounts []storage.Mount
// Env is the agent's environment apart from its network endpoints.
Env []string
// Worker is NATS and the State Store as the daemon reaches them, with the
// task's credentials.
Worker egress.Endpoints
// Providers are the gateway providers whose keys the agent's model calls
// carry, each to its own host.
Providers []Provider
// Hosts are further hosts, "host" or "host:port", any tool may reach.
Hosts []string
}
// Provider is a credentialed gateway provider and the host its key goes to.
type Provider struct {
Gateway string
Host string
Port int
}
// Runtime creates, watches and removes task sandboxes on one gateway.
type Runtime struct {
client v1.ClientInterface
workspace string
instance string
maxUptime func() time.Duration
poll time.Duration
servicesHost string
log *slog.Logger
}
// New returns a Runtime on client.
func New(client v1.ClientInterface, opts Options) *Runtime {
r := &Runtime{
client: client,
workspace: opts.Workspace,
instance: opts.Instance,
maxUptime: opts.MaxUptime,
poll: opts.Poll,
servicesHost: opts.ServicesHost,
log: opts.Log,
}
if r.workspace == "" {
r.workspace = "default"
}
if r.poll <= 0 {
r.poll = 2 * time.Second
}
if r.maxUptime == nil {
r.maxUptime = func() time.Duration { return 0 }
}
if r.log == nil {
r.log = slog.New(slog.DiscardHandler)
}
return r
}
// Sandbox is one task's running sandbox.
type Sandbox struct {
name string
exited chan struct{}
// detail is written before exited closes, so a reader that saw the close
// may read it without a lock.
detail string
once sync.Once
stop context.CancelFunc
done chan struct{}
}
// Exited closes when the agent stops running for any reason: it exited, the
// sandbox failed or was removed, or it reached its maximum uptime.
func (s *Sandbox) Exited() <-chan struct{} { return s.exited }
// ExitDetail says how the sandbox ended. It is empty until Exited closes.
func (s *Sandbox) ExitDetail() string {
select {
case <-s.exited:
return s.detail
default:
return ""
}
}
// Acquire creates a sandbox running the agent l describes, and waits until it
// is ready.
func (r *Runtime) Acquire(ctx context.Context, l Launch) (*Sandbox, error) {
name := fmt.Sprintf("%s-agent-%d", r.instance, time.Now().UnixNano())
spec, err := r.spec(l)
if err != nil {
return nil, err
}
if _, err := r.client.Sandboxes().Create(ctx, r.workspace, name, spec); err != nil {
return nil, fmt.Errorf("create sandbox %s: %w", name, err)
}
if _, err := r.client.Sandboxes().WaitReady(ctx, r.workspace, name); err != nil {
r.remove(ctx, name)
return nil, fmt.Errorf("sandbox %s not ready: %w", name, err)
}
watchCtx, stop := context.WithCancel(context.WithoutCancel(ctx))
s := &Sandbox{name: name, exited: make(chan struct{}), stop: stop, done: make(chan struct{})}
go r.watch(watchCtx, s, r.maxUptime())
r.log.InfoContext(ctx, "sandbox started", "sandbox", name)
return s, nil
}
// Release stops watching s and removes its sandbox.
func (r *Runtime) Release(ctx context.Context, s *Sandbox) {
if s == nil {
return
}
s.stop()
<-s.done
r.remove(ctx, s.name)
r.log.InfoContext(ctx, "sandbox released", "sandbox", s.name)
}
// Reown is not available: the daemon cannot run a privileged container on
// the gateway to chown files back.
func (r *Runtime) Reown(context.Context, string, int, int) error {
return errors.New("the openshell runtime cannot repair file ownership; the sandbox user must match the daemon's")
}
// Sweep removes the sandboxes a previous daemon of this instance left behind.
// Work interrupted by a restart is requeued, so none of them is still needed.
func (r *Runtime) Sweep(ctx context.Context) error {
pager, err := r.client.Sandboxes().List(r.workspace, v1.ListOptions{})
if err != nil {
return fmt.Errorf("list sandboxes: %w", err)
}
all, err := pager.All(ctx)
if err != nil {
return fmt.Errorf("list sandboxes: %w", err)
}
var errs []error
for _, sb := range all {
if sb.Spec.Template == nil || sb.Spec.Template.Labels[instanceLabel] != r.instance {
continue
}
r.log.InfoContext(ctx, "removing orphaned sandbox", "sandbox", sb.Name)
if _, err := r.client.Sandboxes().Delete(ctx, r.workspace, sb.Name); err != nil && !v1.IsNotFound(err) {
errs = append(errs, fmt.Errorf("remove sandbox %s: %w", sb.Name, err))
}
}
return errors.Join(errs...)
}
// spec is the sandbox a task runs in: its environment, mounts and the network
// policy that is all it can reach.
func (r *Runtime) spec(l Launch) (*v1.SandboxSpec, error) {
nats, err := r.service(l.Worker.NATS)
if err != nil {
return nil, fmt.Errorf("nats: %w", err)
}
rules := map[string]v1.NetworkPolicyRule{"archie-nats": serviceRule("archie-nats", nats)}
var stateStore serviceAddr
if l.Worker.StateStore != "" {
if stateStore, err = r.service(l.Worker.StateStore); err != nil {
return nil, fmt.Errorf("state store: %w", err)
}
rules["archie-state-store"] = serviceRule("archie-state-store", stateStore)
}
for _, p := range l.Providers {
name := "model-" + p.Gateway
rules[name] = v1.NetworkPolicyRule{
Name: name,
Endpoints: []v1.PolicyNetworkEndpoint{{
Host: p.Host, Port: uint32(p.Port), Protocol: "rest", Access: v1.NetworkAccessPresetFull, //nolint:gosec // G115: a parsed port is 1 to 65535.
CredentialBinding: &types.NetworkCredentialBinding{Provider: p.Gateway},
}},
Binaries: []v1.PolicyNetworkBinary{{Path: agentBinary}},
}
}
if len(l.Hosts) > 0 {
var endpoints []v1.PolicyNetworkEndpoint
for _, entry := range l.Hosts {
host, port, err := config.EgressHost(entry)
if err != nil {
return nil, err
}
endpoints = append(endpoints, v1.PolicyNetworkEndpoint{Host: host, Port: uint32(port)}) //nolint:gosec // G115: a parsed port is 1 to 65535.
}
rules["egress"] = v1.NetworkPolicyRule{Name: "egress", Endpoints: endpoints, Binaries: []v1.PolicyNetworkBinary{{Path: anyBinary}}}
}
env := append(slices.Clone(l.Env), egress.DirectWorkerEnv(nats.url, stateStore.url, l.Worker)...)
environment := make(map[string]string, len(env))
for _, entry := range env {
key, value, ok := strings.Cut(entry, "=")
if !ok || key == "" {
return nil, fmt.Errorf("environment entry %q is not KEY=value", entry)
}
environment[key] = value
}
readWrite := slices.Clone(writable)
var driverMounts []any
for _, m := range l.Mounts {
mount := map[string]any{"type": m.Type, "source": m.Source, "target": m.Destination, "read_only": m.ReadOnly}
if m.Type == storage.MountTypeBind {
// The same relabelling the Docker pool applies; Docker ignores
// it on a host without SELinux.
mount["selinux_label"] = "private"
if m.Shared {
mount["selinux_label"] = "shared"
}
}
driverMounts = append(driverMounts, mount)
if !m.ReadOnly {
readWrite = append(readWrite, m.Destination)
}
}
return &v1.SandboxSpec{
Template: &v1.SandboxTemplate{
Image: l.Image,
Labels: map[string]string{instanceLabel: r.instance},
DriverConfig: map[string]any{"docker": map[string]any{"mounts": driverMounts}},
},
Environment: environment,
Providers: providerNames(l.Providers),
Command: []string{agentBinary},
Policy: &v1.SandboxPolicy{
Filesystem: &v1.FilesystemPolicy{IncludeWorkdir: true, ReadOnly: slices.Clone(readOnly), ReadWrite: readWrite},
NetworkPolicies: rules,
},
}, nil
}
func providerNames(providers []Provider) []string {
var names []string
for _, p := range providers {
names = append(names, p.Gateway)
}
return names
}
// serviceAddr is one of archie's own services as a sandbox reaches it: the
// URL the agent dials, and the host and port policy admits.
type serviceAddr struct {
url string
host string
port int
}
// service is target as a sandbox must address it. Policy admits TCP only to
// DNS names, and never to loopback, so a host given as an IP or as loopback
// is replaced by the configured services host on the same port. target is a
// URL ("nats://host:port"), a gRPC target ("dns:///host:port") or host:port.
func (r *Runtime) service(target string) (serviceAddr, error) {
prefix, hostport := "", strings.TrimSpace(target)
if i := strings.Index(hostport, "://"); i >= 0 {
prefix, hostport = hostport[:i+3], hostport[i+3:]
hostport = strings.TrimPrefix(hostport, "/")
}
if prefix == "dns://" {
prefix = ""
}
host, portText, err := net.SplitHostPort(hostport)
if err != nil {
return serviceAddr{}, fmt.Errorf("%q: want host:port", target)
}
port, err := strconv.Atoi(portText)
if err != nil || port < 1 || port > 65535 {
return serviceAddr{}, fmt.Errorf("%q: bad port", target)
}
if ip := net.ParseIP(host); ip != nil || strings.EqualFold(host, "localhost") {
if r.servicesHost == "" {
return serviceAddr{}, fmt.Errorf("%q is addressed by IP or loopback, which sandbox policy cannot admit; set the openshell services host", target)
}
host = r.servicesHost
}
hostport = net.JoinHostPort(host, strconv.Itoa(port))
return serviceAddr{url: prefix + hostport, host: host, port: port}, nil
}
// serviceRule admits the agent to one of archie's services over plain TCP.
// The sandbox leaves the stream alone: NATS speaks first, and the State Store
// client does its own TLS.
func serviceRule(name string, s serviceAddr) v1.NetworkPolicyRule {
return v1.NetworkPolicyRule{
Name: name,
Endpoints: []v1.PolicyNetworkEndpoint{{Host: s.host, Port: uint32(s.port), Protocol: "tcp", TLS: v1.NetworkTLSModeSkip}}, //nolint:gosec // G115: a parsed port is 1 to 65535.
Binaries: []v1.PolicyNetworkBinary{{Path: agentBinary}},
}
}
// watch closes s.exited once the sandbox stops running or outlives maxUptime.
// A failed status read says nothing about the sandbox, so it never ends the
// watch: a gateway hiccup must not abort a live run.
func (r *Runtime) watch(ctx context.Context, s *Sandbox, maxUptime time.Duration) {
defer close(s.done)
var deadline <-chan time.Time
if maxUptime > 0 {
timer := time.NewTimer(maxUptime)
defer timer.Stop()
deadline = timer.C
}
ticker := time.NewTicker(r.poll)
defer ticker.Stop()
for {
select {
case <-ctx.Done():
return
case <-deadline:
r.log.WarnContext(ctx, "sandbox reached its maximum uptime", "sandbox", s.name, "max_uptime", maxUptime)
r.remove(ctx, s.name)
r.finish(s, fmt.Sprintf("killed at its maximum uptime of %s", maxUptime))
return
case <-ticker.C:
}
sb, err := r.client.Sandboxes().Get(ctx, r.workspace, s.name)
if v1.IsNotFound(err) {
r.finish(s, "sandbox was removed")
return
}
if err != nil {
continue
}
if detail, ended := exitDetail(sb); ended {
r.finish(s, detail)
return
}
}
}
// finish records how s ended and closes Exited, once.
func (r *Runtime) finish(s *Sandbox, detail string) {
s.once.Do(func() {
s.detail = detail
close(s.exited)
})
}
// exitDetail reports whether sb has stopped running and, if so, how.
func exitDetail(sb *v1.Sandbox) (string, bool) {
st := sb.Status
if st.ExitCode == nil && st.Phase != v1.SandboxError && st.Phase != v1.SandboxCompleted && st.Phase != v1.SandboxStopped {
return "", false
}
detail := fmt.Sprintf("sandbox %s", st.Phase)
if st.ExitCode != nil {
detail = fmt.Sprintf("exit code %d", *st.ExitCode)
}
for _, c := range st.Conditions {
if c.Type == "Ready" && c.Message != "" {
detail += ": " + c.Message
break
}
}
return detail, true
}
// remove deletes a sandbox with a fresh deadline, so a cancelled task still
// cleans up. A sandbox already gone is not an error.
func (r *Runtime) remove(ctx context.Context, name string) {
ctx, cancel := context.WithTimeout(context.WithoutCancel(ctx), 30*time.Second)
defer cancel()
if _, err := r.client.Sandboxes().Delete(ctx, r.workspace, name); err != nil && !v1.IsNotFound(err) {
r.log.WarnContext(ctx, "sandbox remove failed", "sandbox", name, "err", err)
}
}