// Package openshell runs task agents as NVIDIA OpenShell sandboxes. A sandbox // has no network of its own: every connection it makes passes the gateway's // policy, and credentials reach it only as placeholders. package openshell import ( "context" "crypto/rand" "encoding/hex" "errors" "fmt" "log/slog" "net" "slices" "strconv" "strings" "sync" "time" v1 "github.com/ai-openshell/sdk-go/openshell/v1" "github.com/ai-openshell/sdk-go/openshell/v1/types" "github.com/samcharles93/archie-core/internal/config" "github.com/samcharles93/archie-core/internal/infrastructure/egress" "github.com/samcharles93/archie-core/internal/storage" ) // InstanceLabel marks the sandboxes and gateway providers one archie instance // owns; its value is the instance name, so instances sharing a gateway never // remove each other's. const InstanceLabel = "archie.instance" // agentBinary is where the agent image installs archie-agent. The sandbox's // main process names it, since OpenShell replaces the image's entrypoint, // and policy admits archie's own services and model providers to it alone. const agentBinary = "/usr/local/bin/archie-agent" // anyBinary admits every program in the sandbox to an endpoint. const anyBinary = "/**" // writable are container paths tools write to inside the sandbox. They are // the container's own layer, so writing them exposes nothing on the host. var writable = []string{"/tmp", "/var/tmp", "/root", "/home", "/dev/null"} // readOnly are the system paths the agent and its tools read. git and the // shell need /dev/null and /proc. var readOnly = []string{"/usr", "/lib", "/lib64", "/bin", "/sbin", "/etc", "/opt", "/proc", "/dev/urandom"} // Options configure a Runtime. type Options struct { // Workspace is the gateway workspace; empty means "default". Workspace string // Instance names this archie instance. It prefixes sandbox names and is // the instance label's value. Instance string // MaxUptime returns the current lifetime cap; zero or a nil func means // none. It is read per acquire, so a stored change reaches the next task. MaxUptime func() time.Duration // Poll is how often a running sandbox is checked for an exit; zero means // two seconds. Poll time.Duration // ServicesHost is the DNS name a sandbox reaches NATS and the State Store // by when they are addressed by IP or loopback. ServicesHost string Log *slog.Logger } // Launch is one task's agent: what it runs, and everything it may reach. type Launch struct { Image string Mounts []storage.Mount // Env is the agent's environment apart from its network endpoints. Env []string // Worker is NATS and the State Store as the daemon reaches them, with the // task's credentials. Worker egress.Endpoints // Providers are the gateway providers whose keys the agent's model calls // carry, each to its own host. Providers []Provider // Hosts are further hosts, "host" or "host:port", any tool may reach. Hosts []string } // Provider is a credentialed gateway provider and the host its key goes to. type Provider struct { Gateway string Host string Port int } // Runtime creates, watches and removes task sandboxes on one gateway. type Runtime struct { client v1.ClientInterface workspace string instance string maxUptime func() time.Duration poll time.Duration servicesHost string log *slog.Logger } // New returns a Runtime on client. func New(client v1.ClientInterface, opts Options) *Runtime { r := &Runtime{ client: client, workspace: opts.Workspace, instance: opts.Instance, maxUptime: opts.MaxUptime, poll: opts.Poll, servicesHost: opts.ServicesHost, log: opts.Log, } if r.workspace == "" { r.workspace = "default" } if r.poll <= 0 { r.poll = 2 * time.Second } if r.maxUptime == nil { r.maxUptime = func() time.Duration { return 0 } } if r.log == nil { r.log = slog.New(slog.DiscardHandler) } return r } // Sandbox is one task's running sandbox. type Sandbox struct { name string exited chan struct{} // detail is written before exited closes, so a reader that saw the close // may read it without a lock. detail string once sync.Once stop context.CancelFunc done chan struct{} } // Exited closes when the agent stops running for any reason: it exited, the // sandbox failed or was removed, or it reached its maximum uptime. func (s *Sandbox) Exited() <-chan struct{} { return s.exited } // ExitDetail says how the sandbox ended. It is empty until Exited closes. func (s *Sandbox) ExitDetail() string { select { case <-s.exited: return s.detail default: return "" } } // Acquire creates a sandbox running the agent l describes, and waits until it // is ready. func (r *Runtime) Acquire(ctx context.Context, l Launch) (*Sandbox, error) { name, err := sandboxName() if err != nil { return nil, err } spec, err := r.spec(l) if err != nil { return nil, err } if _, err := r.client.Sandboxes().Create(ctx, r.workspace, name, spec); err != nil { return nil, fmt.Errorf("create sandbox %s: %w", name, err) } if _, err := r.client.Sandboxes().WaitReady(ctx, r.workspace, name); err != nil { // The sandbox's own log is the only record of why it failed, and // removing the sandbox discards it. tail := r.logTail(ctx, name) r.remove(ctx, name) return nil, fmt.Errorf("sandbox %s not ready: %w%s", name, err, tail) } watchCtx, stop := context.WithCancel(context.WithoutCancel(ctx)) s := &Sandbox{name: name, exited: make(chan struct{}), stop: stop, done: make(chan struct{})} go r.watch(watchCtx, s, r.maxUptime()) r.log.InfoContext(ctx, "sandbox started", "sandbox", name) return s, nil } // logTail returns the sandbox's last log lines, prefixed for appending to an // error, or "" when there are none or they cannot be read. func (r *Runtime) logTail(ctx context.Context, name string) string { res, err := r.client.Sandboxes().GetLogs(ctx, r.workspace, name, v1.WithLogLines(15)) if err != nil { r.log.WarnContext(ctx, "read failed sandbox log", "sandbox", name, "err", err) return "" } var b strings.Builder for _, l := range res.Lines { b.WriteString("\n") b.WriteString(l.Source) b.WriteString(" ") b.WriteString(l.Level) b.WriteString(" ") b.WriteString(l.Message) } if b.Len() == 0 { return "" } return "\nsandbox log:" + b.String() } // sandboxName is a short random name: the gateway rejects names much longer // than 19 characters, and the instance is carried by the sandbox's label. func sandboxName() (string, error) { var b [6]byte if _, err := rand.Read(b[:]); err != nil { return "", fmt.Errorf("name sandbox: %w", err) } return "archie-" + hex.EncodeToString(b[:]), nil } // Release stops watching s and removes its sandbox. func (r *Runtime) Release(ctx context.Context, s *Sandbox) { if s == nil { return } s.stop() <-s.done r.remove(ctx, s.name) r.log.InfoContext(ctx, "sandbox released", "sandbox", s.name) } // Reown is not available: the daemon cannot run a privileged container on // the gateway to chown files back. func (r *Runtime) Reown(context.Context, string, int, int) error { return errors.New("the openshell runtime cannot repair file ownership; the sandbox user must match the daemon's") } // Sweep removes the sandboxes a previous daemon of this instance left behind. // Work interrupted by a restart is requeued, so none of them is still needed. func (r *Runtime) Sweep(ctx context.Context) error { pager, err := r.client.Sandboxes().List(r.workspace, v1.ListOptions{}) if err != nil { return fmt.Errorf("list sandboxes: %w", err) } all, err := pager.All(ctx) if err != nil { return fmt.Errorf("list sandboxes: %w", err) } var errs []error for _, sb := range all { if sb.Spec.Template == nil || sb.Spec.Template.Labels[InstanceLabel] != r.instance { continue } r.log.InfoContext(ctx, "removing orphaned sandbox", "sandbox", sb.Name) if _, err := r.client.Sandboxes().Delete(ctx, r.workspace, sb.Name); err != nil && !v1.IsNotFound(err) { errs = append(errs, fmt.Errorf("remove sandbox %s: %w", sb.Name, err)) } } return errors.Join(errs...) } // spec is the sandbox a task runs in: its environment, mounts and the network // policy that is all it can reach. func (r *Runtime) spec(l Launch) (*v1.SandboxSpec, error) { nats, err := r.service(l.Worker.NATS) if err != nil { return nil, fmt.Errorf("nats: %w", err) } rules := map[string]v1.NetworkPolicyRule{"archie-nats": serviceRule("archie-nats", nats)} var stateStore serviceAddr if l.Worker.StateStore != "" { if stateStore, err = r.service(l.Worker.StateStore); err != nil { return nil, fmt.Errorf("state store: %w", err) } rules["archie-state-store"] = serviceRule("archie-state-store", stateStore) } for _, p := range l.Providers { name, rule := providerRule(p) rules[name] = rule } if len(l.Hosts) > 0 { var endpoints []v1.PolicyNetworkEndpoint for _, entry := range l.Hosts { host, port, err := config.EgressHost(entry) if err != nil { return nil, err } // Package registries name scoped packages with an encoded slash // (@scope%2Fname), which the gateway's HTTP filter refuses unless // the endpoint allows it, and it allows it only on a REST endpoint. endpoints = append(endpoints, v1.PolicyNetworkEndpoint{ Host: host, Port: uint32(port), //nolint:gosec // G115: a parsed port is 1 to 65535. Protocol: "rest", Access: v1.NetworkAccessPresetFull, AllowEncodedSlash: true, }) } rules["egress"] = v1.NetworkPolicyRule{Name: "egress", Endpoints: endpoints, Binaries: []v1.PolicyNetworkBinary{{Path: anyBinary}}} } env := append(slices.Clone(l.Env), egress.DirectWorkerEnv(nats.url, stateStore.url, l.Worker)...) environment := make(map[string]string, len(env)) for _, entry := range env { key, value, ok := strings.Cut(entry, "=") if !ok || key == "" { return nil, fmt.Errorf("environment entry %q is not KEY=value", entry) } environment[key] = value } readWrite := slices.Clone(writable) var driverMounts []any for _, m := range l.Mounts { mount := map[string]any{"type": m.Type, "source": m.Source, "target": m.Destination, "read_only": m.ReadOnly} if m.Type == storage.MountTypeBind { // The same relabelling the Docker pool applies; Docker ignores // it on a host without SELinux. mount["selinux_label"] = "private" if m.Shared { mount["selinux_label"] = "shared" } } driverMounts = append(driverMounts, mount) if !m.ReadOnly { readWrite = append(readWrite, m.Destination) } } return &v1.SandboxSpec{ Template: &v1.SandboxTemplate{ Image: l.Image, Labels: map[string]string{InstanceLabel: r.instance}, DriverConfig: map[string]any{"docker": map[string]any{"mounts": driverMounts}}, }, Environment: environment, Providers: providerNames(l.Providers), Command: []string{agentBinary}, Policy: &v1.SandboxPolicy{ Filesystem: &v1.FilesystemPolicy{IncludeWorkdir: true, ReadOnly: slices.Clone(readOnly), ReadWrite: readWrite}, NetworkPolicies: rules, }, }, nil } // providerRule is the egress rule for one model provider: a credentialed one // is named by its gateway, a keyless one by the host and port it reaches. func providerRule(p Provider) (string, v1.NetworkPolicyRule) { name := "model-" + p.Gateway endpoint := v1.PolicyNetworkEndpoint{ Host: p.Host, Port: uint32(p.Port), Protocol: "rest", Access: v1.NetworkAccessPresetFull, //nolint:gosec // G115: a parsed port is 1 to 65535. } if p.Gateway != "" { endpoint.CredentialBinding = &types.NetworkCredentialBinding{Provider: p.Gateway} } else { name = fmt.Sprintf("model-%s-%d", p.Host, p.Port) } return name, v1.NetworkPolicyRule{ Name: name, Endpoints: []v1.PolicyNetworkEndpoint{endpoint}, Binaries: []v1.PolicyNetworkBinary{{Path: agentBinary}}, } } func providerNames(providers []Provider) []string { var names []string for _, p := range providers { if p.Gateway == "" { continue } names = append(names, p.Gateway) } return names } // serviceAddr is one of archie's own services as a sandbox reaches it: the // URL the agent dials, and the host and port policy admits. type serviceAddr struct { url string host string port int } // service is target as a sandbox must address it. Policy admits TCP only to // DNS names, and never to loopback, so a host given as an IP or as loopback // is replaced by the configured services host on the same port. target is a // URL ("nats://host:port"), a gRPC target ("dns:///host:port") or host:port. func (r *Runtime) service(target string) (serviceAddr, error) { prefix, hostport := "", strings.TrimSpace(target) if i := strings.Index(hostport, "://"); i >= 0 { prefix, hostport = hostport[:i+3], hostport[i+3:] hostport = strings.TrimPrefix(hostport, "/") } if prefix == "dns://" { prefix = "" } host, portText, err := net.SplitHostPort(hostport) if err != nil { return serviceAddr{}, fmt.Errorf("%q: want host:port", target) } port, err := strconv.Atoi(portText) if err != nil || port < 1 || port > 65535 { return serviceAddr{}, fmt.Errorf("%q: bad port", target) } if ip := net.ParseIP(host); ip != nil || strings.EqualFold(host, "localhost") { if r.servicesHost == "" { return serviceAddr{}, fmt.Errorf("%q is addressed by IP or loopback, which sandbox policy cannot admit; set the openshell services host", target) } host = r.servicesHost } hostport = net.JoinHostPort(host, strconv.Itoa(port)) return serviceAddr{url: prefix + hostport, host: host, port: port}, nil } // serviceRule admits the agent to one of archie's services over plain TCP. // The sandbox leaves the stream alone: NATS speaks first, and the State Store // client does its own TLS. func serviceRule(name string, s serviceAddr) v1.NetworkPolicyRule { return v1.NetworkPolicyRule{ Name: name, Endpoints: []v1.PolicyNetworkEndpoint{{Host: s.host, Port: uint32(s.port), Protocol: "tcp", TLS: v1.NetworkTLSModeSkip}}, //nolint:gosec // G115: a parsed port is 1 to 65535. Binaries: []v1.PolicyNetworkBinary{{Path: agentBinary}}, } } // watch closes s.exited once the sandbox stops running or outlives maxUptime. // A failed status read says nothing about the sandbox, so it never ends the // watch: a gateway hiccup must not abort a live run. func (r *Runtime) watch(ctx context.Context, s *Sandbox, maxUptime time.Duration) { defer close(s.done) var deadline <-chan time.Time if maxUptime > 0 { timer := time.NewTimer(maxUptime) defer timer.Stop() deadline = timer.C } ticker := time.NewTicker(r.poll) defer ticker.Stop() for { select { case <-ctx.Done(): return case <-deadline: r.log.WarnContext(ctx, "sandbox reached its maximum uptime", "sandbox", s.name, "max_uptime", maxUptime) r.remove(ctx, s.name) r.finish(s, fmt.Sprintf("killed at its maximum uptime of %s", maxUptime)) return case <-ticker.C: } sb, err := r.client.Sandboxes().Get(ctx, r.workspace, s.name) if v1.IsNotFound(err) { r.finish(s, "sandbox was removed") return } if err != nil { continue } if detail, ended := exitDetail(sb); ended { r.finish(s, detail) return } } } // finish records how s ended and closes Exited, once. func (r *Runtime) finish(s *Sandbox, detail string) { s.once.Do(func() { s.detail = detail close(s.exited) }) } // exitDetail reports whether sb has stopped running and, if so, how. func exitDetail(sb *v1.Sandbox) (string, bool) { st := sb.Status if st.ExitCode == nil && st.Phase != v1.SandboxError && st.Phase != v1.SandboxCompleted && st.Phase != v1.SandboxStopped { return "", false } detail := fmt.Sprintf("sandbox %s", st.Phase) if st.ExitCode != nil { detail = fmt.Sprintf("exit code %d", *st.ExitCode) } for _, c := range st.Conditions { if c.Type == "Ready" && c.Message != "" { detail += ": " + c.Message break } } return detail, true } // remove deletes a sandbox with a fresh deadline, so a cancelled task still // cleans up. A sandbox already gone is not an error. func (r *Runtime) remove(ctx context.Context, name string) { ctx, cancel := context.WithTimeout(context.WithoutCancel(ctx), 30*time.Second) defer cancel() if _, err := r.client.Sandboxes().Delete(ctx, r.workspace, name); err != nil && !v1.IsNotFound(err) { r.log.WarnContext(ctx, "sandbox remove failed", "sandbox", name, "err", err) } }