Files
archie/internal/infrastructure/openshell/runtime.go
T

483 lines
16 KiB
Go

// Package openshell runs task agents as NVIDIA OpenShell sandboxes. A sandbox
// has no network of its own: every connection it makes passes the gateway's
// policy, and credentials reach it only as placeholders.
package openshell
import (
"context"
"crypto/rand"
"encoding/hex"
"errors"
"fmt"
"log/slog"
"net"
"slices"
"strconv"
"strings"
"sync"
"time"
v1 "github.com/ai-openshell/sdk-go/openshell/v1"
"github.com/ai-openshell/sdk-go/openshell/v1/types"
"github.com/samcharles93/archie-core/internal/config"
"github.com/samcharles93/archie-core/internal/infrastructure/egress"
"github.com/samcharles93/archie-core/internal/storage"
)
// InstanceLabel marks the sandboxes and gateway providers one archie instance
// owns; its value is the instance name, so instances sharing a gateway never
// remove each other's.
const InstanceLabel = "archie.instance"
// agentBinary is where the agent image installs archie-agent. The sandbox's
// main process names it, since OpenShell replaces the image's entrypoint,
// and policy admits archie's own services and model providers to it alone.
const agentBinary = "/usr/local/bin/archie-agent"
// anyBinary admits every program in the sandbox to an endpoint.
const anyBinary = "/**"
// writable are container paths tools write to inside the sandbox. They are
// the container's own layer, so writing them exposes nothing on the host.
var writable = []string{"/tmp", "/var/tmp", "/root", "/home", "/dev/null"}
// readOnly are the system paths the agent and its tools read. git and the
// shell need /dev/null and /proc.
var readOnly = []string{"/usr", "/lib", "/lib64", "/bin", "/sbin", "/etc", "/opt", "/proc", "/dev/urandom"}
// Options configure a Runtime.
type Options struct {
// Workspace is the gateway workspace; empty means "default".
Workspace string
// Instance names this archie instance. It prefixes sandbox names and is
// the instance label's value.
Instance string
// MaxUptime returns the current lifetime cap; zero or a nil func means
// none. It is read per acquire, so a stored change reaches the next task.
MaxUptime func() time.Duration
// Poll is how often a running sandbox is checked for an exit; zero means
// two seconds.
Poll time.Duration
// ServicesHost is the DNS name a sandbox reaches NATS and the State Store
// by when they are addressed by IP or loopback.
ServicesHost string
Log *slog.Logger
}
// Launch is one task's agent: what it runs, and everything it may reach.
type Launch struct {
Image string
Mounts []storage.Mount
// Env is the agent's environment apart from its network endpoints.
Env []string
// Worker is NATS and the State Store as the daemon reaches them, with the
// task's credentials.
Worker egress.Endpoints
// Providers are the gateway providers whose keys the agent's model calls
// carry, each to its own host.
Providers []Provider
// Hosts are further hosts, "host" or "host:port", any tool may reach.
Hosts []string
}
// Provider is a credentialed gateway provider and the host its key goes to.
type Provider struct {
Gateway string
Host string
Port int
}
// Runtime creates, watches and removes task sandboxes on one gateway.
type Runtime struct {
client v1.ClientInterface
workspace string
instance string
maxUptime func() time.Duration
poll time.Duration
servicesHost string
log *slog.Logger
}
// New returns a Runtime on client.
func New(client v1.ClientInterface, opts Options) *Runtime {
r := &Runtime{
client: client,
workspace: opts.Workspace,
instance: opts.Instance,
maxUptime: opts.MaxUptime,
poll: opts.Poll,
servicesHost: opts.ServicesHost,
log: opts.Log,
}
if r.workspace == "" {
r.workspace = "default"
}
if r.poll <= 0 {
r.poll = 2 * time.Second
}
if r.maxUptime == nil {
r.maxUptime = func() time.Duration { return 0 }
}
if r.log == nil {
r.log = slog.New(slog.DiscardHandler)
}
return r
}
// Sandbox is one task's running sandbox.
type Sandbox struct {
name string
exited chan struct{}
// detail is written before exited closes, so a reader that saw the close
// may read it without a lock.
detail string
once sync.Once
stop context.CancelFunc
done chan struct{}
}
// Exited closes when the agent stops running for any reason: it exited, the
// sandbox failed or was removed, or it reached its maximum uptime.
func (s *Sandbox) Exited() <-chan struct{} { return s.exited }
// ExitDetail says how the sandbox ended. It is empty until Exited closes.
func (s *Sandbox) ExitDetail() string {
select {
case <-s.exited:
return s.detail
default:
return ""
}
}
// Acquire creates a sandbox running the agent l describes, and waits until it
// is ready.
func (r *Runtime) Acquire(ctx context.Context, l Launch) (*Sandbox, error) {
name, err := sandboxName()
if err != nil {
return nil, err
}
spec, err := r.spec(l)
if err != nil {
return nil, err
}
if _, err := r.client.Sandboxes().Create(ctx, r.workspace, name, spec); err != nil {
return nil, fmt.Errorf("create sandbox %s: %w", name, err)
}
if _, err := r.client.Sandboxes().WaitReady(ctx, r.workspace, name); err != nil {
// The sandbox's own log is the only record of why it failed, and
// removing the sandbox discards it.
tail := r.logTail(ctx, name)
r.remove(ctx, name)
return nil, fmt.Errorf("sandbox %s not ready: %w%s", name, err, tail)
}
watchCtx, stop := context.WithCancel(context.WithoutCancel(ctx))
s := &Sandbox{name: name, exited: make(chan struct{}), stop: stop, done: make(chan struct{})}
go r.watch(watchCtx, s, r.maxUptime())
r.log.InfoContext(ctx, "sandbox started", "sandbox", name)
return s, nil
}
// logTail returns the sandbox's last log lines, prefixed for appending to an
// error, or "" when there are none or they cannot be read.
func (r *Runtime) logTail(ctx context.Context, name string) string {
res, err := r.client.Sandboxes().GetLogs(ctx, r.workspace, name, v1.WithLogLines(15))
if err != nil {
r.log.WarnContext(ctx, "read failed sandbox log", "sandbox", name, "err", err)
return ""
}
var b strings.Builder
for _, l := range res.Lines {
b.WriteString("\n")
b.WriteString(l.Source)
b.WriteString(" ")
b.WriteString(l.Level)
b.WriteString(" ")
b.WriteString(l.Message)
}
if b.Len() == 0 {
return ""
}
return "\nsandbox log:" + b.String()
}
// sandboxName is a short random name: the gateway rejects names much longer
// than 19 characters, and the instance is carried by the sandbox's label.
func sandboxName() (string, error) {
var b [6]byte
if _, err := rand.Read(b[:]); err != nil {
return "", fmt.Errorf("name sandbox: %w", err)
}
return "archie-" + hex.EncodeToString(b[:]), nil
}
// Release stops watching s and removes its sandbox.
func (r *Runtime) Release(ctx context.Context, s *Sandbox) {
if s == nil {
return
}
s.stop()
<-s.done
r.remove(ctx, s.name)
r.log.InfoContext(ctx, "sandbox released", "sandbox", s.name)
}
// Reown is not available: the daemon cannot run a privileged container on
// the gateway to chown files back.
func (r *Runtime) Reown(context.Context, string, int, int) error {
return errors.New("the openshell runtime cannot repair file ownership; the sandbox user must match the daemon's")
}
// Sweep removes the sandboxes a previous daemon of this instance left behind.
// Work interrupted by a restart is requeued, so none of them is still needed.
func (r *Runtime) Sweep(ctx context.Context) error {
pager, err := r.client.Sandboxes().List(r.workspace, v1.ListOptions{})
if err != nil {
return fmt.Errorf("list sandboxes: %w", err)
}
all, err := pager.All(ctx)
if err != nil {
return fmt.Errorf("list sandboxes: %w", err)
}
var errs []error
for _, sb := range all {
if sb.Spec.Template == nil || sb.Spec.Template.Labels[InstanceLabel] != r.instance {
continue
}
r.log.InfoContext(ctx, "removing orphaned sandbox", "sandbox", sb.Name)
if _, err := r.client.Sandboxes().Delete(ctx, r.workspace, sb.Name); err != nil && !v1.IsNotFound(err) {
errs = append(errs, fmt.Errorf("remove sandbox %s: %w", sb.Name, err))
}
}
return errors.Join(errs...)
}
// spec is the sandbox a task runs in: its environment, mounts and the network
// policy that is all it can reach.
func (r *Runtime) spec(l Launch) (*v1.SandboxSpec, error) {
nats, err := r.service(l.Worker.NATS)
if err != nil {
return nil, fmt.Errorf("nats: %w", err)
}
rules := map[string]v1.NetworkPolicyRule{"archie-nats": serviceRule("archie-nats", nats)}
var stateStore serviceAddr
if l.Worker.StateStore != "" {
if stateStore, err = r.service(l.Worker.StateStore); err != nil {
return nil, fmt.Errorf("state store: %w", err)
}
rules["archie-state-store"] = serviceRule("archie-state-store", stateStore)
}
for _, p := range l.Providers {
name, rule := providerRule(p)
rules[name] = rule
}
if len(l.Hosts) > 0 {
var endpoints []v1.PolicyNetworkEndpoint
for _, entry := range l.Hosts {
host, port, err := config.EgressHost(entry)
if err != nil {
return nil, err
}
endpoints = append(endpoints, v1.PolicyNetworkEndpoint{Host: host, Port: uint32(port)}) //nolint:gosec // G115: a parsed port is 1 to 65535.
}
rules["egress"] = v1.NetworkPolicyRule{Name: "egress", Endpoints: endpoints, Binaries: []v1.PolicyNetworkBinary{{Path: anyBinary}}}
}
env := append(slices.Clone(l.Env), egress.DirectWorkerEnv(nats.url, stateStore.url, l.Worker)...)
environment := make(map[string]string, len(env))
for _, entry := range env {
key, value, ok := strings.Cut(entry, "=")
if !ok || key == "" {
return nil, fmt.Errorf("environment entry %q is not KEY=value", entry)
}
environment[key] = value
}
readWrite := slices.Clone(writable)
var driverMounts []any
for _, m := range l.Mounts {
mount := map[string]any{"type": m.Type, "source": m.Source, "target": m.Destination, "read_only": m.ReadOnly}
if m.Type == storage.MountTypeBind {
// The same relabelling the Docker pool applies; Docker ignores
// it on a host without SELinux.
mount["selinux_label"] = "private"
if m.Shared {
mount["selinux_label"] = "shared"
}
}
driverMounts = append(driverMounts, mount)
if !m.ReadOnly {
readWrite = append(readWrite, m.Destination)
}
}
return &v1.SandboxSpec{
Template: &v1.SandboxTemplate{
Image: l.Image,
Labels: map[string]string{InstanceLabel: r.instance},
DriverConfig: map[string]any{"docker": map[string]any{"mounts": driverMounts}},
},
Environment: environment,
Providers: providerNames(l.Providers),
Command: []string{agentBinary},
Policy: &v1.SandboxPolicy{
Filesystem: &v1.FilesystemPolicy{IncludeWorkdir: true, ReadOnly: slices.Clone(readOnly), ReadWrite: readWrite},
NetworkPolicies: rules,
},
}, nil
}
// providerRule is the egress rule for one model provider: a credentialed one
// is named by its gateway, a keyless one by the host and port it reaches.
func providerRule(p Provider) (string, v1.NetworkPolicyRule) {
name := "model-" + p.Gateway
endpoint := v1.PolicyNetworkEndpoint{
Host: p.Host, Port: uint32(p.Port), Protocol: "rest", Access: v1.NetworkAccessPresetFull, //nolint:gosec // G115: a parsed port is 1 to 65535.
}
if p.Gateway != "" {
endpoint.CredentialBinding = &types.NetworkCredentialBinding{Provider: p.Gateway}
} else {
name = fmt.Sprintf("model-%s-%d", p.Host, p.Port)
}
return name, v1.NetworkPolicyRule{
Name: name,
Endpoints: []v1.PolicyNetworkEndpoint{endpoint},
Binaries: []v1.PolicyNetworkBinary{{Path: agentBinary}},
}
}
func providerNames(providers []Provider) []string {
var names []string
for _, p := range providers {
if p.Gateway == "" {
continue
}
names = append(names, p.Gateway)
}
return names
}
// serviceAddr is one of archie's own services as a sandbox reaches it: the
// URL the agent dials, and the host and port policy admits.
type serviceAddr struct {
url string
host string
port int
}
// service is target as a sandbox must address it. Policy admits TCP only to
// DNS names, and never to loopback, so a host given as an IP or as loopback
// is replaced by the configured services host on the same port. target is a
// URL ("nats://host:port"), a gRPC target ("dns:///host:port") or host:port.
func (r *Runtime) service(target string) (serviceAddr, error) {
prefix, hostport := "", strings.TrimSpace(target)
if i := strings.Index(hostport, "://"); i >= 0 {
prefix, hostport = hostport[:i+3], hostport[i+3:]
hostport = strings.TrimPrefix(hostport, "/")
}
if prefix == "dns://" {
prefix = ""
}
host, portText, err := net.SplitHostPort(hostport)
if err != nil {
return serviceAddr{}, fmt.Errorf("%q: want host:port", target)
}
port, err := strconv.Atoi(portText)
if err != nil || port < 1 || port > 65535 {
return serviceAddr{}, fmt.Errorf("%q: bad port", target)
}
if ip := net.ParseIP(host); ip != nil || strings.EqualFold(host, "localhost") {
if r.servicesHost == "" {
return serviceAddr{}, fmt.Errorf("%q is addressed by IP or loopback, which sandbox policy cannot admit; set the openshell services host", target)
}
host = r.servicesHost
}
hostport = net.JoinHostPort(host, strconv.Itoa(port))
return serviceAddr{url: prefix + hostport, host: host, port: port}, nil
}
// serviceRule admits the agent to one of archie's services over plain TCP.
// The sandbox leaves the stream alone: NATS speaks first, and the State Store
// client does its own TLS.
func serviceRule(name string, s serviceAddr) v1.NetworkPolicyRule {
return v1.NetworkPolicyRule{
Name: name,
Endpoints: []v1.PolicyNetworkEndpoint{{Host: s.host, Port: uint32(s.port), Protocol: "tcp", TLS: v1.NetworkTLSModeSkip}}, //nolint:gosec // G115: a parsed port is 1 to 65535.
Binaries: []v1.PolicyNetworkBinary{{Path: agentBinary}},
}
}
// watch closes s.exited once the sandbox stops running or outlives maxUptime.
// A failed status read says nothing about the sandbox, so it never ends the
// watch: a gateway hiccup must not abort a live run.
func (r *Runtime) watch(ctx context.Context, s *Sandbox, maxUptime time.Duration) {
defer close(s.done)
var deadline <-chan time.Time
if maxUptime > 0 {
timer := time.NewTimer(maxUptime)
defer timer.Stop()
deadline = timer.C
}
ticker := time.NewTicker(r.poll)
defer ticker.Stop()
for {
select {
case <-ctx.Done():
return
case <-deadline:
r.log.WarnContext(ctx, "sandbox reached its maximum uptime", "sandbox", s.name, "max_uptime", maxUptime)
r.remove(ctx, s.name)
r.finish(s, fmt.Sprintf("killed at its maximum uptime of %s", maxUptime))
return
case <-ticker.C:
}
sb, err := r.client.Sandboxes().Get(ctx, r.workspace, s.name)
if v1.IsNotFound(err) {
r.finish(s, "sandbox was removed")
return
}
if err != nil {
continue
}
if detail, ended := exitDetail(sb); ended {
r.finish(s, detail)
return
}
}
}
// finish records how s ended and closes Exited, once.
func (r *Runtime) finish(s *Sandbox, detail string) {
s.once.Do(func() {
s.detail = detail
close(s.exited)
})
}
// exitDetail reports whether sb has stopped running and, if so, how.
func exitDetail(sb *v1.Sandbox) (string, bool) {
st := sb.Status
if st.ExitCode == nil && st.Phase != v1.SandboxError && st.Phase != v1.SandboxCompleted && st.Phase != v1.SandboxStopped {
return "", false
}
detail := fmt.Sprintf("sandbox %s", st.Phase)
if st.ExitCode != nil {
detail = fmt.Sprintf("exit code %d", *st.ExitCode)
}
for _, c := range st.Conditions {
if c.Type == "Ready" && c.Message != "" {
detail += ": " + c.Message
break
}
}
return detail, true
}
// remove deletes a sandbox with a fresh deadline, so a cancelled task still
// cleans up. A sandbox already gone is not an error.
func (r *Runtime) remove(ctx context.Context, name string) {
ctx, cancel := context.WithTimeout(context.WithoutCancel(ctx), 30*time.Second)
defer cancel()
if _, err := r.client.Sandboxes().Delete(ctx, r.workspace, name); err != nil && !v1.IsNotFound(err) {
r.log.WarnContext(ctx, "sandbox remove failed", "sandbox", name, "err", err)
}
}