441 lines
14 KiB
Go
441 lines
14 KiB
Go
// Package openshell runs task agents as NVIDIA OpenShell sandboxes. A sandbox
|
|
// has no network of its own: every connection it makes passes the gateway's
|
|
// policy, and credentials reach it only as placeholders.
|
|
package openshell
|
|
|
|
import (
|
|
"context"
|
|
"crypto/rand"
|
|
"encoding/hex"
|
|
"errors"
|
|
"fmt"
|
|
"log/slog"
|
|
"net"
|
|
"slices"
|
|
"strconv"
|
|
"strings"
|
|
"sync"
|
|
"time"
|
|
|
|
v1 "github.com/ai-openshell/sdk-go/openshell/v1"
|
|
"github.com/ai-openshell/sdk-go/openshell/v1/types"
|
|
|
|
"github.com/samcharles93/archie-core/internal/config"
|
|
"github.com/samcharles93/archie-core/internal/infrastructure/egress"
|
|
"github.com/samcharles93/archie-core/internal/storage"
|
|
)
|
|
|
|
// InstanceLabel marks the sandboxes and gateway providers one archie instance
|
|
// owns; its value is the instance name, so instances sharing a gateway never
|
|
// remove each other's.
|
|
const InstanceLabel = "archie.instance"
|
|
|
|
// agentBinary is where the agent image installs archie-agent. The sandbox's
|
|
// main process names it, since OpenShell replaces the image's entrypoint,
|
|
// and policy admits archie's own services and model providers to it alone.
|
|
const agentBinary = "/usr/local/bin/archie-agent"
|
|
|
|
// anyBinary admits every program in the sandbox to an endpoint.
|
|
const anyBinary = "/**"
|
|
|
|
// writable are container paths tools write to inside the sandbox. They are
|
|
// the container's own layer, so writing them exposes nothing on the host.
|
|
var writable = []string{"/tmp", "/var/tmp", "/root", "/home"}
|
|
|
|
// readOnly are the system paths the agent and its tools read.
|
|
var readOnly = []string{"/usr", "/lib", "/lib64", "/bin", "/sbin", "/etc", "/opt"}
|
|
|
|
// Options configure a Runtime.
|
|
type Options struct {
|
|
// Workspace is the gateway workspace; empty means "default".
|
|
Workspace string
|
|
// Instance names this archie instance. It prefixes sandbox names and is
|
|
// the instance label's value.
|
|
Instance string
|
|
// MaxUptime returns the current lifetime cap; zero or a nil func means
|
|
// none. It is read per acquire, so a stored change reaches the next task.
|
|
MaxUptime func() time.Duration
|
|
// Poll is how often a running sandbox is checked for an exit; zero means
|
|
// two seconds.
|
|
Poll time.Duration
|
|
// ServicesHost is the DNS name a sandbox reaches NATS and the State Store
|
|
// by when they are addressed by IP or loopback.
|
|
ServicesHost string
|
|
Log *slog.Logger
|
|
}
|
|
|
|
// Launch is one task's agent: what it runs, and everything it may reach.
|
|
type Launch struct {
|
|
Image string
|
|
Mounts []storage.Mount
|
|
// Env is the agent's environment apart from its network endpoints.
|
|
Env []string
|
|
// Worker is NATS and the State Store as the daemon reaches them, with the
|
|
// task's credentials.
|
|
Worker egress.Endpoints
|
|
// Providers are the gateway providers whose keys the agent's model calls
|
|
// carry, each to its own host.
|
|
Providers []Provider
|
|
// Hosts are further hosts, "host" or "host:port", any tool may reach.
|
|
Hosts []string
|
|
}
|
|
|
|
// Provider is a credentialed gateway provider and the host its key goes to.
|
|
type Provider struct {
|
|
Gateway string
|
|
Host string
|
|
Port int
|
|
}
|
|
|
|
// Runtime creates, watches and removes task sandboxes on one gateway.
|
|
type Runtime struct {
|
|
client v1.ClientInterface
|
|
workspace string
|
|
instance string
|
|
maxUptime func() time.Duration
|
|
poll time.Duration
|
|
servicesHost string
|
|
log *slog.Logger
|
|
}
|
|
|
|
// New returns a Runtime on client.
|
|
func New(client v1.ClientInterface, opts Options) *Runtime {
|
|
r := &Runtime{
|
|
client: client,
|
|
workspace: opts.Workspace,
|
|
instance: opts.Instance,
|
|
maxUptime: opts.MaxUptime,
|
|
poll: opts.Poll,
|
|
servicesHost: opts.ServicesHost,
|
|
log: opts.Log,
|
|
}
|
|
if r.workspace == "" {
|
|
r.workspace = "default"
|
|
}
|
|
if r.poll <= 0 {
|
|
r.poll = 2 * time.Second
|
|
}
|
|
if r.maxUptime == nil {
|
|
r.maxUptime = func() time.Duration { return 0 }
|
|
}
|
|
if r.log == nil {
|
|
r.log = slog.New(slog.DiscardHandler)
|
|
}
|
|
return r
|
|
}
|
|
|
|
// Sandbox is one task's running sandbox.
|
|
type Sandbox struct {
|
|
name string
|
|
exited chan struct{}
|
|
// detail is written before exited closes, so a reader that saw the close
|
|
// may read it without a lock.
|
|
detail string
|
|
once sync.Once
|
|
stop context.CancelFunc
|
|
done chan struct{}
|
|
}
|
|
|
|
// Exited closes when the agent stops running for any reason: it exited, the
|
|
// sandbox failed or was removed, or it reached its maximum uptime.
|
|
func (s *Sandbox) Exited() <-chan struct{} { return s.exited }
|
|
|
|
// ExitDetail says how the sandbox ended. It is empty until Exited closes.
|
|
func (s *Sandbox) ExitDetail() string {
|
|
select {
|
|
case <-s.exited:
|
|
return s.detail
|
|
default:
|
|
return ""
|
|
}
|
|
}
|
|
|
|
// Acquire creates a sandbox running the agent l describes, and waits until it
|
|
// is ready.
|
|
func (r *Runtime) Acquire(ctx context.Context, l Launch) (*Sandbox, error) {
|
|
name, err := sandboxName()
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
spec, err := r.spec(l)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
if _, err := r.client.Sandboxes().Create(ctx, r.workspace, name, spec); err != nil {
|
|
return nil, fmt.Errorf("create sandbox %s: %w", name, err)
|
|
}
|
|
if _, err := r.client.Sandboxes().WaitReady(ctx, r.workspace, name); err != nil {
|
|
r.remove(ctx, name)
|
|
return nil, fmt.Errorf("sandbox %s not ready: %w", name, err)
|
|
}
|
|
watchCtx, stop := context.WithCancel(context.WithoutCancel(ctx))
|
|
s := &Sandbox{name: name, exited: make(chan struct{}), stop: stop, done: make(chan struct{})}
|
|
go r.watch(watchCtx, s, r.maxUptime())
|
|
r.log.InfoContext(ctx, "sandbox started", "sandbox", name)
|
|
return s, nil
|
|
}
|
|
|
|
// sandboxName is a short random name: the gateway rejects names much longer
|
|
// than 19 characters, and the instance is carried by the sandbox's label.
|
|
func sandboxName() (string, error) {
|
|
var b [6]byte
|
|
if _, err := rand.Read(b[:]); err != nil {
|
|
return "", fmt.Errorf("name sandbox: %w", err)
|
|
}
|
|
return "archie-" + hex.EncodeToString(b[:]), nil
|
|
}
|
|
|
|
// Release stops watching s and removes its sandbox.
|
|
func (r *Runtime) Release(ctx context.Context, s *Sandbox) {
|
|
if s == nil {
|
|
return
|
|
}
|
|
s.stop()
|
|
<-s.done
|
|
r.remove(ctx, s.name)
|
|
r.log.InfoContext(ctx, "sandbox released", "sandbox", s.name)
|
|
}
|
|
|
|
// Reown is not available: the daemon cannot run a privileged container on
|
|
// the gateway to chown files back.
|
|
func (r *Runtime) Reown(context.Context, string, int, int) error {
|
|
return errors.New("the openshell runtime cannot repair file ownership; the sandbox user must match the daemon's")
|
|
}
|
|
|
|
// Sweep removes the sandboxes a previous daemon of this instance left behind.
|
|
// Work interrupted by a restart is requeued, so none of them is still needed.
|
|
func (r *Runtime) Sweep(ctx context.Context) error {
|
|
pager, err := r.client.Sandboxes().List(r.workspace, v1.ListOptions{})
|
|
if err != nil {
|
|
return fmt.Errorf("list sandboxes: %w", err)
|
|
}
|
|
all, err := pager.All(ctx)
|
|
if err != nil {
|
|
return fmt.Errorf("list sandboxes: %w", err)
|
|
}
|
|
var errs []error
|
|
for _, sb := range all {
|
|
if sb.Spec.Template == nil || sb.Spec.Template.Labels[InstanceLabel] != r.instance {
|
|
continue
|
|
}
|
|
r.log.InfoContext(ctx, "removing orphaned sandbox", "sandbox", sb.Name)
|
|
if _, err := r.client.Sandboxes().Delete(ctx, r.workspace, sb.Name); err != nil && !v1.IsNotFound(err) {
|
|
errs = append(errs, fmt.Errorf("remove sandbox %s: %w", sb.Name, err))
|
|
}
|
|
}
|
|
return errors.Join(errs...)
|
|
}
|
|
|
|
// spec is the sandbox a task runs in: its environment, mounts and the network
|
|
// policy that is all it can reach.
|
|
func (r *Runtime) spec(l Launch) (*v1.SandboxSpec, error) {
|
|
nats, err := r.service(l.Worker.NATS)
|
|
if err != nil {
|
|
return nil, fmt.Errorf("nats: %w", err)
|
|
}
|
|
rules := map[string]v1.NetworkPolicyRule{"archie-nats": serviceRule("archie-nats", nats)}
|
|
var stateStore serviceAddr
|
|
if l.Worker.StateStore != "" {
|
|
if stateStore, err = r.service(l.Worker.StateStore); err != nil {
|
|
return nil, fmt.Errorf("state store: %w", err)
|
|
}
|
|
rules["archie-state-store"] = serviceRule("archie-state-store", stateStore)
|
|
}
|
|
for _, p := range l.Providers {
|
|
name := "model-" + p.Gateway
|
|
rules[name] = v1.NetworkPolicyRule{
|
|
Name: name,
|
|
Endpoints: []v1.PolicyNetworkEndpoint{{
|
|
Host: p.Host, Port: uint32(p.Port), Protocol: "rest", Access: v1.NetworkAccessPresetFull, //nolint:gosec // G115: a parsed port is 1 to 65535.
|
|
CredentialBinding: &types.NetworkCredentialBinding{Provider: p.Gateway},
|
|
}},
|
|
Binaries: []v1.PolicyNetworkBinary{{Path: agentBinary}},
|
|
}
|
|
}
|
|
if len(l.Hosts) > 0 {
|
|
var endpoints []v1.PolicyNetworkEndpoint
|
|
for _, entry := range l.Hosts {
|
|
host, port, err := config.EgressHost(entry)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
endpoints = append(endpoints, v1.PolicyNetworkEndpoint{Host: host, Port: uint32(port)}) //nolint:gosec // G115: a parsed port is 1 to 65535.
|
|
}
|
|
rules["egress"] = v1.NetworkPolicyRule{Name: "egress", Endpoints: endpoints, Binaries: []v1.PolicyNetworkBinary{{Path: anyBinary}}}
|
|
}
|
|
env := append(slices.Clone(l.Env), egress.DirectWorkerEnv(nats.url, stateStore.url, l.Worker)...)
|
|
environment := make(map[string]string, len(env))
|
|
for _, entry := range env {
|
|
key, value, ok := strings.Cut(entry, "=")
|
|
if !ok || key == "" {
|
|
return nil, fmt.Errorf("environment entry %q is not KEY=value", entry)
|
|
}
|
|
environment[key] = value
|
|
}
|
|
readWrite := slices.Clone(writable)
|
|
var driverMounts []any
|
|
for _, m := range l.Mounts {
|
|
mount := map[string]any{"type": m.Type, "source": m.Source, "target": m.Destination, "read_only": m.ReadOnly}
|
|
if m.Type == storage.MountTypeBind {
|
|
// The same relabelling the Docker pool applies; Docker ignores
|
|
// it on a host without SELinux.
|
|
mount["selinux_label"] = "private"
|
|
if m.Shared {
|
|
mount["selinux_label"] = "shared"
|
|
}
|
|
}
|
|
driverMounts = append(driverMounts, mount)
|
|
if !m.ReadOnly {
|
|
readWrite = append(readWrite, m.Destination)
|
|
}
|
|
}
|
|
return &v1.SandboxSpec{
|
|
Template: &v1.SandboxTemplate{
|
|
Image: l.Image,
|
|
Labels: map[string]string{InstanceLabel: r.instance},
|
|
DriverConfig: map[string]any{"docker": map[string]any{"mounts": driverMounts}},
|
|
},
|
|
Environment: environment,
|
|
Providers: providerNames(l.Providers),
|
|
Command: []string{agentBinary},
|
|
Policy: &v1.SandboxPolicy{
|
|
Filesystem: &v1.FilesystemPolicy{IncludeWorkdir: true, ReadOnly: slices.Clone(readOnly), ReadWrite: readWrite},
|
|
NetworkPolicies: rules,
|
|
},
|
|
}, nil
|
|
}
|
|
|
|
func providerNames(providers []Provider) []string {
|
|
var names []string
|
|
for _, p := range providers {
|
|
names = append(names, p.Gateway)
|
|
}
|
|
return names
|
|
}
|
|
|
|
// serviceAddr is one of archie's own services as a sandbox reaches it: the
|
|
// URL the agent dials, and the host and port policy admits.
|
|
type serviceAddr struct {
|
|
url string
|
|
host string
|
|
port int
|
|
}
|
|
|
|
// service is target as a sandbox must address it. Policy admits TCP only to
|
|
// DNS names, and never to loopback, so a host given as an IP or as loopback
|
|
// is replaced by the configured services host on the same port. target is a
|
|
// URL ("nats://host:port"), a gRPC target ("dns:///host:port") or host:port.
|
|
func (r *Runtime) service(target string) (serviceAddr, error) {
|
|
prefix, hostport := "", strings.TrimSpace(target)
|
|
if i := strings.Index(hostport, "://"); i >= 0 {
|
|
prefix, hostport = hostport[:i+3], hostport[i+3:]
|
|
hostport = strings.TrimPrefix(hostport, "/")
|
|
}
|
|
if prefix == "dns://" {
|
|
prefix = ""
|
|
}
|
|
host, portText, err := net.SplitHostPort(hostport)
|
|
if err != nil {
|
|
return serviceAddr{}, fmt.Errorf("%q: want host:port", target)
|
|
}
|
|
port, err := strconv.Atoi(portText)
|
|
if err != nil || port < 1 || port > 65535 {
|
|
return serviceAddr{}, fmt.Errorf("%q: bad port", target)
|
|
}
|
|
if ip := net.ParseIP(host); ip != nil || strings.EqualFold(host, "localhost") {
|
|
if r.servicesHost == "" {
|
|
return serviceAddr{}, fmt.Errorf("%q is addressed by IP or loopback, which sandbox policy cannot admit; set the openshell services host", target)
|
|
}
|
|
host = r.servicesHost
|
|
}
|
|
hostport = net.JoinHostPort(host, strconv.Itoa(port))
|
|
return serviceAddr{url: prefix + hostport, host: host, port: port}, nil
|
|
}
|
|
|
|
// serviceRule admits the agent to one of archie's services over plain TCP.
|
|
// The sandbox leaves the stream alone: NATS speaks first, and the State Store
|
|
// client does its own TLS.
|
|
func serviceRule(name string, s serviceAddr) v1.NetworkPolicyRule {
|
|
return v1.NetworkPolicyRule{
|
|
Name: name,
|
|
Endpoints: []v1.PolicyNetworkEndpoint{{Host: s.host, Port: uint32(s.port), Protocol: "tcp", TLS: v1.NetworkTLSModeSkip}}, //nolint:gosec // G115: a parsed port is 1 to 65535.
|
|
Binaries: []v1.PolicyNetworkBinary{{Path: agentBinary}},
|
|
}
|
|
}
|
|
|
|
// watch closes s.exited once the sandbox stops running or outlives maxUptime.
|
|
// A failed status read says nothing about the sandbox, so it never ends the
|
|
// watch: a gateway hiccup must not abort a live run.
|
|
func (r *Runtime) watch(ctx context.Context, s *Sandbox, maxUptime time.Duration) {
|
|
defer close(s.done)
|
|
var deadline <-chan time.Time
|
|
if maxUptime > 0 {
|
|
timer := time.NewTimer(maxUptime)
|
|
defer timer.Stop()
|
|
deadline = timer.C
|
|
}
|
|
ticker := time.NewTicker(r.poll)
|
|
defer ticker.Stop()
|
|
for {
|
|
select {
|
|
case <-ctx.Done():
|
|
return
|
|
case <-deadline:
|
|
r.log.WarnContext(ctx, "sandbox reached its maximum uptime", "sandbox", s.name, "max_uptime", maxUptime)
|
|
r.remove(ctx, s.name)
|
|
r.finish(s, fmt.Sprintf("killed at its maximum uptime of %s", maxUptime))
|
|
return
|
|
case <-ticker.C:
|
|
}
|
|
sb, err := r.client.Sandboxes().Get(ctx, r.workspace, s.name)
|
|
if v1.IsNotFound(err) {
|
|
r.finish(s, "sandbox was removed")
|
|
return
|
|
}
|
|
if err != nil {
|
|
continue
|
|
}
|
|
if detail, ended := exitDetail(sb); ended {
|
|
r.finish(s, detail)
|
|
return
|
|
}
|
|
}
|
|
}
|
|
|
|
// finish records how s ended and closes Exited, once.
|
|
func (r *Runtime) finish(s *Sandbox, detail string) {
|
|
s.once.Do(func() {
|
|
s.detail = detail
|
|
close(s.exited)
|
|
})
|
|
}
|
|
|
|
// exitDetail reports whether sb has stopped running and, if so, how.
|
|
func exitDetail(sb *v1.Sandbox) (string, bool) {
|
|
st := sb.Status
|
|
if st.ExitCode == nil && st.Phase != v1.SandboxError && st.Phase != v1.SandboxCompleted && st.Phase != v1.SandboxStopped {
|
|
return "", false
|
|
}
|
|
detail := fmt.Sprintf("sandbox %s", st.Phase)
|
|
if st.ExitCode != nil {
|
|
detail = fmt.Sprintf("exit code %d", *st.ExitCode)
|
|
}
|
|
for _, c := range st.Conditions {
|
|
if c.Type == "Ready" && c.Message != "" {
|
|
detail += ": " + c.Message
|
|
break
|
|
}
|
|
}
|
|
return detail, true
|
|
}
|
|
|
|
// remove deletes a sandbox with a fresh deadline, so a cancelled task still
|
|
// cleans up. A sandbox already gone is not an error.
|
|
func (r *Runtime) remove(ctx context.Context, name string) {
|
|
ctx, cancel := context.WithTimeout(context.WithoutCancel(ctx), 30*time.Second)
|
|
defer cancel()
|
|
if _, err := r.client.Sandboxes().Delete(ctx, r.workspace, name); err != nil && !v1.IsNotFound(err) {
|
|
r.log.WarnContext(ctx, "sandbox remove failed", "sandbox", name, "err", err)
|
|
}
|
|
}
|