feat(daemon): run a Kit profile's task in its Kit container on the egress path

This commit is contained in:
sam committed 2026-09-26 10:14:33 +10:00
1 parent 25f2299225
commit 70df8d1131
17 files changed
+734 -42

No files matched your search

+1
View File
@@ -90,6 +90,7 @@ require (
google.golang.org/genproto/googleapis/rpc v0.0.0-20260921155816-b14227669459 // indirect
modernc.org/libc v1.77.1 // indirect
modernc.org/sqlite v1.59.0 // indirect
oras.land/oras-go/v2 v2.6.2 // indirect
)
require (
+2
View File
@@ -325,5 +325,7 @@ modernc.org/memory v1.12.1 h1:nFMiWrpStgZczNl6XI9GnIk/rWhYIyHGUaR04pGbp9g=
modernc.org/memory v1.12.1/go.mod h1:/JP4VbVC+K5sU2wZi9bHoq2MAkCnrt2r98UGeSK7Mjw=
modernc.org/sqlite v1.59.0 h1:X1es1GpqBlS/5T+vbM4HLUdaa8OtQx468DF2vrx+38A=
modernc.org/sqlite v1.59.0/go.mod h1:+paeT2A3iPRHkQDwG7oA6Tk0zQd5woMEI8q7orfry8k=
oras.land/oras-go/v2 v2.6.2 h1:N04RXngAp1LJKTG6ifz3xHPipasEkWr+hFmInja5YKo=
oras.land/oras-go/v2 v2.6.2/go.mod h1:PlTtg4JTDJkDe8yVHpM2wz7/YDc00GVas+i4jAW2TZ4=
pgregory.net/rapid v1.2.0 h1:keKAYRcjm+e1F0oAuU5F5+YPAWcyxNNRK2wud503Gnk=
pgregory.net/rapid v1.2.0/go.mod h1:PY5XlDGj0+V1FCq0o192FdRhpKHGTRIWBgqjDBTrq04=
+3
View File
@@ -22,3 +22,6 @@ func runAsHarnessUser(_ *exec.Cmd, name string) error {
// ownForHarness has nothing to do where the harness runs as the worker.
func ownForHarness(string, ...string) error { return nil }
// ownTreeForHarness has nothing to do where the harness runs as the worker.
func ownTreeForHarness(string, string) error { return nil }
+24
View File
@@ -5,9 +5,11 @@ package agentexec
import (
"errors"
"fmt"
"io/fs"
"os"
"os/exec"
"os/user"
"path/filepath"
"strconv"
"strings"
"syscall"
@@ -91,3 +93,25 @@ func ownForHarness(name string, paths ...string) error {
}
return nil
}
// ownTreeForHarness gives the worktree to the harness user when the worker
// is root, except .git: the harness edits files and never writes the
// repository, so its refs and config stay the worker's.
func ownTreeForHarness(name, root string) error {
if name == "" || os.Geteuid() != 0 {
return nil
}
cred, err := harnessCredential(name)
if err != nil {
return err
}
return filepath.WalkDir(root, func(path string, d fs.DirEntry, err error) error {
if err != nil {
return err
}
if d.IsDir() && d.Name() == ".git" && path != root {
return filepath.SkipDir
}
return os.Lchown(path, int(cred.Uid), int(cred.Gid))
})
}
+3
View File
@@ -69,6 +69,9 @@ func (r *HarnessRunner) Run(ctx context.Context, workspace string, req Request,
if err := validateHarness(req); err != nil {
return Result{}, err
}
if err := ownTreeForHarness(req.Harness.User, workspace); err != nil {
return Result{}, fmt.Errorf("give the worktree to the harness user: %w", err)
}
mcpConfig, capturesPath, cleanup, err := r.prepareCaptures(req)
if err != nil {
return Result{}, err
+8 -1
View File
@@ -208,7 +208,10 @@ type boot struct {
natsToken string
containerPool *container.Pool
storeBackend storage.Backend
// kitLauncher starts Kit profile tasks; nil when the egress path cannot
// be built on this host.
kitLauncher daemon.KitLauncher
storeBackend storage.Backend
llm *runtime.Runtime
@@ -1332,6 +1335,7 @@ func (b *boot) buildDaemon() {
WorktreeGrants: b.worktreeGrants,
StateStoreGrants: b.stateStoreGrants,
ContainerPool: b.containerPool,
KitLauncher: b.kitLauncher,
Guardrails: b.guardrails,
ToolRegistry: b.toolReg,
Identities: b.identityRunners,
@@ -1609,6 +1613,9 @@ func (b *boot) startServices(ctx context.Context) error {
func (b *boot) setupBackends(ctx context.Context) error {
closeContainers := b.setupContainers(ctx)
err := b.connectNATS(ctx)
if err == nil {
b.setupKitLauncher(ctx)
}
// Container discovery must precede embedded NATS so it can bind the
// worker bridge gateway, but cleanup registration comes afterwards.
// cleanups run LIFO: workers stop first, then the daemon client closes,
+96
View File
@@ -0,0 +1,96 @@
package archied
import (
"context"
"errors"
"log/slog"
"net"
"net/http"
"os"
"path/filepath"
"strings"
"time"
"github.com/docker/sandbox-kit-spec/v3/fetch"
"github.com/samcharles93/archie-core/internal/config"
"github.com/samcharles93/archie-core/internal/infrastructure/egress"
"github.com/samcharles93/archie-core/internal/infrastructure/kitrun"
)
const relayName = "archie-egress-relay"
// setupKitLauncher builds the egress path Kit tasks run on: the per-daemon
// CA, the proxy on the worker bridge's host gateway, and the relay. It
// degrades to no launcher, which parks Kit tasks, rather than failing boot:
// image profiles do not need it.
func (b *boot) setupKitLauncher(ctx context.Context) {
pool, log := b.containerPool, b.log
if pool == nil || pool.HostGateway() == "" {
log.Info("kit harness runs disabled: they need the managed container pool on a bridge with a host gateway")
return
}
exe, err := os.Executable()
if err != nil {
log.Warn("kit harness runs disabled", "err", err)
return
}
agentBinary := filepath.Join(filepath.Dir(exe), "archie-agent")
caDir := filepath.Join(b.cfg.StateDir, "egress")
ca, err := egress.LoadOrCreateCA(caDir)
if err != nil {
log.Warn("kit harness runs disabled: egress CA", "err", err)
return
}
fetcher, err := fetch.New(fetch.WithDockerCredentials())
if err != nil {
log.Warn("kit harness runs disabled: kit registry client", "err", err)
return
}
ln, err := (&net.ListenConfig{}).Listen(ctx, "tcp", net.JoinHostPort(pool.HostGateway(), "0"))
if err != nil {
log.Warn("kit harness runs disabled: egress proxy listener", "err", err)
return
}
endpoints := kitrun.Endpoints{NATS: b.natsURL, StateStore: strings.TrimSpace(b.cfg.Services.Get(config.ServiceNameState).Target)}
cmd, err := kitrun.RelayCommand(ln.Addr().String(), endpoints)
if err != nil {
_ = ln.Close()
log.Warn("kit harness runs disabled: egress relay", "err", err)
return
}
// No credential resolver yet (archie-core-egkf.11): a Kit's required
// credential is refused at the proxy until run credentials carry secrets.
proxy := egress.NewProxy(ca, egress.ProxyOptions{})
srv := &http.Server{Handler: proxy, ReadHeaderTimeout: 30 * time.Second}
go func() {
if err := srv.Serve(ln); err != nil && !errors.Is(err, http.ErrServerClosed) {
log.Error("egress proxy stopped", "err", err)
}
}()
networks := egress.NewNetworks(pool.Client(), egress.RelaySpec{
Name: relayName, Image: b.cfg.Containers.Image,
Entrypoint: []string{"archie-agent"}, Cmd: cmd, Network: pool.NetworkName(),
})
b.addCleanup(shutdownEgress(networks, srv, log))
b.kitLauncher = &kitrun.Launcher{
Pool: pool, Fetch: fetcher, Proxy: proxy, Networks: networks,
AgentBinary: agentBinary, CAFile: egress.CACertPath(caDir),
}
log.Info("kit harness runs enabled", "proxy", ln.Addr().String())
}
// shutdownEgress removes the relay and stops the proxy. It derives its own
// budget because the boot context is cancelled by the time shutdown runs.
//
//nolint:contextcheck
func shutdownEgress(networks *egress.Networks, srv *http.Server, log *slog.Logger) func() {
return func() {
stopCtx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
defer cancel()
if err := networks.Close(stopCtx); err != nil {
log.Warn("egress relay removal failed", "err", err)
}
_ = srv.Close()
}
}
+59 -22
View File
@@ -10,6 +10,7 @@ import (
"fmt"
"io"
"log/slog"
"maps"
"os"
"path/filepath"
"sync"
@@ -195,16 +196,12 @@ func (p *Pool) Acquire(ctx context.Context, image string, mounts []storage.Mount
if image == "" {
image = p.cfg.Image
}
if err := p.ensureImage(ctx, image); err != nil {
if err := p.EnsureImage(ctx, image); err != nil {
return nil, err
}
p.mu.Lock()
if p.cfg.MaxConcurrency > 0 && p.active >= p.cfg.MaxConcurrency {
p.mu.Unlock()
return nil, fmt.Errorf("max concurrency %d reached", p.cfg.MaxConcurrency)
if err := p.reserve(); err != nil {
return nil, err
}
p.active++
p.mu.Unlock()
name := fmt.Sprintf("archie-agent-%d", time.Now().UnixNano())
@@ -232,9 +229,7 @@ func (p *Pool) Acquire(ctx context.Context, image string, mounts []storage.Mount
HostConfig: hostConfig,
})
if err != nil {
p.mu.Lock()
p.active--
p.mu.Unlock()
p.unreserve()
return nil, fmt.Errorf("container create: %w", err)
}
@@ -243,20 +238,62 @@ func (p *Pool) Acquire(ctx context.Context, image string, mounts []storage.Mount
if _, rmErr := p.cli.ContainerRemove(context.WithoutCancel(ctx), resp.ID, client.ContainerRemoveOptions{Force: true}); rmErr != nil {
p.log.Warn("container remove after start failure", "id", resp.ID[:12], "err", rmErr)
}
p.mu.Lock()
p.active--
p.mu.Unlock()
p.unreserve()
return nil, fmt.Errorf("container start: %w", err)
}
if p.cfg.MaxUptime > 0 {
p.armMaxUptime(ctx, resp.ID)
}
p.log.Info("container started", "id", resp.ID[:12], "name", name)
return &Container{ID: resp.ID, exited: p.watchExit(ctx, resp.ID)}, nil
return p.started(ctx, resp.ID, name), nil
}
// AcquireKit starts a Kit task container under the same concurrency cap,
// lifetime cap and exit watch as Acquire. Release tears it down.
func (p *Pool) AcquireKit(ctx context.Context, s KitSpec) (*Container, error) {
if err := p.EnsureImage(ctx, s.Image); err != nil {
return nil, err
}
if err := p.reserve(); err != nil {
return nil, err
}
s.Labels = maps.Clone(s.Labels)
if s.Labels == nil {
s.Labels = map[string]string{}
}
s.Labels["archie-daemon"] = "true"
id, err := StartKit(ctx, p.cli, s)
if err != nil {
p.unreserve()
return nil, err
}
return p.started(ctx, id, s.Name), nil
}
func (p *Pool) reserve() error {
p.mu.Lock()
defer p.mu.Unlock()
if p.cfg.MaxConcurrency > 0 && p.active >= p.cfg.MaxConcurrency {
return fmt.Errorf("max concurrency %d reached", p.cfg.MaxConcurrency)
}
p.active++
return nil
}
func (p *Pool) unreserve() {
p.mu.Lock()
p.active--
p.mu.Unlock()
}
func (p *Pool) started(ctx context.Context, id, name string) *Container {
if p.cfg.MaxUptime > 0 {
p.armMaxUptime(ctx, id)
}
p.log.Info("container started", "id", id[:12], "name", name)
return &Container{ID: id, exited: p.watchExit(ctx, id)}
}
// Client is the pool's Docker client, for callers that provision what a
// container needs around it.
func (p *Pool) Client() *client.Client { return p.cli }
// watchExit returns a channel closed once Docker reports the container is no
// longer running. A failed wait says nothing about the container, so it never
// closes the channel: a Docker API error must not abort a live run.
@@ -499,10 +536,10 @@ func (p *Pool) Close() error {
// ── helpers ──────────────────────────────────────────────────────────
// ensureImage pulls an image other than the configured one the first time a
// EnsureImage pulls an image other than the configured one the first time a
// task asks for it, under the configured pull policy. The configured image
// was already pulled by NewPool.
func (p *Pool) ensureImage(ctx context.Context, ref string) error {
func (p *Pool) EnsureImage(ctx context.Context, ref string) error {
if ref == p.cfg.Image || (p.cfg.PullPolicy != "always" && p.cfg.PullPolicy != "missing") {
return nil
}
+30 -14
View File
@@ -158,6 +158,8 @@ type Daemon struct {
// set -- acquireTaskContainer parks the task rather than fall back to
// forwarding the daemon's own administrative token.
StateStoreGrants StateStoreGrantIssuer
// KitLauncher starts tasks whose agent profile is a Kit. Nil parks them.
KitLauncher KitLauncher
// TaskRunReadyTimeout bounds how long runViaAgent retries an initial
// taskrun request that fails with nats.ErrNoResponders, giving a
// freshly spawned archie-agent container time to connect to NATS, set
@@ -1261,6 +1263,12 @@ func (d *Daemon) process(ctx context.Context, task *workflow.Task) {
if !ok {
return
}
if profile.IsKit() {
d.runKitTask(ctx, task, repo, workDir, profile)
d.teardownStorage(ctx, task, repo, workDir)
d.cleanupTerminalTaskWorktree(ctx, task, trees)
return
}
ctr, revokeStateStoreGrant, ok := d.acquireTaskContainer(ctx, task, repo, workDir, profile.Image)
if !ok {
return
@@ -1278,12 +1286,17 @@ func (d *Daemon) process(ctx context.Context, task *workflow.Task) {
// task is always served by its own forge client and worktree manager.
limitCtx, stopLimit := withTaskTimeLimit(ctx, d.configFor(task).Budgets.TaskWallClock.Std())
runCtx, stopWatch := withContainerExit(limitCtx, ctr.Exited())
d.runViaAgent(runCtx, task, repo, profile)
d.runViaAgent(runCtx, task, repo, profile, nil)
stopWatch()
stopLimit()
// Teardown storage after workflow completes. The Docker backend is a
// no-op; future backends (temp volumes, NFS leases) use this hook.
d.teardownStorage(ctx, task, repo, workDir)
d.cleanupTerminalTaskWorktree(ctx, task, trees)
}
// teardownStorage runs after the workflow completes. The Docker backend is
// a no-op; future backends (temp volumes, NFS leases) use this hook.
func (d *Daemon) teardownStorage(ctx context.Context, task *workflow.Task, repo config.Repo, workDir string) {
if d.Storage != nil {
_ = d.Storage.Teardown(ctx, storage.TaskRef{
WorktreeDir: workDir,
@@ -1293,8 +1306,6 @@ func (d *Daemon) process(ctx context.Context, task *workflow.Task) {
Repo: task.Repo,
})
}
d.cleanupTerminalTaskWorktree(ctx, task, trees)
}
// prepareWorkspace makes the directory the container binds as its
@@ -1427,14 +1438,7 @@ func (d *Daemon) acquireTaskContainer(
d.parkRunningTask(ctx, task.ID, reason+": "+err.Error(), taskstate.ParkTransient)
}
// Write task.json -- the container's boot-time brief.
if err := container.WriteTaskJSON(workDir, container.TaskPayload{
ID: task.ID, Owner: task.Owner, Repo: task.Repo,
Number: task.IssueNumber, Title: task.Title, Body: task.Body,
Labels: strings.Split(task.Labels, ","),
Workflow: task.Workflow, Branch: task.Branch, Plan: task.Plan,
Inputs: task.Inputs,
}); err != nil {
if err := writeTaskBrief(workDir, task); err != nil {
park("task.json write failed", err)
return nil, nil, false
}
@@ -1496,6 +1500,17 @@ func (d *Daemon) acquireTaskContainer(
return ctr, revokeStateStoreGrant, true
}
// writeTaskBrief writes task.json, the container's boot-time brief.
func writeTaskBrief(workDir string, task *workflow.Task) error {
return container.WriteTaskJSON(workDir, container.TaskPayload{
ID: task.ID, Owner: task.Owner, Repo: task.Repo,
Number: task.IssueNumber, Title: task.Title, Body: task.Body,
Labels: strings.Split(task.Labels, ","),
Workflow: task.Workflow, Branch: task.Branch, Plan: task.Plan,
Inputs: task.Inputs,
})
}
// parkCapabilityUnavailable parks a task whose daemon-side capability is
// missing (container pool or task transport): both are environmental, so
// the park class is transient and the KindParked event carries the class
@@ -1588,7 +1603,7 @@ func (d *Daemon) recordPark(ctx context.Context, taskID int64, reason string) {
// itself when nothing else could have: the request never reached (or was
// never answered by) an archie-agent, or archie-agent failed before its
// own workflow.Run got a chance to record an outcome.
func (d *Daemon) runViaAgent(ctx context.Context, task *workflow.Task, repo config.Repo, profile config.AgentProfile) {
func (d *Daemon) runViaAgent(ctx context.Context, task *workflow.Task, repo config.Repo, profile config.AgentProfile, harness *agentexec.HarnessSpec) {
grant, revoke, ok := d.publicationGrant(ctx, task)
if !ok {
return
@@ -1608,6 +1623,7 @@ func (d *Daemon) runViaAgent(ctx context.Context, task *workflow.Task, repo conf
LabelWorkflows: d.LabelWorkflows,
WorkflowDefinition: task.WorkflowDefinitionYAML,
Tools: profile.Tools,
Harness: harness,
}
data, err := json.Marshal(req)
if err != nil {
+88 -3
View File
@@ -19,6 +19,7 @@ import (
natssrv "github.com/nats-io/nats-server/v2/test"
natsio "github.com/nats-io/nats.go"
"github.com/samcharles93/archie-core/internal/agentexec"
"github.com/samcharles93/archie-core/internal/config"
"github.com/samcharles93/archie-core/internal/container"
"github.com/samcharles93/archie-core/internal/domain/workflow"
@@ -27,6 +28,8 @@ import (
"github.com/samcharles93/archie-core/internal/forge"
agentnats "github.com/samcharles93/archie-core/internal/infrastructure/agenttransport/nats"
arnats "github.com/samcharles93/archie-core/internal/infrastructure/eventbus/nats"
"github.com/samcharles93/archie-core/internal/infrastructure/kit"
"github.com/samcharles93/archie-core/internal/infrastructure/kitrun"
"github.com/samcharles93/archie-core/internal/infrastructure/postgres/pgstore"
"github.com/samcharles93/archie-core/internal/logging"
"github.com/samcharles93/archie-core/internal/secret"
@@ -2330,7 +2333,7 @@ func TestRunViaAgentCarriesTheWorkflowProfile(t *testing.T) {
if !ok || profile.Image != "agent-net:1" {
t.Fatalf("pinTaskProfile() = %+v, %v; want the net profile", profile, ok)
}
d.runViaAgent(ctx, task, config.Repo{Owner: "acme", Name: "widget", Base: "main"}, profile)
d.runViaAgent(ctx, task, config.Repo{Owner: "acme", Name: "widget", Base: "main"}, profile, nil)
select {
case req := <-received:
if len(req.Tools) != 1 || req.Tools[0] != "whois" {
@@ -2378,7 +2381,7 @@ func TestRunViaAgentParksWhenTheContainerExits(t *testing.T) {
defer stop()
done := make(chan struct{})
go func() {
d.runViaAgent(runCtx, task, config.Repo{Owner: "acme", Name: "widget", Base: "main"}, config.AgentProfile{})
d.runViaAgent(runCtx, task, config.Repo{Owner: "acme", Name: "widget", Base: "main"}, config.AgentProfile{}, nil)
close(done)
}()
@@ -2422,7 +2425,7 @@ func TestRunViaAgentParksWhenTheTaskTimeLimitPasses(t *testing.T) {
defer stop()
done := make(chan struct{})
go func() {
d.runViaAgent(runCtx, task, config.Repo{Owner: "acme", Name: "widget", Base: "main"}, config.AgentProfile{})
d.runViaAgent(runCtx, task, config.Repo{Owner: "acme", Name: "widget", Base: "main"}, config.AgentProfile{}, nil)
close(done)
}()
select {
@@ -2436,3 +2439,85 @@ func TestRunViaAgentParksWhenTheTaskTimeLimitPasses(t *testing.T) {
t.Fatalf("task = %+v, %v; want parked naming the time limit", got, err)
}
}
type fakeKitLauncher struct {
got kitrun.Request
released bool
removed []kit.Volume
}
func (f *fakeKitLauncher) Launch(_ context.Context, req kitrun.Request) (*kitrun.Run, error) {
f.got = req
return &kitrun.Run{
Container: &container.Container{},
Harness: agentexec.HarnessSpec{Adapter: agentexec.AdapterClaudeCode, Launch: []string{"claude"}},
Volumes: []kit.Volume{{Name: "v", Path: "/home/agent/.claude"}},
}, nil
}
func (f *fakeKitLauncher) Release(context.Context, *kitrun.Run) error {
f.released = true
return nil
}
func (f *fakeKitLauncher) RemoveVolumes(_ context.Context, v []kit.Volume) error {
f.removed = append(f.removed, v...)
return nil
}
// A Kit profile starts the task through the Kit launcher, hands the worker
// the Kit's harness, keeps model provider keys out of the container, and
// removes the execution's volumes once the task is no longer retryable.
func TestKitProfileRunsTheTaskOnItsHarness(t *testing.T) {
d, s, busClient := daemonWithNATS(t)
launcher := &fakeKitLauncher{}
d.KitLauncher = launcher
d.ConnectedNATS.URL = "nats://172.17.0.1:4222"
t.Setenv("KIT_TEST_PROVIDER_KEY", "sk-real")
cfg := d.Cfg.Get()
cfg.Providers = map[string]config.Provider{"p": {APIKeyEnv: "KIT_TEST_PROVIDER_KEY"}}
d.Cfg.Set(cfg)
ctx := context.Background()
if _, err := s.EnqueueIssue(ctx, "acme", "widget", 31, "t", "b", "", ""); err != nil {
t.Fatal(err)
}
task, err := s.ClaimNext(ctx)
if err != nil || task == nil {
t.Fatalf("claim: (%v, %v)", task, err)
}
received := make(chan taskrun.Request, 1)
sub, err := mustCoreConn(t, busClient).Subscribe(agentnats.SubjectForTask(task.ID), func(msg *natsio.Msg) {
var req taskrun.Request
_ = json.Unmarshal(msg.Data, &req)
received <- req
data, _ := json.Marshal(taskrun.Response{Status: workflow.StatusPROpen})
_ = msg.Respond(data)
})
if err != nil {
t.Fatalf("subscribe: %v", err)
}
t.Cleanup(func() { _ = sub.Unsubscribe() })
profile := config.AgentProfile{Kit: []string{"kit@sha256:ab"}, Adapter: agentexec.AdapterClaudeCode}
d.runKitTask(ctx, task, config.Repo{Owner: "acme", Name: "widget", Base: "main"}, t.TempDir(), profile)
select {
case req := <-received:
if req.Harness == nil || req.Harness.Adapter != agentexec.AdapterClaudeCode {
t.Fatalf("request harness = %+v, want the Kit's", req.Harness)
}
case <-time.After(time.Second):
t.Fatal("archied did not publish a taskrun request")
}
if !slices.Equal(launcher.got.Kit, profile.Kit) || launcher.got.Adapter != profile.Adapter {
t.Fatalf("launch request %+v, want the profile's kits and adapter", launcher.got)
}
for _, e := range launcher.got.WorkerEnv {
if strings.Contains(e, "sk-real") {
t.Fatalf("worker env carries a model provider key: %q", e)
}
}
if !launcher.released || len(launcher.removed) != 1 {
t.Fatalf("released %v, removed %v; want the run released and its volumes removed", launcher.released, launcher.removed)
}
}
+94
View File
@@ -0,0 +1,94 @@
package daemon
import (
"context"
"fmt"
"os"
"time"
"github.com/samcharles93/archie-core/internal/config"
"github.com/samcharles93/archie-core/internal/domain/workflow"
"github.com/samcharles93/archie-core/internal/infrastructure/kit"
"github.com/samcharles93/archie-core/internal/infrastructure/kitrun"
"github.com/samcharles93/archie-core/internal/taskstate"
)
// KitLauncher starts and ends the container of a task whose agent profile
// is a Kit.
type KitLauncher interface {
Launch(ctx context.Context, req kitrun.Request) (*kitrun.Run, error)
Release(ctx context.Context, run *kitrun.Run) error
RemoveVolumes(ctx context.Context, volumes []kit.Volume) error
}
// runKitTask runs a task in its Kit's container, every agent stage on the
// Kit's harness. The worker reaches NATS and the State Store only through
// the egress relay, and its environment carries no model provider key.
func (d *Daemon) runKitTask(ctx context.Context, task *workflow.Task, repo config.Repo, workDir string, profile config.AgentProfile) {
park := func(reason string, err error) {
d.Log.Error(reason, "task", task.ID, "err", err)
d.parkRunningTask(ctx, task.ID, reason+": "+err.Error(), taskstate.ParkTransient)
}
if d.KitLauncher == nil {
park("kit profile", fmt.Errorf("kit harness runs are unavailable on this daemon"))
return
}
if err := writeTaskBrief(workDir, task); err != nil {
park("task.json write failed", err)
return
}
stateStoreToken, revokeStateStoreGrant, err := d.stateStoreGrantToken(task)
if err != nil {
park("state store grant failed", err)
return
}
defer revokeStateStoreGrant()
env, err := kitrun.WorkerEnv(kitrun.Endpoints{
NATS: d.ConnectedNATS.URL, NATSToken: d.ConnectedNATS.Token,
StateStore: d.ConnectedStateStore.URL, StateStoreToken: stateStoreToken,
}, os.Getuid(), os.Getgid())
if err != nil {
park("kit worker environment", err)
return
}
run, err := d.KitLauncher.Launch(ctx, kitrun.Request{
Execution: fmt.Sprintf("task-%d", task.ID),
Kit: profile.Kit,
Adapter: profile.Adapter,
WorkDir: workDir,
WorkerEnv: env,
})
if err != nil {
park("kit launch failed", err)
return
}
limitCtx, stopLimit := withTaskTimeLimit(ctx, d.configFor(task).Budgets.TaskWallClock.Std())
runCtx, stopWatch := withContainerExit(limitCtx, run.Container.Exited())
d.runViaAgent(runCtx, task, repo, profile, &run.Harness)
stopWatch()
stopLimit()
if err := d.KitLauncher.Release(ctx, run); err != nil {
d.Log.Warn("kit release failed", "task", task.ID, "err", err)
}
d.removeEndedKitVolumes(ctx, task.ID, run.Volumes)
}
// removeEndedKitVolumes removes the execution's Kit volumes unless the task
// parked: a retry of a parked task is the same execution and resumes from
// them.
func (d *Daemon) removeEndedKitVolumes(ctx context.Context, taskID int64, volumes []kit.Volume) {
if len(volumes) == 0 || d.Store == nil {
return
}
lookupCtx, cancel := context.WithTimeout(context.WithoutCancel(ctx), 10*time.Second)
defer cancel()
latest, err := d.Store.TaskByID(lookupCtx, taskID)
if err != nil || latest == nil || latest.Status == workflow.StatusParked {
return
}
if err := d.KitLauncher.RemoveVolumes(lookupCtx, volumes); err != nil {
d.Log.Warn("kit volume removal failed", "task", taskID, "err", err)
}
}
+1 -1
View File
@@ -42,5 +42,5 @@ func runPinnedViaAgent(ctx context.Context, d *Daemon, task *workflow.Task, repo
if !ok {
return
}
d.runViaAgent(ctx, task, repo, profile)
d.runViaAgent(ctx, task, repo, profile, nil)
}
+4
View File
@@ -40,6 +40,10 @@ type CA struct {
leaves map[string]*tls.Certificate
}
// CACertPath is where LoadOrCreateCA keeps the CA certificate in dir, the
// file a sandbox container mounts to trust the proxy.
func CACertPath(dir string) string { return filepath.Join(dir, caCertFile) }
// LoadOrCreateCA loads the CA from dir, creating it on first use. A CA that
// exists but will not load is an error: minting a replacement silently
// would break trust in every running container.
+10 -1
View File
@@ -129,10 +129,19 @@ func Admit(d *spec.Descriptor) (*Plan, error) {
// admits the result. archie drives a harness headlessly, so the composition
// must carry an agent-sessions prompt verb.
func Compose(contributions []spec.Contribution) (*Plan, error) {
merged, err := spec.Merge(contributions, spec.MergeOptions{ContextPath: contextPath})
merged, err := spec.Merge(contributions, MergeOptions)
if err != nil {
return nil, err
}
return FromMerge(merged)
}
// MergeOptions are the options every composition archie runs is merged with.
var MergeOptions = spec.MergeOptions{ContextPath: contextPath}
// FromMerge admits an already-merged composition, such as one the Kit spec's
// fetch package assembled from a registry.
func FromMerge(merged *spec.MergeResult) (*Plan, error) {
if merged.Descriptor.Kind != spec.KindWorkload {
return nil, errors.New("kit composition has no workload: a harness needs exactly one base kit")
}
+72
View File
@@ -0,0 +1,72 @@
package kitrun
import (
"fmt"
"net"
"net/url"
"strconv"
"strings"
"github.com/samcharles93/archie-core/internal/infrastructure/egress"
)
// Endpoints are the host-reachable addresses a Kit task's worker needs,
// with the task's own credentials for them.
type Endpoints struct {
NATS string
NATSToken string
StateStore string
StateStoreToken string
}
// WorkerEnv is the worker's entire environment in a Kit container: NATS and
// the State Store at the relay's fixed ports, and the task's credentials.
// Nothing else crosses, so no model provider key enters the container.
func WorkerEnv(e Endpoints, uid, gid int) ([]string, error) {
u, err := url.Parse(e.NATS)
if err != nil || u.Scheme == "" {
return nil, fmt.Errorf("nats url %q: want scheme://host:port", e.NATS)
}
env := []string{"NATS_URL=" + u.Scheme + "://" + relayAddr(egress.RelayNATSPort)}
if e.NATSToken != "" {
env = append(env, "NATS_TOKEN="+e.NATSToken)
}
if e.StateStore != "" {
env = append(env, "STATE_STORE_URL="+relayAddr(egress.RelayStateStorePort))
if e.StateStoreToken != "" {
env = append(env, "STATE_STORE_TOKEN="+e.StateStoreToken)
}
}
return append(env, "WORKTREE_UID="+strconv.Itoa(uid), "WORKTREE_GID="+strconv.Itoa(gid)), nil
}
// relayTargets are the host:port addresses the relay forwards NATS and the
// State Store to.
func relayTargets(e Endpoints) (nats, stateStore string, err error) {
if strings.Contains(e.NATS, ",") {
return "", "", fmt.Errorf("nats url %q lists several servers; the relay forwards one", e.NATS)
}
u, err := url.Parse(e.NATS)
if err != nil || u.Host == "" {
return "", "", fmt.Errorf("nats url %q: want scheme://host:port", e.NATS)
}
stateStore = e.StateStore
if _, rest, ok := strings.Cut(stateStore, ":///"); ok {
stateStore = rest
}
return u.Host, stateStore, nil
}
func relayAddr(port int) string {
return net.JoinHostPort(egress.RelayAlias, strconv.Itoa(port))
}
// RelayCommand is the relay's archie-agent arguments, forwarding its fixed
// ports to the proxy and to the daemon's NATS and State Store.
func RelayCommand(proxy string, e Endpoints) ([]string, error) {
nats, stateStore, err := relayTargets(e)
if err != nil {
return nil, err
}
return egress.RelayArgs(proxy, nats, stateStore), nil
}
@@ -0,0 +1,44 @@
package kitrun
import (
"slices"
"testing"
)
func TestWorkerEnvReachesNATSAndTheStateStoreThroughTheRelay(t *testing.T) {
got, err := WorkerEnv(Endpoints{
NATS: "nats://172.17.0.1:4222", NATSToken: "nt",
StateStore: "10.0.0.5:7443", StateStoreToken: "st",
}, 1001, 1002)
if err != nil {
t.Fatal(err)
}
want := []string{
"NATS_URL=nats://archie-egress:4222", "NATS_TOKEN=nt",
"STATE_STORE_URL=archie-egress:50051", "STATE_STORE_TOKEN=st",
"WORKTREE_UID=1001", "WORKTREE_GID=1002",
}
if !slices.Equal(got, want) {
t.Fatalf("WorkerEnv = %q\nwant %q", got, want)
}
}
func TestWorkerEnvWithoutAStateStoreOmitsIt(t *testing.T) {
got, err := WorkerEnv(Endpoints{NATS: "tls://nats.internal:4222"}, 0, 0)
if err != nil {
t.Fatal(err)
}
if !slices.Equal(got, []string{"NATS_URL=tls://archie-egress:4222", "WORKTREE_UID=0", "WORKTREE_GID=0"}) {
t.Fatalf("WorkerEnv = %q", got)
}
}
func TestRelayTargets(t *testing.T) {
nats, store, err := relayTargets(Endpoints{NATS: "nats://172.17.0.1:4222", StateStore: "dns:///store.lan:7443"})
if err != nil || nats != "172.17.0.1:4222" || store != "store.lan:7443" {
t.Fatalf("relayTargets = %q, %q, %v", nats, store, err)
}
if _, _, err := relayTargets(Endpoints{NATS: "nats://a:1,nats://b:2"}); err == nil {
t.Fatal("a NATS cluster URL list was accepted; the relay forwards one address")
}
}
+195
View File
@@ -0,0 +1,195 @@
// Package kitrun starts a Kit task: it composes the profile's Kits, opens
// the task's egress session and sandbox network, and starts the container
// with archie's worker as root and the harness spec the worker runs stages on.
package kitrun
import (
"context"
"errors"
"fmt"
"maps"
"slices"
"sync"
"github.com/docker/sandbox-kit-spec/v3/fetch"
"github.com/docker/sandbox-kit-spec/v3/spec"
"github.com/samcharles93/archie-core/internal/agentexec"
"github.com/samcharles93/archie-core/internal/container"
"github.com/samcharles93/archie-core/internal/infrastructure/egress"
"github.com/samcharles93/archie-core/internal/infrastructure/kit"
)
const (
agentPath = "/opt/archie/archie-agent"
caPath = "/opt/archie/ca.pem"
)
// Launcher starts Kit task containers.
type Launcher struct {
Pool *container.Pool
Fetch *fetch.Client
Proxy *egress.Proxy
Networks *egress.Networks
// AgentBinary and CAFile are host paths mounted read-only into every
// Kit container: the worker's own binary and the egress CA.
AgentBinary string
CAFile string
mu sync.Mutex
started bool
}
// Request is one Kit task to start.
type Request struct {
// Execution keys the task's Kit volumes and names its container and
// network.
Execution string
Kit []string
Adapter string
WorkDir string
WorkerEnv []string
}
// Run is a started Kit task.
type Run struct {
Container *container.Container
Harness agentexec.HarnessSpec
// Volumes outlive the run; remove them with RemoveVolumes once the
// execution has ended.
Volumes []kit.Volume
network string
token string
}
// Launch composes the request's Kits and starts its container.
func (l *Launcher) Launch(ctx context.Context, req Request) (*Run, error) {
if len(req.Kit) == 0 {
return nil, errors.New("kit profile names no kit")
}
adapter, ok := agentexec.LookupHarnessAdapter(req.Adapter)
if req.Adapter != "" && !ok {
return nil, fmt.Errorf("harness output adapter %q is unknown", req.Adapter)
}
if err := l.startEgress(ctx); err != nil {
return nil, err
}
plan, img, err := l.compose(ctx, req.Kit)
if err != nil {
return nil, err
}
network, err := spec.NetworkPolicyOf(plan.Capabilities)
if err != nil {
return nil, err
}
creds, err := spec.CredentialsOf(plan.Capabilities)
if err != nil {
return nil, err
}
session, err := l.Proxy.Register(egress.SessionOptions{Run: req.Execution, Network: network, Credentials: creds})
if err != nil {
return nil, err
}
run := &Run{network: "archie-kit-" + req.Execution, token: session.Token()}
launch, err := kit.Assemble(plan, img, kit.LaunchParams{Execution: req.Execution, ProxyToken: session.Token(), CAPath: caPath})
if err != nil {
l.Proxy.Revoke(run.token)
return nil, err
}
launch.Harness.Adapter, launch.Harness.MCPConfig = req.Adapter, adapter.MCPConfig
run.Harness, run.Volumes = launch.Harness, launch.Volumes
if err := l.Networks.Create(ctx, run.network); err != nil {
l.Proxy.Revoke(run.token)
return nil, err
}
run.Container, err = l.Pool.AcquireKit(ctx, container.KitSpec{
Name: "archie-kit-" + req.Execution,
Image: req.Kit[0],
Network: run.network,
Worker: []string{agentPath},
WorkerEnv: req.WorkerEnv,
Binds: []string{
l.AgentBinary + ":" + agentPath + ":ro",
l.CAFile + ":" + caPath + ":ro",
req.WorkDir + ":" + kit.WorkspaceDir,
},
Launch: launch,
InstallDone: session.EnterRuntime,
})
if err != nil {
l.Proxy.Revoke(run.token)
return nil, errors.Join(err, l.Networks.Remove(context.WithoutCancel(ctx), run.network))
}
return run, nil
}
// compose assembles the profile's Kits and reads the workload image's
// config, the first Kit being the workload.
func (l *Launcher) compose(ctx context.Context, refs []string) (*kit.Plan, kit.ImageConfig, error) {
reqs := make([]fetch.Request, len(refs))
for i, ref := range refs {
reqs[i] = fetch.Request{Reference: ref}
}
merged, err := l.Fetch.Assemble(ctx, reqs, kit.MergeOptions)
if err != nil {
return nil, kit.ImageConfig{}, fmt.Errorf("assemble kits: %w", err)
}
plan, err := kit.FromMerge(merged.MergeResult)
if err != nil {
return nil, kit.ImageConfig{}, err
}
img, err := l.imageConfig(ctx, refs[0])
if err != nil {
return nil, kit.ImageConfig{}, err
}
for _, name := range slices.Sorted(maps.Keys(merged.Env)) {
img.Env = append(img.Env, name+"="+merged.Env[name])
}
return plan, img, nil
}
// Release stops the run's container, then removes its network and ends
// its egress session.
func (l *Launcher) Release(ctx context.Context, run *Run) error {
l.Pool.Release(ctx, run.Container)
l.Proxy.Revoke(run.token)
return l.Networks.Remove(context.WithoutCancel(ctx), run.network)
}
// RemoveVolumes removes an ended execution's Kit volumes.
func (l *Launcher) RemoveVolumes(ctx context.Context, volumes []kit.Volume) error {
return container.RemoveKitVolumes(ctx, l.Pool.Client(), volumes)
}
// startEgress starts the relay on first use, so a daemon with no Kit
// profile runs no relay.
func (l *Launcher) startEgress(ctx context.Context) error {
l.mu.Lock()
defer l.mu.Unlock()
if l.started {
return nil
}
if err := l.Networks.Start(ctx); err != nil {
return err
}
l.started = true
return nil
}
func (l *Launcher) imageConfig(ctx context.Context, ref string) (kit.ImageConfig, error) {
if err := l.Pool.EnsureImage(ctx, ref); err != nil {
return kit.ImageConfig{}, err
}
inspected, err := l.Pool.Client().ImageInspect(ctx, ref)
if err != nil {
return kit.ImageConfig{}, fmt.Errorf("inspect kit image %s: %w", ref, err)
}
cfg := inspected.Config
if cfg == nil {
return kit.ImageConfig{}, fmt.Errorf("kit image %s has no config", ref)
}
return kit.ImageConfig{
Entrypoint: slices.Clone(cfg.Entrypoint), Cmd: slices.Clone(cfg.Cmd),
Env: slices.Clone(cfg.Env), User: cfg.User,
}, nil
}