diff --git a/go.mod b/go.mod index 545a585be..195fd9297 100644 --- a/go.mod +++ b/go.mod @@ -90,6 +90,7 @@ require ( google.golang.org/genproto/googleapis/rpc v0.0.0-20260921155816-b14227669459 // indirect modernc.org/libc v1.77.1 // indirect modernc.org/sqlite v1.59.0 // indirect + oras.land/oras-go/v2 v2.6.2 // indirect ) require ( diff --git a/go.sum b/go.sum index e16a76e27..09836e4da 100644 --- a/go.sum +++ b/go.sum @@ -325,5 +325,7 @@ modernc.org/memory v1.12.1 h1:nFMiWrpStgZczNl6XI9GnIk/rWhYIyHGUaR04pGbp9g= modernc.org/memory v1.12.1/go.mod h1:/JP4VbVC+K5sU2wZi9bHoq2MAkCnrt2r98UGeSK7Mjw= modernc.org/sqlite v1.59.0 h1:X1es1GpqBlS/5T+vbM4HLUdaa8OtQx468DF2vrx+38A= modernc.org/sqlite v1.59.0/go.mod h1:+paeT2A3iPRHkQDwG7oA6Tk0zQd5woMEI8q7orfry8k= +oras.land/oras-go/v2 v2.6.2 h1:N04RXngAp1LJKTG6ifz3xHPipasEkWr+hFmInja5YKo= +oras.land/oras-go/v2 v2.6.2/go.mod h1:PlTtg4JTDJkDe8yVHpM2wz7/YDc00GVas+i4jAW2TZ4= pgregory.net/rapid v1.2.0 h1:keKAYRcjm+e1F0oAuU5F5+YPAWcyxNNRK2wud503Gnk= pgregory.net/rapid v1.2.0/go.mod h1:PY5XlDGj0+V1FCq0o192FdRhpKHGTRIWBgqjDBTrq04= diff --git a/internal/agentexec/harness_proc_other.go b/internal/agentexec/harness_proc_other.go index 9d32c11a6..870e1e90c 100644 --- a/internal/agentexec/harness_proc_other.go +++ b/internal/agentexec/harness_proc_other.go @@ -22,3 +22,6 @@ func runAsHarnessUser(_ *exec.Cmd, name string) error { // ownForHarness has nothing to do where the harness runs as the worker. func ownForHarness(string, ...string) error { return nil } + +// ownTreeForHarness has nothing to do where the harness runs as the worker. +func ownTreeForHarness(string, string) error { return nil } diff --git a/internal/agentexec/harness_proc_unix.go b/internal/agentexec/harness_proc_unix.go index 00ffd06d1..d500f1052 100644 --- a/internal/agentexec/harness_proc_unix.go +++ b/internal/agentexec/harness_proc_unix.go @@ -5,9 +5,11 @@ package agentexec import ( "errors" "fmt" + "io/fs" "os" "os/exec" "os/user" + "path/filepath" "strconv" "strings" "syscall" @@ -91,3 +93,25 @@ func ownForHarness(name string, paths ...string) error { } return nil } + +// ownTreeForHarness gives the worktree to the harness user when the worker +// is root, except .git: the harness edits files and never writes the +// repository, so its refs and config stay the worker's. +func ownTreeForHarness(name, root string) error { + if name == "" || os.Geteuid() != 0 { + return nil + } + cred, err := harnessCredential(name) + if err != nil { + return err + } + return filepath.WalkDir(root, func(path string, d fs.DirEntry, err error) error { + if err != nil { + return err + } + if d.IsDir() && d.Name() == ".git" && path != root { + return filepath.SkipDir + } + return os.Lchown(path, int(cred.Uid), int(cred.Gid)) + }) +} diff --git a/internal/agentexec/harness_runner.go b/internal/agentexec/harness_runner.go index 1dd480efc..444eb8f48 100644 --- a/internal/agentexec/harness_runner.go +++ b/internal/agentexec/harness_runner.go @@ -69,6 +69,9 @@ func (r *HarnessRunner) Run(ctx context.Context, workspace string, req Request, if err := validateHarness(req); err != nil { return Result{}, err } + if err := ownTreeForHarness(req.Harness.User, workspace); err != nil { + return Result{}, fmt.Errorf("give the worktree to the harness user: %w", err) + } mcpConfig, capturesPath, cleanup, err := r.prepareCaptures(req) if err != nil { return Result{}, err diff --git a/internal/app/archied/bootstrap.go b/internal/app/archied/bootstrap.go index 424d2b10b..f140074e5 100644 --- a/internal/app/archied/bootstrap.go +++ b/internal/app/archied/bootstrap.go @@ -208,7 +208,10 @@ type boot struct { natsToken string containerPool *container.Pool - storeBackend storage.Backend + // kitLauncher starts Kit profile tasks; nil when the egress path cannot + // be built on this host. + kitLauncher daemon.KitLauncher + storeBackend storage.Backend llm *runtime.Runtime @@ -1332,6 +1335,7 @@ func (b *boot) buildDaemon() { WorktreeGrants: b.worktreeGrants, StateStoreGrants: b.stateStoreGrants, ContainerPool: b.containerPool, + KitLauncher: b.kitLauncher, Guardrails: b.guardrails, ToolRegistry: b.toolReg, Identities: b.identityRunners, @@ -1609,6 +1613,9 @@ func (b *boot) startServices(ctx context.Context) error { func (b *boot) setupBackends(ctx context.Context) error { closeContainers := b.setupContainers(ctx) err := b.connectNATS(ctx) + if err == nil { + b.setupKitLauncher(ctx) + } // Container discovery must precede embedded NATS so it can bind the // worker bridge gateway, but cleanup registration comes afterwards. // cleanups run LIFO: workers stop first, then the daemon client closes, diff --git a/internal/app/archied/kit_launcher.go b/internal/app/archied/kit_launcher.go new file mode 100644 index 000000000..a921af320 --- /dev/null +++ b/internal/app/archied/kit_launcher.go @@ -0,0 +1,96 @@ +package archied + +import ( + "context" + "errors" + "log/slog" + "net" + "net/http" + "os" + "path/filepath" + "strings" + "time" + + "github.com/docker/sandbox-kit-spec/v3/fetch" + + "github.com/samcharles93/archie-core/internal/config" + "github.com/samcharles93/archie-core/internal/infrastructure/egress" + "github.com/samcharles93/archie-core/internal/infrastructure/kitrun" +) + +const relayName = "archie-egress-relay" + +// setupKitLauncher builds the egress path Kit tasks run on: the per-daemon +// CA, the proxy on the worker bridge's host gateway, and the relay. It +// degrades to no launcher, which parks Kit tasks, rather than failing boot: +// image profiles do not need it. +func (b *boot) setupKitLauncher(ctx context.Context) { + pool, log := b.containerPool, b.log + if pool == nil || pool.HostGateway() == "" { + log.Info("kit harness runs disabled: they need the managed container pool on a bridge with a host gateway") + return + } + exe, err := os.Executable() + if err != nil { + log.Warn("kit harness runs disabled", "err", err) + return + } + agentBinary := filepath.Join(filepath.Dir(exe), "archie-agent") + caDir := filepath.Join(b.cfg.StateDir, "egress") + ca, err := egress.LoadOrCreateCA(caDir) + if err != nil { + log.Warn("kit harness runs disabled: egress CA", "err", err) + return + } + fetcher, err := fetch.New(fetch.WithDockerCredentials()) + if err != nil { + log.Warn("kit harness runs disabled: kit registry client", "err", err) + return + } + ln, err := (&net.ListenConfig{}).Listen(ctx, "tcp", net.JoinHostPort(pool.HostGateway(), "0")) + if err != nil { + log.Warn("kit harness runs disabled: egress proxy listener", "err", err) + return + } + endpoints := kitrun.Endpoints{NATS: b.natsURL, StateStore: strings.TrimSpace(b.cfg.Services.Get(config.ServiceNameState).Target)} + cmd, err := kitrun.RelayCommand(ln.Addr().String(), endpoints) + if err != nil { + _ = ln.Close() + log.Warn("kit harness runs disabled: egress relay", "err", err) + return + } + // No credential resolver yet (archie-core-egkf.11): a Kit's required + // credential is refused at the proxy until run credentials carry secrets. + proxy := egress.NewProxy(ca, egress.ProxyOptions{}) + srv := &http.Server{Handler: proxy, ReadHeaderTimeout: 30 * time.Second} + go func() { + if err := srv.Serve(ln); err != nil && !errors.Is(err, http.ErrServerClosed) { + log.Error("egress proxy stopped", "err", err) + } + }() + networks := egress.NewNetworks(pool.Client(), egress.RelaySpec{ + Name: relayName, Image: b.cfg.Containers.Image, + Entrypoint: []string{"archie-agent"}, Cmd: cmd, Network: pool.NetworkName(), + }) + b.addCleanup(shutdownEgress(networks, srv, log)) + b.kitLauncher = &kitrun.Launcher{ + Pool: pool, Fetch: fetcher, Proxy: proxy, Networks: networks, + AgentBinary: agentBinary, CAFile: egress.CACertPath(caDir), + } + log.Info("kit harness runs enabled", "proxy", ln.Addr().String()) +} + +// shutdownEgress removes the relay and stops the proxy. It derives its own +// budget because the boot context is cancelled by the time shutdown runs. +// +//nolint:contextcheck +func shutdownEgress(networks *egress.Networks, srv *http.Server, log *slog.Logger) func() { + return func() { + stopCtx, cancel := context.WithTimeout(context.Background(), 10*time.Second) + defer cancel() + if err := networks.Close(stopCtx); err != nil { + log.Warn("egress relay removal failed", "err", err) + } + _ = srv.Close() + } +} diff --git a/internal/container/pool.go b/internal/container/pool.go index cd6b9489c..f13d285d8 100644 --- a/internal/container/pool.go +++ b/internal/container/pool.go @@ -10,6 +10,7 @@ import ( "fmt" "io" "log/slog" + "maps" "os" "path/filepath" "sync" @@ -195,16 +196,12 @@ func (p *Pool) Acquire(ctx context.Context, image string, mounts []storage.Mount if image == "" { image = p.cfg.Image } - if err := p.ensureImage(ctx, image); err != nil { + if err := p.EnsureImage(ctx, image); err != nil { return nil, err } - p.mu.Lock() - if p.cfg.MaxConcurrency > 0 && p.active >= p.cfg.MaxConcurrency { - p.mu.Unlock() - return nil, fmt.Errorf("max concurrency %d reached", p.cfg.MaxConcurrency) + if err := p.reserve(); err != nil { + return nil, err } - p.active++ - p.mu.Unlock() name := fmt.Sprintf("archie-agent-%d", time.Now().UnixNano()) @@ -232,9 +229,7 @@ func (p *Pool) Acquire(ctx context.Context, image string, mounts []storage.Mount HostConfig: hostConfig, }) if err != nil { - p.mu.Lock() - p.active-- - p.mu.Unlock() + p.unreserve() return nil, fmt.Errorf("container create: %w", err) } @@ -243,20 +238,62 @@ func (p *Pool) Acquire(ctx context.Context, image string, mounts []storage.Mount if _, rmErr := p.cli.ContainerRemove(context.WithoutCancel(ctx), resp.ID, client.ContainerRemoveOptions{Force: true}); rmErr != nil { p.log.Warn("container remove after start failure", "id", resp.ID[:12], "err", rmErr) } - p.mu.Lock() - p.active-- - p.mu.Unlock() + p.unreserve() return nil, fmt.Errorf("container start: %w", err) } - - if p.cfg.MaxUptime > 0 { - p.armMaxUptime(ctx, resp.ID) - } - - p.log.Info("container started", "id", resp.ID[:12], "name", name) - return &Container{ID: resp.ID, exited: p.watchExit(ctx, resp.ID)}, nil + return p.started(ctx, resp.ID, name), nil } +// AcquireKit starts a Kit task container under the same concurrency cap, +// lifetime cap and exit watch as Acquire. Release tears it down. +func (p *Pool) AcquireKit(ctx context.Context, s KitSpec) (*Container, error) { + if err := p.EnsureImage(ctx, s.Image); err != nil { + return nil, err + } + if err := p.reserve(); err != nil { + return nil, err + } + s.Labels = maps.Clone(s.Labels) + if s.Labels == nil { + s.Labels = map[string]string{} + } + s.Labels["archie-daemon"] = "true" + id, err := StartKit(ctx, p.cli, s) + if err != nil { + p.unreserve() + return nil, err + } + return p.started(ctx, id, s.Name), nil +} + +func (p *Pool) reserve() error { + p.mu.Lock() + defer p.mu.Unlock() + if p.cfg.MaxConcurrency > 0 && p.active >= p.cfg.MaxConcurrency { + return fmt.Errorf("max concurrency %d reached", p.cfg.MaxConcurrency) + } + p.active++ + return nil +} + +func (p *Pool) unreserve() { + p.mu.Lock() + p.active-- + p.mu.Unlock() +} + +func (p *Pool) started(ctx context.Context, id, name string) *Container { + if p.cfg.MaxUptime > 0 { + p.armMaxUptime(ctx, id) + } + p.log.Info("container started", "id", id[:12], "name", name) + return &Container{ID: id, exited: p.watchExit(ctx, id)} +} + +// Client is the pool's Docker client, for callers that provision what a +// container needs around it. +func (p *Pool) Client() *client.Client { return p.cli } + // watchExit returns a channel closed once Docker reports the container is no // longer running. A failed wait says nothing about the container, so it never // closes the channel: a Docker API error must not abort a live run. @@ -499,10 +536,10 @@ func (p *Pool) Close() error { // ── helpers ────────────────────────────────────────────────────────── -// ensureImage pulls an image other than the configured one the first time a +// EnsureImage pulls an image other than the configured one the first time a // task asks for it, under the configured pull policy. The configured image // was already pulled by NewPool. -func (p *Pool) ensureImage(ctx context.Context, ref string) error { +func (p *Pool) EnsureImage(ctx context.Context, ref string) error { if ref == p.cfg.Image || (p.cfg.PullPolicy != "always" && p.cfg.PullPolicy != "missing") { return nil } diff --git a/internal/daemon/daemon.go b/internal/daemon/daemon.go index e3acb5756..a841f5f9f 100644 --- a/internal/daemon/daemon.go +++ b/internal/daemon/daemon.go @@ -158,6 +158,8 @@ type Daemon struct { // set -- acquireTaskContainer parks the task rather than fall back to // forwarding the daemon's own administrative token. StateStoreGrants StateStoreGrantIssuer + // KitLauncher starts tasks whose agent profile is a Kit. Nil parks them. + KitLauncher KitLauncher // TaskRunReadyTimeout bounds how long runViaAgent retries an initial // taskrun request that fails with nats.ErrNoResponders, giving a // freshly spawned archie-agent container time to connect to NATS, set @@ -1261,6 +1263,12 @@ func (d *Daemon) process(ctx context.Context, task *workflow.Task) { if !ok { return } + if profile.IsKit() { + d.runKitTask(ctx, task, repo, workDir, profile) + d.teardownStorage(ctx, task, repo, workDir) + d.cleanupTerminalTaskWorktree(ctx, task, trees) + return + } ctr, revokeStateStoreGrant, ok := d.acquireTaskContainer(ctx, task, repo, workDir, profile.Image) if !ok { return @@ -1278,12 +1286,17 @@ func (d *Daemon) process(ctx context.Context, task *workflow.Task) { // task is always served by its own forge client and worktree manager. limitCtx, stopLimit := withTaskTimeLimit(ctx, d.configFor(task).Budgets.TaskWallClock.Std()) runCtx, stopWatch := withContainerExit(limitCtx, ctr.Exited()) - d.runViaAgent(runCtx, task, repo, profile) + d.runViaAgent(runCtx, task, repo, profile, nil) stopWatch() stopLimit() - // Teardown storage after workflow completes. The Docker backend is a - // no-op; future backends (temp volumes, NFS leases) use this hook. + d.teardownStorage(ctx, task, repo, workDir) + d.cleanupTerminalTaskWorktree(ctx, task, trees) +} + +// teardownStorage runs after the workflow completes. The Docker backend is +// a no-op; future backends (temp volumes, NFS leases) use this hook. +func (d *Daemon) teardownStorage(ctx context.Context, task *workflow.Task, repo config.Repo, workDir string) { if d.Storage != nil { _ = d.Storage.Teardown(ctx, storage.TaskRef{ WorktreeDir: workDir, @@ -1293,8 +1306,6 @@ func (d *Daemon) process(ctx context.Context, task *workflow.Task) { Repo: task.Repo, }) } - - d.cleanupTerminalTaskWorktree(ctx, task, trees) } // prepareWorkspace makes the directory the container binds as its @@ -1427,14 +1438,7 @@ func (d *Daemon) acquireTaskContainer( d.parkRunningTask(ctx, task.ID, reason+": "+err.Error(), taskstate.ParkTransient) } - // Write task.json -- the container's boot-time brief. - if err := container.WriteTaskJSON(workDir, container.TaskPayload{ - ID: task.ID, Owner: task.Owner, Repo: task.Repo, - Number: task.IssueNumber, Title: task.Title, Body: task.Body, - Labels: strings.Split(task.Labels, ","), - Workflow: task.Workflow, Branch: task.Branch, Plan: task.Plan, - Inputs: task.Inputs, - }); err != nil { + if err := writeTaskBrief(workDir, task); err != nil { park("task.json write failed", err) return nil, nil, false } @@ -1496,6 +1500,17 @@ func (d *Daemon) acquireTaskContainer( return ctr, revokeStateStoreGrant, true } +// writeTaskBrief writes task.json, the container's boot-time brief. +func writeTaskBrief(workDir string, task *workflow.Task) error { + return container.WriteTaskJSON(workDir, container.TaskPayload{ + ID: task.ID, Owner: task.Owner, Repo: task.Repo, + Number: task.IssueNumber, Title: task.Title, Body: task.Body, + Labels: strings.Split(task.Labels, ","), + Workflow: task.Workflow, Branch: task.Branch, Plan: task.Plan, + Inputs: task.Inputs, + }) +} + // parkCapabilityUnavailable parks a task whose daemon-side capability is // missing (container pool or task transport): both are environmental, so // the park class is transient and the KindParked event carries the class @@ -1588,7 +1603,7 @@ func (d *Daemon) recordPark(ctx context.Context, taskID int64, reason string) { // itself when nothing else could have: the request never reached (or was // never answered by) an archie-agent, or archie-agent failed before its // own workflow.Run got a chance to record an outcome. -func (d *Daemon) runViaAgent(ctx context.Context, task *workflow.Task, repo config.Repo, profile config.AgentProfile) { +func (d *Daemon) runViaAgent(ctx context.Context, task *workflow.Task, repo config.Repo, profile config.AgentProfile, harness *agentexec.HarnessSpec) { grant, revoke, ok := d.publicationGrant(ctx, task) if !ok { return @@ -1608,6 +1623,7 @@ func (d *Daemon) runViaAgent(ctx context.Context, task *workflow.Task, repo conf LabelWorkflows: d.LabelWorkflows, WorkflowDefinition: task.WorkflowDefinitionYAML, Tools: profile.Tools, + Harness: harness, } data, err := json.Marshal(req) if err != nil { diff --git a/internal/daemon/daemon_test.go b/internal/daemon/daemon_test.go index c766dc0cc..d34fecaef 100644 --- a/internal/daemon/daemon_test.go +++ b/internal/daemon/daemon_test.go @@ -19,6 +19,7 @@ import ( natssrv "github.com/nats-io/nats-server/v2/test" natsio "github.com/nats-io/nats.go" + "github.com/samcharles93/archie-core/internal/agentexec" "github.com/samcharles93/archie-core/internal/config" "github.com/samcharles93/archie-core/internal/container" "github.com/samcharles93/archie-core/internal/domain/workflow" @@ -27,6 +28,8 @@ import ( "github.com/samcharles93/archie-core/internal/forge" agentnats "github.com/samcharles93/archie-core/internal/infrastructure/agenttransport/nats" arnats "github.com/samcharles93/archie-core/internal/infrastructure/eventbus/nats" + "github.com/samcharles93/archie-core/internal/infrastructure/kit" + "github.com/samcharles93/archie-core/internal/infrastructure/kitrun" "github.com/samcharles93/archie-core/internal/infrastructure/postgres/pgstore" "github.com/samcharles93/archie-core/internal/logging" "github.com/samcharles93/archie-core/internal/secret" @@ -2330,7 +2333,7 @@ func TestRunViaAgentCarriesTheWorkflowProfile(t *testing.T) { if !ok || profile.Image != "agent-net:1" { t.Fatalf("pinTaskProfile() = %+v, %v; want the net profile", profile, ok) } - d.runViaAgent(ctx, task, config.Repo{Owner: "acme", Name: "widget", Base: "main"}, profile) + d.runViaAgent(ctx, task, config.Repo{Owner: "acme", Name: "widget", Base: "main"}, profile, nil) select { case req := <-received: if len(req.Tools) != 1 || req.Tools[0] != "whois" { @@ -2378,7 +2381,7 @@ func TestRunViaAgentParksWhenTheContainerExits(t *testing.T) { defer stop() done := make(chan struct{}) go func() { - d.runViaAgent(runCtx, task, config.Repo{Owner: "acme", Name: "widget", Base: "main"}, config.AgentProfile{}) + d.runViaAgent(runCtx, task, config.Repo{Owner: "acme", Name: "widget", Base: "main"}, config.AgentProfile{}, nil) close(done) }() @@ -2422,7 +2425,7 @@ func TestRunViaAgentParksWhenTheTaskTimeLimitPasses(t *testing.T) { defer stop() done := make(chan struct{}) go func() { - d.runViaAgent(runCtx, task, config.Repo{Owner: "acme", Name: "widget", Base: "main"}, config.AgentProfile{}) + d.runViaAgent(runCtx, task, config.Repo{Owner: "acme", Name: "widget", Base: "main"}, config.AgentProfile{}, nil) close(done) }() select { @@ -2436,3 +2439,85 @@ func TestRunViaAgentParksWhenTheTaskTimeLimitPasses(t *testing.T) { t.Fatalf("task = %+v, %v; want parked naming the time limit", got, err) } } + +type fakeKitLauncher struct { + got kitrun.Request + released bool + removed []kit.Volume +} + +func (f *fakeKitLauncher) Launch(_ context.Context, req kitrun.Request) (*kitrun.Run, error) { + f.got = req + return &kitrun.Run{ + Container: &container.Container{}, + Harness: agentexec.HarnessSpec{Adapter: agentexec.AdapterClaudeCode, Launch: []string{"claude"}}, + Volumes: []kit.Volume{{Name: "v", Path: "/home/agent/.claude"}}, + }, nil +} + +func (f *fakeKitLauncher) Release(context.Context, *kitrun.Run) error { + f.released = true + return nil +} + +func (f *fakeKitLauncher) RemoveVolumes(_ context.Context, v []kit.Volume) error { + f.removed = append(f.removed, v...) + return nil +} + +// A Kit profile starts the task through the Kit launcher, hands the worker +// the Kit's harness, keeps model provider keys out of the container, and +// removes the execution's volumes once the task is no longer retryable. +func TestKitProfileRunsTheTaskOnItsHarness(t *testing.T) { + d, s, busClient := daemonWithNATS(t) + launcher := &fakeKitLauncher{} + d.KitLauncher = launcher + d.ConnectedNATS.URL = "nats://172.17.0.1:4222" + t.Setenv("KIT_TEST_PROVIDER_KEY", "sk-real") + cfg := d.Cfg.Get() + cfg.Providers = map[string]config.Provider{"p": {APIKeyEnv: "KIT_TEST_PROVIDER_KEY"}} + d.Cfg.Set(cfg) + ctx := context.Background() + if _, err := s.EnqueueIssue(ctx, "acme", "widget", 31, "t", "b", "", ""); err != nil { + t.Fatal(err) + } + task, err := s.ClaimNext(ctx) + if err != nil || task == nil { + t.Fatalf("claim: (%v, %v)", task, err) + } + received := make(chan taskrun.Request, 1) + sub, err := mustCoreConn(t, busClient).Subscribe(agentnats.SubjectForTask(task.ID), func(msg *natsio.Msg) { + var req taskrun.Request + _ = json.Unmarshal(msg.Data, &req) + received <- req + data, _ := json.Marshal(taskrun.Response{Status: workflow.StatusPROpen}) + _ = msg.Respond(data) + }) + if err != nil { + t.Fatalf("subscribe: %v", err) + } + t.Cleanup(func() { _ = sub.Unsubscribe() }) + + profile := config.AgentProfile{Kit: []string{"kit@sha256:ab"}, Adapter: agentexec.AdapterClaudeCode} + d.runKitTask(ctx, task, config.Repo{Owner: "acme", Name: "widget", Base: "main"}, t.TempDir(), profile) + + select { + case req := <-received: + if req.Harness == nil || req.Harness.Adapter != agentexec.AdapterClaudeCode { + t.Fatalf("request harness = %+v, want the Kit's", req.Harness) + } + case <-time.After(time.Second): + t.Fatal("archied did not publish a taskrun request") + } + if !slices.Equal(launcher.got.Kit, profile.Kit) || launcher.got.Adapter != profile.Adapter { + t.Fatalf("launch request %+v, want the profile's kits and adapter", launcher.got) + } + for _, e := range launcher.got.WorkerEnv { + if strings.Contains(e, "sk-real") { + t.Fatalf("worker env carries a model provider key: %q", e) + } + } + if !launcher.released || len(launcher.removed) != 1 { + t.Fatalf("released %v, removed %v; want the run released and its volumes removed", launcher.released, launcher.removed) + } +} diff --git a/internal/daemon/kit_task.go b/internal/daemon/kit_task.go new file mode 100644 index 000000000..53afb1178 --- /dev/null +++ b/internal/daemon/kit_task.go @@ -0,0 +1,94 @@ +package daemon + +import ( + "context" + "fmt" + "os" + "time" + + "github.com/samcharles93/archie-core/internal/config" + "github.com/samcharles93/archie-core/internal/domain/workflow" + "github.com/samcharles93/archie-core/internal/infrastructure/kit" + "github.com/samcharles93/archie-core/internal/infrastructure/kitrun" + "github.com/samcharles93/archie-core/internal/taskstate" +) + +// KitLauncher starts and ends the container of a task whose agent profile +// is a Kit. +type KitLauncher interface { + Launch(ctx context.Context, req kitrun.Request) (*kitrun.Run, error) + Release(ctx context.Context, run *kitrun.Run) error + RemoveVolumes(ctx context.Context, volumes []kit.Volume) error +} + +// runKitTask runs a task in its Kit's container, every agent stage on the +// Kit's harness. The worker reaches NATS and the State Store only through +// the egress relay, and its environment carries no model provider key. +func (d *Daemon) runKitTask(ctx context.Context, task *workflow.Task, repo config.Repo, workDir string, profile config.AgentProfile) { + park := func(reason string, err error) { + d.Log.Error(reason, "task", task.ID, "err", err) + d.parkRunningTask(ctx, task.ID, reason+": "+err.Error(), taskstate.ParkTransient) + } + if d.KitLauncher == nil { + park("kit profile", fmt.Errorf("kit harness runs are unavailable on this daemon")) + return + } + if err := writeTaskBrief(workDir, task); err != nil { + park("task.json write failed", err) + return + } + stateStoreToken, revokeStateStoreGrant, err := d.stateStoreGrantToken(task) + if err != nil { + park("state store grant failed", err) + return + } + defer revokeStateStoreGrant() + env, err := kitrun.WorkerEnv(kitrun.Endpoints{ + NATS: d.ConnectedNATS.URL, NATSToken: d.ConnectedNATS.Token, + StateStore: d.ConnectedStateStore.URL, StateStoreToken: stateStoreToken, + }, os.Getuid(), os.Getgid()) + if err != nil { + park("kit worker environment", err) + return + } + run, err := d.KitLauncher.Launch(ctx, kitrun.Request{ + Execution: fmt.Sprintf("task-%d", task.ID), + Kit: profile.Kit, + Adapter: profile.Adapter, + WorkDir: workDir, + WorkerEnv: env, + }) + if err != nil { + park("kit launch failed", err) + return + } + + limitCtx, stopLimit := withTaskTimeLimit(ctx, d.configFor(task).Budgets.TaskWallClock.Std()) + runCtx, stopWatch := withContainerExit(limitCtx, run.Container.Exited()) + d.runViaAgent(runCtx, task, repo, profile, &run.Harness) + stopWatch() + stopLimit() + + if err := d.KitLauncher.Release(ctx, run); err != nil { + d.Log.Warn("kit release failed", "task", task.ID, "err", err) + } + d.removeEndedKitVolumes(ctx, task.ID, run.Volumes) +} + +// removeEndedKitVolumes removes the execution's Kit volumes unless the task +// parked: a retry of a parked task is the same execution and resumes from +// them. +func (d *Daemon) removeEndedKitVolumes(ctx context.Context, taskID int64, volumes []kit.Volume) { + if len(volumes) == 0 || d.Store == nil { + return + } + lookupCtx, cancel := context.WithTimeout(context.WithoutCancel(ctx), 10*time.Second) + defer cancel() + latest, err := d.Store.TaskByID(lookupCtx, taskID) + if err != nil || latest == nil || latest.Status == workflow.StatusParked { + return + } + if err := d.KitLauncher.RemoveVolumes(lookupCtx, volumes); err != nil { + d.Log.Warn("kit volume removal failed", "task", taskID, "err", err) + } +} diff --git a/internal/daemon/scratch_task_test.go b/internal/daemon/scratch_task_test.go index 487545a25..a4bffe0cf 100644 --- a/internal/daemon/scratch_task_test.go +++ b/internal/daemon/scratch_task_test.go @@ -42,5 +42,5 @@ func runPinnedViaAgent(ctx context.Context, d *Daemon, task *workflow.Task, repo if !ok { return } - d.runViaAgent(ctx, task, repo, profile) + d.runViaAgent(ctx, task, repo, profile, nil) } diff --git a/internal/infrastructure/egress/ca.go b/internal/infrastructure/egress/ca.go index 351705ed9..895a8360c 100644 --- a/internal/infrastructure/egress/ca.go +++ b/internal/infrastructure/egress/ca.go @@ -40,6 +40,10 @@ type CA struct { leaves map[string]*tls.Certificate } +// CACertPath is where LoadOrCreateCA keeps the CA certificate in dir, the +// file a sandbox container mounts to trust the proxy. +func CACertPath(dir string) string { return filepath.Join(dir, caCertFile) } + // LoadOrCreateCA loads the CA from dir, creating it on first use. A CA that // exists but will not load is an error: minting a replacement silently // would break trust in every running container. diff --git a/internal/infrastructure/kit/kit.go b/internal/infrastructure/kit/kit.go index 6615beb25..c9b9604a3 100644 --- a/internal/infrastructure/kit/kit.go +++ b/internal/infrastructure/kit/kit.go @@ -129,10 +129,19 @@ func Admit(d *spec.Descriptor) (*Plan, error) { // admits the result. archie drives a harness headlessly, so the composition // must carry an agent-sessions prompt verb. func Compose(contributions []spec.Contribution) (*Plan, error) { - merged, err := spec.Merge(contributions, spec.MergeOptions{ContextPath: contextPath}) + merged, err := spec.Merge(contributions, MergeOptions) if err != nil { return nil, err } + return FromMerge(merged) +} + +// MergeOptions are the options every composition archie runs is merged with. +var MergeOptions = spec.MergeOptions{ContextPath: contextPath} + +// FromMerge admits an already-merged composition, such as one the Kit spec's +// fetch package assembled from a registry. +func FromMerge(merged *spec.MergeResult) (*Plan, error) { if merged.Descriptor.Kind != spec.KindWorkload { return nil, errors.New("kit composition has no workload: a harness needs exactly one base kit") } diff --git a/internal/infrastructure/kitrun/env.go b/internal/infrastructure/kitrun/env.go new file mode 100644 index 000000000..bc08bcfa5 --- /dev/null +++ b/internal/infrastructure/kitrun/env.go @@ -0,0 +1,72 @@ +package kitrun + +import ( + "fmt" + "net" + "net/url" + "strconv" + "strings" + + "github.com/samcharles93/archie-core/internal/infrastructure/egress" +) + +// Endpoints are the host-reachable addresses a Kit task's worker needs, +// with the task's own credentials for them. +type Endpoints struct { + NATS string + NATSToken string + StateStore string + StateStoreToken string +} + +// WorkerEnv is the worker's entire environment in a Kit container: NATS and +// the State Store at the relay's fixed ports, and the task's credentials. +// Nothing else crosses, so no model provider key enters the container. +func WorkerEnv(e Endpoints, uid, gid int) ([]string, error) { + u, err := url.Parse(e.NATS) + if err != nil || u.Scheme == "" { + return nil, fmt.Errorf("nats url %q: want scheme://host:port", e.NATS) + } + env := []string{"NATS_URL=" + u.Scheme + "://" + relayAddr(egress.RelayNATSPort)} + if e.NATSToken != "" { + env = append(env, "NATS_TOKEN="+e.NATSToken) + } + if e.StateStore != "" { + env = append(env, "STATE_STORE_URL="+relayAddr(egress.RelayStateStorePort)) + if e.StateStoreToken != "" { + env = append(env, "STATE_STORE_TOKEN="+e.StateStoreToken) + } + } + return append(env, "WORKTREE_UID="+strconv.Itoa(uid), "WORKTREE_GID="+strconv.Itoa(gid)), nil +} + +// relayTargets are the host:port addresses the relay forwards NATS and the +// State Store to. +func relayTargets(e Endpoints) (nats, stateStore string, err error) { + if strings.Contains(e.NATS, ",") { + return "", "", fmt.Errorf("nats url %q lists several servers; the relay forwards one", e.NATS) + } + u, err := url.Parse(e.NATS) + if err != nil || u.Host == "" { + return "", "", fmt.Errorf("nats url %q: want scheme://host:port", e.NATS) + } + stateStore = e.StateStore + if _, rest, ok := strings.Cut(stateStore, ":///"); ok { + stateStore = rest + } + return u.Host, stateStore, nil +} + +func relayAddr(port int) string { + return net.JoinHostPort(egress.RelayAlias, strconv.Itoa(port)) +} + +// RelayCommand is the relay's archie-agent arguments, forwarding its fixed +// ports to the proxy and to the daemon's NATS and State Store. +func RelayCommand(proxy string, e Endpoints) ([]string, error) { + nats, stateStore, err := relayTargets(e) + if err != nil { + return nil, err + } + return egress.RelayArgs(proxy, nats, stateStore), nil +} diff --git a/internal/infrastructure/kitrun/env_test.go b/internal/infrastructure/kitrun/env_test.go new file mode 100644 index 000000000..38f9424e5 --- /dev/null +++ b/internal/infrastructure/kitrun/env_test.go @@ -0,0 +1,44 @@ +package kitrun + +import ( + "slices" + "testing" +) + +func TestWorkerEnvReachesNATSAndTheStateStoreThroughTheRelay(t *testing.T) { + got, err := WorkerEnv(Endpoints{ + NATS: "nats://172.17.0.1:4222", NATSToken: "nt", + StateStore: "10.0.0.5:7443", StateStoreToken: "st", + }, 1001, 1002) + if err != nil { + t.Fatal(err) + } + want := []string{ + "NATS_URL=nats://archie-egress:4222", "NATS_TOKEN=nt", + "STATE_STORE_URL=archie-egress:50051", "STATE_STORE_TOKEN=st", + "WORKTREE_UID=1001", "WORKTREE_GID=1002", + } + if !slices.Equal(got, want) { + t.Fatalf("WorkerEnv = %q\nwant %q", got, want) + } +} + +func TestWorkerEnvWithoutAStateStoreOmitsIt(t *testing.T) { + got, err := WorkerEnv(Endpoints{NATS: "tls://nats.internal:4222"}, 0, 0) + if err != nil { + t.Fatal(err) + } + if !slices.Equal(got, []string{"NATS_URL=tls://archie-egress:4222", "WORKTREE_UID=0", "WORKTREE_GID=0"}) { + t.Fatalf("WorkerEnv = %q", got) + } +} + +func TestRelayTargets(t *testing.T) { + nats, store, err := relayTargets(Endpoints{NATS: "nats://172.17.0.1:4222", StateStore: "dns:///store.lan:7443"}) + if err != nil || nats != "172.17.0.1:4222" || store != "store.lan:7443" { + t.Fatalf("relayTargets = %q, %q, %v", nats, store, err) + } + if _, _, err := relayTargets(Endpoints{NATS: "nats://a:1,nats://b:2"}); err == nil { + t.Fatal("a NATS cluster URL list was accepted; the relay forwards one address") + } +} diff --git a/internal/infrastructure/kitrun/launcher.go b/internal/infrastructure/kitrun/launcher.go new file mode 100644 index 000000000..f3f9f084a --- /dev/null +++ b/internal/infrastructure/kitrun/launcher.go @@ -0,0 +1,195 @@ +// Package kitrun starts a Kit task: it composes the profile's Kits, opens +// the task's egress session and sandbox network, and starts the container +// with archie's worker as root and the harness spec the worker runs stages on. +package kitrun + +import ( + "context" + "errors" + "fmt" + "maps" + "slices" + "sync" + + "github.com/docker/sandbox-kit-spec/v3/fetch" + "github.com/docker/sandbox-kit-spec/v3/spec" + + "github.com/samcharles93/archie-core/internal/agentexec" + "github.com/samcharles93/archie-core/internal/container" + "github.com/samcharles93/archie-core/internal/infrastructure/egress" + "github.com/samcharles93/archie-core/internal/infrastructure/kit" +) + +const ( + agentPath = "/opt/archie/archie-agent" + caPath = "/opt/archie/ca.pem" +) + +// Launcher starts Kit task containers. +type Launcher struct { + Pool *container.Pool + Fetch *fetch.Client + Proxy *egress.Proxy + Networks *egress.Networks + // AgentBinary and CAFile are host paths mounted read-only into every + // Kit container: the worker's own binary and the egress CA. + AgentBinary string + CAFile string + + mu sync.Mutex + started bool +} + +// Request is one Kit task to start. +type Request struct { + // Execution keys the task's Kit volumes and names its container and + // network. + Execution string + Kit []string + Adapter string + WorkDir string + WorkerEnv []string +} + +// Run is a started Kit task. +type Run struct { + Container *container.Container + Harness agentexec.HarnessSpec + // Volumes outlive the run; remove them with RemoveVolumes once the + // execution has ended. + Volumes []kit.Volume + network string + token string +} + +// Launch composes the request's Kits and starts its container. +func (l *Launcher) Launch(ctx context.Context, req Request) (*Run, error) { + if len(req.Kit) == 0 { + return nil, errors.New("kit profile names no kit") + } + adapter, ok := agentexec.LookupHarnessAdapter(req.Adapter) + if req.Adapter != "" && !ok { + return nil, fmt.Errorf("harness output adapter %q is unknown", req.Adapter) + } + if err := l.startEgress(ctx); err != nil { + return nil, err + } + plan, img, err := l.compose(ctx, req.Kit) + if err != nil { + return nil, err + } + network, err := spec.NetworkPolicyOf(plan.Capabilities) + if err != nil { + return nil, err + } + creds, err := spec.CredentialsOf(plan.Capabilities) + if err != nil { + return nil, err + } + session, err := l.Proxy.Register(egress.SessionOptions{Run: req.Execution, Network: network, Credentials: creds}) + if err != nil { + return nil, err + } + run := &Run{network: "archie-kit-" + req.Execution, token: session.Token()} + launch, err := kit.Assemble(plan, img, kit.LaunchParams{Execution: req.Execution, ProxyToken: session.Token(), CAPath: caPath}) + if err != nil { + l.Proxy.Revoke(run.token) + return nil, err + } + launch.Harness.Adapter, launch.Harness.MCPConfig = req.Adapter, adapter.MCPConfig + run.Harness, run.Volumes = launch.Harness, launch.Volumes + if err := l.Networks.Create(ctx, run.network); err != nil { + l.Proxy.Revoke(run.token) + return nil, err + } + run.Container, err = l.Pool.AcquireKit(ctx, container.KitSpec{ + Name: "archie-kit-" + req.Execution, + Image: req.Kit[0], + Network: run.network, + Worker: []string{agentPath}, + WorkerEnv: req.WorkerEnv, + Binds: []string{ + l.AgentBinary + ":" + agentPath + ":ro", + l.CAFile + ":" + caPath + ":ro", + req.WorkDir + ":" + kit.WorkspaceDir, + }, + Launch: launch, + InstallDone: session.EnterRuntime, + }) + if err != nil { + l.Proxy.Revoke(run.token) + return nil, errors.Join(err, l.Networks.Remove(context.WithoutCancel(ctx), run.network)) + } + return run, nil +} + +// compose assembles the profile's Kits and reads the workload image's +// config, the first Kit being the workload. +func (l *Launcher) compose(ctx context.Context, refs []string) (*kit.Plan, kit.ImageConfig, error) { + reqs := make([]fetch.Request, len(refs)) + for i, ref := range refs { + reqs[i] = fetch.Request{Reference: ref} + } + merged, err := l.Fetch.Assemble(ctx, reqs, kit.MergeOptions) + if err != nil { + return nil, kit.ImageConfig{}, fmt.Errorf("assemble kits: %w", err) + } + plan, err := kit.FromMerge(merged.MergeResult) + if err != nil { + return nil, kit.ImageConfig{}, err + } + img, err := l.imageConfig(ctx, refs[0]) + if err != nil { + return nil, kit.ImageConfig{}, err + } + for _, name := range slices.Sorted(maps.Keys(merged.Env)) { + img.Env = append(img.Env, name+"="+merged.Env[name]) + } + return plan, img, nil +} + +// Release stops the run's container, then removes its network and ends +// its egress session. +func (l *Launcher) Release(ctx context.Context, run *Run) error { + l.Pool.Release(ctx, run.Container) + l.Proxy.Revoke(run.token) + return l.Networks.Remove(context.WithoutCancel(ctx), run.network) +} + +// RemoveVolumes removes an ended execution's Kit volumes. +func (l *Launcher) RemoveVolumes(ctx context.Context, volumes []kit.Volume) error { + return container.RemoveKitVolumes(ctx, l.Pool.Client(), volumes) +} + +// startEgress starts the relay on first use, so a daemon with no Kit +// profile runs no relay. +func (l *Launcher) startEgress(ctx context.Context) error { + l.mu.Lock() + defer l.mu.Unlock() + if l.started { + return nil + } + if err := l.Networks.Start(ctx); err != nil { + return err + } + l.started = true + return nil +} + +func (l *Launcher) imageConfig(ctx context.Context, ref string) (kit.ImageConfig, error) { + if err := l.Pool.EnsureImage(ctx, ref); err != nil { + return kit.ImageConfig{}, err + } + inspected, err := l.Pool.Client().ImageInspect(ctx, ref) + if err != nil { + return kit.ImageConfig{}, fmt.Errorf("inspect kit image %s: %w", ref, err) + } + cfg := inspected.Config + if cfg == nil { + return kit.ImageConfig{}, fmt.Errorf("kit image %s has no config", ref) + } + return kit.ImageConfig{ + Entrypoint: slices.Clone(cfg.Entrypoint), Cmd: slices.Clone(cfg.Cmd), + Env: slices.Clone(cfg.Env), User: cfg.User, + }, nil +}