feat(daemon): run a Kit profile's task in its Kit container on the egress path
This commit is contained in:
17 files changed
+734
-42
No files matched your search
@@ -90,6 +90,7 @@ require (
|
|||||||
google.golang.org/genproto/googleapis/rpc v0.0.0-20260921155816-b14227669459 // indirect
|
google.golang.org/genproto/googleapis/rpc v0.0.0-20260921155816-b14227669459 // indirect
|
||||||
modernc.org/libc v1.77.1 // indirect
|
modernc.org/libc v1.77.1 // indirect
|
||||||
modernc.org/sqlite v1.59.0 // indirect
|
modernc.org/sqlite v1.59.0 // indirect
|
||||||
|
oras.land/oras-go/v2 v2.6.2 // indirect
|
||||||
)
|
)
|
||||||
|
|
||||||
require (
|
require (
|
||||||
|
|||||||
@@ -325,5 +325,7 @@ modernc.org/memory v1.12.1 h1:nFMiWrpStgZczNl6XI9GnIk/rWhYIyHGUaR04pGbp9g=
|
|||||||
modernc.org/memory v1.12.1/go.mod h1:/JP4VbVC+K5sU2wZi9bHoq2MAkCnrt2r98UGeSK7Mjw=
|
modernc.org/memory v1.12.1/go.mod h1:/JP4VbVC+K5sU2wZi9bHoq2MAkCnrt2r98UGeSK7Mjw=
|
||||||
modernc.org/sqlite v1.59.0 h1:X1es1GpqBlS/5T+vbM4HLUdaa8OtQx468DF2vrx+38A=
|
modernc.org/sqlite v1.59.0 h1:X1es1GpqBlS/5T+vbM4HLUdaa8OtQx468DF2vrx+38A=
|
||||||
modernc.org/sqlite v1.59.0/go.mod h1:+paeT2A3iPRHkQDwG7oA6Tk0zQd5woMEI8q7orfry8k=
|
modernc.org/sqlite v1.59.0/go.mod h1:+paeT2A3iPRHkQDwG7oA6Tk0zQd5woMEI8q7orfry8k=
|
||||||
|
oras.land/oras-go/v2 v2.6.2 h1:N04RXngAp1LJKTG6ifz3xHPipasEkWr+hFmInja5YKo=
|
||||||
|
oras.land/oras-go/v2 v2.6.2/go.mod h1:PlTtg4JTDJkDe8yVHpM2wz7/YDc00GVas+i4jAW2TZ4=
|
||||||
pgregory.net/rapid v1.2.0 h1:keKAYRcjm+e1F0oAuU5F5+YPAWcyxNNRK2wud503Gnk=
|
pgregory.net/rapid v1.2.0 h1:keKAYRcjm+e1F0oAuU5F5+YPAWcyxNNRK2wud503Gnk=
|
||||||
pgregory.net/rapid v1.2.0/go.mod h1:PY5XlDGj0+V1FCq0o192FdRhpKHGTRIWBgqjDBTrq04=
|
pgregory.net/rapid v1.2.0/go.mod h1:PY5XlDGj0+V1FCq0o192FdRhpKHGTRIWBgqjDBTrq04=
|
||||||
@@ -22,3 +22,6 @@ func runAsHarnessUser(_ *exec.Cmd, name string) error {
|
|||||||
|
|
||||||
// ownForHarness has nothing to do where the harness runs as the worker.
|
// ownForHarness has nothing to do where the harness runs as the worker.
|
||||||
func ownForHarness(string, ...string) error { return nil }
|
func ownForHarness(string, ...string) error { return nil }
|
||||||
|
|
||||||
|
// ownTreeForHarness has nothing to do where the harness runs as the worker.
|
||||||
|
func ownTreeForHarness(string, string) error { return nil }
|
||||||
@@ -5,9 +5,11 @@ package agentexec
|
|||||||
import (
|
import (
|
||||||
"errors"
|
"errors"
|
||||||
"fmt"
|
"fmt"
|
||||||
|
"io/fs"
|
||||||
"os"
|
"os"
|
||||||
"os/exec"
|
"os/exec"
|
||||||
"os/user"
|
"os/user"
|
||||||
|
"path/filepath"
|
||||||
"strconv"
|
"strconv"
|
||||||
"strings"
|
"strings"
|
||||||
"syscall"
|
"syscall"
|
||||||
@@ -91,3 +93,25 @@ func ownForHarness(name string, paths ...string) error {
|
|||||||
}
|
}
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// ownTreeForHarness gives the worktree to the harness user when the worker
|
||||||
|
// is root, except .git: the harness edits files and never writes the
|
||||||
|
// repository, so its refs and config stay the worker's.
|
||||||
|
func ownTreeForHarness(name, root string) error {
|
||||||
|
if name == "" || os.Geteuid() != 0 {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
cred, err := harnessCredential(name)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
return filepath.WalkDir(root, func(path string, d fs.DirEntry, err error) error {
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
if d.IsDir() && d.Name() == ".git" && path != root {
|
||||||
|
return filepath.SkipDir
|
||||||
|
}
|
||||||
|
return os.Lchown(path, int(cred.Uid), int(cred.Gid))
|
||||||
|
})
|
||||||
|
}
|
||||||
@@ -69,6 +69,9 @@ func (r *HarnessRunner) Run(ctx context.Context, workspace string, req Request,
|
|||||||
if err := validateHarness(req); err != nil {
|
if err := validateHarness(req); err != nil {
|
||||||
return Result{}, err
|
return Result{}, err
|
||||||
}
|
}
|
||||||
|
if err := ownTreeForHarness(req.Harness.User, workspace); err != nil {
|
||||||
|
return Result{}, fmt.Errorf("give the worktree to the harness user: %w", err)
|
||||||
|
}
|
||||||
mcpConfig, capturesPath, cleanup, err := r.prepareCaptures(req)
|
mcpConfig, capturesPath, cleanup, err := r.prepareCaptures(req)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return Result{}, err
|
return Result{}, err
|
||||||
|
|||||||
@@ -208,7 +208,10 @@ type boot struct {
|
|||||||
natsToken string
|
natsToken string
|
||||||
|
|
||||||
containerPool *container.Pool
|
containerPool *container.Pool
|
||||||
storeBackend storage.Backend
|
// kitLauncher starts Kit profile tasks; nil when the egress path cannot
|
||||||
|
// be built on this host.
|
||||||
|
kitLauncher daemon.KitLauncher
|
||||||
|
storeBackend storage.Backend
|
||||||
|
|
||||||
llm *runtime.Runtime
|
llm *runtime.Runtime
|
||||||
|
|
||||||
@@ -1332,6 +1335,7 @@ func (b *boot) buildDaemon() {
|
|||||||
WorktreeGrants: b.worktreeGrants,
|
WorktreeGrants: b.worktreeGrants,
|
||||||
StateStoreGrants: b.stateStoreGrants,
|
StateStoreGrants: b.stateStoreGrants,
|
||||||
ContainerPool: b.containerPool,
|
ContainerPool: b.containerPool,
|
||||||
|
KitLauncher: b.kitLauncher,
|
||||||
Guardrails: b.guardrails,
|
Guardrails: b.guardrails,
|
||||||
ToolRegistry: b.toolReg,
|
ToolRegistry: b.toolReg,
|
||||||
Identities: b.identityRunners,
|
Identities: b.identityRunners,
|
||||||
@@ -1609,6 +1613,9 @@ func (b *boot) startServices(ctx context.Context) error {
|
|||||||
func (b *boot) setupBackends(ctx context.Context) error {
|
func (b *boot) setupBackends(ctx context.Context) error {
|
||||||
closeContainers := b.setupContainers(ctx)
|
closeContainers := b.setupContainers(ctx)
|
||||||
err := b.connectNATS(ctx)
|
err := b.connectNATS(ctx)
|
||||||
|
if err == nil {
|
||||||
|
b.setupKitLauncher(ctx)
|
||||||
|
}
|
||||||
// Container discovery must precede embedded NATS so it can bind the
|
// Container discovery must precede embedded NATS so it can bind the
|
||||||
// worker bridge gateway, but cleanup registration comes afterwards.
|
// worker bridge gateway, but cleanup registration comes afterwards.
|
||||||
// cleanups run LIFO: workers stop first, then the daemon client closes,
|
// cleanups run LIFO: workers stop first, then the daemon client closes,
|
||||||
|
|||||||
@@ -0,0 +1,96 @@
|
|||||||
|
package archied
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"errors"
|
||||||
|
"log/slog"
|
||||||
|
"net"
|
||||||
|
"net/http"
|
||||||
|
"os"
|
||||||
|
"path/filepath"
|
||||||
|
"strings"
|
||||||
|
"time"
|
||||||
|
|
||||||
|
"github.com/docker/sandbox-kit-spec/v3/fetch"
|
||||||
|
|
||||||
|
"github.com/samcharles93/archie-core/internal/config"
|
||||||
|
"github.com/samcharles93/archie-core/internal/infrastructure/egress"
|
||||||
|
"github.com/samcharles93/archie-core/internal/infrastructure/kitrun"
|
||||||
|
)
|
||||||
|
|
||||||
|
const relayName = "archie-egress-relay"
|
||||||
|
|
||||||
|
// setupKitLauncher builds the egress path Kit tasks run on: the per-daemon
|
||||||
|
// CA, the proxy on the worker bridge's host gateway, and the relay. It
|
||||||
|
// degrades to no launcher, which parks Kit tasks, rather than failing boot:
|
||||||
|
// image profiles do not need it.
|
||||||
|
func (b *boot) setupKitLauncher(ctx context.Context) {
|
||||||
|
pool, log := b.containerPool, b.log
|
||||||
|
if pool == nil || pool.HostGateway() == "" {
|
||||||
|
log.Info("kit harness runs disabled: they need the managed container pool on a bridge with a host gateway")
|
||||||
|
return
|
||||||
|
}
|
||||||
|
exe, err := os.Executable()
|
||||||
|
if err != nil {
|
||||||
|
log.Warn("kit harness runs disabled", "err", err)
|
||||||
|
return
|
||||||
|
}
|
||||||
|
agentBinary := filepath.Join(filepath.Dir(exe), "archie-agent")
|
||||||
|
caDir := filepath.Join(b.cfg.StateDir, "egress")
|
||||||
|
ca, err := egress.LoadOrCreateCA(caDir)
|
||||||
|
if err != nil {
|
||||||
|
log.Warn("kit harness runs disabled: egress CA", "err", err)
|
||||||
|
return
|
||||||
|
}
|
||||||
|
fetcher, err := fetch.New(fetch.WithDockerCredentials())
|
||||||
|
if err != nil {
|
||||||
|
log.Warn("kit harness runs disabled: kit registry client", "err", err)
|
||||||
|
return
|
||||||
|
}
|
||||||
|
ln, err := (&net.ListenConfig{}).Listen(ctx, "tcp", net.JoinHostPort(pool.HostGateway(), "0"))
|
||||||
|
if err != nil {
|
||||||
|
log.Warn("kit harness runs disabled: egress proxy listener", "err", err)
|
||||||
|
return
|
||||||
|
}
|
||||||
|
endpoints := kitrun.Endpoints{NATS: b.natsURL, StateStore: strings.TrimSpace(b.cfg.Services.Get(config.ServiceNameState).Target)}
|
||||||
|
cmd, err := kitrun.RelayCommand(ln.Addr().String(), endpoints)
|
||||||
|
if err != nil {
|
||||||
|
_ = ln.Close()
|
||||||
|
log.Warn("kit harness runs disabled: egress relay", "err", err)
|
||||||
|
return
|
||||||
|
}
|
||||||
|
// No credential resolver yet (archie-core-egkf.11): a Kit's required
|
||||||
|
// credential is refused at the proxy until run credentials carry secrets.
|
||||||
|
proxy := egress.NewProxy(ca, egress.ProxyOptions{})
|
||||||
|
srv := &http.Server{Handler: proxy, ReadHeaderTimeout: 30 * time.Second}
|
||||||
|
go func() {
|
||||||
|
if err := srv.Serve(ln); err != nil && !errors.Is(err, http.ErrServerClosed) {
|
||||||
|
log.Error("egress proxy stopped", "err", err)
|
||||||
|
}
|
||||||
|
}()
|
||||||
|
networks := egress.NewNetworks(pool.Client(), egress.RelaySpec{
|
||||||
|
Name: relayName, Image: b.cfg.Containers.Image,
|
||||||
|
Entrypoint: []string{"archie-agent"}, Cmd: cmd, Network: pool.NetworkName(),
|
||||||
|
})
|
||||||
|
b.addCleanup(shutdownEgress(networks, srv, log))
|
||||||
|
b.kitLauncher = &kitrun.Launcher{
|
||||||
|
Pool: pool, Fetch: fetcher, Proxy: proxy, Networks: networks,
|
||||||
|
AgentBinary: agentBinary, CAFile: egress.CACertPath(caDir),
|
||||||
|
}
|
||||||
|
log.Info("kit harness runs enabled", "proxy", ln.Addr().String())
|
||||||
|
}
|
||||||
|
|
||||||
|
// shutdownEgress removes the relay and stops the proxy. It derives its own
|
||||||
|
// budget because the boot context is cancelled by the time shutdown runs.
|
||||||
|
//
|
||||||
|
//nolint:contextcheck
|
||||||
|
func shutdownEgress(networks *egress.Networks, srv *http.Server, log *slog.Logger) func() {
|
||||||
|
return func() {
|
||||||
|
stopCtx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
|
||||||
|
defer cancel()
|
||||||
|
if err := networks.Close(stopCtx); err != nil {
|
||||||
|
log.Warn("egress relay removal failed", "err", err)
|
||||||
|
}
|
||||||
|
_ = srv.Close()
|
||||||
|
}
|
||||||
|
}
|
||||||
+59
-22
@@ -10,6 +10,7 @@ import (
|
|||||||
"fmt"
|
"fmt"
|
||||||
"io"
|
"io"
|
||||||
"log/slog"
|
"log/slog"
|
||||||
|
"maps"
|
||||||
"os"
|
"os"
|
||||||
"path/filepath"
|
"path/filepath"
|
||||||
"sync"
|
"sync"
|
||||||
@@ -195,16 +196,12 @@ func (p *Pool) Acquire(ctx context.Context, image string, mounts []storage.Mount
|
|||||||
if image == "" {
|
if image == "" {
|
||||||
image = p.cfg.Image
|
image = p.cfg.Image
|
||||||
}
|
}
|
||||||
if err := p.ensureImage(ctx, image); err != nil {
|
if err := p.EnsureImage(ctx, image); err != nil {
|
||||||
return nil, err
|
return nil, err
|
||||||
}
|
}
|
||||||
p.mu.Lock()
|
if err := p.reserve(); err != nil {
|
||||||
if p.cfg.MaxConcurrency > 0 && p.active >= p.cfg.MaxConcurrency {
|
return nil, err
|
||||||
p.mu.Unlock()
|
|
||||||
return nil, fmt.Errorf("max concurrency %d reached", p.cfg.MaxConcurrency)
|
|
||||||
}
|
}
|
||||||
p.active++
|
|
||||||
p.mu.Unlock()
|
|
||||||
|
|
||||||
name := fmt.Sprintf("archie-agent-%d", time.Now().UnixNano())
|
name := fmt.Sprintf("archie-agent-%d", time.Now().UnixNano())
|
||||||
|
|
||||||
@@ -232,9 +229,7 @@ func (p *Pool) Acquire(ctx context.Context, image string, mounts []storage.Mount
|
|||||||
HostConfig: hostConfig,
|
HostConfig: hostConfig,
|
||||||
})
|
})
|
||||||
if err != nil {
|
if err != nil {
|
||||||
p.mu.Lock()
|
p.unreserve()
|
||||||
p.active--
|
|
||||||
p.mu.Unlock()
|
|
||||||
return nil, fmt.Errorf("container create: %w", err)
|
return nil, fmt.Errorf("container create: %w", err)
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -243,20 +238,62 @@ func (p *Pool) Acquire(ctx context.Context, image string, mounts []storage.Mount
|
|||||||
if _, rmErr := p.cli.ContainerRemove(context.WithoutCancel(ctx), resp.ID, client.ContainerRemoveOptions{Force: true}); rmErr != nil {
|
if _, rmErr := p.cli.ContainerRemove(context.WithoutCancel(ctx), resp.ID, client.ContainerRemoveOptions{Force: true}); rmErr != nil {
|
||||||
p.log.Warn("container remove after start failure", "id", resp.ID[:12], "err", rmErr)
|
p.log.Warn("container remove after start failure", "id", resp.ID[:12], "err", rmErr)
|
||||||
}
|
}
|
||||||
p.mu.Lock()
|
p.unreserve()
|
||||||
p.active--
|
|
||||||
p.mu.Unlock()
|
|
||||||
return nil, fmt.Errorf("container start: %w", err)
|
return nil, fmt.Errorf("container start: %w", err)
|
||||||
}
|
}
|
||||||
|
return p.started(ctx, resp.ID, name), nil
|
||||||
if p.cfg.MaxUptime > 0 {
|
|
||||||
p.armMaxUptime(ctx, resp.ID)
|
|
||||||
}
|
|
||||||
|
|
||||||
p.log.Info("container started", "id", resp.ID[:12], "name", name)
|
|
||||||
return &Container{ID: resp.ID, exited: p.watchExit(ctx, resp.ID)}, nil
|
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// AcquireKit starts a Kit task container under the same concurrency cap,
|
||||||
|
// lifetime cap and exit watch as Acquire. Release tears it down.
|
||||||
|
func (p *Pool) AcquireKit(ctx context.Context, s KitSpec) (*Container, error) {
|
||||||
|
if err := p.EnsureImage(ctx, s.Image); err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
if err := p.reserve(); err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
s.Labels = maps.Clone(s.Labels)
|
||||||
|
if s.Labels == nil {
|
||||||
|
s.Labels = map[string]string{}
|
||||||
|
}
|
||||||
|
s.Labels["archie-daemon"] = "true"
|
||||||
|
id, err := StartKit(ctx, p.cli, s)
|
||||||
|
if err != nil {
|
||||||
|
p.unreserve()
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
return p.started(ctx, id, s.Name), nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func (p *Pool) reserve() error {
|
||||||
|
p.mu.Lock()
|
||||||
|
defer p.mu.Unlock()
|
||||||
|
if p.cfg.MaxConcurrency > 0 && p.active >= p.cfg.MaxConcurrency {
|
||||||
|
return fmt.Errorf("max concurrency %d reached", p.cfg.MaxConcurrency)
|
||||||
|
}
|
||||||
|
p.active++
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func (p *Pool) unreserve() {
|
||||||
|
p.mu.Lock()
|
||||||
|
p.active--
|
||||||
|
p.mu.Unlock()
|
||||||
|
}
|
||||||
|
|
||||||
|
func (p *Pool) started(ctx context.Context, id, name string) *Container {
|
||||||
|
if p.cfg.MaxUptime > 0 {
|
||||||
|
p.armMaxUptime(ctx, id)
|
||||||
|
}
|
||||||
|
p.log.Info("container started", "id", id[:12], "name", name)
|
||||||
|
return &Container{ID: id, exited: p.watchExit(ctx, id)}
|
||||||
|
}
|
||||||
|
|
||||||
|
// Client is the pool's Docker client, for callers that provision what a
|
||||||
|
// container needs around it.
|
||||||
|
func (p *Pool) Client() *client.Client { return p.cli }
|
||||||
|
|
||||||
// watchExit returns a channel closed once Docker reports the container is no
|
// watchExit returns a channel closed once Docker reports the container is no
|
||||||
// longer running. A failed wait says nothing about the container, so it never
|
// longer running. A failed wait says nothing about the container, so it never
|
||||||
// closes the channel: a Docker API error must not abort a live run.
|
// closes the channel: a Docker API error must not abort a live run.
|
||||||
@@ -499,10 +536,10 @@ func (p *Pool) Close() error {
|
|||||||
|
|
||||||
// ── helpers ──────────────────────────────────────────────────────────
|
// ── helpers ──────────────────────────────────────────────────────────
|
||||||
|
|
||||||
// ensureImage pulls an image other than the configured one the first time a
|
// EnsureImage pulls an image other than the configured one the first time a
|
||||||
// task asks for it, under the configured pull policy. The configured image
|
// task asks for it, under the configured pull policy. The configured image
|
||||||
// was already pulled by NewPool.
|
// was already pulled by NewPool.
|
||||||
func (p *Pool) ensureImage(ctx context.Context, ref string) error {
|
func (p *Pool) EnsureImage(ctx context.Context, ref string) error {
|
||||||
if ref == p.cfg.Image || (p.cfg.PullPolicy != "always" && p.cfg.PullPolicy != "missing") {
|
if ref == p.cfg.Image || (p.cfg.PullPolicy != "always" && p.cfg.PullPolicy != "missing") {
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
|||||||
+30
-14
@@ -158,6 +158,8 @@ type Daemon struct {
|
|||||||
// set -- acquireTaskContainer parks the task rather than fall back to
|
// set -- acquireTaskContainer parks the task rather than fall back to
|
||||||
// forwarding the daemon's own administrative token.
|
// forwarding the daemon's own administrative token.
|
||||||
StateStoreGrants StateStoreGrantIssuer
|
StateStoreGrants StateStoreGrantIssuer
|
||||||
|
// KitLauncher starts tasks whose agent profile is a Kit. Nil parks them.
|
||||||
|
KitLauncher KitLauncher
|
||||||
// TaskRunReadyTimeout bounds how long runViaAgent retries an initial
|
// TaskRunReadyTimeout bounds how long runViaAgent retries an initial
|
||||||
// taskrun request that fails with nats.ErrNoResponders, giving a
|
// taskrun request that fails with nats.ErrNoResponders, giving a
|
||||||
// freshly spawned archie-agent container time to connect to NATS, set
|
// freshly spawned archie-agent container time to connect to NATS, set
|
||||||
@@ -1261,6 +1263,12 @@ func (d *Daemon) process(ctx context.Context, task *workflow.Task) {
|
|||||||
if !ok {
|
if !ok {
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
|
if profile.IsKit() {
|
||||||
|
d.runKitTask(ctx, task, repo, workDir, profile)
|
||||||
|
d.teardownStorage(ctx, task, repo, workDir)
|
||||||
|
d.cleanupTerminalTaskWorktree(ctx, task, trees)
|
||||||
|
return
|
||||||
|
}
|
||||||
ctr, revokeStateStoreGrant, ok := d.acquireTaskContainer(ctx, task, repo, workDir, profile.Image)
|
ctr, revokeStateStoreGrant, ok := d.acquireTaskContainer(ctx, task, repo, workDir, profile.Image)
|
||||||
if !ok {
|
if !ok {
|
||||||
return
|
return
|
||||||
@@ -1278,12 +1286,17 @@ func (d *Daemon) process(ctx context.Context, task *workflow.Task) {
|
|||||||
// task is always served by its own forge client and worktree manager.
|
// task is always served by its own forge client and worktree manager.
|
||||||
limitCtx, stopLimit := withTaskTimeLimit(ctx, d.configFor(task).Budgets.TaskWallClock.Std())
|
limitCtx, stopLimit := withTaskTimeLimit(ctx, d.configFor(task).Budgets.TaskWallClock.Std())
|
||||||
runCtx, stopWatch := withContainerExit(limitCtx, ctr.Exited())
|
runCtx, stopWatch := withContainerExit(limitCtx, ctr.Exited())
|
||||||
d.runViaAgent(runCtx, task, repo, profile)
|
d.runViaAgent(runCtx, task, repo, profile, nil)
|
||||||
stopWatch()
|
stopWatch()
|
||||||
stopLimit()
|
stopLimit()
|
||||||
|
|
||||||
// Teardown storage after workflow completes. The Docker backend is a
|
d.teardownStorage(ctx, task, repo, workDir)
|
||||||
// no-op; future backends (temp volumes, NFS leases) use this hook.
|
d.cleanupTerminalTaskWorktree(ctx, task, trees)
|
||||||
|
}
|
||||||
|
|
||||||
|
// teardownStorage runs after the workflow completes. The Docker backend is
|
||||||
|
// a no-op; future backends (temp volumes, NFS leases) use this hook.
|
||||||
|
func (d *Daemon) teardownStorage(ctx context.Context, task *workflow.Task, repo config.Repo, workDir string) {
|
||||||
if d.Storage != nil {
|
if d.Storage != nil {
|
||||||
_ = d.Storage.Teardown(ctx, storage.TaskRef{
|
_ = d.Storage.Teardown(ctx, storage.TaskRef{
|
||||||
WorktreeDir: workDir,
|
WorktreeDir: workDir,
|
||||||
@@ -1293,8 +1306,6 @@ func (d *Daemon) process(ctx context.Context, task *workflow.Task) {
|
|||||||
Repo: task.Repo,
|
Repo: task.Repo,
|
||||||
})
|
})
|
||||||
}
|
}
|
||||||
|
|
||||||
d.cleanupTerminalTaskWorktree(ctx, task, trees)
|
|
||||||
}
|
}
|
||||||
|
|
||||||
// prepareWorkspace makes the directory the container binds as its
|
// prepareWorkspace makes the directory the container binds as its
|
||||||
@@ -1427,14 +1438,7 @@ func (d *Daemon) acquireTaskContainer(
|
|||||||
d.parkRunningTask(ctx, task.ID, reason+": "+err.Error(), taskstate.ParkTransient)
|
d.parkRunningTask(ctx, task.ID, reason+": "+err.Error(), taskstate.ParkTransient)
|
||||||
}
|
}
|
||||||
|
|
||||||
// Write task.json -- the container's boot-time brief.
|
if err := writeTaskBrief(workDir, task); err != nil {
|
||||||
if err := container.WriteTaskJSON(workDir, container.TaskPayload{
|
|
||||||
ID: task.ID, Owner: task.Owner, Repo: task.Repo,
|
|
||||||
Number: task.IssueNumber, Title: task.Title, Body: task.Body,
|
|
||||||
Labels: strings.Split(task.Labels, ","),
|
|
||||||
Workflow: task.Workflow, Branch: task.Branch, Plan: task.Plan,
|
|
||||||
Inputs: task.Inputs,
|
|
||||||
}); err != nil {
|
|
||||||
park("task.json write failed", err)
|
park("task.json write failed", err)
|
||||||
return nil, nil, false
|
return nil, nil, false
|
||||||
}
|
}
|
||||||
@@ -1496,6 +1500,17 @@ func (d *Daemon) acquireTaskContainer(
|
|||||||
return ctr, revokeStateStoreGrant, true
|
return ctr, revokeStateStoreGrant, true
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// writeTaskBrief writes task.json, the container's boot-time brief.
|
||||||
|
func writeTaskBrief(workDir string, task *workflow.Task) error {
|
||||||
|
return container.WriteTaskJSON(workDir, container.TaskPayload{
|
||||||
|
ID: task.ID, Owner: task.Owner, Repo: task.Repo,
|
||||||
|
Number: task.IssueNumber, Title: task.Title, Body: task.Body,
|
||||||
|
Labels: strings.Split(task.Labels, ","),
|
||||||
|
Workflow: task.Workflow, Branch: task.Branch, Plan: task.Plan,
|
||||||
|
Inputs: task.Inputs,
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
// parkCapabilityUnavailable parks a task whose daemon-side capability is
|
// parkCapabilityUnavailable parks a task whose daemon-side capability is
|
||||||
// missing (container pool or task transport): both are environmental, so
|
// missing (container pool or task transport): both are environmental, so
|
||||||
// the park class is transient and the KindParked event carries the class
|
// the park class is transient and the KindParked event carries the class
|
||||||
@@ -1588,7 +1603,7 @@ func (d *Daemon) recordPark(ctx context.Context, taskID int64, reason string) {
|
|||||||
// itself when nothing else could have: the request never reached (or was
|
// itself when nothing else could have: the request never reached (or was
|
||||||
// never answered by) an archie-agent, or archie-agent failed before its
|
// never answered by) an archie-agent, or archie-agent failed before its
|
||||||
// own workflow.Run got a chance to record an outcome.
|
// own workflow.Run got a chance to record an outcome.
|
||||||
func (d *Daemon) runViaAgent(ctx context.Context, task *workflow.Task, repo config.Repo, profile config.AgentProfile) {
|
func (d *Daemon) runViaAgent(ctx context.Context, task *workflow.Task, repo config.Repo, profile config.AgentProfile, harness *agentexec.HarnessSpec) {
|
||||||
grant, revoke, ok := d.publicationGrant(ctx, task)
|
grant, revoke, ok := d.publicationGrant(ctx, task)
|
||||||
if !ok {
|
if !ok {
|
||||||
return
|
return
|
||||||
@@ -1608,6 +1623,7 @@ func (d *Daemon) runViaAgent(ctx context.Context, task *workflow.Task, repo conf
|
|||||||
LabelWorkflows: d.LabelWorkflows,
|
LabelWorkflows: d.LabelWorkflows,
|
||||||
WorkflowDefinition: task.WorkflowDefinitionYAML,
|
WorkflowDefinition: task.WorkflowDefinitionYAML,
|
||||||
Tools: profile.Tools,
|
Tools: profile.Tools,
|
||||||
|
Harness: harness,
|
||||||
}
|
}
|
||||||
data, err := json.Marshal(req)
|
data, err := json.Marshal(req)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
|
|||||||
@@ -19,6 +19,7 @@ import (
|
|||||||
natssrv "github.com/nats-io/nats-server/v2/test"
|
natssrv "github.com/nats-io/nats-server/v2/test"
|
||||||
natsio "github.com/nats-io/nats.go"
|
natsio "github.com/nats-io/nats.go"
|
||||||
|
|
||||||
|
"github.com/samcharles93/archie-core/internal/agentexec"
|
||||||
"github.com/samcharles93/archie-core/internal/config"
|
"github.com/samcharles93/archie-core/internal/config"
|
||||||
"github.com/samcharles93/archie-core/internal/container"
|
"github.com/samcharles93/archie-core/internal/container"
|
||||||
"github.com/samcharles93/archie-core/internal/domain/workflow"
|
"github.com/samcharles93/archie-core/internal/domain/workflow"
|
||||||
@@ -27,6 +28,8 @@ import (
|
|||||||
"github.com/samcharles93/archie-core/internal/forge"
|
"github.com/samcharles93/archie-core/internal/forge"
|
||||||
agentnats "github.com/samcharles93/archie-core/internal/infrastructure/agenttransport/nats"
|
agentnats "github.com/samcharles93/archie-core/internal/infrastructure/agenttransport/nats"
|
||||||
arnats "github.com/samcharles93/archie-core/internal/infrastructure/eventbus/nats"
|
arnats "github.com/samcharles93/archie-core/internal/infrastructure/eventbus/nats"
|
||||||
|
"github.com/samcharles93/archie-core/internal/infrastructure/kit"
|
||||||
|
"github.com/samcharles93/archie-core/internal/infrastructure/kitrun"
|
||||||
"github.com/samcharles93/archie-core/internal/infrastructure/postgres/pgstore"
|
"github.com/samcharles93/archie-core/internal/infrastructure/postgres/pgstore"
|
||||||
"github.com/samcharles93/archie-core/internal/logging"
|
"github.com/samcharles93/archie-core/internal/logging"
|
||||||
"github.com/samcharles93/archie-core/internal/secret"
|
"github.com/samcharles93/archie-core/internal/secret"
|
||||||
@@ -2330,7 +2333,7 @@ func TestRunViaAgentCarriesTheWorkflowProfile(t *testing.T) {
|
|||||||
if !ok || profile.Image != "agent-net:1" {
|
if !ok || profile.Image != "agent-net:1" {
|
||||||
t.Fatalf("pinTaskProfile() = %+v, %v; want the net profile", profile, ok)
|
t.Fatalf("pinTaskProfile() = %+v, %v; want the net profile", profile, ok)
|
||||||
}
|
}
|
||||||
d.runViaAgent(ctx, task, config.Repo{Owner: "acme", Name: "widget", Base: "main"}, profile)
|
d.runViaAgent(ctx, task, config.Repo{Owner: "acme", Name: "widget", Base: "main"}, profile, nil)
|
||||||
select {
|
select {
|
||||||
case req := <-received:
|
case req := <-received:
|
||||||
if len(req.Tools) != 1 || req.Tools[0] != "whois" {
|
if len(req.Tools) != 1 || req.Tools[0] != "whois" {
|
||||||
@@ -2378,7 +2381,7 @@ func TestRunViaAgentParksWhenTheContainerExits(t *testing.T) {
|
|||||||
defer stop()
|
defer stop()
|
||||||
done := make(chan struct{})
|
done := make(chan struct{})
|
||||||
go func() {
|
go func() {
|
||||||
d.runViaAgent(runCtx, task, config.Repo{Owner: "acme", Name: "widget", Base: "main"}, config.AgentProfile{})
|
d.runViaAgent(runCtx, task, config.Repo{Owner: "acme", Name: "widget", Base: "main"}, config.AgentProfile{}, nil)
|
||||||
close(done)
|
close(done)
|
||||||
}()
|
}()
|
||||||
|
|
||||||
@@ -2422,7 +2425,7 @@ func TestRunViaAgentParksWhenTheTaskTimeLimitPasses(t *testing.T) {
|
|||||||
defer stop()
|
defer stop()
|
||||||
done := make(chan struct{})
|
done := make(chan struct{})
|
||||||
go func() {
|
go func() {
|
||||||
d.runViaAgent(runCtx, task, config.Repo{Owner: "acme", Name: "widget", Base: "main"}, config.AgentProfile{})
|
d.runViaAgent(runCtx, task, config.Repo{Owner: "acme", Name: "widget", Base: "main"}, config.AgentProfile{}, nil)
|
||||||
close(done)
|
close(done)
|
||||||
}()
|
}()
|
||||||
select {
|
select {
|
||||||
@@ -2436,3 +2439,85 @@ func TestRunViaAgentParksWhenTheTaskTimeLimitPasses(t *testing.T) {
|
|||||||
t.Fatalf("task = %+v, %v; want parked naming the time limit", got, err)
|
t.Fatalf("task = %+v, %v; want parked naming the time limit", got, err)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
type fakeKitLauncher struct {
|
||||||
|
got kitrun.Request
|
||||||
|
released bool
|
||||||
|
removed []kit.Volume
|
||||||
|
}
|
||||||
|
|
||||||
|
func (f *fakeKitLauncher) Launch(_ context.Context, req kitrun.Request) (*kitrun.Run, error) {
|
||||||
|
f.got = req
|
||||||
|
return &kitrun.Run{
|
||||||
|
Container: &container.Container{},
|
||||||
|
Harness: agentexec.HarnessSpec{Adapter: agentexec.AdapterClaudeCode, Launch: []string{"claude"}},
|
||||||
|
Volumes: []kit.Volume{{Name: "v", Path: "/home/agent/.claude"}},
|
||||||
|
}, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func (f *fakeKitLauncher) Release(context.Context, *kitrun.Run) error {
|
||||||
|
f.released = true
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func (f *fakeKitLauncher) RemoveVolumes(_ context.Context, v []kit.Volume) error {
|
||||||
|
f.removed = append(f.removed, v...)
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// A Kit profile starts the task through the Kit launcher, hands the worker
|
||||||
|
// the Kit's harness, keeps model provider keys out of the container, and
|
||||||
|
// removes the execution's volumes once the task is no longer retryable.
|
||||||
|
func TestKitProfileRunsTheTaskOnItsHarness(t *testing.T) {
|
||||||
|
d, s, busClient := daemonWithNATS(t)
|
||||||
|
launcher := &fakeKitLauncher{}
|
||||||
|
d.KitLauncher = launcher
|
||||||
|
d.ConnectedNATS.URL = "nats://172.17.0.1:4222"
|
||||||
|
t.Setenv("KIT_TEST_PROVIDER_KEY", "sk-real")
|
||||||
|
cfg := d.Cfg.Get()
|
||||||
|
cfg.Providers = map[string]config.Provider{"p": {APIKeyEnv: "KIT_TEST_PROVIDER_KEY"}}
|
||||||
|
d.Cfg.Set(cfg)
|
||||||
|
ctx := context.Background()
|
||||||
|
if _, err := s.EnqueueIssue(ctx, "acme", "widget", 31, "t", "b", "", ""); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
task, err := s.ClaimNext(ctx)
|
||||||
|
if err != nil || task == nil {
|
||||||
|
t.Fatalf("claim: (%v, %v)", task, err)
|
||||||
|
}
|
||||||
|
received := make(chan taskrun.Request, 1)
|
||||||
|
sub, err := mustCoreConn(t, busClient).Subscribe(agentnats.SubjectForTask(task.ID), func(msg *natsio.Msg) {
|
||||||
|
var req taskrun.Request
|
||||||
|
_ = json.Unmarshal(msg.Data, &req)
|
||||||
|
received <- req
|
||||||
|
data, _ := json.Marshal(taskrun.Response{Status: workflow.StatusPROpen})
|
||||||
|
_ = msg.Respond(data)
|
||||||
|
})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("subscribe: %v", err)
|
||||||
|
}
|
||||||
|
t.Cleanup(func() { _ = sub.Unsubscribe() })
|
||||||
|
|
||||||
|
profile := config.AgentProfile{Kit: []string{"kit@sha256:ab"}, Adapter: agentexec.AdapterClaudeCode}
|
||||||
|
d.runKitTask(ctx, task, config.Repo{Owner: "acme", Name: "widget", Base: "main"}, t.TempDir(), profile)
|
||||||
|
|
||||||
|
select {
|
||||||
|
case req := <-received:
|
||||||
|
if req.Harness == nil || req.Harness.Adapter != agentexec.AdapterClaudeCode {
|
||||||
|
t.Fatalf("request harness = %+v, want the Kit's", req.Harness)
|
||||||
|
}
|
||||||
|
case <-time.After(time.Second):
|
||||||
|
t.Fatal("archied did not publish a taskrun request")
|
||||||
|
}
|
||||||
|
if !slices.Equal(launcher.got.Kit, profile.Kit) || launcher.got.Adapter != profile.Adapter {
|
||||||
|
t.Fatalf("launch request %+v, want the profile's kits and adapter", launcher.got)
|
||||||
|
}
|
||||||
|
for _, e := range launcher.got.WorkerEnv {
|
||||||
|
if strings.Contains(e, "sk-real") {
|
||||||
|
t.Fatalf("worker env carries a model provider key: %q", e)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if !launcher.released || len(launcher.removed) != 1 {
|
||||||
|
t.Fatalf("released %v, removed %v; want the run released and its volumes removed", launcher.released, launcher.removed)
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -0,0 +1,94 @@
|
|||||||
|
package daemon
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"fmt"
|
||||||
|
"os"
|
||||||
|
"time"
|
||||||
|
|
||||||
|
"github.com/samcharles93/archie-core/internal/config"
|
||||||
|
"github.com/samcharles93/archie-core/internal/domain/workflow"
|
||||||
|
"github.com/samcharles93/archie-core/internal/infrastructure/kit"
|
||||||
|
"github.com/samcharles93/archie-core/internal/infrastructure/kitrun"
|
||||||
|
"github.com/samcharles93/archie-core/internal/taskstate"
|
||||||
|
)
|
||||||
|
|
||||||
|
// KitLauncher starts and ends the container of a task whose agent profile
|
||||||
|
// is a Kit.
|
||||||
|
type KitLauncher interface {
|
||||||
|
Launch(ctx context.Context, req kitrun.Request) (*kitrun.Run, error)
|
||||||
|
Release(ctx context.Context, run *kitrun.Run) error
|
||||||
|
RemoveVolumes(ctx context.Context, volumes []kit.Volume) error
|
||||||
|
}
|
||||||
|
|
||||||
|
// runKitTask runs a task in its Kit's container, every agent stage on the
|
||||||
|
// Kit's harness. The worker reaches NATS and the State Store only through
|
||||||
|
// the egress relay, and its environment carries no model provider key.
|
||||||
|
func (d *Daemon) runKitTask(ctx context.Context, task *workflow.Task, repo config.Repo, workDir string, profile config.AgentProfile) {
|
||||||
|
park := func(reason string, err error) {
|
||||||
|
d.Log.Error(reason, "task", task.ID, "err", err)
|
||||||
|
d.parkRunningTask(ctx, task.ID, reason+": "+err.Error(), taskstate.ParkTransient)
|
||||||
|
}
|
||||||
|
if d.KitLauncher == nil {
|
||||||
|
park("kit profile", fmt.Errorf("kit harness runs are unavailable on this daemon"))
|
||||||
|
return
|
||||||
|
}
|
||||||
|
if err := writeTaskBrief(workDir, task); err != nil {
|
||||||
|
park("task.json write failed", err)
|
||||||
|
return
|
||||||
|
}
|
||||||
|
stateStoreToken, revokeStateStoreGrant, err := d.stateStoreGrantToken(task)
|
||||||
|
if err != nil {
|
||||||
|
park("state store grant failed", err)
|
||||||
|
return
|
||||||
|
}
|
||||||
|
defer revokeStateStoreGrant()
|
||||||
|
env, err := kitrun.WorkerEnv(kitrun.Endpoints{
|
||||||
|
NATS: d.ConnectedNATS.URL, NATSToken: d.ConnectedNATS.Token,
|
||||||
|
StateStore: d.ConnectedStateStore.URL, StateStoreToken: stateStoreToken,
|
||||||
|
}, os.Getuid(), os.Getgid())
|
||||||
|
if err != nil {
|
||||||
|
park("kit worker environment", err)
|
||||||
|
return
|
||||||
|
}
|
||||||
|
run, err := d.KitLauncher.Launch(ctx, kitrun.Request{
|
||||||
|
Execution: fmt.Sprintf("task-%d", task.ID),
|
||||||
|
Kit: profile.Kit,
|
||||||
|
Adapter: profile.Adapter,
|
||||||
|
WorkDir: workDir,
|
||||||
|
WorkerEnv: env,
|
||||||
|
})
|
||||||
|
if err != nil {
|
||||||
|
park("kit launch failed", err)
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
|
limitCtx, stopLimit := withTaskTimeLimit(ctx, d.configFor(task).Budgets.TaskWallClock.Std())
|
||||||
|
runCtx, stopWatch := withContainerExit(limitCtx, run.Container.Exited())
|
||||||
|
d.runViaAgent(runCtx, task, repo, profile, &run.Harness)
|
||||||
|
stopWatch()
|
||||||
|
stopLimit()
|
||||||
|
|
||||||
|
if err := d.KitLauncher.Release(ctx, run); err != nil {
|
||||||
|
d.Log.Warn("kit release failed", "task", task.ID, "err", err)
|
||||||
|
}
|
||||||
|
d.removeEndedKitVolumes(ctx, task.ID, run.Volumes)
|
||||||
|
}
|
||||||
|
|
||||||
|
// removeEndedKitVolumes removes the execution's Kit volumes unless the task
|
||||||
|
// parked: a retry of a parked task is the same execution and resumes from
|
||||||
|
// them.
|
||||||
|
func (d *Daemon) removeEndedKitVolumes(ctx context.Context, taskID int64, volumes []kit.Volume) {
|
||||||
|
if len(volumes) == 0 || d.Store == nil {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
lookupCtx, cancel := context.WithTimeout(context.WithoutCancel(ctx), 10*time.Second)
|
||||||
|
defer cancel()
|
||||||
|
latest, err := d.Store.TaskByID(lookupCtx, taskID)
|
||||||
|
if err != nil || latest == nil || latest.Status == workflow.StatusParked {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
if err := d.KitLauncher.RemoveVolumes(lookupCtx, volumes); err != nil {
|
||||||
|
d.Log.Warn("kit volume removal failed", "task", taskID, "err", err)
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -42,5 +42,5 @@ func runPinnedViaAgent(ctx context.Context, d *Daemon, task *workflow.Task, repo
|
|||||||
if !ok {
|
if !ok {
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
d.runViaAgent(ctx, task, repo, profile)
|
d.runViaAgent(ctx, task, repo, profile, nil)
|
||||||
}
|
}
|
||||||
@@ -40,6 +40,10 @@ type CA struct {
|
|||||||
leaves map[string]*tls.Certificate
|
leaves map[string]*tls.Certificate
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// CACertPath is where LoadOrCreateCA keeps the CA certificate in dir, the
|
||||||
|
// file a sandbox container mounts to trust the proxy.
|
||||||
|
func CACertPath(dir string) string { return filepath.Join(dir, caCertFile) }
|
||||||
|
|
||||||
// LoadOrCreateCA loads the CA from dir, creating it on first use. A CA that
|
// LoadOrCreateCA loads the CA from dir, creating it on first use. A CA that
|
||||||
// exists but will not load is an error: minting a replacement silently
|
// exists but will not load is an error: minting a replacement silently
|
||||||
// would break trust in every running container.
|
// would break trust in every running container.
|
||||||
|
|||||||
@@ -129,10 +129,19 @@ func Admit(d *spec.Descriptor) (*Plan, error) {
|
|||||||
// admits the result. archie drives a harness headlessly, so the composition
|
// admits the result. archie drives a harness headlessly, so the composition
|
||||||
// must carry an agent-sessions prompt verb.
|
// must carry an agent-sessions prompt verb.
|
||||||
func Compose(contributions []spec.Contribution) (*Plan, error) {
|
func Compose(contributions []spec.Contribution) (*Plan, error) {
|
||||||
merged, err := spec.Merge(contributions, spec.MergeOptions{ContextPath: contextPath})
|
merged, err := spec.Merge(contributions, MergeOptions)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, err
|
return nil, err
|
||||||
}
|
}
|
||||||
|
return FromMerge(merged)
|
||||||
|
}
|
||||||
|
|
||||||
|
// MergeOptions are the options every composition archie runs is merged with.
|
||||||
|
var MergeOptions = spec.MergeOptions{ContextPath: contextPath}
|
||||||
|
|
||||||
|
// FromMerge admits an already-merged composition, such as one the Kit spec's
|
||||||
|
// fetch package assembled from a registry.
|
||||||
|
func FromMerge(merged *spec.MergeResult) (*Plan, error) {
|
||||||
if merged.Descriptor.Kind != spec.KindWorkload {
|
if merged.Descriptor.Kind != spec.KindWorkload {
|
||||||
return nil, errors.New("kit composition has no workload: a harness needs exactly one base kit")
|
return nil, errors.New("kit composition has no workload: a harness needs exactly one base kit")
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -0,0 +1,72 @@
|
|||||||
|
package kitrun
|
||||||
|
|
||||||
|
import (
|
||||||
|
"fmt"
|
||||||
|
"net"
|
||||||
|
"net/url"
|
||||||
|
"strconv"
|
||||||
|
"strings"
|
||||||
|
|
||||||
|
"github.com/samcharles93/archie-core/internal/infrastructure/egress"
|
||||||
|
)
|
||||||
|
|
||||||
|
// Endpoints are the host-reachable addresses a Kit task's worker needs,
|
||||||
|
// with the task's own credentials for them.
|
||||||
|
type Endpoints struct {
|
||||||
|
NATS string
|
||||||
|
NATSToken string
|
||||||
|
StateStore string
|
||||||
|
StateStoreToken string
|
||||||
|
}
|
||||||
|
|
||||||
|
// WorkerEnv is the worker's entire environment in a Kit container: NATS and
|
||||||
|
// the State Store at the relay's fixed ports, and the task's credentials.
|
||||||
|
// Nothing else crosses, so no model provider key enters the container.
|
||||||
|
func WorkerEnv(e Endpoints, uid, gid int) ([]string, error) {
|
||||||
|
u, err := url.Parse(e.NATS)
|
||||||
|
if err != nil || u.Scheme == "" {
|
||||||
|
return nil, fmt.Errorf("nats url %q: want scheme://host:port", e.NATS)
|
||||||
|
}
|
||||||
|
env := []string{"NATS_URL=" + u.Scheme + "://" + relayAddr(egress.RelayNATSPort)}
|
||||||
|
if e.NATSToken != "" {
|
||||||
|
env = append(env, "NATS_TOKEN="+e.NATSToken)
|
||||||
|
}
|
||||||
|
if e.StateStore != "" {
|
||||||
|
env = append(env, "STATE_STORE_URL="+relayAddr(egress.RelayStateStorePort))
|
||||||
|
if e.StateStoreToken != "" {
|
||||||
|
env = append(env, "STATE_STORE_TOKEN="+e.StateStoreToken)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return append(env, "WORKTREE_UID="+strconv.Itoa(uid), "WORKTREE_GID="+strconv.Itoa(gid)), nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// relayTargets are the host:port addresses the relay forwards NATS and the
|
||||||
|
// State Store to.
|
||||||
|
func relayTargets(e Endpoints) (nats, stateStore string, err error) {
|
||||||
|
if strings.Contains(e.NATS, ",") {
|
||||||
|
return "", "", fmt.Errorf("nats url %q lists several servers; the relay forwards one", e.NATS)
|
||||||
|
}
|
||||||
|
u, err := url.Parse(e.NATS)
|
||||||
|
if err != nil || u.Host == "" {
|
||||||
|
return "", "", fmt.Errorf("nats url %q: want scheme://host:port", e.NATS)
|
||||||
|
}
|
||||||
|
stateStore = e.StateStore
|
||||||
|
if _, rest, ok := strings.Cut(stateStore, ":///"); ok {
|
||||||
|
stateStore = rest
|
||||||
|
}
|
||||||
|
return u.Host, stateStore, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func relayAddr(port int) string {
|
||||||
|
return net.JoinHostPort(egress.RelayAlias, strconv.Itoa(port))
|
||||||
|
}
|
||||||
|
|
||||||
|
// RelayCommand is the relay's archie-agent arguments, forwarding its fixed
|
||||||
|
// ports to the proxy and to the daemon's NATS and State Store.
|
||||||
|
func RelayCommand(proxy string, e Endpoints) ([]string, error) {
|
||||||
|
nats, stateStore, err := relayTargets(e)
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
return egress.RelayArgs(proxy, nats, stateStore), nil
|
||||||
|
}
|
||||||
@@ -0,0 +1,44 @@
|
|||||||
|
package kitrun
|
||||||
|
|
||||||
|
import (
|
||||||
|
"slices"
|
||||||
|
"testing"
|
||||||
|
)
|
||||||
|
|
||||||
|
func TestWorkerEnvReachesNATSAndTheStateStoreThroughTheRelay(t *testing.T) {
|
||||||
|
got, err := WorkerEnv(Endpoints{
|
||||||
|
NATS: "nats://172.17.0.1:4222", NATSToken: "nt",
|
||||||
|
StateStore: "10.0.0.5:7443", StateStoreToken: "st",
|
||||||
|
}, 1001, 1002)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
want := []string{
|
||||||
|
"NATS_URL=nats://archie-egress:4222", "NATS_TOKEN=nt",
|
||||||
|
"STATE_STORE_URL=archie-egress:50051", "STATE_STORE_TOKEN=st",
|
||||||
|
"WORKTREE_UID=1001", "WORKTREE_GID=1002",
|
||||||
|
}
|
||||||
|
if !slices.Equal(got, want) {
|
||||||
|
t.Fatalf("WorkerEnv = %q\nwant %q", got, want)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestWorkerEnvWithoutAStateStoreOmitsIt(t *testing.T) {
|
||||||
|
got, err := WorkerEnv(Endpoints{NATS: "tls://nats.internal:4222"}, 0, 0)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if !slices.Equal(got, []string{"NATS_URL=tls://archie-egress:4222", "WORKTREE_UID=0", "WORKTREE_GID=0"}) {
|
||||||
|
t.Fatalf("WorkerEnv = %q", got)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestRelayTargets(t *testing.T) {
|
||||||
|
nats, store, err := relayTargets(Endpoints{NATS: "nats://172.17.0.1:4222", StateStore: "dns:///store.lan:7443"})
|
||||||
|
if err != nil || nats != "172.17.0.1:4222" || store != "store.lan:7443" {
|
||||||
|
t.Fatalf("relayTargets = %q, %q, %v", nats, store, err)
|
||||||
|
}
|
||||||
|
if _, _, err := relayTargets(Endpoints{NATS: "nats://a:1,nats://b:2"}); err == nil {
|
||||||
|
t.Fatal("a NATS cluster URL list was accepted; the relay forwards one address")
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -0,0 +1,195 @@
|
|||||||
|
// Package kitrun starts a Kit task: it composes the profile's Kits, opens
|
||||||
|
// the task's egress session and sandbox network, and starts the container
|
||||||
|
// with archie's worker as root and the harness spec the worker runs stages on.
|
||||||
|
package kitrun
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"errors"
|
||||||
|
"fmt"
|
||||||
|
"maps"
|
||||||
|
"slices"
|
||||||
|
"sync"
|
||||||
|
|
||||||
|
"github.com/docker/sandbox-kit-spec/v3/fetch"
|
||||||
|
"github.com/docker/sandbox-kit-spec/v3/spec"
|
||||||
|
|
||||||
|
"github.com/samcharles93/archie-core/internal/agentexec"
|
||||||
|
"github.com/samcharles93/archie-core/internal/container"
|
||||||
|
"github.com/samcharles93/archie-core/internal/infrastructure/egress"
|
||||||
|
"github.com/samcharles93/archie-core/internal/infrastructure/kit"
|
||||||
|
)
|
||||||
|
|
||||||
|
const (
|
||||||
|
agentPath = "/opt/archie/archie-agent"
|
||||||
|
caPath = "/opt/archie/ca.pem"
|
||||||
|
)
|
||||||
|
|
||||||
|
// Launcher starts Kit task containers.
|
||||||
|
type Launcher struct {
|
||||||
|
Pool *container.Pool
|
||||||
|
Fetch *fetch.Client
|
||||||
|
Proxy *egress.Proxy
|
||||||
|
Networks *egress.Networks
|
||||||
|
// AgentBinary and CAFile are host paths mounted read-only into every
|
||||||
|
// Kit container: the worker's own binary and the egress CA.
|
||||||
|
AgentBinary string
|
||||||
|
CAFile string
|
||||||
|
|
||||||
|
mu sync.Mutex
|
||||||
|
started bool
|
||||||
|
}
|
||||||
|
|
||||||
|
// Request is one Kit task to start.
|
||||||
|
type Request struct {
|
||||||
|
// Execution keys the task's Kit volumes and names its container and
|
||||||
|
// network.
|
||||||
|
Execution string
|
||||||
|
Kit []string
|
||||||
|
Adapter string
|
||||||
|
WorkDir string
|
||||||
|
WorkerEnv []string
|
||||||
|
}
|
||||||
|
|
||||||
|
// Run is a started Kit task.
|
||||||
|
type Run struct {
|
||||||
|
Container *container.Container
|
||||||
|
Harness agentexec.HarnessSpec
|
||||||
|
// Volumes outlive the run; remove them with RemoveVolumes once the
|
||||||
|
// execution has ended.
|
||||||
|
Volumes []kit.Volume
|
||||||
|
network string
|
||||||
|
token string
|
||||||
|
}
|
||||||
|
|
||||||
|
// Launch composes the request's Kits and starts its container.
|
||||||
|
func (l *Launcher) Launch(ctx context.Context, req Request) (*Run, error) {
|
||||||
|
if len(req.Kit) == 0 {
|
||||||
|
return nil, errors.New("kit profile names no kit")
|
||||||
|
}
|
||||||
|
adapter, ok := agentexec.LookupHarnessAdapter(req.Adapter)
|
||||||
|
if req.Adapter != "" && !ok {
|
||||||
|
return nil, fmt.Errorf("harness output adapter %q is unknown", req.Adapter)
|
||||||
|
}
|
||||||
|
if err := l.startEgress(ctx); err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
plan, img, err := l.compose(ctx, req.Kit)
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
network, err := spec.NetworkPolicyOf(plan.Capabilities)
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
creds, err := spec.CredentialsOf(plan.Capabilities)
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
session, err := l.Proxy.Register(egress.SessionOptions{Run: req.Execution, Network: network, Credentials: creds})
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
run := &Run{network: "archie-kit-" + req.Execution, token: session.Token()}
|
||||||
|
launch, err := kit.Assemble(plan, img, kit.LaunchParams{Execution: req.Execution, ProxyToken: session.Token(), CAPath: caPath})
|
||||||
|
if err != nil {
|
||||||
|
l.Proxy.Revoke(run.token)
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
launch.Harness.Adapter, launch.Harness.MCPConfig = req.Adapter, adapter.MCPConfig
|
||||||
|
run.Harness, run.Volumes = launch.Harness, launch.Volumes
|
||||||
|
if err := l.Networks.Create(ctx, run.network); err != nil {
|
||||||
|
l.Proxy.Revoke(run.token)
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
run.Container, err = l.Pool.AcquireKit(ctx, container.KitSpec{
|
||||||
|
Name: "archie-kit-" + req.Execution,
|
||||||
|
Image: req.Kit[0],
|
||||||
|
Network: run.network,
|
||||||
|
Worker: []string{agentPath},
|
||||||
|
WorkerEnv: req.WorkerEnv,
|
||||||
|
Binds: []string{
|
||||||
|
l.AgentBinary + ":" + agentPath + ":ro",
|
||||||
|
l.CAFile + ":" + caPath + ":ro",
|
||||||
|
req.WorkDir + ":" + kit.WorkspaceDir,
|
||||||
|
},
|
||||||
|
Launch: launch,
|
||||||
|
InstallDone: session.EnterRuntime,
|
||||||
|
})
|
||||||
|
if err != nil {
|
||||||
|
l.Proxy.Revoke(run.token)
|
||||||
|
return nil, errors.Join(err, l.Networks.Remove(context.WithoutCancel(ctx), run.network))
|
||||||
|
}
|
||||||
|
return run, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// compose assembles the profile's Kits and reads the workload image's
|
||||||
|
// config, the first Kit being the workload.
|
||||||
|
func (l *Launcher) compose(ctx context.Context, refs []string) (*kit.Plan, kit.ImageConfig, error) {
|
||||||
|
reqs := make([]fetch.Request, len(refs))
|
||||||
|
for i, ref := range refs {
|
||||||
|
reqs[i] = fetch.Request{Reference: ref}
|
||||||
|
}
|
||||||
|
merged, err := l.Fetch.Assemble(ctx, reqs, kit.MergeOptions)
|
||||||
|
if err != nil {
|
||||||
|
return nil, kit.ImageConfig{}, fmt.Errorf("assemble kits: %w", err)
|
||||||
|
}
|
||||||
|
plan, err := kit.FromMerge(merged.MergeResult)
|
||||||
|
if err != nil {
|
||||||
|
return nil, kit.ImageConfig{}, err
|
||||||
|
}
|
||||||
|
img, err := l.imageConfig(ctx, refs[0])
|
||||||
|
if err != nil {
|
||||||
|
return nil, kit.ImageConfig{}, err
|
||||||
|
}
|
||||||
|
for _, name := range slices.Sorted(maps.Keys(merged.Env)) {
|
||||||
|
img.Env = append(img.Env, name+"="+merged.Env[name])
|
||||||
|
}
|
||||||
|
return plan, img, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// Release stops the run's container, then removes its network and ends
|
||||||
|
// its egress session.
|
||||||
|
func (l *Launcher) Release(ctx context.Context, run *Run) error {
|
||||||
|
l.Pool.Release(ctx, run.Container)
|
||||||
|
l.Proxy.Revoke(run.token)
|
||||||
|
return l.Networks.Remove(context.WithoutCancel(ctx), run.network)
|
||||||
|
}
|
||||||
|
|
||||||
|
// RemoveVolumes removes an ended execution's Kit volumes.
|
||||||
|
func (l *Launcher) RemoveVolumes(ctx context.Context, volumes []kit.Volume) error {
|
||||||
|
return container.RemoveKitVolumes(ctx, l.Pool.Client(), volumes)
|
||||||
|
}
|
||||||
|
|
||||||
|
// startEgress starts the relay on first use, so a daemon with no Kit
|
||||||
|
// profile runs no relay.
|
||||||
|
func (l *Launcher) startEgress(ctx context.Context) error {
|
||||||
|
l.mu.Lock()
|
||||||
|
defer l.mu.Unlock()
|
||||||
|
if l.started {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
if err := l.Networks.Start(ctx); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
l.started = true
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func (l *Launcher) imageConfig(ctx context.Context, ref string) (kit.ImageConfig, error) {
|
||||||
|
if err := l.Pool.EnsureImage(ctx, ref); err != nil {
|
||||||
|
return kit.ImageConfig{}, err
|
||||||
|
}
|
||||||
|
inspected, err := l.Pool.Client().ImageInspect(ctx, ref)
|
||||||
|
if err != nil {
|
||||||
|
return kit.ImageConfig{}, fmt.Errorf("inspect kit image %s: %w", ref, err)
|
||||||
|
}
|
||||||
|
cfg := inspected.Config
|
||||||
|
if cfg == nil {
|
||||||
|
return kit.ImageConfig{}, fmt.Errorf("kit image %s has no config", ref)
|
||||||
|
}
|
||||||
|
return kit.ImageConfig{
|
||||||
|
Entrypoint: slices.Clone(cfg.Entrypoint), Cmd: slices.Clone(cfg.Cmd),
|
||||||
|
Env: slices.Clone(cfg.Env), User: cfg.User,
|
||||||
|
}, nil
|
||||||
|
}
|
||||||
Reference in new issue
Block a user