fix(container): scope agent containers, volumes and the egress relay to the instance

This commit is contained in:
sam committed 2026-10-19 10:15:47 +11:00
1 parent ffdb1ce583
commit a33776db82
10 files changed
+138 -46

No files matched your search

+1 -1
View File
@@ -246,7 +246,7 @@ func (b *boot) runView() *runView {
return &runView{
tasks: b.stateStore,
follow: func(ctx context.Context, taskID, attempt int64) (io.ReadCloser, error) {
return container.FollowLogs(ctx, cli, kitrun.TaskContainer(taskID, attempt), runTail)
return container.FollowLogs(ctx, cli, kitrun.TaskContainer(b.containerPool.Instance(), taskID, attempt), runTail)
},
}
}
+1 -3
View File
@@ -21,8 +21,6 @@ import (
"github.com/samcharles93/archie-core/internal/infrastructure/tasktools"
)
const relayName = "archie-egress-relay"
// setupKitLauncher builds the egress path Kit tasks run on: the per-daemon
// CA, the proxy on the worker bridge's host gateway, and the relay. It
// degrades to no launcher, which parks Kit tasks, rather than failing boot:
@@ -89,7 +87,7 @@ func (b *boot) setupKitLauncher(ctx context.Context) {
return
}
networks := egress.NewNetworks(pool.Client(), egress.RelaySpec{
Name: relayName, Image: b.cfg.Containers.Image,
Name: b.cfg.Instance + "-egress-relay", Image: b.cfg.Containers.Image,
Entrypoint: []string{"archie-agent"}, Cmd: cmd, Network: pool.NetworkName(),
})
b.addCleanup(shutdownEgress(networks, srv, log))
+3 -2
View File
@@ -430,18 +430,19 @@ func startContainers(
// Only the embedded broker binds a discovered host gateway. The
// standalone State Store uses its configured, container-reachable target.
RequireHostGateway: cfg.NATS.Mode == config.NATSModeEmbedded,
Instance: cfg.Instance,
}, containerLiveSettings(cfgHolder), log)
if err != nil {
// A missing image is recoverable by hand.
log.Error("autonomous workflows unavailable: container pool unavailable", "err", err,
"hint", "run `docker compose pull agent` (or `build agent`) so the image is present locally")
return nil, storage.NewDockerBackend(dockerCli), closeDocker
return nil, storage.NewDockerBackend(dockerCli, cfg.Instance), closeDocker
}
// contextcheck: Pool.Close takes no context by design -- it is a shutdown
// path that must run to completion after ctx is already cancelled.
//nolint:contextcheck
return pool, storage.NewDockerBackend(dockerCli), func() {
return pool, storage.NewDockerBackend(dockerCli, cfg.Instance), func() {
if err := pool.Close(); err != nil {
log.Warn("container pool close failed", "err", err)
}
+1 -1
View File
@@ -124,7 +124,7 @@ type Repo struct {
// ecosystem default.
TestGlob string `toml:"test_glob" json:"test_glob" yaml:"test_glob"`
// PersistentStorage creates a named Docker volume for this repo
// (archie-repo-<owner>-<repo>) mounted at /data/repo. The volume
// (<instance>-repo-<owner>-<repo>) mounted at /data/repo. The volume
// survives task completion and retains data across tasks. Use for
// repos with expensive build artifacts or large dependency trees.
PersistentStorage bool `toml:"persistent_storage" json:"persistent_storage" yaml:"persistent_storage"`
+15 -5
View File
@@ -123,6 +123,12 @@ func (p *Pool) current() Config {
return cfg
}
// daemonLabel marks containers a pool owns; its value is the instance name.
const daemonLabel = "archie-daemon"
// Instance is the name of the archie instance the pool belongs to.
func (p *Pool) Instance() string { return p.cfg.Instance }
// Config is the subset of daemon container configuration the pool needs.
type Config struct {
Image string
@@ -150,6 +156,10 @@ type Config struct {
// RequireHostGateway resolves and validates a local Docker bridge gateway.
// Embedded NATS needs this address; external brokers do not.
RequireHostGateway bool
// Instance names this archie instance. It prefixes container names and
// is the value of the daemon label the reaper filters on, so instances
// sharing a Docker host never remove each other's containers.
Instance string
}
// NewPool connects to Docker, optionally pulls the image and removes orphaned
@@ -231,7 +241,7 @@ func (p *Pool) Acquire(ctx context.Context, image string, mounts []storage.Mount
return nil, err
}
name := fmt.Sprintf("archie-agent-%d", time.Now().UnixNano())
name := fmt.Sprintf("%s-agent-%d", cfg.Instance, time.Now().UnixNano())
hostConfig := &container.HostConfig{AutoRemove: true}
for i, m := range storage.ConvertMounts(mounts) {
@@ -265,7 +275,7 @@ func (p *Pool) Acquire(ctx context.Context, image string, mounts []storage.Mount
Image: image,
Env: env,
Labels: map[string]string{
"archie-daemon": "true",
daemonLabel: cfg.Instance,
},
},
HostConfig: hostConfig,
@@ -300,7 +310,7 @@ func (p *Pool) AcquireKit(ctx context.Context, s KitSpec) (*Container, error) {
if s.Labels == nil {
s.Labels = map[string]string{}
}
s.Labels["archie-daemon"] = "true"
s.Labels[daemonLabel] = p.cfg.Instance
id, err := StartKit(ctx, p.cli, s)
if err != nil {
p.unreserve()
@@ -499,7 +509,7 @@ func (p *Pool) Close() error {
list, err := p.cli.ContainerList(ctx, client.ContainerListOptions{
All: true,
Filters: client.Filters{}.Add("label", "archie-daemon=true"),
Filters: client.Filters{}.Add("label", daemonLabel+"="+p.cfg.Instance),
})
if err != nil {
p.log.Warn("container list failed during close", "err", err)
@@ -652,7 +662,7 @@ func selfNetwork(ctx context.Context, cli *client.Client, log *slog.Logger) stri
func (p *Pool) recoverOrphans(ctx context.Context) {
list, err := p.cli.ContainerList(ctx, client.ContainerListOptions{
All: true,
Filters: client.Filters{}.Add("label", "archie-daemon=true"),
Filters: client.Filters{}.Add("label", daemonLabel+"="+p.cfg.Instance),
})
if err != nil {
return
+76
View File
@@ -0,0 +1,76 @@
package container
import (
"encoding/json"
"log/slog"
"net/http"
"net/http/httptest"
"slices"
"strings"
"sync"
"testing"
"github.com/moby/moby/client"
)
// TestPoolReapsOnlyOwnInstance runs the reaper against a fake Engine that
// applies label filters the way Docker does, so instances sharing a host
// never remove each other's containers.
func TestPoolReapsOnlyOwnInstance(t *testing.T) {
containers := map[string]string{ // ID -> archie-daemon label value
"own000000000": "archie",
"other0000000": "staging",
"legacy000000": "true",
}
tests := []struct {
name string
reap func(*Pool)
}{
{"close", func(p *Pool) { _ = p.Close() }},
{"orphan recovery", func(p *Pool) { p.recoverOrphans(t.Context()) }},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
var (
mu sync.Mutex
removed []string
)
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
switch {
case r.Method == http.MethodGet && strings.HasSuffix(r.URL.Path, "/containers/json"):
var filters map[string]map[string]bool
if err := json.Unmarshal([]byte(r.URL.Query().Get("filters")), &filters); err != nil {
t.Error(err)
}
var items []map[string]any
for id, value := range containers {
if filters["label"]["archie-daemon="+value] {
items = append(items, map[string]any{"Id": id})
}
}
w.Header().Set("Content-Type", "application/json")
_ = json.NewEncoder(w).Encode(items)
case r.Method == http.MethodPost && strings.HasSuffix(r.URL.Path, "/stop"):
w.WriteHeader(http.StatusNoContent)
case r.Method == http.MethodDelete:
mu.Lock()
removed = append(removed, r.URL.Path[strings.LastIndex(r.URL.Path, "/")+1:])
mu.Unlock()
w.WriteHeader(http.StatusNoContent)
default:
http.NotFound(w, r)
}
}))
defer server.Close()
cli, err := client.New(client.WithHost(server.URL), client.WithAPIVersion("1.54"))
if err != nil {
t.Fatal(err)
}
defer cli.Close()
tt.reap(&Pool{cli: cli, cfg: Config{Instance: "archie"}, log: slog.New(slog.DiscardHandler)})
if !slices.Equal(removed, []string{"own000000000"}) {
t.Fatalf("removed %v, want only the own instance's container", removed)
}
})
}
}
+1 -1
View File
@@ -22,7 +22,7 @@ func (p *Pool) Reown(ctx context.Context, dir string, uid, gid int) error {
Config: &container.Config{
Image: cfg.Image,
Entrypoint: []string{"chown", "-R", strconv.Itoa(uid) + ":" + strconv.Itoa(gid), "/w"},
Labels: map[string]string{"archie-daemon": "true"},
Labels: map[string]string{daemonLabel: cfg.Instance},
},
HostConfig: &container.HostConfig{
NetworkMode: "none",
+6 -3
View File
@@ -54,6 +54,9 @@ type OAuthFacts struct {
// LaunchParams are the run-specific inputs to a Kit container.
type LaunchParams struct {
// Instance prefixes the Kit's volume names, so instances sharing a
// Docker host never mount each other's volumes.
Instance string
// Execution keys the Kit's volumes, so they survive retries of one
// WorkflowExecution and are never shared with another.
Execution string
@@ -164,7 +167,7 @@ func Assemble(p *Plan, img ImageConfig, params LaunchParams) (Launch, error) {
}
l.Files = append(l.Files, credential...)
for _, v := range volumes {
l.Volumes = append(l.Volumes, Volume{Name: volumeName(params.Execution, v.Path), Path: v.Path})
l.Volumes = append(l.Volumes, Volume{Name: volumeName(params.Instance, params.Execution, v.Path), Path: v.Path})
}
if l.Context, err = contextFiles(p); err != nil {
return Launch{}, err
@@ -306,7 +309,7 @@ func orDefault(v, fallback string) string {
return v
}
func volumeName(execution, path string) string {
func volumeName(instance, execution, path string) string {
sum := sha256.Sum256([]byte(path))
return "archie-kit-" + unsafeVolumeChars.ReplaceAllString(execution, "_") + "-" + hex.EncodeToString(sum[:6])
return instance + "-kit-" + unsafeVolumeChars.ReplaceAllString(execution, "_") + "-" + hex.EncodeToString(sum[:6])
}
+7 -7
View File
@@ -174,12 +174,12 @@ func (l *Launcher) LaunchSetup(ctx context.Context, req SetupRequest) (*Run, err
if err != nil {
return nil, err
}
run := &Run{network: "archie-kit-" + req.Session, token: session.Token(), execution: req.Session, grants: l.Grants, ephemeral: true}
run := &Run{network: l.Pool.Instance() + "-kit-" + req.Session, token: session.Token(), execution: req.Session, grants: l.Grants, ephemeral: true}
// Bound is deliberately empty here: a credential file renders the stored
// scopes and expiry, and there is no stored token set yet -- that is the
// state this session exists to change. The Kit's sentinels still reach the
// harness environment.
launch, err := kit.Assemble(k.plan, k.img, kit.LaunchParams{Execution: req.Session, ProxyToken: session.Token(), CAPath: caPath})
launch, err := kit.Assemble(k.plan, k.img, kit.LaunchParams{Instance: l.Pool.Instance(), Execution: req.Session, ProxyToken: session.Token(), CAPath: caPath})
if err != nil {
l.release(ctx, run)
return nil, err
@@ -196,7 +196,7 @@ func (l *Launcher) LaunchSetup(ctx context.Context, req SetupRequest) (*Run, err
// environment stays empty (WorkerEnv nil): the harness env is passed
// explicitly on the exec, and no worker credential reaches it.
run.Container, err = l.Pool.AcquireKit(ctx, container.KitSpec{
Name: "archie-kit-" + req.Session,
Name: run.network,
Image: req.Kit,
Network: run.network,
Worker: []string{agentPath, "hold"},
@@ -251,14 +251,14 @@ func newProxyToken() (string, error) {
func TaskExecution(taskID int64) string { return fmt.Sprintf("task-%d", taskID) }
// TaskContainer is the container name of a task's Kit run.
func TaskContainer(taskID, attempt int64) string {
return fmt.Sprintf("archie-kit-%s-attempt-%d", TaskExecution(taskID), attempt)
func TaskContainer(instance string, taskID, attempt int64) string {
return fmt.Sprintf("%s-kit-%s-attempt-%d", instance, TaskExecution(taskID), attempt)
}
// Launch reads the request's Kit and starts its container.
func (l *Launcher) Launch(ctx context.Context, req Request) (*Run, error) {
execution := TaskExecution(req.TaskID)
containerName := TaskContainer(req.TaskID, int64(req.Attempt))
containerName := TaskContainer(l.Pool.Instance(), req.TaskID, int64(req.Attempt))
if req.Kit == "" {
return nil, errors.New("kit profile names no kit")
}
@@ -283,7 +283,7 @@ func (l *Launcher) Launch(ctx context.Context, req Request) (*Run, error) {
return nil, err
}
run := &Run{network: containerName, token: session.Token(), execution: execution}
launch, err := kit.Assemble(k.plan, k.img, kit.LaunchParams{Execution: execution, ProxyToken: session.Token(), CAPath: caPath, Bound: kinds, OAuth: facts})
launch, err := kit.Assemble(k.plan, k.img, kit.LaunchParams{Instance: l.Pool.Instance(), Execution: execution, ProxyToken: session.Token(), CAPath: caPath, Bound: kinds, OAuth: facts})
if err != nil {
l.release(ctx, run)
return nil, err
+27 -23
View File
@@ -142,46 +142,46 @@ type Backend interface {
// cacheVolume describes a named Docker volume that is shared across tasks.
type cacheVolume struct {
Name string // Docker volume name
Name string // Docker volume name after the instance prefix
MountPath string // container destination path
}
// alwaysOnCacheVolumes are mounted on every task container.
var alwaysOnCacheVolumes = []cacheVolume{
{Name: "archie-cache-mcp-npm", MountPath: MCPNPMCacheMountDir},
{Name: "cache-mcp-npm", MountPath: MCPNPMCacheMountDir},
}
// cacheVolumesByEcosystem maps ecosystem names to their shared cache volumes.
// Volumes are created once by the pool and never deleted.
var cacheVolumesByEcosystem = map[string][]cacheVolume{
"go": {
{Name: "archie-cache-go", MountPath: "/data/cache/go"},
{Name: "cache-go", MountPath: "/data/cache/go"},
},
"node": {
{Name: "archie-cache-node", MountPath: "/data/cache/node"},
{Name: "archie-cache-pnpm", MountPath: "/data/cache/pnpm"},
{Name: "cache-node", MountPath: "/data/cache/node"},
{Name: "cache-pnpm", MountPath: "/data/cache/pnpm"},
},
"typescript": {
{Name: "archie-cache-node", MountPath: "/data/cache/node"},
{Name: "archie-cache-pnpm", MountPath: "/data/cache/pnpm"},
{Name: "cache-node", MountPath: "/data/cache/node"},
{Name: "cache-pnpm", MountPath: "/data/cache/pnpm"},
},
"deno": {
{Name: "archie-cache-deno", MountPath: "/data/cache/deno"},
{Name: "cache-deno", MountPath: "/data/cache/deno"},
},
"bun": {
{Name: "archie-cache-bun", MountPath: "/data/cache/bun"},
{Name: "cache-bun", MountPath: "/data/cache/bun"},
},
"python": {
{Name: "archie-cache-pip", MountPath: "/data/cache/pip"},
{Name: "cache-pip", MountPath: "/data/cache/pip"},
},
"rust": {
{Name: "archie-cache-cargo", MountPath: "/data/cache/cargo"},
{Name: "cache-cargo", MountPath: "/data/cache/cargo"},
},
}
// cacheMounts returns the always-on mounts plus the ecosystem's, sorted by
// destination. Ecosystem is case-insensitive.
func cacheMounts(ecosystem string) []Mount {
func cacheMounts(instance, ecosystem string) []Mount {
vols := append([]cacheVolume(nil), alwaysOnCacheVolumes...)
vols = append(vols, cacheVolumesByEcosystem[strings.ToLower(ecosystem)]...)
@@ -189,7 +189,7 @@ func cacheMounts(ecosystem string) []Mount {
for i, v := range vols {
mounts[i] = Mount{
Type: MountTypeVolume,
Source: v.Name,
Source: instance + "-" + v.Name,
Destination: v.MountPath,
}
}
@@ -219,17 +219,21 @@ func (e *ErrVolumeCreate) Unwrap() error { return e.Cause }
type DockerBackend struct {
cli *client.Client
now func() time.Time
// instance prefixes volume names and labels them, so instances sharing
// a Docker host never mount or expire each other's volumes.
instance string
}
const (
storageLabel = "com.samcharles93.archie.storage"
storageKindRepo = "repo"
storageKindCache = "cache"
instanceLabel = "com.samcharles93.archie.instance"
)
// NewDockerBackend creates a Docker storage backend.
func NewDockerBackend(cli *client.Client) *DockerBackend {
return &DockerBackend{cli: cli}
func NewDockerBackend(cli *client.Client, instance string) *DockerBackend {
return &DockerBackend{cli: cli, instance: instance}
}
// ensureVolume creates a Docker volume unless it exists. A nil client does
@@ -262,8 +266,8 @@ func (d *DockerBackend) ensureVolume(ctx context.Context, name string, labels ma
}
// repoVolumeName returns the Docker volume name for a repo's persistent storage.
func repoVolumeName(owner, repo string) string {
return fmt.Sprintf("archie-repo-%s-%s", owner, repo)
func repoVolumeName(instance, owner, repo string) string {
return fmt.Sprintf("%s-repo-%s-%s", instance, owner, repo)
}
// Setup returns a task container's mounts: the worktree, the repo volume
@@ -279,8 +283,8 @@ func (d *DockerBackend) Setup(ctx context.Context, task TaskRef) ([]Mount, error
// Per-repo persistent volume.
if task.PersistentStorage && task.Owner != "" && task.Repo != "" {
volName := repoVolumeName(task.Owner, task.Repo)
if err := d.ensureVolume(ctx, volName, map[string]string{storageLabel: storageKindRepo}); err != nil {
volName := repoVolumeName(d.instance, task.Owner, task.Repo)
if err := d.ensureVolume(ctx, volName, map[string]string{storageLabel: storageKindRepo, instanceLabel: d.instance}); err != nil {
return nil, err
}
mounts = append(mounts, Mount{
@@ -290,9 +294,9 @@ func (d *DockerBackend) Setup(ctx context.Context, task TaskRef) ([]Mount, error
})
}
caches := cacheMounts(task.Ecosystem)
caches := cacheMounts(d.instance, task.Ecosystem)
for _, m := range caches {
if err := d.ensureVolume(ctx, m.Source, map[string]string{storageLabel: storageKindCache}); err != nil {
if err := d.ensureVolume(ctx, m.Source, map[string]string{storageLabel: storageKindCache, instanceLabel: d.instance}); err != nil {
return nil, err
}
}
@@ -316,7 +320,7 @@ func (d *DockerBackend) CleanupExpired(ctx context.Context, ttl time.Duration) (
return 0, nil
}
result, err := d.cli.VolumeList(ctx, client.VolumeListOptions{
Filters: client.Filters{}.Add("label", storageLabel+"="+storageKindRepo),
Filters: client.Filters{}.Add("label", storageLabel+"="+storageKindRepo).Add("label", instanceLabel+"="+d.instance),
})
if err != nil {
return 0, fmt.Errorf("list persistent volumes: %w", err)
@@ -332,7 +336,7 @@ func (d *DockerBackend) CleanupExpired(ctx context.Context, ttl time.Duration) (
)
for _, vol := range result.Items {
// Defend against Docker implementations that ignore label filters.
if vol.Labels[storageLabel] != storageKindRepo {
if vol.Labels[storageLabel] != storageKindRepo || vol.Labels[instanceLabel] != d.instance {
continue
}
created, err := time.Parse(time.RFC3339Nano, vol.CreatedAt)