diff --git a/internal/app/archied/harness.go b/internal/app/archied/harness.go index 91cdae396..b7731bf78 100644 --- a/internal/app/archied/harness.go +++ b/internal/app/archied/harness.go @@ -246,7 +246,7 @@ func (b *boot) runView() *runView { return &runView{ tasks: b.stateStore, follow: func(ctx context.Context, taskID, attempt int64) (io.ReadCloser, error) { - return container.FollowLogs(ctx, cli, kitrun.TaskContainer(taskID, attempt), runTail) + return container.FollowLogs(ctx, cli, kitrun.TaskContainer(b.containerPool.Instance(), taskID, attempt), runTail) }, } } diff --git a/internal/app/archied/kit_launcher.go b/internal/app/archied/kit_launcher.go index 958a3aee0..10de7c272 100644 --- a/internal/app/archied/kit_launcher.go +++ b/internal/app/archied/kit_launcher.go @@ -21,8 +21,6 @@ import ( "github.com/samcharles93/archie-core/internal/infrastructure/tasktools" ) -const relayName = "archie-egress-relay" - // setupKitLauncher builds the egress path Kit tasks run on: the per-daemon // CA, the proxy on the worker bridge's host gateway, and the relay. It // degrades to no launcher, which parks Kit tasks, rather than failing boot: @@ -89,7 +87,7 @@ func (b *boot) setupKitLauncher(ctx context.Context) { return } networks := egress.NewNetworks(pool.Client(), egress.RelaySpec{ - Name: relayName, Image: b.cfg.Containers.Image, + Name: b.cfg.Instance + "-egress-relay", Image: b.cfg.Containers.Image, Entrypoint: []string{"archie-agent"}, Cmd: cmd, Network: pool.NetworkName(), }) b.addCleanup(shutdownEgress(networks, srv, log)) diff --git a/internal/app/archied/main.go b/internal/app/archied/main.go index 6e4a2846e..9823ef027 100644 --- a/internal/app/archied/main.go +++ b/internal/app/archied/main.go @@ -430,18 +430,19 @@ func startContainers( // Only the embedded broker binds a discovered host gateway. The // standalone State Store uses its configured, container-reachable target. RequireHostGateway: cfg.NATS.Mode == config.NATSModeEmbedded, + Instance: cfg.Instance, }, containerLiveSettings(cfgHolder), log) if err != nil { // A missing image is recoverable by hand. log.Error("autonomous workflows unavailable: container pool unavailable", "err", err, "hint", "run `docker compose pull agent` (or `build agent`) so the image is present locally") - return nil, storage.NewDockerBackend(dockerCli), closeDocker + return nil, storage.NewDockerBackend(dockerCli, cfg.Instance), closeDocker } // contextcheck: Pool.Close takes no context by design -- it is a shutdown // path that must run to completion after ctx is already cancelled. //nolint:contextcheck - return pool, storage.NewDockerBackend(dockerCli), func() { + return pool, storage.NewDockerBackend(dockerCli, cfg.Instance), func() { if err := pool.Close(); err != nil { log.Warn("container pool close failed", "err", err) } diff --git a/internal/config/config.go b/internal/config/config.go index b3590f99f..df2d2a1ea 100644 --- a/internal/config/config.go +++ b/internal/config/config.go @@ -124,7 +124,7 @@ type Repo struct { // ecosystem default. TestGlob string `toml:"test_glob" json:"test_glob" yaml:"test_glob"` // PersistentStorage creates a named Docker volume for this repo - // (archie-repo--) mounted at /data/repo. The volume + // (-repo--) mounted at /data/repo. The volume // survives task completion and retains data across tasks. Use for // repos with expensive build artifacts or large dependency trees. PersistentStorage bool `toml:"persistent_storage" json:"persistent_storage" yaml:"persistent_storage"` diff --git a/internal/container/pool.go b/internal/container/pool.go index ccf25e283..a6c463186 100644 --- a/internal/container/pool.go +++ b/internal/container/pool.go @@ -123,6 +123,12 @@ func (p *Pool) current() Config { return cfg } +// daemonLabel marks containers a pool owns; its value is the instance name. +const daemonLabel = "archie-daemon" + +// Instance is the name of the archie instance the pool belongs to. +func (p *Pool) Instance() string { return p.cfg.Instance } + // Config is the subset of daemon container configuration the pool needs. type Config struct { Image string @@ -150,6 +156,10 @@ type Config struct { // RequireHostGateway resolves and validates a local Docker bridge gateway. // Embedded NATS needs this address; external brokers do not. RequireHostGateway bool + // Instance names this archie instance. It prefixes container names and + // is the value of the daemon label the reaper filters on, so instances + // sharing a Docker host never remove each other's containers. + Instance string } // NewPool connects to Docker, optionally pulls the image and removes orphaned @@ -231,7 +241,7 @@ func (p *Pool) Acquire(ctx context.Context, image string, mounts []storage.Mount return nil, err } - name := fmt.Sprintf("archie-agent-%d", time.Now().UnixNano()) + name := fmt.Sprintf("%s-agent-%d", cfg.Instance, time.Now().UnixNano()) hostConfig := &container.HostConfig{AutoRemove: true} for i, m := range storage.ConvertMounts(mounts) { @@ -265,7 +275,7 @@ func (p *Pool) Acquire(ctx context.Context, image string, mounts []storage.Mount Image: image, Env: env, Labels: map[string]string{ - "archie-daemon": "true", + daemonLabel: cfg.Instance, }, }, HostConfig: hostConfig, @@ -300,7 +310,7 @@ func (p *Pool) AcquireKit(ctx context.Context, s KitSpec) (*Container, error) { if s.Labels == nil { s.Labels = map[string]string{} } - s.Labels["archie-daemon"] = "true" + s.Labels[daemonLabel] = p.cfg.Instance id, err := StartKit(ctx, p.cli, s) if err != nil { p.unreserve() @@ -499,7 +509,7 @@ func (p *Pool) Close() error { list, err := p.cli.ContainerList(ctx, client.ContainerListOptions{ All: true, - Filters: client.Filters{}.Add("label", "archie-daemon=true"), + Filters: client.Filters{}.Add("label", daemonLabel+"="+p.cfg.Instance), }) if err != nil { p.log.Warn("container list failed during close", "err", err) @@ -652,7 +662,7 @@ func selfNetwork(ctx context.Context, cli *client.Client, log *slog.Logger) stri func (p *Pool) recoverOrphans(ctx context.Context) { list, err := p.cli.ContainerList(ctx, client.ContainerListOptions{ All: true, - Filters: client.Filters{}.Add("label", "archie-daemon=true"), + Filters: client.Filters{}.Add("label", daemonLabel+"="+p.cfg.Instance), }) if err != nil { return diff --git a/internal/container/pool_test.go b/internal/container/pool_test.go new file mode 100644 index 000000000..613d494c5 --- /dev/null +++ b/internal/container/pool_test.go @@ -0,0 +1,76 @@ +package container + +import ( + "encoding/json" + "log/slog" + "net/http" + "net/http/httptest" + "slices" + "strings" + "sync" + "testing" + + "github.com/moby/moby/client" +) + +// TestPoolReapsOnlyOwnInstance runs the reaper against a fake Engine that +// applies label filters the way Docker does, so instances sharing a host +// never remove each other's containers. +func TestPoolReapsOnlyOwnInstance(t *testing.T) { + containers := map[string]string{ // ID -> archie-daemon label value + "own000000000": "archie", + "other0000000": "staging", + "legacy000000": "true", + } + tests := []struct { + name string + reap func(*Pool) + }{ + {"close", func(p *Pool) { _ = p.Close() }}, + {"orphan recovery", func(p *Pool) { p.recoverOrphans(t.Context()) }}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + var ( + mu sync.Mutex + removed []string + ) + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + switch { + case r.Method == http.MethodGet && strings.HasSuffix(r.URL.Path, "/containers/json"): + var filters map[string]map[string]bool + if err := json.Unmarshal([]byte(r.URL.Query().Get("filters")), &filters); err != nil { + t.Error(err) + } + var items []map[string]any + for id, value := range containers { + if filters["label"]["archie-daemon="+value] { + items = append(items, map[string]any{"Id": id}) + } + } + w.Header().Set("Content-Type", "application/json") + _ = json.NewEncoder(w).Encode(items) + case r.Method == http.MethodPost && strings.HasSuffix(r.URL.Path, "/stop"): + w.WriteHeader(http.StatusNoContent) + case r.Method == http.MethodDelete: + mu.Lock() + removed = append(removed, r.URL.Path[strings.LastIndex(r.URL.Path, "/")+1:]) + mu.Unlock() + w.WriteHeader(http.StatusNoContent) + default: + http.NotFound(w, r) + } + })) + defer server.Close() + cli, err := client.New(client.WithHost(server.URL), client.WithAPIVersion("1.54")) + if err != nil { + t.Fatal(err) + } + defer cli.Close() + tt.reap(&Pool{cli: cli, cfg: Config{Instance: "archie"}, log: slog.New(slog.DiscardHandler)}) + if !slices.Equal(removed, []string{"own000000000"}) { + t.Fatalf("removed %v, want only the own instance's container", removed) + } + }) + } +} diff --git a/internal/container/reown.go b/internal/container/reown.go index 9225301a2..a2f3aa64b 100644 --- a/internal/container/reown.go +++ b/internal/container/reown.go @@ -22,7 +22,7 @@ func (p *Pool) Reown(ctx context.Context, dir string, uid, gid int) error { Config: &container.Config{ Image: cfg.Image, Entrypoint: []string{"chown", "-R", strconv.Itoa(uid) + ":" + strconv.Itoa(gid), "/w"}, - Labels: map[string]string{"archie-daemon": "true"}, + Labels: map[string]string{daemonLabel: cfg.Instance}, }, HostConfig: &container.HostConfig{ NetworkMode: "none", diff --git a/internal/infrastructure/kit/launch.go b/internal/infrastructure/kit/launch.go index 847b58bc9..dc8a44d44 100644 --- a/internal/infrastructure/kit/launch.go +++ b/internal/infrastructure/kit/launch.go @@ -54,6 +54,9 @@ type OAuthFacts struct { // LaunchParams are the run-specific inputs to a Kit container. type LaunchParams struct { + // Instance prefixes the Kit's volume names, so instances sharing a + // Docker host never mount each other's volumes. + Instance string // Execution keys the Kit's volumes, so they survive retries of one // WorkflowExecution and are never shared with another. Execution string @@ -164,7 +167,7 @@ func Assemble(p *Plan, img ImageConfig, params LaunchParams) (Launch, error) { } l.Files = append(l.Files, credential...) for _, v := range volumes { - l.Volumes = append(l.Volumes, Volume{Name: volumeName(params.Execution, v.Path), Path: v.Path}) + l.Volumes = append(l.Volumes, Volume{Name: volumeName(params.Instance, params.Execution, v.Path), Path: v.Path}) } if l.Context, err = contextFiles(p); err != nil { return Launch{}, err @@ -306,7 +309,7 @@ func orDefault(v, fallback string) string { return v } -func volumeName(execution, path string) string { +func volumeName(instance, execution, path string) string { sum := sha256.Sum256([]byte(path)) - return "archie-kit-" + unsafeVolumeChars.ReplaceAllString(execution, "_") + "-" + hex.EncodeToString(sum[:6]) + return instance + "-kit-" + unsafeVolumeChars.ReplaceAllString(execution, "_") + "-" + hex.EncodeToString(sum[:6]) } diff --git a/internal/infrastructure/kitrun/launcher.go b/internal/infrastructure/kitrun/launcher.go index e8b1f0d92..c91ad1dfc 100644 --- a/internal/infrastructure/kitrun/launcher.go +++ b/internal/infrastructure/kitrun/launcher.go @@ -174,12 +174,12 @@ func (l *Launcher) LaunchSetup(ctx context.Context, req SetupRequest) (*Run, err if err != nil { return nil, err } - run := &Run{network: "archie-kit-" + req.Session, token: session.Token(), execution: req.Session, grants: l.Grants, ephemeral: true} + run := &Run{network: l.Pool.Instance() + "-kit-" + req.Session, token: session.Token(), execution: req.Session, grants: l.Grants, ephemeral: true} // Bound is deliberately empty here: a credential file renders the stored // scopes and expiry, and there is no stored token set yet -- that is the // state this session exists to change. The Kit's sentinels still reach the // harness environment. - launch, err := kit.Assemble(k.plan, k.img, kit.LaunchParams{Execution: req.Session, ProxyToken: session.Token(), CAPath: caPath}) + launch, err := kit.Assemble(k.plan, k.img, kit.LaunchParams{Instance: l.Pool.Instance(), Execution: req.Session, ProxyToken: session.Token(), CAPath: caPath}) if err != nil { l.release(ctx, run) return nil, err @@ -196,7 +196,7 @@ func (l *Launcher) LaunchSetup(ctx context.Context, req SetupRequest) (*Run, err // environment stays empty (WorkerEnv nil): the harness env is passed // explicitly on the exec, and no worker credential reaches it. run.Container, err = l.Pool.AcquireKit(ctx, container.KitSpec{ - Name: "archie-kit-" + req.Session, + Name: run.network, Image: req.Kit, Network: run.network, Worker: []string{agentPath, "hold"}, @@ -251,14 +251,14 @@ func newProxyToken() (string, error) { func TaskExecution(taskID int64) string { return fmt.Sprintf("task-%d", taskID) } // TaskContainer is the container name of a task's Kit run. -func TaskContainer(taskID, attempt int64) string { - return fmt.Sprintf("archie-kit-%s-attempt-%d", TaskExecution(taskID), attempt) +func TaskContainer(instance string, taskID, attempt int64) string { + return fmt.Sprintf("%s-kit-%s-attempt-%d", instance, TaskExecution(taskID), attempt) } // Launch reads the request's Kit and starts its container. func (l *Launcher) Launch(ctx context.Context, req Request) (*Run, error) { execution := TaskExecution(req.TaskID) - containerName := TaskContainer(req.TaskID, int64(req.Attempt)) + containerName := TaskContainer(l.Pool.Instance(), req.TaskID, int64(req.Attempt)) if req.Kit == "" { return nil, errors.New("kit profile names no kit") } @@ -283,7 +283,7 @@ func (l *Launcher) Launch(ctx context.Context, req Request) (*Run, error) { return nil, err } run := &Run{network: containerName, token: session.Token(), execution: execution} - launch, err := kit.Assemble(k.plan, k.img, kit.LaunchParams{Execution: execution, ProxyToken: session.Token(), CAPath: caPath, Bound: kinds, OAuth: facts}) + launch, err := kit.Assemble(k.plan, k.img, kit.LaunchParams{Instance: l.Pool.Instance(), Execution: execution, ProxyToken: session.Token(), CAPath: caPath, Bound: kinds, OAuth: facts}) if err != nil { l.release(ctx, run) return nil, err diff --git a/internal/storage/storage.go b/internal/storage/storage.go index 8c158a151..dd79af6da 100644 --- a/internal/storage/storage.go +++ b/internal/storage/storage.go @@ -142,46 +142,46 @@ type Backend interface { // cacheVolume describes a named Docker volume that is shared across tasks. type cacheVolume struct { - Name string // Docker volume name + Name string // Docker volume name after the instance prefix MountPath string // container destination path } // alwaysOnCacheVolumes are mounted on every task container. var alwaysOnCacheVolumes = []cacheVolume{ - {Name: "archie-cache-mcp-npm", MountPath: MCPNPMCacheMountDir}, + {Name: "cache-mcp-npm", MountPath: MCPNPMCacheMountDir}, } // cacheVolumesByEcosystem maps ecosystem names to their shared cache volumes. // Volumes are created once by the pool and never deleted. var cacheVolumesByEcosystem = map[string][]cacheVolume{ "go": { - {Name: "archie-cache-go", MountPath: "/data/cache/go"}, + {Name: "cache-go", MountPath: "/data/cache/go"}, }, "node": { - {Name: "archie-cache-node", MountPath: "/data/cache/node"}, - {Name: "archie-cache-pnpm", MountPath: "/data/cache/pnpm"}, + {Name: "cache-node", MountPath: "/data/cache/node"}, + {Name: "cache-pnpm", MountPath: "/data/cache/pnpm"}, }, "typescript": { - {Name: "archie-cache-node", MountPath: "/data/cache/node"}, - {Name: "archie-cache-pnpm", MountPath: "/data/cache/pnpm"}, + {Name: "cache-node", MountPath: "/data/cache/node"}, + {Name: "cache-pnpm", MountPath: "/data/cache/pnpm"}, }, "deno": { - {Name: "archie-cache-deno", MountPath: "/data/cache/deno"}, + {Name: "cache-deno", MountPath: "/data/cache/deno"}, }, "bun": { - {Name: "archie-cache-bun", MountPath: "/data/cache/bun"}, + {Name: "cache-bun", MountPath: "/data/cache/bun"}, }, "python": { - {Name: "archie-cache-pip", MountPath: "/data/cache/pip"}, + {Name: "cache-pip", MountPath: "/data/cache/pip"}, }, "rust": { - {Name: "archie-cache-cargo", MountPath: "/data/cache/cargo"}, + {Name: "cache-cargo", MountPath: "/data/cache/cargo"}, }, } // cacheMounts returns the always-on mounts plus the ecosystem's, sorted by // destination. Ecosystem is case-insensitive. -func cacheMounts(ecosystem string) []Mount { +func cacheMounts(instance, ecosystem string) []Mount { vols := append([]cacheVolume(nil), alwaysOnCacheVolumes...) vols = append(vols, cacheVolumesByEcosystem[strings.ToLower(ecosystem)]...) @@ -189,7 +189,7 @@ func cacheMounts(ecosystem string) []Mount { for i, v := range vols { mounts[i] = Mount{ Type: MountTypeVolume, - Source: v.Name, + Source: instance + "-" + v.Name, Destination: v.MountPath, } } @@ -219,17 +219,21 @@ func (e *ErrVolumeCreate) Unwrap() error { return e.Cause } type DockerBackend struct { cli *client.Client now func() time.Time + // instance prefixes volume names and labels them, so instances sharing + // a Docker host never mount or expire each other's volumes. + instance string } const ( storageLabel = "com.samcharles93.archie.storage" storageKindRepo = "repo" storageKindCache = "cache" + instanceLabel = "com.samcharles93.archie.instance" ) // NewDockerBackend creates a Docker storage backend. -func NewDockerBackend(cli *client.Client) *DockerBackend { - return &DockerBackend{cli: cli} +func NewDockerBackend(cli *client.Client, instance string) *DockerBackend { + return &DockerBackend{cli: cli, instance: instance} } // ensureVolume creates a Docker volume unless it exists. A nil client does @@ -262,8 +266,8 @@ func (d *DockerBackend) ensureVolume(ctx context.Context, name string, labels ma } // repoVolumeName returns the Docker volume name for a repo's persistent storage. -func repoVolumeName(owner, repo string) string { - return fmt.Sprintf("archie-repo-%s-%s", owner, repo) +func repoVolumeName(instance, owner, repo string) string { + return fmt.Sprintf("%s-repo-%s-%s", instance, owner, repo) } // Setup returns a task container's mounts: the worktree, the repo volume @@ -279,8 +283,8 @@ func (d *DockerBackend) Setup(ctx context.Context, task TaskRef) ([]Mount, error // Per-repo persistent volume. if task.PersistentStorage && task.Owner != "" && task.Repo != "" { - volName := repoVolumeName(task.Owner, task.Repo) - if err := d.ensureVolume(ctx, volName, map[string]string{storageLabel: storageKindRepo}); err != nil { + volName := repoVolumeName(d.instance, task.Owner, task.Repo) + if err := d.ensureVolume(ctx, volName, map[string]string{storageLabel: storageKindRepo, instanceLabel: d.instance}); err != nil { return nil, err } mounts = append(mounts, Mount{ @@ -290,9 +294,9 @@ func (d *DockerBackend) Setup(ctx context.Context, task TaskRef) ([]Mount, error }) } - caches := cacheMounts(task.Ecosystem) + caches := cacheMounts(d.instance, task.Ecosystem) for _, m := range caches { - if err := d.ensureVolume(ctx, m.Source, map[string]string{storageLabel: storageKindCache}); err != nil { + if err := d.ensureVolume(ctx, m.Source, map[string]string{storageLabel: storageKindCache, instanceLabel: d.instance}); err != nil { return nil, err } } @@ -316,7 +320,7 @@ func (d *DockerBackend) CleanupExpired(ctx context.Context, ttl time.Duration) ( return 0, nil } result, err := d.cli.VolumeList(ctx, client.VolumeListOptions{ - Filters: client.Filters{}.Add("label", storageLabel+"="+storageKindRepo), + Filters: client.Filters{}.Add("label", storageLabel+"="+storageKindRepo).Add("label", instanceLabel+"="+d.instance), }) if err != nil { return 0, fmt.Errorf("list persistent volumes: %w", err) @@ -332,7 +336,7 @@ func (d *DockerBackend) CleanupExpired(ctx context.Context, ttl time.Duration) ( ) for _, vol := range result.Items { // Defend against Docker implementations that ignore label filters. - if vol.Labels[storageLabel] != storageKindRepo { + if vol.Labels[storageLabel] != storageKindRepo || vol.Labels[instanceLabel] != d.instance { continue } created, err := time.Parse(time.RFC3339Nano, vol.CreatedAt)