// Package container manages Docker containers running archie-agent. // A Pool acquires and releases containers per task, handling image pull, // creation, startup, health check, and teardown. The daemon hands each task to // its container as one core-NATS request. package container import ( "bytes" "context" "encoding/base64" "encoding/json" "errors" "fmt" "io" "log/slog" "os" "path/filepath" "strconv" "strings" "sync" "time" "github.com/moby/moby/api/pkg/stdcopy" "github.com/moby/moby/api/types/container" "github.com/moby/moby/api/types/registry" "github.com/moby/moby/client" "github.com/samcharles93/archie-core/internal/storage" ) // Container wraps a running Docker container. type Container struct { ID string exited <-chan struct{} // network is the task's sandbox network, removed by Release. Empty when the // pool runs without a sandbox. network string // detail is written before exited closes, so a reader that saw the close // may read it without a lock. detail *string } // Exited closes when the container stops running for any reason: the // max-uptime reaper, an OOM kill, a crash or Release. A Container the pool did // not start returns nil, which never closes. func (c *Container) Exited() <-chan struct{} { return c.exited } // ExitDetail says how the container ended: its exit code, and the runtime's // crash line when it panicked. It is empty until Exited closes, and for a // container the pool did not start. func (c *Container) ExitDetail() string { if c.detail == nil { return "" } select { case <-c.exited: return *c.detail default: return "" } } // TaskPayload is the boot-time brief written to /data/worktree/.git/task.json // before the container starts. type TaskPayload struct { ID int64 `json:"id"` Owner string `json:"owner"` Repo string `json:"repo"` Number int `json:"issue_number"` Title string `json:"title"` Body string `json:"body"` Labels []string `json:"labels"` Workflow string `json:"workflow"` Branch string `json:"branch,omitempty"` Plan string `json:"plan,omitempty"` // Inputs are the workflow inputs a binding assigned, as structured data. Inputs map[string]any `json:"inputs,omitempty"` } // WriteTaskJSON writes the task payload to /.git/task.json, where // it is never committed. func WriteTaskJSON(workspace string, payload TaskPayload) error { if workspace == "" { return fmt.Errorf("task.json: empty workspace path") } if info, err := os.Stat(workspace); err != nil { return fmt.Errorf("task.json: workspace %s: %w", workspace, err) } else if !info.IsDir() { return fmt.Errorf("task.json: workspace %s is not a directory", workspace) } data, err := json.Marshal(payload) if err != nil { return err } gitDir := filepath.Join(workspace, ".git") if err := os.MkdirAll(gitDir, 0o700); err != nil { return fmt.Errorf("task.json: %w", err) } return os.WriteFile(filepath.Join(gitDir, "task.json"), data, 0o644) //nolint:gosec // G306: the task container reads its brief as another user. } // SandboxNetworks provisions one isolated network per task container. The // container can reach only what is attached to that network. type SandboxNetworks interface { Create(ctx context.Context, name string) error Remove(ctx context.Context, name string) error } // Pool manages a set of Docker containers running archie-agent. type Pool struct { cli *client.Client sandbox SandboxNetworks ownCli bool // true when the pool created the client (owns Close) cfg Config log *slog.Logger // network is the Docker network spawned containers join. External broker // deployments may leave it empty to use Docker's implicit default bridge; // embedded deployments resolve that default explicitly as "bridge" so its // host gateway can be inspected. network string // hostGateway is the local IPv4 gateway of network. It is populated when // embedded broker reachability is required, so native composition can bind // the listener to an address visible only to workers on this bridge. hostGateway string // live, when set, returns the pool's current settings. A production pool // is given one that reads the running config, so a container-runtime- // policies change reaches the next acquire without a restart; a test pool // leaves it nil and cfg is the whole truth. live func() Config mu sync.Mutex active int // pulled records the images already made available, so a profile's image // is pulled once rather than on every task. Guarded by mu. pulled map[string]bool // teardowns holds the one-shot teardown state for each live container, // keyed by ID. See containerTeardown. teardowns map[string]*containerTeardown } // current returns the per-acquire settings from the live source when set, // else the construction Config. func (p *Pool) current() Config { if p.live == nil { return p.cfg } cfg := p.cfg live := p.live() cfg.Image = live.Image cfg.MaxConcurrency = live.MaxConcurrency cfg.MaxUptime = live.MaxUptime cfg.PullPolicy = live.PullPolicy return cfg } // daemonLabel marks containers a pool owns; its value is the instance name. const daemonLabel = "archie-daemon" // Config is the subset of daemon container configuration the pool needs. type Config struct { Image string MaxConcurrency int // MaxUptime is the total container lifetime cap from creation. When // exceeded, the container is killed regardless of task state. MaxUptime time.Duration // GracePeriod is the idle time after task completion before the container // is killed. The agent stays alive to handle follow-ups (gate re-runs, // chat replies) during this window. GracePeriod time.Duration PullPolicy string // RegistryAuth is the resolved value of [containers].registry_auth: either // empty (anonymous pulls -- the behaviour for a deployment with no private // registry) or a Docker registry.AuthConfig JSON document. pullImage // encodes it into the X-Registry-Auth header the Engine API expects. RegistryAuth string // Network is the host-reachable Docker network the egress relay joins, // read once at construction. Empty falls back to selfNetwork's // best-effort auto-detection. Network string // DockerClient is an optional pre-connected Docker client. When nil, // NewPool creates its own via client.New(client.FromEnv). Pass a // shared client to avoid multiple independent connections. DockerClient *client.Client // RequireHostGateway resolves and validates a local Docker bridge gateway. // Embedded NATS needs this address; external brokers do not. RequireHostGateway bool // Sandbox provisions the isolated network each task container runs on. // It is required: no task container shares a network with the host. Sandbox SandboxNetworks // Instance names this archie instance. It prefixes container names and // is the value of the daemon label the reaper filters on, so instances // sharing a Docker host never remove each other's containers. Instance string } // NewPool connects to Docker, optionally pulls the image and removes orphaned // containers. A set cfg.DockerClient is reused and not closed. live, when // non-nil, supplies Image, MaxConcurrency, MaxUptime and PullPolicy // on every acquire. func NewPool(ctx context.Context, cfg Config, live func() Config, log *slog.Logger) (*Pool, error) { if cfg.Sandbox == nil { return nil, errors.New("container pool needs a sandbox network provisioner") } cli := cfg.DockerClient if cli == nil { var err error cli, err = client.New(client.FromEnv) if err != nil { return nil, fmt.Errorf("docker client: %w", err) } } network := cfg.Network if network == "" { network = selfNetwork(ctx, cli, log) } if cfg.RequireHostGateway && network == "" { // A native daemon is not itself attached to a Docker network. Its // managed workers use Docker's default bridge unless configured // otherwise, so that is the bridge whose host gateway must be bound. network = "bridge" } hostGateway, err := resolveHostGateway(ctx, cli, network, cfg.RequireHostGateway) if err != nil { if cfg.DockerClient == nil { _ = cli.Close() } return nil, err } p := &Pool{ cli: cli, sandbox: cfg.Sandbox, cfg: cfg, live: live, log: log, ownCli: cfg.DockerClient == nil, network: network, hostGateway: hostGateway, } // Pull image if needed. The configured image is recorded as pulled, so an // acquire that names it skips a second pull even though the live settings // may have changed the configured image since. if cfg.PullPolicy == "always" || cfg.PullPolicy == "missing" { if err := p.pullImage(ctx, cfg.Image); err != nil { if err := cli.Close(); err != nil { log.WarnContext(ctx, "docker client close failed during pull error", "err", err) } return nil, err } p.markPulled(cfg.Image) } // Recover orphaned containers from a previous daemon crash. p.recoverOrphans(ctx) return p, nil } // Acquire creates and starts a container from image (empty means the // configured one). With MaxUptime set, the container is stopped and removed // once it elapses. func (p *Pool) Acquire(ctx context.Context, image string, mounts []storage.Mount, env []string) (*Container, error) { // One snapshot for the whole acquire: the image, pull policy, network, // concurrency cap and max-uptime cap are all read once, so a settings // change landing mid-acquire cannot mix two versions. cfg := p.current() if image == "" { image = cfg.Image } if err := p.ensureImage(ctx, cfg, image); err != nil { return nil, err } if err := p.reserve(cfg.MaxConcurrency); err != nil { return nil, err } name := fmt.Sprintf("%s-agent-%d", cfg.Instance, time.Now().UnixNano()) // Release owns removal rather than Docker's auto-remove: a crash writes its // panic trace to stderr, and the container's log is unreadable the moment // Docker removes it. hostConfig := &container.HostConfig{} for i, m := range storage.ConvertMounts(mounts) { if string(m.Type) == storage.MountTypeBind { label := "Z" if mounts[i].Shared { label = "z" } hostConfig.Binds = append(hostConfig.Binds, m.Source+":"+m.Target+":"+label) } else { hostConfig.Mounts = append(hostConfig.Mounts, m) } } // Every task container runs on its own sandbox network, which reaches the // egress relay and nothing else. if err := p.sandbox.Create(ctx, name); err != nil { p.unreserve() return nil, fmt.Errorf("sandbox network: %w", err) } hostConfig.NetworkMode = container.NetworkMode(name) resp, err := p.cli.ContainerCreate(ctx, client.ContainerCreateOptions{ Name: name, Config: &container.Config{ Image: image, Env: env, Labels: map[string]string{ daemonLabel: cfg.Instance, }, }, HostConfig: hostConfig, }) if err != nil { p.dropSandbox(ctx, name) p.unreserve() return nil, fmt.Errorf("container create: %w", err) } if _, err := p.cli.ContainerStart(ctx, resp.ID, client.ContainerStartOptions{}); err != nil { // Best-effort cleanup. if _, rmErr := p.cli.ContainerRemove(context.WithoutCancel(ctx), resp.ID, client.ContainerRemoveOptions{Force: true}); rmErr != nil { p.log.WarnContext(ctx, "container remove after start failure", "id", resp.ID[:12], "err", rmErr) } p.dropSandbox(ctx, name) p.unreserve() return nil, fmt.Errorf("container start: %w", err) } c := p.started(ctx, cfg.MaxUptime, resp.ID, name) c.network = name return c, nil } // dropSandbox removes a task's sandbox network, after a failed acquire or on // release. Best effort: the boot sweep removes whatever this misses. func (p *Pool) dropSandbox(ctx context.Context, name string) { if name == "" { return } if err := p.sandbox.Remove(context.WithoutCancel(ctx), name); err != nil { p.log.WarnContext(ctx, "sandbox network remove failed", "network", name, "err", err) } } func (p *Pool) reserve(maxConcurrency int) error { p.mu.Lock() defer p.mu.Unlock() if maxConcurrency > 0 && p.active >= maxConcurrency { return fmt.Errorf("max concurrency %d reached", maxConcurrency) } p.active++ return nil } func (p *Pool) unreserve() { p.mu.Lock() p.active-- p.mu.Unlock() } func (p *Pool) started(ctx context.Context, maxUptime time.Duration, id, name string) *Container { if maxUptime > 0 { p.armMaxUptime(ctx, maxUptime, id) } p.log.InfoContext(ctx, "container started", "id", id[:12], "name", name) detail := new(string) return &Container{ID: id, exited: p.watchExit(ctx, id, detail), detail: detail} } // Client is the pool's Docker client, for callers that provision what a // container needs around it. func (p *Pool) Client() *client.Client { return p.cli } // watchExit returns a channel closed once Docker reports the container is no // longer running. A failed wait says nothing about the container, so it never // closes the channel: a Docker API error must not abort a live run. func (p *Pool) watchExit(ctx context.Context, id string, detail *string) <-chan struct{} { exited := make(chan struct{}) wait := p.cli.ContainerWait(context.WithoutCancel(ctx), id, client.ContainerWaitOptions{Condition: container.WaitConditionNotRunning}) go func() { select { case result := <-wait.Result: *detail = p.exitDetail(ctx, id, result.StatusCode) close(exited) case <-wait.Error: } }() return exited } // exitDetail describes a finished container for the operator. The detail // becomes the task's park reason, which reaches the dashboard and chat, and an // agent's output can carry a credential, so it takes only the crash line. The // full stderr tail, the only account of a Go panic, goes to the daemon log. func (p *Pool) exitDetail(ctx context.Context, id string, code int64) string { detail := exitDetail(code) if code == 0 { return detail } tail := p.stderrTail(ctx, id) if tail == "" { return detail } p.log.ErrorContext(ctx, "agent container crashed", "id", id[:12], "code", code, "stderr", tail) if line := crashLine(tail); line != "" { detail += ": " + line } return detail } // crashLineBytes bounds the crash line carried into the park reason. const crashLineBytes = 200 // crashLine picks the runtime's own summary of a crash from a stderr tail. func crashLine(tail string) string { for line := range strings.Lines(tail) { line = strings.TrimSpace(line) if strings.HasPrefix(line, "panic: ") || strings.HasPrefix(line, "fatal error: ") { if len(line) > crashLineBytes { line = line[:crashLineBytes] + "…" } return line } } return "" } const ( // crashTailLines bounds the stderr tail read after a crash: enough for a // panic's opening line and the frames around the fault. crashTailLines = 100 // crashTailBytes bounds the same tail by size, for a container that writes // one enormous line or a multi-megabyte stack dump. crashTailBytes = 8 << 10 ) // stderrTail returns what the container last wrote to stderr. It reads the log // of a stopped container, which exists until Release removes it. Best effort: a // container with no output, or a Docker error, yields "". func (p *Pool) stderrTail(ctx context.Context, id string) string { ctx, cancel := context.WithTimeout(context.WithoutCancel(ctx), 5*time.Second) defer cancel() reader, err := p.cli.ContainerLogs(ctx, id, client.ContainerLogsOptions{ ShowStderr: true, Tail: strconv.Itoa(crashTailLines), }) if err != nil { p.log.WarnContext(ctx, "container stderr unavailable after exit", "id", id[:12], "err", err) return "" } defer func() { if err := reader.Close(); err != nil { p.log.WarnContext(ctx, "container log reader close failed", "id", id[:12], "err", err) } }() var stderr bytes.Buffer // stdout is not part of the detail. Hitting the byte cap cuts the stream // short, so the partial text is kept rather than the read reported as a // failure. _, _ = stdcopy.StdCopy(io.Discard, &stderr, io.LimitReader(reader, crashTailBytes)) return strings.TrimSpace(stderr.String()) } // exitDetail is the operator-facing one-line cause of a container's exit. func exitDetail(code int64) string { if code == 137 { return "exit code 137 (killed; often out of memory)" } return fmt.Sprintf("exit code %d", code) } // containerTeardown is a live container's max-uptime timer and whether its // Docker teardown has been claimed. type containerTeardown struct { timer *time.Timer closed bool } // armMaxUptime schedules a stop and remove after maxUptime, recording the // entry and timer under p.mu. func (p *Pool) armMaxUptime(ctx context.Context, maxUptime time.Duration, id string) { p.mu.Lock() defer p.mu.Unlock() t := p.teardownLocked(id) t.timer = time.AfterFunc(maxUptime, func() { p.reapMaxUptime(ctx, id) }) } // reapMaxUptime is the max-uptime callback body: it claims the container's // Docker teardown and, if it won, stops and removes the container. func (p *Pool) reapMaxUptime(ctx context.Context, id string) { if !p.claimReaper(id) { return } zero := 0 stopCtx, cancel := context.WithTimeout(context.WithoutCancel(ctx), 10*time.Second) defer cancel() if _, err := p.cli.ContainerStop(stopCtx, id, client.ContainerStopOptions{Timeout: &zero}); err != nil { p.log.WarnContext(ctx, "max uptime stop failed", "id", id[:12], "err", err) } if _, err := p.cli.ContainerRemove(context.WithoutCancel(ctx), id, client.ContainerRemoveOptions{Force: true}); err != nil { p.log.WarnContext(ctx, "max uptime remove failed", "id", id[:12], "err", err) } } // claimReaper reports whether the max-uptime callback may tear the container // down. A missing entry is a loss. func (p *Pool) claimReaper(id string) bool { p.mu.Lock() defer p.mu.Unlock() t := p.teardowns[id] if t == nil || t.closed { return false } t.closed = true return true } // claimRelease reports whether Release may tear the container down. A // missing entry is a win. func (p *Pool) claimRelease(id string) bool { p.mu.Lock() defer p.mu.Unlock() t := p.teardowns[id] if t == nil { return true } if t.closed { return false } t.closed = true return true } // forgetTeardown stops the timer and deletes the container's entry. func (p *Pool) forgetTeardown(id string) { p.mu.Lock() t := p.teardowns[id] delete(p.teardowns, id) p.mu.Unlock() if t != nil && t.timer != nil { t.timer.Stop() } } // teardownLocked returns the container's teardown state, creating it on first // use. armMaxUptime is its only caller: entries are created by the producer of // the reaper, under the same lock that lets the callback claim them, and never // by a path that might be racing Release. Callers must hold p.mu. func (p *Pool) teardownLocked(id string) *containerTeardown { if p.teardowns == nil { p.teardowns = make(map[string]*containerTeardown) } t := p.teardowns[id] if t == nil { t = &containerTeardown{} p.teardowns[id] = t } return t } // releaseDecision reports whether Release honours the grace period and the // stop timeout to use. A cancelled ctx skips the grace period and kills // immediately. func releaseDecision(ctx context.Context, grace time.Duration) (honorGrace bool, stopTimeout *int) { if ctx.Err() != nil { zero := 0 return false, &zero } return grace > 0, nil } // Release stops and removes a container after a task completes. If GracePeriod // is configured, the container stays alive for that duration before being // killed -- the agent can handle follow-ups (gate re-runs, chat replies) // during this window. func (p *Pool) Release(ctx context.Context, c *Container) { if c == nil { return } honorGrace, stopTimeout := releaseDecision(ctx, p.cfg.GracePeriod) if honorGrace { p.log.InfoContext(ctx, "container keeping alive for grace period", "id", c.ID[:12], "grace", p.cfg.GracePeriod) time.Sleep(p.cfg.GracePeriod) } // Claim after the grace period so MaxUptime still applies during it. if p.claimRelease(c.ID) { // Stop with a fresh deadline even if ctx is cancelled. stopCtx, cancel := context.WithTimeout(context.WithoutCancel(ctx), 10*time.Second) defer cancel() if _, err := p.cli.ContainerStop(stopCtx, c.ID, client.ContainerStopOptions{Timeout: stopTimeout}); err != nil { p.log.WarnContext(ctx, "container stop failed", "id", c.ID[:12], "err", err) } if _, err := p.cli.ContainerRemove(context.WithoutCancel(ctx), c.ID, client.ContainerRemoveOptions{Force: true}); err != nil { p.log.WarnContext(ctx, "container remove failed", "id", c.ID[:12], "err", err) } } p.forgetTeardown(c.ID) p.mu.Lock() p.active-- p.mu.Unlock() p.dropSandbox(ctx, c.network) p.log.InfoContext(ctx, "container released", "id", c.ID[:12]) } // Close stops and removes all active containers. If the pool created its // own Docker client, it is closed. Shared clients are not closed. func (p *Pool) Close() error { ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second) defer cancel() list, err := p.cli.ContainerList(ctx, client.ContainerListOptions{ All: true, Filters: client.Filters{}.Add("label", daemonLabel+"="+p.cfg.Instance), }) if err != nil { p.log.WarnContext(ctx, "container list failed during close", "err", err) } else { for _, c := range list.Items { p.log.InfoContext(ctx, "cleaning up container", "id", c.ID[:12]) if _, err := p.cli.ContainerStop(ctx, c.ID, client.ContainerStopOptions{}); err != nil { p.log.WarnContext(ctx, "container stop during close", "id", c.ID[:12], "err", err) } if _, err := p.cli.ContainerRemove(ctx, c.ID, client.ContainerRemoveOptions{Force: true}); err != nil { p.log.WarnContext(ctx, "container remove during close", "id", c.ID[:12], "err", err) } } } if p.ownCli { return p.cli.Close() } return nil } // ── helpers ────────────────────────────────────────────────────────── // ensureImage pulls ref under cfg's pull policy unless it is already present // in the pool. It is the live read an acquire uses; EnsureImage is the // exported form for callers that only name a ref. func (p *Pool) ensureImage(ctx context.Context, cfg Config, ref string) error { if cfg.PullPolicy != "always" && cfg.PullPolicy != "missing" { return nil } p.mu.Lock() done := p.pulled[ref] p.mu.Unlock() if done { return nil } if err := p.pullImage(ctx, ref); err != nil { return err } p.markPulled(ref) return nil } // markPulled records that ref is available locally, so a later acquire skips // a second pull. func (p *Pool) markPulled(ref string) { p.mu.Lock() if p.pulled == nil { p.pulled = map[string]bool{} } p.pulled[ref] = true p.mu.Unlock() } // pullImage pulls ref under the live pull policy. On "missing" policy, skips // if the image already exists locally. func (p *Pool) pullImage(ctx context.Context, ref string) error { cfg := p.current() if cfg.PullPolicy == "missing" { _, err := p.cli.ImageInspect(ctx, ref) if err == nil { p.log.InfoContext(ctx, "image already present", "image", ref) return nil } } p.log.InfoContext(ctx, "pulling image", "image", ref) rc, err := p.cli.ImagePull(ctx, ref, client.ImagePullOptions{RegistryAuth: p.registryAuthHeader(ctx, cfg, ref)}) if err != nil { return fmt.Errorf("pull %s: %w", ref, err) } defer func() { if err := rc.Close(); err != nil { p.log.WarnContext(ctx, "image pull reader close failed", "err", err) } }() if _, err := io.Copy(io.Discard, rc); err != nil { return fmt.Errorf("pull %s: discard: %w", ref, err) } p.log.InfoContext(ctx, "image pulled", "image", ref) return nil } // registryAuthHeader returns the X-Registry-Auth value, or "". A credential // that cannot be rendered logs a warning and pulls anonymously. func (p *Pool) registryAuthHeader(ctx context.Context, cfg Config, ref string) string { if cfg.RegistryAuth == "" { return "" } encoded, err := encodeRegistryAuth(cfg.RegistryAuth) if err != nil { p.log.WarnContext(ctx, "registry credential unusable; pulling without auth", "image", ref, "err", err) return "" } return encoded } // encodeRegistryAuth encodes credential, a registry.AuthConfig JSON document, // as the base64url JSON the Engine API documents for // [registry.AuthHeader]. It is the shape docker login's config.json stores, // minus the surrounding "auths" map. func encodeRegistryAuth(credential string) (string, error) { var auth registry.AuthConfig if err := json.Unmarshal([]byte(credential), &auth); err != nil { return "", fmt.Errorf("decode registry credential: %w", err) } if auth.Username == "" && auth.Password == "" && auth.IdentityToken == "" && auth.RegistryToken == "" { return "", errors.New("registry credential names no username, password, identity token or registry token") } payload, err := json.Marshal(auth) //nolint:gosec // G117: the registry credential is meant for the image pull's auth header. if err != nil { return "", fmt.Errorf("encode registry credential: %w", err) } return base64.URLEncoding.EncodeToString(payload), nil } // inContainer reports whether this process runs in a container, where Docker // and Podman each leave a marker file. func inContainer() bool { for _, marker := range []string{"/.dockerenv", "/run/.containerenv"} { if _, err := os.Stat(marker); err == nil { return true } } return false } // selfNetwork returns the user-defined network this process's container is // on, or "". func selfNetwork(ctx context.Context, cli *client.Client, log *slog.Logger) string { if !inContainer() { return "" } hostname, err := os.Hostname() if err != nil { log.WarnContext(ctx, "container network auto-detect: os.Hostname failed, spawned containers will use the default bridge network and won't resolve sibling services by name — set containers.network explicitly", "err", err) return "" } self, err := cli.ContainerInspect(ctx, hostname, client.ContainerInspectOptions{}) if err != nil { log.WarnContext(ctx, "container network auto-detect: could not inspect self by hostname, spawned containers will use the default bridge network and won't resolve sibling services by name — set containers.network explicitly", "hostname", hostname, "err", err) return "" } if self.Container.NetworkSettings == nil { log.WarnContext(ctx, "container network auto-detect: self container has no NetworkSettings, spawned containers will use the default bridge network — set containers.network explicitly", "hostname", hostname) return "" } for name := range self.Container.NetworkSettings.Networks { if name != "bridge" { log.InfoContext(ctx, "spawned containers will join daemon's network", "network", name) return name } } return "" } // recoverOrphans stops and removes containers left by a previous daemon. func (p *Pool) recoverOrphans(ctx context.Context) { list, err := p.cli.ContainerList(ctx, client.ContainerListOptions{ All: true, Filters: client.Filters{}.Add("label", daemonLabel+"="+p.cfg.Instance), }) if err != nil { return } for _, c := range list.Items { p.log.InfoContext(ctx, "recovering orphaned container", "id", c.ID[:12]) if _, err := p.cli.ContainerStop(ctx, c.ID, client.ContainerStopOptions{}); err != nil { p.log.WarnContext(ctx, "container stop during orphan recovery", "id", c.ID[:12], "err", err) } if _, err := p.cli.ContainerRemove(ctx, c.ID, client.ContainerRemoveOptions{Force: true}); err != nil { p.log.WarnContext(ctx, "container remove during orphan recovery", "id", c.ID[:12], "err", err) } } if len(list.Items) > 0 { p.log.InfoContext(ctx, "orphan recovery complete", "count", len(list.Items)) } } // Active is the number of containers this pool currently holds: acquired but // not yet released. It reads the pool's own counter rather than listing // Docker containers, which would include containers this pool does not own // (orphans from a crashed daemon, another instance's workers). func (p *Pool) Active() int { p.mu.Lock() defer p.mu.Unlock() return p.active } // Cap is the concurrency cap this pool enforces, from the live // containers.max_concurrency. Zero means unlimited (Acquire only enforces a // cap when MaxConcurrency > 0), so a caller rendering "active/cap" must not // treat zero as a cap of zero. func (p *Pool) Cap() int { return p.current().MaxConcurrency }