Timer.Stop cannot unwind a max-uptime callback that has already begun, so cancelling still let the reaper and Release both stop and remove the same container, which is the duplicate teardown and the misleading 'No such container' warnings this was meant to remove. Teardown is now claimed once per container: whichever of the two gets there first talks to Docker and the loser skips to its own bookkeeping. Release still claims after the grace period, so MaxUptime keeps biting during it. Also table-drives the event pump priming cases, per the repo's test directive.
497 lines
18 KiB
Go
497 lines
18 KiB
Go
// Package container manages Docker containers running archie-agent.
|
|
// A Pool acquires and releases containers per task, handling image pull,
|
|
// creation, startup, health check, and teardown. The daemon hands each task to
|
|
// its container as one core-NATS request.
|
|
package container
|
|
|
|
import (
|
|
"context"
|
|
"encoding/json"
|
|
"fmt"
|
|
"io"
|
|
"log/slog"
|
|
"os"
|
|
"path/filepath"
|
|
"sync"
|
|
"time"
|
|
|
|
"github.com/moby/moby/api/types/container"
|
|
"github.com/moby/moby/client"
|
|
|
|
"github.com/samcharles93/archie-core/internal/storage"
|
|
)
|
|
|
|
// Container wraps a running Docker container.
|
|
type Container struct {
|
|
ID string
|
|
}
|
|
|
|
// TaskPayload is the boot-time brief written to /data/worktree/.git/task.json
|
|
// before the container starts, per PRD section 3.
|
|
type TaskPayload struct {
|
|
ID int64 `json:"id"`
|
|
Owner string `json:"owner"`
|
|
Repo string `json:"repo"`
|
|
Number int `json:"issue_number"`
|
|
Title string `json:"title"`
|
|
Body string `json:"body"`
|
|
Labels []string `json:"labels"`
|
|
Workflow string `json:"workflow"`
|
|
Branch string `json:"branch,omitempty"`
|
|
Plan string `json:"plan,omitempty"`
|
|
}
|
|
|
|
// WriteTaskJSON writes the task payload to <workspace>/.git/task.json.
|
|
// The workspace directory must already exist.
|
|
//
|
|
// It goes under .git deliberately. The workspace is the task worktree the
|
|
// agent commits from, and worktree.CommitAll stages with go-git's All
|
|
// option, which does not honour .gitignore or .git/info/exclude -- a brief
|
|
// at the worktree root is swept into the agent's commit and pushed onto the
|
|
// task branch. Nothing under .git can ever be tracked, which is the same
|
|
// reasoning that moved the prepared sentinel there.
|
|
func WriteTaskJSON(workspace string, payload TaskPayload) error {
|
|
if workspace == "" {
|
|
return fmt.Errorf("task.json: empty workspace path")
|
|
}
|
|
if info, err := os.Stat(workspace); err != nil {
|
|
return fmt.Errorf("task.json: workspace %s: %w", workspace, err)
|
|
} else if !info.IsDir() {
|
|
return fmt.Errorf("task.json: workspace %s is not a directory", workspace)
|
|
}
|
|
data, err := json.Marshal(payload)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
gitDir := filepath.Join(workspace, ".git")
|
|
if err := os.MkdirAll(gitDir, 0o700); err != nil {
|
|
return fmt.Errorf("task.json: %w", err)
|
|
}
|
|
return os.WriteFile(filepath.Join(gitDir, "task.json"), data, 0o644)
|
|
}
|
|
|
|
// Pool manages a set of Docker containers running archie-agent.
|
|
type Pool struct {
|
|
cli *client.Client
|
|
ownCli bool // true when the pool created the client (owns Close)
|
|
cfg Config
|
|
log *slog.Logger
|
|
// network is the Docker network spawned containers join. External broker
|
|
// deployments may leave it empty to use Docker's implicit default bridge;
|
|
// embedded deployments resolve that default explicitly as "bridge" so its
|
|
// host gateway can be inspected.
|
|
network string
|
|
// hostGateway is the local IPv4 gateway of network. It is populated when
|
|
// embedded broker reachability is required, so native composition can bind
|
|
// the listener to an address visible only to workers on this bridge.
|
|
hostGateway string
|
|
|
|
mu sync.Mutex
|
|
active int
|
|
// teardowns holds the one-shot teardown state for each live container,
|
|
// keyed by ID. See containerTeardown.
|
|
teardowns map[string]*containerTeardown
|
|
}
|
|
|
|
// Config is the subset of daemon container configuration the pool needs.
|
|
type Config struct {
|
|
Image string
|
|
MaxConcurrency int
|
|
// MaxUptime is the total container lifetime cap from creation. When
|
|
// exceeded, the container is killed regardless of task state. PRD §4.
|
|
MaxUptime time.Duration
|
|
// GracePeriod is the idle time after task completion before the
|
|
// container is killed. The agent stays alive to handle follow-ups
|
|
// (gate re-runs, human replies) during this window. PRD §1.
|
|
GracePeriod time.Duration
|
|
PullPolicy string
|
|
// Network is the Docker network spawned agent containers join. Empty
|
|
// falls back to selfNetwork's best-effort auto-detection.
|
|
Network string
|
|
// DockerClient is an optional pre-connected Docker client. When nil,
|
|
// NewPool creates its own via client.New(client.FromEnv). Pass a
|
|
// shared client to avoid multiple independent connections.
|
|
DockerClient *client.Client
|
|
// RequireHostGateway resolves and validates a local Docker bridge gateway.
|
|
// Embedded NATS needs this address; external brokers do not.
|
|
RequireHostGateway bool
|
|
}
|
|
|
|
// NewPool connects to the Docker daemon, optionally pulls the image,
|
|
// and cleans up orphaned containers from a previous daemon crash.
|
|
//
|
|
// If cfg.DockerClient is set, it is reused (caller owns Close).
|
|
// Otherwise a new client is created via client.FromEnv (pool owns Close).
|
|
func NewPool(ctx context.Context, cfg Config, log *slog.Logger) (*Pool, error) {
|
|
cli := cfg.DockerClient
|
|
if cli == nil {
|
|
var err error
|
|
cli, err = client.New(client.FromEnv)
|
|
if err != nil {
|
|
return nil, fmt.Errorf("docker client: %w", err)
|
|
}
|
|
}
|
|
|
|
network := cfg.Network
|
|
if network == "" {
|
|
network = selfNetwork(ctx, cli, log)
|
|
}
|
|
if cfg.RequireHostGateway && network == "" {
|
|
// A native daemon is not itself attached to a Docker network. Its
|
|
// managed workers use Docker's default bridge unless configured
|
|
// otherwise, so that is the bridge whose host gateway must be bound.
|
|
network = "bridge"
|
|
}
|
|
hostGateway, err := resolveHostGateway(ctx, cli, network, cfg.RequireHostGateway)
|
|
if err != nil {
|
|
if cfg.DockerClient == nil {
|
|
_ = cli.Close()
|
|
}
|
|
return nil, err
|
|
}
|
|
|
|
p := &Pool{
|
|
cli: cli,
|
|
cfg: cfg,
|
|
log: log,
|
|
ownCli: cfg.DockerClient == nil,
|
|
network: network,
|
|
hostGateway: hostGateway,
|
|
}
|
|
|
|
// Pull image if needed.
|
|
if cfg.PullPolicy == "always" || cfg.PullPolicy == "missing" {
|
|
if err := p.pullImage(ctx); err != nil {
|
|
if err := cli.Close(); err != nil {
|
|
log.Warn("docker client close failed during pull error", "err", err)
|
|
}
|
|
return nil, err
|
|
}
|
|
}
|
|
|
|
// Recover orphaned containers from a previous daemon crash.
|
|
p.recoverOrphans(ctx)
|
|
|
|
return p, nil
|
|
}
|
|
|
|
// Acquire creates and starts a container with the given mounts and
|
|
// environment variables. Mounts are provided by the caller (typically
|
|
// from a storage.Backend). If MaxUptime is set, the pool schedules a hard
|
|
// stop and remove of the container once that lifetime cap elapses,
|
|
// regardless of task state.
|
|
func (p *Pool) Acquire(ctx context.Context, mounts []storage.Mount, env []string) (*Container, error) {
|
|
p.mu.Lock()
|
|
if p.cfg.MaxConcurrency > 0 && p.active >= p.cfg.MaxConcurrency {
|
|
p.mu.Unlock()
|
|
return nil, fmt.Errorf("max concurrency %d reached", p.cfg.MaxConcurrency)
|
|
}
|
|
p.active++
|
|
p.mu.Unlock()
|
|
|
|
name := fmt.Sprintf("archie-agent-%d", time.Now().UnixNano())
|
|
|
|
hostConfig := &container.HostConfig{
|
|
Mounts: storage.ConvertMounts(mounts),
|
|
AutoRemove: true,
|
|
}
|
|
if p.network != "" {
|
|
// Join the same user-defined network the daemon itself is on, so
|
|
// the agent container can resolve sibling compose services (nats,
|
|
// etc.) by name -- otherwise Docker attaches it to the default
|
|
// bridge network, where those hostnames don't resolve.
|
|
hostConfig.NetworkMode = container.NetworkMode(p.network)
|
|
}
|
|
|
|
resp, err := p.cli.ContainerCreate(ctx, client.ContainerCreateOptions{
|
|
Name: name,
|
|
Config: &container.Config{
|
|
Image: p.cfg.Image,
|
|
Env: env,
|
|
Labels: map[string]string{
|
|
"archie-daemon": "true",
|
|
},
|
|
},
|
|
HostConfig: hostConfig,
|
|
})
|
|
if err != nil {
|
|
p.mu.Lock()
|
|
p.active--
|
|
p.mu.Unlock()
|
|
return nil, fmt.Errorf("container create: %w", err)
|
|
}
|
|
|
|
if _, err := p.cli.ContainerStart(ctx, resp.ID, client.ContainerStartOptions{}); err != nil {
|
|
// Best-effort cleanup.
|
|
if _, rmErr := p.cli.ContainerRemove(context.WithoutCancel(ctx), resp.ID, client.ContainerRemoveOptions{Force: true}); rmErr != nil {
|
|
p.log.Warn("container remove after start failure", "id", resp.ID[:12], "err", rmErr)
|
|
}
|
|
p.mu.Lock()
|
|
p.active--
|
|
p.mu.Unlock()
|
|
return nil, fmt.Errorf("container start: %w", err)
|
|
}
|
|
|
|
if p.cfg.MaxUptime > 0 {
|
|
p.armMaxUptime(ctx, resp.ID)
|
|
}
|
|
|
|
p.log.Info("container started", "id", resp.ID[:12], "name", name)
|
|
return &Container{ID: resp.ID}, nil
|
|
}
|
|
|
|
// containerTeardown is the one-shot teardown state for a live container: its
|
|
// armed max-uptime reaper, and whether Docker stop/remove has been claimed.
|
|
//
|
|
// Both the reaper and Release tear a container down, and Timer.Stop cannot
|
|
// unwind a callback that has already begun, so cancelling the timer is not
|
|
// enough to keep them from overlapping. Claiming decides the winner instead:
|
|
// exactly one path talks to Docker, and the loser skips straight to its own
|
|
// bookkeeping.
|
|
type containerTeardown struct {
|
|
timer *time.Timer
|
|
closed bool
|
|
}
|
|
|
|
// armMaxUptime schedules a hard stop and remove for a container once its
|
|
// lifetime cap elapses. The timer never touches p.active: Release (or Close)
|
|
// remains the only owner of the active slot.
|
|
func (p *Pool) armMaxUptime(ctx context.Context, id string) {
|
|
timer := time.AfterFunc(p.cfg.MaxUptime, func() {
|
|
if !p.claimTeardown(id) {
|
|
return
|
|
}
|
|
defer p.forgetTeardown(id)
|
|
zero := 0
|
|
stopCtx, cancel := context.WithTimeout(context.WithoutCancel(ctx), 10*time.Second)
|
|
defer cancel()
|
|
if _, err := p.cli.ContainerStop(stopCtx, id, client.ContainerStopOptions{Timeout: &zero}); err != nil {
|
|
p.log.Warn("max uptime stop failed", "id", id[:12], "err", err)
|
|
}
|
|
if _, err := p.cli.ContainerRemove(context.WithoutCancel(ctx), id, client.ContainerRemoveOptions{Force: true}); err != nil {
|
|
p.log.Warn("max uptime remove failed", "id", id[:12], "err", err)
|
|
}
|
|
})
|
|
|
|
p.mu.Lock()
|
|
defer p.mu.Unlock()
|
|
p.teardownLocked(id).timer = timer
|
|
}
|
|
|
|
// claimTeardown reports whether the caller owns this container's Docker
|
|
// teardown. The first caller wins; every later one is told to stay out.
|
|
func (p *Pool) claimTeardown(id string) bool {
|
|
p.mu.Lock()
|
|
defer p.mu.Unlock()
|
|
t := p.teardownLocked(id)
|
|
if t.closed {
|
|
return false
|
|
}
|
|
t.closed = true
|
|
return true
|
|
}
|
|
|
|
// forgetTeardown disarms the reaper and drops the container's bookkeeping, so
|
|
// the map does not grow for the life of the pool.
|
|
func (p *Pool) forgetTeardown(id string) {
|
|
p.mu.Lock()
|
|
t := p.teardowns[id]
|
|
delete(p.teardowns, id)
|
|
p.mu.Unlock()
|
|
if t != nil && t.timer != nil {
|
|
t.timer.Stop()
|
|
}
|
|
}
|
|
|
|
// teardownLocked returns the container's teardown state, creating it on first
|
|
// use. Callers must hold p.mu.
|
|
func (p *Pool) teardownLocked(id string) *containerTeardown {
|
|
if p.teardowns == nil {
|
|
p.teardowns = make(map[string]*containerTeardown)
|
|
}
|
|
t := p.teardowns[id]
|
|
if t == nil {
|
|
t = &containerTeardown{}
|
|
p.teardowns[id] = t
|
|
}
|
|
return t
|
|
}
|
|
|
|
// releaseDecision reports how Release should tear a container down: whether
|
|
// to honour the configured post-completion grace period, and what Docker
|
|
// stop timeout to request (nil means the container's configured timeout or
|
|
// the engine default; a non-nil zero means stop immediately with SIGKILL,
|
|
// skipping the graceful SIGTERM wait).
|
|
//
|
|
// A cancelled ctx means the release is happening because the task was
|
|
// stopped, not because it finished -- /stop is an emergency brake, so
|
|
// neither the grace period (meant for a task that completed on its own and
|
|
// might get a follow-up) nor a graceful stop (meant to let a healthy
|
|
// process wind down) apply. A still-live ctx is a normal completion, so
|
|
// both keep their existing meaning.
|
|
func releaseDecision(ctx context.Context, grace time.Duration) (honorGrace bool, stopTimeout *int) {
|
|
if ctx.Err() != nil {
|
|
zero := 0
|
|
return false, &zero
|
|
}
|
|
return grace > 0, nil
|
|
}
|
|
|
|
// Release stops and removes a container after a task completes. If
|
|
// GracePeriod is configured, the container stays alive for that duration
|
|
// before being killed -- the agent can handle follow-ups (gate re-runs,
|
|
// human replies) during this window. PRD section 1.
|
|
func (p *Pool) Release(ctx context.Context, c *Container) {
|
|
if c == nil {
|
|
return
|
|
}
|
|
honorGrace, stopTimeout := releaseDecision(ctx, p.cfg.GracePeriod)
|
|
if honorGrace {
|
|
p.log.Info("container keeping alive for grace period", "id", c.ID[:12], "grace", p.cfg.GracePeriod)
|
|
time.Sleep(p.cfg.GracePeriod)
|
|
}
|
|
|
|
// Claim only now. MaxUptime is a cap from creation enforced regardless of
|
|
// task state, so the reaper must still be free to bite during the grace
|
|
// period above; claiming before the sleep would quietly extend the cap by
|
|
// GracePeriod. Losing the claim means the reaper already removed this
|
|
// container, so there is nothing left to stop.
|
|
if p.claimTeardown(c.ID) {
|
|
// Detach from ctx before stopping. Release runs on the way out of a
|
|
// task, and the most important reason a task is on its way out is
|
|
// that it was cancelled -- at which point ctx is already dead, the
|
|
// graceful stop fails immediately, and the container is left to the
|
|
// forced remove below. Teardown is cleanup, so it gets its own
|
|
// deadline rather than inheriting the caller's cancellation.
|
|
stopCtx, cancel := context.WithTimeout(context.WithoutCancel(ctx), 10*time.Second)
|
|
defer cancel()
|
|
|
|
if _, err := p.cli.ContainerStop(stopCtx, c.ID, client.ContainerStopOptions{Timeout: stopTimeout}); err != nil {
|
|
p.log.Warn("container stop failed", "id", c.ID[:12], "err", err)
|
|
}
|
|
if _, err := p.cli.ContainerRemove(context.WithoutCancel(ctx), c.ID, client.ContainerRemoveOptions{Force: true}); err != nil {
|
|
p.log.Warn("container remove failed", "id", c.ID[:12], "err", err)
|
|
}
|
|
}
|
|
p.forgetTeardown(c.ID)
|
|
|
|
p.mu.Lock()
|
|
p.active--
|
|
p.mu.Unlock()
|
|
p.log.Info("container released", "id", c.ID[:12])
|
|
}
|
|
|
|
// Close stops and removes all active containers. If the pool created its
|
|
// own Docker client, it is closed. Shared clients are not closed.
|
|
func (p *Pool) Close() error {
|
|
ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
|
|
defer cancel()
|
|
|
|
list, err := p.cli.ContainerList(ctx, client.ContainerListOptions{
|
|
All: true,
|
|
Filters: client.Filters{}.Add("label", "archie-daemon=true"),
|
|
})
|
|
if err != nil {
|
|
p.log.Warn("container list failed during close", "err", err)
|
|
} else {
|
|
for _, c := range list.Items {
|
|
p.log.Info("cleaning up container", "id", c.ID[:12])
|
|
if _, err := p.cli.ContainerStop(ctx, c.ID, client.ContainerStopOptions{}); err != nil {
|
|
p.log.Warn("container stop during close", "id", c.ID[:12], "err", err)
|
|
}
|
|
if _, err := p.cli.ContainerRemove(ctx, c.ID, client.ContainerRemoveOptions{Force: true}); err != nil {
|
|
p.log.Warn("container remove during close", "id", c.ID[:12], "err", err)
|
|
}
|
|
}
|
|
}
|
|
if p.ownCli {
|
|
return p.cli.Close()
|
|
}
|
|
return nil
|
|
}
|
|
|
|
// ── helpers ──────────────────────────────────────────────────────────
|
|
|
|
// pullImage pulls the configured image. On "missing" policy, skips if
|
|
// the image already exists locally.
|
|
func (p *Pool) pullImage(ctx context.Context) error {
|
|
ref := p.cfg.Image
|
|
|
|
if p.cfg.PullPolicy == "missing" {
|
|
_, err := p.cli.ImageInspect(ctx, ref)
|
|
if err == nil {
|
|
p.log.Info("image already present", "image", ref)
|
|
return nil
|
|
}
|
|
}
|
|
|
|
p.log.Info("pulling image", "image", ref)
|
|
rc, err := p.cli.ImagePull(ctx, ref, client.ImagePullOptions{})
|
|
if err != nil {
|
|
return fmt.Errorf("pull %s: %w", ref, err)
|
|
}
|
|
defer func() {
|
|
if err := rc.Close(); err != nil {
|
|
p.log.Warn("image pull reader close failed", "err", err)
|
|
}
|
|
}()
|
|
if _, err := io.Copy(io.Discard, rc); err != nil {
|
|
return fmt.Errorf("pull %s: discard: %w", ref, err)
|
|
}
|
|
p.log.Info("image pulled", "image", ref)
|
|
return nil
|
|
}
|
|
|
|
// selfNetwork detects the user-defined Docker network the current
|
|
// process's own container is attached to, by inspecting the container
|
|
// named after our hostname (Docker sets a container's hostname to its
|
|
// short ID by default). Returns "" if we're not running in a container,
|
|
// the inspect fails, or we're only on the default bridge network -- in
|
|
// all of those cases Acquire falls back to Docker's normal default.
|
|
func selfNetwork(ctx context.Context, cli *client.Client, log *slog.Logger) string {
|
|
hostname, err := os.Hostname()
|
|
if err != nil {
|
|
log.Warn("container network auto-detect: os.Hostname failed, spawned containers will use the default bridge network and won't resolve sibling services by name — set containers.network explicitly", "err", err)
|
|
return ""
|
|
}
|
|
self, err := cli.ContainerInspect(ctx, hostname, client.ContainerInspectOptions{})
|
|
if err != nil {
|
|
log.Warn("container network auto-detect: could not inspect self by hostname, spawned containers will use the default bridge network and won't resolve sibling services by name — set containers.network explicitly", "hostname", hostname, "err", err)
|
|
return ""
|
|
}
|
|
if self.Container.NetworkSettings == nil {
|
|
log.Warn("container network auto-detect: self container has no NetworkSettings, spawned containers will use the default bridge network — set containers.network explicitly", "hostname", hostname)
|
|
return ""
|
|
}
|
|
for name := range self.Container.NetworkSettings.Networks {
|
|
if name != "bridge" {
|
|
log.Info("spawned containers will join daemon's network", "network", name)
|
|
return name
|
|
}
|
|
}
|
|
return ""
|
|
}
|
|
|
|
// recoverOrphans stops and removes containers left by a previous daemon.
|
|
func (p *Pool) recoverOrphans(ctx context.Context) {
|
|
list, err := p.cli.ContainerList(ctx, client.ContainerListOptions{
|
|
All: true,
|
|
Filters: client.Filters{}.Add("label", "archie-daemon=true"),
|
|
})
|
|
if err != nil {
|
|
return
|
|
}
|
|
for _, c := range list.Items {
|
|
p.log.Info("recovering orphaned container", "id", c.ID[:12])
|
|
if _, err := p.cli.ContainerStop(ctx, c.ID, client.ContainerStopOptions{}); err != nil {
|
|
p.log.Warn("container stop during orphan recovery", "id", c.ID[:12], "err", err)
|
|
}
|
|
if _, err := p.cli.ContainerRemove(ctx, c.ID, client.ContainerRemoveOptions{Force: true}); err != nil {
|
|
p.log.Warn("container remove during orphan recovery", "id", c.ID[:12], "err", err)
|
|
}
|
|
}
|
|
if len(list.Items) > 0 {
|
|
p.log.Info("orphan recovery complete", "count", len(list.Items))
|
|
}
|
|
}
|