diff --git a/internal/app/archied/bootstrap.go b/internal/app/archied/bootstrap.go index 8b7c6dae0..afe58917b 100644 --- a/internal/app/archied/bootstrap.go +++ b/internal/app/archied/bootstrap.go @@ -42,6 +42,7 @@ import ( infraaccess "github.com/samcharles93/archie-core/internal/infrastructure/access" "github.com/samcharles93/archie-core/internal/infrastructure/configuration" "github.com/samcharles93/archie-core/internal/infrastructure/controlplanestore" + "github.com/samcharles93/archie-core/internal/infrastructure/egress" "github.com/samcharles93/archie-core/internal/infrastructure/eventbus/nats" "github.com/samcharles93/archie-core/internal/infrastructure/extension" "github.com/samcharles93/archie-core/internal/infrastructure/issuerpc" @@ -159,10 +160,16 @@ type boot struct { natsToken string containerPool *container.Pool + // sandboxes provisions each task container's isolated network and runs + // the egress relay that is its only way out. + sandboxes *egress.Networks // modelEgress gives native tasks model access through the egress proxy; // nil when the proxy cannot start. - modelEgress daemon.ModelEgress - storeBackend storage.Backend + modelEgress daemon.ModelEgress + // egressProxyAddr is where the egress proxy listens, the address the relay + // forwards its proxy port to. Empty when the proxy is not running. + egressProxyAddr string + storeBackend storage.Backend // runtimeVersions records the version of every control-plane kind boot's // layering applied. Set once at boot before the runtime-resource watches @@ -484,8 +491,9 @@ func (b *boot) startEmbeddedNATS(ctx context.Context) (string, string, error) { } func (b *boot) setupContainers(ctx context.Context) func() { - containerPool, storeBackend, closeDocker := startContainers(ctx, b.cfgHolder, b.secrets, b.log) + containerPool, sandboxes, storeBackend, closeDocker := startContainers(ctx, b.cfgHolder, b.secrets, b.log) b.containerPool = containerPool + b.sandboxes = sandboxes b.storeBackend = storeBackend return closeDocker } @@ -725,6 +733,7 @@ func (b *boot) setupBackends(ctx context.Context) error { err := b.connectNATS(ctx) if err == nil { b.setupModelEgress(ctx) + b.startEgressRelay(ctx) } return err } diff --git a/internal/app/archied/egress_relay.go b/internal/app/archied/egress_relay.go new file mode 100644 index 000000000..7dbfa5ece --- /dev/null +++ b/internal/app/archied/egress_relay.go @@ -0,0 +1,57 @@ +package archied + +import ( + "context" + "strings" + "time" + + "github.com/samcharles93/archie-core/internal/config" + "github.com/samcharles93/archie-core/internal/daemon" + "github.com/samcharles93/archie-core/internal/infrastructure/egress" +) + +// startEgressRelay removes the sandbox networks and relay a previous daemon +// left, then starts the relay: the only peer on every task container's +// sandbox network, forwarding to NATS, the State Store and the model proxy. +// Boot carries on when it cannot start; each task then parks on the failed +// sandbox attach rather than starting with no way out. +func (b *boot) startEgressRelay(ctx context.Context) { + if b.containerPool == nil || b.sandboxes == nil { + return + } + log := b.log + if err := b.sandboxes.Sweep(ctx); err != nil { + log.WarnContext(ctx, "egress relay: stale sandbox networks remain", "err", err) + } + stateStore := "" + if target := strings.TrimSpace(b.cfg.Services.Get(config.ServiceNameState).Target); target != "" { + addr, err := daemon.RelayStateStoreTarget(target, b.containerPool.HostGateway()) + if err != nil { + log.ErrorContext(ctx, "egress relay: task containers cannot reach the state store", "err", err) + } + stateStore = addr + } + cmd, err := egress.RelayCommand(b.egressProxyAddr, egress.Endpoints{NATS: b.natsURL, StateStore: stateStore}) + if err != nil { + log.ErrorContext(ctx, "egress relay not started: task containers cannot run", "err", err) + return + } + spec := egress.RelaySpec{ + Image: b.cfg.Containers.Image, + Entrypoint: []string{"archie-agent"}, + Cmd: cmd, + Network: b.containerPool.NetworkName(), + } + if err := b.sandboxes.Start(ctx, spec); err != nil { + log.ErrorContext(ctx, "egress relay not started: task containers cannot run", "err", err) + return + } + b.addCleanup(func() { + closeCtx, cancel := context.WithTimeout(context.WithoutCancel(ctx), 10*time.Second) + defer cancel() + if err := b.sandboxes.Close(closeCtx); err != nil { + log.WarnContext(ctx, "egress relay remove failed", "err", err) + } + }) + log.InfoContext(ctx, "egress relay started", "network", spec.Network, "proxy", b.egressProxyAddr != "", "state_store", stateStore != "") +} diff --git a/internal/app/archied/main.go b/internal/app/archied/main.go index 30a250b0e..2e98b48c5 100644 --- a/internal/app/archied/main.go +++ b/internal/app/archied/main.go @@ -37,6 +37,7 @@ import ( "github.com/samcharles93/archie-core/internal/forge" "github.com/samcharles93/archie-core/internal/forgerpc" "github.com/samcharles93/archie-core/internal/infrastructure/configuration" + "github.com/samcharles93/archie-core/internal/infrastructure/egress" "github.com/samcharles93/archie-core/internal/infrastructure/readiness" "github.com/samcharles93/archie-core/internal/logging" "github.com/samcharles93/archie-core/internal/messagerpc" @@ -380,14 +381,14 @@ func startContainers( cfgHolder *config.Holder, secrets *secret.Registry, log *slog.Logger, -) (*container.Pool, storage.Backend, func()) { +) (*container.Pool, *egress.Networks, storage.Backend, func()) { noop := func() {} cfg := cfgHolder.Get() // Single Docker client shared between pool and storage backend. dockerCli, err := client.New(client.FromEnv) if err != nil { log.ErrorContext(ctx, "autonomous workflows unavailable: docker client unavailable", "err", err) - return nil, nil, noop + return nil, nil, nil, noop } closeDocker := func() { if err := dockerCli.Close(); err != nil { @@ -395,6 +396,7 @@ func startContainers( } } + sandboxes := egress.NewNetworks(dockerCli, cfg.Instance+"-egress-relay") pool, err := container.NewPool(ctx, container.Config{ Image: cfg.Containers.Image, MaxConcurrency: cfg.Containers.MaxConcurrency, @@ -403,23 +405,24 @@ func startContainers( RegistryAuth: resolveRegistryAuth(ctx, cfg.Containers.RegistryAuth, secrets, log), Network: cfg.Containers.Network, DockerClient: dockerCli, - // The embedded broker binds the host gateway, and a loopback State - // Store is relayed from it: containers reach neither otherwise. + // The embedded broker binds the host gateway, and the egress relay + // reaches a loopback State Store there: it reaches neither otherwise. RequireHostGateway: cfg.NATS.Mode == config.NATSModeEmbedded || daemon.StateStoreNeedsRelay(cfg.Services.Get(config.ServiceNameState).Target), + Sandbox: sandboxes, Instance: cfg.Instance, }, containerLiveSettings(cfgHolder), log) if err != nil { // A missing image is recoverable by hand. log.ErrorContext(ctx, "autonomous workflows unavailable: container pool unavailable", "err", err, "hint", "run `docker compose pull agent` (or `build agent`) so the image is present locally") - return nil, storage.NewDockerBackend(dockerCli, cfg.Instance), closeDocker + return nil, nil, storage.NewDockerBackend(dockerCli, cfg.Instance), closeDocker } // contextcheck: Pool.Close takes no context by design -- it is a shutdown // path that must run to completion after ctx is already cancelled. //nolint:contextcheck - return pool, storage.NewDockerBackend(dockerCli, cfg.Instance), func() { + return pool, sandboxes, storage.NewDockerBackend(dockerCli, cfg.Instance), func() { if err := pool.Close(); err != nil { log.WarnContext(ctx, "container pool close failed", "err", err) } @@ -431,9 +434,9 @@ func startContainers( // running config. The pool calls it on every acquire, so a stored // container-runtime-policies change reaches the next container without a // restart. Only fields the resource document carries are taken live: -// RegistryAuth is file-owned and stays the boot-resolved value, DockerClient -// and RequireHostGateway are construction-time, and GracePeriod is not part -// of the document. +// RegistryAuth is file-owned and stays the boot-resolved value; DockerClient, +// RequireHostGateway and Network, which the egress relay joins at boot, are +// construction-time; and GracePeriod is not part of the document. func containerLiveSettings(cfgHolder *config.Holder) func() container.Config { return func() container.Config { cfg := cfgHolder.Get() @@ -442,7 +445,6 @@ func containerLiveSettings(cfgHolder *config.Holder) func() container.Config { MaxConcurrency: cfg.Containers.MaxConcurrency, MaxUptime: cfg.Containers.MaxUptime.Std(), PullPolicy: cfg.Containers.PullPolicy, - Network: cfg.Containers.Network, } } } diff --git a/internal/app/archied/model_egress_setup.go b/internal/app/archied/model_egress_setup.go index b1bd3f76e..787a2c713 100644 --- a/internal/app/archied/model_egress_setup.go +++ b/internal/app/archied/model_egress_setup.go @@ -6,6 +6,7 @@ import ( "net" "net/http" "path/filepath" + "strconv" "time" "github.com/samcharles93/archie-core/internal/infrastructure/egress" @@ -41,6 +42,9 @@ func (b *boot) setupModelEgress(ctx context.Context) { } }() b.addCleanup(func() { _ = srv.Close() }) - b.modelEgress = modelEgress{proxy: proxy, addr: ln.Addr().String(), caFile: egress.CACertPath(caDir), config: b.cfgHolder, secrets: b.secrets} + // Task containers reach the proxy only through the egress relay, which + // forwards its proxy port here. + b.egressProxyAddr = ln.Addr().String() + b.modelEgress = modelEgress{proxy: proxy, addr: net.JoinHostPort(egress.RelayAlias, strconv.Itoa(egress.RelayPort)), caFile: egress.CACertPath(caDir), config: b.cfgHolder, secrets: b.secrets} log.InfoContext(ctx, "model egress enabled", "proxy", ln.Addr().String()) } diff --git a/internal/container/pool.go b/internal/container/pool.go index 219411d49..15d9ccdb5 100644 --- a/internal/container/pool.go +++ b/internal/container/pool.go @@ -32,6 +32,9 @@ import ( type Container struct { ID string exited <-chan struct{} + // network is the task's sandbox network, removed by Release. Empty when the + // pool runs without a sandbox. + network string // detail is written before exited closes, so a reader that saw the close // may read it without a lock. detail *string @@ -96,12 +99,20 @@ func WriteTaskJSON(workspace string, payload TaskPayload) error { return os.WriteFile(filepath.Join(gitDir, "task.json"), data, 0o644) //nolint:gosec // G306: the task container reads its brief as another user. } +// SandboxNetworks provisions one isolated network per task container. The +// container can reach only what is attached to that network. +type SandboxNetworks interface { + Create(ctx context.Context, name string) error + Remove(ctx context.Context, name string) error +} + // Pool manages a set of Docker containers running archie-agent. type Pool struct { - cli *client.Client - ownCli bool // true when the pool created the client (owns Close) - cfg Config - log *slog.Logger + cli *client.Client + sandbox SandboxNetworks + ownCli bool // true when the pool created the client (owns Close) + cfg Config + log *slog.Logger // network is the Docker network spawned containers join. External broker // deployments may leave it empty to use Docker's implicit default bridge; // embedded deployments resolve that default explicitly as "bridge" so its @@ -140,7 +151,6 @@ func (p *Pool) current() Config { cfg.MaxConcurrency = live.MaxConcurrency cfg.MaxUptime = live.MaxUptime cfg.PullPolicy = live.PullPolicy - cfg.Network = live.Network return cfg } @@ -164,8 +174,9 @@ type Config struct { // registry) or a Docker registry.AuthConfig JSON document. pullImage // encodes it into the X-Registry-Auth header the Engine API expects. RegistryAuth string - // Network is the Docker network spawned agent containers join. Empty - // falls back to selfNetwork's best-effort auto-detection. + // Network is the host-reachable Docker network the egress relay joins, + // read once at construction. Empty falls back to selfNetwork's + // best-effort auto-detection. Network string // DockerClient is an optional pre-connected Docker client. When nil, // NewPool creates its own via client.New(client.FromEnv). Pass a @@ -174,6 +185,9 @@ type Config struct { // RequireHostGateway resolves and validates a local Docker bridge gateway. // Embedded NATS needs this address; external brokers do not. RequireHostGateway bool + // Sandbox provisions the isolated network each task container runs on. + // It is required: no task container shares a network with the host. + Sandbox SandboxNetworks // Instance names this archie instance. It prefixes container names and // is the value of the daemon label the reaper filters on, so instances // sharing a Docker host never remove each other's containers. @@ -182,9 +196,12 @@ type Config struct { // NewPool connects to Docker, optionally pulls the image and removes orphaned // containers. A set cfg.DockerClient is reused and not closed. live, when -// non-nil, supplies Image, MaxConcurrency, MaxUptime, PullPolicy and Network +// non-nil, supplies Image, MaxConcurrency, MaxUptime and PullPolicy // on every acquire. func NewPool(ctx context.Context, cfg Config, live func() Config, log *slog.Logger) (*Pool, error) { + if cfg.Sandbox == nil { + return nil, errors.New("container pool needs a sandbox network provisioner") + } cli := cfg.DockerClient if cli == nil { var err error @@ -214,6 +231,7 @@ func NewPool(ctx context.Context, cfg Config, live func() Config, log *slog.Logg p := &Pool{ cli: cli, + sandbox: cfg.Sandbox, cfg: cfg, live: live, log: log, @@ -276,19 +294,13 @@ func (p *Pool) Acquire(ctx context.Context, image string, mounts []storage.Mount hostConfig.Mounts = append(hostConfig.Mounts, m) } } - // The live network wins when it names one; otherwise the network resolved - // at construction (the configured one, or self-detection) stays in force. - network := cfg.Network - if network == "" { - network = p.network - } - if network != "" { - // Join the same user-defined network the daemon itself is on, so - // the agent container can resolve sibling compose services (nats, - // etc.) by name -- otherwise Docker attaches it to the default - // bridge network, where those hostnames don't resolve. - hostConfig.NetworkMode = container.NetworkMode(network) + // Every task container runs on its own sandbox network, which reaches the + // egress relay and nothing else. + if err := p.sandbox.Create(ctx, name); err != nil { + p.unreserve() + return nil, fmt.Errorf("sandbox network: %w", err) } + hostConfig.NetworkMode = container.NetworkMode(name) resp, err := p.cli.ContainerCreate(ctx, client.ContainerCreateOptions{ Name: name, @@ -302,6 +314,7 @@ func (p *Pool) Acquire(ctx context.Context, image string, mounts []storage.Mount HostConfig: hostConfig, }) if err != nil { + p.dropSandbox(ctx, name) p.unreserve() return nil, fmt.Errorf("container create: %w", err) } @@ -311,10 +324,24 @@ func (p *Pool) Acquire(ctx context.Context, image string, mounts []storage.Mount if _, rmErr := p.cli.ContainerRemove(context.WithoutCancel(ctx), resp.ID, client.ContainerRemoveOptions{Force: true}); rmErr != nil { p.log.WarnContext(ctx, "container remove after start failure", "id", resp.ID[:12], "err", rmErr) } + p.dropSandbox(ctx, name) p.unreserve() return nil, fmt.Errorf("container start: %w", err) } - return p.started(ctx, cfg.MaxUptime, resp.ID, name), nil + c := p.started(ctx, cfg.MaxUptime, resp.ID, name) + c.network = name + return c, nil +} + +// dropSandbox removes a task's sandbox network, after a failed acquire or on +// release. Best effort: the boot sweep removes whatever this misses. +func (p *Pool) dropSandbox(ctx context.Context, name string) { + if name == "" { + return + } + if err := p.sandbox.Remove(context.WithoutCancel(ctx), name); err != nil { + p.log.WarnContext(ctx, "sandbox network remove failed", "network", name, "err", err) + } } func (p *Pool) reserve(maxConcurrency int) error { @@ -579,6 +606,7 @@ func (p *Pool) Release(ctx context.Context, c *Container) { p.mu.Lock() p.active-- p.mu.Unlock() + p.dropSandbox(ctx, c.network) p.log.InfoContext(ctx, "container released", "id", c.ID[:12]) } diff --git a/internal/container/sandbox_test.go b/internal/container/sandbox_test.go new file mode 100644 index 000000000..d5e0735f8 --- /dev/null +++ b/internal/container/sandbox_test.go @@ -0,0 +1,132 @@ +package container + +import ( + "context" + "encoding/json" + "log/slog" + "net/http" + "net/http/httptest" + "slices" + "strings" + "sync" + "testing" + + "github.com/moby/moby/client" +) + +// fakeSandbox records the sandbox networks a pool creates and removes. +type fakeSandbox struct { + mu sync.Mutex + created []string + removed []string +} + +func (f *fakeSandbox) Create(_ context.Context, name string) error { + f.mu.Lock() + defer f.mu.Unlock() + f.created = append(f.created, name) + return nil +} + +func (f *fakeSandbox) Remove(_ context.Context, name string) error { + f.mu.Lock() + defer f.mu.Unlock() + f.removed = append(f.removed, name) + return nil +} + +// TestTaskContainerRunsOnItsOwnSandboxNetwork guards the task container's +// network boundary and its cleanup: the container joins only its own sandbox +// network, never the daemon's, so the egress relay is all it can reach; and +// the network goes with the task, so a host never fills with leaked +// networks. +func TestTaskContainerRunsOnItsOwnSandboxNetwork(t *testing.T) { + const daemonNetwork = "daemon-net" + tests := []struct { + name string + failStart bool + }{ + {name: "released task"}, + {name: "container that fails to start", failStart: true}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + var ( + mu sync.Mutex + attached []string + ) + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + switch { + case r.Method == http.MethodPost && strings.HasSuffix(r.URL.Path, "/containers/create"): + var req struct { + HostConfig struct { + NetworkMode string `json:"NetworkMode"` + } `json:"HostConfig"` + NetworkingConfig struct { + EndpointsConfig map[string]any `json:"EndpointsConfig"` + } `json:"NetworkingConfig"` + } + if err := json.NewDecoder(r.Body).Decode(&req); err != nil { + t.Error(err) + } + mu.Lock() + attached = append(attached, req.HostConfig.NetworkMode) + for name := range req.NetworkingConfig.EndpointsConfig { + attached = append(attached, name) + } + mu.Unlock() + w.Header().Set("Content-Type", "application/json") + _, _ = w.Write([]byte(`{"Id":"0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef"}`)) + case r.Method == http.MethodPost && strings.HasSuffix(r.URL.Path, "/start"): + if tt.failStart { + http.Error(w, `{"message":"start failed"}`, http.StatusInternalServerError) + return + } + w.WriteHeader(http.StatusNoContent) + case r.Method == http.MethodGet && strings.HasSuffix(r.URL.Path, "/wait"): + w.Header().Set("Content-Type", "application/json") + _, _ = w.Write([]byte(`{"StatusCode":0}`)) + case r.Method == http.MethodPost && strings.HasSuffix(r.URL.Path, "/stop"), + r.Method == http.MethodDelete: + w.WriteHeader(http.StatusNoContent) + default: + http.NotFound(w, r) + } + })) + defer server.Close() + cli, err := client.New(client.WithHost(server.URL), client.WithAPIVersion("1.54")) + if err != nil { + t.Fatal(err) + } + defer cli.Close() + + sandbox := &fakeSandbox{} + p := &Pool{ + cli: cli, + cfg: Config{Instance: "archie", Network: daemonNetwork}, + sandbox: sandbox, + network: daemonNetwork, + log: slog.New(slog.DiscardHandler), + } + ctr, err := p.Acquire(t.Context(), "agent:test", nil, nil) + if tt.failStart != (err != nil) { + t.Fatalf("Acquire error = %v, want failure %t", err, tt.failStart) + } + p.Release(t.Context(), ctr) + + mu.Lock() + defer mu.Unlock() + sandbox.mu.Lock() + defer sandbox.mu.Unlock() + if len(sandbox.created) != 1 { + t.Fatalf("sandbox networks created = %v, want exactly one for the task", sandbox.created) + } + if want := sandbox.created[:1]; !slices.Equal(attached, want) { + t.Fatalf("container attached to %v, want only its sandbox network %v", attached, want) + } + if !slices.Equal(sandbox.removed, sandbox.created) { + t.Fatalf("sandbox networks removed = %v, want %v removed with the task", sandbox.removed, sandbox.created) + } + }) + } +} diff --git a/internal/daemon/container_run.go b/internal/daemon/container_run.go index c68144561..b8201a774 100644 --- a/internal/daemon/container_run.go +++ b/internal/daemon/container_run.go @@ -19,6 +19,7 @@ import ( "github.com/samcharles93/archie-core/internal/eventbus" "github.com/samcharles93/archie-core/internal/events" agentnats "github.com/samcharles93/archie-core/internal/infrastructure/agenttransport/nats" + "github.com/samcharles93/archie-core/internal/infrastructure/egress" "github.com/samcharles93/archie-core/internal/storage" "github.com/samcharles93/archie-core/internal/taskrun" "github.com/samcharles93/archie-core/internal/taskstate" @@ -95,19 +96,15 @@ func (d *Daemon) acquireTaskContainer( revokeRun := revokeCredential revokeCredential = func() { closeModel(); revokeRun() } - // The container lives in its own network namespace, so a loopback State - // Store target names the container itself. Give this task an address on - // the host bridge that forwards to it, closed with the task. - stateStoreURL, relay, release, err := d.taskStateStoreEndpoint(ctx, revokeCredential) + env, err := d.containerEnv(credential) if err != nil { revokeCredential() d.teardownStorage(ctx, task, repo, workDir) - park("state store relay failed", err) + park("container environment failed", err) return nil, "", nil, false } - revokeCredential = release - ctr, err := d.ContainerPool.Acquire(ctx, image, append(mounts, model.Mounts...), append(d.containerEnv(task, credential, stateStoreURL), model.Env...)) + ctr, err := d.ContainerPool.Acquire(ctx, image, append(mounts, model.Mounts...), append(env, model.Env...)) if err != nil { revokeCredential() // Roll back the storage we just set up: the mounts were created @@ -118,90 +115,26 @@ func (d *Daemon) acquireTaskContainer( park("container acquire failed", err) return nil, "", nil, false } - d.admitToRelay(ctx, relay, task, ctr) return ctr, credential, revokeCredential, true } -// admitToRelay lets ctr, and only ctr, through relay. A container whose -// addresses cannot be read is refused rather than let anyone through. -func (d *Daemon) admitToRelay(ctx context.Context, relay *stateStoreRelay, task *workflow.Task, ctr *container.Container) { - if relay == nil { - return - } - addrs, err := d.ContainerPool.ContainerAddrs(ctx, ctr.ID) - if err != nil { - d.Log.WarnContext(ctx, "state store relay: container addresses unknown, refusing its connections", "task", task.ID, "err", err) - } - relay.Admit(addrs) -} - -// taskStateStoreEndpoint is the State Store address one task container is -// handed, the relay carrying it (nil when none is needed) and the release that -// ends that relay. The container lives -// in its own network namespace, so a loopback target names the container -// itself; behind a relay, the address is the host bridge's. -func (d *Daemon) taskStateStoreEndpoint(ctx context.Context, release func()) (string, *stateStoreRelay, func(), error) { - relay, err := d.openTaskStateStoreRelay(ctx) - if err != nil { - return "", nil, release, err - } - if relay == nil { - return d.ConnectedStateStore.URL, nil, release, nil - } - return relay.Addr(), relay, func() { relay.Close(); release() }, nil -} - -// openTaskStateStoreRelay starts the relay that carries one task container's -// State Store traffic to the daemon's target. The target stays bound where the -// operator put it; the container is handed an address on the host bridge, the -// same way it is handed NATS's. Nil when the target already reaches the -// containers, or when no bridge address is known. -func (d *Daemon) openTaskStateStoreRelay(ctx context.Context) (*stateStoreRelay, error) { - addr, loopback := loopbackStateStoreTarget(d.ConnectedStateStore.URL) - if !loopback { - return nil, nil - } - gateway := "" - if d.ContainerPool != nil { - gateway = d.ContainerPool.HostGateway() - } - if gateway == "" { - d.Log.WarnContext(ctx, "state store target is loopback and the container pool has no host bridge address; task containers cannot reach it", - "target", d.ConnectedStateStore.URL) - return nil, nil - } - return openStateStoreRelay(ctx, gateway, addr) -} - -// containerEnv is the environment a task container starts with. stateStoreURL -// is the address the container can reach the State Store by, which is the -// relay's rather than the daemon's own target. -func (d *Daemon) containerEnv(task *workflow.Task, stateStoreToken, stateStoreURL string) []string { - var env []string - // The endpoint the daemon's own client connected with at startup, not - // the live config: a reloaded [nats] section must not point new - // containers at a server the daemon is not publishing on. - env = append(env, "NATS_URL="+d.ConnectedNATS.URL) - if token := d.ConnectedNATS.ContainerToken(); token != "" { - env = append(env, "NATS_TOKEN="+token) - } - // Pass the container-reachable endpoint with this task's scoped token, - // never the daemon's own. - if stateStoreURL != "" { - env = append(env, "STATE_STORE_URL="+stateStoreURL) - if len(d.ConnectedStateStore.CA) > 0 { - env = append(env, "STATE_STORE_CA="+string(d.ConnectedStateStore.CA)) - } - if stateStoreToken != "" { - env = append(env, "STATE_STORE_TOKEN="+stateStoreToken) - } - } +// containerEnv is the environment a task container starts with. The container +// sits on a sandbox network whose only peer is the egress relay, so NATS and +// the State Store are the relay's ports; the credentials are this task's, +// never the daemon's own. NATS is the endpoint the daemon connected with at +// startup, not live config: a reloaded [nats] section must not point new +// containers at a server the daemon is not publishing on. +func (d *Daemon) containerEnv(stateStoreToken string) ([]string, error) { // The agent runs as root in the container, so its commits land owned by // UID 0 on the host. The daemon's UID/GID let the agent hand the worktree // back before the daemon reads it to push. - env = append(env, fmt.Sprintf("WORKTREE_UID=%d", os.Getuid())) - env = append(env, fmt.Sprintf("WORKTREE_GID=%d", os.Getgid())) - return env + return egress.WorkerEnv(egress.Endpoints{ + NATS: d.ConnectedNATS.URL, + NATSToken: d.ConnectedNATS.ContainerToken(), + StateStore: d.ConnectedStateStore.URL, + StateStoreToken: stateStoreToken, + StateStoreCA: d.ConnectedStateStore.CA, + }, os.Getuid(), os.Getgid()) } // runCredential issues the task's run credential, or "" when no State Store diff --git a/internal/daemon/statestore_relay.go b/internal/daemon/statestore_relay.go deleted file mode 100644 index 747e6ac50..000000000 --- a/internal/daemon/statestore_relay.go +++ /dev/null @@ -1,167 +0,0 @@ -package daemon - -import ( - "context" - "fmt" - "io" - "net" - "net/netip" - "slices" - "strings" - "sync" - "time" -) - -// stateStoreDialTimeout bounds one forwarded connection's dial. The target is -// local, so a dial that has not connected by then is a dead State Store rather -// than a slow one. -const stateStoreDialTimeout = 10 * time.Second - -// stateStoreRelay carries one task's State Store traffic from an address on -// the host's container bridge to the target the daemon dialed. A container has -// its own network namespace, so a loopback target there names the container -// itself; the relay address is what the container is handed instead. The State -// Store keeps listening exactly where the operator bound it. -type stateStoreRelay struct { - target string - ln net.Listener - - // admitted is closed once allowed holds the task container's addresses; - // closed is closed by Close. A connection waits for one of them, so a - // container that dials before it is admitted is held rather than refused. - admitted chan struct{} - closed chan struct{} - closeOnce sync.Once - allowed []netip.Addr -} - -// openStateStoreRelay listens on gateway and forwards to addr. The listener is -// per task: it forwards only connections from the addresses Admit names, and -// closing it is what ends the task's access. -func openStateStoreRelay(ctx context.Context, gateway, addr string) (*stateStoreRelay, error) { - ln, err := (&net.ListenConfig{}).Listen(ctx, "tcp", net.JoinHostPort(gateway, "0")) - if err != nil { - return nil, fmt.Errorf("state store relay: listen on %s: %w", gateway, err) - } - r := &stateStoreRelay{target: addr, ln: ln, admitted: make(chan struct{}), closed: make(chan struct{})} - go r.serve(ctx) - return r, nil -} - -// Admit names the task container's addresses, the only peers the relay -// forwards. It is called once, after the container exists. -func (r *stateStoreRelay) Admit(addrs []netip.Addr) { - r.allowed = addrs - close(r.admitted) -} - -// admits waits until the relay knows its container, then reports whether -// conn comes from it. -func (r *stateStoreRelay) admits(ctx context.Context, conn net.Conn) bool { - select { - case <-r.admitted: - case <-r.closed: - return false - case <-ctx.Done(): - return false - } - peer, err := netip.ParseAddrPort(conn.RemoteAddr().String()) - if err != nil { - return false - } - return slices.Contains(r.allowed, peer.Addr().Unmap()) -} - -// Addr is the address the task container is handed. -func (r *stateStoreRelay) Addr() string { return r.ln.Addr().String() } - -// Close stops accepting. Streams already open end when the container that owns -// them goes away, and a restarted daemon leaves no listener behind: the -// sockets belong to its process. -func (r *stateStoreRelay) Close() { - r.closeOnce.Do(func() { close(r.closed) }) - _ = r.ln.Close() -} - -func (r *stateStoreRelay) serve(ctx context.Context) { - for { - downstream, err := r.ln.Accept() - if err != nil { - return - } - go r.forward(ctx, downstream) - } -} - -// forward joins one container stream to the target, half-closing each side so -// a gRPC stream drains in both directions before the pair ends. -func (r *stateStoreRelay) forward(ctx context.Context, downstream net.Conn) { - if !r.admits(ctx, downstream) { - _ = downstream.Close() - return - } - ctx, cancel := context.WithTimeout(ctx, stateStoreDialTimeout) - defer cancel() - upstream, err := (&net.Dialer{}).DialContext(ctx, "tcp", r.target) - if err != nil { - _ = downstream.Close() - return - } - var wg sync.WaitGroup - wg.Add(2) - go func() { defer wg.Done(); copyAndClose(upstream, downstream) }() - go func() { defer wg.Done(); copyAndClose(downstream, upstream) }() - wg.Wait() - _ = downstream.Close() - _ = upstream.Close() -} - -func copyAndClose(dst, src net.Conn) { - _, _ = io.Copy(dst, src) - if half, ok := dst.(interface{ CloseWrite() error }); ok { - _ = half.CloseWrite() - return - } - _ = dst.Close() -} - -// loopbackStateStoreTarget splits target into the address to forward to, and -// reports whether it is this host's loopback: the one case a task container -// cannot reach on its own. A target already addressed for the containers' -// network needs no relay. -func loopbackStateStoreTarget(target string) (addr string, loopback bool) { - addr, host, ok := stateStoreTargetAddr(target) - if !ok || !loopbackHost(host) { - return "", false - } - return addr, true -} - -// StateStoreNeedsRelay reports whether task containers can reach target only -// through a relay on the host bridge. -func StateStoreNeedsRelay(target string) bool { - _, loopback := loopbackStateStoreTarget(target) - return loopback -} - -// stateStoreTargetAddr is target as host:port, accepting the gRPC scheme form -// ("dns:///host:port") the configuration allows. -func stateStoreTargetAddr(target string) (addr, host string, ok bool) { - target = strings.TrimSpace(target) - if _, rest, found := strings.Cut(target, ":///"); found { - target = rest - } - host, _, err := net.SplitHostPort(target) - if err != nil { - return "", "", false - } - return target, host, true -} - -func loopbackHost(host string) bool { - if host == "localhost" { - return true - } - ip := net.ParseIP(host) - return ip != nil && ip.IsLoopback() -} diff --git a/internal/daemon/statestore_relay_test.go b/internal/daemon/statestore_relay_test.go deleted file mode 100644 index 2bf1bda63..000000000 --- a/internal/daemon/statestore_relay_test.go +++ /dev/null @@ -1,172 +0,0 @@ -package daemon - -import ( - "context" - "io" - "net" - "net/netip" - "strings" - "testing" - "time" - - "github.com/samcharles93/archie-core/internal/domain/workflow" -) - -// The relay is what makes a loopback State Store reachable from a container: -// bytes the task's container writes to its address arrive at the target, and -// no other peer on the bridge gets through. -func TestStateStoreRelayForwards(t *testing.T) { - t.Parallel() - - tests := []struct { - name string - admit []netip.Addr - carried bool - }{ - {name: "the task container", admit: []netip.Addr{netip.MustParseAddr("127.0.0.1")}, carried: true}, - {name: "another peer", admit: []netip.Addr{netip.MustParseAddr("127.0.0.2")}}, - } - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - upstream := echoListener(t) - relay, err := openStateStoreRelay(context.Background(), "127.0.0.1", upstream.Addr().String()) - if err != nil { - t.Fatalf("open relay: %v", err) - } - t.Cleanup(relay.Close) - if relay.Addr() == upstream.Addr().String() { - t.Fatalf("relay listens on the target's own address %s", relay.Addr()) - } - - conn, err := (&net.Dialer{}).DialContext(context.Background(), "tcp", relay.Addr()) - if err != nil { - t.Fatalf("dial relay: %v", err) - } - defer func() { _ = conn.Close() }() - relay.Admit(tt.admit) - _ = conn.SetDeadline(time.Now().Add(5 * time.Second)) - _, _ = conn.Write([]byte("ping")) - got := make([]byte, 4) - _, err = io.ReadFull(conn, got) - if carried := err == nil && string(got) == "ping"; carried != tt.carried { - t.Fatalf("carried = %v (read %q, %v), want %v", carried, got, err, tt.carried) - } - }) - } -} - -// The relay is per task, so releasing the task has to end the access it gave. -func TestStateStoreRelayClosesWithTask(t *testing.T) { - t.Parallel() - - upstream := echoListener(t) - relay, err := openStateStoreRelay(context.Background(), "127.0.0.1", upstream.Addr().String()) - if err != nil { - t.Fatalf("open relay: %v", err) - } - addr := relay.Addr() - relay.Close() - - if conn, err := (&net.Dialer{Timeout: time.Second}).DialContext(context.Background(), "tcp", addr); err == nil { - _ = conn.Close() - t.Fatalf("relay still accepts at %s after the task released it", addr) - } -} - -// An ordinary image task is handed the relay's address, never the loopback -// target the daemon dialed, with the task-scoped credentials unchanged. -func TestContainerEnvStateStore(t *testing.T) { - t.Parallel() - - upstream := echoListener(t) - relay, err := openStateStoreRelay(context.Background(), "127.0.0.1", upstream.Addr().String()) - if err != nil { - t.Fatalf("open relay: %v", err) - } - t.Cleanup(relay.Close) - - d := wired(&Daemon{ - ConnectedNATS: NATSEndpoint{URL: "nats://10.0.0.5:4222"}, - ConnectedStateStore: StateStoreEndpoint{URL: upstream.Addr().String(), CA: []byte("ca-pem")}, - }) - env := envValues(d.containerEnv(&workflow.Task{ID: 7}, "task-token", relay.Addr())) - - if got := env["STATE_STORE_URL"]; got != relay.Addr() { - t.Fatalf("STATE_STORE_URL = %q, want the relay address %q", got, relay.Addr()) - } - if env["STATE_STORE_TOKEN"] != "task-token" { - t.Fatalf("STATE_STORE_TOKEN = %q, want the task's own token", env["STATE_STORE_TOKEN"]) - } - if env["STATE_STORE_CA"] != "ca-pem" { - t.Fatalf("STATE_STORE_CA = %q, want the CA passed through", env["STATE_STORE_CA"]) - } - if got := env["NATS_URL"]; got != "nats://10.0.0.5:4222" { - t.Fatalf("NATS_URL = %q, want the connected broker", got) - } -} - -// Only a target the container cannot reach is relayed. An address already -// configured for the containers' network is passed through untouched, and a -// target the relay cannot forward is left alone rather than guessed at. -func TestLoopbackStateStoreTarget(t *testing.T) { - t.Parallel() - - cases := []struct { - name string - target string - addr string - loopback bool - }{ - {name: "loopback ipv4", target: "127.0.0.1:9090", addr: "127.0.0.1:9090", loopback: true}, - {name: "loopback ipv6", target: "[::1]:9090", addr: "[::1]:9090", loopback: true}, - {name: "localhost", target: "localhost:9090", addr: "localhost:9090", loopback: true}, - {name: "grpc scheme form", target: "dns:///127.0.0.1:9090", addr: "127.0.0.1:9090", loopback: true}, - {name: "host address", target: "10.0.0.5:9090"}, - {name: "service name", target: "state-store:9090"}, - {name: "unset", target: ""}, - } - for _, tc := range cases { - t.Run(tc.name, func(t *testing.T) { - t.Parallel() - addr, loopback := loopbackStateStoreTarget(tc.target) - if loopback != tc.loopback { - t.Fatalf("loopbackStateStoreTarget(%q) loopback = %t, want %t", tc.target, loopback, tc.loopback) - } - if addr != tc.addr { - t.Fatalf("loopbackStateStoreTarget(%q) addr = %q, want %q", tc.target, addr, tc.addr) - } - }) - } -} - -// echoListener accepts connections on loopback and echoes what it reads. -func echoListener(t *testing.T) net.Listener { - t.Helper() - ln, err := (&net.ListenConfig{}).Listen(context.Background(), "tcp", "127.0.0.1:0") - if err != nil { - t.Fatalf("listen: %v", err) - } - t.Cleanup(func() { _ = ln.Close() }) - go func() { - for { - conn, err := ln.Accept() - if err != nil { - return - } - go func() { - defer func() { _ = conn.Close() }() - _, _ = io.Copy(conn, conn) - }() - } - }() - return ln -} - -func envValues(env []string) map[string]string { - out := make(map[string]string, len(env)) - for _, entry := range env { - key, value, _ := strings.Cut(entry, "=") - out[key] = value - } - return out -} diff --git a/internal/daemon/statestore_target.go b/internal/daemon/statestore_target.go new file mode 100644 index 000000000..a42afcfc5 --- /dev/null +++ b/internal/daemon/statestore_target.go @@ -0,0 +1,56 @@ +package daemon + +import ( + "fmt" + "net" + "strings" +) + +// StateStoreNeedsRelay reports whether target is bound to this host's +// loopback, which the egress relay can reach only through the host bridge's +// gateway address. +func StateStoreNeedsRelay(target string) bool { + _, host, ok := stateStoreTargetAddr(target) + return ok && loopbackHost(host) +} + +// RelayStateStoreTarget is the host:port the egress relay forwards task State +// Store traffic to. The relay has its own network namespace, so a loopback +// target would name the relay itself: it is reached at gateway, the host's +// address on the relay's bridge, on the same port. +func RelayStateStoreTarget(target, gateway string) (string, error) { + addr, host, ok := stateStoreTargetAddr(target) + if !ok { + return "", fmt.Errorf("state store target %q: want host:port", target) + } + if !loopbackHost(host) { + return addr, nil + } + if gateway == "" { + return "", fmt.Errorf("state store target %q is loopback and the container network has no host gateway", target) + } + _, port, _ := net.SplitHostPort(addr) + return net.JoinHostPort(gateway, port), nil +} + +// stateStoreTargetAddr is target as host:port, accepting the gRPC scheme form +// ("dns:///host:port") the configuration allows. +func stateStoreTargetAddr(target string) (addr, host string, ok bool) { + target = strings.TrimSpace(target) + if _, rest, found := strings.Cut(target, ":///"); found { + target = rest + } + host, _, err := net.SplitHostPort(target) + if err != nil { + return "", "", false + } + return target, host, true +} + +func loopbackHost(host string) bool { + if strings.EqualFold(host, "localhost") { + return true + } + ip := net.ParseIP(host) + return ip != nil && ip.IsLoopback() +} diff --git a/internal/daemon/statestore_target_test.go b/internal/daemon/statestore_target_test.go new file mode 100644 index 000000000..b5a118139 --- /dev/null +++ b/internal/daemon/statestore_target_test.go @@ -0,0 +1,40 @@ +package daemon + +import "testing" + +// TestRelayStateStoreTarget guards where the egress relay sends task State +// Store traffic: a loopback target is reached at the host gateway on the same +// port, since loopback inside the relay names the relay itself, and any other +// target passes through untouched. +func TestRelayStateStoreTarget(t *testing.T) { + t.Parallel() + + cases := []struct { + name string + target string + gateway string + want string + wantErr bool + }{ + {name: "loopback ipv4", target: "127.0.0.1:9090", gateway: "172.17.0.1", want: "172.17.0.1:9090"}, + {name: "loopback ipv6", target: "[::1]:9090", gateway: "172.17.0.1", want: "172.17.0.1:9090"}, + {name: "localhost", target: "localhost:9090", gateway: "172.17.0.1", want: "172.17.0.1:9090"}, + {name: "grpc scheme form", target: "dns:///127.0.0.1:9090", gateway: "172.17.0.1", want: "172.17.0.1:9090"}, + {name: "host address", target: "10.0.0.5:9090", gateway: "172.17.0.1", want: "10.0.0.5:9090"}, + {name: "service name", target: "dns:///state-store:9090", want: "state-store:9090"}, + {name: "loopback without a gateway", target: "127.0.0.1:9090", wantErr: true}, + {name: "unset", target: "", wantErr: true}, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + t.Parallel() + got, err := RelayStateStoreTarget(tc.target, tc.gateway) + if (err != nil) != tc.wantErr { + t.Fatalf("RelayStateStoreTarget(%q, %q) error = %v, want error %t", tc.target, tc.gateway, err, tc.wantErr) + } + if got != tc.want { + t.Fatalf("RelayStateStoreTarget(%q, %q) = %q, want %q", tc.target, tc.gateway, got, tc.want) + } + }) + } +} diff --git a/internal/infrastructure/egress/networks.go b/internal/infrastructure/egress/networks.go index 919568af2..a00ba2265 100644 --- a/internal/infrastructure/egress/networks.go +++ b/internal/infrastructure/egress/networks.go @@ -29,12 +29,9 @@ const ( ) // RelaySpec is how the relay container is started. Network is the -// host-reachable network it joins to reach the proxy; Entrypoint and Cmd -// run archie-agent relay forwarding to the proxy's address on that network. +// host-reachable network it joins to reach the proxy, NATS and the State +// Store; Entrypoint and Cmd run archie-agent relay forwarding to them. type RelaySpec struct { - // Name is the relay container's name. It also scopes the labels Start - // prunes by, so two Networks on one host never remove each other's. - Name string Image string Entrypoint []string Cmd []string @@ -49,41 +46,50 @@ type RelaySpec struct { // other host service, including ones listening on all interfaces, is out of // reach. type Networks struct { - cli *client.Client - relay RelaySpec + cli *client.Client + // relayName is the relay container's name. It also scopes the labels + // Sweep prunes by, so two instances on one host never remove each + // other's networks. + relayName string } -func NewNetworks(cli *client.Client, relay RelaySpec) *Networks { - return &Networks{cli: cli, relay: relay} +// NewNetworks is the sandbox provisioner for the relay named relayName. The +// relay itself is started by Start, once the addresses it forwards to exist. +func NewNetworks(cli *client.Client, relayName string) *Networks { + return &Networks{cli: cli, relayName: relayName} } -// Start removes sandbox networks a previous daemon left behind and starts a -// fresh relay. Work interrupted by a restart is requeued, so no live sandbox -// depends on the old ones. -func (n *Networks) Start(ctx context.Context) error { +// Sweep removes the relay and every sandbox network a previous daemon left +// behind. Work interrupted by a restart is requeued, so no live sandbox +// depends on them. It removes what it can and reports the rest. +func (n *Networks) Sweep(ctx context.Context) error { + errs := []error{n.removeRelay(ctx)} + list, err := n.cli.NetworkList(ctx, client.NetworkListOptions{Filters: client.Filters{}.Add("label", sandboxLabel+"="+n.relayName)}) + if err != nil { + return errors.Join(append(errs, fmt.Errorf("list sandbox networks: %w", err))...) + } + for _, stale := range list.Items { + errs = append(errs, n.Remove(ctx, stale.ID)) + } + return errors.Join(errs...) +} + +// Start starts a fresh relay, replacing any left running. +func (n *Networks) Start(ctx context.Context, relay RelaySpec) error { if err := n.removeRelay(ctx); err != nil { return err } - list, err := n.cli.NetworkList(ctx, client.NetworkListOptions{Filters: client.Filters{}.Add("label", sandboxLabel+"="+n.relay.Name)}) - if err != nil { - return fmt.Errorf("list sandbox networks: %w", err) - } - for _, stale := range list.Items { - if _, err := n.cli.NetworkRemove(ctx, stale.ID, client.NetworkRemoveOptions{}); err != nil { - return fmt.Errorf("remove stale sandbox network %s: %w", stale.Name, err) - } - } created, err := n.cli.ContainerCreate(ctx, client.ContainerCreateOptions{ - Name: n.relay.Name, + Name: n.relayName, Config: &container.Config{ - Image: n.relay.Image, - Entrypoint: n.relay.Entrypoint, - Cmd: n.relay.Cmd, - Labels: map[string]string{relayLabel: n.relay.Name}, + Image: relay.Image, + Entrypoint: relay.Entrypoint, + Cmd: relay.Cmd, + Labels: map[string]string{relayLabel: n.relayName}, }, HostConfig: &container.HostConfig{ - NetworkMode: container.NetworkMode(n.relay.Network), - Binds: n.relay.Binds, + NetworkMode: container.NetworkMode(relay.Network), + Binds: relay.Binds, RestartPolicy: container.RestartPolicy{Name: container.RestartPolicyUnlessStopped}, }, }) @@ -102,7 +108,7 @@ func (n *Networks) Close(ctx context.Context) error { return n.removeRelay(ctx) func (n *Networks) removeRelay(ctx context.Context) error { list, err := n.cli.ContainerList(ctx, client.ContainerListOptions{ All: true, - Filters: client.Filters{}.Add("label", relayLabel+"="+n.relay.Name), + Filters: client.Filters{}.Add("label", relayLabel+"="+n.relayName), }) if err != nil { return fmt.Errorf("list egress relays: %w", err) @@ -124,12 +130,12 @@ func (n *Networks) Create(ctx context.Context, name string) error { // routable from the sandbox; internal alone still exposes every // host service through the gateway. Options: map[string]string{"com.docker.network.bridge.inhibit_ipv4": "true"}, - Labels: map[string]string{sandboxLabel: n.relay.Name}, + Labels: map[string]string{sandboxLabel: n.relayName}, }); err != nil { return fmt.Errorf("create sandbox network %s: %w", name, err) } if _, err := n.cli.NetworkConnect(ctx, name, client.NetworkConnectOptions{ - Container: n.relay.Name, + Container: n.relayName, EndpointConfig: &network.EndpointSettings{Aliases: []string{RelayAlias}}, }); err != nil { return errors.Join(fmt.Errorf("attach egress relay to %s: %w", name, err), n.Remove(context.WithoutCancel(ctx), name)) @@ -137,11 +143,23 @@ func (n *Networks) Create(ctx context.Context, name string) error { return nil } -// Remove detaches the relay from a sandbox network and deletes it. +// Remove deletes a sandbox network. Every endpoint still on it is detached +// first: the task container may not be gone yet when the max-uptime reaper +// and Release race, and an attached endpoint would otherwise block removal +// and leak the network's subnet. func (n *Networks) Remove(ctx context.Context, name string) error { - _, detachErr := n.cli.NetworkDisconnect(ctx, name, client.NetworkDisconnectOptions{Container: n.relay.Name, Force: true}) + inspected, err := n.cli.NetworkInspect(ctx, name, client.NetworkInspectOptions{}) + if err != nil { + return fmt.Errorf("inspect sandbox network %s: %w", name, err) + } + var errs []error + for id := range inspected.Network.Containers { + if _, err := n.cli.NetworkDisconnect(ctx, name, client.NetworkDisconnectOptions{Container: id, Force: true}); err != nil { + errs = append(errs, fmt.Errorf("detach %s from sandbox network %s: %w", id, name, err)) + } + } if _, err := n.cli.NetworkRemove(ctx, name, client.NetworkRemoveOptions{}); err != nil { - return errors.Join(detachErr, fmt.Errorf("remove sandbox network %s: %w", name, err)) + return errors.Join(append(errs, fmt.Errorf("remove sandbox network %s: %w", name, err))...) } return nil }