fix(container): task containers run on their own sandbox network behind the egress relay (#1867)
Each task container now gets an internal bridge with no host address, shared only with the egress relay. NATS, the State Store and the model proxy are reached through the relay's ports; a loopback State Store is forwarded to the host gateway. The network is removed with the task, and boot sweeps any a previous daemon left. The per-task State Store relay on the host bridge is gone.
This commit is contained in:
12 files changed
+434
-494
No files matched your search
@@ -42,6 +42,7 @@ import (
|
||||
infraaccess "github.com/samcharles93/archie-core/internal/infrastructure/access"
|
||||
"github.com/samcharles93/archie-core/internal/infrastructure/configuration"
|
||||
"github.com/samcharles93/archie-core/internal/infrastructure/controlplanestore"
|
||||
"github.com/samcharles93/archie-core/internal/infrastructure/egress"
|
||||
"github.com/samcharles93/archie-core/internal/infrastructure/eventbus/nats"
|
||||
"github.com/samcharles93/archie-core/internal/infrastructure/extension"
|
||||
"github.com/samcharles93/archie-core/internal/infrastructure/issuerpc"
|
||||
@@ -159,10 +160,16 @@ type boot struct {
|
||||
natsToken string
|
||||
|
||||
containerPool *container.Pool
|
||||
// sandboxes provisions each task container's isolated network and runs
|
||||
// the egress relay that is its only way out.
|
||||
sandboxes *egress.Networks
|
||||
// modelEgress gives native tasks model access through the egress proxy;
|
||||
// nil when the proxy cannot start.
|
||||
modelEgress daemon.ModelEgress
|
||||
storeBackend storage.Backend
|
||||
modelEgress daemon.ModelEgress
|
||||
// egressProxyAddr is where the egress proxy listens, the address the relay
|
||||
// forwards its proxy port to. Empty when the proxy is not running.
|
||||
egressProxyAddr string
|
||||
storeBackend storage.Backend
|
||||
|
||||
// runtimeVersions records the version of every control-plane kind boot's
|
||||
// layering applied. Set once at boot before the runtime-resource watches
|
||||
@@ -484,8 +491,9 @@ func (b *boot) startEmbeddedNATS(ctx context.Context) (string, string, error) {
|
||||
}
|
||||
|
||||
func (b *boot) setupContainers(ctx context.Context) func() {
|
||||
containerPool, storeBackend, closeDocker := startContainers(ctx, b.cfgHolder, b.secrets, b.log)
|
||||
containerPool, sandboxes, storeBackend, closeDocker := startContainers(ctx, b.cfgHolder, b.secrets, b.log)
|
||||
b.containerPool = containerPool
|
||||
b.sandboxes = sandboxes
|
||||
b.storeBackend = storeBackend
|
||||
return closeDocker
|
||||
}
|
||||
@@ -725,6 +733,7 @@ func (b *boot) setupBackends(ctx context.Context) error {
|
||||
err := b.connectNATS(ctx)
|
||||
if err == nil {
|
||||
b.setupModelEgress(ctx)
|
||||
b.startEgressRelay(ctx)
|
||||
}
|
||||
return err
|
||||
}
|
||||
|
||||
@@ -0,0 +1,57 @@
|
||||
package archied
|
||||
|
||||
import (
|
||||
"context"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/samcharles93/archie-core/internal/config"
|
||||
"github.com/samcharles93/archie-core/internal/daemon"
|
||||
"github.com/samcharles93/archie-core/internal/infrastructure/egress"
|
||||
)
|
||||
|
||||
// startEgressRelay removes the sandbox networks and relay a previous daemon
|
||||
// left, then starts the relay: the only peer on every task container's
|
||||
// sandbox network, forwarding to NATS, the State Store and the model proxy.
|
||||
// Boot carries on when it cannot start; each task then parks on the failed
|
||||
// sandbox attach rather than starting with no way out.
|
||||
func (b *boot) startEgressRelay(ctx context.Context) {
|
||||
if b.containerPool == nil || b.sandboxes == nil {
|
||||
return
|
||||
}
|
||||
log := b.log
|
||||
if err := b.sandboxes.Sweep(ctx); err != nil {
|
||||
log.WarnContext(ctx, "egress relay: stale sandbox networks remain", "err", err)
|
||||
}
|
||||
stateStore := ""
|
||||
if target := strings.TrimSpace(b.cfg.Services.Get(config.ServiceNameState).Target); target != "" {
|
||||
addr, err := daemon.RelayStateStoreTarget(target, b.containerPool.HostGateway())
|
||||
if err != nil {
|
||||
log.ErrorContext(ctx, "egress relay: task containers cannot reach the state store", "err", err)
|
||||
}
|
||||
stateStore = addr
|
||||
}
|
||||
cmd, err := egress.RelayCommand(b.egressProxyAddr, egress.Endpoints{NATS: b.natsURL, StateStore: stateStore})
|
||||
if err != nil {
|
||||
log.ErrorContext(ctx, "egress relay not started: task containers cannot run", "err", err)
|
||||
return
|
||||
}
|
||||
spec := egress.RelaySpec{
|
||||
Image: b.cfg.Containers.Image,
|
||||
Entrypoint: []string{"archie-agent"},
|
||||
Cmd: cmd,
|
||||
Network: b.containerPool.NetworkName(),
|
||||
}
|
||||
if err := b.sandboxes.Start(ctx, spec); err != nil {
|
||||
log.ErrorContext(ctx, "egress relay not started: task containers cannot run", "err", err)
|
||||
return
|
||||
}
|
||||
b.addCleanup(func() {
|
||||
closeCtx, cancel := context.WithTimeout(context.WithoutCancel(ctx), 10*time.Second)
|
||||
defer cancel()
|
||||
if err := b.sandboxes.Close(closeCtx); err != nil {
|
||||
log.WarnContext(ctx, "egress relay remove failed", "err", err)
|
||||
}
|
||||
})
|
||||
log.InfoContext(ctx, "egress relay started", "network", spec.Network, "proxy", b.egressProxyAddr != "", "state_store", stateStore != "")
|
||||
}
|
||||
@@ -37,6 +37,7 @@ import (
|
||||
"github.com/samcharles93/archie-core/internal/forge"
|
||||
"github.com/samcharles93/archie-core/internal/forgerpc"
|
||||
"github.com/samcharles93/archie-core/internal/infrastructure/configuration"
|
||||
"github.com/samcharles93/archie-core/internal/infrastructure/egress"
|
||||
"github.com/samcharles93/archie-core/internal/infrastructure/readiness"
|
||||
"github.com/samcharles93/archie-core/internal/logging"
|
||||
"github.com/samcharles93/archie-core/internal/messagerpc"
|
||||
@@ -380,14 +381,14 @@ func startContainers(
|
||||
cfgHolder *config.Holder,
|
||||
secrets *secret.Registry,
|
||||
log *slog.Logger,
|
||||
) (*container.Pool, storage.Backend, func()) {
|
||||
) (*container.Pool, *egress.Networks, storage.Backend, func()) {
|
||||
noop := func() {}
|
||||
cfg := cfgHolder.Get()
|
||||
// Single Docker client shared between pool and storage backend.
|
||||
dockerCli, err := client.New(client.FromEnv)
|
||||
if err != nil {
|
||||
log.ErrorContext(ctx, "autonomous workflows unavailable: docker client unavailable", "err", err)
|
||||
return nil, nil, noop
|
||||
return nil, nil, nil, noop
|
||||
}
|
||||
closeDocker := func() {
|
||||
if err := dockerCli.Close(); err != nil {
|
||||
@@ -395,6 +396,7 @@ func startContainers(
|
||||
}
|
||||
}
|
||||
|
||||
sandboxes := egress.NewNetworks(dockerCli, cfg.Instance+"-egress-relay")
|
||||
pool, err := container.NewPool(ctx, container.Config{
|
||||
Image: cfg.Containers.Image,
|
||||
MaxConcurrency: cfg.Containers.MaxConcurrency,
|
||||
@@ -403,23 +405,24 @@ func startContainers(
|
||||
RegistryAuth: resolveRegistryAuth(ctx, cfg.Containers.RegistryAuth, secrets, log),
|
||||
Network: cfg.Containers.Network,
|
||||
DockerClient: dockerCli,
|
||||
// The embedded broker binds the host gateway, and a loopback State
|
||||
// Store is relayed from it: containers reach neither otherwise.
|
||||
// The embedded broker binds the host gateway, and the egress relay
|
||||
// reaches a loopback State Store there: it reaches neither otherwise.
|
||||
RequireHostGateway: cfg.NATS.Mode == config.NATSModeEmbedded ||
|
||||
daemon.StateStoreNeedsRelay(cfg.Services.Get(config.ServiceNameState).Target),
|
||||
Sandbox: sandboxes,
|
||||
Instance: cfg.Instance,
|
||||
}, containerLiveSettings(cfgHolder), log)
|
||||
if err != nil {
|
||||
// A missing image is recoverable by hand.
|
||||
log.ErrorContext(ctx, "autonomous workflows unavailable: container pool unavailable", "err", err,
|
||||
"hint", "run `docker compose pull agent` (or `build agent`) so the image is present locally")
|
||||
return nil, storage.NewDockerBackend(dockerCli, cfg.Instance), closeDocker
|
||||
return nil, nil, storage.NewDockerBackend(dockerCli, cfg.Instance), closeDocker
|
||||
}
|
||||
|
||||
// contextcheck: Pool.Close takes no context by design -- it is a shutdown
|
||||
// path that must run to completion after ctx is already cancelled.
|
||||
//nolint:contextcheck
|
||||
return pool, storage.NewDockerBackend(dockerCli, cfg.Instance), func() {
|
||||
return pool, sandboxes, storage.NewDockerBackend(dockerCli, cfg.Instance), func() {
|
||||
if err := pool.Close(); err != nil {
|
||||
log.WarnContext(ctx, "container pool close failed", "err", err)
|
||||
}
|
||||
@@ -431,9 +434,9 @@ func startContainers(
|
||||
// running config. The pool calls it on every acquire, so a stored
|
||||
// container-runtime-policies change reaches the next container without a
|
||||
// restart. Only fields the resource document carries are taken live:
|
||||
// RegistryAuth is file-owned and stays the boot-resolved value, DockerClient
|
||||
// and RequireHostGateway are construction-time, and GracePeriod is not part
|
||||
// of the document.
|
||||
// RegistryAuth is file-owned and stays the boot-resolved value; DockerClient,
|
||||
// RequireHostGateway and Network, which the egress relay joins at boot, are
|
||||
// construction-time; and GracePeriod is not part of the document.
|
||||
func containerLiveSettings(cfgHolder *config.Holder) func() container.Config {
|
||||
return func() container.Config {
|
||||
cfg := cfgHolder.Get()
|
||||
@@ -442,7 +445,6 @@ func containerLiveSettings(cfgHolder *config.Holder) func() container.Config {
|
||||
MaxConcurrency: cfg.Containers.MaxConcurrency,
|
||||
MaxUptime: cfg.Containers.MaxUptime.Std(),
|
||||
PullPolicy: cfg.Containers.PullPolicy,
|
||||
Network: cfg.Containers.Network,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -6,6 +6,7 @@ import (
|
||||
"net"
|
||||
"net/http"
|
||||
"path/filepath"
|
||||
"strconv"
|
||||
"time"
|
||||
|
||||
"github.com/samcharles93/archie-core/internal/infrastructure/egress"
|
||||
@@ -41,6 +42,9 @@ func (b *boot) setupModelEgress(ctx context.Context) {
|
||||
}
|
||||
}()
|
||||
b.addCleanup(func() { _ = srv.Close() })
|
||||
b.modelEgress = modelEgress{proxy: proxy, addr: ln.Addr().String(), caFile: egress.CACertPath(caDir), config: b.cfgHolder, secrets: b.secrets}
|
||||
// Task containers reach the proxy only through the egress relay, which
|
||||
// forwards its proxy port here.
|
||||
b.egressProxyAddr = ln.Addr().String()
|
||||
b.modelEgress = modelEgress{proxy: proxy, addr: net.JoinHostPort(egress.RelayAlias, strconv.Itoa(egress.RelayPort)), caFile: egress.CACertPath(caDir), config: b.cfgHolder, secrets: b.secrets}
|
||||
log.InfoContext(ctx, "model egress enabled", "proxy", ln.Addr().String())
|
||||
}
|
||||
+49
-21
@@ -32,6 +32,9 @@ import (
|
||||
type Container struct {
|
||||
ID string
|
||||
exited <-chan struct{}
|
||||
// network is the task's sandbox network, removed by Release. Empty when the
|
||||
// pool runs without a sandbox.
|
||||
network string
|
||||
// detail is written before exited closes, so a reader that saw the close
|
||||
// may read it without a lock.
|
||||
detail *string
|
||||
@@ -96,12 +99,20 @@ func WriteTaskJSON(workspace string, payload TaskPayload) error {
|
||||
return os.WriteFile(filepath.Join(gitDir, "task.json"), data, 0o644) //nolint:gosec // G306: the task container reads its brief as another user.
|
||||
}
|
||||
|
||||
// SandboxNetworks provisions one isolated network per task container. The
|
||||
// container can reach only what is attached to that network.
|
||||
type SandboxNetworks interface {
|
||||
Create(ctx context.Context, name string) error
|
||||
Remove(ctx context.Context, name string) error
|
||||
}
|
||||
|
||||
// Pool manages a set of Docker containers running archie-agent.
|
||||
type Pool struct {
|
||||
cli *client.Client
|
||||
ownCli bool // true when the pool created the client (owns Close)
|
||||
cfg Config
|
||||
log *slog.Logger
|
||||
cli *client.Client
|
||||
sandbox SandboxNetworks
|
||||
ownCli bool // true when the pool created the client (owns Close)
|
||||
cfg Config
|
||||
log *slog.Logger
|
||||
// network is the Docker network spawned containers join. External broker
|
||||
// deployments may leave it empty to use Docker's implicit default bridge;
|
||||
// embedded deployments resolve that default explicitly as "bridge" so its
|
||||
@@ -140,7 +151,6 @@ func (p *Pool) current() Config {
|
||||
cfg.MaxConcurrency = live.MaxConcurrency
|
||||
cfg.MaxUptime = live.MaxUptime
|
||||
cfg.PullPolicy = live.PullPolicy
|
||||
cfg.Network = live.Network
|
||||
return cfg
|
||||
}
|
||||
|
||||
@@ -164,8 +174,9 @@ type Config struct {
|
||||
// registry) or a Docker registry.AuthConfig JSON document. pullImage
|
||||
// encodes it into the X-Registry-Auth header the Engine API expects.
|
||||
RegistryAuth string
|
||||
// Network is the Docker network spawned agent containers join. Empty
|
||||
// falls back to selfNetwork's best-effort auto-detection.
|
||||
// Network is the host-reachable Docker network the egress relay joins,
|
||||
// read once at construction. Empty falls back to selfNetwork's
|
||||
// best-effort auto-detection.
|
||||
Network string
|
||||
// DockerClient is an optional pre-connected Docker client. When nil,
|
||||
// NewPool creates its own via client.New(client.FromEnv). Pass a
|
||||
@@ -174,6 +185,9 @@ type Config struct {
|
||||
// RequireHostGateway resolves and validates a local Docker bridge gateway.
|
||||
// Embedded NATS needs this address; external brokers do not.
|
||||
RequireHostGateway bool
|
||||
// Sandbox provisions the isolated network each task container runs on.
|
||||
// It is required: no task container shares a network with the host.
|
||||
Sandbox SandboxNetworks
|
||||
// Instance names this archie instance. It prefixes container names and
|
||||
// is the value of the daemon label the reaper filters on, so instances
|
||||
// sharing a Docker host never remove each other's containers.
|
||||
@@ -182,9 +196,12 @@ type Config struct {
|
||||
|
||||
// NewPool connects to Docker, optionally pulls the image and removes orphaned
|
||||
// containers. A set cfg.DockerClient is reused and not closed. live, when
|
||||
// non-nil, supplies Image, MaxConcurrency, MaxUptime, PullPolicy and Network
|
||||
// non-nil, supplies Image, MaxConcurrency, MaxUptime and PullPolicy
|
||||
// on every acquire.
|
||||
func NewPool(ctx context.Context, cfg Config, live func() Config, log *slog.Logger) (*Pool, error) {
|
||||
if cfg.Sandbox == nil {
|
||||
return nil, errors.New("container pool needs a sandbox network provisioner")
|
||||
}
|
||||
cli := cfg.DockerClient
|
||||
if cli == nil {
|
||||
var err error
|
||||
@@ -214,6 +231,7 @@ func NewPool(ctx context.Context, cfg Config, live func() Config, log *slog.Logg
|
||||
|
||||
p := &Pool{
|
||||
cli: cli,
|
||||
sandbox: cfg.Sandbox,
|
||||
cfg: cfg,
|
||||
live: live,
|
||||
log: log,
|
||||
@@ -276,19 +294,13 @@ func (p *Pool) Acquire(ctx context.Context, image string, mounts []storage.Mount
|
||||
hostConfig.Mounts = append(hostConfig.Mounts, m)
|
||||
}
|
||||
}
|
||||
// The live network wins when it names one; otherwise the network resolved
|
||||
// at construction (the configured one, or self-detection) stays in force.
|
||||
network := cfg.Network
|
||||
if network == "" {
|
||||
network = p.network
|
||||
}
|
||||
if network != "" {
|
||||
// Join the same user-defined network the daemon itself is on, so
|
||||
// the agent container can resolve sibling compose services (nats,
|
||||
// etc.) by name -- otherwise Docker attaches it to the default
|
||||
// bridge network, where those hostnames don't resolve.
|
||||
hostConfig.NetworkMode = container.NetworkMode(network)
|
||||
// Every task container runs on its own sandbox network, which reaches the
|
||||
// egress relay and nothing else.
|
||||
if err := p.sandbox.Create(ctx, name); err != nil {
|
||||
p.unreserve()
|
||||
return nil, fmt.Errorf("sandbox network: %w", err)
|
||||
}
|
||||
hostConfig.NetworkMode = container.NetworkMode(name)
|
||||
|
||||
resp, err := p.cli.ContainerCreate(ctx, client.ContainerCreateOptions{
|
||||
Name: name,
|
||||
@@ -302,6 +314,7 @@ func (p *Pool) Acquire(ctx context.Context, image string, mounts []storage.Mount
|
||||
HostConfig: hostConfig,
|
||||
})
|
||||
if err != nil {
|
||||
p.dropSandbox(ctx, name)
|
||||
p.unreserve()
|
||||
return nil, fmt.Errorf("container create: %w", err)
|
||||
}
|
||||
@@ -311,10 +324,24 @@ func (p *Pool) Acquire(ctx context.Context, image string, mounts []storage.Mount
|
||||
if _, rmErr := p.cli.ContainerRemove(context.WithoutCancel(ctx), resp.ID, client.ContainerRemoveOptions{Force: true}); rmErr != nil {
|
||||
p.log.WarnContext(ctx, "container remove after start failure", "id", resp.ID[:12], "err", rmErr)
|
||||
}
|
||||
p.dropSandbox(ctx, name)
|
||||
p.unreserve()
|
||||
return nil, fmt.Errorf("container start: %w", err)
|
||||
}
|
||||
return p.started(ctx, cfg.MaxUptime, resp.ID, name), nil
|
||||
c := p.started(ctx, cfg.MaxUptime, resp.ID, name)
|
||||
c.network = name
|
||||
return c, nil
|
||||
}
|
||||
|
||||
// dropSandbox removes a task's sandbox network, after a failed acquire or on
|
||||
// release. Best effort: the boot sweep removes whatever this misses.
|
||||
func (p *Pool) dropSandbox(ctx context.Context, name string) {
|
||||
if name == "" {
|
||||
return
|
||||
}
|
||||
if err := p.sandbox.Remove(context.WithoutCancel(ctx), name); err != nil {
|
||||
p.log.WarnContext(ctx, "sandbox network remove failed", "network", name, "err", err)
|
||||
}
|
||||
}
|
||||
|
||||
func (p *Pool) reserve(maxConcurrency int) error {
|
||||
@@ -579,6 +606,7 @@ func (p *Pool) Release(ctx context.Context, c *Container) {
|
||||
p.mu.Lock()
|
||||
p.active--
|
||||
p.mu.Unlock()
|
||||
p.dropSandbox(ctx, c.network)
|
||||
p.log.InfoContext(ctx, "container released", "id", c.ID[:12])
|
||||
}
|
||||
|
||||
|
||||
@@ -0,0 +1,132 @@
|
||||
package container
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"log/slog"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"slices"
|
||||
"strings"
|
||||
"sync"
|
||||
"testing"
|
||||
|
||||
"github.com/moby/moby/client"
|
||||
)
|
||||
|
||||
// fakeSandbox records the sandbox networks a pool creates and removes.
|
||||
type fakeSandbox struct {
|
||||
mu sync.Mutex
|
||||
created []string
|
||||
removed []string
|
||||
}
|
||||
|
||||
func (f *fakeSandbox) Create(_ context.Context, name string) error {
|
||||
f.mu.Lock()
|
||||
defer f.mu.Unlock()
|
||||
f.created = append(f.created, name)
|
||||
return nil
|
||||
}
|
||||
|
||||
func (f *fakeSandbox) Remove(_ context.Context, name string) error {
|
||||
f.mu.Lock()
|
||||
defer f.mu.Unlock()
|
||||
f.removed = append(f.removed, name)
|
||||
return nil
|
||||
}
|
||||
|
||||
// TestTaskContainerRunsOnItsOwnSandboxNetwork guards the task container's
|
||||
// network boundary and its cleanup: the container joins only its own sandbox
|
||||
// network, never the daemon's, so the egress relay is all it can reach; and
|
||||
// the network goes with the task, so a host never fills with leaked
|
||||
// networks.
|
||||
func TestTaskContainerRunsOnItsOwnSandboxNetwork(t *testing.T) {
|
||||
const daemonNetwork = "daemon-net"
|
||||
tests := []struct {
|
||||
name string
|
||||
failStart bool
|
||||
}{
|
||||
{name: "released task"},
|
||||
{name: "container that fails to start", failStart: true},
|
||||
}
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
var (
|
||||
mu sync.Mutex
|
||||
attached []string
|
||||
)
|
||||
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
switch {
|
||||
case r.Method == http.MethodPost && strings.HasSuffix(r.URL.Path, "/containers/create"):
|
||||
var req struct {
|
||||
HostConfig struct {
|
||||
NetworkMode string `json:"NetworkMode"`
|
||||
} `json:"HostConfig"`
|
||||
NetworkingConfig struct {
|
||||
EndpointsConfig map[string]any `json:"EndpointsConfig"`
|
||||
} `json:"NetworkingConfig"`
|
||||
}
|
||||
if err := json.NewDecoder(r.Body).Decode(&req); err != nil {
|
||||
t.Error(err)
|
||||
}
|
||||
mu.Lock()
|
||||
attached = append(attached, req.HostConfig.NetworkMode)
|
||||
for name := range req.NetworkingConfig.EndpointsConfig {
|
||||
attached = append(attached, name)
|
||||
}
|
||||
mu.Unlock()
|
||||
w.Header().Set("Content-Type", "application/json")
|
||||
_, _ = w.Write([]byte(`{"Id":"0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef"}`))
|
||||
case r.Method == http.MethodPost && strings.HasSuffix(r.URL.Path, "/start"):
|
||||
if tt.failStart {
|
||||
http.Error(w, `{"message":"start failed"}`, http.StatusInternalServerError)
|
||||
return
|
||||
}
|
||||
w.WriteHeader(http.StatusNoContent)
|
||||
case r.Method == http.MethodGet && strings.HasSuffix(r.URL.Path, "/wait"):
|
||||
w.Header().Set("Content-Type", "application/json")
|
||||
_, _ = w.Write([]byte(`{"StatusCode":0}`))
|
||||
case r.Method == http.MethodPost && strings.HasSuffix(r.URL.Path, "/stop"),
|
||||
r.Method == http.MethodDelete:
|
||||
w.WriteHeader(http.StatusNoContent)
|
||||
default:
|
||||
http.NotFound(w, r)
|
||||
}
|
||||
}))
|
||||
defer server.Close()
|
||||
cli, err := client.New(client.WithHost(server.URL), client.WithAPIVersion("1.54"))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer cli.Close()
|
||||
|
||||
sandbox := &fakeSandbox{}
|
||||
p := &Pool{
|
||||
cli: cli,
|
||||
cfg: Config{Instance: "archie", Network: daemonNetwork},
|
||||
sandbox: sandbox,
|
||||
network: daemonNetwork,
|
||||
log: slog.New(slog.DiscardHandler),
|
||||
}
|
||||
ctr, err := p.Acquire(t.Context(), "agent:test", nil, nil)
|
||||
if tt.failStart != (err != nil) {
|
||||
t.Fatalf("Acquire error = %v, want failure %t", err, tt.failStart)
|
||||
}
|
||||
p.Release(t.Context(), ctr)
|
||||
|
||||
mu.Lock()
|
||||
defer mu.Unlock()
|
||||
sandbox.mu.Lock()
|
||||
defer sandbox.mu.Unlock()
|
||||
if len(sandbox.created) != 1 {
|
||||
t.Fatalf("sandbox networks created = %v, want exactly one for the task", sandbox.created)
|
||||
}
|
||||
if want := sandbox.created[:1]; !slices.Equal(attached, want) {
|
||||
t.Fatalf("container attached to %v, want only its sandbox network %v", attached, want)
|
||||
}
|
||||
if !slices.Equal(sandbox.removed, sandbox.created) {
|
||||
t.Fatalf("sandbox networks removed = %v, want %v removed with the task", sandbox.removed, sandbox.created)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
@@ -19,6 +19,7 @@ import (
|
||||
"github.com/samcharles93/archie-core/internal/eventbus"
|
||||
"github.com/samcharles93/archie-core/internal/events"
|
||||
agentnats "github.com/samcharles93/archie-core/internal/infrastructure/agenttransport/nats"
|
||||
"github.com/samcharles93/archie-core/internal/infrastructure/egress"
|
||||
"github.com/samcharles93/archie-core/internal/storage"
|
||||
"github.com/samcharles93/archie-core/internal/taskrun"
|
||||
"github.com/samcharles93/archie-core/internal/taskstate"
|
||||
@@ -95,19 +96,15 @@ func (d *Daemon) acquireTaskContainer(
|
||||
revokeRun := revokeCredential
|
||||
revokeCredential = func() { closeModel(); revokeRun() }
|
||||
|
||||
// The container lives in its own network namespace, so a loopback State
|
||||
// Store target names the container itself. Give this task an address on
|
||||
// the host bridge that forwards to it, closed with the task.
|
||||
stateStoreURL, relay, release, err := d.taskStateStoreEndpoint(ctx, revokeCredential)
|
||||
env, err := d.containerEnv(credential)
|
||||
if err != nil {
|
||||
revokeCredential()
|
||||
d.teardownStorage(ctx, task, repo, workDir)
|
||||
park("state store relay failed", err)
|
||||
park("container environment failed", err)
|
||||
return nil, "", nil, false
|
||||
}
|
||||
revokeCredential = release
|
||||
|
||||
ctr, err := d.ContainerPool.Acquire(ctx, image, append(mounts, model.Mounts...), append(d.containerEnv(task, credential, stateStoreURL), model.Env...))
|
||||
ctr, err := d.ContainerPool.Acquire(ctx, image, append(mounts, model.Mounts...), append(env, model.Env...))
|
||||
if err != nil {
|
||||
revokeCredential()
|
||||
// Roll back the storage we just set up: the mounts were created
|
||||
@@ -118,90 +115,26 @@ func (d *Daemon) acquireTaskContainer(
|
||||
park("container acquire failed", err)
|
||||
return nil, "", nil, false
|
||||
}
|
||||
d.admitToRelay(ctx, relay, task, ctr)
|
||||
return ctr, credential, revokeCredential, true
|
||||
}
|
||||
|
||||
// admitToRelay lets ctr, and only ctr, through relay. A container whose
|
||||
// addresses cannot be read is refused rather than let anyone through.
|
||||
func (d *Daemon) admitToRelay(ctx context.Context, relay *stateStoreRelay, task *workflow.Task, ctr *container.Container) {
|
||||
if relay == nil {
|
||||
return
|
||||
}
|
||||
addrs, err := d.ContainerPool.ContainerAddrs(ctx, ctr.ID)
|
||||
if err != nil {
|
||||
d.Log.WarnContext(ctx, "state store relay: container addresses unknown, refusing its connections", "task", task.ID, "err", err)
|
||||
}
|
||||
relay.Admit(addrs)
|
||||
}
|
||||
|
||||
// taskStateStoreEndpoint is the State Store address one task container is
|
||||
// handed, the relay carrying it (nil when none is needed) and the release that
|
||||
// ends that relay. The container lives
|
||||
// in its own network namespace, so a loopback target names the container
|
||||
// itself; behind a relay, the address is the host bridge's.
|
||||
func (d *Daemon) taskStateStoreEndpoint(ctx context.Context, release func()) (string, *stateStoreRelay, func(), error) {
|
||||
relay, err := d.openTaskStateStoreRelay(ctx)
|
||||
if err != nil {
|
||||
return "", nil, release, err
|
||||
}
|
||||
if relay == nil {
|
||||
return d.ConnectedStateStore.URL, nil, release, nil
|
||||
}
|
||||
return relay.Addr(), relay, func() { relay.Close(); release() }, nil
|
||||
}
|
||||
|
||||
// openTaskStateStoreRelay starts the relay that carries one task container's
|
||||
// State Store traffic to the daemon's target. The target stays bound where the
|
||||
// operator put it; the container is handed an address on the host bridge, the
|
||||
// same way it is handed NATS's. Nil when the target already reaches the
|
||||
// containers, or when no bridge address is known.
|
||||
func (d *Daemon) openTaskStateStoreRelay(ctx context.Context) (*stateStoreRelay, error) {
|
||||
addr, loopback := loopbackStateStoreTarget(d.ConnectedStateStore.URL)
|
||||
if !loopback {
|
||||
return nil, nil
|
||||
}
|
||||
gateway := ""
|
||||
if d.ContainerPool != nil {
|
||||
gateway = d.ContainerPool.HostGateway()
|
||||
}
|
||||
if gateway == "" {
|
||||
d.Log.WarnContext(ctx, "state store target is loopback and the container pool has no host bridge address; task containers cannot reach it",
|
||||
"target", d.ConnectedStateStore.URL)
|
||||
return nil, nil
|
||||
}
|
||||
return openStateStoreRelay(ctx, gateway, addr)
|
||||
}
|
||||
|
||||
// containerEnv is the environment a task container starts with. stateStoreURL
|
||||
// is the address the container can reach the State Store by, which is the
|
||||
// relay's rather than the daemon's own target.
|
||||
func (d *Daemon) containerEnv(task *workflow.Task, stateStoreToken, stateStoreURL string) []string {
|
||||
var env []string
|
||||
// The endpoint the daemon's own client connected with at startup, not
|
||||
// the live config: a reloaded [nats] section must not point new
|
||||
// containers at a server the daemon is not publishing on.
|
||||
env = append(env, "NATS_URL="+d.ConnectedNATS.URL)
|
||||
if token := d.ConnectedNATS.ContainerToken(); token != "" {
|
||||
env = append(env, "NATS_TOKEN="+token)
|
||||
}
|
||||
// Pass the container-reachable endpoint with this task's scoped token,
|
||||
// never the daemon's own.
|
||||
if stateStoreURL != "" {
|
||||
env = append(env, "STATE_STORE_URL="+stateStoreURL)
|
||||
if len(d.ConnectedStateStore.CA) > 0 {
|
||||
env = append(env, "STATE_STORE_CA="+string(d.ConnectedStateStore.CA))
|
||||
}
|
||||
if stateStoreToken != "" {
|
||||
env = append(env, "STATE_STORE_TOKEN="+stateStoreToken)
|
||||
}
|
||||
}
|
||||
// containerEnv is the environment a task container starts with. The container
|
||||
// sits on a sandbox network whose only peer is the egress relay, so NATS and
|
||||
// the State Store are the relay's ports; the credentials are this task's,
|
||||
// never the daemon's own. NATS is the endpoint the daemon connected with at
|
||||
// startup, not live config: a reloaded [nats] section must not point new
|
||||
// containers at a server the daemon is not publishing on.
|
||||
func (d *Daemon) containerEnv(stateStoreToken string) ([]string, error) {
|
||||
// The agent runs as root in the container, so its commits land owned by
|
||||
// UID 0 on the host. The daemon's UID/GID let the agent hand the worktree
|
||||
// back before the daemon reads it to push.
|
||||
env = append(env, fmt.Sprintf("WORKTREE_UID=%d", os.Getuid()))
|
||||
env = append(env, fmt.Sprintf("WORKTREE_GID=%d", os.Getgid()))
|
||||
return env
|
||||
return egress.WorkerEnv(egress.Endpoints{
|
||||
NATS: d.ConnectedNATS.URL,
|
||||
NATSToken: d.ConnectedNATS.ContainerToken(),
|
||||
StateStore: d.ConnectedStateStore.URL,
|
||||
StateStoreToken: stateStoreToken,
|
||||
StateStoreCA: d.ConnectedStateStore.CA,
|
||||
}, os.Getuid(), os.Getgid())
|
||||
}
|
||||
|
||||
// runCredential issues the task's run credential, or "" when no State Store
|
||||
|
||||
@@ -1,167 +0,0 @@
|
||||
package daemon
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"io"
|
||||
"net"
|
||||
"net/netip"
|
||||
"slices"
|
||||
"strings"
|
||||
"sync"
|
||||
"time"
|
||||
)
|
||||
|
||||
// stateStoreDialTimeout bounds one forwarded connection's dial. The target is
|
||||
// local, so a dial that has not connected by then is a dead State Store rather
|
||||
// than a slow one.
|
||||
const stateStoreDialTimeout = 10 * time.Second
|
||||
|
||||
// stateStoreRelay carries one task's State Store traffic from an address on
|
||||
// the host's container bridge to the target the daemon dialed. A container has
|
||||
// its own network namespace, so a loopback target there names the container
|
||||
// itself; the relay address is what the container is handed instead. The State
|
||||
// Store keeps listening exactly where the operator bound it.
|
||||
type stateStoreRelay struct {
|
||||
target string
|
||||
ln net.Listener
|
||||
|
||||
// admitted is closed once allowed holds the task container's addresses;
|
||||
// closed is closed by Close. A connection waits for one of them, so a
|
||||
// container that dials before it is admitted is held rather than refused.
|
||||
admitted chan struct{}
|
||||
closed chan struct{}
|
||||
closeOnce sync.Once
|
||||
allowed []netip.Addr
|
||||
}
|
||||
|
||||
// openStateStoreRelay listens on gateway and forwards to addr. The listener is
|
||||
// per task: it forwards only connections from the addresses Admit names, and
|
||||
// closing it is what ends the task's access.
|
||||
func openStateStoreRelay(ctx context.Context, gateway, addr string) (*stateStoreRelay, error) {
|
||||
ln, err := (&net.ListenConfig{}).Listen(ctx, "tcp", net.JoinHostPort(gateway, "0"))
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("state store relay: listen on %s: %w", gateway, err)
|
||||
}
|
||||
r := &stateStoreRelay{target: addr, ln: ln, admitted: make(chan struct{}), closed: make(chan struct{})}
|
||||
go r.serve(ctx)
|
||||
return r, nil
|
||||
}
|
||||
|
||||
// Admit names the task container's addresses, the only peers the relay
|
||||
// forwards. It is called once, after the container exists.
|
||||
func (r *stateStoreRelay) Admit(addrs []netip.Addr) {
|
||||
r.allowed = addrs
|
||||
close(r.admitted)
|
||||
}
|
||||
|
||||
// admits waits until the relay knows its container, then reports whether
|
||||
// conn comes from it.
|
||||
func (r *stateStoreRelay) admits(ctx context.Context, conn net.Conn) bool {
|
||||
select {
|
||||
case <-r.admitted:
|
||||
case <-r.closed:
|
||||
return false
|
||||
case <-ctx.Done():
|
||||
return false
|
||||
}
|
||||
peer, err := netip.ParseAddrPort(conn.RemoteAddr().String())
|
||||
if err != nil {
|
||||
return false
|
||||
}
|
||||
return slices.Contains(r.allowed, peer.Addr().Unmap())
|
||||
}
|
||||
|
||||
// Addr is the address the task container is handed.
|
||||
func (r *stateStoreRelay) Addr() string { return r.ln.Addr().String() }
|
||||
|
||||
// Close stops accepting. Streams already open end when the container that owns
|
||||
// them goes away, and a restarted daemon leaves no listener behind: the
|
||||
// sockets belong to its process.
|
||||
func (r *stateStoreRelay) Close() {
|
||||
r.closeOnce.Do(func() { close(r.closed) })
|
||||
_ = r.ln.Close()
|
||||
}
|
||||
|
||||
func (r *stateStoreRelay) serve(ctx context.Context) {
|
||||
for {
|
||||
downstream, err := r.ln.Accept()
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
go r.forward(ctx, downstream)
|
||||
}
|
||||
}
|
||||
|
||||
// forward joins one container stream to the target, half-closing each side so
|
||||
// a gRPC stream drains in both directions before the pair ends.
|
||||
func (r *stateStoreRelay) forward(ctx context.Context, downstream net.Conn) {
|
||||
if !r.admits(ctx, downstream) {
|
||||
_ = downstream.Close()
|
||||
return
|
||||
}
|
||||
ctx, cancel := context.WithTimeout(ctx, stateStoreDialTimeout)
|
||||
defer cancel()
|
||||
upstream, err := (&net.Dialer{}).DialContext(ctx, "tcp", r.target)
|
||||
if err != nil {
|
||||
_ = downstream.Close()
|
||||
return
|
||||
}
|
||||
var wg sync.WaitGroup
|
||||
wg.Add(2)
|
||||
go func() { defer wg.Done(); copyAndClose(upstream, downstream) }()
|
||||
go func() { defer wg.Done(); copyAndClose(downstream, upstream) }()
|
||||
wg.Wait()
|
||||
_ = downstream.Close()
|
||||
_ = upstream.Close()
|
||||
}
|
||||
|
||||
func copyAndClose(dst, src net.Conn) {
|
||||
_, _ = io.Copy(dst, src)
|
||||
if half, ok := dst.(interface{ CloseWrite() error }); ok {
|
||||
_ = half.CloseWrite()
|
||||
return
|
||||
}
|
||||
_ = dst.Close()
|
||||
}
|
||||
|
||||
// loopbackStateStoreTarget splits target into the address to forward to, and
|
||||
// reports whether it is this host's loopback: the one case a task container
|
||||
// cannot reach on its own. A target already addressed for the containers'
|
||||
// network needs no relay.
|
||||
func loopbackStateStoreTarget(target string) (addr string, loopback bool) {
|
||||
addr, host, ok := stateStoreTargetAddr(target)
|
||||
if !ok || !loopbackHost(host) {
|
||||
return "", false
|
||||
}
|
||||
return addr, true
|
||||
}
|
||||
|
||||
// StateStoreNeedsRelay reports whether task containers can reach target only
|
||||
// through a relay on the host bridge.
|
||||
func StateStoreNeedsRelay(target string) bool {
|
||||
_, loopback := loopbackStateStoreTarget(target)
|
||||
return loopback
|
||||
}
|
||||
|
||||
// stateStoreTargetAddr is target as host:port, accepting the gRPC scheme form
|
||||
// ("dns:///host:port") the configuration allows.
|
||||
func stateStoreTargetAddr(target string) (addr, host string, ok bool) {
|
||||
target = strings.TrimSpace(target)
|
||||
if _, rest, found := strings.Cut(target, ":///"); found {
|
||||
target = rest
|
||||
}
|
||||
host, _, err := net.SplitHostPort(target)
|
||||
if err != nil {
|
||||
return "", "", false
|
||||
}
|
||||
return target, host, true
|
||||
}
|
||||
|
||||
func loopbackHost(host string) bool {
|
||||
if host == "localhost" {
|
||||
return true
|
||||
}
|
||||
ip := net.ParseIP(host)
|
||||
return ip != nil && ip.IsLoopback()
|
||||
}
|
||||
@@ -1,172 +0,0 @@
|
||||
package daemon
|
||||
|
||||
import (
|
||||
"context"
|
||||
"io"
|
||||
"net"
|
||||
"net/netip"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/samcharles93/archie-core/internal/domain/workflow"
|
||||
)
|
||||
|
||||
// The relay is what makes a loopback State Store reachable from a container:
|
||||
// bytes the task's container writes to its address arrive at the target, and
|
||||
// no other peer on the bridge gets through.
|
||||
func TestStateStoreRelayForwards(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
tests := []struct {
|
||||
name string
|
||||
admit []netip.Addr
|
||||
carried bool
|
||||
}{
|
||||
{name: "the task container", admit: []netip.Addr{netip.MustParseAddr("127.0.0.1")}, carried: true},
|
||||
{name: "another peer", admit: []netip.Addr{netip.MustParseAddr("127.0.0.2")}},
|
||||
}
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
upstream := echoListener(t)
|
||||
relay, err := openStateStoreRelay(context.Background(), "127.0.0.1", upstream.Addr().String())
|
||||
if err != nil {
|
||||
t.Fatalf("open relay: %v", err)
|
||||
}
|
||||
t.Cleanup(relay.Close)
|
||||
if relay.Addr() == upstream.Addr().String() {
|
||||
t.Fatalf("relay listens on the target's own address %s", relay.Addr())
|
||||
}
|
||||
|
||||
conn, err := (&net.Dialer{}).DialContext(context.Background(), "tcp", relay.Addr())
|
||||
if err != nil {
|
||||
t.Fatalf("dial relay: %v", err)
|
||||
}
|
||||
defer func() { _ = conn.Close() }()
|
||||
relay.Admit(tt.admit)
|
||||
_ = conn.SetDeadline(time.Now().Add(5 * time.Second))
|
||||
_, _ = conn.Write([]byte("ping"))
|
||||
got := make([]byte, 4)
|
||||
_, err = io.ReadFull(conn, got)
|
||||
if carried := err == nil && string(got) == "ping"; carried != tt.carried {
|
||||
t.Fatalf("carried = %v (read %q, %v), want %v", carried, got, err, tt.carried)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// The relay is per task, so releasing the task has to end the access it gave.
|
||||
func TestStateStoreRelayClosesWithTask(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
upstream := echoListener(t)
|
||||
relay, err := openStateStoreRelay(context.Background(), "127.0.0.1", upstream.Addr().String())
|
||||
if err != nil {
|
||||
t.Fatalf("open relay: %v", err)
|
||||
}
|
||||
addr := relay.Addr()
|
||||
relay.Close()
|
||||
|
||||
if conn, err := (&net.Dialer{Timeout: time.Second}).DialContext(context.Background(), "tcp", addr); err == nil {
|
||||
_ = conn.Close()
|
||||
t.Fatalf("relay still accepts at %s after the task released it", addr)
|
||||
}
|
||||
}
|
||||
|
||||
// An ordinary image task is handed the relay's address, never the loopback
|
||||
// target the daemon dialed, with the task-scoped credentials unchanged.
|
||||
func TestContainerEnvStateStore(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
upstream := echoListener(t)
|
||||
relay, err := openStateStoreRelay(context.Background(), "127.0.0.1", upstream.Addr().String())
|
||||
if err != nil {
|
||||
t.Fatalf("open relay: %v", err)
|
||||
}
|
||||
t.Cleanup(relay.Close)
|
||||
|
||||
d := wired(&Daemon{
|
||||
ConnectedNATS: NATSEndpoint{URL: "nats://10.0.0.5:4222"},
|
||||
ConnectedStateStore: StateStoreEndpoint{URL: upstream.Addr().String(), CA: []byte("ca-pem")},
|
||||
})
|
||||
env := envValues(d.containerEnv(&workflow.Task{ID: 7}, "task-token", relay.Addr()))
|
||||
|
||||
if got := env["STATE_STORE_URL"]; got != relay.Addr() {
|
||||
t.Fatalf("STATE_STORE_URL = %q, want the relay address %q", got, relay.Addr())
|
||||
}
|
||||
if env["STATE_STORE_TOKEN"] != "task-token" {
|
||||
t.Fatalf("STATE_STORE_TOKEN = %q, want the task's own token", env["STATE_STORE_TOKEN"])
|
||||
}
|
||||
if env["STATE_STORE_CA"] != "ca-pem" {
|
||||
t.Fatalf("STATE_STORE_CA = %q, want the CA passed through", env["STATE_STORE_CA"])
|
||||
}
|
||||
if got := env["NATS_URL"]; got != "nats://10.0.0.5:4222" {
|
||||
t.Fatalf("NATS_URL = %q, want the connected broker", got)
|
||||
}
|
||||
}
|
||||
|
||||
// Only a target the container cannot reach is relayed. An address already
|
||||
// configured for the containers' network is passed through untouched, and a
|
||||
// target the relay cannot forward is left alone rather than guessed at.
|
||||
func TestLoopbackStateStoreTarget(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
cases := []struct {
|
||||
name string
|
||||
target string
|
||||
addr string
|
||||
loopback bool
|
||||
}{
|
||||
{name: "loopback ipv4", target: "127.0.0.1:9090", addr: "127.0.0.1:9090", loopback: true},
|
||||
{name: "loopback ipv6", target: "[::1]:9090", addr: "[::1]:9090", loopback: true},
|
||||
{name: "localhost", target: "localhost:9090", addr: "localhost:9090", loopback: true},
|
||||
{name: "grpc scheme form", target: "dns:///127.0.0.1:9090", addr: "127.0.0.1:9090", loopback: true},
|
||||
{name: "host address", target: "10.0.0.5:9090"},
|
||||
{name: "service name", target: "state-store:9090"},
|
||||
{name: "unset", target: ""},
|
||||
}
|
||||
for _, tc := range cases {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
t.Parallel()
|
||||
addr, loopback := loopbackStateStoreTarget(tc.target)
|
||||
if loopback != tc.loopback {
|
||||
t.Fatalf("loopbackStateStoreTarget(%q) loopback = %t, want %t", tc.target, loopback, tc.loopback)
|
||||
}
|
||||
if addr != tc.addr {
|
||||
t.Fatalf("loopbackStateStoreTarget(%q) addr = %q, want %q", tc.target, addr, tc.addr)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// echoListener accepts connections on loopback and echoes what it reads.
|
||||
func echoListener(t *testing.T) net.Listener {
|
||||
t.Helper()
|
||||
ln, err := (&net.ListenConfig{}).Listen(context.Background(), "tcp", "127.0.0.1:0")
|
||||
if err != nil {
|
||||
t.Fatalf("listen: %v", err)
|
||||
}
|
||||
t.Cleanup(func() { _ = ln.Close() })
|
||||
go func() {
|
||||
for {
|
||||
conn, err := ln.Accept()
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
go func() {
|
||||
defer func() { _ = conn.Close() }()
|
||||
_, _ = io.Copy(conn, conn)
|
||||
}()
|
||||
}
|
||||
}()
|
||||
return ln
|
||||
}
|
||||
|
||||
func envValues(env []string) map[string]string {
|
||||
out := make(map[string]string, len(env))
|
||||
for _, entry := range env {
|
||||
key, value, _ := strings.Cut(entry, "=")
|
||||
out[key] = value
|
||||
}
|
||||
return out
|
||||
}
|
||||
@@ -0,0 +1,56 @@
|
||||
package daemon
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"net"
|
||||
"strings"
|
||||
)
|
||||
|
||||
// StateStoreNeedsRelay reports whether target is bound to this host's
|
||||
// loopback, which the egress relay can reach only through the host bridge's
|
||||
// gateway address.
|
||||
func StateStoreNeedsRelay(target string) bool {
|
||||
_, host, ok := stateStoreTargetAddr(target)
|
||||
return ok && loopbackHost(host)
|
||||
}
|
||||
|
||||
// RelayStateStoreTarget is the host:port the egress relay forwards task State
|
||||
// Store traffic to. The relay has its own network namespace, so a loopback
|
||||
// target would name the relay itself: it is reached at gateway, the host's
|
||||
// address on the relay's bridge, on the same port.
|
||||
func RelayStateStoreTarget(target, gateway string) (string, error) {
|
||||
addr, host, ok := stateStoreTargetAddr(target)
|
||||
if !ok {
|
||||
return "", fmt.Errorf("state store target %q: want host:port", target)
|
||||
}
|
||||
if !loopbackHost(host) {
|
||||
return addr, nil
|
||||
}
|
||||
if gateway == "" {
|
||||
return "", fmt.Errorf("state store target %q is loopback and the container network has no host gateway", target)
|
||||
}
|
||||
_, port, _ := net.SplitHostPort(addr)
|
||||
return net.JoinHostPort(gateway, port), nil
|
||||
}
|
||||
|
||||
// stateStoreTargetAddr is target as host:port, accepting the gRPC scheme form
|
||||
// ("dns:///host:port") the configuration allows.
|
||||
func stateStoreTargetAddr(target string) (addr, host string, ok bool) {
|
||||
target = strings.TrimSpace(target)
|
||||
if _, rest, found := strings.Cut(target, ":///"); found {
|
||||
target = rest
|
||||
}
|
||||
host, _, err := net.SplitHostPort(target)
|
||||
if err != nil {
|
||||
return "", "", false
|
||||
}
|
||||
return target, host, true
|
||||
}
|
||||
|
||||
func loopbackHost(host string) bool {
|
||||
if strings.EqualFold(host, "localhost") {
|
||||
return true
|
||||
}
|
||||
ip := net.ParseIP(host)
|
||||
return ip != nil && ip.IsLoopback()
|
||||
}
|
||||
@@ -0,0 +1,40 @@
|
||||
package daemon
|
||||
|
||||
import "testing"
|
||||
|
||||
// TestRelayStateStoreTarget guards where the egress relay sends task State
|
||||
// Store traffic: a loopback target is reached at the host gateway on the same
|
||||
// port, since loopback inside the relay names the relay itself, and any other
|
||||
// target passes through untouched.
|
||||
func TestRelayStateStoreTarget(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
cases := []struct {
|
||||
name string
|
||||
target string
|
||||
gateway string
|
||||
want string
|
||||
wantErr bool
|
||||
}{
|
||||
{name: "loopback ipv4", target: "127.0.0.1:9090", gateway: "172.17.0.1", want: "172.17.0.1:9090"},
|
||||
{name: "loopback ipv6", target: "[::1]:9090", gateway: "172.17.0.1", want: "172.17.0.1:9090"},
|
||||
{name: "localhost", target: "localhost:9090", gateway: "172.17.0.1", want: "172.17.0.1:9090"},
|
||||
{name: "grpc scheme form", target: "dns:///127.0.0.1:9090", gateway: "172.17.0.1", want: "172.17.0.1:9090"},
|
||||
{name: "host address", target: "10.0.0.5:9090", gateway: "172.17.0.1", want: "10.0.0.5:9090"},
|
||||
{name: "service name", target: "dns:///state-store:9090", want: "state-store:9090"},
|
||||
{name: "loopback without a gateway", target: "127.0.0.1:9090", wantErr: true},
|
||||
{name: "unset", target: "", wantErr: true},
|
||||
}
|
||||
for _, tc := range cases {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
t.Parallel()
|
||||
got, err := RelayStateStoreTarget(tc.target, tc.gateway)
|
||||
if (err != nil) != tc.wantErr {
|
||||
t.Fatalf("RelayStateStoreTarget(%q, %q) error = %v, want error %t", tc.target, tc.gateway, err, tc.wantErr)
|
||||
}
|
||||
if got != tc.want {
|
||||
t.Fatalf("RelayStateStoreTarget(%q, %q) = %q, want %q", tc.target, tc.gateway, got, tc.want)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
@@ -29,12 +29,9 @@ const (
|
||||
)
|
||||
|
||||
// RelaySpec is how the relay container is started. Network is the
|
||||
// host-reachable network it joins to reach the proxy; Entrypoint and Cmd
|
||||
// run archie-agent relay forwarding to the proxy's address on that network.
|
||||
// host-reachable network it joins to reach the proxy, NATS and the State
|
||||
// Store; Entrypoint and Cmd run archie-agent relay forwarding to them.
|
||||
type RelaySpec struct {
|
||||
// Name is the relay container's name. It also scopes the labels Start
|
||||
// prunes by, so two Networks on one host never remove each other's.
|
||||
Name string
|
||||
Image string
|
||||
Entrypoint []string
|
||||
Cmd []string
|
||||
@@ -49,41 +46,50 @@ type RelaySpec struct {
|
||||
// other host service, including ones listening on all interfaces, is out of
|
||||
// reach.
|
||||
type Networks struct {
|
||||
cli *client.Client
|
||||
relay RelaySpec
|
||||
cli *client.Client
|
||||
// relayName is the relay container's name. It also scopes the labels
|
||||
// Sweep prunes by, so two instances on one host never remove each
|
||||
// other's networks.
|
||||
relayName string
|
||||
}
|
||||
|
||||
func NewNetworks(cli *client.Client, relay RelaySpec) *Networks {
|
||||
return &Networks{cli: cli, relay: relay}
|
||||
// NewNetworks is the sandbox provisioner for the relay named relayName. The
|
||||
// relay itself is started by Start, once the addresses it forwards to exist.
|
||||
func NewNetworks(cli *client.Client, relayName string) *Networks {
|
||||
return &Networks{cli: cli, relayName: relayName}
|
||||
}
|
||||
|
||||
// Start removes sandbox networks a previous daemon left behind and starts a
|
||||
// fresh relay. Work interrupted by a restart is requeued, so no live sandbox
|
||||
// depends on the old ones.
|
||||
func (n *Networks) Start(ctx context.Context) error {
|
||||
// Sweep removes the relay and every sandbox network a previous daemon left
|
||||
// behind. Work interrupted by a restart is requeued, so no live sandbox
|
||||
// depends on them. It removes what it can and reports the rest.
|
||||
func (n *Networks) Sweep(ctx context.Context) error {
|
||||
errs := []error{n.removeRelay(ctx)}
|
||||
list, err := n.cli.NetworkList(ctx, client.NetworkListOptions{Filters: client.Filters{}.Add("label", sandboxLabel+"="+n.relayName)})
|
||||
if err != nil {
|
||||
return errors.Join(append(errs, fmt.Errorf("list sandbox networks: %w", err))...)
|
||||
}
|
||||
for _, stale := range list.Items {
|
||||
errs = append(errs, n.Remove(ctx, stale.ID))
|
||||
}
|
||||
return errors.Join(errs...)
|
||||
}
|
||||
|
||||
// Start starts a fresh relay, replacing any left running.
|
||||
func (n *Networks) Start(ctx context.Context, relay RelaySpec) error {
|
||||
if err := n.removeRelay(ctx); err != nil {
|
||||
return err
|
||||
}
|
||||
list, err := n.cli.NetworkList(ctx, client.NetworkListOptions{Filters: client.Filters{}.Add("label", sandboxLabel+"="+n.relay.Name)})
|
||||
if err != nil {
|
||||
return fmt.Errorf("list sandbox networks: %w", err)
|
||||
}
|
||||
for _, stale := range list.Items {
|
||||
if _, err := n.cli.NetworkRemove(ctx, stale.ID, client.NetworkRemoveOptions{}); err != nil {
|
||||
return fmt.Errorf("remove stale sandbox network %s: %w", stale.Name, err)
|
||||
}
|
||||
}
|
||||
created, err := n.cli.ContainerCreate(ctx, client.ContainerCreateOptions{
|
||||
Name: n.relay.Name,
|
||||
Name: n.relayName,
|
||||
Config: &container.Config{
|
||||
Image: n.relay.Image,
|
||||
Entrypoint: n.relay.Entrypoint,
|
||||
Cmd: n.relay.Cmd,
|
||||
Labels: map[string]string{relayLabel: n.relay.Name},
|
||||
Image: relay.Image,
|
||||
Entrypoint: relay.Entrypoint,
|
||||
Cmd: relay.Cmd,
|
||||
Labels: map[string]string{relayLabel: n.relayName},
|
||||
},
|
||||
HostConfig: &container.HostConfig{
|
||||
NetworkMode: container.NetworkMode(n.relay.Network),
|
||||
Binds: n.relay.Binds,
|
||||
NetworkMode: container.NetworkMode(relay.Network),
|
||||
Binds: relay.Binds,
|
||||
RestartPolicy: container.RestartPolicy{Name: container.RestartPolicyUnlessStopped},
|
||||
},
|
||||
})
|
||||
@@ -102,7 +108,7 @@ func (n *Networks) Close(ctx context.Context) error { return n.removeRelay(ctx)
|
||||
func (n *Networks) removeRelay(ctx context.Context) error {
|
||||
list, err := n.cli.ContainerList(ctx, client.ContainerListOptions{
|
||||
All: true,
|
||||
Filters: client.Filters{}.Add("label", relayLabel+"="+n.relay.Name),
|
||||
Filters: client.Filters{}.Add("label", relayLabel+"="+n.relayName),
|
||||
})
|
||||
if err != nil {
|
||||
return fmt.Errorf("list egress relays: %w", err)
|
||||
@@ -124,12 +130,12 @@ func (n *Networks) Create(ctx context.Context, name string) error {
|
||||
// routable from the sandbox; internal alone still exposes every
|
||||
// host service through the gateway.
|
||||
Options: map[string]string{"com.docker.network.bridge.inhibit_ipv4": "true"},
|
||||
Labels: map[string]string{sandboxLabel: n.relay.Name},
|
||||
Labels: map[string]string{sandboxLabel: n.relayName},
|
||||
}); err != nil {
|
||||
return fmt.Errorf("create sandbox network %s: %w", name, err)
|
||||
}
|
||||
if _, err := n.cli.NetworkConnect(ctx, name, client.NetworkConnectOptions{
|
||||
Container: n.relay.Name,
|
||||
Container: n.relayName,
|
||||
EndpointConfig: &network.EndpointSettings{Aliases: []string{RelayAlias}},
|
||||
}); err != nil {
|
||||
return errors.Join(fmt.Errorf("attach egress relay to %s: %w", name, err), n.Remove(context.WithoutCancel(ctx), name))
|
||||
@@ -137,11 +143,23 @@ func (n *Networks) Create(ctx context.Context, name string) error {
|
||||
return nil
|
||||
}
|
||||
|
||||
// Remove detaches the relay from a sandbox network and deletes it.
|
||||
// Remove deletes a sandbox network. Every endpoint still on it is detached
|
||||
// first: the task container may not be gone yet when the max-uptime reaper
|
||||
// and Release race, and an attached endpoint would otherwise block removal
|
||||
// and leak the network's subnet.
|
||||
func (n *Networks) Remove(ctx context.Context, name string) error {
|
||||
_, detachErr := n.cli.NetworkDisconnect(ctx, name, client.NetworkDisconnectOptions{Container: n.relay.Name, Force: true})
|
||||
inspected, err := n.cli.NetworkInspect(ctx, name, client.NetworkInspectOptions{})
|
||||
if err != nil {
|
||||
return fmt.Errorf("inspect sandbox network %s: %w", name, err)
|
||||
}
|
||||
var errs []error
|
||||
for id := range inspected.Network.Containers {
|
||||
if _, err := n.cli.NetworkDisconnect(ctx, name, client.NetworkDisconnectOptions{Container: id, Force: true}); err != nil {
|
||||
errs = append(errs, fmt.Errorf("detach %s from sandbox network %s: %w", id, name, err))
|
||||
}
|
||||
}
|
||||
if _, err := n.cli.NetworkRemove(ctx, name, client.NetworkRemoveOptions{}); err != nil {
|
||||
return errors.Join(detachErr, fmt.Errorf("remove sandbox network %s: %w", name, err))
|
||||
return errors.Join(append(errs, fmt.Errorf("remove sandbox network %s: %w", name, err))...)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
Reference in new issue
Block a user