diff --git a/internal/app/archied/main.go b/internal/app/archied/main.go index 9823ef027..bd61981a0 100644 --- a/internal/app/archied/main.go +++ b/internal/app/archied/main.go @@ -427,10 +427,11 @@ func startContainers( RegistryAuth: resolveRegistryAuth(cfg.Containers.RegistryAuth, secrets, log), Network: cfg.Containers.Network, DockerClient: dockerCli, - // Only the embedded broker binds a discovered host gateway. The - // standalone State Store uses its configured, container-reachable target. - RequireHostGateway: cfg.NATS.Mode == config.NATSModeEmbedded, - Instance: cfg.Instance, + // The embedded broker binds the host gateway, and a loopback State + // Store is relayed from it: containers reach neither otherwise. + RequireHostGateway: cfg.NATS.Mode == config.NATSModeEmbedded || + daemon.StateStoreNeedsRelay(cfg.Services.Get(config.ServiceNameState).Target), + Instance: cfg.Instance, }, containerLiveSettings(cfgHolder), log) if err != nil { // A missing image is recoverable by hand. diff --git a/internal/container/network.go b/internal/container/network.go index fd60e60fd..02af46922 100644 --- a/internal/container/network.go +++ b/internal/container/network.go @@ -3,6 +3,7 @@ package container import ( "context" "fmt" + "net/netip" "github.com/moby/moby/client" ) @@ -11,7 +12,7 @@ import ( func (p *Pool) NetworkName() string { return p.network } // HostGateway is the host-side IPv4 gateway of NetworkName. It is non-empty -// only when Config.RequireHostGateway requested embedded broker reachability. +// only when Config.RequireHostGateway asked for it. func (p *Pool) HostGateway() string { return p.hostGateway } func resolveHostGateway( @@ -25,14 +26,14 @@ func resolveHostGateway( } result, err := cli.NetworkInspect(ctx, networkName, client.NetworkInspectOptions{}) if err != nil { - return "", fmt.Errorf("inspect Docker network %q for embedded NATS: %w", networkName, err) + return "", fmt.Errorf("inspect Docker network %q for task containers: %w", networkName, err) } network := result.Network.Network if network.Driver != "bridge" { - return "", fmt.Errorf("docker network %q uses driver %q, want bridge for embedded NATS", networkName, network.Driver) + return "", fmt.Errorf("docker network %q uses driver %q, want bridge for task containers", networkName, network.Driver) } if network.Scope != "" && network.Scope != "local" { - return "", fmt.Errorf("docker network %q has scope %q, want local for embedded NATS", networkName, network.Scope) + return "", fmt.Errorf("docker network %q has scope %q, want local for task containers", networkName, network.Scope) } for _, ipam := range network.IPAM.Config { gateway := ipam.Gateway @@ -40,5 +41,22 @@ func resolveHostGateway( return gateway.String(), nil } } - return "", fmt.Errorf("docker network %q has no usable IPv4 host gateway for embedded NATS", networkName) + return "", fmt.Errorf("docker network %q has no usable IPv4 host gateway for task containers", networkName) +} + +// ContainerAddrs is every address container id holds on its networks. +func (p *Pool) ContainerAddrs(ctx context.Context, id string) ([]netip.Addr, error) { + inspected, err := p.cli.ContainerInspect(ctx, id, client.ContainerInspectOptions{}) + if err != nil { + return nil, fmt.Errorf("inspect container %s: %w", id, err) + } + var addrs []netip.Addr + if settings := inspected.Container.NetworkSettings; settings != nil { + for _, endpoint := range settings.Networks { + if endpoint != nil && endpoint.IPAddress.IsValid() { + addrs = append(addrs, endpoint.IPAddress) + } + } + } + return addrs, nil } diff --git a/internal/daemon/daemon.go b/internal/daemon/daemon.go index c4a6765db..d55d669a5 100644 --- a/internal/daemon/daemon.go +++ b/internal/daemon/daemon.go @@ -1761,7 +1761,7 @@ func (d *Daemon) acquireTaskContainer( // The container lives in its own network namespace, so a loopback State // Store target names the container itself. Give this task an address on // the host bridge that forwards to it, closed with the task. - stateStoreURL, release, err := d.taskStateStoreEndpoint(ctx, revokeCredential) + stateStoreURL, relay, release, err := d.taskStateStoreEndpoint(ctx, revokeCredential) if err != nil { revokeCredential() d.teardownStorage(ctx, task, repo, workDir) @@ -1781,6 +1781,7 @@ func (d *Daemon) acquireTaskContainer( park("container acquire failed", err) return nil, "", nil, false } + d.admitToRelay(ctx, relay, task, ctr) return ctr, credential, revokeCredential, true } @@ -2222,18 +2223,32 @@ func (d *Daemon) requestTaskRun(ctx context.Context, taskID int64, data []byte) } // taskStateStoreEndpoint is the State Store address one task container is -// handed, and the release that ends the relay carrying it. The container lives +// handed, the relay carrying it (nil when none is needed) and the release that +// ends that relay. The container lives // in its own network namespace, so a loopback target names the container // itself; behind a relay, the address is the host bridge's. -func (d *Daemon) taskStateStoreEndpoint(ctx context.Context, release func()) (string, func(), error) { +func (d *Daemon) taskStateStoreEndpoint(ctx context.Context, release func()) (string, *stateStoreRelay, func(), error) { relay, err := d.openTaskStateStoreRelay(ctx) if err != nil { - return "", release, err + return "", nil, release, err } if relay == nil { - return d.ConnectedStateStore.URL, release, nil + return d.ConnectedStateStore.URL, nil, release, nil } - return relay.Addr(), func() { relay.Close(); release() }, nil + return relay.Addr(), relay, func() { relay.Close(); release() }, nil +} + +// admitToRelay lets ctr, and only ctr, through relay. A container whose +// addresses cannot be read is refused rather than let anyone through. +func (d *Daemon) admitToRelay(ctx context.Context, relay *stateStoreRelay, task *workflow.Task, ctr *container.Container) { + if relay == nil { + return + } + addrs, err := d.ContainerPool.ContainerAddrs(ctx, ctr.ID) + if err != nil { + d.Log.Warn("state store relay: container addresses unknown, refusing its connections", "task", task.ID, "err", err) + } + relay.Admit(addrs) } // openTaskStateStoreRelay starts the relay that carries one task container's diff --git a/internal/daemon/statestore_relay.go b/internal/daemon/statestore_relay.go index 567adeaad..747e6ac50 100644 --- a/internal/daemon/statestore_relay.go +++ b/internal/daemon/statestore_relay.go @@ -5,6 +5,8 @@ import ( "fmt" "io" "net" + "net/netip" + "slices" "strings" "sync" "time" @@ -23,28 +25,63 @@ const stateStoreDialTimeout = 10 * time.Second type stateStoreRelay struct { target string ln net.Listener + + // admitted is closed once allowed holds the task container's addresses; + // closed is closed by Close. A connection waits for one of them, so a + // container that dials before it is admitted is held rather than refused. + admitted chan struct{} + closed chan struct{} + closeOnce sync.Once + allowed []netip.Addr } // openStateStoreRelay listens on gateway and forwards to addr. The listener is -// per task: it accepts only from the bridge the container joins, and closing -// it is what ends the task's access. +// per task: it forwards only connections from the addresses Admit names, and +// closing it is what ends the task's access. func openStateStoreRelay(ctx context.Context, gateway, addr string) (*stateStoreRelay, error) { ln, err := (&net.ListenConfig{}).Listen(ctx, "tcp", net.JoinHostPort(gateway, "0")) if err != nil { return nil, fmt.Errorf("state store relay: listen on %s: %w", gateway, err) } - r := &stateStoreRelay{target: addr, ln: ln} + r := &stateStoreRelay{target: addr, ln: ln, admitted: make(chan struct{}), closed: make(chan struct{})} go r.serve(ctx) return r, nil } +// Admit names the task container's addresses, the only peers the relay +// forwards. It is called once, after the container exists. +func (r *stateStoreRelay) Admit(addrs []netip.Addr) { + r.allowed = addrs + close(r.admitted) +} + +// admits waits until the relay knows its container, then reports whether +// conn comes from it. +func (r *stateStoreRelay) admits(ctx context.Context, conn net.Conn) bool { + select { + case <-r.admitted: + case <-r.closed: + return false + case <-ctx.Done(): + return false + } + peer, err := netip.ParseAddrPort(conn.RemoteAddr().String()) + if err != nil { + return false + } + return slices.Contains(r.allowed, peer.Addr().Unmap()) +} + // Addr is the address the task container is handed. func (r *stateStoreRelay) Addr() string { return r.ln.Addr().String() } // Close stops accepting. Streams already open end when the container that owns // them goes away, and a restarted daemon leaves no listener behind: the // sockets belong to its process. -func (r *stateStoreRelay) Close() { _ = r.ln.Close() } +func (r *stateStoreRelay) Close() { + r.closeOnce.Do(func() { close(r.closed) }) + _ = r.ln.Close() +} func (r *stateStoreRelay) serve(ctx context.Context) { for { @@ -59,6 +96,10 @@ func (r *stateStoreRelay) serve(ctx context.Context) { // forward joins one container stream to the target, half-closing each side so // a gRPC stream drains in both directions before the pair ends. func (r *stateStoreRelay) forward(ctx context.Context, downstream net.Conn) { + if !r.admits(ctx, downstream) { + _ = downstream.Close() + return + } ctx, cancel := context.WithTimeout(ctx, stateStoreDialTimeout) defer cancel() upstream, err := (&net.Dialer{}).DialContext(ctx, "tcp", r.target) @@ -96,6 +137,13 @@ func loopbackStateStoreTarget(target string) (addr string, loopback bool) { return addr, true } +// StateStoreNeedsRelay reports whether task containers can reach target only +// through a relay on the host bridge. +func StateStoreNeedsRelay(target string) bool { + _, loopback := loopbackStateStoreTarget(target) + return loopback +} + // stateStoreTargetAddr is target as host:port, accepting the gRPC scheme form // ("dns:///host:port") the configuration allows. func stateStoreTargetAddr(target string) (addr, host string, ok bool) { diff --git a/internal/daemon/statestore_relay_test.go b/internal/daemon/statestore_relay_test.go index 015d1c623..e06548d12 100644 --- a/internal/daemon/statestore_relay_test.go +++ b/internal/daemon/statestore_relay_test.go @@ -4,6 +4,7 @@ import ( "context" "io" "net" + "net/netip" "strings" "testing" "time" @@ -12,35 +13,45 @@ import ( ) // The relay is what makes a loopback State Store reachable from a container: -// bytes written to its address arrive at the target. +// bytes the task's container writes to its address arrive at the target, and +// no other peer on the bridge gets through. func TestStateStoreRelayForwards(t *testing.T) { t.Parallel() - upstream := echoListener(t) - relay, err := openStateStoreRelay(context.Background(), "127.0.0.1", upstream.Addr().String()) - if err != nil { - t.Fatalf("open relay: %v", err) - } - t.Cleanup(relay.Close) - if relay.Addr() == upstream.Addr().String() { - t.Fatalf("relay listens on the target's own address %s", relay.Addr()) + tests := []struct { + name string + admit []netip.Addr + carried bool + }{ + {name: "the task container", admit: []netip.Addr{netip.MustParseAddr("127.0.0.1")}, carried: true}, + {name: "another peer", admit: []netip.Addr{netip.MustParseAddr("127.0.0.2")}}, } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + upstream := echoListener(t) + relay, err := openStateStoreRelay(context.Background(), "127.0.0.1", upstream.Addr().String()) + if err != nil { + t.Fatalf("open relay: %v", err) + } + t.Cleanup(relay.Close) + if relay.Addr() == upstream.Addr().String() { + t.Fatalf("relay listens on the target's own address %s", relay.Addr()) + } - dialer := &net.Dialer{} - conn, err := dialer.DialContext(context.Background(), "tcp", relay.Addr()) - if err != nil { - t.Fatalf("dial relay: %v", err) - } - defer func() { _ = conn.Close() }() - if _, err := conn.Write([]byte("ping")); err != nil { - t.Fatalf("write through relay: %v", err) - } - got := make([]byte, 4) - if _, err := io.ReadFull(conn, got); err != nil { - t.Fatalf("read through relay: %v", err) - } - if string(got) != "ping" { - t.Fatalf("relay carried %q, want %q", got, "ping") + conn, err := (&net.Dialer{}).DialContext(context.Background(), "tcp", relay.Addr()) + if err != nil { + t.Fatalf("dial relay: %v", err) + } + defer func() { _ = conn.Close() }() + relay.Admit(tt.admit) + _ = conn.SetDeadline(time.Now().Add(5 * time.Second)) + _, _ = conn.Write([]byte("ping")) + got := make([]byte, 4) + _, err = io.ReadFull(conn, got) + if carried := err == nil && string(got) == "ping"; carried != tt.carried { + t.Fatalf("carried = %v (read %q, %v), want %v", carried, got, err, tt.carried) + } + }) } }