fix(daemon): relay a loopback State Store with external NATS, and only for the task's container

This commit is contained in:
sam committed 2026-10-22 17:35:24 +11:00
1 parent b395eec893
commit 2ca7c9b2b9
5 files changed
+136 -43

No files matched your search

+5 -4
View File
@@ -427,10 +427,11 @@ func startContainers(
RegistryAuth: resolveRegistryAuth(cfg.Containers.RegistryAuth, secrets, log),
Network: cfg.Containers.Network,
DockerClient: dockerCli,
// Only the embedded broker binds a discovered host gateway. The
// standalone State Store uses its configured, container-reachable target.
RequireHostGateway: cfg.NATS.Mode == config.NATSModeEmbedded,
Instance: cfg.Instance,
// The embedded broker binds the host gateway, and a loopback State
// Store is relayed from it: containers reach neither otherwise.
RequireHostGateway: cfg.NATS.Mode == config.NATSModeEmbedded ||
daemon.StateStoreNeedsRelay(cfg.Services.Get(config.ServiceNameState).Target),
Instance: cfg.Instance,
}, containerLiveSettings(cfgHolder), log)
if err != nil {
// A missing image is recoverable by hand.
+23 -5
View File
@@ -3,6 +3,7 @@ package container
import (
"context"
"fmt"
"net/netip"
"github.com/moby/moby/client"
)
@@ -11,7 +12,7 @@ import (
func (p *Pool) NetworkName() string { return p.network }
// HostGateway is the host-side IPv4 gateway of NetworkName. It is non-empty
// only when Config.RequireHostGateway requested embedded broker reachability.
// only when Config.RequireHostGateway asked for it.
func (p *Pool) HostGateway() string { return p.hostGateway }
func resolveHostGateway(
@@ -25,14 +26,14 @@ func resolveHostGateway(
}
result, err := cli.NetworkInspect(ctx, networkName, client.NetworkInspectOptions{})
if err != nil {
return "", fmt.Errorf("inspect Docker network %q for embedded NATS: %w", networkName, err)
return "", fmt.Errorf("inspect Docker network %q for task containers: %w", networkName, err)
}
network := result.Network.Network
if network.Driver != "bridge" {
return "", fmt.Errorf("docker network %q uses driver %q, want bridge for embedded NATS", networkName, network.Driver)
return "", fmt.Errorf("docker network %q uses driver %q, want bridge for task containers", networkName, network.Driver)
}
if network.Scope != "" && network.Scope != "local" {
return "", fmt.Errorf("docker network %q has scope %q, want local for embedded NATS", networkName, network.Scope)
return "", fmt.Errorf("docker network %q has scope %q, want local for task containers", networkName, network.Scope)
}
for _, ipam := range network.IPAM.Config {
gateway := ipam.Gateway
@@ -40,5 +41,22 @@ func resolveHostGateway(
return gateway.String(), nil
}
}
return "", fmt.Errorf("docker network %q has no usable IPv4 host gateway for embedded NATS", networkName)
return "", fmt.Errorf("docker network %q has no usable IPv4 host gateway for task containers", networkName)
}
// ContainerAddrs is every address container id holds on its networks.
func (p *Pool) ContainerAddrs(ctx context.Context, id string) ([]netip.Addr, error) {
inspected, err := p.cli.ContainerInspect(ctx, id, client.ContainerInspectOptions{})
if err != nil {
return nil, fmt.Errorf("inspect container %s: %w", id, err)
}
var addrs []netip.Addr
if settings := inspected.Container.NetworkSettings; settings != nil {
for _, endpoint := range settings.Networks {
if endpoint != nil && endpoint.IPAddress.IsValid() {
addrs = append(addrs, endpoint.IPAddress)
}
}
}
return addrs, nil
}
+21 -6
View File
@@ -1761,7 +1761,7 @@ func (d *Daemon) acquireTaskContainer(
// The container lives in its own network namespace, so a loopback State
// Store target names the container itself. Give this task an address on
// the host bridge that forwards to it, closed with the task.
stateStoreURL, release, err := d.taskStateStoreEndpoint(ctx, revokeCredential)
stateStoreURL, relay, release, err := d.taskStateStoreEndpoint(ctx, revokeCredential)
if err != nil {
revokeCredential()
d.teardownStorage(ctx, task, repo, workDir)
@@ -1781,6 +1781,7 @@ func (d *Daemon) acquireTaskContainer(
park("container acquire failed", err)
return nil, "", nil, false
}
d.admitToRelay(ctx, relay, task, ctr)
return ctr, credential, revokeCredential, true
}
@@ -2222,18 +2223,32 @@ func (d *Daemon) requestTaskRun(ctx context.Context, taskID int64, data []byte)
}
// taskStateStoreEndpoint is the State Store address one task container is
// handed, and the release that ends the relay carrying it. The container lives
// handed, the relay carrying it (nil when none is needed) and the release that
// ends that relay. The container lives
// in its own network namespace, so a loopback target names the container
// itself; behind a relay, the address is the host bridge's.
func (d *Daemon) taskStateStoreEndpoint(ctx context.Context, release func()) (string, func(), error) {
func (d *Daemon) taskStateStoreEndpoint(ctx context.Context, release func()) (string, *stateStoreRelay, func(), error) {
relay, err := d.openTaskStateStoreRelay(ctx)
if err != nil {
return "", release, err
return "", nil, release, err
}
if relay == nil {
return d.ConnectedStateStore.URL, release, nil
return d.ConnectedStateStore.URL, nil, release, nil
}
return relay.Addr(), func() { relay.Close(); release() }, nil
return relay.Addr(), relay, func() { relay.Close(); release() }, nil
}
// admitToRelay lets ctr, and only ctr, through relay. A container whose
// addresses cannot be read is refused rather than let anyone through.
func (d *Daemon) admitToRelay(ctx context.Context, relay *stateStoreRelay, task *workflow.Task, ctr *container.Container) {
if relay == nil {
return
}
addrs, err := d.ContainerPool.ContainerAddrs(ctx, ctr.ID)
if err != nil {
d.Log.Warn("state store relay: container addresses unknown, refusing its connections", "task", task.ID, "err", err)
}
relay.Admit(addrs)
}
// openTaskStateStoreRelay starts the relay that carries one task container's
+52 -4
View File
@@ -5,6 +5,8 @@ import (
"fmt"
"io"
"net"
"net/netip"
"slices"
"strings"
"sync"
"time"
@@ -23,28 +25,63 @@ const stateStoreDialTimeout = 10 * time.Second
type stateStoreRelay struct {
target string
ln net.Listener
// admitted is closed once allowed holds the task container's addresses;
// closed is closed by Close. A connection waits for one of them, so a
// container that dials before it is admitted is held rather than refused.
admitted chan struct{}
closed chan struct{}
closeOnce sync.Once
allowed []netip.Addr
}
// openStateStoreRelay listens on gateway and forwards to addr. The listener is
// per task: it accepts only from the bridge the container joins, and closing
// it is what ends the task's access.
// per task: it forwards only connections from the addresses Admit names, and
// closing it is what ends the task's access.
func openStateStoreRelay(ctx context.Context, gateway, addr string) (*stateStoreRelay, error) {
ln, err := (&net.ListenConfig{}).Listen(ctx, "tcp", net.JoinHostPort(gateway, "0"))
if err != nil {
return nil, fmt.Errorf("state store relay: listen on %s: %w", gateway, err)
}
r := &stateStoreRelay{target: addr, ln: ln}
r := &stateStoreRelay{target: addr, ln: ln, admitted: make(chan struct{}), closed: make(chan struct{})}
go r.serve(ctx)
return r, nil
}
// Admit names the task container's addresses, the only peers the relay
// forwards. It is called once, after the container exists.
func (r *stateStoreRelay) Admit(addrs []netip.Addr) {
r.allowed = addrs
close(r.admitted)
}
// admits waits until the relay knows its container, then reports whether
// conn comes from it.
func (r *stateStoreRelay) admits(ctx context.Context, conn net.Conn) bool {
select {
case <-r.admitted:
case <-r.closed:
return false
case <-ctx.Done():
return false
}
peer, err := netip.ParseAddrPort(conn.RemoteAddr().String())
if err != nil {
return false
}
return slices.Contains(r.allowed, peer.Addr().Unmap())
}
// Addr is the address the task container is handed.
func (r *stateStoreRelay) Addr() string { return r.ln.Addr().String() }
// Close stops accepting. Streams already open end when the container that owns
// them goes away, and a restarted daemon leaves no listener behind: the
// sockets belong to its process.
func (r *stateStoreRelay) Close() { _ = r.ln.Close() }
func (r *stateStoreRelay) Close() {
r.closeOnce.Do(func() { close(r.closed) })
_ = r.ln.Close()
}
func (r *stateStoreRelay) serve(ctx context.Context) {
for {
@@ -59,6 +96,10 @@ func (r *stateStoreRelay) serve(ctx context.Context) {
// forward joins one container stream to the target, half-closing each side so
// a gRPC stream drains in both directions before the pair ends.
func (r *stateStoreRelay) forward(ctx context.Context, downstream net.Conn) {
if !r.admits(ctx, downstream) {
_ = downstream.Close()
return
}
ctx, cancel := context.WithTimeout(ctx, stateStoreDialTimeout)
defer cancel()
upstream, err := (&net.Dialer{}).DialContext(ctx, "tcp", r.target)
@@ -96,6 +137,13 @@ func loopbackStateStoreTarget(target string) (addr string, loopback bool) {
return addr, true
}
// StateStoreNeedsRelay reports whether task containers can reach target only
// through a relay on the host bridge.
func StateStoreNeedsRelay(target string) bool {
_, loopback := loopbackStateStoreTarget(target)
return loopback
}
// stateStoreTargetAddr is target as host:port, accepting the gRPC scheme form
// ("dns:///host:port") the configuration allows.
func stateStoreTargetAddr(target string) (addr, host string, ok bool) {
+35 -24
View File
@@ -4,6 +4,7 @@ import (
"context"
"io"
"net"
"net/netip"
"strings"
"testing"
"time"
@@ -12,35 +13,45 @@ import (
)
// The relay is what makes a loopback State Store reachable from a container:
// bytes written to its address arrive at the target.
// bytes the task's container writes to its address arrive at the target, and
// no other peer on the bridge gets through.
func TestStateStoreRelayForwards(t *testing.T) {
t.Parallel()
upstream := echoListener(t)
relay, err := openStateStoreRelay(context.Background(), "127.0.0.1", upstream.Addr().String())
if err != nil {
t.Fatalf("open relay: %v", err)
}
t.Cleanup(relay.Close)
if relay.Addr() == upstream.Addr().String() {
t.Fatalf("relay listens on the target's own address %s", relay.Addr())
tests := []struct {
name string
admit []netip.Addr
carried bool
}{
{name: "the task container", admit: []netip.Addr{netip.MustParseAddr("127.0.0.1")}, carried: true},
{name: "another peer", admit: []netip.Addr{netip.MustParseAddr("127.0.0.2")}},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
upstream := echoListener(t)
relay, err := openStateStoreRelay(context.Background(), "127.0.0.1", upstream.Addr().String())
if err != nil {
t.Fatalf("open relay: %v", err)
}
t.Cleanup(relay.Close)
if relay.Addr() == upstream.Addr().String() {
t.Fatalf("relay listens on the target's own address %s", relay.Addr())
}
dialer := &net.Dialer{}
conn, err := dialer.DialContext(context.Background(), "tcp", relay.Addr())
if err != nil {
t.Fatalf("dial relay: %v", err)
}
defer func() { _ = conn.Close() }()
if _, err := conn.Write([]byte("ping")); err != nil {
t.Fatalf("write through relay: %v", err)
}
got := make([]byte, 4)
if _, err := io.ReadFull(conn, got); err != nil {
t.Fatalf("read through relay: %v", err)
}
if string(got) != "ping" {
t.Fatalf("relay carried %q, want %q", got, "ping")
conn, err := (&net.Dialer{}).DialContext(context.Background(), "tcp", relay.Addr())
if err != nil {
t.Fatalf("dial relay: %v", err)
}
defer func() { _ = conn.Close() }()
relay.Admit(tt.admit)
_ = conn.SetDeadline(time.Now().Add(5 * time.Second))
_, _ = conn.Write([]byte("ping"))
got := make([]byte, 4)
_, err = io.ReadFull(conn, got)
if carried := err == nil && string(got) == "ping"; carried != tt.carried {
t.Fatalf("carried = %v (read %q, %v), want %v", carried, got, err, tt.carried)
}
})
}
}