fix(daemon): relay a loopback State Store with external NATS, and only for the task's container
This commit is contained in:
1 parent
b395eec893
commit
2ca7c9b2b9
5 files changed
+136
-43
No files matched your search
@@ -427,10 +427,11 @@ func startContainers(
|
||||
RegistryAuth: resolveRegistryAuth(cfg.Containers.RegistryAuth, secrets, log),
|
||||
Network: cfg.Containers.Network,
|
||||
DockerClient: dockerCli,
|
||||
// Only the embedded broker binds a discovered host gateway. The
|
||||
// standalone State Store uses its configured, container-reachable target.
|
||||
RequireHostGateway: cfg.NATS.Mode == config.NATSModeEmbedded,
|
||||
Instance: cfg.Instance,
|
||||
// The embedded broker binds the host gateway, and a loopback State
|
||||
// Store is relayed from it: containers reach neither otherwise.
|
||||
RequireHostGateway: cfg.NATS.Mode == config.NATSModeEmbedded ||
|
||||
daemon.StateStoreNeedsRelay(cfg.Services.Get(config.ServiceNameState).Target),
|
||||
Instance: cfg.Instance,
|
||||
}, containerLiveSettings(cfgHolder), log)
|
||||
if err != nil {
|
||||
// A missing image is recoverable by hand.
|
||||
|
||||
@@ -3,6 +3,7 @@ package container
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"net/netip"
|
||||
|
||||
"github.com/moby/moby/client"
|
||||
)
|
||||
@@ -11,7 +12,7 @@ import (
|
||||
func (p *Pool) NetworkName() string { return p.network }
|
||||
|
||||
// HostGateway is the host-side IPv4 gateway of NetworkName. It is non-empty
|
||||
// only when Config.RequireHostGateway requested embedded broker reachability.
|
||||
// only when Config.RequireHostGateway asked for it.
|
||||
func (p *Pool) HostGateway() string { return p.hostGateway }
|
||||
|
||||
func resolveHostGateway(
|
||||
@@ -25,14 +26,14 @@ func resolveHostGateway(
|
||||
}
|
||||
result, err := cli.NetworkInspect(ctx, networkName, client.NetworkInspectOptions{})
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("inspect Docker network %q for embedded NATS: %w", networkName, err)
|
||||
return "", fmt.Errorf("inspect Docker network %q for task containers: %w", networkName, err)
|
||||
}
|
||||
network := result.Network.Network
|
||||
if network.Driver != "bridge" {
|
||||
return "", fmt.Errorf("docker network %q uses driver %q, want bridge for embedded NATS", networkName, network.Driver)
|
||||
return "", fmt.Errorf("docker network %q uses driver %q, want bridge for task containers", networkName, network.Driver)
|
||||
}
|
||||
if network.Scope != "" && network.Scope != "local" {
|
||||
return "", fmt.Errorf("docker network %q has scope %q, want local for embedded NATS", networkName, network.Scope)
|
||||
return "", fmt.Errorf("docker network %q has scope %q, want local for task containers", networkName, network.Scope)
|
||||
}
|
||||
for _, ipam := range network.IPAM.Config {
|
||||
gateway := ipam.Gateway
|
||||
@@ -40,5 +41,22 @@ func resolveHostGateway(
|
||||
return gateway.String(), nil
|
||||
}
|
||||
}
|
||||
return "", fmt.Errorf("docker network %q has no usable IPv4 host gateway for embedded NATS", networkName)
|
||||
return "", fmt.Errorf("docker network %q has no usable IPv4 host gateway for task containers", networkName)
|
||||
}
|
||||
|
||||
// ContainerAddrs is every address container id holds on its networks.
|
||||
func (p *Pool) ContainerAddrs(ctx context.Context, id string) ([]netip.Addr, error) {
|
||||
inspected, err := p.cli.ContainerInspect(ctx, id, client.ContainerInspectOptions{})
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("inspect container %s: %w", id, err)
|
||||
}
|
||||
var addrs []netip.Addr
|
||||
if settings := inspected.Container.NetworkSettings; settings != nil {
|
||||
for _, endpoint := range settings.Networks {
|
||||
if endpoint != nil && endpoint.IPAddress.IsValid() {
|
||||
addrs = append(addrs, endpoint.IPAddress)
|
||||
}
|
||||
}
|
||||
}
|
||||
return addrs, nil
|
||||
}
|
||||
@@ -1761,7 +1761,7 @@ func (d *Daemon) acquireTaskContainer(
|
||||
// The container lives in its own network namespace, so a loopback State
|
||||
// Store target names the container itself. Give this task an address on
|
||||
// the host bridge that forwards to it, closed with the task.
|
||||
stateStoreURL, release, err := d.taskStateStoreEndpoint(ctx, revokeCredential)
|
||||
stateStoreURL, relay, release, err := d.taskStateStoreEndpoint(ctx, revokeCredential)
|
||||
if err != nil {
|
||||
revokeCredential()
|
||||
d.teardownStorage(ctx, task, repo, workDir)
|
||||
@@ -1781,6 +1781,7 @@ func (d *Daemon) acquireTaskContainer(
|
||||
park("container acquire failed", err)
|
||||
return nil, "", nil, false
|
||||
}
|
||||
d.admitToRelay(ctx, relay, task, ctr)
|
||||
return ctr, credential, revokeCredential, true
|
||||
}
|
||||
|
||||
@@ -2222,18 +2223,32 @@ func (d *Daemon) requestTaskRun(ctx context.Context, taskID int64, data []byte)
|
||||
}
|
||||
|
||||
// taskStateStoreEndpoint is the State Store address one task container is
|
||||
// handed, and the release that ends the relay carrying it. The container lives
|
||||
// handed, the relay carrying it (nil when none is needed) and the release that
|
||||
// ends that relay. The container lives
|
||||
// in its own network namespace, so a loopback target names the container
|
||||
// itself; behind a relay, the address is the host bridge's.
|
||||
func (d *Daemon) taskStateStoreEndpoint(ctx context.Context, release func()) (string, func(), error) {
|
||||
func (d *Daemon) taskStateStoreEndpoint(ctx context.Context, release func()) (string, *stateStoreRelay, func(), error) {
|
||||
relay, err := d.openTaskStateStoreRelay(ctx)
|
||||
if err != nil {
|
||||
return "", release, err
|
||||
return "", nil, release, err
|
||||
}
|
||||
if relay == nil {
|
||||
return d.ConnectedStateStore.URL, release, nil
|
||||
return d.ConnectedStateStore.URL, nil, release, nil
|
||||
}
|
||||
return relay.Addr(), func() { relay.Close(); release() }, nil
|
||||
return relay.Addr(), relay, func() { relay.Close(); release() }, nil
|
||||
}
|
||||
|
||||
// admitToRelay lets ctr, and only ctr, through relay. A container whose
|
||||
// addresses cannot be read is refused rather than let anyone through.
|
||||
func (d *Daemon) admitToRelay(ctx context.Context, relay *stateStoreRelay, task *workflow.Task, ctr *container.Container) {
|
||||
if relay == nil {
|
||||
return
|
||||
}
|
||||
addrs, err := d.ContainerPool.ContainerAddrs(ctx, ctr.ID)
|
||||
if err != nil {
|
||||
d.Log.Warn("state store relay: container addresses unknown, refusing its connections", "task", task.ID, "err", err)
|
||||
}
|
||||
relay.Admit(addrs)
|
||||
}
|
||||
|
||||
// openTaskStateStoreRelay starts the relay that carries one task container's
|
||||
|
||||
@@ -5,6 +5,8 @@ import (
|
||||
"fmt"
|
||||
"io"
|
||||
"net"
|
||||
"net/netip"
|
||||
"slices"
|
||||
"strings"
|
||||
"sync"
|
||||
"time"
|
||||
@@ -23,28 +25,63 @@ const stateStoreDialTimeout = 10 * time.Second
|
||||
type stateStoreRelay struct {
|
||||
target string
|
||||
ln net.Listener
|
||||
|
||||
// admitted is closed once allowed holds the task container's addresses;
|
||||
// closed is closed by Close. A connection waits for one of them, so a
|
||||
// container that dials before it is admitted is held rather than refused.
|
||||
admitted chan struct{}
|
||||
closed chan struct{}
|
||||
closeOnce sync.Once
|
||||
allowed []netip.Addr
|
||||
}
|
||||
|
||||
// openStateStoreRelay listens on gateway and forwards to addr. The listener is
|
||||
// per task: it accepts only from the bridge the container joins, and closing
|
||||
// it is what ends the task's access.
|
||||
// per task: it forwards only connections from the addresses Admit names, and
|
||||
// closing it is what ends the task's access.
|
||||
func openStateStoreRelay(ctx context.Context, gateway, addr string) (*stateStoreRelay, error) {
|
||||
ln, err := (&net.ListenConfig{}).Listen(ctx, "tcp", net.JoinHostPort(gateway, "0"))
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("state store relay: listen on %s: %w", gateway, err)
|
||||
}
|
||||
r := &stateStoreRelay{target: addr, ln: ln}
|
||||
r := &stateStoreRelay{target: addr, ln: ln, admitted: make(chan struct{}), closed: make(chan struct{})}
|
||||
go r.serve(ctx)
|
||||
return r, nil
|
||||
}
|
||||
|
||||
// Admit names the task container's addresses, the only peers the relay
|
||||
// forwards. It is called once, after the container exists.
|
||||
func (r *stateStoreRelay) Admit(addrs []netip.Addr) {
|
||||
r.allowed = addrs
|
||||
close(r.admitted)
|
||||
}
|
||||
|
||||
// admits waits until the relay knows its container, then reports whether
|
||||
// conn comes from it.
|
||||
func (r *stateStoreRelay) admits(ctx context.Context, conn net.Conn) bool {
|
||||
select {
|
||||
case <-r.admitted:
|
||||
case <-r.closed:
|
||||
return false
|
||||
case <-ctx.Done():
|
||||
return false
|
||||
}
|
||||
peer, err := netip.ParseAddrPort(conn.RemoteAddr().String())
|
||||
if err != nil {
|
||||
return false
|
||||
}
|
||||
return slices.Contains(r.allowed, peer.Addr().Unmap())
|
||||
}
|
||||
|
||||
// Addr is the address the task container is handed.
|
||||
func (r *stateStoreRelay) Addr() string { return r.ln.Addr().String() }
|
||||
|
||||
// Close stops accepting. Streams already open end when the container that owns
|
||||
// them goes away, and a restarted daemon leaves no listener behind: the
|
||||
// sockets belong to its process.
|
||||
func (r *stateStoreRelay) Close() { _ = r.ln.Close() }
|
||||
func (r *stateStoreRelay) Close() {
|
||||
r.closeOnce.Do(func() { close(r.closed) })
|
||||
_ = r.ln.Close()
|
||||
}
|
||||
|
||||
func (r *stateStoreRelay) serve(ctx context.Context) {
|
||||
for {
|
||||
@@ -59,6 +96,10 @@ func (r *stateStoreRelay) serve(ctx context.Context) {
|
||||
// forward joins one container stream to the target, half-closing each side so
|
||||
// a gRPC stream drains in both directions before the pair ends.
|
||||
func (r *stateStoreRelay) forward(ctx context.Context, downstream net.Conn) {
|
||||
if !r.admits(ctx, downstream) {
|
||||
_ = downstream.Close()
|
||||
return
|
||||
}
|
||||
ctx, cancel := context.WithTimeout(ctx, stateStoreDialTimeout)
|
||||
defer cancel()
|
||||
upstream, err := (&net.Dialer{}).DialContext(ctx, "tcp", r.target)
|
||||
@@ -96,6 +137,13 @@ func loopbackStateStoreTarget(target string) (addr string, loopback bool) {
|
||||
return addr, true
|
||||
}
|
||||
|
||||
// StateStoreNeedsRelay reports whether task containers can reach target only
|
||||
// through a relay on the host bridge.
|
||||
func StateStoreNeedsRelay(target string) bool {
|
||||
_, loopback := loopbackStateStoreTarget(target)
|
||||
return loopback
|
||||
}
|
||||
|
||||
// stateStoreTargetAddr is target as host:port, accepting the gRPC scheme form
|
||||
// ("dns:///host:port") the configuration allows.
|
||||
func stateStoreTargetAddr(target string) (addr, host string, ok bool) {
|
||||
|
||||
@@ -4,6 +4,7 @@ import (
|
||||
"context"
|
||||
"io"
|
||||
"net"
|
||||
"net/netip"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
@@ -12,35 +13,45 @@ import (
|
||||
)
|
||||
|
||||
// The relay is what makes a loopback State Store reachable from a container:
|
||||
// bytes written to its address arrive at the target.
|
||||
// bytes the task's container writes to its address arrive at the target, and
|
||||
// no other peer on the bridge gets through.
|
||||
func TestStateStoreRelayForwards(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
upstream := echoListener(t)
|
||||
relay, err := openStateStoreRelay(context.Background(), "127.0.0.1", upstream.Addr().String())
|
||||
if err != nil {
|
||||
t.Fatalf("open relay: %v", err)
|
||||
}
|
||||
t.Cleanup(relay.Close)
|
||||
if relay.Addr() == upstream.Addr().String() {
|
||||
t.Fatalf("relay listens on the target's own address %s", relay.Addr())
|
||||
tests := []struct {
|
||||
name string
|
||||
admit []netip.Addr
|
||||
carried bool
|
||||
}{
|
||||
{name: "the task container", admit: []netip.Addr{netip.MustParseAddr("127.0.0.1")}, carried: true},
|
||||
{name: "another peer", admit: []netip.Addr{netip.MustParseAddr("127.0.0.2")}},
|
||||
}
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
upstream := echoListener(t)
|
||||
relay, err := openStateStoreRelay(context.Background(), "127.0.0.1", upstream.Addr().String())
|
||||
if err != nil {
|
||||
t.Fatalf("open relay: %v", err)
|
||||
}
|
||||
t.Cleanup(relay.Close)
|
||||
if relay.Addr() == upstream.Addr().String() {
|
||||
t.Fatalf("relay listens on the target's own address %s", relay.Addr())
|
||||
}
|
||||
|
||||
dialer := &net.Dialer{}
|
||||
conn, err := dialer.DialContext(context.Background(), "tcp", relay.Addr())
|
||||
if err != nil {
|
||||
t.Fatalf("dial relay: %v", err)
|
||||
}
|
||||
defer func() { _ = conn.Close() }()
|
||||
if _, err := conn.Write([]byte("ping")); err != nil {
|
||||
t.Fatalf("write through relay: %v", err)
|
||||
}
|
||||
got := make([]byte, 4)
|
||||
if _, err := io.ReadFull(conn, got); err != nil {
|
||||
t.Fatalf("read through relay: %v", err)
|
||||
}
|
||||
if string(got) != "ping" {
|
||||
t.Fatalf("relay carried %q, want %q", got, "ping")
|
||||
conn, err := (&net.Dialer{}).DialContext(context.Background(), "tcp", relay.Addr())
|
||||
if err != nil {
|
||||
t.Fatalf("dial relay: %v", err)
|
||||
}
|
||||
defer func() { _ = conn.Close() }()
|
||||
relay.Admit(tt.admit)
|
||||
_ = conn.SetDeadline(time.Now().Add(5 * time.Second))
|
||||
_, _ = conn.Write([]byte("ping"))
|
||||
got := make([]byte, 4)
|
||||
_, err = io.ReadFull(conn, got)
|
||||
if carried := err == nil && string(got) == "ping"; carried != tt.carried {
|
||||
t.Fatalf("carried = %v (read %q, %v), want %v", carried, got, err, tt.carried)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
Reference in new issue
Block a user