fix(daemon): give a task container a reachable State Store endpoint
This commit is contained in:
1 parent
89cbbfc871
commit
92bc0f692f
4 files changed
+381
-6
No files matched your search
@@ -1758,7 +1758,19 @@ func (d *Daemon) acquireTaskContainer(
|
||||
revokeRun := revokeCredential
|
||||
revokeCredential = func() { closeModel(); revokeRun() }
|
||||
|
||||
ctr, err := d.ContainerPool.Acquire(ctx, image, append(mounts, model.Mounts...), append(d.containerEnv(task, credential), model.Env...))
|
||||
// The container lives in its own network namespace, so a loopback State
|
||||
// Store target names the container itself. Give this task an address on
|
||||
// the host bridge that forwards to it, closed with the task.
|
||||
stateStoreURL, release, err := d.taskStateStoreEndpoint(ctx, revokeCredential)
|
||||
if err != nil {
|
||||
revokeCredential()
|
||||
d.teardownStorage(ctx, task, repo, workDir)
|
||||
park("state store relay failed", err)
|
||||
return nil, "", nil, false
|
||||
}
|
||||
revokeCredential = release
|
||||
|
||||
ctr, err := d.ContainerPool.Acquire(ctx, image, append(mounts, model.Mounts...), append(d.containerEnv(task, credential, stateStoreURL), model.Env...))
|
||||
if err != nil {
|
||||
revokeCredential()
|
||||
// Roll back the storage we just set up: the mounts were created
|
||||
@@ -2190,7 +2202,47 @@ func (d *Daemon) requestTaskRun(ctx context.Context, taskID int64, data []byte)
|
||||
}
|
||||
}
|
||||
|
||||
func (d *Daemon) containerEnv(task *workflow.Task, stateStoreToken string) []string {
|
||||
// taskStateStoreEndpoint is the State Store address one task container is
|
||||
// handed, and the release that ends the relay carrying it. The container lives
|
||||
// in its own network namespace, so a loopback target names the container
|
||||
// itself; behind a relay, the address is the host bridge's.
|
||||
func (d *Daemon) taskStateStoreEndpoint(ctx context.Context, release func()) (string, func(), error) {
|
||||
relay, err := d.openTaskStateStoreRelay(ctx)
|
||||
if err != nil {
|
||||
return "", release, err
|
||||
}
|
||||
if relay == nil {
|
||||
return d.ConnectedStateStore.URL, release, nil
|
||||
}
|
||||
return relay.Addr(), func() { relay.Close(); release() }, nil
|
||||
}
|
||||
|
||||
// openTaskStateStoreRelay starts the relay that carries one task container's
|
||||
// State Store traffic to the daemon's target. The target stays bound where the
|
||||
// operator put it; the container is handed an address on the host bridge, the
|
||||
// same way it is handed NATS's. Nil when the target already reaches the
|
||||
// containers, or when no bridge address is known.
|
||||
func (d *Daemon) openTaskStateStoreRelay(ctx context.Context) (*stateStoreRelay, error) {
|
||||
addr, loopback := loopbackStateStoreTarget(d.ConnectedStateStore.URL)
|
||||
if !loopback {
|
||||
return nil, nil
|
||||
}
|
||||
gateway := ""
|
||||
if d.ContainerPool != nil {
|
||||
gateway = d.ContainerPool.HostGateway()
|
||||
}
|
||||
if gateway == "" {
|
||||
d.Log.Warn("state store target is loopback and the container pool has no host bridge address; task containers cannot reach it",
|
||||
"target", d.ConnectedStateStore.URL)
|
||||
return nil, nil
|
||||
}
|
||||
return openStateStoreRelay(ctx, gateway, addr)
|
||||
}
|
||||
|
||||
// containerEnv is the environment a task container starts with. stateStoreURL
|
||||
// is the address the container can reach the State Store by, which is the
|
||||
// relay's rather than the daemon's own target.
|
||||
func (d *Daemon) containerEnv(task *workflow.Task, stateStoreToken, stateStoreURL string) []string {
|
||||
var env []string
|
||||
// The endpoint the daemon's own client connected with at startup, not
|
||||
// the live config: a reloaded [nats] section must not point new
|
||||
@@ -2199,10 +2251,10 @@ func (d *Daemon) containerEnv(task *workflow.Task, stateStoreToken string) []str
|
||||
if token := d.ConnectedNATS.ContainerToken(); token != "" {
|
||||
env = append(env, "NATS_TOKEN="+token)
|
||||
}
|
||||
// Pass the State Store target with this task's scoped token, never the
|
||||
// daemon's own.
|
||||
if d.ConnectedStateStore.URL != "" {
|
||||
env = append(env, "STATE_STORE_URL="+d.ConnectedStateStore.URL)
|
||||
// Pass the container-reachable endpoint with this task's scoped token,
|
||||
// never the daemon's own.
|
||||
if stateStoreURL != "" {
|
||||
env = append(env, "STATE_STORE_URL="+stateStoreURL)
|
||||
if len(d.ConnectedStateStore.CA) > 0 {
|
||||
env = append(env, "STATE_STORE_CA="+string(d.ConnectedStateStore.CA))
|
||||
}
|
||||
|
||||
@@ -0,0 +1,119 @@
|
||||
package daemon
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"io"
|
||||
"net"
|
||||
"strings"
|
||||
"sync"
|
||||
"time"
|
||||
)
|
||||
|
||||
// stateStoreDialTimeout bounds one forwarded connection's dial. The target is
|
||||
// local, so a dial that has not connected by then is a dead State Store rather
|
||||
// than a slow one.
|
||||
const stateStoreDialTimeout = 10 * time.Second
|
||||
|
||||
// stateStoreRelay carries one task's State Store traffic from an address on
|
||||
// the host's container bridge to the target the daemon dialed. A container has
|
||||
// its own network namespace, so a loopback target there names the container
|
||||
// itself; the relay address is what the container is handed instead. The State
|
||||
// Store keeps listening exactly where the operator bound it.
|
||||
type stateStoreRelay struct {
|
||||
target string
|
||||
ln net.Listener
|
||||
}
|
||||
|
||||
// openStateStoreRelay listens on gateway and forwards to addr. The listener is
|
||||
// per task: it accepts only from the bridge the container joins, and closing
|
||||
// it is what ends the task's access.
|
||||
func openStateStoreRelay(ctx context.Context, gateway, addr string) (*stateStoreRelay, error) {
|
||||
ln, err := (&net.ListenConfig{}).Listen(ctx, "tcp", net.JoinHostPort(gateway, "0"))
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("state store relay: listen on %s: %w", gateway, err)
|
||||
}
|
||||
r := &stateStoreRelay{target: addr, ln: ln}
|
||||
go r.serve(ctx)
|
||||
return r, nil
|
||||
}
|
||||
|
||||
// Addr is the address the task container is handed.
|
||||
func (r *stateStoreRelay) Addr() string { return r.ln.Addr().String() }
|
||||
|
||||
// Close stops accepting. Streams already open end when the container that owns
|
||||
// them goes away, and a restarted daemon leaves no listener behind: the
|
||||
// sockets belong to its process.
|
||||
func (r *stateStoreRelay) Close() { _ = r.ln.Close() }
|
||||
|
||||
func (r *stateStoreRelay) serve(ctx context.Context) {
|
||||
for {
|
||||
downstream, err := r.ln.Accept()
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
go r.forward(ctx, downstream)
|
||||
}
|
||||
}
|
||||
|
||||
// forward joins one container stream to the target, half-closing each side so
|
||||
// a gRPC stream drains in both directions before the pair ends.
|
||||
func (r *stateStoreRelay) forward(ctx context.Context, downstream net.Conn) {
|
||||
ctx, cancel := context.WithTimeout(ctx, stateStoreDialTimeout)
|
||||
defer cancel()
|
||||
upstream, err := (&net.Dialer{}).DialContext(ctx, "tcp", r.target)
|
||||
if err != nil {
|
||||
_ = downstream.Close()
|
||||
return
|
||||
}
|
||||
var wg sync.WaitGroup
|
||||
wg.Add(2)
|
||||
go func() { defer wg.Done(); copyAndClose(upstream, downstream) }()
|
||||
go func() { defer wg.Done(); copyAndClose(downstream, upstream) }()
|
||||
wg.Wait()
|
||||
_ = downstream.Close()
|
||||
_ = upstream.Close()
|
||||
}
|
||||
|
||||
func copyAndClose(dst, src net.Conn) {
|
||||
_, _ = io.Copy(dst, src)
|
||||
if half, ok := dst.(interface{ CloseWrite() error }); ok {
|
||||
_ = half.CloseWrite()
|
||||
return
|
||||
}
|
||||
_ = dst.Close()
|
||||
}
|
||||
|
||||
// loopbackStateStoreTarget splits target into the address to forward to, and
|
||||
// reports whether it is this host's loopback: the one case a task container
|
||||
// cannot reach on its own. A target already addressed for the containers'
|
||||
// network needs no relay.
|
||||
func loopbackStateStoreTarget(target string) (addr string, loopback bool) {
|
||||
addr, host, ok := stateStoreTargetAddr(target)
|
||||
if !ok || !loopbackHost(host) {
|
||||
return "", false
|
||||
}
|
||||
return addr, true
|
||||
}
|
||||
|
||||
// stateStoreTargetAddr is target as host:port, accepting the gRPC scheme form
|
||||
// ("dns:///host:port") the configuration allows.
|
||||
func stateStoreTargetAddr(target string) (addr, host string, ok bool) {
|
||||
target = strings.TrimSpace(target)
|
||||
if _, rest, found := strings.Cut(target, ":///"); found {
|
||||
target = rest
|
||||
}
|
||||
host, _, err := net.SplitHostPort(target)
|
||||
if err != nil {
|
||||
return "", "", false
|
||||
}
|
||||
return target, host, true
|
||||
}
|
||||
|
||||
func loopbackHost(host string) bool {
|
||||
if host == "localhost" {
|
||||
return true
|
||||
}
|
||||
ip := net.ParseIP(host)
|
||||
return ip != nil && ip.IsLoopback()
|
||||
}
|
||||
@@ -0,0 +1,161 @@
|
||||
package daemon
|
||||
|
||||
import (
|
||||
"context"
|
||||
"io"
|
||||
"net"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/samcharles93/archie-core/internal/domain/workflow"
|
||||
)
|
||||
|
||||
// The relay is what makes a loopback State Store reachable from a container:
|
||||
// bytes written to its address arrive at the target.
|
||||
func TestStateStoreRelayForwards(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
upstream := echoListener(t)
|
||||
relay, err := openStateStoreRelay(context.Background(), "127.0.0.1", upstream.Addr().String())
|
||||
if err != nil {
|
||||
t.Fatalf("open relay: %v", err)
|
||||
}
|
||||
t.Cleanup(relay.Close)
|
||||
if relay.Addr() == upstream.Addr().String() {
|
||||
t.Fatalf("relay listens on the target's own address %s", relay.Addr())
|
||||
}
|
||||
|
||||
dialer := &net.Dialer{}
|
||||
conn, err := dialer.DialContext(context.Background(), "tcp", relay.Addr())
|
||||
if err != nil {
|
||||
t.Fatalf("dial relay: %v", err)
|
||||
}
|
||||
defer func() { _ = conn.Close() }()
|
||||
if _, err := conn.Write([]byte("ping")); err != nil {
|
||||
t.Fatalf("write through relay: %v", err)
|
||||
}
|
||||
got := make([]byte, 4)
|
||||
if _, err := io.ReadFull(conn, got); err != nil {
|
||||
t.Fatalf("read through relay: %v", err)
|
||||
}
|
||||
if string(got) != "ping" {
|
||||
t.Fatalf("relay carried %q, want %q", got, "ping")
|
||||
}
|
||||
}
|
||||
|
||||
// The relay is per task, so releasing the task has to end the access it gave.
|
||||
func TestStateStoreRelayClosesWithTask(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
upstream := echoListener(t)
|
||||
relay, err := openStateStoreRelay(context.Background(), "127.0.0.1", upstream.Addr().String())
|
||||
if err != nil {
|
||||
t.Fatalf("open relay: %v", err)
|
||||
}
|
||||
addr := relay.Addr()
|
||||
relay.Close()
|
||||
|
||||
if conn, err := (&net.Dialer{Timeout: time.Second}).DialContext(context.Background(), "tcp", addr); err == nil {
|
||||
_ = conn.Close()
|
||||
t.Fatalf("relay still accepts at %s after the task released it", addr)
|
||||
}
|
||||
}
|
||||
|
||||
// An ordinary image task is handed the relay's address, never the loopback
|
||||
// target the daemon dialed, with the task-scoped credentials unchanged.
|
||||
func TestContainerEnvStateStore(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
upstream := echoListener(t)
|
||||
relay, err := openStateStoreRelay(context.Background(), "127.0.0.1", upstream.Addr().String())
|
||||
if err != nil {
|
||||
t.Fatalf("open relay: %v", err)
|
||||
}
|
||||
t.Cleanup(relay.Close)
|
||||
|
||||
d := &Daemon{
|
||||
ConnectedNATS: NATSEndpoint{URL: "nats://10.0.0.5:4222"},
|
||||
ConnectedStateStore: StateStoreEndpoint{URL: upstream.Addr().String(), CA: []byte("ca-pem")},
|
||||
}
|
||||
env := envValues(d.containerEnv(&workflow.Task{ID: 7}, "task-token", relay.Addr()))
|
||||
|
||||
if got := env["STATE_STORE_URL"]; got != relay.Addr() {
|
||||
t.Fatalf("STATE_STORE_URL = %q, want the relay address %q", got, relay.Addr())
|
||||
}
|
||||
if env["STATE_STORE_TOKEN"] != "task-token" {
|
||||
t.Fatalf("STATE_STORE_TOKEN = %q, want the task's own token", env["STATE_STORE_TOKEN"])
|
||||
}
|
||||
if env["STATE_STORE_CA"] != "ca-pem" {
|
||||
t.Fatalf("STATE_STORE_CA = %q, want the CA passed through", env["STATE_STORE_CA"])
|
||||
}
|
||||
if got := env["NATS_URL"]; got != "nats://10.0.0.5:4222" {
|
||||
t.Fatalf("NATS_URL = %q, want the connected broker", got)
|
||||
}
|
||||
}
|
||||
|
||||
// Only a target the container cannot reach is relayed. An address already
|
||||
// configured for the containers' network is passed through untouched, and a
|
||||
// target the relay cannot forward is left alone rather than guessed at.
|
||||
func TestLoopbackStateStoreTarget(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
cases := []struct {
|
||||
name string
|
||||
target string
|
||||
addr string
|
||||
loopback bool
|
||||
}{
|
||||
{name: "loopback ipv4", target: "127.0.0.1:9090", addr: "127.0.0.1:9090", loopback: true},
|
||||
{name: "loopback ipv6", target: "[::1]:9090", addr: "[::1]:9090", loopback: true},
|
||||
{name: "localhost", target: "localhost:9090", addr: "localhost:9090", loopback: true},
|
||||
{name: "grpc scheme form", target: "dns:///127.0.0.1:9090", addr: "127.0.0.1:9090", loopback: true},
|
||||
{name: "host address", target: "10.0.0.5:9090"},
|
||||
{name: "service name", target: "state-store:9090"},
|
||||
{name: "unset", target: ""},
|
||||
}
|
||||
for _, tc := range cases {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
t.Parallel()
|
||||
addr, loopback := loopbackStateStoreTarget(tc.target)
|
||||
if loopback != tc.loopback {
|
||||
t.Fatalf("loopbackStateStoreTarget(%q) loopback = %t, want %t", tc.target, loopback, tc.loopback)
|
||||
}
|
||||
if addr != tc.addr {
|
||||
t.Fatalf("loopbackStateStoreTarget(%q) addr = %q, want %q", tc.target, addr, tc.addr)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// echoListener accepts connections on loopback and echoes what it reads.
|
||||
func echoListener(t *testing.T) net.Listener {
|
||||
t.Helper()
|
||||
ln, err := (&net.ListenConfig{}).Listen(context.Background(), "tcp", "127.0.0.1:0")
|
||||
if err != nil {
|
||||
t.Fatalf("listen: %v", err)
|
||||
}
|
||||
t.Cleanup(func() { _ = ln.Close() })
|
||||
go func() {
|
||||
for {
|
||||
conn, err := ln.Accept()
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
go func() {
|
||||
defer func() { _ = conn.Close() }()
|
||||
_, _ = io.Copy(conn, conn)
|
||||
}()
|
||||
}
|
||||
}()
|
||||
return ln
|
||||
}
|
||||
|
||||
func envValues(env []string) map[string]string {
|
||||
out := make(map[string]string, len(env))
|
||||
for _, entry := range env {
|
||||
key, value, _ := strings.Cut(entry, "=")
|
||||
out[key] = value
|
||||
}
|
||||
return out
|
||||
}
|
||||
@@ -0,0 +1,43 @@
|
||||
package kitrun
|
||||
|
||||
import (
|
||||
"net"
|
||||
"strconv"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"github.com/samcharles93/archie-core/internal/infrastructure/egress"
|
||||
)
|
||||
|
||||
// A Kit worker reaches the State Store through its relay, never through the
|
||||
// daemon's own target: ordinary image tasks got a per-task relay instead, and
|
||||
// this path must not follow them onto it.
|
||||
func TestWorkerEnvKeepsRelayAddress(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
env, err := WorkerEnv(Endpoints{
|
||||
NATS: "nats://127.0.0.1:4222",
|
||||
StateStore: "127.0.0.1:9090",
|
||||
StateStoreToken: "task-token",
|
||||
StateStoreCA: []byte("ca-pem"),
|
||||
}, 0, 0)
|
||||
if err != nil {
|
||||
t.Fatalf("WorkerEnv: %v", err)
|
||||
}
|
||||
values := make(map[string]string, len(env))
|
||||
for _, entry := range env {
|
||||
key, value, _ := strings.Cut(entry, "=")
|
||||
values[key] = value
|
||||
}
|
||||
|
||||
want := net.JoinHostPort(egress.RelayAlias, strconv.Itoa(egress.RelayStateStorePort))
|
||||
if got := values["STATE_STORE_URL"]; got != want {
|
||||
t.Fatalf("STATE_STORE_URL = %q, want the relay address %q", got, want)
|
||||
}
|
||||
if values["STATE_STORE_TOKEN"] != "task-token" {
|
||||
t.Fatalf("STATE_STORE_TOKEN = %q, want the task's own token", values["STATE_STORE_TOKEN"])
|
||||
}
|
||||
if values["STATE_STORE_CA"] != "ca-pem" {
|
||||
t.Fatalf("STATE_STORE_CA = %q, want the CA passed through", values["STATE_STORE_CA"])
|
||||
}
|
||||
}
|
||||
Reference in new issue
Block a user